From 70cbd7989dea19cb56af7f4d272eaf7c0510980b Mon Sep 17 00:00:00 2001 From: Levon Akopian Date: Wed, 26 Jun 2024 14:49:13 +0300 Subject: [PATCH 01/27] wlc mowed to from dataplane to controlplane --- controlplane/balancer.cpp | 152 +++++++++++++++++++++++++++++++++++++- controlplane/balancer.h | 6 ++ dataplane/globalbase.cpp | 47 ------------ 3 files changed, 157 insertions(+), 48 deletions(-) diff --git a/controlplane/balancer.cpp b/controlplane/balancer.cpp index 4774ef76..ec601c55 100644 --- a/controlplane/balancer.cpp +++ b/controlplane/balancer.cpp @@ -883,6 +883,12 @@ void balancer_t::flush_reals(common::idp::updateGlobalBaseBalancer::request& bal } } + auto reals_wlc = reals_wlc_weight.find(key); + if (reals_wlc != reals_wlc_weight.end() && reals_wlc->second.has_value()) + { + effective_weight = reals_wlc->second.value(); + } + uint32_t real_unordered_id = 0; { std::lock_guard guard(reals_unordered_mutex); @@ -926,8 +932,152 @@ void balancer_t::reconfigure_wlc_thread() { while (!flagStop) { - balancer_real_flush(); + if (balancer_t::reconfigure_wlc()) + { + balancer_real_flush(); + } std::this_thread::sleep_for(std::chrono::seconds(YANET_CONFIG_BALANCER_WLC_RECONFIGURE)); } } + +bool balancer_t::reconfigure_wlc() +{ + bool wlc_weight_changed = false; + + common::idp::updateGlobalBaseBalancer::request balancer; + const auto balancer_real_connections = dataplane.balancer_real_connections(); + + std::lock_guard guard(reals_enabled_mutex); + + for (const auto& [module_name, balancer] : generations_config.current().config_balancers) + { + + for (const auto& [service_id, + virtual_ip, + proto, + virtual_port, + version, + scheduler, + scheduler_params, + forwarding_method, + flags, + ipv4_outer_source_network, + ipv6_outer_source_network, + reals] : balancer.services) + { + (void)flags; + (void)version; + (void)forwarding_method; + (void)ipv4_outer_source_network; + (void)ipv6_outer_source_network; + + if (scheduler != ::balancer::scheduler::wlc) + { + continue; + } + + if (service_id >= YANET_CONFIG_BALANCER_SERVICES_SIZE) + { + continue; + } + + std::vector> service_reals_usage_info; + uint32_t connection_sum = 0; + uint32_t weight_sum = 0; + + for (const auto& [real_ip, real_port, weight] : reals) + { + balancer::real_key_global_t key = {module_name, {virtual_ip, proto, virtual_port}, {real_ip, real_port}}; + uint32_t effective_weight = weight; + { + auto it = reals_enabled.find(key); + if (it != reals_enabled.end()) + { + if (it->second.has_value()) + { + effective_weight = it->second.value(); + } + } + } + + weight_sum += effective_weight; + + // don`t count connections for disabled reals - it can make other reals "feel" underloaded + if (effective_weight == 0) + { + continue; + } + + common::idp::balancer_real_connections::real_key_t real_connections_key = {balancer.balancer_id, + virtual_ip, + proto, + virtual_port.value(), + real_ip, + real_port.value()}; + uint32_t connections = 0; + for (auto& [socket_id, real_connections] : balancer_real_connections) + { + (void)socket_id; + + auto it = real_connections.find(real_connections_key); + if (it == real_connections.end()) + { + continue; + } + connections += it->second; + } + + connection_sum += connections; + + service_reals_usage_info.emplace_back(key, + effective_weight, + connections); + } + + for (auto [key, + effective_weight, + connections] : service_reals_usage_info) + { + uint32_t wlc_power = scheduler_params.wlc_power; + if (wlc_power < 1 || wlc_power > 100) + { + wlc_power = YANET_CONFIG_BALANCER_WLC_DEFAULT_POWER; + } + + effective_weight = calculate_wlc_weight(effective_weight, connections, weight_sum, connection_sum, wlc_power); + + if (reals_wlc_weight[key] != effective_weight) + { + reals_wlc_weight[key] = effective_weight; + real_updates.insert(key); + if (in_reload) + { + real_reload_updates.insert(key); + } + wlc_weight_changed = true; + } + } + } + } + + return wlc_weight_changed; +} + +uint32_t balancer_t::calculate_wlc_weight(uint32_t weight, uint32_t connections, uint32_t weight_sum, uint32_t connection_sum, uint32_t wlc_power) +{ + if (weight == 0 || weight_sum == 0 || connection_sum < weight_sum) + { + return weight; + } + + auto wlc_ratio = std::max(1.0, wlc_power * (1 - 1.0 * connections * weight_sum / connection_sum / weight)); + auto wlc_weight = (uint32_t)(weight * wlc_ratio); + + if (wlc_weight > YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX) + { + wlc_weight = YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX; + } + + return wlc_weight; +} \ No newline at end of file diff --git a/controlplane/balancer.h b/controlplane/balancer.h index 3bcba0cf..b0a3b10a 100644 --- a/controlplane/balancer.h +++ b/controlplane/balancer.h @@ -113,6 +113,8 @@ class balancer_t : public module_t, common::icp_proto::BalancerService std::map> reals_enabled; + std::map> reals_wlc_weight; + // The set contains all reals touched after the last one flush operation std::set real_updates; @@ -133,7 +135,11 @@ class balancer_t : public module_t, common::icp_proto::BalancerService friend class telegraf_t; counter_t service_counters; counter_t real_counters; + void RealFind(google::protobuf::RpcController* controller, const common::icp_proto::BalancerRealFindRequest* request, common::icp_proto::BalancerRealFindResponse* response, google::protobuf::Closure* done) override; void Real(google::protobuf::RpcController* controller, const ::common::icp_proto::BalancerRealRequest* request, ::common::icp_proto::Empty* response, ::google::protobuf::Closure* done) override; void RealFlush(google::protobuf::RpcController* controller, const ::common::icp_proto::Empty* request, ::common::icp_proto::Empty* response, ::google::protobuf::Closure* done) override; + + bool reconfigure_wlc(); + uint32_t calculate_wlc_weight(uint32_t weight, uint32_t connections, uint32_t weight_sum, uint32_t connection_sum, uint32_t wlc_power); }; diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index e199c004..363a3754 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -1584,16 +1584,6 @@ eResult generation::update_balancer_unordered_real(const common::idp::updateGlob return eResult::success; } -double wlc_ratio(uint32_t weight, uint32_t connections, uint32_t weight_sum, uint32_t connection_sum, uint32_t power) -{ - if (weight == 0 || weight_sum == 0 || connection_sum < weight_sum) - { - return 1; - } - auto a = power * (1 - 1.0 * connections * weight_sum / connection_sum / weight); - return std::max(1.0, a); -} - inline uint64_t generation::count_real_connections(uint32_t counter_id) { uint64_t sessions_created = 0; @@ -1626,27 +1616,6 @@ void generation::evaluate_service_ring() for (uint32_t service_idx = 0; service_idx < balancer_services_count; ++service_idx) { balancer_service_t* service = balancer_services + balancer_active_services[service_idx]; - uint64_t connection_sum = 0; - uint32_t weight_sum = 0; - if (service->scheduler == ::balancer::scheduler::wlc) - { - for (uint32_t real_idx = service->real_start; - real_idx < service->real_start + service->real_size; - ++real_idx) - { - uint32_t real_id = balancer_service_reals[real_idx]; - balancer_real_state_t* state = balancer_real_states + real_id; - // don`t count connections for disabled reals - it can make other reals "feel" underloaded - if (state->weight == 0) - { - continue; - } - weight_sum += state->weight; - - const balancer_real_t& real = balancer_reals[real_id]; - connection_sum += count_real_connections(real.counter_id); - } - } balancer_service_range_t* range = ring->ranges + balancer_active_services[service_idx]; @@ -1656,7 +1625,6 @@ void generation::evaluate_service_ring() ++real_idx) { uint32_t real_id = balancer_service_reals[real_idx]; - const balancer_real_t& real = balancer_reals[real_id]; balancer_real_state_t* state = balancer_real_states + real_id; if (state->weight == 0) @@ -1666,21 +1634,6 @@ void generation::evaluate_service_ring() auto weight = state->weight; - if (service->scheduler == ::balancer::scheduler::wlc) - { - uint64_t real_connections = count_real_connections(real.counter_id); - - weight = (int)(weight * wlc_ratio(state->weight, real_connections, weight_sum, connection_sum, service->wlc_power)); - // todo check weight change - } - - // clamp weight to a maximum possible value - if (weight > YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX) - { - // TODO: think about accounting the clamping - weight = YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX; - } - while (weight-- > 0) { ring->reals[weight_pos++] = real_id; From 59e9b3e5b93ec970cebe5550f850d2fd6eb68ac1 Mon Sep 17 00:00:00 2001 From: vimes Date: Thu, 14 Nov 2024 17:12:46 +0300 Subject: [PATCH 02/27] Refactors service evaluation --- dataplane/globalbase.cpp | 67 ++++++++++++++++++++++++++-------------- dataplane/globalbase.h | 4 +++ 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index 363a3754..0b825ca8 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -1609,41 +1609,60 @@ inline uint64_t generation::count_real_connections(uint32_t counter_id) return (sessions_created - sessions_destroyed + sessions_created_gc - sessions_destroyed_gc) / dataPlane->numaNodesInUse; } -void generation::evaluate_service_ring() +balancer_real_id_t* generation::evaluate_service_ring_one(balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service, + balancer_service_range_t& range) { - balancer_service_ring_t* ring = &balancer_service_ring; - uint32_t weight_pos = 0; - for (uint32_t service_idx = 0; service_idx < balancer_services_count; ++service_idx) + for (uint32_t real_idx = service.real_start; + real_idx < service.real_start + service.real_size; + ++real_idx) { - balancer_service_t* service = balancer_services + balancer_active_services[service_idx]; - - balancer_service_range_t* range = ring->ranges + balancer_active_services[service_idx]; + uint32_t real_id = balancer_service_reals[real_idx]; + balancer_real_state_t* state = balancer_real_states + real_id; - range->start = weight_pos; - for (uint32_t real_idx = service->real_start; - real_idx < service->real_start + service->real_size; - ++real_idx) + if (state->weight == 0) { - uint32_t real_id = balancer_service_reals[real_idx]; - balancer_real_state_t* state = balancer_real_states + real_id; - - if (state->weight == 0) - { - continue; - } + continue; + } - auto weight = state->weight; + auto weight = state->weight; - while (weight-- > 0) + while (weight-- > 0) + { + if (start == do_not_exceed) { - ring->reals[weight_pos++] = real_id; + YANET_LOG_ERROR("Balancer service exceeded ring chunk bounds\n"); + goto finish; } + *start = real_id; + ++start; } + } + +finish: + YADECAP_MEMORY_BARRIER_COMPILE; + return start; +} + +void generation::evaluate_service_ring() +{ + balancer_service_ring_t* ring = &balancer_service_ring; + balancer_real_id_t* service_start = ring->reals; + for (uint32_t service_idx = 0; + service_idx < balancer_services_count; + ++service_idx, service_start += YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX) + { + balancer_service_t* service = balancer_services + balancer_active_services[service_idx]; - YADECAP_MEMORY_BARRIER_COMPILE; + balancer_service_range_t* range = ring->ranges + balancer_active_services[service_idx]; - range->size = weight_pos - range->start; - weight_pos = range->start + service->real_size * YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX; + range->start = std::distance(ring->reals, service_start); + auto service_end = evaluate_service_ring_one(service_start, + service_start + YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX, + *service, + *range); + range->size = std::distance(service_start, service_end); } } diff --git a/dataplane/globalbase.h b/dataplane/globalbase.h index fbb38538..e0e8d7c8 100644 --- a/dataplane/globalbase.h +++ b/dataplane/globalbase.h @@ -186,6 +186,10 @@ class generation eResult tsc_state_update(const common::idp::updateGlobalBase::tsc_state_update::request& request); eResult tscs_base_value_update(const common::idp::updateGlobalBase::tscs_base_value_update::request& request); + balancer_real_id_t* evaluate_service_ring_one(balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service, + balancer_service_range_t& range); void evaluate_service_ring(); inline uint64_t count_real_connections(uint32_t counter_id); From b98de2009ec6d67e0ee49e1fe761649f19b1b5b3 Mon Sep 17 00:00:00 2001 From: vimes Date: Thu, 14 Nov 2024 19:30:11 +0300 Subject: [PATCH 03/27] Adds chash service evaluation stub --- common/scheduler.h | 13 +++++------- dataplane/globalbase.cpp | 44 ++++++++++++++++++++++++++++++++++++---- dataplane/globalbase.h | 8 ++++++++ 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/common/scheduler.h b/common/scheduler.h index 6fe773ac..54fa7857 100644 --- a/common/scheduler.h +++ b/common/scheduler.h @@ -9,6 +9,7 @@ enum class scheduler : uint8_t rr, wrr, wlc, + chash }; class scheduler_params @@ -23,20 +24,16 @@ class scheduler_params switch (scheduler) { case scheduler::rr: - { return "rr"; - } case scheduler::wrr: - { return "wrr"; - } case scheduler::wlc: - { return "wlc"; - } + case scheduler::chash: + return "chash"; + default: + return "unknown"; } - - return "unknown"; } } diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index 0b825ca8..cc43d5c8 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -1609,10 +1609,11 @@ inline uint64_t generation::count_real_connections(uint32_t counter_id) return (sessions_created - sessions_destroyed + sessions_created_gc - sessions_destroyed_gc) / dataPlane->numaNodesInUse; } -balancer_real_id_t* generation::evaluate_service_ring_one(balancer_real_id_t* start, - const balancer_real_id_t* const do_not_exceed, - const balancer_service_t& service, - balancer_service_range_t& range) +balancer_real_id_t* generation::evaluate_service_ring_one_wrr( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service, + balancer_service_range_t& range) { for (uint32_t real_idx = service.real_start; real_idx < service.real_start + service.real_size; @@ -1645,6 +1646,41 @@ balancer_real_id_t* generation::evaluate_service_ring_one(balancer_real_id_t* st return start; } +balancer_real_id_t* generation::evaluate_service_ring_one_chash( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service, + balancer_service_range_t& range) +{ + return start; +} + +balancer_real_id_t* generation::evaluate_service_ring_one( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service, + balancer_service_range_t& range) +{ + balancer_real_id_t* end{start}; + using scheduler = ::balancer::scheduler; + switch (service.scheduler) + { + case scheduler::rr: + case scheduler::wrr: + end = evaluate_service_ring_one_wrr( + start, do_not_exceed, service, range); + break; + case scheduler::wlc: + case scheduler::chash: + end = evaluate_service_ring_one_chash( + start, do_not_exceed, service, range); + break; + default: + YANET_LOG_ERROR("Unknown balancer service scheduler type\n"); + } + return end; +} + void generation::evaluate_service_ring() { balancer_service_ring_t* ring = &balancer_service_ring; diff --git a/dataplane/globalbase.h b/dataplane/globalbase.h index e0e8d7c8..d1425342 100644 --- a/dataplane/globalbase.h +++ b/dataplane/globalbase.h @@ -186,6 +186,14 @@ class generation eResult tsc_state_update(const common::idp::updateGlobalBase::tsc_state_update::request& request); eResult tscs_base_value_update(const common::idp::updateGlobalBase::tscs_base_value_update::request& request); + balancer_real_id_t* evaluate_service_ring_one_wrr(balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service, + balancer_service_range_t& range); + balancer_real_id_t* evaluate_service_ring_one_chash(balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service, + balancer_service_range_t& range); balancer_real_id_t* evaluate_service_ring_one(balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, const balancer_service_t& service, From 95e01ce84deca3a52e3fb0bfce887bb5c2345158 Mon Sep 17 00:00:00 2001 From: vimes Date: Mon, 18 Nov 2024 22:48:02 +0300 Subject: [PATCH 04/27] Adds handling of chash services to dataplane --- common/idp.h | 1 - dataplane/globalbase.cpp | 93 +++++++++++++++++++++++++++++++--------- dataplane/globalbase.h | 44 +++++++++++++------ dataplane/meson.build | 1 + dataplane/type.h | 8 +++- meson.build | 2 + 6 files changed, 114 insertions(+), 35 deletions(-) diff --git a/common/idp.h b/common/idp.h index 8719cd0d..745eb3a8 100644 --- a/common/idp.h +++ b/common/idp.h @@ -283,7 +283,6 @@ using service = std::tuple, ///< ipv4_outer_source_network diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index cc43d5c8..43f08d81 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -2,6 +2,8 @@ #include +#include + #include "common.h" #include "dataplane.h" #include "globalbase.h" @@ -1424,7 +1426,6 @@ eResult generation::update_balancer_services(const common::idp::updateGlobalBase counter_id, scheduler, forwarding_method, - default_wlc_power, real_start, real_size, ipv4_outer_source_network, @@ -1475,7 +1476,6 @@ eResult generation::update_balancer_services(const common::idp::updateGlobalBase balancer_service.real_start = real_start; balancer_service.real_size = real_size; balancer_service.scheduler = scheduler; - balancer_service.wlc_power = default_wlc_power; balancer_service.forwarding_method = forwarding_method; balancer_service.outer_source_network_flag = outer_source_network_flag; balancer_service.ipv4_outer_source_network = ipv4_prefix; @@ -1556,7 +1556,7 @@ eResult generation::update_balancer_services(const common::idp::updateGlobalBase std::copy(binding.begin(), binding.end(), balancer_service_reals); - evaluate_service_ring(); + evaluate_service_ring(ServiceRingOp::Rebuild); return eResult::success; } @@ -1579,7 +1579,7 @@ eResult generation::update_balancer_unordered_real(const common::idp::updateGlob real_state = new_state; } - evaluate_service_ring(); + evaluate_service_ring(ServiceRingOp::Update); return eResult::success; } @@ -1609,14 +1609,14 @@ inline uint64_t generation::count_real_connections(uint32_t counter_id) return (sessions_created - sessions_destroyed + sessions_created_gc - sessions_destroyed_gc) / dataPlane->numaNodesInUse; } -balancer_real_id_t* generation::evaluate_service_ring_one_wrr( +balancer_real_id_t* generation::rebuild_service_ring_one_wrr( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t& service, + const balancer_service_t* service, balancer_service_range_t& range) { - for (uint32_t real_idx = service.real_start; - real_idx < service.real_start + service.real_size; + for (uint32_t real_idx = service->real_start; + real_idx < service->real_start + service->real_size; ++real_idx) { uint32_t real_id = balancer_service_reals[real_idx]; @@ -1646,34 +1646,85 @@ balancer_real_id_t* generation::evaluate_service_ring_one_wrr( return start; } -balancer_real_id_t* generation::evaluate_service_ring_one_chash( +std::vector> +generation::ServiceWeights(const balancer_service_t* service) +{ + std::vector> weights; + for (uint32_t real_idx = service->real_start; + real_idx < service->real_start + service->real_size; + ++real_idx) + { + uint32_t real_id = balancer_service_reals[real_idx]; + uint32_t weight = (balancer_real_states + real_id)->weight; + + weights.emplace_back(real_id, weight); + } +} + +balancer_real_id_t* generation::rebuild_service_ring_one_chash( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t& service, + const balancer_service_t* service, balancer_service_range_t& range) { + std::vector> reals; + for (uint32_t real_idx = service->real_start; + real_idx < service->real_start + service->real_size; + ++real_idx) + { + uint32_t real_id = balancer_service_reals[real_idx]; + ipv6_address_t& ip = balancer_reals[real_id].destination; + reals.emplace_back(ip, real_id); + } + auto updater = chash::WeightUpdater::MakeWeightUpdater(reals, service->details->segments_per_real); + updater.InitLookup(ServiceWeights(service), start); + + chash_updaters.emplace(service, std::move(updater)); return start; } +void generation::update_service_ring_one_chash( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t* service, + balancer_service_range_t& range) +{ + if (chash_updaters.find(service) == chash_updaters.end()) + { + YANET_LOG_ERROR("No state information for updating requested service.\n"); + return; + } + auto& updater = chash_updaters.at(service); + updater.UpdateLookup(ServiceWeights(service), start); +} + balancer_real_id_t* generation::evaluate_service_ring_one( + ServiceRingOp op, balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t& service, + const balancer_service_t* service, balancer_service_range_t& range) { balancer_real_id_t* end{start}; using scheduler = ::balancer::scheduler; - switch (service.scheduler) + switch (service->scheduler) { case scheduler::rr: case scheduler::wrr: - end = evaluate_service_ring_one_wrr( + end = rebuild_service_ring_one_wrr( start, do_not_exceed, service, range); break; case scheduler::wlc: case scheduler::chash: - end = evaluate_service_ring_one_chash( - start, do_not_exceed, service, range); + if (op == ServiceRingOp::Rebuild) + { + end = rebuild_service_ring_one_chash( + start, do_not_exceed, service, range); + } + else + { + update_service_ring_one_chash(start, do_not_exceed, service, range); + } break; default: YANET_LOG_ERROR("Unknown balancer service scheduler type\n"); @@ -1681,7 +1732,7 @@ balancer_real_id_t* generation::evaluate_service_ring_one( return end; } -void generation::evaluate_service_ring() +void generation::evaluate_service_ring(ServiceRingOp op) { balancer_service_ring_t* ring = &balancer_service_ring; balancer_real_id_t* service_start = ring->reals; @@ -1694,10 +1745,12 @@ void generation::evaluate_service_ring() balancer_service_range_t* range = ring->ranges + balancer_active_services[service_idx]; range->start = std::distance(ring->reals, service_start); - auto service_end = evaluate_service_ring_one(service_start, - service_start + YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX, - *service, - *range); + auto service_end = evaluate_service_ring_one( + op, + service_start, + service_start + YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX, + service, + *range); range->size = std::distance(service_start, service_end); } } diff --git a/dataplane/globalbase.h b/dataplane/globalbase.h index d1425342..fdb9338e 100644 --- a/dataplane/globalbase.h +++ b/dataplane/globalbase.h @@ -186,19 +186,36 @@ class generation eResult tsc_state_update(const common::idp::updateGlobalBase::tsc_state_update::request& request); eResult tscs_base_value_update(const common::idp::updateGlobalBase::tscs_base_value_update::request& request); - balancer_real_id_t* evaluate_service_ring_one_wrr(balancer_real_id_t* start, - const balancer_real_id_t* const do_not_exceed, - const balancer_service_t& service, - balancer_service_range_t& range); - balancer_real_id_t* evaluate_service_ring_one_chash(balancer_real_id_t* start, - const balancer_real_id_t* const do_not_exceed, - const balancer_service_t& service, - balancer_service_range_t& range); - balancer_real_id_t* evaluate_service_ring_one(balancer_real_id_t* start, - const balancer_real_id_t* const do_not_exceed, - const balancer_service_t& service, - balancer_service_range_t& range); - void evaluate_service_ring(); + enum class ServiceRingOp + { + Update, + Rebuild + }; + + std::vector> + generation::ServiceWeights(const balancer_service_t* service); + balancer_real_id_t* rebuild_service_ring_one_wrr( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t* service, + balancer_service_range_t& range); + balancer_real_id_t* rebuild_service_ring_one_chash( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t* service, + balancer_service_range_t& range); + void update_service_ring_one_chash( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t* service, + balancer_service_range_t& range); + balancer_real_id_t* evaluate_service_ring_one( + ServiceRingOp op, + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t* service, + balancer_service_range_t& range); + void evaluate_service_ring(ServiceRingOp op); inline uint64_t count_real_connections(uint32_t counter_id); public: ///< @todo @@ -321,6 +338,7 @@ class generation balancer_real_state_t balancer_real_states[YANET_CONFIG_BALANCER_REALS_SIZE]; balancer_service_ring_t balancer_service_ring; + std::map chash_updaters; int64_t dump_id_to_tag[YANET_CONFIG_DUMP_ID_TO_TAG_SIZE]; diff --git a/dataplane/meson.build b/dataplane/meson.build index 73962ecd..07fb895c 100644 --- a/dataplane/meson.build +++ b/dataplane/meson.build @@ -1,6 +1,7 @@ dependencies = [] dependencies += libdpdk.get_variable('dpdk_dep') dependencies += libjson.get_variable('nlohmann_json_dep') +dependencies += libchash.get_variable('balancer_dep') dependencies += dependency('libsystemd') dependencies += dependency('threads') diff --git a/dataplane/type.h b/dataplane/type.h index cf55b213..f8316bc5 100644 --- a/dataplane/type.h +++ b/dataplane/type.h @@ -559,6 +559,11 @@ struct balancer_service_ring_t static_assert(YANET_CONFIG_COUNTERS_SIZE <= 0xFFFFFF, "invalid size"); +struct ChashServiceConfig +{ + std::size_t segments_per_real; +}; + struct balancer_service_t { /// @todo @@ -578,7 +583,8 @@ struct balancer_service_t uint32_t real_size; ::balancer::scheduler scheduler; ::balancer::forwarding_method forwarding_method; - uint32_t wlc_power; + + std::optional details; /* outer_source_network_flag: diff --git a/meson.build b/meson.build index 08c20a68..97447027 100644 --- a/meson.build +++ b/meson.build @@ -58,6 +58,8 @@ libdpdk = subproject('dpdk', default_options: [ libjson = subproject('json') +libchash = subproject('balancer') + if target_option.contains('buildenv') subdir('libprotobuf') subdir('libfwparser') From e5ae4d4d14fb325c0f37982d3cbb7098ca9df6fa Mon Sep 17 00:00:00 2001 From: vimes Date: Tue, 3 Dec 2024 13:39:30 +0300 Subject: [PATCH 05/27] Works. Maybe --- common/idp.h | 1 + common/scheduler.h | 13 ++++++++++--- controlplane/balancer.cpp | 4 ++-- controlplane/configparser.cpp | 15 ++++++++++++++- dataplane/globalbase.cpp | 19 ++++++++++++++----- dataplane/globalbase.h | 4 +++- dataplane/type.h | 12 ++++++------ 7 files changed, 50 insertions(+), 18 deletions(-) diff --git a/common/idp.h b/common/idp.h index 745eb3a8..b91deb88 100644 --- a/common/idp.h +++ b/common/idp.h @@ -283,6 +283,7 @@ using service = std::tuple, ///< ipv4_outer_source_network diff --git a/common/scheduler.h b/common/scheduler.h index 54fa7857..bb722c1d 100644 --- a/common/scheduler.h +++ b/common/scheduler.h @@ -1,4 +1,5 @@ #pragma once +#include #include namespace balancer @@ -12,13 +13,19 @@ enum class scheduler : uint8_t chash }; -class scheduler_params +struct wlc_params { -public: - scheduler_params() = default; uint32_t wlc_power; }; +struct chash_params +{ + uint32_t siderings_count; + uint32_t segments_per_weight; +}; + +using scheduler_params = std::variant; + [[maybe_unused]] constexpr const char* to_string(const scheduler& scheduler) { switch (scheduler) diff --git a/controlplane/balancer.cpp b/controlplane/balancer.cpp index ec601c55..a0c3df95 100644 --- a/controlplane/balancer.cpp +++ b/controlplane/balancer.cpp @@ -808,7 +808,7 @@ void balancer_t::compile(common::idp::updateGlobalBase::request& globalbase, counter_id, scheduler, forwarding_method, - balancer.default_wlc_power, // todo use scheduler_params.wlc_power when other services will be able to set it + scheduler_params, (uint32_t)real_start, (uint32_t)(req_reals.size() - real_start), ipv4_outer_source_network, @@ -1039,7 +1039,7 @@ bool balancer_t::reconfigure_wlc() effective_weight, connections] : service_reals_usage_info) { - uint32_t wlc_power = scheduler_params.wlc_power; + uint32_t wlc_power = std::get(scheduler_params).wlc_power; if (wlc_power < 1 || wlc_power > 100) { wlc_power = YANET_CONFIG_BALANCER_WLC_DEFAULT_POWER; diff --git a/controlplane/configparser.cpp b/controlplane/configparser.cpp index 92054e4c..ce8de8c2 100644 --- a/controlplane/configparser.cpp +++ b/controlplane/configparser.cpp @@ -1664,7 +1664,20 @@ void config_parser_t::loadConfig_balancer_services(controlplane::base_t& baseNex scheduler = balancer::scheduler::wlc; if (exist(service_json, "scheduler_params") && exist(service_json["scheduler_params"], "wlc_power")) { - scheduler_params.wlc_power = std::stoll(service_json["scheduler_params"]["wlc_power"].get(), nullptr, 10); + auto& params = std::get(scheduler_params); + params.wlc_power = std::stoll(service_json["scheduler_params"]["wlc_power"].get(), nullptr, 10); + } + } + else if (scheduler_string == "chash") + { + scheduler = balancer::scheduler::chash; + if (exist(service_json, "scheduler_params")) + { + auto& params = std::get(scheduler_params); + params.segments_per_weight = service_json.value( + "/scheduler_params/segments_per_weight"_json_pointer, 20); + params.siderings_count = service_json.value( + "/scheduler_params/siderings_count"_json_pointer, 10000); } } else diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index 43f08d81..7432b5b5 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -2,8 +2,6 @@ #include -#include - #include "common.h" #include "dataplane.h" #include "globalbase.h" @@ -1426,6 +1424,7 @@ eResult generation::update_balancer_services(const common::idp::updateGlobalBase counter_id, scheduler, forwarding_method, + scheduler_params, real_start, real_size, ipv4_outer_source_network, @@ -1477,6 +1476,7 @@ eResult generation::update_balancer_services(const common::idp::updateGlobalBase balancer_service.real_size = real_size; balancer_service.scheduler = scheduler; balancer_service.forwarding_method = forwarding_method; + balancer_service.details = (scheduler == ::balancer::scheduler::chash) ? std::make_optional(std::get<::balancer::chash_params>(scheduler_params)) : std::nullopt; balancer_service.outer_source_network_flag = outer_source_network_flag; balancer_service.ipv4_outer_source_network = ipv4_prefix; balancer_service.ipv6_outer_source_network = ipv6_prefix; @@ -1659,6 +1659,7 @@ generation::ServiceWeights(const balancer_service_t* service) weights.emplace_back(real_id, weight); } + return weights; } balancer_real_id_t* generation::rebuild_service_ring_one_chash( @@ -1676,10 +1677,18 @@ balancer_real_id_t* generation::rebuild_service_ring_one_chash( ipv6_address_t& ip = balancer_reals[real_id].destination; reals.emplace_back(ip, real_id); } - auto updater = chash::WeightUpdater::MakeWeightUpdater(reals, service->details->segments_per_real); - updater.InitLookup(ServiceWeights(service), start); + auto updater = chash::WeightUpdater::MakeWeightUpdater( + reals, + service->details->siderings_count, + service->details->segments_per_weight); + if (!updater) + { + YANET_LOG_ERROR("Failed to intialize updater for balancer service"); + std::abort(); + } + updater.value().InitLookup(ServiceWeights(service), start); - chash_updaters.emplace(service, std::move(updater)); + chash_updaters.emplace(service, std::move(updater.value())); return start; } diff --git a/dataplane/globalbase.h b/dataplane/globalbase.h index fdb9338e..95476bc5 100644 --- a/dataplane/globalbase.h +++ b/dataplane/globalbase.h @@ -7,6 +7,8 @@ #include #include +#include + #include "common/idp.h" #include "common/result.h" #include "common/tsc_deltas.h" @@ -193,7 +195,7 @@ class generation }; std::vector> - generation::ServiceWeights(const balancer_service_t* service); + ServiceWeights(const balancer_service_t* service); balancer_real_id_t* rebuild_service_ring_one_wrr( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, diff --git a/dataplane/type.h b/dataplane/type.h index f8316bc5..88cb56f3 100644 --- a/dataplane/type.h +++ b/dataplane/type.h @@ -206,6 +206,11 @@ struct ipv6_address_t memset(bytes, 0, std::size(bytes)); } + bool operator<(ipv6_address_t other) const + { + return std::memcmp(bytes, other.bytes, 16) < 0; + } + union { uint8_t bytes[16]; ///< @todo: rename to address @@ -559,11 +564,6 @@ struct balancer_service_ring_t static_assert(YANET_CONFIG_COUNTERS_SIZE <= 0xFFFFFF, "invalid size"); -struct ChashServiceConfig -{ - std::size_t segments_per_real; -}; - struct balancer_service_t { /// @todo @@ -584,7 +584,7 @@ struct balancer_service_t ::balancer::scheduler scheduler; ::balancer::forwarding_method forwarding_method; - std::optional details; + std::optional<::balancer::chash_params> details; /* outer_source_network_flag: From bfa8e0facb2e38b2cc88e7e07b8ccf769ea4aead Mon Sep 17 00:00:00 2001 From: vimes Date: Tue, 3 Dec 2024 14:40:00 +0300 Subject: [PATCH 06/27] Adds chash submodule --- .gitmodules | 3 +++ subprojects/chash | 1 + 2 files changed, 4 insertions(+) create mode 160000 subprojects/chash diff --git a/.gitmodules b/.gitmodules index 288c73af..08160a99 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "subprojects/json"] path = subprojects/json url = https://github.com/nlohmann/json.git +[submodule "subprojects/chash"] + path = subprojects/chash + url = https://github.com/yanet-platform/chash.git diff --git a/subprojects/chash b/subprojects/chash new file mode 160000 index 00000000..8930a3ec --- /dev/null +++ b/subprojects/chash @@ -0,0 +1 @@ +Subproject commit 8930a3ec77028e33fd5ff03c297a83cf53061984 From 69fbfa9532bd9278a7a363c6b4e29dea0ce18eea Mon Sep 17 00:00:00 2001 From: vimes Date: Tue, 3 Dec 2024 14:49:25 +0300 Subject: [PATCH 07/27] Updates chash --- subprojects/chash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/chash b/subprojects/chash index 8930a3ec..903817aa 160000 --- a/subprojects/chash +++ b/subprojects/chash @@ -1 +1 @@ -Subproject commit 8930a3ec77028e33fd5ff03c297a83cf53061984 +Subproject commit 903817aac431b206a0c3c5f97e5941e589d5a943 From cb856e7488b992bbbf53664f44db08b9828d5dee Mon Sep 17 00:00:00 2001 From: vimes Date: Tue, 3 Dec 2024 15:35:57 +0300 Subject: [PATCH 08/27] Fixes build --- dataplane/meson.build | 2 +- meson.build | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dataplane/meson.build b/dataplane/meson.build index 07fb895c..3fc2f067 100644 --- a/dataplane/meson.build +++ b/dataplane/meson.build @@ -1,7 +1,7 @@ dependencies = [] dependencies += libdpdk.get_variable('dpdk_dep') dependencies += libjson.get_variable('nlohmann_json_dep') -dependencies += libchash.get_variable('balancer_dep') +dependencies += libchash.get_variable('chash_dep') dependencies += dependency('libsystemd') dependencies += dependency('threads') diff --git a/meson.build b/meson.build index 97447027..d0ffc84e 100644 --- a/meson.build +++ b/meson.build @@ -58,7 +58,7 @@ libdpdk = subproject('dpdk', default_options: [ libjson = subproject('json') -libchash = subproject('balancer') +libchash = subproject('chash') if target_option.contains('buildenv') subdir('libprotobuf') From ad4c864f82c6c5c631bef62677c5a98a59d90412 Mon Sep 17 00:00:00 2001 From: vimes Date: Tue, 3 Dec 2024 18:18:35 +0300 Subject: [PATCH 09/27] Updates chash --- subprojects/chash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/chash b/subprojects/chash index 903817aa..31788157 160000 --- a/subprojects/chash +++ b/subprojects/chash @@ -1 +1 @@ -Subproject commit 903817aac431b206a0c3c5f97e5941e589d5a943 +Subproject commit 3178815725bf229a4954708123c45615f60a16b1 From 7d8b42ddf4d389921d7a2a9ba2a98aab73331edd Mon Sep 17 00:00:00 2001 From: vimes Date: Wed, 4 Dec 2024 11:59:55 +0300 Subject: [PATCH 10/27] Changes chash submodule to track master --- .gitmodules | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitmodules b/.gitmodules index 08160a99..0b8a4cbd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,4 @@ [submodule "subprojects/chash"] path = subprojects/chash url = https://github.com/yanet-platform/chash.git + branch = master From c07d5a7889287a394dc23ef4ea9e5824aa4d2cad Mon Sep 17 00:00:00 2001 From: vimes Date: Wed, 4 Dec 2024 15:13:06 +0300 Subject: [PATCH 11/27] Update chash --- subprojects/chash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/chash b/subprojects/chash index 31788157..45ab197b 160000 --- a/subprojects/chash +++ b/subprojects/chash @@ -1 +1 @@ -Subproject commit 3178815725bf229a4954708123c45615f60a16b1 +Subproject commit 45ab197bd848b495c8ce80705c9346e9e5d12970 From f61379979f772700b6d6e216d0713529e64a9188 Mon Sep 17 00:00:00 2001 From: vimes Date: Wed, 4 Dec 2024 16:07:46 +0300 Subject: [PATCH 12/27] Disables unittests in chash subproject --- meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meson.build b/meson.build index d0ffc84e..95650d18 100644 --- a/meson.build +++ b/meson.build @@ -58,7 +58,7 @@ libdpdk = subproject('dpdk', default_options: [ libjson = subproject('json') -libchash = subproject('chash') +libchash = subproject('chash', default_options: ['tests=false']) if target_option.contains('buildenv') subdir('libprotobuf') From 61a6aa86f92b0fde43a5c4edc7233fc99c64f14c Mon Sep 17 00:00:00 2001 From: vimes Date: Wed, 4 Dec 2024 18:27:13 +0300 Subject: [PATCH 13/27] Fixes build --- meson.build | 2 +- subprojects/chash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meson.build b/meson.build index 95650d18..29e283b9 100644 --- a/meson.build +++ b/meson.build @@ -58,7 +58,7 @@ libdpdk = subproject('dpdk', default_options: [ libjson = subproject('json') -libchash = subproject('chash', default_options: ['tests=false']) +libchash = subproject('chash', default_options: [ 'tests=false' ]) if target_option.contains('buildenv') subdir('libprotobuf') diff --git a/subprojects/chash b/subprojects/chash index 45ab197b..ac8ee6af 160000 --- a/subprojects/chash +++ b/subprojects/chash @@ -1 +1 @@ -Subproject commit 45ab197bd848b495c8ce80705c9346e9e5d12970 +Subproject commit ac8ee6afc06a91a6cb75285cc99b213e2ca522f4 From 3b6dfc75318ac57f2bb811ea0f9c3f809a00513f Mon Sep 17 00:00:00 2001 From: vimes Date: Thu, 5 Dec 2024 16:05:16 +0300 Subject: [PATCH 14/27] Fixes ubuntu 18 includes --- subprojects/chash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/chash b/subprojects/chash index ac8ee6af..711c3b22 160000 --- a/subprojects/chash +++ b/subprojects/chash @@ -1 +1 @@ -Subproject commit ac8ee6afc06a91a6cb75285cc99b213e2ca522f4 +Subproject commit 711c3b22a033be4640295ec5f522ee12bc1e9c58 From df6aeb4f9aabbead34e49b41a567ce4efc3f11bf Mon Sep 17 00:00:00 2001 From: vimes Date: Thu, 5 Dec 2024 17:21:45 +0300 Subject: [PATCH 15/27] Updates chash submodule --- subprojects/chash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/chash b/subprojects/chash index 711c3b22..8f232baa 160000 --- a/subprojects/chash +++ b/subprojects/chash @@ -1 +1 @@ -Subproject commit 711c3b22a033be4640295ec5f522ee12bc1e9c58 +Subproject commit 8f232baac7b6b9084393bd9e222b1e0bee9a46c4 From 0cf759006e6e8b915c2274dbfa1fbc56c49a29f2 Mon Sep 17 00:00:00 2001 From: vimes Date: Thu, 5 Dec 2024 19:17:26 +0300 Subject: [PATCH 16/27] Switches default meson option to link statically --- librib/meson.build | 6 ------ meson.build | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/librib/meson.build b/librib/meson.build index c2748905..890e9aff 100644 --- a/librib/meson.build +++ b/librib/meson.build @@ -11,9 +11,3 @@ libyabird = library('yanet-rib', include_directories: yanet_rootdir, dependencies: dependencies, install: true) - -libyabird_static = static_library('yanet-rib', - sources, - include_directories: yanet_rootdir, - dependencies: dependencies, - install: true) diff --git a/meson.build b/meson.build index 29e283b9..d6d2621d 100644 --- a/meson.build +++ b/meson.build @@ -4,6 +4,7 @@ project('yanet', 'cpp', 'buildtype=release', 'warning_level=2', 'werror=true', + 'default_library=static', 'b_lto=true']) yanet_rootdir = include_directories('.') From 7089d5a19e3fe201d279b2d876dbf6d9c2a826f4 Mon Sep 17 00:00:00 2001 From: vimes Date: Mon, 9 Dec 2024 14:52:38 +0300 Subject: [PATCH 17/27] No config params --- common/controlplaneconfig.h | 2 +- common/idp.h | 1 - common/scheduler.h | 7 ------- controlplane/balancer.cpp | 2 +- controlplane/configparser.cpp | 15 +++------------ dataplane/globalbase.cpp | 9 ++++----- dataplane/type.h | 2 +- 7 files changed, 10 insertions(+), 28 deletions(-) diff --git a/common/controlplaneconfig.h b/common/controlplaneconfig.h index da9a6d4d..dcb1e820 100644 --- a/common/controlplaneconfig.h +++ b/common/controlplaneconfig.h @@ -206,7 +206,7 @@ using service_t = std::tuple, ///< vport std::optional, ///< version ::balancer::scheduler, - ::balancer::scheduler_params, + uint32_t, ///< wlc power ::balancer::forwarding_method, uint8_t, ///< flags: mss_fix|ops std::optional, ///< ipv4_outer_source_network diff --git a/common/idp.h b/common/idp.h index b91deb88..745eb3a8 100644 --- a/common/idp.h +++ b/common/idp.h @@ -283,7 +283,6 @@ using service = std::tuple, ///< ipv4_outer_source_network diff --git a/common/scheduler.h b/common/scheduler.h index bb722c1d..b76da094 100644 --- a/common/scheduler.h +++ b/common/scheduler.h @@ -13,19 +13,12 @@ enum class scheduler : uint8_t chash }; -struct wlc_params -{ - uint32_t wlc_power; -}; - struct chash_params { uint32_t siderings_count; uint32_t segments_per_weight; }; -using scheduler_params = std::variant; - [[maybe_unused]] constexpr const char* to_string(const scheduler& scheduler) { switch (scheduler) diff --git a/controlplane/balancer.cpp b/controlplane/balancer.cpp index a0c3df95..a2220b3d 100644 --- a/controlplane/balancer.cpp +++ b/controlplane/balancer.cpp @@ -1039,7 +1039,7 @@ bool balancer_t::reconfigure_wlc() effective_weight, connections] : service_reals_usage_info) { - uint32_t wlc_power = std::get(scheduler_params).wlc_power; + uint32_t wlc_power = scheduler_params; if (wlc_power < 1 || wlc_power > 100) { wlc_power = YANET_CONFIG_BALANCER_WLC_DEFAULT_POWER; diff --git a/controlplane/configparser.cpp b/controlplane/configparser.cpp index ce8de8c2..742dbb0e 100644 --- a/controlplane/configparser.cpp +++ b/controlplane/configparser.cpp @@ -1650,7 +1650,7 @@ void config_parser_t::loadConfig_balancer_services(controlplane::base_t& baseNex std::string scheduler_string = service_json["scheduler"]; balancer::scheduler scheduler{}; - balancer::scheduler_params scheduler_params{}; + std::uint32_t wlc_power{}; if (scheduler_string == "rr") { scheduler = balancer::scheduler::rr; @@ -1664,21 +1664,12 @@ void config_parser_t::loadConfig_balancer_services(controlplane::base_t& baseNex scheduler = balancer::scheduler::wlc; if (exist(service_json, "scheduler_params") && exist(service_json["scheduler_params"], "wlc_power")) { - auto& params = std::get(scheduler_params); - params.wlc_power = std::stoll(service_json["scheduler_params"]["wlc_power"].get(), nullptr, 10); + wlc_power = std::stoll(service_json["scheduler_params"]["wlc_power"].get(), nullptr, 10); } } else if (scheduler_string == "chash") { scheduler = balancer::scheduler::chash; - if (exist(service_json, "scheduler_params")) - { - auto& params = std::get(scheduler_params); - params.segments_per_weight = service_json.value( - "/scheduler_params/segments_per_weight"_json_pointer, 20); - params.siderings_count = service_json.value( - "/scheduler_params/siderings_count"_json_pointer, 10000); - } } else { @@ -1762,7 +1753,7 @@ void config_parser_t::loadConfig_balancer_services(controlplane::base_t& baseNex exist(service_json, "vport") ? std::make_optional(std::stoll(service_json["vport"].get(), nullptr, 0)) : std::nullopt, service_version, scheduler, - scheduler_params, + wlc_power, forwarding_method, flags, ipv4_outer_source_network, diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index 7432b5b5..3b35bf26 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -1424,7 +1424,6 @@ eResult generation::update_balancer_services(const common::idp::updateGlobalBase counter_id, scheduler, forwarding_method, - scheduler_params, real_start, real_size, ipv4_outer_source_network, @@ -1476,7 +1475,6 @@ eResult generation::update_balancer_services(const common::idp::updateGlobalBase balancer_service.real_size = real_size; balancer_service.scheduler = scheduler; balancer_service.forwarding_method = forwarding_method; - balancer_service.details = (scheduler == ::balancer::scheduler::chash) ? std::make_optional(std::get<::balancer::chash_params>(scheduler_params)) : std::nullopt; balancer_service.outer_source_network_flag = outer_source_network_flag; balancer_service.ipv4_outer_source_network = ipv4_prefix; balancer_service.ipv6_outer_source_network = ipv6_prefix; @@ -1679,11 +1677,12 @@ balancer_real_id_t* generation::rebuild_service_ring_one_chash( } auto updater = chash::WeightUpdater::MakeWeightUpdater( reals, - service->details->siderings_count, - service->details->segments_per_weight); + 20000, + 20); if (!updater) { - YANET_LOG_ERROR("Failed to intialize updater for balancer service"); + YANET_LOG_ERROR("Failed to intialize updater for balancer service reals: %ld\n", + reals.size()); std::abort(); } updater.value().InitLookup(ServiceWeights(service), start); diff --git a/dataplane/type.h b/dataplane/type.h index 88cb56f3..8ffe1486 100644 --- a/dataplane/type.h +++ b/dataplane/type.h @@ -584,7 +584,7 @@ struct balancer_service_t ::balancer::scheduler scheduler; ::balancer::forwarding_method forwarding_method; - std::optional<::balancer::chash_params> details; + std::uint32_t wlc_power; /* outer_source_network_flag: From 4f3e395a5eb4717e559a4ed274f529e6370f9fe8 Mon Sep 17 00:00:00 2001 From: vimes Date: Thu, 12 Dec 2024 13:26:39 +0300 Subject: [PATCH 18/27] Updates chash fixing edge cases --- subprojects/chash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/chash b/subprojects/chash index 8f232baa..3a374315 160000 --- a/subprojects/chash +++ b/subprojects/chash @@ -1 +1 @@ -Subproject commit 8f232baac7b6b9084393bd9e222b1e0bee9a46c4 +Subproject commit 3a37431565d8a43dc47ac3c970ea0411c6edbaf5 From 5122d784dd57b66c76a45fc4f0c2568b291ce915 Mon Sep 17 00:00:00 2001 From: vimes Date: Thu, 12 Dec 2024 14:09:01 +0300 Subject: [PATCH 19/27] Fixes wlc using chash --- common/scheduler.h | 6 ------ controlplane/balancer.cpp | 9 ++++----- dataplane/globalbase.cpp | 39 +++++++++++++++++++++------------------ dataplane/globalbase.h | 14 +++++--------- 4 files changed, 30 insertions(+), 38 deletions(-) diff --git a/common/scheduler.h b/common/scheduler.h index b76da094..e213b7ca 100644 --- a/common/scheduler.h +++ b/common/scheduler.h @@ -13,12 +13,6 @@ enum class scheduler : uint8_t chash }; -struct chash_params -{ - uint32_t siderings_count; - uint32_t segments_per_weight; -}; - [[maybe_unused]] constexpr const char* to_string(const scheduler& scheduler) { switch (scheduler) diff --git a/controlplane/balancer.cpp b/controlplane/balancer.cpp index a2220b3d..cf681aa0 100644 --- a/controlplane/balancer.cpp +++ b/controlplane/balancer.cpp @@ -757,14 +757,14 @@ void balancer_t::compile(common::idp::updateGlobalBase::request& globalbase, virtual_port, version, scheduler, - scheduler_params, + wlc_power, forwarding_method, flags, ipv4_outer_source_network, ipv6_outer_source_network, reals] : balancer.services) { - YANET_GCC_BUG_UNUSED(scheduler_params); + YANET_GCC_BUG_UNUSED(wlc_power); YANET_GCC_BUG_UNUSED(version); if (service_id >= YANET_CONFIG_BALANCER_SERVICES_SIZE) @@ -808,7 +808,6 @@ void balancer_t::compile(common::idp::updateGlobalBase::request& globalbase, counter_id, scheduler, forwarding_method, - scheduler_params, (uint32_t)real_start, (uint32_t)(req_reals.size() - real_start), ipv4_outer_source_network, @@ -959,7 +958,7 @@ bool balancer_t::reconfigure_wlc() virtual_port, version, scheduler, - scheduler_params, + requested_wlc_power, forwarding_method, flags, ipv4_outer_source_network, @@ -1039,7 +1038,7 @@ bool balancer_t::reconfigure_wlc() effective_weight, connections] : service_reals_usage_info) { - uint32_t wlc_power = scheduler_params; + uint32_t wlc_power = requested_wlc_power; if (wlc_power < 1 || wlc_power > 100) { wlc_power = YANET_CONFIG_BALANCER_WLC_DEFAULT_POWER; diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index 3b35bf26..178aff0d 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -1610,8 +1610,7 @@ inline uint64_t generation::count_real_connections(uint32_t counter_id) balancer_real_id_t* generation::rebuild_service_ring_one_wrr( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service, - balancer_service_range_t& range) + const balancer_service_t* service) { for (uint32_t real_idx = service->real_start; real_idx < service->real_start + service->real_size; @@ -1663,8 +1662,7 @@ generation::ServiceWeights(const balancer_service_t* service) balancer_real_id_t* generation::rebuild_service_ring_one_chash( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service, - balancer_service_range_t& range) + const balancer_service_t* service) { std::vector> reals; for (uint32_t real_idx = service->real_start; @@ -1677,8 +1675,8 @@ balancer_real_id_t* generation::rebuild_service_ring_one_chash( } auto updater = chash::WeightUpdater::MakeWeightUpdater( reals, - 20000, - 20); + 20000, + 20); if (!updater) { YANET_LOG_ERROR("Failed to intialize updater for balancer service reals: %ld\n", @@ -1688,30 +1686,29 @@ balancer_real_id_t* generation::rebuild_service_ring_one_chash( updater.value().InitLookup(ServiceWeights(service), start); chash_updaters.emplace(service, std::move(updater.value())); - return start; + return start + updater.value().LookupSize(); } -void generation::update_service_ring_one_chash( +balancer_real_id_t* generation::update_service_ring_one_chash( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service, - balancer_service_range_t& range) + const balancer_service_t* service) { if (chash_updaters.find(service) == chash_updaters.end()) { YANET_LOG_ERROR("No state information for updating requested service.\n"); - return; + return start; } auto& updater = chash_updaters.at(service); updater.UpdateLookup(ServiceWeights(service), start); + return start + updater.LookupSize(); } balancer_real_id_t* generation::evaluate_service_ring_one( ServiceRingOp op, balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service, - balancer_service_range_t& range) + const balancer_service_t* service) { balancer_real_id_t* end{start}; using scheduler = ::balancer::scheduler; @@ -1720,18 +1717,25 @@ balancer_real_id_t* generation::evaluate_service_ring_one( case scheduler::rr: case scheduler::wrr: end = rebuild_service_ring_one_wrr( - start, do_not_exceed, service, range); + start, do_not_exceed, service); break; case scheduler::wlc: case scheduler::chash: if (op == ServiceRingOp::Rebuild) { end = rebuild_service_ring_one_chash( - start, do_not_exceed, service, range); + start, do_not_exceed, service); } else { - update_service_ring_one_chash(start, do_not_exceed, service, range); + end = update_service_ring_one_chash(start, do_not_exceed, service); + } + { + std::unordered_map freq; + for (auto c = start; c != end; ++c) + { + freq[*c]++; + } } break; default: @@ -1757,8 +1761,7 @@ void generation::evaluate_service_ring(ServiceRingOp op) op, service_start, service_start + YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX, - service, - *range); + service); range->size = std::distance(service_start, service_end); } } diff --git a/dataplane/globalbase.h b/dataplane/globalbase.h index 95476bc5..491c2e3d 100644 --- a/dataplane/globalbase.h +++ b/dataplane/globalbase.h @@ -199,24 +199,20 @@ class generation balancer_real_id_t* rebuild_service_ring_one_wrr( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service, - balancer_service_range_t& range); + const balancer_service_t* service); balancer_real_id_t* rebuild_service_ring_one_chash( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service, - balancer_service_range_t& range); - void update_service_ring_one_chash( + const balancer_service_t* service); + balancer_real_id_t* update_service_ring_one_chash( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service, - balancer_service_range_t& range); + const balancer_service_t* service); balancer_real_id_t* evaluate_service_ring_one( ServiceRingOp op, balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service, - balancer_service_range_t& range); + const balancer_service_t* service); void evaluate_service_ring(ServiceRingOp op); inline uint64_t count_real_connections(uint32_t counter_id); From a5ee74f8563b3049ac9f30415c8374321559a4e3 Mon Sep 17 00:00:00 2001 From: vimes Date: Thu, 12 Dec 2024 14:09:31 +0300 Subject: [PATCH 20/27] Adds feedback on autotests errors --- autotest/autotest.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/autotest/autotest.cpp b/autotest/autotest.cpp index 5a3cef86..8656a80a 100644 --- a/autotest/autotest.cpp +++ b/autotest/autotest.cpp @@ -291,6 +291,7 @@ bool readTimeLimited(int fd, u_char* buf, ssize_t len, std::chrono::system_clock switch (ret) { case 0: + YANET_LOG_ERROR("Failed to read packets: end of file\n"); return false; case -1: if ((errno == EAGAIN) || (errno = EWOULDBLOCK)) @@ -299,6 +300,7 @@ bool readTimeLimited(int fd, u_char* buf, ssize_t len, std::chrono::system_clock } else { + YANET_LOG_ERROR("Failed to read packets: %s\n", strerror(errno)); return false; } break; @@ -319,6 +321,7 @@ static bool readPacket(int fd, pcap_pkthdr* header, u_char* data, Duration timel struct packHeader hdr; if (!readTimeLimited(fd, hdr, time_to_give_up)) { + YANET_LOG_ERROR("Failed to read packet header\n"); return false; } From 7259efa11c445b34990a07f6829ffb109315a4f7 Mon Sep 17 00:00:00 2001 From: vimes Date: Thu, 12 Dec 2024 14:11:38 +0300 Subject: [PATCH 21/27] Updates autotest 055_balancer_wlc expected data --- .../055_balancer_wlc/001-expect.pcap | Bin 3672 -> 3672 bytes .../055_balancer_wlc/002-expect.pcap | Bin 3672 -> 3672 bytes .../055_balancer_wlc/003-expect.pcap | Bin 3672 -> 3672 bytes .../055_balancer_wlc/autotest.yaml | 14 +++++++------- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/autotest/units/001_one_port/055_balancer_wlc/001-expect.pcap b/autotest/units/001_one_port/055_balancer_wlc/001-expect.pcap index 85cd0cb89836322bbab65d5da81fea2c162d6759..7db6cdc600d98694a5e11b014a55ce77305f06bb 100644 GIT binary patch delta 234 zcmca1b3?9QxhIy#OrFohJ@MYn$@R?KlMgYb18JVgrc5`0DtIPKGN%Eh z#U@W>1}mE^!_7UJoh1gWJC6mTCx{h9^Gr5p1F7YkyqYZqtXG;H#OI!v!##OAJ6Oxa zd)$*bInIG?dc*;-DU>q}tot~Sp3lWI*^(;_ta~+><_6lB4l-qOACLyQ@Bs_=WNw}q aFuRBcV#ZS*h#BF$5WOdXG{`J#K3M?2tWy5~ diff --git a/autotest/units/001_one_port/055_balancer_wlc/002-expect.pcap b/autotest/units/001_one_port/055_balancer_wlc/002-expect.pcap index 61fbaec326e61a3c45e17ccfd9ca80e60fbc4cbe..1d2adba45d01f224de95fb5547d6bce7a4b9dffe 100644 GIT binary patch delta 50 zcmca1b3E8}EuUhd5SjQUKI=W~f|p2}Rs3}oE92yPX70%a9Ne3u8C94VnKm~t_cJpxZx&#;W@BWW?8Ez*k!i94p9TOw CPYdw? diff --git a/autotest/units/001_one_port/055_balancer_wlc/003-expect.pcap b/autotest/units/001_one_port/055_balancer_wlc/003-expect.pcap index 553d1483dce0606e34b0b108ab113decc05e7a0a..d097ce8ece6554f96f8baa1e899e9ce420e4bd11 100644 GIT binary patch delta 152 zcmca1b3hkNpCmKdNE z_v9c}kPy%0qpWE_KHp?Zwv@@yHvQPwn&rpe}PX+S-ElUK7{1IbU!;hsF59irtQJ48zX2S^L|=0_a* fOkgwq06FU14vb(<5zkGa1w50(c_9X!=IsRlQB^b$ diff --git a/autotest/units/001_one_port/055_balancer_wlc/autotest.yaml b/autotest/units/001_one_port/055_balancer_wlc/autotest.yaml index 15ae3176..c7c308c4 100644 --- a/autotest/units/001_one_port/055_balancer_wlc/autotest.yaml +++ b/autotest/units/001_one_port/055_balancer_wlc/autotest.yaml @@ -14,9 +14,9 @@ steps: YANET_FORMAT_COLUMNS=module,virtual_ip,proto,virtual_port,scheduler,real_ip,real_port,enabled,weight,connections,packets,bytes balancer real any module virtual_ip proto virtual_port scheduler real_ip real_port enabled weight connections packets bytes --------- ---------- ----- ------------ --------- ------- --------- ------- ------ ----------- ------- ----- - balancer0 10.1.0.55 tcp 443 wlc 2443::1 443 true 2 16 16 1568 + balancer0 10.1.0.55 tcp 443 wlc 2443::1 443 true 2 15 15 1470 balancer0 10.1.0.55 tcp 443 wlc 2443::2 443 true 1 8 8 784 - balancer0 10.1.0.55 tcp 443 wlc 2443::3 443 true 1 8 8 784 + balancer0 10.1.0.55 tcp 443 wlc 2443::3 443 true 1 9 9 882 balancer0 10.1.0.55 tcp 443 wlc 2443::4 443 false 4 0 0 0 - cli: @@ -30,10 +30,10 @@ steps: YANET_FORMAT_COLUMNS=module,virtual_ip,proto,virtual_port,scheduler,real_ip,real_port,enabled,weight,connections,packets,bytes balancer real any module virtual_ip proto virtual_port scheduler real_ip real_port enabled weight connections packets bytes --------- ---------- ----- ------------ --------- ------- --------- ------- ------ ----------- ------- ----- - balancer0 10.1.0.55 tcp 443 wlc 2443::1 443 true 2 17 17 1666 - balancer0 10.1.0.55 tcp 443 wlc 2443::2 443 true 1 10 10 980 + balancer0 10.1.0.55 tcp 443 wlc 2443::1 443 true 2 16 16 1568 + balancer0 10.1.0.55 tcp 443 wlc 2443::2 443 true 1 8 8 784 balancer0 10.1.0.55 tcp 443 wlc 2443::3 443 true 1 9 9 882 - balancer0 10.1.0.55 tcp 443 wlc 2443::4 443 true 4 28 28 2744 + balancer0 10.1.0.55 tcp 443 wlc 2443::4 443 true 4 31 31 3038 - sleep: 2 - sendPackets: @@ -45,6 +45,6 @@ steps: module virtual_ip proto virtual_port scheduler real_ip real_port enabled weight connections packets bytes --------- ---------- ----- ------------ --------- ------- --------- ------- ------ ----------- ------- ----- balancer0 10.1.0.55 tcp 443 wlc 2443::1 443 true 2 24 24 2352 - balancer0 10.1.0.55 tcp 443 wlc 2443::2 443 true 1 11 11 1078 + balancer0 10.1.0.55 tcp 443 wlc 2443::2 443 true 1 12 12 1176 balancer0 10.1.0.55 tcp 443 wlc 2443::3 443 true 1 12 12 1176 - balancer0 10.1.0.55 tcp 443 wlc 2443::4 443 true 4 49 49 4802 + balancer0 10.1.0.55 tcp 443 wlc 2443::4 443 true 4 48 48 4704 From cd20fd0cdf18696db72ae968ecc34e50986cd412 Mon Sep 17 00:00:00 2001 From: vimes Date: Fri, 6 Dec 2024 13:22:08 +0300 Subject: [PATCH 22/27] Replaces goto with deferer object --- common/deferer.h | 23 +++++++++++++++++++++++ dataplane/globalbase.cpp | 6 +++--- 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 common/deferer.h diff --git a/common/deferer.h b/common/deferer.h new file mode 100644 index 00000000..9502ddbc --- /dev/null +++ b/common/deferer.h @@ -0,0 +1,23 @@ +namespace utils +{ + +template +class Deferer +{ + F action_; + +public: + Deferer(F&& action) : + action_{std::move(action)} + {} + Deferer(const Deferer& other) = delete; + Deferer(Deferer&& other) = delete; + Deferer& operator=(const Deferer& other) = delete; + Deferer& operator=(Deferer&& other) = delete; + ~Deferer() + { + action_(); + } +}; + +} // namespace utils \ No newline at end of file diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index 178aff0d..58af843f 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -9,6 +9,7 @@ #include "worker_gc.h" #include "common/counters.h" +#include "common/deferer.h" #include "common/define.h" #include "debug_latch.h" @@ -1612,6 +1613,7 @@ balancer_real_id_t* generation::rebuild_service_ring_one_wrr( const balancer_real_id_t* const do_not_exceed, const balancer_service_t* service) { + utils::Deferer defer([]() { YADECAP_MEMORY_BARRIER_COMPILE; }); for (uint32_t real_idx = service->real_start; real_idx < service->real_start + service->real_size; ++real_idx) @@ -1631,15 +1633,13 @@ balancer_real_id_t* generation::rebuild_service_ring_one_wrr( if (start == do_not_exceed) { YANET_LOG_ERROR("Balancer service exceeded ring chunk bounds\n"); - goto finish; + return start; } *start = real_id; ++start; } } -finish: - YADECAP_MEMORY_BARRIER_COMPILE; return start; } From ed5a0908d4d1ca4c9d8dccb51f57f2a7880305d7 Mon Sep 17 00:00:00 2001 From: vimes Date: Tue, 17 Dec 2024 17:08:11 +0300 Subject: [PATCH 23/27] Changes autotest --- autotest/autotest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/autotest/autotest.cpp b/autotest/autotest.cpp index 8656a80a..5a8bacd7 100644 --- a/autotest/autotest.cpp +++ b/autotest/autotest.cpp @@ -321,7 +321,6 @@ static bool readPacket(int fd, pcap_pkthdr* header, u_char* data, Duration timel struct packHeader hdr; if (!readTimeLimited(fd, hdr, time_to_give_up)) { - YANET_LOG_ERROR("Failed to read packet header\n"); return false; } From db3d789077d93eeb5fcef657d148994893959025 Mon Sep 17 00:00:00 2001 From: vimes Date: Tue, 17 Dec 2024 17:12:56 +0300 Subject: [PATCH 24/27] Constants --- common/define.h | 2 ++ controlplane/balancer.cpp | 12 ++++++------ dataplane/globalbase.cpp | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/common/define.h b/common/define.h index 47cc5fd0..760702c5 100644 --- a/common/define.h +++ b/common/define.h @@ -98,6 +98,8 @@ extern LogPriority logPriority; #define YANET_BALANCER_DEFAULT_MSS_SIZE 536 #define YANET_BALANCER_FIX_MSS_SIZE 1220 #define YANET_BALANCER_FIX_MSS_FLAG ((uint8_t)(1u << 0)) +static constexpr size_t YANET_DEFAULT_BALANCER_REAL_MAPPINGS_LIMIT = 20000; +static constexpr size_t YANET_DEFAULT_BALANCER_CELLS_PER_WEIGHT_UNIT = 20; #define YANET_BALANCER_OPS_FLAG ((uint8_t)(1u << 1)) #define YANET_BALANCER_PURE_L3 ((uint8_t)(1u << 2)) diff --git a/controlplane/balancer.cpp b/controlplane/balancer.cpp index cf681aa0..66ddfb6c 100644 --- a/controlplane/balancer.cpp +++ b/controlplane/balancer.cpp @@ -965,11 +965,11 @@ bool balancer_t::reconfigure_wlc() ipv6_outer_source_network, reals] : balancer.services) { - (void)flags; - (void)version; - (void)forwarding_method; - (void)ipv4_outer_source_network; - (void)ipv6_outer_source_network; + YANET_GCC_BUG_UNUSED(flags); + YANET_GCC_BUG_UNUSED(version); + YANET_GCC_BUG_UNUSED(forwarding_method); + YANET_GCC_BUG_UNUSED(ipv4_outer_source_network); + YANET_GCC_BUG_UNUSED(ipv6_outer_source_network); if (scheduler != ::balancer::scheduler::wlc) { @@ -1017,7 +1017,7 @@ bool balancer_t::reconfigure_wlc() uint32_t connections = 0; for (auto& [socket_id, real_connections] : balancer_real_connections) { - (void)socket_id; + YANET_GCC_BUG_UNUSED(socket_id); auto it = real_connections.find(real_connections_key); if (it == real_connections.end()) diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index 58af843f..83f7249a 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -1675,8 +1675,8 @@ balancer_real_id_t* generation::rebuild_service_ring_one_chash( } auto updater = chash::WeightUpdater::MakeWeightUpdater( reals, - 20000, - 20); + YANET_DEFAULT_BALANCER_REAL_MAPPINGS_LIMIT, + YANET_DEFAULT_BALANCER_CELLS_PER_WEIGHT_UNIT); if (!updater) { YANET_LOG_ERROR("Failed to intialize updater for balancer service reals: %ld\n", From 20297e6cd869a69ac26785eae37db0e1d62ead91 Mon Sep 17 00:00:00 2001 From: vimes Date: Tue, 17 Dec 2024 18:09:05 +0300 Subject: [PATCH 25/27] Some stylefix --- controlplane/balancer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/controlplane/balancer.cpp b/controlplane/balancer.cpp index 66ddfb6c..4c1df123 100644 --- a/controlplane/balancer.cpp +++ b/controlplane/balancer.cpp @@ -1019,12 +1019,12 @@ bool balancer_t::reconfigure_wlc() { YANET_GCC_BUG_UNUSED(socket_id); - auto it = real_connections.find(real_connections_key); - if (it == real_connections.end()) + if (auto it = real_connections.find(real_connections_key); it != real_connections.end()) { + connections += it->second; continue; + connections += it->second; } - connections += it->second; } connection_sum += connections; From 9eb04b3ac8475b66d7fe6e5334d169c6589564e3 Mon Sep 17 00:00:00 2001 From: vimes Date: Tue, 17 Dec 2024 18:13:49 +0300 Subject: [PATCH 26/27] References --- dataplane/globalbase.cpp | 62 ++++++++++++++++++++-------------------- dataplane/globalbase.h | 13 +++++---- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index 83f7249a..d802f671 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -1611,22 +1611,22 @@ inline uint64_t generation::count_real_connections(uint32_t counter_id) balancer_real_id_t* generation::rebuild_service_ring_one_wrr( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service) + const balancer_service_t& service) { utils::Deferer defer([]() { YADECAP_MEMORY_BARRIER_COMPILE; }); - for (uint32_t real_idx = service->real_start; - real_idx < service->real_start + service->real_size; + for (uint32_t real_idx = service.real_start; + real_idx < service.real_start + service.real_size; ++real_idx) { - uint32_t real_id = balancer_service_reals[real_idx]; - balancer_real_state_t* state = balancer_real_states + real_id; + balancer_real_id_t real_id = balancer_service_reals[real_idx]; + const balancer_real_state_t& state = balancer_real_states[real_id]; - if (state->weight == 0) + if (state.weight == 0) { continue; } - auto weight = state->weight; + auto weight = state.weight; while (weight-- > 0) { @@ -1643,16 +1643,15 @@ balancer_real_id_t* generation::rebuild_service_ring_one_wrr( return start; } -std::vector> -generation::ServiceWeights(const balancer_service_t* service) +std::vector generation::ServiceWeights(const balancer_service_t& service) { - std::vector> weights; - for (uint32_t real_idx = service->real_start; - real_idx < service->real_start + service->real_size; + std::vector weights; + for (uint32_t real_idx = service.real_start; + real_idx < service.real_start + service.real_size; ++real_idx) { - uint32_t real_id = balancer_service_reals[real_idx]; - uint32_t weight = (balancer_real_states + real_id)->weight; + balancer_real_id_t real_id = balancer_service_reals[real_idx]; + uint32_t weight = balancer_real_states[real_id].weight; weights.emplace_back(real_id, weight); } @@ -1662,14 +1661,15 @@ generation::ServiceWeights(const balancer_service_t* service) balancer_real_id_t* generation::rebuild_service_ring_one_chash( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service) + const balancer_service_t& service) { std::vector> reals; - for (uint32_t real_idx = service->real_start; - real_idx < service->real_start + service->real_size; + reals.reserve(service.real_size); + for (uint32_t real_idx = service.real_start; + real_idx < service.real_start + service.real_size; ++real_idx) { - uint32_t real_id = balancer_service_reals[real_idx]; + balancer_real_id_t real_id = balancer_service_reals[real_idx]; ipv6_address_t& ip = balancer_reals[real_id].destination; reals.emplace_back(ip, real_id); } @@ -1679,27 +1679,27 @@ balancer_real_id_t* generation::rebuild_service_ring_one_chash( YANET_DEFAULT_BALANCER_CELLS_PER_WEIGHT_UNIT); if (!updater) { - YANET_LOG_ERROR("Failed to intialize updater for balancer service reals: %ld\n", - reals.size()); + YANET_THROW("Failed to intialize updater for balancer service reals"); std::abort(); } updater.value().InitLookup(ServiceWeights(service), start); - chash_updaters.emplace(service, std::move(updater.value())); + chash_updaters.emplace(&service, std::move(updater.value())); return start + updater.value().LookupSize(); } balancer_real_id_t* generation::update_service_ring_one_chash( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service) + const balancer_service_t& service) { - if (chash_updaters.find(service) == chash_updaters.end()) + auto up = chash_updaters.find(&service); + if (up == chash_updaters.end()) { YANET_LOG_ERROR("No state information for updating requested service.\n"); return start; } - auto& updater = chash_updaters.at(service); + auto& updater = up->second; updater.UpdateLookup(ServiceWeights(service), start); return start + updater.LookupSize(); } @@ -1708,11 +1708,11 @@ balancer_real_id_t* generation::evaluate_service_ring_one( ServiceRingOp op, balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service) + const balancer_service_t& service) { balancer_real_id_t* end{start}; using scheduler = ::balancer::scheduler; - switch (service->scheduler) + switch (service.scheduler) { case scheduler::rr: case scheduler::wrr: @@ -1739,7 +1739,7 @@ balancer_real_id_t* generation::evaluate_service_ring_one( } break; default: - YANET_LOG_ERROR("Unknown balancer service scheduler type\n"); + YANET_THROW("Unknown balancer service scheduler type"); } return end; } @@ -1752,17 +1752,17 @@ void generation::evaluate_service_ring(ServiceRingOp op) service_idx < balancer_services_count; ++service_idx, service_start += YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX) { - balancer_service_t* service = balancer_services + balancer_active_services[service_idx]; + const balancer_service_t& service = balancer_services[balancer_active_services[service_idx]]; - balancer_service_range_t* range = ring->ranges + balancer_active_services[service_idx]; + balancer_service_range_t& range = ring->ranges[balancer_active_services[service_idx]]; - range->start = std::distance(ring->reals, service_start); + range.start = std::distance(ring->reals, service_start); auto service_end = evaluate_service_ring_one( op, service_start, service_start + YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX, service); - range->size = std::distance(service_start, service_end); + range.size = std::distance(service_start, service_end); } } diff --git a/dataplane/globalbase.h b/dataplane/globalbase.h index 491c2e3d..59c44ee0 100644 --- a/dataplane/globalbase.h +++ b/dataplane/globalbase.h @@ -194,25 +194,26 @@ class generation Rebuild }; - std::vector> - ServiceWeights(const balancer_service_t* service); + using RealWeight = std::pair; + + std::vector ServiceWeights(const balancer_service_t& service); balancer_real_id_t* rebuild_service_ring_one_wrr( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service); + const balancer_service_t& service); balancer_real_id_t* rebuild_service_ring_one_chash( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service); + const balancer_service_t& service); balancer_real_id_t* update_service_ring_one_chash( balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service); + const balancer_service_t& service); balancer_real_id_t* evaluate_service_ring_one( ServiceRingOp op, balancer_real_id_t* start, const balancer_real_id_t* const do_not_exceed, - const balancer_service_t* service); + const balancer_service_t& service); void evaluate_service_ring(ServiceRingOp op); inline uint64_t count_real_connections(uint32_t counter_id); From 92d22549ea0d15c1cfa99e5532b915497285c71a Mon Sep 17 00:00:00 2001 From: vimes Date: Tue, 17 Dec 2024 18:14:50 +0300 Subject: [PATCH 27/27] Reserves --- controlplane/balancer.cpp | 1 + dataplane/globalbase.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/controlplane/balancer.cpp b/controlplane/balancer.cpp index 4c1df123..339ec3b9 100644 --- a/controlplane/balancer.cpp +++ b/controlplane/balancer.cpp @@ -982,6 +982,7 @@ bool balancer_t::reconfigure_wlc() } std::vector> service_reals_usage_info; + service_reals_usage_info.reserve(reals.size()); uint32_t connection_sum = 0; uint32_t weight_sum = 0; diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index d802f671..2a761c8c 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -1646,6 +1646,7 @@ balancer_real_id_t* generation::rebuild_service_ring_one_wrr( std::vector generation::ServiceWeights(const balancer_service_t& service) { std::vector weights; + weights.reserve(service.real_size); for (uint32_t real_idx = service.real_start; real_idx < service.real_start + service.real_size; ++real_idx)