diff --git a/.gitmodules b/.gitmodules index 288c73af..0b8a4cbd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,7 @@ [submodule "subprojects/json"] path = subprojects/json url = https://github.com/nlohmann/json.git +[submodule "subprojects/chash"] + path = subprojects/chash + url = https://github.com/yanet-platform/chash.git + branch = master diff --git a/autotest/autotest.cpp b/autotest/autotest.cpp index 5a3cef86..5a8bacd7 100644 --- a/autotest/autotest.cpp +++ b/autotest/autotest.cpp @@ -291,6 +291,7 @@ bool readTimeLimited(int fd, u_char* buf, ssize_t len, std::chrono::system_clock switch (ret) { case 0: + YANET_LOG_ERROR("Failed to read packets: end of file\n"); return false; case -1: if ((errno == EAGAIN) || (errno = EWOULDBLOCK)) @@ -299,6 +300,7 @@ bool readTimeLimited(int fd, u_char* buf, ssize_t len, std::chrono::system_clock } else { + YANET_LOG_ERROR("Failed to read packets: %s\n", strerror(errno)); return false; } break; diff --git a/autotest/units/001_one_port/055_balancer_wlc/001-expect.pcap b/autotest/units/001_one_port/055_balancer_wlc/001-expect.pcap index 85cd0cb8..7db6cdc6 100644 Binary files a/autotest/units/001_one_port/055_balancer_wlc/001-expect.pcap and b/autotest/units/001_one_port/055_balancer_wlc/001-expect.pcap differ diff --git a/autotest/units/001_one_port/055_balancer_wlc/002-expect.pcap b/autotest/units/001_one_port/055_balancer_wlc/002-expect.pcap index 61fbaec3..1d2adba4 100644 Binary files a/autotest/units/001_one_port/055_balancer_wlc/002-expect.pcap and b/autotest/units/001_one_port/055_balancer_wlc/002-expect.pcap differ diff --git a/autotest/units/001_one_port/055_balancer_wlc/003-expect.pcap b/autotest/units/001_one_port/055_balancer_wlc/003-expect.pcap index 553d1483..d097ce8e 100644 Binary files a/autotest/units/001_one_port/055_balancer_wlc/003-expect.pcap and b/autotest/units/001_one_port/055_balancer_wlc/003-expect.pcap differ diff --git a/autotest/units/001_one_port/055_balancer_wlc/autotest.yaml b/autotest/units/001_one_port/055_balancer_wlc/autotest.yaml index 15ae3176..c7c308c4 100644 --- a/autotest/units/001_one_port/055_balancer_wlc/autotest.yaml +++ b/autotest/units/001_one_port/055_balancer_wlc/autotest.yaml @@ -14,9 +14,9 @@ steps: YANET_FORMAT_COLUMNS=module,virtual_ip,proto,virtual_port,scheduler,real_ip,real_port,enabled,weight,connections,packets,bytes balancer real any module virtual_ip proto virtual_port scheduler real_ip real_port enabled weight connections packets bytes --------- ---------- ----- ------------ --------- ------- --------- ------- ------ ----------- ------- ----- - balancer0 10.1.0.55 tcp 443 wlc 2443::1 443 true 2 16 16 1568 + balancer0 10.1.0.55 tcp 443 wlc 2443::1 443 true 2 15 15 1470 balancer0 10.1.0.55 tcp 443 wlc 2443::2 443 true 1 8 8 784 - balancer0 10.1.0.55 tcp 443 wlc 2443::3 443 true 1 8 8 784 + balancer0 10.1.0.55 tcp 443 wlc 2443::3 443 true 1 9 9 882 balancer0 10.1.0.55 tcp 443 wlc 2443::4 443 false 4 0 0 0 - cli: @@ -30,10 +30,10 @@ steps: YANET_FORMAT_COLUMNS=module,virtual_ip,proto,virtual_port,scheduler,real_ip,real_port,enabled,weight,connections,packets,bytes balancer real any module virtual_ip proto virtual_port scheduler real_ip real_port enabled weight connections packets bytes --------- ---------- ----- ------------ --------- ------- --------- ------- ------ ----------- ------- ----- - balancer0 10.1.0.55 tcp 443 wlc 2443::1 443 true 2 17 17 1666 - balancer0 10.1.0.55 tcp 443 wlc 2443::2 443 true 1 10 10 980 + balancer0 10.1.0.55 tcp 443 wlc 2443::1 443 true 2 16 16 1568 + balancer0 10.1.0.55 tcp 443 wlc 2443::2 443 true 1 8 8 784 balancer0 10.1.0.55 tcp 443 wlc 2443::3 443 true 1 9 9 882 - balancer0 10.1.0.55 tcp 443 wlc 2443::4 443 true 4 28 28 2744 + balancer0 10.1.0.55 tcp 443 wlc 2443::4 443 true 4 31 31 3038 - sleep: 2 - sendPackets: @@ -45,6 +45,6 @@ steps: module virtual_ip proto virtual_port scheduler real_ip real_port enabled weight connections packets bytes --------- ---------- ----- ------------ --------- ------- --------- ------- ------ ----------- ------- ----- balancer0 10.1.0.55 tcp 443 wlc 2443::1 443 true 2 24 24 2352 - balancer0 10.1.0.55 tcp 443 wlc 2443::2 443 true 1 11 11 1078 + balancer0 10.1.0.55 tcp 443 wlc 2443::2 443 true 1 12 12 1176 balancer0 10.1.0.55 tcp 443 wlc 2443::3 443 true 1 12 12 1176 - balancer0 10.1.0.55 tcp 443 wlc 2443::4 443 true 4 49 49 4802 + balancer0 10.1.0.55 tcp 443 wlc 2443::4 443 true 4 48 48 4704 diff --git a/common/controlplaneconfig.h b/common/controlplaneconfig.h index da9a6d4d..dcb1e820 100644 --- a/common/controlplaneconfig.h +++ b/common/controlplaneconfig.h @@ -206,7 +206,7 @@ using service_t = std::tuple, ///< vport std::optional, ///< version ::balancer::scheduler, - ::balancer::scheduler_params, + uint32_t, ///< wlc power ::balancer::forwarding_method, uint8_t, ///< flags: mss_fix|ops std::optional, ///< ipv4_outer_source_network diff --git a/common/deferer.h b/common/deferer.h new file mode 100644 index 00000000..9502ddbc --- /dev/null +++ b/common/deferer.h @@ -0,0 +1,23 @@ +namespace utils +{ + +template +class Deferer +{ + F action_; + +public: + Deferer(F&& action) : + action_{std::move(action)} + {} + Deferer(const Deferer& other) = delete; + Deferer(Deferer&& other) = delete; + Deferer& operator=(const Deferer& other) = delete; + Deferer& operator=(Deferer&& other) = delete; + ~Deferer() + { + action_(); + } +}; + +} // namespace utils \ No newline at end of file diff --git a/common/define.h b/common/define.h index 47cc5fd0..760702c5 100644 --- a/common/define.h +++ b/common/define.h @@ -98,6 +98,8 @@ extern LogPriority logPriority; #define YANET_BALANCER_DEFAULT_MSS_SIZE 536 #define YANET_BALANCER_FIX_MSS_SIZE 1220 #define YANET_BALANCER_FIX_MSS_FLAG ((uint8_t)(1u << 0)) +static constexpr size_t YANET_DEFAULT_BALANCER_REAL_MAPPINGS_LIMIT = 20000; +static constexpr size_t YANET_DEFAULT_BALANCER_CELLS_PER_WEIGHT_UNIT = 20; #define YANET_BALANCER_OPS_FLAG ((uint8_t)(1u << 1)) #define YANET_BALANCER_PURE_L3 ((uint8_t)(1u << 2)) diff --git a/common/idp.h b/common/idp.h index 8719cd0d..745eb3a8 100644 --- a/common/idp.h +++ b/common/idp.h @@ -283,7 +283,6 @@ using service = std::tuple, ///< ipv4_outer_source_network diff --git a/common/scheduler.h b/common/scheduler.h index 6fe773ac..e213b7ca 100644 --- a/common/scheduler.h +++ b/common/scheduler.h @@ -1,4 +1,5 @@ #pragma once +#include #include namespace balancer @@ -9,13 +10,7 @@ enum class scheduler : uint8_t rr, wrr, wlc, -}; - -class scheduler_params -{ -public: - scheduler_params() = default; - uint32_t wlc_power; + chash }; [[maybe_unused]] constexpr const char* to_string(const scheduler& scheduler) @@ -23,20 +18,16 @@ class scheduler_params switch (scheduler) { case scheduler::rr: - { return "rr"; - } case scheduler::wrr: - { return "wrr"; - } case scheduler::wlc: - { return "wlc"; - } + case scheduler::chash: + return "chash"; + default: + return "unknown"; } - - return "unknown"; } } diff --git a/controlplane/balancer.cpp b/controlplane/balancer.cpp index 4774ef76..339ec3b9 100644 --- a/controlplane/balancer.cpp +++ b/controlplane/balancer.cpp @@ -757,14 +757,14 @@ void balancer_t::compile(common::idp::updateGlobalBase::request& globalbase, virtual_port, version, scheduler, - scheduler_params, + wlc_power, forwarding_method, flags, ipv4_outer_source_network, ipv6_outer_source_network, reals] : balancer.services) { - YANET_GCC_BUG_UNUSED(scheduler_params); + YANET_GCC_BUG_UNUSED(wlc_power); YANET_GCC_BUG_UNUSED(version); if (service_id >= YANET_CONFIG_BALANCER_SERVICES_SIZE) @@ -808,7 +808,6 @@ void balancer_t::compile(common::idp::updateGlobalBase::request& globalbase, counter_id, scheduler, forwarding_method, - balancer.default_wlc_power, // todo use scheduler_params.wlc_power when other services will be able to set it (uint32_t)real_start, (uint32_t)(req_reals.size() - real_start), ipv4_outer_source_network, @@ -883,6 +882,12 @@ void balancer_t::flush_reals(common::idp::updateGlobalBaseBalancer::request& bal } } + auto reals_wlc = reals_wlc_weight.find(key); + if (reals_wlc != reals_wlc_weight.end() && reals_wlc->second.has_value()) + { + effective_weight = reals_wlc->second.value(); + } + uint32_t real_unordered_id = 0; { std::lock_guard guard(reals_unordered_mutex); @@ -926,8 +931,153 @@ void balancer_t::reconfigure_wlc_thread() { while (!flagStop) { - balancer_real_flush(); + if (balancer_t::reconfigure_wlc()) + { + balancer_real_flush(); + } std::this_thread::sleep_for(std::chrono::seconds(YANET_CONFIG_BALANCER_WLC_RECONFIGURE)); } } + +bool balancer_t::reconfigure_wlc() +{ + bool wlc_weight_changed = false; + + common::idp::updateGlobalBaseBalancer::request balancer; + const auto balancer_real_connections = dataplane.balancer_real_connections(); + + std::lock_guard guard(reals_enabled_mutex); + + for (const auto& [module_name, balancer] : generations_config.current().config_balancers) + { + + for (const auto& [service_id, + virtual_ip, + proto, + virtual_port, + version, + scheduler, + requested_wlc_power, + forwarding_method, + flags, + ipv4_outer_source_network, + ipv6_outer_source_network, + reals] : balancer.services) + { + YANET_GCC_BUG_UNUSED(flags); + YANET_GCC_BUG_UNUSED(version); + YANET_GCC_BUG_UNUSED(forwarding_method); + YANET_GCC_BUG_UNUSED(ipv4_outer_source_network); + YANET_GCC_BUG_UNUSED(ipv6_outer_source_network); + + if (scheduler != ::balancer::scheduler::wlc) + { + continue; + } + + if (service_id >= YANET_CONFIG_BALANCER_SERVICES_SIZE) + { + continue; + } + + std::vector> service_reals_usage_info; + service_reals_usage_info.reserve(reals.size()); + uint32_t connection_sum = 0; + uint32_t weight_sum = 0; + + for (const auto& [real_ip, real_port, weight] : reals) + { + balancer::real_key_global_t key = {module_name, {virtual_ip, proto, virtual_port}, {real_ip, real_port}}; + uint32_t effective_weight = weight; + { + auto it = reals_enabled.find(key); + if (it != reals_enabled.end()) + { + if (it->second.has_value()) + { + effective_weight = it->second.value(); + } + } + } + + weight_sum += effective_weight; + + // don`t count connections for disabled reals - it can make other reals "feel" underloaded + if (effective_weight == 0) + { + continue; + } + + common::idp::balancer_real_connections::real_key_t real_connections_key = {balancer.balancer_id, + virtual_ip, + proto, + virtual_port.value(), + real_ip, + real_port.value()}; + uint32_t connections = 0; + for (auto& [socket_id, real_connections] : balancer_real_connections) + { + YANET_GCC_BUG_UNUSED(socket_id); + + if (auto it = real_connections.find(real_connections_key); it != real_connections.end()) + { + connections += it->second; + continue; + connections += it->second; + } + } + + connection_sum += connections; + + service_reals_usage_info.emplace_back(key, + effective_weight, + connections); + } + + for (auto [key, + effective_weight, + connections] : service_reals_usage_info) + { + uint32_t wlc_power = requested_wlc_power; + if (wlc_power < 1 || wlc_power > 100) + { + wlc_power = YANET_CONFIG_BALANCER_WLC_DEFAULT_POWER; + } + + effective_weight = calculate_wlc_weight(effective_weight, connections, weight_sum, connection_sum, wlc_power); + + if (reals_wlc_weight[key] != effective_weight) + { + reals_wlc_weight[key] = effective_weight; + real_updates.insert(key); + if (in_reload) + { + real_reload_updates.insert(key); + } + wlc_weight_changed = true; + } + } + } + } + + return wlc_weight_changed; +} + +uint32_t balancer_t::calculate_wlc_weight(uint32_t weight, uint32_t connections, uint32_t weight_sum, uint32_t connection_sum, uint32_t wlc_power) +{ + if (weight == 0 || weight_sum == 0 || connection_sum < weight_sum) + { + return weight; + } + + auto wlc_ratio = std::max(1.0, wlc_power * (1 - 1.0 * connections * weight_sum / connection_sum / weight)); + auto wlc_weight = (uint32_t)(weight * wlc_ratio); + + if (wlc_weight > YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX) + { + wlc_weight = YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX; + } + + return wlc_weight; +} \ No newline at end of file diff --git a/controlplane/balancer.h b/controlplane/balancer.h index 3bcba0cf..b0a3b10a 100644 --- a/controlplane/balancer.h +++ b/controlplane/balancer.h @@ -113,6 +113,8 @@ class balancer_t : public module_t, common::icp_proto::BalancerService std::map> reals_enabled; + std::map> reals_wlc_weight; + // The set contains all reals touched after the last one flush operation std::set real_updates; @@ -133,7 +135,11 @@ class balancer_t : public module_t, common::icp_proto::BalancerService friend class telegraf_t; counter_t service_counters; counter_t real_counters; + void RealFind(google::protobuf::RpcController* controller, const common::icp_proto::BalancerRealFindRequest* request, common::icp_proto::BalancerRealFindResponse* response, google::protobuf::Closure* done) override; void Real(google::protobuf::RpcController* controller, const ::common::icp_proto::BalancerRealRequest* request, ::common::icp_proto::Empty* response, ::google::protobuf::Closure* done) override; void RealFlush(google::protobuf::RpcController* controller, const ::common::icp_proto::Empty* request, ::common::icp_proto::Empty* response, ::google::protobuf::Closure* done) override; + + bool reconfigure_wlc(); + uint32_t calculate_wlc_weight(uint32_t weight, uint32_t connections, uint32_t weight_sum, uint32_t connection_sum, uint32_t wlc_power); }; diff --git a/controlplane/configparser.cpp b/controlplane/configparser.cpp index 92054e4c..742dbb0e 100644 --- a/controlplane/configparser.cpp +++ b/controlplane/configparser.cpp @@ -1650,7 +1650,7 @@ void config_parser_t::loadConfig_balancer_services(controlplane::base_t& baseNex std::string scheduler_string = service_json["scheduler"]; balancer::scheduler scheduler{}; - balancer::scheduler_params scheduler_params{}; + std::uint32_t wlc_power{}; if (scheduler_string == "rr") { scheduler = balancer::scheduler::rr; @@ -1664,9 +1664,13 @@ void config_parser_t::loadConfig_balancer_services(controlplane::base_t& baseNex scheduler = balancer::scheduler::wlc; if (exist(service_json, "scheduler_params") && exist(service_json["scheduler_params"], "wlc_power")) { - scheduler_params.wlc_power = std::stoll(service_json["scheduler_params"]["wlc_power"].get(), nullptr, 10); + wlc_power = std::stoll(service_json["scheduler_params"]["wlc_power"].get(), nullptr, 10); } } + else if (scheduler_string == "chash") + { + scheduler = balancer::scheduler::chash; + } else { throw error_result_t(eResult::invalidConfigurationFile, "unknown scheduler: " + scheduler_string); @@ -1749,7 +1753,7 @@ void config_parser_t::loadConfig_balancer_services(controlplane::base_t& baseNex exist(service_json, "vport") ? std::make_optional(std::stoll(service_json["vport"].get(), nullptr, 0)) : std::nullopt, service_version, scheduler, - scheduler_params, + wlc_power, forwarding_method, flags, ipv4_outer_source_network, diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index e199c004..2a761c8c 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -9,6 +9,7 @@ #include "worker_gc.h" #include "common/counters.h" +#include "common/deferer.h" #include "common/define.h" #include "debug_latch.h" @@ -1424,7 +1425,6 @@ eResult generation::update_balancer_services(const common::idp::updateGlobalBase counter_id, scheduler, forwarding_method, - default_wlc_power, real_start, real_size, ipv4_outer_source_network, @@ -1475,7 +1475,6 @@ eResult generation::update_balancer_services(const common::idp::updateGlobalBase balancer_service.real_start = real_start; balancer_service.real_size = real_size; balancer_service.scheduler = scheduler; - balancer_service.wlc_power = default_wlc_power; balancer_service.forwarding_method = forwarding_method; balancer_service.outer_source_network_flag = outer_source_network_flag; balancer_service.ipv4_outer_source_network = ipv4_prefix; @@ -1556,7 +1555,7 @@ eResult generation::update_balancer_services(const common::idp::updateGlobalBase std::copy(binding.begin(), binding.end(), balancer_service_reals); - evaluate_service_ring(); + evaluate_service_ring(ServiceRingOp::Rebuild); return eResult::success; } @@ -1579,21 +1578,11 @@ eResult generation::update_balancer_unordered_real(const common::idp::updateGlob real_state = new_state; } - evaluate_service_ring(); + evaluate_service_ring(ServiceRingOp::Update); return eResult::success; } -double wlc_ratio(uint32_t weight, uint32_t connections, uint32_t weight_sum, uint32_t connection_sum, uint32_t power) -{ - if (weight == 0 || weight_sum == 0 || connection_sum < weight_sum) - { - return 1; - } - auto a = power * (1 - 1.0 * connections * weight_sum / connection_sum / weight); - return std::max(1.0, a); -} - inline uint64_t generation::count_real_connections(uint32_t counter_id) { uint64_t sessions_created = 0; @@ -1619,78 +1608,162 @@ inline uint64_t generation::count_real_connections(uint32_t counter_id) return (sessions_created - sessions_destroyed + sessions_created_gc - sessions_destroyed_gc) / dataPlane->numaNodesInUse; } -void generation::evaluate_service_ring() +balancer_real_id_t* generation::rebuild_service_ring_one_wrr( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service) { - balancer_service_ring_t* ring = &balancer_service_ring; - uint32_t weight_pos = 0; - for (uint32_t service_idx = 0; service_idx < balancer_services_count; ++service_idx) + utils::Deferer defer([]() { YADECAP_MEMORY_BARRIER_COMPILE; }); + for (uint32_t real_idx = service.real_start; + real_idx < service.real_start + service.real_size; + ++real_idx) { - balancer_service_t* service = balancer_services + balancer_active_services[service_idx]; - uint64_t connection_sum = 0; - uint32_t weight_sum = 0; - if (service->scheduler == ::balancer::scheduler::wlc) - { - for (uint32_t real_idx = service->real_start; - real_idx < service->real_start + service->real_size; - ++real_idx) - { - uint32_t real_id = balancer_service_reals[real_idx]; - balancer_real_state_t* state = balancer_real_states + real_id; - // don`t count connections for disabled reals - it can make other reals "feel" underloaded - if (state->weight == 0) - { - continue; - } - weight_sum += state->weight; + balancer_real_id_t real_id = balancer_service_reals[real_idx]; + const balancer_real_state_t& state = balancer_real_states[real_id]; - const balancer_real_t& real = balancer_reals[real_id]; - connection_sum += count_real_connections(real.counter_id); - } + if (state.weight == 0) + { + continue; } - balancer_service_range_t* range = ring->ranges + balancer_active_services[service_idx]; + auto weight = state.weight; - range->start = weight_pos; - for (uint32_t real_idx = service->real_start; - real_idx < service->real_start + service->real_size; - ++real_idx) + while (weight-- > 0) { - uint32_t real_id = balancer_service_reals[real_idx]; - const balancer_real_t& real = balancer_reals[real_id]; - balancer_real_state_t* state = balancer_real_states + real_id; - - if (state->weight == 0) + if (start == do_not_exceed) { - continue; + YANET_LOG_ERROR("Balancer service exceeded ring chunk bounds\n"); + return start; } + *start = real_id; + ++start; + } + } - auto weight = state->weight; + return start; +} - if (service->scheduler == ::balancer::scheduler::wlc) - { - uint64_t real_connections = count_real_connections(real.counter_id); +std::vector generation::ServiceWeights(const balancer_service_t& service) +{ + std::vector weights; + weights.reserve(service.real_size); + for (uint32_t real_idx = service.real_start; + real_idx < service.real_start + service.real_size; + ++real_idx) + { + balancer_real_id_t real_id = balancer_service_reals[real_idx]; + uint32_t weight = balancer_real_states[real_id].weight; - weight = (int)(weight * wlc_ratio(state->weight, real_connections, weight_sum, connection_sum, service->wlc_power)); - // todo check weight change - } + weights.emplace_back(real_id, weight); + } + return weights; +} + +balancer_real_id_t* generation::rebuild_service_ring_one_chash( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service) +{ + std::vector> reals; + reals.reserve(service.real_size); + for (uint32_t real_idx = service.real_start; + real_idx < service.real_start + service.real_size; + ++real_idx) + { + balancer_real_id_t real_id = balancer_service_reals[real_idx]; + ipv6_address_t& ip = balancer_reals[real_id].destination; + reals.emplace_back(ip, real_id); + } + auto updater = chash::WeightUpdater::MakeWeightUpdater( + reals, + YANET_DEFAULT_BALANCER_REAL_MAPPINGS_LIMIT, + YANET_DEFAULT_BALANCER_CELLS_PER_WEIGHT_UNIT); + if (!updater) + { + YANET_THROW("Failed to intialize updater for balancer service reals"); + std::abort(); + } + updater.value().InitLookup(ServiceWeights(service), start); + + chash_updaters.emplace(&service, std::move(updater.value())); + return start + updater.value().LookupSize(); +} - // clamp weight to a maximum possible value - if (weight > YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX) +balancer_real_id_t* generation::update_service_ring_one_chash( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service) +{ + auto up = chash_updaters.find(&service); + if (up == chash_updaters.end()) + { + YANET_LOG_ERROR("No state information for updating requested service.\n"); + return start; + } + auto& updater = up->second; + updater.UpdateLookup(ServiceWeights(service), start); + return start + updater.LookupSize(); +} + +balancer_real_id_t* generation::evaluate_service_ring_one( + ServiceRingOp op, + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service) +{ + balancer_real_id_t* end{start}; + using scheduler = ::balancer::scheduler; + switch (service.scheduler) + { + case scheduler::rr: + case scheduler::wrr: + end = rebuild_service_ring_one_wrr( + start, do_not_exceed, service); + break; + case scheduler::wlc: + case scheduler::chash: + if (op == ServiceRingOp::Rebuild) { - // TODO: think about accounting the clamping - weight = YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX; + end = rebuild_service_ring_one_chash( + start, do_not_exceed, service); } - - while (weight-- > 0) + else { - ring->reals[weight_pos++] = real_id; + end = update_service_ring_one_chash(start, do_not_exceed, service); } - } - - YADECAP_MEMORY_BARRIER_COMPILE; + { + std::unordered_map freq; + for (auto c = start; c != end; ++c) + { + freq[*c]++; + } + } + break; + default: + YANET_THROW("Unknown balancer service scheduler type"); + } + return end; +} - range->size = weight_pos - range->start; - weight_pos = range->start + service->real_size * YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX; +void generation::evaluate_service_ring(ServiceRingOp op) +{ + balancer_service_ring_t* ring = &balancer_service_ring; + balancer_real_id_t* service_start = ring->reals; + for (uint32_t service_idx = 0; + service_idx < balancer_services_count; + ++service_idx, service_start += YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX) + { + const balancer_service_t& service = balancer_services[balancer_active_services[service_idx]]; + + balancer_service_range_t& range = ring->ranges[balancer_active_services[service_idx]]; + + range.start = std::distance(ring->reals, service_start); + auto service_end = evaluate_service_ring_one( + op, + service_start, + service_start + YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX, + service); + range.size = std::distance(service_start, service_end); } } diff --git a/dataplane/globalbase.h b/dataplane/globalbase.h index fbb38538..59c44ee0 100644 --- a/dataplane/globalbase.h +++ b/dataplane/globalbase.h @@ -7,6 +7,8 @@ #include #include +#include + #include "common/idp.h" #include "common/result.h" #include "common/tsc_deltas.h" @@ -186,7 +188,33 @@ class generation eResult tsc_state_update(const common::idp::updateGlobalBase::tsc_state_update::request& request); eResult tscs_base_value_update(const common::idp::updateGlobalBase::tscs_base_value_update::request& request); - void evaluate_service_ring(); + enum class ServiceRingOp + { + Update, + Rebuild + }; + + using RealWeight = std::pair; + + std::vector ServiceWeights(const balancer_service_t& service); + balancer_real_id_t* rebuild_service_ring_one_wrr( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service); + balancer_real_id_t* rebuild_service_ring_one_chash( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service); + balancer_real_id_t* update_service_ring_one_chash( + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service); + balancer_real_id_t* evaluate_service_ring_one( + ServiceRingOp op, + balancer_real_id_t* start, + const balancer_real_id_t* const do_not_exceed, + const balancer_service_t& service); + void evaluate_service_ring(ServiceRingOp op); inline uint64_t count_real_connections(uint32_t counter_id); public: ///< @todo @@ -309,6 +337,7 @@ class generation balancer_real_state_t balancer_real_states[YANET_CONFIG_BALANCER_REALS_SIZE]; balancer_service_ring_t balancer_service_ring; + std::map chash_updaters; int64_t dump_id_to_tag[YANET_CONFIG_DUMP_ID_TO_TAG_SIZE]; diff --git a/dataplane/meson.build b/dataplane/meson.build index 73962ecd..3fc2f067 100644 --- a/dataplane/meson.build +++ b/dataplane/meson.build @@ -1,6 +1,7 @@ dependencies = [] dependencies += libdpdk.get_variable('dpdk_dep') dependencies += libjson.get_variable('nlohmann_json_dep') +dependencies += libchash.get_variable('chash_dep') dependencies += dependency('libsystemd') dependencies += dependency('threads') diff --git a/dataplane/type.h b/dataplane/type.h index cf55b213..8ffe1486 100644 --- a/dataplane/type.h +++ b/dataplane/type.h @@ -206,6 +206,11 @@ struct ipv6_address_t memset(bytes, 0, std::size(bytes)); } + bool operator<(ipv6_address_t other) const + { + return std::memcmp(bytes, other.bytes, 16) < 0; + } + union { uint8_t bytes[16]; ///< @todo: rename to address @@ -578,7 +583,8 @@ struct balancer_service_t uint32_t real_size; ::balancer::scheduler scheduler; ::balancer::forwarding_method forwarding_method; - uint32_t wlc_power; + + std::uint32_t wlc_power; /* outer_source_network_flag: diff --git a/librib/meson.build b/librib/meson.build index c2748905..890e9aff 100644 --- a/librib/meson.build +++ b/librib/meson.build @@ -11,9 +11,3 @@ libyabird = library('yanet-rib', include_directories: yanet_rootdir, dependencies: dependencies, install: true) - -libyabird_static = static_library('yanet-rib', - sources, - include_directories: yanet_rootdir, - dependencies: dependencies, - install: true) diff --git a/meson.build b/meson.build index 08c20a68..d6d2621d 100644 --- a/meson.build +++ b/meson.build @@ -4,6 +4,7 @@ project('yanet', 'cpp', 'buildtype=release', 'warning_level=2', 'werror=true', + 'default_library=static', 'b_lto=true']) yanet_rootdir = include_directories('.') @@ -58,6 +59,8 @@ libdpdk = subproject('dpdk', default_options: [ libjson = subproject('json') +libchash = subproject('chash', default_options: [ 'tests=false' ]) + if target_option.contains('buildenv') subdir('libprotobuf') subdir('libfwparser') diff --git a/subprojects/chash b/subprojects/chash new file mode 160000 index 00000000..3a374315 --- /dev/null +++ b/subprojects/chash @@ -0,0 +1 @@ +Subproject commit 3a37431565d8a43dc47ac3c970ea0411c6edbaf5