Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: wlc moved to from dataplane to controlplane #211

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

WIP: wlc moved to from dataplane to controlplane #211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 151 additions & 1 deletion controlplane/balancer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -886,6 +886,12 @@ void balancer_t::flush_reals(common::idp::updateGlobalBaseBalancer::request& bal
}
}

auto reals_wlc = reals_wlc_weight.find(key);
if (reals_wlc != reals_wlc_weight.end() && reals_wlc->second.has_value())
{
effective_weight = reals_wlc->second.value();
}
Comment on lines +889 to +893
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We will have code duplication here with the following lines:

auto it = reals_enabled.find(key);
if (it != reals_enabled.end())
{
	if (it->second.has_value())
	{
		effective_weight = it->second.value();
	}
}

I think we can add a private method in balancer_t like

 template <typename Map>
    std::optional<typename Map::mapped_type::value_type> get_effective_weight(const Map& map, const balancer::real_key_global_t& key) const {
        auto it = map.find(key);
        if (it != map.end() && it->second.has_value()) {
            return it->second.value();
        }
        return std::nullopt;
    }

And then use it, for example, with compound initialization:

if (auto found_weight = get_effective_weight(reals_wlc_weight, key); found_weight) {
    effective_weight = *found_weight;
}


uint32_t real_unordered_id = 0;
{
std::lock_guard<std::mutex> guard(reals_unordered_mutex);
Expand Down Expand Up @@ -929,8 +935,152 @@ void balancer_t::reconfigure_wlc_thread()
{
while (!flagStop)
{
balancer_real_flush();
if (balancer_t::reconfigure_wlc())
{
balancer_real_flush();
}

std::this_thread::sleep_for(std::chrono::seconds(YANET_CONFIG_BALANCER_WLC_RECONFIGURE));
}
}

bool balancer_t::reconfigure_wlc()
{
bool wlc_weight_changed = false;

common::idp::updateGlobalBaseBalancer::request balancer;
const auto balancer_real_connections = dataplane.balancer_real_connections();

std::lock_guard<std::mutex> guard(reals_enabled_mutex);

for (const auto& [module_name, balancer] : generations_config.current().config_balancers)
{

for (const auto& [service_id,
virtual_ip,
proto,
virtual_port,
version,
scheduler,
scheduler_params,
forwarding_method,
flags,
ipv4_outer_source_network,
ipv6_outer_source_network,
reals] : balancer.services)
{
(void)flags;
(void)version;
(void)forwarding_method;
(void)ipv4_outer_source_network;
(void)ipv6_outer_source_network;

if (scheduler != ::balancer::scheduler::wlc)
{
continue;
}

if (service_id >= YANET_CONFIG_BALANCER_SERVICES_SIZE)
{
continue;
}

std::vector<std::tuple<balancer::real_key_global_t, uint32_t, uint32_t>> service_reals_usage_info;
uint32_t connection_sum = 0;
uint32_t weight_sum = 0;

for (const auto& [real_ip, real_port, weight] : reals)
{
balancer::real_key_global_t key = {module_name, {virtual_ip, proto, virtual_port}, {real_ip, real_port}};
uint32_t effective_weight = weight;
{
auto it = reals_enabled.find(key);
if (it != reals_enabled.end())
{
if (it->second.has_value())
{
effective_weight = it->second.value();
}
}
}
Comment on lines +995 to +1005
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

auto found_weight = get_effective_weight(reals_enabled, key);
uint32_t effective_weight = found_weight ? *found_weight : weight;


weight_sum += effective_weight;

// don`t count connections for disabled reals - it can make other reals "feel" underloaded
if (effective_weight == 0)
{
continue;
}

common::idp::balancer_real_connections::real_key_t real_connections_key = {balancer.balancer_id,
virtual_ip,
proto,
virtual_port.value(),
real_ip,
real_port.value()};
uint32_t connections = 0;
for (auto& [socket_id, real_connections] : balancer_real_connections)
{
(void)socket_id;

auto it = real_connections.find(real_connections_key);
if (it == real_connections.end())
{
continue;
}
connections += it->second;
Comment on lines +1027 to +1031
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMHO

if (it != real_connections.end())
{
    connections += it->second;
}

is cleaner, but feel free no ignore

}

connection_sum += connections;

service_reals_usage_info.emplace_back(key,
effective_weight,
connections);
}

for (auto [key,
effective_weight,
connections] : service_reals_usage_info)
{
uint32_t wlc_power = scheduler_params.wlc_power;
if (wlc_power < 1 || wlc_power > 100)
{
wlc_power = YANET_CONFIG_BALANCER_WLC_DEFAULT_POWER;
}

effective_weight = calculate_wlc_weight(effective_weight, connections, weight_sum, connection_sum, wlc_power);

if (reals_wlc_weight[key] != effective_weight)
{
reals_wlc_weight[key] = effective_weight;
real_updates.insert(key);
if (in_reload)
{
real_reload_updates.insert(key);
}
wlc_weight_changed = true;
}
}
}
}

return wlc_weight_changed;
}

uint32_t balancer_t::calculate_wlc_weight(uint32_t weight, uint32_t connections, uint32_t weight_sum, uint32_t connection_sum, uint32_t wlc_power)
{
if (weight == 0 || weight_sum == 0 || connection_sum < weight_sum)
{
return weight;
}

auto wlc_ratio = std::max(1.0, wlc_power * (1 - 1.0 * connections * weight_sum / connection_sum / weight));
auto wlc_weight = (uint32_t)(weight * wlc_ratio);

if (wlc_weight > YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX)
{
wlc_weight = YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX;
}

return wlc_weight;
}
6 changes: 6 additions & 0 deletions controlplane/balancer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ class balancer_t : public module_t, common::icp_proto::BalancerService

std::map<balancer::real_key_global_t, std::optional<uint32_t>> reals_enabled;

std::map<balancer::real_key_global_t, std::optional<uint32_t>> reals_wlc_weight;

// The set contains all reals touched after the last one flush operation
std::set<balancer::real_key_global_t> real_updates;

Expand All @@ -143,7 +145,11 @@ class balancer_t : public module_t, common::icp_proto::BalancerService
friend class telegraf_t;
counter_t<balancer::service_counter_key_t, (size_t)balancer::service_counter::size> service_counters;
counter_t<balancer::real_counter_key_t, (size_t)balancer::real_counter::size> real_counters;

void RealFind(google::protobuf::RpcController* controller, const common::icp_proto::BalancerRealFindRequest* request, common::icp_proto::BalancerRealFindResponse* response, google::protobuf::Closure* done) override;
void Real(google::protobuf::RpcController* controller, const ::common::icp_proto::BalancerRealRequest* request, ::common::icp_proto::Empty* response, ::google::protobuf::Closure* done) override;
void RealFlush(google::protobuf::RpcController* controller, const ::common::icp_proto::Empty* request, ::common::icp_proto::Empty* response, ::google::protobuf::Closure* done) override;

bool reconfigure_wlc();
uint32_t calculate_wlc_weight(uint32_t weight, uint32_t connections, uint32_t weight_sum, uint32_t connection_sum, uint32_t wlc_power);
};
47 changes: 0 additions & 47 deletions dataplane/globalbase.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1610,16 +1610,6 @@ eResult generation::update_balancer_unordered_real(const common::idp::updateGlob
return eResult::success;
}

double wlc_ratio(uint32_t weight, uint32_t connections, uint32_t weight_sum, uint32_t connection_sum, uint32_t power)
{
if (weight == 0 || weight_sum == 0 || connection_sum < weight_sum)
{
return 1;
}
auto a = power * (1 - 1.0 * connections * weight_sum / connection_sum / weight);
return std::max(1.0, a);
}

inline uint64_t generation::count_real_connections(uint32_t counter_id)
{
uint64_t sessions_created = 0;
Expand Down Expand Up @@ -1653,27 +1643,6 @@ void generation::evaluate_service_ring()
for (uint32_t service_idx = 0; service_idx < balancer_services_count; ++service_idx)
{
balancer_service_t* service = balancer_services + balancer_active_services[service_idx];
uint64_t connection_sum = 0;
uint32_t weight_sum = 0;
if (service->scheduler == ::balancer::scheduler::wlc)
{
for (uint32_t real_idx = service->real_start;
real_idx < service->real_start + service->real_size;
++real_idx)
{
uint32_t real_id = balancer_service_reals[real_idx];
balancer_real_state_t* state = balancer_real_states + real_id;
// don`t count connections for disabled reals - it can make other reals "feel" underloaded
if (state->weight == 0)
{
continue;
}
weight_sum += state->weight;

const balancer_real_t& real = balancer_reals[real_id];
connection_sum += count_real_connections(real.counter_id);
}
}

balancer_service_range_t* range = ring->ranges + balancer_active_services[service_idx];

Expand All @@ -1683,7 +1652,6 @@ void generation::evaluate_service_ring()
++real_idx)
{
uint32_t real_id = balancer_service_reals[real_idx];
const balancer_real_t& real = balancer_reals[real_id];
balancer_real_state_t* state = balancer_real_states + real_id;

if (state->weight == 0)
Expand All @@ -1693,21 +1661,6 @@ void generation::evaluate_service_ring()

auto weight = state->weight;

if (service->scheduler == ::balancer::scheduler::wlc)
{
uint64_t real_connections = count_real_connections(real.counter_id);

weight = (int)(weight * wlc_ratio(state->weight, real_connections, weight_sum, connection_sum, service->wlc_power));
// todo check weight change
}

// clamp weight to a maximum possible value
if (weight > YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX)
{
// TODO: think about accounting the clamping
weight = YANET_CONFIG_BALANCER_REAL_WEIGHT_MAX;
}

while (weight-- > 0)
{
ring->reals[weight_pos++] = real_id;
Expand Down
Loading