From 1f170574ce5eb649164405444acdc40693f9a339 Mon Sep 17 00:00:00 2001 From: Timur Aitov Date: Fri, 26 Jan 2024 19:13:31 +0300 Subject: [PATCH] Dynamic allocation memory 1) allocate memory on update for some acl structs 2) move acl to module in dataplane 3) update controlplane <=> dataplane API for sync multiple modules --- autotest/autotest.cpp | 2 - .../070_neighbor_resolve/autotest.yaml | 4 - .../autotest.yaml | 1 - cli/main.cpp | 1 - cli/neighbor.h | 6 - cli/show.h | 12 +- common/acl.h | 22 ++ common/generation.h | 22 +- common/idataplane.h | 60 +++- common/idp.h | 194 ++++++----- common/limit.h | 16 + common/memory_manager.h | 61 ++++ common/neighbor.h | 54 ++++ common/stream.h | 17 + common/type.h | 5 + controlplane/acl.h | 11 +- controlplane/acl_transport_table.h | 2 +- controlplane/base.h | 2 + controlplane/configconverter.cpp | 49 ++- controlplane/configconverter.h | 6 + controlplane/configparser.cpp | 31 ++ controlplane/configparser.h | 1 + controlplane/controlplane.cpp | 25 +- controlplane/controlplane.h | 2 + controlplane/dregress.cpp | 9 +- controlplane/memory_manager.cpp | 12 + controlplane/memory_manager.h | 21 ++ controlplane/meson.build | 1 + controlplane/route.cpp | 13 +- dataplane/acl.cpp | 191 +++++++++++ dataplane/acl.h | 69 ++++ dataplane/base.h | 15 +- dataplane/bus.cpp | 32 +- dataplane/common.h | 10 + dataplane/controlplane.cpp | 178 ++--------- dataplane/controlplane.h | 6 +- dataplane/dataplane.cpp | 203 +++++++++--- dataplane/dataplane.h | 22 +- dataplane/globalbase.cpp | 91 +----- dataplane/globalbase.h | 24 -- dataplane/hashtable.h | 302 ++++++++---------- dataplane/lpm.h | 296 ++++++++--------- dataplane/main.cpp | 24 ++ dataplane/memory_manager.cpp | 142 ++++++++ dataplane/memory_manager.h | 75 +++++ dataplane/meson.build | 4 +- dataplane/neighbor.cpp | 297 ++++++++--------- dataplane/neighbor.h | 23 +- dataplane/report.cpp | 7 +- dataplane/updater.h | 219 +++++++++++++ dataplane/worker.cpp | 96 +++--- 51 files changed, 1900 insertions(+), 1088 deletions(-) create mode 100644 common/limit.h create mode 100644 common/memory_manager.h create mode 100644 controlplane/memory_manager.cpp create mode 100644 controlplane/memory_manager.h create mode 100644 dataplane/acl.cpp create mode 100644 dataplane/acl.h create mode 100644 dataplane/memory_manager.cpp create mode 100644 dataplane/memory_manager.h create mode 100644 dataplane/updater.h diff --git a/autotest/autotest.cpp b/autotest/autotest.cpp index 53cd151a..480addb5 100644 --- a/autotest/autotest.cpp +++ b/autotest/autotest.cpp @@ -1308,8 +1308,6 @@ void tAutotest::mainThread() this->request.swap(request); } - dataPlane.neighbor_flush(); - YAML::Node yamlRoot = YAML::LoadFile(configFilePath + "/autotest.yaml"); for (const YAML::Node& yamlStep : yamlRoot["steps"]) diff --git a/autotest/units/001_one_port/070_neighbor_resolve/autotest.yaml b/autotest/units/001_one_port/070_neighbor_resolve/autotest.yaml index c8c13c23..02f50d58 100644 --- a/autotest/units/001_one_port/070_neighbor_resolve/autotest.yaml +++ b/autotest/units/001_one_port/070_neighbor_resolve/autotest.yaml @@ -31,8 +31,6 @@ steps: route0 kni0.200 fe80::1 42:42:A4:59:BE:A5 - cli: neighbor insert route0 kni0.100 200.0.0.2 00:11:22:33:44:55 -- cli: neighbor flush - - cli_check: | YANET_FORMAT_COLUMNS=route_name,interface_name,ip_address,mac_address neighbor show route_name interface_name ip_address mac_address @@ -42,8 +40,6 @@ steps: route0 kni0.200 fe80::1 42:42:A4:59:BE:A5 - cli: neighbor remove route0 kni0.100 200.0.0.2 -- cli: neighbor flush - - cli_check: | YANET_FORMAT_COLUMNS=route_name,interface_name,ip_address,mac_address neighbor show route_name interface_name ip_address mac_address diff --git a/autotest/units/001_one_port/071_route_directly_connected/autotest.yaml b/autotest/units/001_one_port/071_route_directly_connected/autotest.yaml index 94de4271..3c378c15 100644 --- a/autotest/units/001_one_port/071_route_directly_connected/autotest.yaml +++ b/autotest/units/001_one_port/071_route_directly_connected/autotest.yaml @@ -8,7 +8,6 @@ steps: - neighbor insert route0 kni0.200 10.20.0.250 00:00:ee:20:44:fa - neighbor insert route0 kni0.200 2000:200::2 00:00:ee:20:66:02 - neighbor insert route0 kni0.200 2000:200::fa 00:00:ee:20:66:fa - - neighbor flush - sendPackets: - port: kni0 send: 001-send.pcap diff --git a/cli/main.cpp b/cli/main.cpp index 0ed1855d..3d9b637d 100644 --- a/cli/main.cpp +++ b/cli/main.cpp @@ -79,7 +79,6 @@ std::vector; using ranges_uint16_t = ranges_t; +// + +namespace idp +{ + +using network_ipv4_source = std::vector; +using network_ipv4_destination = std::vector; +using network_ipv6_destination_ht = std::vector>; +using transport_table = std::vector>; +using total_table = std::vector>; + +using request = std::tuple; + +using response = eResult; + +} + } diff --git a/common/generation.h b/common/generation.h index 6c434be1..ab53e7da 100644 --- a/common/generation.h +++ b/common/generation.h @@ -137,6 +137,11 @@ class generation_manager next_mutex.unlock(); } + [[nodiscard]] std::unique_lock next_lock_guard() const + { + return std::move(std::unique_lock(next_mutex)); + } + Type& next() { return generations[id ^ 1]; @@ -186,18 +191,28 @@ class generation_manager template Type_Update_Result update(const function_t& function) { - next_lock(); auto result = function(next()); requests.emplace_back(function); - next_unlock(); return result; } + /// apply all previous requests for next generation + void next_apply() + { + if (requests.size()) + { + for (const auto& function : requests) + { + function(next()); + } + requests.clear(); + } + } + /// switch generation and apply all previous requests for next generation template void switch_generation_with_update(const wait_function_t& wait_function) { - next_lock(); if (requests.size()) { /// update current pointer @@ -213,7 +228,6 @@ class generation_manager } requests.clear(); } - next_unlock(); } protected: diff --git a/common/idataplane.h b/common/idataplane.h index aeeb26d7..011285a1 100644 --- a/common/idataplane.h +++ b/common/idataplane.h @@ -28,9 +28,17 @@ class dataPlane } public: - auto updateGlobalBase(const common::idp::updateGlobalBase::request& request) const + auto update(const common::idp::update::request& request) const { - return get(request); + return get(request); + } + + auto update_globalbase(const common::idp::updateGlobalBase::request& request) const + { + common::idp::update::request update_request; + update_request.globalbase() = std::move(request); + const auto update_response = update(update_request); + return std::move(*update_response.globalbase()); } eResult updateGlobalBaseBalancer(const common::idp::updateGlobalBaseBalancer::request& request) const @@ -225,37 +233,61 @@ class dataPlane auto neighbor_show() const { - return get(); + common::idp::update::request update_request; + update_request.neighbor() = {common::neighbor::idp::type::show, std::tuple<>()}; + + const auto update_response = update(update_request); + return std::get(*update_response.neighbor()); } - auto neighbor_insert(const common::idp::neighbor_insert::request& request) const + auto neighbor_insert(const common::neighbor::idp::insert& request) const { - return get(request); + common::idp::update::request update_request; + update_request.neighbor() = {common::neighbor::idp::type::insert, request}; + + const auto update_response = update(update_request); + return std::get(*update_response.neighbor()); } - auto neighbor_remove(const common::idp::neighbor_remove::request& request) const + auto neighbor_remove(const common::neighbor::idp::remove& request) const { - return get(request); + common::idp::update::request update_request; + update_request.neighbor() = {common::neighbor::idp::type::remove, request}; + + const auto update_response = update(update_request); + return std::get(*update_response.neighbor()); } auto neighbor_clear() const { - return get(); + common::idp::update::request update_request; + update_request.neighbor() = {common::neighbor::idp::type::clear, std::tuple<>()}; + + const auto update_response = update(update_request); + return std::get(*update_response.neighbor()); } - auto neighbor_flush() const + auto neighbor_update_interfaces(const common::neighbor::idp::update_interfaces& request) const { - return get(); + common::idp::update::request update_request; + update_request.neighbor() = {common::neighbor::idp::type::update_interfaces, request}; + + const auto update_response = update(update_request); + return std::get(*update_response.neighbor()); } - auto neighbor_update_interfaces(const common::idp::neighbor_update_interfaces::request& request) const + auto neighbor_stats() const { - return get(request); + common::idp::update::request update_request; + update_request.neighbor() = {common::neighbor::idp::type::stats, std::tuple<>()}; + + const auto update_response = update(update_request); + return std::get(*update_response.neighbor()); } - auto neighbor_stats() const + auto memory_manager_update(const common::idp::memory_manager_update::request& request) const { - return get(); + return get(request); } protected: diff --git a/common/idp.h b/common/idp.h index 40e921db..88b356db 100644 --- a/common/idp.h +++ b/common/idp.h @@ -15,6 +15,8 @@ #include "acl.h" #include "balancer.h" #include "config.h" +#include "limit.h" +#include "memory_manager.h" #include "neighbor.h" #include "result.h" #include "scheduler.h" @@ -35,7 +37,7 @@ enum class errorType : uint32_t enum class requestType : uint32_t { - updateGlobalBase, + update, ///< update dataplane bases updateGlobalBaseBalancer, getGlobalBase, getWorkerStats, @@ -75,13 +77,8 @@ enum class requestType : uint32_t set_shm_tsc_state, dump_physical_port, balancer_state_clear, - neighbor_show, - neighbor_insert, - neighbor_remove, - neighbor_clear, - neighbor_flush, - neighbor_update_interfaces, - neighbor_stats, + memory_manager_update, + memory_manager_stats, size, // size should always be at the bottom of the list, this enum allows us to find out the size of the enum list }; @@ -141,16 +138,11 @@ enum class requestType : uint32_t route_tunnel_weight_update, route_tunnel_value_update, early_decap_flags, - acl_network_ipv4_source, - acl_network_ipv4_destination, acl_network_ipv6_source, - acl_network_ipv6_destination_ht, acl_network_ipv6_destination, acl_network_table, acl_network_flags, acl_transport_layers, - acl_transport_table, - acl_total_table, acl_values, dregress_prefix_update, dregress_prefix_remove, @@ -305,26 +297,11 @@ namespace update_early_decap_flags using request = bool; } -namespace acl_network_ipv4_source -{ -using request = std::vector; -} - -namespace acl_network_ipv4_destination -{ -using request = std::vector; -} - namespace acl_network_ipv6_source { using request = std::vector; } -namespace acl_network_ipv6_destination_ht -{ -using request = std::vector>; -} - namespace acl_network_ipv6_destination { using request = std::vector; @@ -354,16 +331,6 @@ using layer = std::tuple, ///< protocol using request = std::vector; } -namespace acl_transport_table -{ -using request = std::vector>; -} - -namespace acl_total_table -{ -using request = std::vector>; -} - namespace acl_values { using request = std::vector; @@ -517,13 +484,10 @@ using requestVariant = std::variant, update_balancer::request, update_balancer_services::request, route_tunnel_weight_update::request, - acl_network_ipv4_source::request, /// + acl_network_ipv4_destination, acl_network_ipv6_source, acl_network_ipv6_destination - acl_network_ipv6_destination_ht::request, + acl_network_ipv6_source::request, /// + acl_network_ipv4_destination, acl_network_ipv6_source, acl_network_ipv6_destination acl_network_table::request, /// + aclTransportDestination acl_network_flags::request, acl_transport_layers::request, - acl_transport_table::request, - acl_total_table::request, acl_values::request, dump_tags_ids::request, lpm::request, @@ -909,12 +873,7 @@ using request = std::tuple, - uint64_t, ///< current - uint64_t>; ///< maximum - -using response = std::vector; +using response = common::limit::limits; } namespace samples @@ -948,47 +907,121 @@ using request = std::tuple>>; ///< last_update_timestamp +using request = memory_manager::memory_group; } -namespace neighbor_insert +namespace update { -using request = std::tuple; ///< mac_address -} -namespace neighbor_remove +class request { -using request = std::tuple; ///< ip_address -} +public: + auto& globalbase() + { + return std::get<0>(request); + } -namespace neighbor_update_interfaces -{ -using request = std::vector>; ///< interface_name -} + const auto& globalbase() const + { + return std::get<0>(request); + } + + auto& acl() + { + return std::get<1>(request); + } + + const auto& acl() const + { + return std::get<1>(request); + } + + auto& neighbor() + { + return std::get<2>(request); + } + + const auto& neighbor() const + { + return std::get<2>(request); + } + + void pop(stream_in_t& stream) + { + stream.pop(request); + } + + void push(stream_out_t& stream) const + { + stream.push(request); + } + +public: + std::tuple, + std::optional, + std::optional> + request; +}; + +class response +{ +public: + auto& globalbase() + { + return std::get<0>(response); + } + + const auto& globalbase() const + { + return std::get<0>(response); + } + + auto& acl() + { + return std::get<1>(response); + } + + const auto& acl() const + { + return std::get<1>(response); + } + + auto& neighbor() + { + return std::get<2>(response); + } + + const auto& neighbor() const + { + return std::get<2>(response); + } + + void pop(stream_in_t& stream) + { + stream.pop(response); + } + + void push(stream_out_t& stream) const + { + stream.push(response); + } + +public: + std::tuple, + std::optional, + std::optional> + response; +}; -namespace neighbor_stats -{ -using response = common::neighbor::stats; } // using request = std::tuple, - updateGlobalBase::request, + update::request, updateGlobalBaseBalancer::request, getGlobalBase::request, getControlPlanePortStats::request, @@ -1003,12 +1036,11 @@ using request = std::tuple>; + memory_manager_update::request>>; using response = std::variant, - updateGlobalBase::response, ///< + others which have eResult as response + eResult, + update::response, getGlobalBase::response, getWorkerStats::response, getSlowWorkerStats::response, @@ -1037,7 +1069,5 @@ using response = std::variant, samples::response, get_counter_by_name::response, get_shm_info::response, - get_shm_tsc_info::response, - neighbor_show::response, - neighbor_stats::response>; + get_shm_tsc_info::response>; } diff --git a/common/limit.h b/common/limit.h new file mode 100644 index 00000000..0d25af3a --- /dev/null +++ b/common/limit.h @@ -0,0 +1,16 @@ +#pragma once + +#include +#include + +#include "type.h" + +namespace common::limit +{ +using item = std::tuple, + uint64_t, ///< current + uint64_t>; ///< maximum + +using limits = std::vector; +} diff --git a/common/memory_manager.h b/common/memory_manager.h new file mode 100644 index 00000000..a9e0b5f2 --- /dev/null +++ b/common/memory_manager.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include + +#include "stream.h" + +namespace common::memory_manager +{ + +inline uint64_t convert_string_to_bytes(std::string string) +{ + static std::map multipliers = + {{'k', 1024ull}, + {'K', 1024ull}, + {'m', 1024ull * 1024ull}, + {'M', 1024ull * 1024ull}, + {'g', 1024ull * 1024ull * 1024ull}, + {'G', 1024ull * 1024ull * 1024ull}}; + + if (string.empty()) + { + return 0; + } + + uint64_t multiplier = 1; + + auto iter = multipliers.find(string.back()); + if (iter != multipliers.end()) + { + multiplier = iter->second; + string.pop_back(); + } + + return std::stoll(string) * multiplier; +} + +class memory_group +{ +public: + void pop(common::stream_in_t& stream) + { + stream.pop(name); + stream.pop(limit); + stream.pop(memory_groups); + } + + void push(common::stream_out_t& stream) const + { + stream.push(name); + stream.push(limit); + stream.push(memory_groups); + } + +public: + std::string name; + uint64_t limit; + std::vector> memory_groups; +}; + +} diff --git a/common/neighbor.h b/common/neighbor.h index 01032091..294c84ea 100644 --- a/common/neighbor.h +++ b/common/neighbor.h @@ -1,6 +1,12 @@ #pragma once #include +#include +#include +#include + +#include "result.h" +#include "type.h" namespace common::neighbor { @@ -15,4 +21,52 @@ struct stats uint64_t resolve; }; +// + +namespace idp +{ + +enum class type +{ + show, + insert, + remove, + clear, + update_interfaces, + stats +}; + +using show = std::vector>>; ///< last_update_timestamp + +using insert = std::tuple; ///< mac_address + +using remove = std::tuple; ///< ip_address + +using update_interfaces = std::vector>; ///< interface_name + +using stats = common::neighbor::stats; + +using request = std::tuple, + insert, + remove, + update_interfaces>>; + +using response = std::variant; + +} + } diff --git a/common/stream.h b/common/stream.h index 74291275..a853f6d2 100644 --- a/common/stream.h +++ b/common/stream.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -61,6 +62,9 @@ class stream_in_t template inline void pop(std::optional& optional); + template + inline void pop(std::shared_ptr& pointer); + inline bool isFailed(); protected: @@ -221,6 +225,12 @@ class stream_out_t } } + template + inline void push(const std::shared_ptr& pointer) + { + push(*pointer.get()); + } + inline const std::vector& getBuffer() { return outBuffer; @@ -450,6 +460,13 @@ inline void stream_in_t::pop(std::optional& optional) } } +template +inline void stream_in_t::pop(std::shared_ptr& pointer) +{ + pointer = std::make_shared(); + pop(*pointer.get()); +} + inline bool stream_in_t::isFailed() { return failed; diff --git a/common/type.h b/common/type.h index 9a9d0cf2..7c4dd4b6 100644 --- a/common/type.h +++ b/common/type.h @@ -236,6 +236,11 @@ class mac_address_t return buffer; } + uint8_t* data() + { + return address.data(); + } + const uint8_t* data() const { return address.data(); diff --git a/controlplane/acl.h b/controlplane/acl.h index c52b2001..b7607986 100644 --- a/controlplane/acl.h +++ b/controlplane/acl.h @@ -3,6 +3,7 @@ #include #include "base.h" +#include "common/acl.h" namespace acl { @@ -16,16 +17,16 @@ typedef std::vector ids_t; struct result_t { - common::idp::updateGlobalBase::acl_network_ipv4_source::request acl_network_ipv4_source; - common::idp::updateGlobalBase::acl_network_ipv4_destination::request acl_network_ipv4_destination; + common::acl::idp::network_ipv4_source acl_network_ipv4_source; + common::acl::idp::network_ipv4_destination acl_network_ipv4_destination; common::idp::updateGlobalBase::acl_network_ipv6_source::request acl_network_ipv6_source; - common::idp::updateGlobalBase::acl_network_ipv6_destination_ht::request acl_network_ipv6_destination_ht; + common::acl::idp::network_ipv6_destination_ht acl_network_ipv6_destination_ht; common::idp::updateGlobalBase::acl_network_ipv6_destination::request acl_network_ipv6_destination; common::idp::updateGlobalBase::acl_network_table::request acl_network_table; common::idp::updateGlobalBase::acl_network_flags::request acl_network_flags; common::idp::updateGlobalBase::acl_transport_layers::request acl_transport_layers; - std::vector acl_transport_tables; - common::idp::updateGlobalBase::acl_total_table::request acl_total_table; + std::vector acl_transport_tables; + common::acl::idp::total_table acl_total_table; common::idp::updateGlobalBase::acl_values::request acl_values; std::vector ids_map; diff --git a/controlplane/acl_transport_table.h b/controlplane/acl_transport_table.h index 789b3197..23a0c662 100644 --- a/controlplane/acl_transport_table.h +++ b/controlplane/acl_transport_table.h @@ -69,7 +69,7 @@ class thread_t std::map> group_id_filter_ids; std::vector> transport_table_filter_id_group_ids; - common::idp::updateGlobalBase::acl_transport_table::request acl_transport_table; + common::acl::idp::transport_table acl_transport_table; std::optional exception; }; diff --git a/controlplane/base.h b/controlplane/base.h index 5103415c..6b06b13d 100644 --- a/controlplane/base.h +++ b/controlplane/base.h @@ -484,6 +484,8 @@ class base_t std::map> rib; + + common::memory_manager::memory_group root_memory_group; }; // diff --git a/controlplane/configconverter.cpp b/controlplane/configconverter.cpp index 708c10c8..9c48f510 100644 --- a/controlplane/configconverter.cpp +++ b/controlplane/configconverter.cpp @@ -1198,13 +1198,13 @@ void config_converter_t::acl_rules_nat64stateless_ingress(controlplane::base::ac /* @todo { - controlplane::base::acl_rule_transport_icmpv6_t rule_transport{range_t{0x00, 0xFF}, - range_t{0x00, 0xFF}, - ingressPortRange}; - acl.nextModuleRules.emplace_back(rule_network, - fragState::firstFragment, - rule_transport, - flow_fragmentation); + controlplane::base::acl_rule_transport_icmpv6_t rule_transport{range_t{0x00, 0xFF}, + range_t{0x00, 0xFF}, + ingressPortRange}; + acl.nextModuleRules.emplace_back(rule_network, + fragState::firstFragment, + rule_transport, + flow_fragmentation); } */ @@ -1456,13 +1456,13 @@ void config_converter_t::acl_rules_nat64stateless_egress(controlplane::base::acl /* @todo { - controlplane::base::acl_rule_transport_icmpv4_t rule_transport{range_t{0x00, 0xFF}, - range_t{0x00, 0xFF}, - egressPortRange}; - acl.nextModuleRules.emplace_back(rule_network, - fragState::firstFragment, - rule_transport, - flow_fragmentation); + controlplane::base::acl_rule_transport_icmpv4_t rule_transport{range_t{0x00, 0xFF}, + range_t{0x00, 0xFF}, + egressPortRange}; + acl.nextModuleRules.emplace_back(rule_network, + fragState::firstFragment, + rule_transport, + flow_fragmentation); } */ @@ -1880,24 +1880,23 @@ void config_converter_t::buildAcl() serializeLogicalPorts(); serializeRoutes(); - globalbase.emplace_back(common::idp::updateGlobalBase::requestType::acl_network_ipv4_source, std::move(result.acl_network_ipv4_source)); - globalbase.emplace_back(common::idp::updateGlobalBase::requestType::acl_network_ipv4_destination, std::move(result.acl_network_ipv4_destination)); + if (result.acl_transport_tables.size() != 1) + { + throw std::runtime_error("support multithread here"); + } + globalbase.emplace_back(common::idp::updateGlobalBase::requestType::acl_network_ipv6_source, std::move(result.acl_network_ipv6_source)); - globalbase.emplace_back(common::idp::updateGlobalBase::requestType::acl_network_ipv6_destination_ht, std::move(result.acl_network_ipv6_destination_ht)); globalbase.emplace_back(common::idp::updateGlobalBase::requestType::acl_network_ipv6_destination, std::move(result.acl_network_ipv6_destination)); globalbase.emplace_back(common::idp::updateGlobalBase::requestType::acl_network_table, std::move(result.acl_network_table)); globalbase.emplace_back(common::idp::updateGlobalBase::requestType::acl_network_flags, std::move(result.acl_network_flags)); globalbase.emplace_back(common::idp::updateGlobalBase::requestType::acl_transport_layers, std::move(result.acl_transport_layers)); - { - if (result.acl_transport_tables.size() != 1) - { - throw std::runtime_error("support multithread here"); - } - globalbase.emplace_back(common::idp::updateGlobalBase::requestType::acl_transport_table, std::move(result.acl_transport_tables[0])); - } + acl_request = {std::move(result.acl_network_ipv4_source), + std::move(result.acl_network_ipv4_destination), + std::move(result.acl_network_ipv6_destination_ht), + std::move(result.acl_transport_tables[0]), + std::move(result.acl_total_table)}; - globalbase.emplace_back(common::idp::updateGlobalBase::requestType::acl_total_table, std::move(result.acl_total_table)); globalbase.emplace_back(common::idp::updateGlobalBase::requestType::acl_values, std::move(result.acl_values)); globalbase.emplace_back(common::idp::updateGlobalBase::requestType::dump_tags_ids, std::move(result.dump_id_to_tag)); diff --git a/controlplane/configconverter.h b/controlplane/configconverter.h index 12c2da88..dfc9dc72 100644 --- a/controlplane/configconverter.h +++ b/controlplane/configconverter.h @@ -26,6 +26,11 @@ class config_converter_t return globalbase; } + common::acl::idp::request& get_acl() + { + return acl_request; + } + protected: void processLogicalPorts(); void processRoutes(); @@ -68,5 +73,6 @@ class config_converter_t controlplane::base_t baseNext; common::idp::updateGlobalBase::request globalbase; + common::acl::idp::request acl_request; common::idp::limits::response limits; }; diff --git a/controlplane/configparser.cpp b/controlplane/configparser.cpp index a9a038e0..2491367f 100644 --- a/controlplane/configparser.cpp +++ b/controlplane/configparser.cpp @@ -130,6 +130,11 @@ controlplane::base_t config_parser_t::loadConfig(const std::string& rootFilePath { loadConfig_rib(baseNext, rootJson["rib"]); } + + if (exist(rootJson, "memory_groups")) + { + loadConfig_memory_group(baseNext.root_memory_group, rootJson["memory_groups"]); + } } catch (const error_result_t& err) { @@ -1901,3 +1906,29 @@ void config_parser_t::loadConfig_rib(controlplane::base_t& baseNext, } } } + +void config_parser_t::loadConfig_memory_group(common::memory_manager::memory_group& memory_group, + const nlohmann::json& json) +{ + for (const auto& json_iter : json) + { + auto memory_group_next = std::make_shared(); + + std::string name = json_iter["name"].get(); + std::string limit = "0"; + if (exist(json_iter, "limit")) + { + limit = json_iter["limit"].get(); + } + + memory_group_next->name = name; + memory_group_next->limit = common::memory_manager::convert_string_to_bytes(std::move(limit)); + + if (exist(json_iter, "memory_groups")) + { + loadConfig_memory_group(*memory_group_next.get(), json_iter["memory_groups"]); + } + + memory_group.memory_groups.emplace_back(memory_group_next); + } +} diff --git a/controlplane/configparser.h b/controlplane/configparser.h index e041fb10..ad8af05f 100644 --- a/controlplane/configparser.h +++ b/controlplane/configparser.h @@ -39,6 +39,7 @@ class config_parser_t void loadConfig_variables(controlplane::base_t& baseNext, const nlohmann::json& json); void loadConfig_fqdns(controlplane::base_t& baseNext, const nlohmann::json& json, const std::string& rootFilePath, const std::map& jsons); void loadConfig_rib(controlplane::base_t& baseNext, const nlohmann::json& json); + void loadConfig_memory_group(common::memory_manager::memory_group& memory_group, const nlohmann::json& json); private: common::idp::getConfig::response dataPlaneConfig; diff --git a/controlplane/controlplane.cpp b/controlplane/controlplane.cpp index 52c6e32d..c66a69f1 100644 --- a/controlplane/controlplane.cpp +++ b/controlplane/controlplane.cpp @@ -54,6 +54,7 @@ eResult cControlPlane::init(const std::string& jsonFilePath) modules.emplace_back(&durations); modules.emplace_back(&nat64stateful); modules.emplace_back(&nat46clat); + modules.emplace_back(&memory_manager); for (auto* module : modules) { @@ -176,10 +177,7 @@ eResult cControlPlane::init(const std::string& jsonFilePath) } else { - common::idp::updateGlobalBase::request globalbase; - globalbase.emplace_back(common::idp::updateGlobalBase::requestType::clear, - std::tuple<>{}); - dataPlane.updateGlobalBase(globalbase); + dataPlane.update_globalbase({{common::idp::updateGlobalBase::requestType::clear, std::tuple<>()}}); } durations.add("init", start); loadConfigStatus = true; @@ -904,13 +902,26 @@ eResult cControlPlane::loadConfig(const std::string& rootFilePath, YANET_LOG_INFO("updating globalbase (stage 5)\n"); addConfig(serial, converter.getBaseNext()); - const auto result = dataPlane.updateGlobalBase(std::move(globalbase)); - if (result != eResult::success) + + common::idp::update::request update_request; + update_request.globalbase() = std::move(globalbase); + update_request.acl() = std::move(converter.get_acl()); + + const auto update_response = dataPlane.update(update_request); + if (*update_response.globalbase() != eResult::success) + { + // Since now the dataplane is locked for further changes + // and considered broken. + return *update_response.globalbase(); + } + + if (*update_response.acl() != eResult::success) { // Since now the dataplane is locked for further changes // and considered broken. - return result; + return *update_response.acl(); } + start = durations.add("reload.dataplane", start); YANET_LOG_INFO("globalbase updated (stage 6), serial %d\n", serial); diff --git a/controlplane/controlplane.h b/controlplane/controlplane.h index b80b2281..8d038a22 100644 --- a/controlplane/controlplane.h +++ b/controlplane/controlplane.h @@ -25,6 +25,7 @@ #include "durations.h" #include "fqdn.h" #include "isystem.h" +#include "memory_manager.h" #include "module.h" #include "nat46clat.h" #include "nat64stateful.h" @@ -172,6 +173,7 @@ class cControlPlane durations_t durations; nat64stateful_t nat64stateful; nat46clat::manager nat46clat; + controlplane::memory_manager::memory_manager memory_manager; counter_manager_t counter_manager; diff --git a/controlplane/dregress.cpp b/controlplane/dregress.cpp index c5a10764..b56f7d97 100644 --- a/controlplane/dregress.cpp +++ b/controlplane/dregress.cpp @@ -8,10 +8,9 @@ dregress_t::dregress_t() : eResult dregress_t::init() { - dataplane.updateGlobalBase( - {{common::idp::updateGlobalBase::requestType::dregress_prefix_clear, std::tuple<>()}, - {common::idp::updateGlobalBase::requestType::dregress_local_prefix_update, - common::idp::updateGlobalBase::dregress_local_prefix_update::request()}}); + dataplane.update_globalbase({{common::idp::updateGlobalBase::requestType::dregress_prefix_clear, std::tuple<>()}, + {common::idp::updateGlobalBase::requestType::dregress_local_prefix_update, + common::idp::updateGlobalBase::dregress_local_prefix_update::request()}}); controlPlane->register_command(common::icp::requestType::dregress_config, [this]() { return dregress_config(); @@ -202,7 +201,7 @@ void dregress_t::prefix_flush() compile(globalbase, generations.current()); } - dataplane.updateGlobalBase(globalbase); + dataplane.update_globalbase(globalbase); } common::icp::dregress_config::response dregress_t::dregress_config() const diff --git a/controlplane/memory_manager.cpp b/controlplane/memory_manager.cpp new file mode 100644 index 00000000..4f2990d9 --- /dev/null +++ b/controlplane/memory_manager.cpp @@ -0,0 +1,12 @@ +#include "memory_manager.h" + +using namespace controlplane::memory_manager; + +void memory_manager::reload(const base_t& base_prev, + const base_t& base_next, + common::idp::updateGlobalBase::request& globalbase) +{ + (void)base_prev; + (void)globalbase; + dataplane.memory_manager_update(base_next.root_memory_group); +} diff --git a/controlplane/memory_manager.h b/controlplane/memory_manager.h new file mode 100644 index 00000000..4f901a1c --- /dev/null +++ b/controlplane/memory_manager.h @@ -0,0 +1,21 @@ +#pragma once + +#include "base.h" +#include "module.h" +#include "type.h" + +#include "common/idataplane.h" + +namespace controlplane::memory_manager +{ + +class memory_manager : public module_t +{ +public: + void reload(const controlplane::base_t& base_prev, const controlplane::base_t& base_next, common::idp::updateGlobalBase::request& globalbase) override; + +protected: + interface::dataPlane dataplane; +}; + +} diff --git a/controlplane/meson.build b/controlplane/meson.build index 91e9e67c..daf6efa6 100644 --- a/controlplane/meson.build +++ b/controlplane/meson.build @@ -22,6 +22,7 @@ sources = files('acl_compiler.cpp', 'fqdn.cpp', 'isystem.cpp', 'main.cpp', + 'memory_manager.cpp', 'module.cpp', 'nat46clat.cpp', 'nat64stateful.cpp', diff --git a/controlplane/route.cpp b/controlplane/route.cpp index 4f0afc9a..0f1cd7be 100644 --- a/controlplane/route.cpp +++ b/controlplane/route.cpp @@ -3,12 +3,8 @@ eResult route_t::init() { - { - common::idp::updateGlobalBase::request globalbase; - globalbase.emplace_back(common::idp::updateGlobalBase::requestType::route_lpm_update, - common::idp::lpm::request({common::idp::lpm::clear()})); - dataplane.updateGlobalBase(std::move(globalbase)); - } + dataplane.update_globalbase({{common::idp::updateGlobalBase::requestType::route_lpm_update, + common::idp::lpm::request({common::idp::lpm::clear()})}}); tunnel_counter.init(&controlPlane->counter_manager); tunnel_counter.insert({true, 0, ip_address_t(), 0}); ///< fallback v4 @@ -389,7 +385,7 @@ void route_t::prefix_flush() tunnel_counter.allocate(); compile(globalbase, generations.current()); - dataplane.updateGlobalBase(globalbase); ///< может вызвать исключение, которое никто не поймает, и это приведёт к abort() + dataplane.update_globalbase(globalbase); ///< может вызвать исключение, которое никто не поймает, и это приведёт к abort() tunnel_counter.release(); @@ -750,8 +746,9 @@ void route_t::compile_interface(common::idp::updateGlobalBase::request& globalba const route::generation_t& generation, route::generation_neighbors_t& generation_neighbors) { + /// @todo: use dataplane.update() { - common::idp::neighbor_update_interfaces::request request; + common::neighbor::idp::update_interfaces request; for (const auto& [route_name, route] : generation.routes) { for (auto& [interface_name, interface] : route.interfaces) diff --git a/dataplane/acl.cpp b/dataplane/acl.cpp new file mode 100644 index 00000000..df6cc3f5 --- /dev/null +++ b/dataplane/acl.cpp @@ -0,0 +1,191 @@ +#include "acl.h" +#include "dataplane.h" +#include "worker.h" + +using namespace dataplane::acl; + +base::base(memory_manager* memory_manager, + const tSocketId socket_id) : + network_ipv4_source("acl.network.v4.source.lpm", memory_manager, socket_id), + network_ipv4_destination("acl.network.v4.destination.lpm", memory_manager, socket_id), + network_ipv6_destination_ht("acl.network.v6.destination.ht", memory_manager, socket_id), + transport_table("acl.transport.ht", memory_manager, socket_id), + total_table("acl.total.ht", memory_manager, socket_id) +{ +} + +module::module() : + dataplane(nullptr) +{ +} + +eResult module::init(cDataPlane* dataplane) +{ + eResult result = eResult::success; + + this->dataplane = dataplane; + + generations.fill([&](acl::generation& generation) { + for (const auto socket_id : dataplane->get_socket_ids()) + { + generation.bases.try_emplace(socket_id, &dataplane->memory_manager, socket_id); + } + }); + + /// initial allocate objects + acl_update({}); + + return result; +} + +void module::update_worker_base(const std::vector>& worker_base_nexts) +{ + auto lock = generations.current_lock_guard(); + + const auto& generation = generations.current(); + for (auto& [socket_id, worker_base] : worker_base_nexts) + { + const auto& base = generation.bases.find(socket_id)->second; + + worker_base->acl_network_ipv4_source = base.network_ipv4_source.pointer; + worker_base->acl_network_ipv4_destination = base.network_ipv4_destination.pointer; + worker_base->acl_network_ipv6_destination_ht = base.network_ipv6_destination_ht.pointer; + worker_base->acl_transport_table = base.transport_table.pointer; + worker_base->acl_total_table = base.total_table.pointer; + } +} + +void module::report(nlohmann::json& json) +{ + auto lock = generations.current_lock_guard(); + + const auto& generation = generations.current(); + for (const auto socket_id : dataplane->get_socket_ids()) + { + const auto& base = generation.bases.find(socket_id)->second; + + base.network_ipv4_source.report(json["acl"]["network"]["ipv4"]["source"]); + base.network_ipv4_destination.report(json["acl"]["network"]["ipv4"]["destination"]); + base.network_ipv6_destination_ht.report(json["acl"]["network"]["ipv6"]["destination_ht"]); + base.transport_table.report(json["acl"]["transport_table"]); + base.total_table.report(json["acl"]["total_table"]); + } +} + +void module::limits(common::idp::limits::response& response) +{ + auto lock = generations.current_lock_guard(); + + const auto& generation = generations.current(); + for (const auto socket_id : dataplane->get_socket_ids()) + { + const auto& base = generation.bases.find(socket_id)->second; + + base.network_ipv4_source.limits(response); + base.network_ipv4_destination.limits(response); + base.network_ipv6_destination_ht.limits(response); + base.transport_table.limits(response); + base.total_table.limits(response); + } +} + +void module::update_before(const common::idp::update::request& request) +{ + if (!request.acl()) + { + return; + } + + generations.next_lock(); +} + +void module::update(const common::idp::update::request& request, + common::idp::update::response& response) +{ + if (!request.acl()) + { + return; + } + + response.acl() = acl_update(*request.acl()); + return; +} + +void module::update_after(const common::idp::update::request& request) +{ + if (!request.acl()) + { + return; + } + + auto& generation = generations.next(); + for (const auto socket_id : dataplane->get_socket_ids()) + { + auto& base = generation.bases.find(socket_id)->second; + + base.network_ipv4_source.clear(); + base.network_ipv4_destination.clear(); + base.network_ipv6_destination_ht.clear(); + base.transport_table.clear(); + base.total_table.clear(); + } + + generations.next_unlock(); +} + +eResult module::acl_update(const common::acl::idp::request& request_acl) +{ + eResult result = eResult::success; + const auto& [request_network_ipv4_source, + request_network_ipv4_destination, + request_network_ipv6_destination_ht, + request_transport_table, + request_total_table] = request_acl; + + auto& generation = generations.next(); + for (const auto socket_id : dataplane->get_socket_ids()) + { + auto& base = generation.bases.find(socket_id)->second; + + result = base.network_ipv4_source.update(request_network_ipv4_source); + if (result != eResult::success) + { + return result; + } + + result = base.network_ipv4_destination.update(request_network_ipv4_destination); + if (result != eResult::success) + { + return result; + } + + { + std::vector> request_convert; + for (const auto& [address, group_id] : request_network_ipv6_destination_ht) + { + request_convert.emplace_back(ipv6_address_t::convert(address), group_id); + } + + result = base.network_ipv6_destination_ht.update(request_convert); + if (result != eResult::success) + { + return result; + } + } + + result = base.transport_table.update(request_transport_table); + if (result != eResult::success) + { + return result; + } + + result = base.total_table.update(request_total_table); + if (result != eResult::success) + { + return result; + } + } + + generations.switch_generation_without_gc(); + return result; +} diff --git a/dataplane/acl.h b/dataplane/acl.h new file mode 100644 index 00000000..e4ade693 --- /dev/null +++ b/dataplane/acl.h @@ -0,0 +1,69 @@ +#pragma once + +#include + +#include + +#include "common/acl.h" +#include "common/generation.h" +#include "common/idp.h" + +#include "lpm.h" +#include "type.h" +#include "updater.h" + +namespace dataplane::acl +{ + +using network_ipv4_source = dataplane::updater_lpm4_24bit_8bit_id32; +using network_ipv4_destination = dataplane::updater_lpm4_24bit_8bit_id32; +using network_ipv6_destination_ht = dataplane::updater_hashtable_mod_id32; +using transport_table = dataplane::updater_hashtable_mod_id32; +using total_table = dataplane::updater_hashtable_mod_id32; + +class base +{ +public: + base(dataplane::memory_manager* memory_manager, const tSocketId socket_id); + +public: + acl::network_ipv4_source network_ipv4_source; + acl::network_ipv4_destination network_ipv4_destination; + acl::network_ipv6_destination_ht network_ipv6_destination_ht; + acl::transport_table transport_table; + acl::total_table total_table; +}; + +class generation +{ +public: + std::map bases; +}; + +// + +class module +{ +public: + module(); + +public: + eResult init(cDataPlane* dataplane); + void update_worker_base(const std::vector>& worker_base_nexts); + void report(nlohmann::json& json); + void limits(common::idp::limits::response& response); + + void update_before(const common::idp::update::request& request); + void update(const common::idp::update::request& request, common::idp::update::response& response); + void update_after(const common::idp::update::request& request); + +protected: + eResult acl_update(const common::acl::idp::request& request_acl); + +protected: + cDataPlane* dataplane; + std::vector threads; + generation_manager generations; +}; + +} diff --git a/dataplane/base.h b/dataplane/base.h index 45afe6ea..1b1e44d1 100644 --- a/dataplane/base.h +++ b/dataplane/base.h @@ -8,6 +8,7 @@ #include #include +#include "acl.h" #include "neighbor.h" #include "type.h" @@ -81,12 +82,22 @@ class generation public: generation() : globalBase(nullptr), - neighbor_hashtable(nullptr) + neighbor_hashtable(nullptr), + acl_network_ipv4_source(nullptr), + acl_network_ipv4_destination(nullptr), + acl_network_ipv6_destination_ht(nullptr), + acl_transport_table(nullptr), + acl_total_table(nullptr) { } dataplane::globalBase::generation* globalBase; - dataplane::neighbor::hashtable const* neighbor_hashtable; + const dataplane::neighbor::hashtable* neighbor_hashtable; + const dataplane::acl::network_ipv4_source::object_type* acl_network_ipv4_source; + const dataplane::acl::network_ipv4_destination::object_type* acl_network_ipv4_destination; + const dataplane::acl::network_ipv6_destination_ht::object_type* acl_network_ipv6_destination_ht; + const dataplane::acl::transport_table::object_type* acl_transport_table; + const dataplane::acl::total_table::object_type* acl_total_table; } __rte_aligned(2 * RTE_CACHE_LINE_SIZE); } diff --git a/dataplane/bus.cpp b/dataplane/bus.cpp index c72d7831..ce44c887 100644 --- a/dataplane/bus.cpp +++ b/dataplane/bus.cpp @@ -194,9 +194,9 @@ void cBus::clientThread(int clientSocket) const common::idp::requestType& type = std::get<0>(request); YANET_LOG_DEBUG("request type %d\n", (int)type); - if (type == common::idp::requestType::updateGlobalBase) + if (type == common::idp::requestType::update) { - response = callWithResponse(&cControlPlane::updateGlobalBase, request); + response = callWithResponse(&cControlPlane::update, request); } else if (type == common::idp::requestType::updateGlobalBaseBalancer) { @@ -350,33 +350,9 @@ void cBus::clientThread(int clientSocket) { response = callWithResponse(&cControlPlane::balancer_state_clear, request); } - else if (type == common::idp::requestType::neighbor_show) + else if (type == common::idp::requestType::memory_manager_update) { - response = dataPlane->neighbor.neighbor_show(); - } - else if (type == common::idp::requestType::neighbor_insert) - { - response = dataPlane->neighbor.neighbor_insert(std::get(std::get<1>(request))); - } - else if (type == common::idp::requestType::neighbor_remove) - { - response = dataPlane->neighbor.neighbor_remove(std::get(std::get<1>(request))); - } - else if (type == common::idp::requestType::neighbor_clear) - { - response = dataPlane->neighbor.neighbor_clear(); - } - else if (type == common::idp::requestType::neighbor_flush) - { - response = dataPlane->neighbor.neighbor_flush(); - } - else if (type == common::idp::requestType::neighbor_update_interfaces) - { - response = dataPlane->neighbor.neighbor_update_interfaces(std::get(std::get<1>(request))); - } - else if (type == common::idp::requestType::neighbor_stats) - { - response = dataPlane->neighbor.neighbor_stats(); + response = dataPlane->memory_manager.memory_manager_update(std::get(std::get<1>(request))); } else { diff --git a/dataplane/common.h b/dataplane/common.h index 0f2d7bf5..041528a6 100644 --- a/dataplane/common.h +++ b/dataplane/common.h @@ -90,3 +90,13 @@ inline uint32_t yanet_hash_crc(const void* data, uint32_t init) // static_assert(CONFIG_YADECAP_PORTS_SIZE <= 0xFF, "invalid CONFIG_YADECAP_PORTS_SIZE"); + +// + +namespace dataplane +{ + +class memory_pointer; +class memory_manager; + +} diff --git a/dataplane/controlplane.cpp b/dataplane/controlplane.cpp index 55123a1b..395eee2a 100644 --- a/dataplane/controlplane.cpp +++ b/dataplane/controlplane.cpp @@ -150,70 +150,29 @@ void cControlPlane::start() mainThread(); } -common::idp::updateGlobalBase::response cControlPlane::updateGlobalBase(const common::idp::updateGlobalBase::request& request) +common::idp::update::response cControlPlane::update(const common::idp::update::request& request) { - std::lock_guard guard(mutex); - if (!errors.empty()) - { - return eResult::dataplaneIsBroken; - } - - YADECAP_MEMORY_BARRIER_COMPILE; - - auto result = eResult::success; - for (auto& iter : dataPlane->globalBases) - { - auto* globalBaseNext = iter.second[dataPlane->currentGlobalBaseId ^ 1]; - DEBUG_LATCH_WAIT(common::idp::debug_latch_update::id::global_base_pre_update); - result = globalBaseNext->update(request); - DEBUG_LATCH_WAIT(common::idp::debug_latch_update::id::global_base_post_update); - if (result != eResult::success) - { - ++errors["updateGlobalBase"]; - break; - } - } - - if (result != eResult::success) - { - return result; - } - - DEBUG_LATCH_WAIT(common::idp::debug_latch_update::id::global_base_switch); + common::idp::update::response response; - YADECAP_MEMORY_BARRIER_COMPILE; + /// lock + dataPlane->globalbase_update_before(request); + dataPlane->neighbor.update_before(request); + dataPlane->acl_module.update_before(request); - switchGlobalBase(); + /// update + dataPlane->globalbase_update(request, response); + dataPlane->neighbor.update(request, response); + dataPlane->acl_module.update(request, response); - YADECAP_MEMORY_BARRIER_COMPILE; - - result = eResult::success; - for (auto& iter : dataPlane->globalBases) - { - auto* globalBaseNext = iter.second[dataPlane->currentGlobalBaseId ^ 1]; - DEBUG_LATCH_WAIT(common::idp::debug_latch_update::id::global_base_pre_update); - result = globalBaseNext->update(request); - DEBUG_LATCH_WAIT(common::idp::debug_latch_update::id::global_base_post_update); - if (result != eResult::success) - { - // Practically unreachable. - ++errors["updateGlobalBase"]; - break; - } - } - - if (result != eResult::success) - { - return result; - } - - YADECAP_MEMORY_BARRIER_COMPILE; - - waitAllWorkers(); + /// switch + dataPlane->switch_worker_base(); - YADECAP_MEMORY_BARRIER_COMPILE; + /// unlock + dataPlane->globalbase_update_after(request); + dataPlane->neighbor.update_after(request); + dataPlane->acl_module.update_after(request); - return eResult::success; + return response; } eResult cControlPlane::updateGlobalBaseBalancer(const common::idp::updateGlobalBaseBalancer::request& request) @@ -253,7 +212,7 @@ eResult cControlPlane::updateGlobalBaseBalancer(const common::idp::updateGlobalB YADECAP_MEMORY_BARRIER_COMPILE; - waitAllWorkers(); + dataPlane->wait_all_workers(); YADECAP_MEMORY_BARRIER_COMPILE; @@ -973,10 +932,6 @@ common::idp::limits::response cControlPlane::limits() YANET_CONFIG_ROUTE_TUNNEL_LPM6_EXTENDED_SIZE); globalBase->updater.acl.network_table.limits(response, "acl.network.ht"); - globalBase->updater.acl.transport_table.limits(response, "acl.transport.ht"); - globalBase->updater.acl.total_table.limits(response, "acl.total.ht"); - globalBase->updater.acl.network_ipv4_source.limits(response, "acl.network.v4.source.lpm"); - globalBase->updater.acl.network_ipv4_destination.limits(response, "acl.network.v4.destination.lpm"); globalBase->updater.acl.network_ipv6_source.limits(response, "acl.network.v6.source.lpm"); /// globalBase->acl.network_ipv6_destination_ht is not critical @@ -1003,6 +958,9 @@ common::idp::limits::response cControlPlane::limits() } dregress.limits(response); + dataPlane->memory_manager.limits(response); + dataPlane->neighbor.limits(response); + dataPlane->acl_module.limits(response); return response; } @@ -1319,100 +1277,6 @@ common::idp::nat64stateful_state::response cControlPlane::nat64stateful_state(co return response; } -void cControlPlane::switchBase() -{ - YADECAP_MEMORY_BARRIER_COMPILE; - - for (auto& iter : dataPlane->workers) - { - auto* worker = iter.second; - - worker->currentBaseId ^= 1; - } - - for (auto& iter : dataPlane->worker_gcs) - { - auto* worker = iter.second; - - worker->current_base_id ^= 1; - } - - YADECAP_MEMORY_BARRIER_COMPILE; - - waitAllWorkers(); - - YADECAP_MEMORY_BARRIER_COMPILE; - - for (auto& iter : dataPlane->workers) - { - auto* worker = iter.second; - auto& base = worker->bases[worker->currentBaseId]; - auto& baseNext = worker->bases[worker->currentBaseId ^ 1]; - - baseNext = base; - } - - for (auto& iter : dataPlane->worker_gcs) - { - auto* worker = iter.second; - auto& base = worker->bases[worker->current_base_id]; - auto& baseNext = worker->bases[worker->current_base_id ^ 1]; - - baseNext = base; - } - - YADECAP_MEMORY_BARRIER_COMPILE; -} - -void cControlPlane::switchGlobalBase() -{ - YADECAP_MEMORY_BARRIER_COMPILE; - - { - std::lock_guard guard(dataPlane->currentGlobalBaseId_mutex); - dataPlane->currentGlobalBaseId ^= 1; - } - - YADECAP_MEMORY_BARRIER_COMPILE; - - dataPlane->switch_worker_base(); - - YADECAP_MEMORY_BARRIER_COMPILE; -} - -void cControlPlane::waitAllWorkers() -{ - YADECAP_MEMORY_BARRIER_COMPILE; - - for (const auto& [core_id, worker] : dataPlane->workers) - { - (void)core_id; - - uint64_t startIteration = worker->iteration; - uint64_t nextIteration = startIteration; - while (nextIteration - startIteration <= (uint64_t)16) - { - YADECAP_MEMORY_BARRIER_COMPILE; - nextIteration = worker->iteration; - } - } - - for (const auto& [core_id, worker] : dataPlane->worker_gcs) - { - (void)core_id; - - uint64_t startIteration = worker->iteration; - uint64_t nextIteration = startIteration; - while (nextIteration - startIteration <= (uint64_t)16) - { - YADECAP_MEMORY_BARRIER_COMPILE; - nextIteration = worker->iteration; - } - } - - YADECAP_MEMORY_BARRIER_COMPILE; -} - eResult cControlPlane::initMempool() { mempool = rte_mempool_create("cp", diff --git a/dataplane/controlplane.h b/dataplane/controlplane.h index e1885c52..c1f610ea 100644 --- a/dataplane/controlplane.h +++ b/dataplane/controlplane.h @@ -32,7 +32,7 @@ class cControlPlane ///< @todo: move to cDataPlane void stop(); void join(); - common::idp::updateGlobalBase::response updateGlobalBase(const common::idp::updateGlobalBase::request& request); + common::idp::update::response update(const common::idp::update::request& request); eResult updateGlobalBaseBalancer(const common::idp::updateGlobalBaseBalancer::request& request); common::idp::getGlobalBase::response getGlobalBase(const common::idp::getGlobalBase::request& request); common::idp::getWorkerStats::response getWorkerStats(const common::idp::getWorkerStats::request& request); @@ -72,10 +72,6 @@ class cControlPlane ///< @todo: move to cDataPlane eResult dump_physical_port(const common::idp::dump_physical_port::request& request); eResult balancer_state_clear(); - void switchBase(); - void switchGlobalBase(); - virtual void waitAllWorkers(); - void sendPacketToSlowWorker(rte_mbuf* mbuf, const common::globalBase::tFlow& flow); ///< @todo: remove flow void freeWorkerPacket(rte_ring* ring_to_free_mbuf, rte_mbuf* mbuf); diff --git a/dataplane/dataplane.cpp b/dataplane/dataplane.cpp index 83c2626d..26785d99 100644 --- a/dataplane/dataplane.cpp +++ b/dataplane/dataplane.cpp @@ -32,6 +32,7 @@ #include "common/result.h" #include "common/tsc_deltas.h" #include "dataplane.h" +#include "debug_latch.h" #include "globalbase.h" #include "report.h" #include "sock_dev.h" @@ -226,6 +227,12 @@ eResult cDataPlane::init(const std::string& binaryPath, return result; } + result = memory_manager.init(this); + if (result != eResult::success) + { + return result; + } + result = controlPlane->init(config.use_kernel_interface); if (result != eResult::success) { @@ -244,6 +251,12 @@ eResult cDataPlane::init(const std::string& binaryPath, return result; } + result = acl_module.init(this); + if (result != eResult::success) + { + return result; + } + init_worker_base(); /// init sync barrier @@ -655,30 +668,12 @@ eResult cDataPlane::initGlobalBases() } { - auto* acl_network_ipv4_source = hugepage_create_dynamic(socket_id, getConfigValue(eConfigType::acl_network_lpm4_chunks_size), globalbase->updater.acl.network_ipv4_source); - if (!acl_network_ipv4_source) - { - return nullptr; - } - - auto* acl_network_ipv4_destination = hugepage_create_dynamic(socket_id, getConfigValue(eConfigType::acl_network_lpm4_chunks_size), globalbase->updater.acl.network_ipv4_destination); - if (!acl_network_ipv4_destination) - { - return nullptr; - } - auto* acl_network_ipv6_source = hugepage_create_dynamic(socket_id, getConfigValue(eConfigType::acl_network_source_lpm6_chunks_size), globalbase->updater.acl.network_ipv6_source); if (!acl_network_ipv6_source) { return nullptr; } - auto* acl_network_ipv6_destination_ht = hugepage_create_dynamic(socket_id, getConfigValue(eConfigType::acl_network_destination_ht_size), globalbase->updater.acl.network_ipv6_destination_ht); - if (!acl_network_ipv6_destination_ht) - { - return nullptr; - } - auto* acl_network_ipv6_destination = hugepage_create_dynamic(socket_id, getConfigValue(eConfigType::acl_network_destination_lpm6_chunks_size), globalbase->updater.acl.network_ipv6_destination); if (!acl_network_ipv6_destination) { @@ -711,18 +706,6 @@ eResult cDataPlane::initGlobalBases() return nullptr; } - auto* acl_transport_table = hugepage_create_dynamic(socket_id, getConfigValue(eConfigType::acl_transport_ht_size), globalbase->updater.acl.transport_table); - if (!acl_transport_table) - { - return nullptr; - } - - auto* acl_total_table = hugepage_create_dynamic(socket_id, getConfigValue(eConfigType::acl_total_ht_size), globalbase->updater.acl.total_table); - if (!acl_total_table) - { - return nullptr; - } - const auto acl_values_size = getConfigValue(eConfigType::acl_values_size); if (acl_values_size < 2) { @@ -736,16 +719,11 @@ eResult cDataPlane::initGlobalBases() return nullptr; } - globalbase->acl.network.ipv4.source = acl_network_ipv4_source; - globalbase->acl.network.ipv4.destination = acl_network_ipv4_destination; globalbase->acl.network.ipv6.source = acl_network_ipv6_source; - globalbase->acl.network.ipv6.destination_ht = acl_network_ipv6_destination_ht; globalbase->acl.network.ipv6.destination = acl_network_ipv6_destination; globalbase->acl.network_table = acl_network_table; globalbase->acl.transport_layers_mask = acl_transport_layers_size - 1; globalbase->acl.transport_layers = acl_transport_layers; - globalbase->acl.transport_table = acl_transport_table; - globalbase->acl.total_table = acl_total_table; globalbase->acl.values = acl_values; } @@ -1166,6 +1144,7 @@ void cDataPlane::init_worker_base() } neighbor.update_worker_base(base_nexts); + acl_module.update_worker_base(base_nexts); } void cDataPlane::hugepage_destroy(void* pointer) @@ -1244,6 +1223,78 @@ void cDataPlane::timestamp_thread() } } +void cDataPlane::globalbase_update_worker_base(const std::vector>& worker_base_nexts) +{ + std::lock_guard guard(currentGlobalBaseId_mutex); + for (const auto& [socket_id, base_next] : worker_base_nexts) + { + base_next->globalBase = globalBases[socket_id][currentGlobalBaseId]; + } +} + +void cDataPlane::globalbase_update_before(const common::idp::update::request& request) +{ + if (!request.globalbase()) + { + return; + } + + nextGlobalBaseId_mutex.lock(); +} + +void cDataPlane::globalbase_update(const common::idp::update::request& request, + common::idp::update::response& response) +{ + if (!request.globalbase()) + { + return; + } + + for (auto& iter : globalBases) + { + auto* globalBaseNext = iter.second[currentGlobalBaseId ^ 1]; + DEBUG_LATCH_WAIT(common::idp::debug_latch_update::id::global_base_pre_update); + response.globalbase() = globalBaseNext->update(*request.globalbase()); + DEBUG_LATCH_WAIT(common::idp::debug_latch_update::id::global_base_post_update); + if (response.globalbase() != eResult::success) + { + /// @todo: write error + return; + } + } + + { + std::lock_guard guard(currentGlobalBaseId_mutex); + currentGlobalBaseId ^= 1; + } + + return; +} + +void cDataPlane::globalbase_update_after(const common::idp::update::request& request) +{ + if (!request.globalbase()) + { + return; + } + + for (auto& iter : globalBases) + { + auto* globalBaseNext = iter.second[currentGlobalBaseId ^ 1]; + DEBUG_LATCH_WAIT(common::idp::debug_latch_update::id::global_base_pre_update); + eResult result = globalBaseNext->update(*request.globalbase()); + DEBUG_LATCH_WAIT(common::idp::debug_latch_update::id::global_base_post_update); + if (result != eResult::success) + { + // Practically unreachable. + /// @todo: write error + break; + } + } + + nextGlobalBaseId_mutex.unlock(); +} + void cDataPlane::start() { threads.emplace_back([this]() { @@ -1630,17 +1681,87 @@ void cDataPlane::switch_worker_base() } /// update base_next + globalbase_update_worker_base(base_nexts); + neighbor.update_worker_base(base_nexts); + acl_module.update_worker_base(base_nexts); + + /// switch { - std::lock_guard guard(currentGlobalBaseId_mutex); - for (const auto& [socket_id, base_next] : base_nexts) + YADECAP_MEMORY_BARRIER_COMPILE; + + for (auto& [core_id, worker] : workers) + { + (void)core_id; + worker->currentBaseId ^= 1; + } + + for (auto& [core_id, worker] : worker_gcs) + { + (void)core_id; + worker->current_base_id ^= 1; + } + + YADECAP_MEMORY_BARRIER_COMPILE; + + wait_all_workers(); + + YADECAP_MEMORY_BARRIER_COMPILE; + + for (auto& [core_id, worker] : workers) { - base_next->globalBase = globalBases[socket_id][currentGlobalBaseId]; + (void)core_id; + + auto& base = worker->bases[worker->currentBaseId]; + auto& base_next = worker->bases[worker->currentBaseId ^ 1]; + + base_next = base; } + + for (auto& [core_id, worker] : worker_gcs) + { + (void)core_id; + + auto& base = worker->bases[worker->current_base_id]; + auto& base_next = worker->bases[worker->current_base_id ^ 1]; + + base_next = base; + } + + YADECAP_MEMORY_BARRIER_COMPILE; } - neighbor.update_worker_base(base_nexts); +} - /// switch - controlPlane->switchBase(); +void cDataPlane::wait_all_workers() +{ + YADECAP_MEMORY_BARRIER_COMPILE; + + for (const auto& [core_id, worker] : workers) + { + (void)core_id; + + uint64_t startIteration = worker->iteration; + uint64_t nextIteration = startIteration; + while (nextIteration - startIteration <= (uint64_t)16) + { + YADECAP_MEMORY_BARRIER_COMPILE; + nextIteration = worker->iteration; + } + } + + for (const auto& [core_id, worker] : worker_gcs) + { + (void)core_id; + + uint64_t startIteration = worker->iteration; + uint64_t nextIteration = startIteration; + while (nextIteration - startIteration <= (uint64_t)16) + { + YADECAP_MEMORY_BARRIER_COMPILE; + nextIteration = worker->iteration; + } + } + + YADECAP_MEMORY_BARRIER_COMPILE; } eResult cDataPlane::parseConfig(const std::string& configFilePath) diff --git a/dataplane/dataplane.h b/dataplane/dataplane.h index cecfd06e..96dfa9d1 100644 --- a/dataplane/dataplane.h +++ b/dataplane/dataplane.h @@ -21,6 +21,7 @@ #include "bus.h" #include "controlplane.h" #include "globalbase.h" +#include "memory_manager.h" #include "neighbor.h" #include "report.h" #include "type.h" @@ -141,6 +142,7 @@ class cDataPlane void run_on_worker_gc(const tSocketId socket_id, const std::function& callback); void switch_worker_base(); + void wait_all_workers(); inline uint32_t get_current_time() const { @@ -309,6 +311,11 @@ class cDataPlane static int lcoreThread(void* args); void timestamp_thread(); + void globalbase_update_worker_base(const std::vector>& worker_base_nexts); + void globalbase_update_before(const common::idp::update::request& request); + void globalbase_update(const common::idp::update::request& request, common::idp::update::response& response); + void globalbase_update_after(const common::idp::update::request& request); + protected: friend class cWorker; friend class cReport; @@ -331,6 +338,7 @@ class cDataPlane std::map workers; std::map worker_gcs; + std::mutex nextGlobalBaseId_mutex; std::mutex currentGlobalBaseId_mutex; uint8_t currentGlobalBaseId; std::map globalBaseAtomics; @@ -356,12 +364,6 @@ class cDataPlane common::idp::get_shm_tsc_info::response tscs_meta; - /// modules - cReport report; - std::unique_ptr controlPlane; - cBus bus; - dataplane::neighbor::module neighbor; - // array instead of the table - how many coreIds can be there? std::unordered_map> coreId_to_stats_tables; @@ -380,4 +382,12 @@ class cDataPlane std::vector threads; mutable std::mutex dpdk_mutex; + +public: ///< modules + cReport report; + std::unique_ptr controlPlane; + cBus bus; + dataplane::memory_manager memory_manager; + dataplane::neighbor::module neighbor; + dataplane::acl::module acl_module; }; diff --git a/dataplane/globalbase.cpp b/dataplane/globalbase.cpp index 0b019e4e..1ad82c51 100644 --- a/dataplane/globalbase.cpp +++ b/dataplane/globalbase.cpp @@ -5,6 +5,7 @@ #include "common.h" #include "dataplane.h" #include "globalbase.h" +#include "memory_manager.h" #include "worker.h" #include "common/counters.h" @@ -140,22 +141,10 @@ eResult generation::update(const common::idp::updateGlobalBase::request& request { result = update_early_decap_flags(std::get(data)); } - else if (type == common::idp::updateGlobalBase::requestType::acl_network_ipv4_source) - { - result = acl_network_ipv4_source(std::get(data)); - } - else if (type == common::idp::updateGlobalBase::requestType::acl_network_ipv4_destination) - { - result = acl_network_ipv4_destination(std::get(data)); - } else if (type == common::idp::updateGlobalBase::requestType::acl_network_ipv6_source) { result = acl_network_ipv6_source(std::get(data)); } - else if (type == common::idp::updateGlobalBase::requestType::acl_network_ipv6_destination_ht) - { - result = acl_network_ipv6_destination_ht(std::get(data)); - } else if (type == common::idp::updateGlobalBase::requestType::acl_network_ipv6_destination) { result = acl_network_ipv6_destination(std::get(data)); @@ -172,14 +161,6 @@ eResult generation::update(const common::idp::updateGlobalBase::request& request { result = acl_transport_layers(std::get(data)); } - else if (type == common::idp::updateGlobalBase::requestType::acl_transport_table) - { - result = acl_transport_table(std::get(data)); - } - else if (type == common::idp::updateGlobalBase::requestType::acl_total_table) - { - result = acl_total_table(std::get(data)); - } else if (type == common::idp::updateGlobalBase::requestType::acl_values) { result = acl_values(std::get(data)); @@ -418,9 +399,6 @@ eResult generation::clear() sampler_enabled = 0; serial = 0; - updater.acl.network_ipv6_destination_ht.clear(); - updater.acl.transport_table.clear(); - updater.acl.total_table.clear(); acl.values[0] = {}; tun64mappingsTable.clear(); @@ -1856,34 +1834,6 @@ eResult generation::update_early_decap_flags(const common::idp::updateGlobalBase return result; } -eResult generation::acl_network_ipv4_source(const common::idp::updateGlobalBase::acl_network_ipv4_source::request& request) -{ - eResult result = eResult::success; - - result = updater.acl.network_ipv4_source.update(request); - if (result != eResult::success) - { - YANET_LOG_ERROR("acl.network.ipv4.source.update(): %s\n", result_to_c_str(result)); - return result; - } - - return result; -} - -eResult generation::acl_network_ipv4_destination(const common::idp::updateGlobalBase::acl_network_ipv4_destination::request& request) -{ - eResult result = eResult::success; - - result = updater.acl.network_ipv4_destination.update(request); - if (result != eResult::success) - { - YANET_LOG_ERROR("acl.network.ipv4.destination.update(): %s\n", result_to_c_str(result)); - return result; - } - - return result; -} - eResult generation::acl_network_ipv6_source(const common::idp::updateGlobalBase::acl_network_ipv6_source::request& request) { eResult result = eResult::success; @@ -1898,17 +1848,6 @@ eResult generation::acl_network_ipv6_source(const common::idp::updateGlobalBase: return result; } -eResult generation::acl_network_ipv6_destination_ht(const common::idp::updateGlobalBase::acl_network_ipv6_destination_ht::request& request) -{ - auto result = updater.acl.network_ipv6_destination_ht.update(request); - if (result != eResult::success) - { - /// dont panic. this is fine - } - - return eResult::success; -} - eResult generation::acl_network_ipv6_destination(const common::idp::updateGlobalBase::acl_network_ipv6_destination::request& request) { eResult result = eResult::success; @@ -2049,34 +1988,6 @@ eResult generation::acl_transport_layers(const common::idp::updateGlobalBase::ac return result; } -eResult generation::acl_transport_table(const common::idp::updateGlobalBase::acl_transport_table::request& request) -{ - eResult result = eResult::success; - - result = updater.acl.transport_table.update(request); - if (result != eResult::success) - { - YANET_LOG_ERROR("acl.transport_table.update(): %s\n", result_to_c_str(result)); - return result; - } - - return result; -} - -eResult generation::acl_total_table(const common::idp::updateGlobalBase::acl_total_table::request& request) -{ - eResult result = eResult::success; - - result = updater.acl.total_table.update(request); - if (result != eResult::success) - { - YANET_LOG_ERROR("acl.total_table.update(): %s\n", result_to_c_str(result)); - return result; - } - - return result; -} - eResult generation::acl_values(const common::idp::updateGlobalBase::acl_values::request& request) { if (request.size() > dataPlane->getConfigValue(eConfigType::acl_values_size)) diff --git a/dataplane/globalbase.h b/dataplane/globalbase.h index 78629343..ea1fba7a 100644 --- a/dataplane/globalbase.h +++ b/dataplane/globalbase.h @@ -61,14 +61,9 @@ struct transport_layer_t }; /// @todo: move to config -using network_ipv4_source = lpm4_24bit_8bit_id32_dynamic; -using network_ipv4_destination = lpm4_24bit_8bit_id32_dynamic; using network_ipv6_source = YANET_CONFIG_ACL_NETWORK_LPM6_TYPE; -using network_ipv6_destination_ht = hashtable_mod_id32_dynamic; using network_ipv6_destination = YANET_CONFIG_ACL_NETWORK_LPM6_TYPE; using network_table = dynamic_table; -using transport_table = hashtable_mod_id32_dynamic; -using total_table = hashtable_mod_id32_dynamic; } namespace nat64stateful @@ -159,16 +154,11 @@ class generation eResult route_tunnel_weight_update(const common::idp::updateGlobalBase::route_tunnel_weight_update::request& request); eResult route_tunnel_value_update(const common::idp::updateGlobalBase::route_tunnel_value_update::request& request); eResult update_early_decap_flags(const common::idp::updateGlobalBase::update_early_decap_flags::request& request); - eResult acl_network_ipv4_source(const common::idp::updateGlobalBase::acl_network_ipv4_source::request& request); - eResult acl_network_ipv4_destination(const common::idp::updateGlobalBase::acl_network_ipv4_destination::request& request); eResult acl_network_ipv6_source(const common::idp::updateGlobalBase::acl_network_ipv6_source::request& request); - eResult acl_network_ipv6_destination_ht(const common::idp::updateGlobalBase::acl_network_ipv6_destination_ht::request& request); eResult acl_network_ipv6_destination(const common::idp::updateGlobalBase::acl_network_ipv6_destination::request& request); eResult acl_network_table(const common::idp::updateGlobalBase::acl_network_table::request& request); eResult acl_network_flags(const common::idp::updateGlobalBase::acl_network_flags::request& request); eResult acl_transport_layers(const common::idp::updateGlobalBase::acl_transport_layers::request& request); - eResult acl_transport_table(const common::idp::updateGlobalBase::acl_transport_table::request& request); - eResult acl_total_table(const common::idp::updateGlobalBase::acl_total_table::request& request); eResult acl_values(const common::idp::updateGlobalBase::acl_values::request& request); eResult dump_tags_ids(const common::idp::updateGlobalBase::dump_tags_ids::request& request); eResult dregress_prefix_update(const common::idp::updateGlobalBase::dregress_prefix_update::request& request); @@ -193,14 +183,9 @@ class generation { struct { - acl::network_ipv4_source::updater network_ipv4_source; - acl::network_ipv4_destination::updater network_ipv4_destination; acl::network_ipv6_source::updater network_ipv6_source; - acl::network_ipv6_destination_ht::updater network_ipv6_destination_ht; acl::network_ipv6_destination::updater network_ipv6_destination; acl::network_table::updater network_table; - acl::transport_table::updater transport_table; - acl::total_table::updater total_table; } acl; } updater; @@ -255,16 +240,9 @@ class generation { struct { - struct - { - acl::network_ipv4_source* source; - acl::network_ipv4_destination* destination; - } ipv4; - struct { acl::network_ipv6_source* source; - acl::network_ipv6_destination_ht* destination_ht; acl::network_ipv6_destination* destination; } ipv6; } network; @@ -275,8 +253,6 @@ class generation uint32_t transport_layers_mask; acl::transport_layer_t* transport_layers; - acl::transport_table* transport_table; - acl::total_table* total_table; common::acl::value_t* values; } acl; diff --git a/dataplane/hashtable.h b/dataplane/hashtable.h index 622cd67f..1ea30019 100644 --- a/dataplane/hashtable.h +++ b/dataplane/hashtable.h @@ -230,7 +230,7 @@ class hashtable_chain_t } inline bool lookup(const TKey& key, - TValue*& value) + TValue*& value) const { lookup(&key, &value, 1); return (value != nullptr); @@ -1771,191 +1771,50 @@ class hashtable_mod_id32_dynamic constexpr static uint32_t mask_full = 0xFFFFFFFFu; constexpr static uint32_t shift_valid = (32 - 1 - valid_bit_offset); constexpr static uint64_t longest_chain_size = chunk_size; + constexpr static uint32_t pairs_size_min = 128; static_assert(__builtin_popcount(chunk_size) == 1); static_assert(valid_bit_offset < 32); -public: - class updater + struct stats_t { - public: - void update_pointer(hashtable_t* hashtable, - const tSocketId socket_id, - const uint32_t total_size) - { - this->hashtable = hashtable; - this->socket_id = socket_id; - this->total_size = total_size; - - hashtable->total_mask = total_size - 1; - } - - template - eResult update(const std::vector>& keys) - { - eResult result = eResult::success; - - keys_count = 0; - keys_in_chunks.fill(0); - longest_chain = 0; - insert_failed = 0; - rewrites = 0; - - for (const auto& [key, value] : keys) - { - eResult insert_result; - if constexpr (std::is_same_v) - { - insert_result = insert(key, value); - } - else - { - insert_result = insert(key_t::convert(key), value); - } - - if (insert_result != eResult::success) - { - result = insert_result; - } - } - - for (uint32_t chunk_i = 0; - chunk_i < total_size / chunk_size; - chunk_i++) - { - unsigned int count = 0; - - for (uint32_t pair_i = 0; - pair_i < chunk_size; - pair_i++) - { - if (hashtable->is_valid(chunk_i * chunk_size + pair_i)) - { - count++; - } - } - - keys_in_chunks[count]++; - } - - return result; - } - - void clear() - { - for (uint32_t i = 0; - i < total_size; - i++) - { - hashtable->pairs[i].value = 0; - } - } - - public: - template ///< @todo: common::idp::limits::response - void limits(list_T& list, - const std::string& name) const - { - list.emplace_back(name + ".keys", - socket_id, - keys_count, - total_size); - list.emplace_back(name + ".longest_collision", - socket_id, - longest_chain, - chunk_size); - } - - template ///< @todo: nlohmann::json - void report(json_t& json) const - { - json["total_size"] = total_size; - json["keys_count"] = keys_count; - for (unsigned int i = 0; - i < keys_in_chunks.size(); - i++) - { - json["keys_in_chunks"][i] = keys_in_chunks[i]; - } - json["longest_chain"] = longest_chain; - json["insert_failed"] = insert_failed; - json["rewrites"] = rewrites; - } - - protected: - eResult insert(const key_t& key, - const uint32_t value) - { - const uint32_t hash = calculate_hash(key) & (total_size - 1); - - for (unsigned int try_i = 0; - try_i < chunk_size; - try_i++) - { - const uint32_t index = (hash + try_i) % total_size; - - if (!hashtable->is_valid(index)) - { - memcpy(&hashtable->pairs[index].key, &key, sizeof(key_t)); - hashtable->pairs[index].value = value; - hashtable->pairs[index].value |= 1u << shift_valid; - - keys_count++; - - uint64_t longest_chain = try_i + 1; - if (this->longest_chain < longest_chain) - { - this->longest_chain = longest_chain; - } - - return eResult::success; - } - else if (hashtable->is_valid_and_equal(index, key)) - { - hashtable->pairs[index].value = value; - hashtable->pairs[index].value |= 1u << shift_valid; - - rewrites++; - - return eResult::success; - } - } - - insert_failed++; - - return eResult::isFull; - } - - public: - hashtable_t* hashtable; - tSocketId socket_id; - uint32_t total_size; - uint32_t keys_count; - std::array keys_in_chunks; + uint32_t pairs_count; + uint32_t pairs_size; + std::array pairs_in_chunks; uint32_t longest_chain; uint64_t insert_failed; uint64_t rewrites; }; -public: - static size_t calculate_sizeof(const uint32_t total_size) + static uint64_t calculate_sizeof(const uint32_t pairs_size) { - if (!total_size) + if (!pairs_size) { - YANET_LOG_ERROR("wrong total_size: %u\n", total_size); + YANET_LOG_ERROR("wrong pairs_size: %u\n", pairs_size); return 0; } - if (__builtin_popcount(total_size) != 1) + if (__builtin_popcount(pairs_size) != 1) { - YANET_LOG_ERROR("wrong total_size: %u is non power of 2\n", total_size); + YANET_LOG_ERROR("wrong pairs_size: %u is non power of 2\n", pairs_size); return 0; } - return sizeof(hashtable_t) + (size_t)total_size * sizeof(pair); + return sizeof(hashtable_t) + pairs_size * sizeof(pair); } public: + hashtable_mod_id32_dynamic(const uint32_t pairs_size) : + total_mask(pairs_size - 1) + { + for (uint32_t i = 0; + i < pairs_size; + i++) + { + pairs[i].value = 0; + } + } + /// value: /// valid invalid /// = 0VV = 1VV @@ -2035,14 +1894,90 @@ class hashtable_mod_id32_dynamic return mask; } -protected: - uint32_t total_mask; + eResult fill(stats_t& stats, const std::vector>& pairs) + { + eResult result = eResult::success; - struct pair + stats.pairs_count = 0; + stats.pairs_in_chunks.fill(0); + stats.longest_chain = 0; + stats.insert_failed = 0; + stats.rewrites = 0; + + for (const auto& [key, value] : pairs) + { + eResult insert_result = insert(stats, key, value); + if (insert_result != eResult::success) + { + result = insert_result; + } + } + + for (uint32_t chunk_i = 0; + chunk_i < stats.pairs_size / chunk_size; + chunk_i++) + { + unsigned int count = 0; + + for (uint32_t pair_i = 0; + pair_i < chunk_size; + pair_i++) + { + if (is_valid(chunk_i * chunk_size + pair_i)) + { + count++; + } + } + + stats.pairs_in_chunks[count]++; + } + + return result; + } + + eResult insert(stats_t& stats, + const key_t& key, + const uint32_t value) { - key_t key; - uint32_t value; - } pairs[]; + const uint32_t hash = calculate_hash(key) & total_mask; + + for (unsigned int try_i = 0; + try_i < chunk_size; + try_i++) + { + const uint32_t index = (hash + try_i) & total_mask; + + if (!is_valid(index)) + { + memcpy(&pairs[index].key, &key, sizeof(key_t)); + pairs[index].value = value; + pairs[index].value |= 1u << shift_valid; + + stats.pairs_count++; + + uint64_t longest_chain = try_i + 1; + if (stats.longest_chain < longest_chain) + { + stats.longest_chain = longest_chain; + } + + return eResult::success; + } + else if (is_valid_and_equal(index, key)) + { + pairs[index].value = value; + pairs[index].value |= 1u << shift_valid; + + stats.rewrites++; + + return eResult::success; + } + } + + stats.insert_failed++; + + return eResult::isFull; + } protected: inline bool is_valid(const uint32_t index) const @@ -2059,6 +1994,15 @@ class hashtable_mod_id32_dynamic { return is_valid(index) && is_equal(index, key); } + +protected: + uint32_t total_mask; + + struct pair + { + key_t key; + uint32_t value; + } pairs[]; }; class hashtable_mod_spinlock_stats ///< @todo: move to class::updater @@ -2184,7 +2128,7 @@ class hashtable_mod_spinlock } /* else if (is_equal(chunk, pair_index, key)) { - /// hashtable is broken + /// hashtable is broken } */ if (chunk_size == 1) @@ -2214,7 +2158,7 @@ class hashtable_mod_spinlock } /* else if (is_equal(chunk, pair_index, key)) { - /// hashtable is broken + /// hashtable is broken } */ } @@ -2661,7 +2605,7 @@ class hashtable_mod_spinlock_dynamic } /* else if (is_equal(chunk, pair_index, key)) { - /// hashtable is broken + /// hashtable is broken } */ if (chunk_size == 1) @@ -2691,7 +2635,7 @@ class hashtable_mod_spinlock_dynamic } /* else if (is_equal(chunk, pair_index, key)) { - /// hashtable is broken + /// hashtable is broken } */ } @@ -2989,6 +2933,11 @@ class hashtable_mod_dynamic class updater { public: + updater() : + hashtable(nullptr) + { + } + void update_pointer(hashtable_t* hashtable, const tSocketId socket_id, const uint32_t total_size) @@ -3011,7 +2960,6 @@ class hashtable_mod_dynamic return hashtable; } - public: range_t range(uint32_t& offset, const uint32_t step) { @@ -3242,7 +3190,7 @@ class hashtable_mod_dynamic } /* else if (is_equal(chunk, pair_index, key)) { - /// hashtable is broken + /// hashtable is broken } */ if (chunk_size == 1) @@ -3272,7 +3220,7 @@ class hashtable_mod_dynamic } /* else if (is_equal(chunk, pair_index, key)) { - /// hashtable is broken + /// hashtable is broken } */ } diff --git a/dataplane/lpm.h b/dataplane/lpm.h index 0afe4ed9..0f6d7eb1 100644 --- a/dataplane/lpm.h +++ b/dataplane/lpm.h @@ -1922,223 +1922,205 @@ class lpm4_24bit_8bit_id32 class lpm4_24bit_8bit_id32_dynamic { public: - class updater + constexpr static uint64_t extended_chunks_size_min = 8; + + struct stats_t { - public: - updater() : - extended_chunks_count(1) + uint32_t extended_chunks_size; + unsigned int extended_chunks_count; + std::vector remap_chunks; + }; + + static uint64_t calculate_sizeof(const uint64_t extended_chunks_size) + { + if (!extended_chunks_size) { + YANET_LOG_ERROR("wrong extended_chunks_size: %lu\n", extended_chunks_size); + return 0; } - void update_pointer(lpm4_24bit_8bit_id32_dynamic* lpm, - const tSocketId socket_id, - const uint32_t extended_chunks_size) + return sizeof(lpm4_24bit_8bit_id32_dynamic) + extended_chunks_size * sizeof(chunk_step2_t); + } + +public: + lpm4_24bit_8bit_id32_dynamic() + { + for (auto& value : root_chunk.values) { - this->lpm = lpm; - this->socket_id = socket_id; - this->extended_chunks_size = extended_chunks_size; + value.id = 0; } + } - eResult update(const std::vector& from_chunks) + template + inline void lookup(const ipv4_address_t (&addresses)[burst_size], + uint32_t (&group_ids)[burst_size], + const unsigned int count) const + { + /// @todo: OPT: le -> be + + for (unsigned int address_i = 0; + address_i < count; + address_i++) { - extended_chunks_count = 1; - remap_chunks.resize(0); - remap_chunks.resize(from_chunks.size(), 0); - return update_root_chunk<0>(from_chunks, 0, 0); + const auto& address = addresses[address_i].address; + auto& group_id = group_ids[address_i]; + + const auto& root_chunk_value = root_chunk.values[rte_be_to_cpu_32(address) >> 8]; + if (root_chunk_value.id & 0x80000000u) + { + const auto& extended_chunk = extended_chunks[root_chunk_value.id ^ 0x80000000u]; + group_id = extended_chunk.values[address >> 24].id; + } + else + { + group_id = root_chunk_value.id; + } } + } - public: - template ///< @todo: common::idp::limits::response - void limits(list_T& list, - const std::string& name) const + eResult fill(stats_t& stats, const std::vector& values) + { + stats.extended_chunks_count = 1; + stats.remap_chunks.resize(0); + stats.remap_chunks.resize(values.size(), 0); + + if (values.empty()) { - list.emplace_back(name + ".extended_chunks", - socket_id, - extended_chunks_count, - extended_chunks_size); + return eResult::success; } - template ///< @todo: nlohmann::json - void report(json_t& json) const + return update_root_chunk<0>(stats, values, 0, 0); + } + +protected: + constexpr static unsigned int bits_step1 = 24; + constexpr static unsigned int bits_step2 = 8; + constexpr static uint32_t mask_full = 0xFFFFFFFFu; + + unsigned int allocate_extended_chunk(stats_t& stats) + { + if (stats.extended_chunks_count >= stats.extended_chunks_size) { - json["extended_chunks_count"] = extended_chunks_count; + return 0; } - protected: - inline unsigned int allocate_extended_chunk() - { - if (extended_chunks_count >= extended_chunks_size) - { - return 0; - } + unsigned int new_chunk_id = stats.extended_chunks_count; + stats.extended_chunks_count++; - unsigned int new_chunk_id = extended_chunks_count; - extended_chunks_count++; + return new_chunk_id; + } - return new_chunk_id; - } + template + eResult update_root_chunk(stats_t& stats, + const std::vector& from_chunks, + const unsigned int from_chunk_id, + const unsigned int root_chunk_values_offset) + { + eResult result = eResult::success; - template - eResult update_root_chunk(const std::vector& from_chunks, - const unsigned int from_chunk_id, - const unsigned int root_chunk_values_offset) + const auto& from_chunk = from_chunks[from_chunk_id]; + for (uint32_t i = 0; + i < (1u << 8); + i++) { - eResult result = eResult::success; + const unsigned int root_chunk_values_i = root_chunk_values_offset + (i << (bits_step1 - bits_offset - 8)); + const auto& from_chunk_value = from_chunk.values[i]; - const auto& from_chunk = from_chunks[from_chunk_id]; - for (uint32_t i = 0; - i < (1u << 8); - i++) + if (from_chunk_value.is_chunk_id()) { - const unsigned int root_chunk_values_i = root_chunk_values_offset + (i << (bits_step1 - bits_offset - 8)); - const auto& from_chunk_value = from_chunk.values[i]; - - if (from_chunk_value.is_chunk_id()) + if constexpr (bits_offset < bits_step1 - bits_step2) { - if constexpr (bits_offset < bits_step1 - bits_step2) + result = update_root_chunk(stats, + from_chunks, + from_chunk_value.get_chunk_id(), + root_chunk_values_i); + if (result != eResult::success) { - result = update_root_chunk(from_chunks, - from_chunk_value.get_chunk_id(), - root_chunk_values_i); - if (result != eResult::success) - { - return result; - } + return result; } - else if constexpr (bits_offset < bits_step1) - { - auto& root_chunk_value = lpm->root_chunk.values[root_chunk_values_i]; + } + else if constexpr (bits_offset < bits_step1) + { + auto& root_chunk_value = root_chunk.values[root_chunk_values_i]; - auto& extended_chunk_id = remap_chunks[from_chunk_value.get_chunk_id()]; + auto& extended_chunk_id = stats.remap_chunks[from_chunk_value.get_chunk_id()]; + if (!extended_chunk_id) + { + extended_chunk_id = allocate_extended_chunk(stats); if (!extended_chunk_id) { - extended_chunk_id = allocate_extended_chunk(); - if (!extended_chunk_id) - { - YANET_LOG_ERROR("lpm is full\n"); - return eResult::isFull; - } - - result = update_extended_chunk(from_chunks, - from_chunk_value.get_chunk_id(), - extended_chunk_id); - if (result != eResult::success) - { - return result; - } + YANET_LOG_ERROR("lpm is full\n"); + return eResult::isFull; } - root_chunk_value.id = extended_chunk_id ^ 0x80000000u; - } - else - { - YANET_LOG_ERROR("tree broken\n"); - return eResult::invalidArguments; + result = update_extended_chunk(from_chunks, + from_chunk_value.get_chunk_id(), + extended_chunk_id); + if (result != eResult::success) + { + return result; + } } + + root_chunk_value.id = extended_chunk_id ^ 0x80000000u; } else { - if constexpr (bits_offset < bits_step1) - { - for (uint32_t j = 0; - j < (1u << (bits_step1 - bits_offset - 8)); - j++) - { - lpm->root_chunk.values[root_chunk_values_i + j].id = from_chunk_value.get_group_id(); - } - } - else - { - YANET_LOG_ERROR("tree broken\n"); - return eResult::invalidArguments; - } + YANET_LOG_ERROR("tree broken\n"); + return eResult::invalidArguments; } } - - return result; - } - - eResult update_extended_chunk(const std::vector& from_chunks, - const unsigned int from_chunk_id, - const unsigned int extended_chunk_id) - { - const auto& from_chunk = from_chunks[from_chunk_id]; - auto& extended_chunk = lpm->extended_chunks[extended_chunk_id]; - - for (uint32_t i = 0; - i < (1u << 8); - i++) + else { - const auto& from_chunk_value = from_chunk.values[i]; - auto& extended_chunk_value = extended_chunk.values[i]; - - if (from_chunk_value.is_chunk_id()) + if constexpr (bits_offset < bits_step1) { - YANET_LOG_ERROR("is_chunk_id\n"); - return eResult::invalidArguments; + for (uint32_t j = 0; + j < (1u << (bits_step1 - bits_offset - 8)); + j++) + { + root_chunk.values[root_chunk_values_i + j].id = from_chunk_value.get_group_id(); + } } else { - extended_chunk_value.id = from_chunk_value.get_group_id(); + YANET_LOG_ERROR("tree broken\n"); + return eResult::invalidArguments; } } - - return eResult::success; - } - - public: - lpm4_24bit_8bit_id32_dynamic* lpm; - tSocketId socket_id; - uint32_t extended_chunks_size; - unsigned int extended_chunks_count; - std::vector remap_chunks; - }; - -public: - static size_t calculate_sizeof(const uint32_t extended_chunks_size) - { - if (!extended_chunks_size) - { - YANET_LOG_ERROR("wrong extended_chunks_size: %u\n", extended_chunks_size); - return 0; } - return sizeof(lpm4_24bit_8bit_id32_dynamic) + extended_chunks_size * sizeof(chunk_step2_t); + return result; } -public: - template - inline void lookup(const ipv4_address_t (&addresses)[burst_size], - uint32_t (&group_ids)[burst_size], - const unsigned int count) const + eResult update_extended_chunk(const std::vector& from_chunks, + const unsigned int from_chunk_id, + const unsigned int extended_chunk_id) { - /// @todo: OPT: le -> be + const auto& from_chunk = from_chunks[from_chunk_id]; + auto& extended_chunk = extended_chunks[extended_chunk_id]; - for (unsigned int address_i = 0; - address_i < count; - address_i++) + for (uint32_t i = 0; + i < (1u << 8); + i++) { - const auto& address = addresses[address_i].address; - auto& group_id = group_ids[address_i]; + const auto& from_chunk_value = from_chunk.values[i]; + auto& extended_chunk_value = extended_chunk.values[i]; - const auto& root_chunk_value = root_chunk.values[rte_be_to_cpu_32(address) >> 8]; - if (root_chunk_value.id & 0x80000000u) + if (from_chunk_value.is_chunk_id()) { - const auto& extended_chunk = extended_chunks[root_chunk_value.id ^ 0x80000000u]; - group_id = extended_chunk.values[address >> 24].id; + YANET_LOG_ERROR("is_chunk_id\n"); + return eResult::invalidArguments; } else { - group_id = root_chunk_value.id; + extended_chunk_value.id = from_chunk_value.get_group_id(); } } + + return eResult::success; } protected: - constexpr static unsigned int bits_step1 = 24; - constexpr static unsigned int bits_step2 = 8; - constexpr static uint32_t mask_full = 0xFFFFFFFFu; - struct value_t { uint32_t id; diff --git a/dataplane/main.cpp b/dataplane/main.cpp index e88d5ab3..4e73932c 100644 --- a/dataplane/main.cpp +++ b/dataplane/main.cpp @@ -1,5 +1,9 @@ +#include #include +#include +#include #include +#include #include @@ -21,6 +25,21 @@ void handleSignal(int signalType) { YADECAP_LOG_INFO("signal: SIGPIPE\n"); } + else if (signalType == SIGABRT) + { + YADECAP_LOG_INFO("signal: SIGABRT\n"); + + void* array[10]; + size_t size; + + // get void*'s for all entries on the stack + size = backtrace(array, 10); + + // print out all the frames to stderr + fprintf(stderr, "Error: signal %d:\n", signalType); + backtrace_symbols_fd(array, size, STDERR_FILENO); + exit(1); + } } int main(int argc, @@ -67,6 +86,11 @@ int main(int argc, return 3; } + if (signal(SIGABRT, handleSignal) == SIG_ERR) + { + return 3; + } + sd_notify(0, "READY=1"); dataPlane.start(); diff --git a/dataplane/memory_manager.cpp b/dataplane/memory_manager.cpp new file mode 100644 index 00000000..1d039cea --- /dev/null +++ b/dataplane/memory_manager.cpp @@ -0,0 +1,142 @@ +#include "memory_manager.h" + +using namespace dataplane; + +memory_pointer::memory_pointer(const char* name, + const tSocketId socket_id, + const size_t size, + void* pointer, + const std::function& destructor) : + name(name), + socket_id(socket_id), + size(size), + pointer(pointer), + destructor(destructor) +{ +} + +memory_pointer::~memory_pointer() +{ + YANET_LOG_INFO("yanet_free(name: '%s', socket: %u, size: %lu)\n", + name.data(), + socket_id, + size); + destructor(pointer); +} + +memory_manager::memory_manager() : + dataplane(nullptr) +{ +} + +eResult memory_manager::init(cDataPlane* dataplane) +{ + this->dataplane = dataplane; + return eResult::success; +} + +inline std::string to_hex(const void* pointer) +{ + char buffer[128]; + snprintf(buffer, 128, "%p", pointer); + return buffer; +} + +void memory_manager::report(nlohmann::json& json) +{ + std::lock_guard guard(pointers_mutex); + for (const auto& [pointer, memory_pointer] : pointers) + { + nlohmann::json json_object; + json_object["pointer"] = to_hex(pointer); + json_object["name"] = memory_pointer.name; + json_object["socket_id"] = memory_pointer.socket_id; + json_object["size"] = memory_pointer.size; + json["memory_manager"].emplace_back(json_object); + } +} + +eResult memory_manager::memory_manager_update(const common::idp::memory_manager_update::request& request) +{ + root_memory_group = request; + return eResult::success; +} + +void* memory_manager::alloc(const char* name, + const tSocketId socket_id, + uint64_t size, + const std::function& destructor) +{ + if (!size) + { + YANET_LOG_ERROR("error allocation memory (name: '%s', socket: %u, size: %lu)\n", + name, + socket_id, + size); + return nullptr; + } + + size += 2 * RTE_CACHE_LINE_SIZE; + + YANET_LOG_INFO("yanet_alloc(name: '%s', socket: %u, size: %lu)\n", + name, + socket_id, + size); + + void* pointer = rte_malloc_socket(nullptr, + size, + RTE_CACHE_LINE_SIZE, + socket_id); + if (pointer == nullptr) + { + YANET_LOG_ERROR("error allocation memory (name: '%s', socket: %u, size: %lu)\n", + name, + socket_id, + size); + debug(socket_id); + return nullptr; + } + + { + std::lock_guard guard(pointers_mutex); + pointers.try_emplace(pointer, name, socket_id, size, pointer, [destructor](void* pointer) { + destructor(pointer); + rte_free(pointer); + }); + } + + return pointer; +} + +void memory_manager::destroy(void* pointer) +{ + std::lock_guard guard(pointers_mutex); + + auto it = pointers.find(pointer); + if (it == pointers.end()) + { + YANET_LOG_ERROR("unknown pointer: %p\n", pointer); + return; + } + + pointers.erase(it); +} + +void memory_manager::debug(tSocketId socket_id) +{ + rte_malloc_socket_stats stats; + if (rte_malloc_get_socket_stats(socket_id, &stats) == 0) + { + YANET_LOG_INFO("heap_totalsz_bytes: %lu MB\n", stats.heap_totalsz_bytes / (1024 * 1024)); + YANET_LOG_INFO("heap_freesz_bytes: %lu MB\n", stats.heap_freesz_bytes / (1024 * 1024)); + YANET_LOG_INFO("greatest_free_size: %lu MB\n", stats.greatest_free_size / (1024 * 1024)); + YANET_LOG_INFO("free_count: %u\n", stats.free_count); + YANET_LOG_INFO("alloc_count: %u\n", stats.alloc_count); + YANET_LOG_INFO("heap_allocsz_bytes: %lu MB\n", stats.heap_allocsz_bytes / (1024 * 1024)); + } +} + +void dataplane::memory_manager::limits(common::idp::limits::response& response) +{ + (void)response; +} diff --git a/dataplane/memory_manager.h b/dataplane/memory_manager.h new file mode 100644 index 00000000..4032d22e --- /dev/null +++ b/dataplane/memory_manager.h @@ -0,0 +1,75 @@ +#pragma once + +#include + +#include + +#include "common/idp.h" +#include "common/memory_manager.h" +#include "common/result.h" +#include "type.h" + +namespace dataplane +{ + +class memory_pointer +{ +public: + memory_pointer(const char* name, const tSocketId socket_id, const size_t size, void* pointer, const std::function& destructor); + ~memory_pointer(); + +public: + std::string name; + tSocketId socket_id; + size_t size; + void* pointer; + std::function destructor; +}; + +class memory_manager +{ +public: + memory_manager(); + + eResult init(cDataPlane* dataplane); + void report(nlohmann::json& json); + void limits(common::idp::limits::response& response); + eResult memory_manager_update(const common::idp::memory_manager_update::request& request); + + void* alloc( + const char* name, + const tSocketId socket_id, + const uint64_t size, + const std::function& destructor = [](void*) {}); + + template + type* create(const char* name, + const tSocketId socket_id, + const uint64_t size, + const args_t&... args) + { + void* pointer = alloc(name, socket_id, size, [](void* pointer) { + reinterpret_cast(pointer)->~type(); + }); + + if (pointer == nullptr) + { + return nullptr; + } + + return new (reinterpret_cast(pointer)) type(args...); + } + + void destroy(void* pointer); + void debug(tSocketId socket_id); + +protected: + cDataPlane* dataplane; + common::memory_manager::memory_group root_memory_group; + + std::mutex pointers_mutex; + std::map pointers; +}; + +} diff --git a/dataplane/meson.build b/dataplane/meson.build index 1a8d0712..a949999c 100644 --- a/dataplane/meson.build +++ b/dataplane/meson.build @@ -3,7 +3,8 @@ dependencies += dependency('libdpdk', static: true) dependencies += dependency('libsystemd') dependencies += dependency('threads') -sources = files('bus.cpp', +sources = files('acl.cpp', + 'bus.cpp', 'controlplane.cpp', 'dataplane.cpp', 'debug_latch.cpp', @@ -11,6 +12,7 @@ sources = files('bus.cpp', 'fragmentation.cpp', 'globalbase.cpp', 'main.cpp', + 'memory_manager.cpp', 'neighbor.cpp', 'report.cpp', 'sharedmemory.cpp', diff --git a/dataplane/neighbor.cpp b/dataplane/neighbor.cpp index c94dd4d0..850d5bea 100644 --- a/dataplane/neighbor.cpp +++ b/dataplane/neighbor.cpp @@ -114,6 +114,10 @@ static void netlink_parse(const std::function& callback) { +#ifdef CONFIG_YADECAP_AUTOTEST + return; +#endif // CONFIG_YADECAP_AUTOTEST + int nl_socket = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); if (nl_socket < 0) { @@ -151,6 +155,10 @@ static void netlink_neighbor_monitor(const std::function& callback) { +#ifdef CONFIG_YADECAP_AUTOTEST + return; +#endif // CONFIG_YADECAP_AUTOTEST + int nl_socket = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE); if (nl_socket < 0) { @@ -183,11 +191,11 @@ static void netlink_neighbor_dump(const std::function buffer(64 * 1024); + int buffer_length = recv(nl_socket, buffer.data(), buffer.size(), 0); if (buffer_length > 0) { - netlink_parse(callback, buffer, buffer_length); + netlink_parse(callback, buffer.data(), buffer_length); } close(nl_socket); @@ -233,9 +241,9 @@ void module::update_worker_base(const std::vectorswitch_worker_base(); - }); - return eResult::success; -} - -eResult module::neighbor_update_interfaces(const common::idp::neighbor_update_interfaces::request& request) -{ - generation_interface.next_lock(); - auto& generation = generation_interface.next(); for (const auto& [interface_id, route_name, @@ -430,85 +428,118 @@ eResult module::neighbor_update_interfaces(const common::idp::neighbor_update_in generation.interface_id_to_name[interface_id] = {route_name, interface_name}; } - generation_interface.switch_generation(); - generation_interface.next_unlock(); + return eResult::success; +} - netlink_neighbor_dump([this](const std::string& interface_name, - const common::ip_address_t& ip_address, - const common::mac_address_t& mac_address) { - tInterfaceId interface_id = 0; - { - auto lock = generation_interface.current_lock_guard(); +common::neighbor::idp::stats module::neighbor_stats() const +{ + return stats; +} - const auto& interface_name_to_id = generation_interface.current().interface_name_to_id; - auto it = interface_name_to_id.find(interface_name); - if (it == interface_name_to_id.end()) - { - return; - } - interface_id = it->second; - } +void module::report(nlohmann::json& json) +{ + json["neighbor"]["hashtable_insert_success"] = stats.hashtable_insert_success; + json["neighbor"]["hashtable_insert_error"] = stats.hashtable_insert_error; + json["neighbor"]["hashtable_remove_success"] = stats.hashtable_remove_success; + json["neighbor"]["hashtable_remove_error"] = stats.hashtable_remove_error; + json["neighbor"]["netlink_neighbor_update"] = stats.netlink_neighbor_update; + json["neighbor"]["resolve"] = stats.resolve; +} - YANET_LOG_DEBUG("netlink: %s, %s -> %s\n", - interface_name.data(), - ip_address.toString().data(), - mac_address.toString().data()); +void module::limits(common::idp::limits::response& response) +{ + (void)response; +} - stats.netlink_neighbor_update++; +void module::update_before(const common::idp::update::request& request) +{ + if (!request.neighbor()) + { + return; + } - dataplane::neighbor::key key; - memset(&key, 0, sizeof(key)); - key.interface_id = interface_id; - key.flags = 0; + /// @todo: lock generation_interface only on `update_interfaces` + generation_interface.next_lock(); - if (ip_address.is_ipv4()) - { - key.address.mapped_ipv4_address = ipv4_address_t::convert(ip_address.get_ipv4()); - } - else - { - key.address = ipv6_address_t::convert(ip_address.get_ipv6()); - key.flags |= flag_is_ipv6; - } + /// @todo: lock generation_hashtable only on write to hashtable + generation_hashtable.next_lock(); +} + +void module::update(const common::idp::update::request& request, + common::idp::update::response& response) +{ + if (!request.neighbor()) + { + return; + } - value value; - memcpy(value.ether_address.addr_bytes, mac_address.data(), RTE_ETHER_ADDR_LEN); - value.flags = 0; - value.last_update_timestamp = dataplane->get_current_time(); + const auto& [type, variant] = *request.neighbor(); - generation_hashtable.update([this, key, value](neighbor::generation_hashtable& hashtable) { - for (auto& [socket_id, hashtable_updater] : hashtable.hashtable_updater) + auto& response_neighbor = response.neighbor(); + if (type == common::neighbor::idp::type::show) + { + response_neighbor = neighbor_show(); + } + else if (type == common::neighbor::idp::type::insert) + { + response_neighbor = neighbor_insert(std::get(variant)); + } + else if (type == common::neighbor::idp::type::remove) + { + response_neighbor = neighbor_remove(std::get(variant)); + } + else if (type == common::neighbor::idp::type::clear) + { + response_neighbor = neighbor_clear(); + } + else if (type == common::neighbor::idp::type::update_interfaces) + { + response_neighbor = neighbor_update_interfaces(std::get(variant)); + + generation_interface.switch_generation_without_gc(); + + netlink_neighbor_dump([this](const std::string& interface_name, + const common::ip_address_t& ip_address, + const common::mac_address_t& mac_address) { + common::neighbor::idp::insert insert("", ///< @todo: route_name + interface_name, + ip_address, + mac_address); + auto result = neighbor_insert(insert); + if (result != eResult::success) { - (void)socket_id; - if (!hashtable_updater.get_pointer()->insert_or_update(key, value)) - { - stats.hashtable_insert_error++; - } - else - { - stats.hashtable_insert_success++; - } + YANET_LOG_WARNING("neighbor_insert(%s, %s): %s\n", + interface_name.data(), + ip_address.toString().data(), + common::result_to_c_str(result)); } - return eResult::success; }); - }); + } + else if (type == common::neighbor::idp::type::stats) + { + response_neighbor = neighbor_stats(); + } + else + { + response_neighbor = eResult::invalidType; + } - return eResult::success; + generation_hashtable.switch_generation_without_gc(); } -common::idp::neighbor_stats::response module::neighbor_stats() const +void module::update_after(const common::idp::update::request& request) { - return stats; -} + if (!request.neighbor()) + { + return; + } -void module::report(nlohmann::json& json) -{ - json["neighbor"]["hashtable_insert_success"] = stats.hashtable_insert_success; - json["neighbor"]["hashtable_insert_error"] = stats.hashtable_insert_error; - json["neighbor"]["hashtable_remove_success"] = stats.hashtable_remove_success; - json["neighbor"]["hashtable_remove_error"] = stats.hashtable_remove_error; - json["neighbor"]["netlink_neighbor_update"] = stats.netlink_neighbor_update; - json["neighbor"]["resolve"] = stats.resolve; + generation_interface.next().interface_name_to_id.clear(); + generation_interface.next().interface_id_to_name.clear(); + generation_hashtable.next_apply(); + + generation_interface.next_unlock(); + generation_hashtable.next_unlock(); } void module::main_thread() @@ -548,10 +579,6 @@ void module::main_thread() resolve(key); } - generation_hashtable.switch_generation_with_update([this]() { - dataplane->switch_worker_base(); - }); - std::this_thread::sleep_for(std::chrono::milliseconds(10)); } } @@ -567,61 +594,14 @@ void module::netlink_thread() netlink_neighbor_monitor([&](const std::string& interface_name, const common::ip_address_t& ip_address, const common::mac_address_t& mac_address) { - tInterfaceId interface_id = 0; - { - auto lock = generation_interface.current_lock_guard(); - - const auto& interface_name_to_id = generation_interface.current().interface_name_to_id; - auto it = interface_name_to_id.find(interface_name); - if (it == interface_name_to_id.end()) - { - return; - } - interface_id = it->second; - } - - YANET_LOG_DEBUG("netlink: %s, %s -> %s\n", - interface_name.data(), - ip_address.toString().data(), - mac_address.toString().data()); - - stats.netlink_neighbor_update++; - - dataplane::neighbor::key key; - memset(&key, 0, sizeof(key)); - key.interface_id = interface_id; - key.flags = 0; - - if (ip_address.is_ipv4()) - { - key.address.mapped_ipv4_address = ipv4_address_t::convert(ip_address.get_ipv4()); - } - else - { - key.address = ipv6_address_t::convert(ip_address.get_ipv6()); - key.flags |= flag_is_ipv6; - } - - value value; - memcpy(value.ether_address.addr_bytes, mac_address.data(), 6); - value.flags = 0; - value.last_update_timestamp = dataplane->get_current_time(); - - generation_hashtable.update([this, key, value](neighbor::generation_hashtable& hashtable) { - for (auto& [socket_id, hashtable_updater] : hashtable.hashtable_updater) - { - (void)socket_id; - if (!hashtable_updater.get_pointer()->insert_or_update(key, value)) - { - stats.hashtable_insert_error++; - } - else - { - stats.hashtable_insert_success++; - } - } - return eResult::success; - }); + common::neighbor::idp::insert insert("", ///< @todo: route_name + interface_name, + ip_address, + mac_address); + common::idp::update::request update; + update.neighbor() = {common::neighbor::idp::type::insert, insert}; + + dataplane->controlPlane->update(update); }); YANET_LOG_WARNING("restart neighbor_monitor\n"); @@ -658,33 +638,24 @@ void module::resolve(const dataplane::neighbor::key& key) ip_address.toString().data()); #ifdef CONFIG_YADECAP_AUTOTEST - value value; - value.ether_address.addr_bytes[0] = 44; - value.ether_address.addr_bytes[1] = 44; + common::mac_address_t mac_address; + mac_address.data()[0] = 44; + mac_address.data()[1] = 44; if (key.flags & flag_is_ipv6) { - value.ether_address.addr_bytes[0] = 66; - value.ether_address.addr_bytes[1] = 66; + mac_address.data()[0] = 66; + mac_address.data()[1] = 66; } - *((uint32_t*)&value.ether_address.addr_bytes[2]) = rte_hash_crc(key.address.bytes, 16, 0); - value.flags = 0; - value.last_update_timestamp = dataplane->get_current_time(); + *((uint32_t*)&mac_address.data()[2]) = rte_hash_crc(key.address.bytes, 16, 0); - generation_hashtable.update([this, key, value](neighbor::generation_hashtable& hashtable) { - for (auto& [socket_id, hashtable_updater] : hashtable.hashtable_updater) - { - (void)socket_id; - if (!hashtable_updater.get_pointer()->insert_or_update(key, value)) - { - stats.hashtable_insert_error++; - } - else - { - stats.hashtable_insert_success++; - } - } - return eResult::success; - }); + common::neighbor::idp::insert insert("", ///< @todo: route_name + interface_name, + ip_address, + mac_address); + common::idp::update::request update; + update.neighbor() = {common::neighbor::idp::type::insert, insert}; + + dataplane->controlPlane->update(update); #else // CONFIG_YADECAP_AUTOTEST /// @todo: in first, try resolve like 'ip neig show 10.0.0.1 dev eth0' diff --git a/dataplane/neighbor.h b/dataplane/neighbor.h index e163862b..975fe69f 100644 --- a/dataplane/neighbor.h +++ b/dataplane/neighbor.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include @@ -67,18 +66,13 @@ class module public: eResult init(cDataPlane* dataplane); - void update_worker_base(const std::vector>& base_nexts); - - common::idp::neighbor_show::response neighbor_show() const; - eResult neighbor_insert(const common::idp::neighbor_insert::request& request); - eResult neighbor_remove(const common::idp::neighbor_remove::request& request); - eResult neighbor_clear(); - eResult neighbor_flush(); - eResult neighbor_update_interfaces(const common::idp::neighbor_update_interfaces::request& request); - common::idp::neighbor_stats::response neighbor_stats() const; - void report(nlohmann::json& json); + void limits(common::idp::limits::response& response); + + void update_before(const common::idp::update::request& request); + void update(const common::idp::update::request& request, common::idp::update::response& response); + void update_after(const common::idp::update::request& request); protected: void main_thread(); @@ -86,6 +80,13 @@ class module void resolve(const dataplane::neighbor::key& key); + common::neighbor::idp::show neighbor_show() const; + eResult neighbor_insert(const common::neighbor::idp::insert& request); + eResult neighbor_remove(const common::neighbor::idp::remove& request); + eResult neighbor_clear(); + eResult neighbor_update_interfaces(const common::neighbor::idp::update_interfaces& request); + common::neighbor::idp::stats neighbor_stats() const; + protected: cDataPlane* dataplane; diff --git a/dataplane/report.cpp b/dataplane/report.cpp index 78677c76..a0876557 100644 --- a/dataplane/report.cpp +++ b/dataplane/report.cpp @@ -106,6 +106,8 @@ nlohmann::json cReport::getReport() jsonReport["memory_total"] = memory_total; dataPlane->neighbor.report(jsonReport); + dataPlane->memory_manager.report(jsonReport); + dataPlane->acl_module.report(jsonReport); return jsonReport; } @@ -686,12 +688,7 @@ nlohmann::json cReport::convertGlobalBase(const dataplane::globalBase::generatio } globalBase->updater.acl.network_table.report(json["acl"]["network_table"]); - globalBase->updater.acl.transport_table.report(json["acl"]["transport_table"]); - globalBase->updater.acl.total_table.report(json["acl"]["total_table"]); - globalBase->updater.acl.network_ipv4_source.report(json["acl"]["network"]["ipv4"]["source"]); - globalBase->updater.acl.network_ipv4_destination.report(json["acl"]["network"]["ipv4"]["destination"]); globalBase->updater.acl.network_ipv6_source.report(json["acl"]["network"]["ipv6"]["source"]); - globalBase->updater.acl.network_ipv6_destination_ht.report(json["acl"]["network"]["ipv6"]["destination_ht"]); globalBase->updater.acl.network_ipv6_destination.report(json["acl"]["network"]["ipv6"]["destination"]); json["serial"] = globalBase->serial; diff --git a/dataplane/updater.h b/dataplane/updater.h new file mode 100644 index 00000000..4dceb73a --- /dev/null +++ b/dataplane/updater.h @@ -0,0 +1,219 @@ +#pragma once + +#include +#include +#include + +#include "common.h" +#include "common/limit.h" +#include "hashtable.h" +#include "lpm.h" +#include "memory_manager.h" + +namespace dataplane +{ + +[[maybe_unused]] static uint32_t upper_power_of_two(const uint32_t value) +{ + /// @todo: use __builtin_clz + + uint32_t result = 1; + while (result < value) + { + result <<= 1; + if (!result) + { + return 0; + } + } + return result; +} + +[[maybe_unused]] static std::string to_hex(const void* pointer) +{ + char buffer[128]; + snprintf(buffer, 128, "%p", pointer); + return buffer; +} + +class updater_lpm4_24bit_8bit_id32 +{ +public: + using object_type = lpm4_24bit_8bit_id32_dynamic; + + updater_lpm4_24bit_8bit_id32(const char* name, + dataplane::memory_manager* memory_manager, + const tSocketId socket_id) : + name(name), + memory_manager(memory_manager), + socket_id(socket_id), + pointer(nullptr) + { + } + + eResult update(const std::vector& values) + { + stats.extended_chunks_size = std::max((uint64_t)object_type::extended_chunks_size_min, + values.size() / 16); + + for (;;) + { + pointer = memory_manager->create(name.data(), + socket_id, + object_type::calculate_sizeof(stats.extended_chunks_size)); + if (pointer == nullptr) + { + return eResult::errorAllocatingMemory; + } + + eResult result = pointer->fill(stats, values); + if (result != eResult::success) + { + /// try again + memory_manager->destroy(pointer); + stats.extended_chunks_size *= 2; + continue; + } + + break; + } + + return eResult::success; + } + + void clear() + { + if (pointer) + { + memory_manager->destroy(pointer); + pointer = nullptr; + } + } + + void limits(common::limit::limits& limits) const + { + limits.emplace_back(name + ".extended_chunks", + socket_id, + stats.extended_chunks_count, + stats.extended_chunks_size); + } + + void report(nlohmann::json& report) const + { + report["pointer"] = to_hex(pointer); + report["extended_chunks_count"] = stats.extended_chunks_count; + report["extended_chunks_size"] = stats.extended_chunks_size; + } + +protected: + std::string name; + dataplane::memory_manager* memory_manager; + tSocketId socket_id; + + object_type::stats_t stats; + +public: + object_type* pointer; +}; + +// + +template calculate_hash = calculate_hash_crc> +class updater_hashtable_mod_id32 +{ +public: + using object_type = hashtable_mod_id32_dynamic; + + updater_hashtable_mod_id32(const char* name, + dataplane::memory_manager* memory_manager, + const tSocketId socket_id) : + name(name), + memory_manager(memory_manager), + socket_id(socket_id), + pointer(nullptr) + { + } + + eResult update(const std::vector>& values) + { + stats.pairs_size = upper_power_of_two(std::max(object_type::pairs_size_min, + (uint32_t)(4ull * values.size()))); + + for (;;) + { + pointer = memory_manager->create(name.data(), + socket_id, + object_type::calculate_sizeof(stats.pairs_size), + stats.pairs_size); + if (pointer == nullptr) + { + return eResult::errorAllocatingMemory; + } + + eResult result = pointer->fill(stats, values); + if (result != eResult::success) + { + /// try again + memory_manager->destroy(pointer); + stats.pairs_size *= 2; + continue; + } + + break; + } + + return eResult::success; + } + + void clear() + { + if (pointer) + { + memory_manager->destroy(pointer); + pointer = nullptr; + } + } + + void limits(common::limit::limits& limits) const + { + limits.emplace_back(name + ".keys", + socket_id, + stats.pairs_count, + stats.pairs_size); + limits.emplace_back(name + ".longest_collision", + socket_id, + stats.longest_chain, + chunk_size); + } + + void report(nlohmann::json& report) const + { + report["pointer"] = to_hex(pointer); + report["pairs_count"] = stats.pairs_count; + report["pairs_size"] = stats.pairs_size; + for (unsigned int i = 0; + i < stats.pairs_in_chunks.size(); + i++) + { + report["pairs_in_chunks"][i] = stats.pairs_in_chunks[i]; + } + report["longest_chain"] = stats.longest_chain; + report["insert_failed"] = stats.insert_failed; + report["rewrites"] = stats.rewrites; + } + +protected: + std::string name; + dataplane::memory_manager* memory_manager; + tSocketId socket_id; + + typename object_type::stats_t stats; + +public: + object_type* pointer; +}; + +} diff --git a/dataplane/worker.cpp b/dataplane/worker.cpp index 6cf9d9b2..0b1a2f02 100644 --- a/dataplane/worker.cpp +++ b/dataplane/worker.cpp @@ -1423,14 +1423,14 @@ inline void cWorker::acl_ingress_handle4() key_acl.ipv4_destinations[mbuf_i].address = ipv4Header->dst_addr; } - acl.network.ipv4.source->lookup(key_acl.ipv4_sources, - value_acl.ipv4_sources, - acl_ingress_stack4.mbufsCount); - - acl.network.ipv4.destination->lookup(key_acl.ipv4_destinations, - value_acl.ipv4_destinations, + base.acl_network_ipv4_source->lookup(key_acl.ipv4_sources, + value_acl.ipv4_sources, acl_ingress_stack4.mbufsCount); + base.acl_network_ipv4_destination->lookup(key_acl.ipv4_destinations, + value_acl.ipv4_destinations, + acl_ingress_stack4.mbufsCount); + acl.network_table->lookup(value_acl.ipv4_sources, value_acl.ipv4_destinations, value_acl.networks, @@ -1494,10 +1494,10 @@ inline void cWorker::acl_ingress_handle4() transport_key.network_flags = acl.network_flags.array[metadata->network_flags]; } - acl.transport_table->lookup(hashes, - key_acl.transports, - value_acl.transports, - acl_ingress_stack4.mbufsCount); + base.acl_transport_table->lookup(hashes, + key_acl.transports, + value_acl.transports, + acl_ingress_stack4.mbufsCount); for (unsigned int mbuf_i = 0; mbuf_i < acl_ingress_stack4.mbufsCount; @@ -1513,10 +1513,10 @@ inline void cWorker::acl_ingress_handle4() total_key.transport_id = transport_value; } - acl.total_table->lookup(hashes, - key_acl.totals, - value_acl.totals, - acl_ingress_stack4.mbufsCount); + base.acl_total_table->lookup(hashes, + key_acl.totals, + value_acl.totals, + acl_ingress_stack4.mbufsCount); for (unsigned int mbuf_i = 0; mbuf_i < acl_ingress_stack4.mbufsCount; @@ -1612,10 +1612,10 @@ inline void cWorker::acl_ingress_handle6() acl_ingress_stack6.mbufsCount); { - uint32_t mask = acl.network.ipv6.destination_ht->lookup(hashes, - key_acl.ipv6_destinations, - value_acl.ipv6_destinations, - acl_ingress_stack6.mbufsCount); + uint32_t mask = base.acl_network_ipv6_destination_ht->lookup(hashes, + key_acl.ipv6_destinations, + value_acl.ipv6_destinations, + acl_ingress_stack6.mbufsCount); acl.network.ipv6.destination->lookup(mask, key_acl.ipv6_destinations, value_acl.ipv6_destinations, @@ -1685,10 +1685,10 @@ inline void cWorker::acl_ingress_handle6() transport_key.network_flags = acl.network_flags.array[metadata->network_flags]; } - acl.transport_table->lookup(hashes, - key_acl.transports, - value_acl.transports, - acl_ingress_stack6.mbufsCount); + base.acl_transport_table->lookup(hashes, + key_acl.transports, + value_acl.transports, + acl_ingress_stack6.mbufsCount); for (unsigned int mbuf_i = 0; mbuf_i < acl_ingress_stack6.mbufsCount; @@ -1704,10 +1704,10 @@ inline void cWorker::acl_ingress_handle6() total_key.transport_id = transport_value; } - acl.total_table->lookup(hashes, - key_acl.totals, - value_acl.totals, - acl_ingress_stack6.mbufsCount); + base.acl_total_table->lookup(hashes, + key_acl.totals, + value_acl.totals, + acl_ingress_stack6.mbufsCount); for (unsigned int mbuf_i = 0; mbuf_i < acl_ingress_stack6.mbufsCount; @@ -5134,14 +5134,14 @@ inline void cWorker::acl_egress_handle4() key_acl.ipv4_destinations[mbuf_i].address = ipv4Header->dst_addr; } - acl.network.ipv4.source->lookup(key_acl.ipv4_sources, - value_acl.ipv4_sources, - acl_egress_stack4.mbufsCount); - - acl.network.ipv4.destination->lookup(key_acl.ipv4_destinations, - value_acl.ipv4_destinations, + base.acl_network_ipv4_source->lookup(key_acl.ipv4_sources, + value_acl.ipv4_sources, acl_egress_stack4.mbufsCount); + base.acl_network_ipv4_destination->lookup(key_acl.ipv4_destinations, + value_acl.ipv4_destinations, + acl_egress_stack4.mbufsCount); + acl.network_table->lookup(value_acl.ipv4_sources, value_acl.ipv4_destinations, value_acl.networks, @@ -5205,10 +5205,10 @@ inline void cWorker::acl_egress_handle4() transport_key.network_flags = acl.network_flags.array[metadata->network_flags]; } - acl.transport_table->lookup(hashes, - key_acl.transports, - value_acl.transports, - acl_egress_stack4.mbufsCount); + base.acl_transport_table->lookup(hashes, + key_acl.transports, + value_acl.transports, + acl_egress_stack4.mbufsCount); for (unsigned int mbuf_i = 0; mbuf_i < acl_egress_stack4.mbufsCount; @@ -5224,10 +5224,10 @@ inline void cWorker::acl_egress_handle4() total_key.transport_id = transport_value; } - acl.total_table->lookup(hashes, - key_acl.totals, - value_acl.totals, - acl_egress_stack4.mbufsCount); + base.acl_total_table->lookup(hashes, + key_acl.totals, + value_acl.totals, + acl_egress_stack4.mbufsCount); for (unsigned int mbuf_i = 0; mbuf_i < acl_egress_stack4.mbufsCount; @@ -5389,10 +5389,10 @@ inline void cWorker::acl_egress_handle6() transport_key.network_flags = acl.network_flags.array[metadata->network_flags]; } - acl.transport_table->lookup(hashes, - key_acl.transports, - value_acl.transports, - acl_egress_stack6.mbufsCount); + base.acl_transport_table->lookup(hashes, + key_acl.transports, + value_acl.transports, + acl_egress_stack6.mbufsCount); for (unsigned int mbuf_i = 0; mbuf_i < acl_egress_stack6.mbufsCount; @@ -5408,10 +5408,10 @@ inline void cWorker::acl_egress_handle6() total_key.transport_id = transport_value; } - acl.total_table->lookup(hashes, - key_acl.totals, - value_acl.totals, - acl_egress_stack6.mbufsCount); + base.acl_total_table->lookup(hashes, + key_acl.totals, + value_acl.totals, + acl_egress_stack6.mbufsCount); for (unsigned int mbuf_i = 0; mbuf_i < acl_egress_stack6.mbufsCount;