Skip to content

Commit

Permalink
Non-contiguous DPDK port range
Browse files Browse the repository at this point in the history
In some cases we do not need to use all ports provided by `DPDK`.
Also the commit enables us to use port name instead of pci function
address as there are some devices like Chelsio having more than one port
sharing same pci function.

Closes #59
  • Loading branch information
Georgy Kirichenko authored and GeorgyKirichenko committed Feb 12, 2024
1 parent 8c5b411 commit 6b58af2
Show file tree
Hide file tree
Showing 6 changed files with 44 additions and 47 deletions.
1 change: 1 addition & 0 deletions dataplane/base.h
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ class permanently
tQueueId inQueueId;
} workerPorts[CONFIG_YADECAP_WORKER_PORTS_SIZE];

tPortId ports[CONFIG_YADECAP_PORTS_SIZE];
unsigned int ports_count;
tQueueId outQueueId;

Expand Down
16 changes: 7 additions & 9 deletions dataplane/controlplane.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,10 @@ void cControlPlane::start()
}

/// start devices
for (tPortId portId = 0; portId < rte_eth_dev_count_avail(); portId++)
for (const auto& portIter : dataPlane->ports)
{
const tPortId& portId = portIter.first;

int rc = rte_eth_dev_start(portId);
if (rc)
{
Expand Down Expand Up @@ -319,11 +321,9 @@ common::idp::getWorkerStats::response cControlPlane::getWorkerStats(const common
const auto& worker = dataPlane->workers.find(coreId)->second;

std::map<tPortId, common::worker::stats::port> portsStats;
for (tPortId portId = 0;
portId < dataPlane->ports.size();
portId++)
for (const auto& portIter : dataPlane->ports)
{
portsStats[portId] = worker->statsPorts[portId];
portsStats[portIter.first] = worker->statsPorts[portIter.first];
}

response[coreId] = {worker->iteration,
Expand All @@ -338,11 +338,9 @@ common::idp::getWorkerStats::response cControlPlane::getWorkerStats(const common
for (const auto& [coreId, worker] : dataPlane->workers)
{
std::map<tPortId, common::worker::stats::port> portsStats;
for (tPortId portId = 0;
portId < dataPlane->ports.size();
portId++)
for (const auto& portIter : dataPlane->ports)
{
portsStats[portId] = worker->statsPorts[portId];
portsStats[portIter.first] = worker->statsPorts[portIter.first];
}

response[coreId] = {worker->iteration,
Expand Down
64 changes: 30 additions & 34 deletions dataplane/dataplane.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -160,15 +160,6 @@ eResult cDataPlane::init(const std::string& binaryPath,
return result;
}

/// sanity check
if (rte_eth_dev_count_avail() != ports.size())
{
YADECAP_LOG_ERROR("invalid ports count: %u != %lu\n",
rte_eth_dev_count_avail(),
ports.size());
return eResult::invalidPortsCount;
}

mempool_log = rte_mempool_create("log", YANET_CONFIG_SAMPLES_SIZE, sizeof(samples::sample_t), 0, 0, NULL, NULL, NULL, NULL, SOCKET_ID_ANY, MEMPOOL_F_NO_IOVA_CONTIG);

result = initGlobalBases();
Expand Down Expand Up @@ -465,16 +456,17 @@ eResult cDataPlane::initPorts()
for (const auto& configPortIter : config.ports)
{
const std::string& interfaceName = configPortIter.first;
const auto& [pci, symmetric_mode, rss_flags] = configPortIter.second;
const auto& [pci, name, symmetric_mode, rss_flags] = configPortIter.second;
(void)pci;

tPortId portId;
if (strncmp(pci.data(), SOCK_DEV_PREFIX, strlen(SOCK_DEV_PREFIX)) == 0)
if (strncmp(name.data(), SOCK_DEV_PREFIX, strlen(SOCK_DEV_PREFIX)) == 0)
{
portId = sock_dev_create(pci.data(), 0);
portId = sock_dev_create(name.data(), 0);
}
else if (rte_eth_dev_get_port_by_name(pci.data(), &portId))
else if (rte_eth_dev_get_port_by_name(name.data(), &portId))
{
YADECAP_LOG_ERROR("invalid pci: '%s'\n", pci.data());
YADECAP_LOG_ERROR("invalid name: '%s'\n", name.data());
remove_keys.emplace_back(interfaceName);
continue;
}
Expand Down Expand Up @@ -588,17 +580,6 @@ eResult cDataPlane::initPorts()
config.ports.erase(interface_name);
}

for (const auto& [port_id, port] : ports)
{
(void)port;

if (port_id >= ports.size())
{
YADECAP_LOG_ERROR("invalid portId: '%u'\n", port_id);
return eResult::invalidPortId;
}
}

return eResult::success;
}

Expand Down Expand Up @@ -853,7 +834,12 @@ eResult cDataPlane::initWorkers()
dataplane::base::permanently basePermanently;
basePermanently.globalBaseAtomic = globalBaseAtomics[socket_id];
basePermanently.outQueueId = outQueueId; ///< 0
basePermanently.ports_count = ports.size();
basePermanently.ports_count = 0;
for (const auto& portIter : ports)
{
basePermanently.ports[basePermanently.ports_count++] = portIter.first;
}

basePermanently.SWNormalPriorityRateLimitPerWorker = config.SWNormalPriorityRateLimitPerWorker;

dataplane::base::generation base;
Expand Down Expand Up @@ -929,12 +915,15 @@ eResult cDataPlane::initWorkers()
basePermanently.nat64stateful_numa_id = rte_cpu_to_be_16(socket_id);
}

basePermanently.ports_count = 0;
for (const auto& [port_id, port] : ports)
{
const auto& [interface_name, rx_queues, tx_queues_count, mac_address, pci, symmetric_mode] = port;
(void)mac_address;
(void)pci;

basePermanently.ports[basePermanently.ports_count++] = port_id;

if (exist(rx_queues, coreId))
{
YANET_LOG_DEBUG("worker[%u]: add_worker_port(port_id: %u, queue_id: %u)\n",
Expand Down Expand Up @@ -999,7 +988,6 @@ eResult cDataPlane::initWorkers()
}

basePermanently.outQueueId = outQueueId;
basePermanently.ports_count = ports.size();

dataplane::base::generation base;
{
Expand Down Expand Up @@ -1759,6 +1747,7 @@ eResult cDataPlane::parseJsonPorts(const nlohmann::json& json)
{
std::string interfaceName = portJson["interfaceName"];
std::string pci = portJson["pci"];
std::string name = pci;
bool symmetric_mode = false;
uint64_t rss_flags = 0;

Expand All @@ -1768,6 +1757,11 @@ eResult cDataPlane::parseJsonPorts(const nlohmann::json& json)
return eResult::invalidConfigurationFile;
}

if (exist(portJson, "name"))
{
name = portJson["name"];
}

if (exist(portJson, "symmetric_mode"))
{
symmetric_mode = portJson["symmetric_mode"];
Expand All @@ -1787,7 +1781,7 @@ eResult cDataPlane::parseJsonPorts(const nlohmann::json& json)
rss_flags = RTE_ETH_RSS_IP;
}

config.ports[interfaceName] = {pci, symmetric_mode, rss_flags};
config.ports[interfaceName] = {pci, name, symmetric_mode, rss_flags};

for (tCoreId coreId : portJson["coreIds"])
{
Expand Down Expand Up @@ -2036,20 +2030,21 @@ eResult cDataPlane::checkConfig()
}

{
std::set<std::string> pcis;
std::set<std::string> names;
for (const auto& portIter : config.ports)
{
const auto& [pci, symmetric_mode, rss_flags] = portIter.second;
const auto& [pci, name, symmetric_mode, rss_flags] = portIter.second;
(void)pci;
(void)symmetric_mode;
(void)rss_flags;

if (exist(pcis, pci))
if (exist(names, name))
{
YADECAP_LOG_ERROR("pci '%s' already exist\n", pci.data());
YADECAP_LOG_ERROR("pci '%s' already exist\n", name.data());
return eResult::invalidConfigurationFile;
}

pcis.emplace(pci);
names.emplace(name);
}
}

Expand Down Expand Up @@ -2133,7 +2128,8 @@ eResult cDataPlane::initEal(const std::string& binaryPath,

for (const auto& port : config.ports)
{
const auto& [pci, symmetric_mode, rss_flags] = port.second;
const auto& [pci, name, symmetric_mode, rss_flags] = port.second;
(void)name;
(void)symmetric_mode;
(void)rss_flags;

Expand Down
1 change: 1 addition & 0 deletions dataplane/dataplane.h
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ struct tDataPlaneConfig
*/
std::map<std::string, ///< interfaceName
std::tuple<std::string, ///< pci
std::string, ///< name
bool, ///< symmetric_mode
uint64_t ///< rssFlags
>>
Expand Down
2 changes: 1 addition & 1 deletion dataplane/globalbase.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -623,7 +623,7 @@ eResult generation::updateLogicalPort(const common::idp::updateGlobalBase::updat
YADECAP_LOG_ERROR("invalid logicalPortId: '%u'\n", logicalPortId);
return eResult::invalidLogicalPortId;
}
if (portId >= dataPlane->ports.size())
if (!exist(dataPlane->ports, portId))
{
YADECAP_LOG_ERROR("invalid portId: '%u'\n", portId);
return eResult::invalidPortId;
Expand Down
7 changes: 4 additions & 3 deletions dataplane/worker.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1091,10 +1091,11 @@ inline void cWorker::physicalPort_ingress_handle(const unsigned int& worker_port

inline void cWorker::physicalPort_egress_handle()
{
for (tPortId portId = 0;
portId < basePermanently.ports_count;
portId++)
for (uint32_t portId_i = 0;
portId_i < basePermanently.ports_count;
portId_i++)
{
const auto portId = basePermanently.ports[portId_i];
if (unlikely(physicalPort_stack[portId].mbufsCount == 0))
{
continue;
Expand Down

0 comments on commit 6b58af2

Please sign in to comment.