From 44f6f689aa01d1f8215f958b4e5b320de0cba4e3 Mon Sep 17 00:00:00 2001 From: kishanps Date: Wed, 24 Jan 2024 07:56:39 -0800 Subject: [PATCH 1/2] Create test skeleton with mock helpers, validators, and programmers. --- tests/integration/system/nsf/BUILD.bazel | 25 +++ tests/integration/system/nsf/upgrade_test.cc | 165 +++++++++++++++++++ tests/integration/system/nsf/upgrade_test.h | 50 ++++++ 3 files changed, 240 insertions(+) create mode 100644 tests/integration/system/nsf/upgrade_test.cc create mode 100644 tests/integration/system/nsf/upgrade_test.h diff --git a/tests/integration/system/nsf/BUILD.bazel b/tests/integration/system/nsf/BUILD.bazel index c9f3080b..f3cdcf0e 100644 --- a/tests/integration/system/nsf/BUILD.bazel +++ b/tests/integration/system/nsf/BUILD.bazel @@ -34,3 +34,28 @@ cc_library( "@com_google_absl//absl/types:span", ], ) + +cc_library( + name = "upgrade_test", + testonly = True, + srcs = ["upgrade_test.cc"], + hdrs = ["upgrade_test.h"], + deps = [ + ":util", + "//gutil:status_matchers", + "//gutil:testing", + "//tests/integration/system/nsf/interfaces:component_validator", + "//tests/integration/system/nsf/interfaces:flow_programmer", + "//tests/integration/system/nsf/interfaces:test_params", + "//tests/integration/system/nsf/interfaces:traffic_helper", + "//thinkit:generic_testbed", + "//thinkit:generic_testbed_fixture", + "//thinkit/proto:generic_testbed_cc_proto", + "@com_github_google_glog//:glog", + "@com_github_p4lang_p4runtime//:p4info_cc_proto", + "@com_github_p4lang_p4runtime//:p4runtime_cc_proto", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/tests/integration/system/nsf/upgrade_test.cc b/tests/integration/system/nsf/upgrade_test.cc new file mode 100644 index 00000000..24f7ca8a --- /dev/null +++ b/tests/integration/system/nsf/upgrade_test.cc @@ -0,0 +1,165 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tests/integration/system/nsf/upgrade_test.h" + +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "glog/logging.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "gutil/status.h" +#include "gutil/status_matchers.h" // NOLINT: Need to add status_matchers.h for using `ASSERT_OK` in upstream code. +#include "gutil/testing.h" +#include "p4/config/v1/p4info.pb.h" +#include "p4/v1/p4runtime.pb.h" +#include "tests/integration/system/nsf/interfaces/component_validator.h" +#include "tests/integration/system/nsf/interfaces/flow_programmer.h" +#include "tests/integration/system/nsf/interfaces/traffic_helper.h" +#include "tests/integration/system/nsf/util.h" +#include "thinkit/generic_testbed.h" +#include "thinkit/proto/generic_testbed.pb.h" + +namespace pins_test { + +using ::p4::v1::ReadResponse; + +constexpr int kErrorPercentage = 1; + +void NsfUpgradeTest::SetUp() { + flow_programmer_ = GetParam().create_flow_programmer(); + traffic_helper_ = GetParam().create_traffic_helper(); + testbed_interface_ = GetParam().create_testbed_interface(); + component_validators_ = GetParam().create_component_validators(); + testbed_interface_->SetUp(); +} +void NsfUpgradeTest::TearDown() { testbed_interface_->TearDown(); } + +// Assumption: Valid config (gNMI and P4Info) has been pushed (to avoid +// duplicate config push) +absl::Status NsfUpgradeTest::NsfUpgrade(absl::string_view prev_version, + absl::string_view version) { + // Pick a testbed with SUT connected to an Ixia on 2 ports, one ingress and + // one egress port. + auto requirements = gutil::ParseProtoOrDie( + R"pb( + interface_requirements { count: 2 interface_mode: TRAFFIC_GENERATOR } + )pb"); + ASSIGN_OR_RETURN( + std::unique_ptr testbed, + testbed_interface_->GetTestbedWithRequirements(requirements)); + + RETURN_IF_ERROR(ValidateSwitchState(prev_version)); + RETURN_IF_ERROR(ValidateComponents(&ComponentValidator::OnInit, + component_validators_, prev_version, + *testbed)); + RETURN_IF_ERROR(CaptureDbState()); + + // P4 Snapshot before programming flows and starting the + // traffic. + ReadResponse snapshot1 = TakeP4FlowSnapshot(); + + // Program all the flows + RETURN_IF_ERROR(flow_programmer_->ProgramFlows(IpVersion::kIpv4, + Protocol::kTcp, *testbed)); + RETURN_IF_ERROR(ValidateComponents(&ComponentValidator::OnFlowProgram, + component_validators_, prev_version, + *testbed)); + + RETURN_IF_ERROR(traffic_helper_->StartTraffic(*testbed)); + RETURN_IF_ERROR(ValidateComponents(&ComponentValidator::OnStartTraffic, + component_validators_, prev_version, + *testbed)); + RETURN_IF_ERROR(traffic_helper_->ValidateTraffic(kErrorPercentage, *testbed)); + // Since the validation is while the traffic is in progress, + // error margin needs to be defined. + + // P4 Snapshot before Upgrade and NSF reboot. + ReadResponse snapshot2 = TakeP4FlowSnapshot(); + + // Perform Upgrade + RETURN_IF_ERROR(Upgrade(version)); + RETURN_IF_ERROR(ValidateComponents(&ComponentValidator::OnUpgrade, + component_validators_, version, *testbed)); + + // Perform NSF Reboot + RETURN_IF_ERROR(NsfReboot(version)); + RETURN_IF_ERROR(ValidateComponents(&ComponentValidator::OnNsfReboot, + component_validators_, version, *testbed)); + + RETURN_IF_ERROR(ValidateSwitchState(version)); + RETURN_IF_ERROR(ValidateDbState()); + + // P4 Snapshot after upgrade and NSF reboot. + ReadResponse snapshot3 = TakeP4FlowSnapshot(); + + RETURN_IF_ERROR(traffic_helper_->ValidateTraffic(kErrorPercentage, *testbed)); + RETURN_IF_ERROR(PushConfig(version)); // Push the newer config + + RETURN_IF_ERROR(ValidateSwitchState(version)); + RETURN_IF_ERROR(ValidateComponents(&ComponentValidator::OnConfigPush, + component_validators_, version, *testbed)); + + // Stop and validate traffic + RETURN_IF_ERROR(traffic_helper_->ValidateTraffic(kErrorPercentage, *testbed)); + RETURN_IF_ERROR(traffic_helper_->StopTraffic(*testbed)); + + // For now, we validate traffic only after stopping traffic. Ideally we + // would want to validate traffic while injection is in progress to narrow + // down when the traffic loss occurred (i.e. before reboot, during reboot or + // after reconciliation). Although this is possible in OTG traffic + // generators, DVaaS traffic generator for now does not support traffic + // validation until only after stopping the traffic. This is a good-to-have + // feature and we will update the skeleton to validate traffic while + // injection is ongoing once this feature is available in DVaaS (more + // details). + RETURN_IF_ERROR(traffic_helper_->ValidateTraffic(kErrorPercentage, *testbed)); + RETURN_IF_ERROR(ValidateComponents(&ComponentValidator::OnStopTraffic, + component_validators_, version, *testbed)); + + // Selectively clear flows (eg. not clearing nexthop entries for host + // testbeds) + RETURN_IF_ERROR(flow_programmer_->ClearFlows(*testbed)); + + RETURN_IF_ERROR(ValidateComponents(&ComponentValidator::OnFlowCleanup, + component_validators_, version, *testbed)); + + ReadResponse snapshot4 = TakeP4FlowSnapshot(); + + RETURN_IF_ERROR(CompareP4FlowSnapshots(snapshot1, snapshot4)); + return CompareP4FlowSnapshots(snapshot2, snapshot3); +} + +TEST_P(NsfUpgradeTest, UpgradeAndReboot) { + static constexpr absl::string_view kThirdLastImage = "third_last_image"; + static constexpr absl::string_view kSecondLastImage = "second_last_image"; + static constexpr absl::string_view kLastImage = "last_image"; + static constexpr absl::string_view kCurrentImage = "current_image"; + + ASSERT_OK(InstallRebootPushConfig(kThirdLastImage)); + + // NSF Upgrade to N - 2 (once a week) + ASSERT_OK(NsfUpgrade(kThirdLastImage, kSecondLastImage)); + + // NSF Upgrade to N - 1 + ASSERT_OK(NsfUpgrade(kSecondLastImage, kLastImage)); + + // NSF Upgrade to N + ASSERT_OK(NsfUpgrade(kLastImage, kCurrentImage)); +} + +} // namespace pins_test diff --git a/tests/integration/system/nsf/upgrade_test.h b/tests/integration/system/nsf/upgrade_test.h new file mode 100644 index 00000000..3043cc61 --- /dev/null +++ b/tests/integration/system/nsf/upgrade_test.h @@ -0,0 +1,50 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PINS_TESTS_INTEGRATION_SYSTEM_NSF_UPGRADE_TEST_H_ +#define PINS_TESTS_INTEGRATION_SYSTEM_NSF_UPGRADE_TEST_H_ + +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "gtest/gtest.h" +#include "tests/integration/system/nsf/interfaces/component_validator.h" +#include "tests/integration/system/nsf/interfaces/flow_programmer.h" +#include "tests/integration/system/nsf/interfaces/test_params.h" +#include "tests/integration/system/nsf/interfaces/traffic_helper.h" +#include "thinkit/generic_testbed_fixture.h" + +namespace pins_test { + +class NsfUpgradeTest : public testing::TestWithParam { + protected: + void SetUp() override; + void TearDown() override; + + // Assumption: Valid config (gNMI and P4Info) has already been pushed. + absl::Status NsfUpgrade(absl::string_view prev_version, + absl::string_view version); + + private: + std::unique_ptr flow_programmer_; + std::unique_ptr traffic_helper_; + std::unique_ptr testbed_interface_; + std::vector> component_validators_; +}; + +} // namespace pins_test + +#endif // PINS_TESTS_INTEGRATION_SYSTEM_NSF_UPGRADE_TEST_H_ From 141f1f1c64b23e5f61bad5c92ecef687ef8b7c9a Mon Sep 17 00:00:00 2001 From: kishanps Date: Wed, 24 Jan 2024 09:20:55 -0800 Subject: [PATCH 2/2] [Thinkit] Create milestones for controlling test behavior in PINs NSF integration tests.Implement OTG traffic helper for the NSF Upgrade test skeleton. --- tests/integration/system/nsf/BUILD.bazel | 13 ++ tests/integration/system/nsf/milestone.cc | 59 ++++++ tests/integration/system/nsf/milestone.h | 36 ++++ .../system/nsf/traffic_helpers/BUILD.bazel | 7 + .../system/nsf/traffic_helpers/otg_helper.cc | 199 ++++++++++++++++++ .../system/nsf/traffic_helpers/otg_helper.h | 12 +- tests/integration/system/nsf/upgrade_test.cc | 5 + 7 files changed, 322 insertions(+), 9 deletions(-) create mode 100644 tests/integration/system/nsf/milestone.cc create mode 100644 tests/integration/system/nsf/milestone.h create mode 100644 tests/integration/system/nsf/traffic_helpers/otg_helper.cc diff --git a/tests/integration/system/nsf/BUILD.bazel b/tests/integration/system/nsf/BUILD.bazel index f3cdcf0e..1ed98405 100644 --- a/tests/integration/system/nsf/BUILD.bazel +++ b/tests/integration/system/nsf/BUILD.bazel @@ -41,6 +41,7 @@ cc_library( srcs = ["upgrade_test.cc"], hdrs = ["upgrade_test.h"], deps = [ + ":milestone", ":util", "//gutil:status_matchers", "//gutil:testing", @@ -54,8 +55,20 @@ cc_library( "@com_github_google_glog//:glog", "@com_github_p4lang_p4runtime//:p4info_cc_proto", "@com_github_p4lang_p4runtime//:p4runtime_cc_proto", + "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/status", "@com_google_absl//absl/strings:string_view", "@com_google_googletest//:gtest_main", ], ) + +cc_library( + name = "milestone", + srcs = ["milestone.cc"], + hdrs = ["milestone.h"], + deps = [ + "@com_github_google_glog//:glog", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:string_view", + ], +) diff --git a/tests/integration/system/nsf/milestone.cc b/tests/integration/system/nsf/milestone.cc new file mode 100644 index 00000000..94dc724d --- /dev/null +++ b/tests/integration/system/nsf/milestone.cc @@ -0,0 +1,59 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tests/integration/system/nsf/milestone.h" + +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "glog/logging.h" + +namespace pins_test { + +namespace { + +std::vector> MilestoneToStringMapping() { + return { + {NsfMilestone::kAll, "all"}, + {NsfMilestone::kShutdown, "shutdown"}, + {NsfMilestone::kBootup, "bootup"}, + }; +} + +} // namespace + +bool AbslParseFlag(absl::string_view milestone_text, NsfMilestone* milestone, + std::string* error) { + for (auto& [milestone_, string] : MilestoneToStringMapping()) { + if (string == milestone_text) { + *milestone = milestone_; + return true; + } + } + absl::StrAppend(error, "unknown milestone: '", milestone_text, "'"); + return false; +} + +std::string AbslUnparseFlag(NsfMilestone milestone) { + for (auto& [milestone_, string] : MilestoneToStringMapping()) { + if (milestone_ == milestone) return string; + } + LOG(DFATAL) << "invalid milestone: " << static_cast(milestone); + return ""; +} + +} // namespace pins_test diff --git a/tests/integration/system/nsf/milestone.h b/tests/integration/system/nsf/milestone.h new file mode 100644 index 00000000..8efc8a02 --- /dev/null +++ b/tests/integration/system/nsf/milestone.h @@ -0,0 +1,36 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PINS_TESTS_INTEGRATION_SYSTEM_NSF_MILESTONE_H_ +#define PINS_TESTS_INTEGRATION_SYSTEM_NSF_MILESTONE_H_ + +#include + +#include "absl/strings/string_view.h" + +namespace pins_test { + +enum class NsfMilestone { + kAll, // All NSF milestones in one. + kShutdown, + kBootup, +}; + +bool AbslParseFlag(absl::string_view milestone_text, NsfMilestone* milestone, + std::string* error); +std::string AbslUnparseFlag(NsfMilestone milestone); + +} // namespace pins_test + +#endif // PINS_TESTS_INTEGRATION_SYSTEM_NSF_MILESTONE_H_ diff --git a/tests/integration/system/nsf/traffic_helpers/BUILD.bazel b/tests/integration/system/nsf/traffic_helpers/BUILD.bazel index c8c9dc40..97b91642 100644 --- a/tests/integration/system/nsf/traffic_helpers/BUILD.bazel +++ b/tests/integration/system/nsf/traffic_helpers/BUILD.bazel @@ -33,10 +33,17 @@ cc_library( cc_library( name = "otg_helper", testonly = True, + srcs = ["otg_helper.cc"], hdrs = ["otg_helper.h"], deps = [ + "//gutil:status", + "//lib/utils:generic_testbed_utils", "//tests/integration/system/nsf/interfaces:traffic_helper", "//thinkit:generic_testbed", + "@com_github_grpc_grpc//:grpc++", + "@com_github_otg_models//:otg_cc_proto", + "@com_github_otg_models//:otg_grpc_proto", "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", ], ) diff --git a/tests/integration/system/nsf/traffic_helpers/otg_helper.cc b/tests/integration/system/nsf/traffic_helpers/otg_helper.cc new file mode 100644 index 00000000..0727654f --- /dev/null +++ b/tests/integration/system/nsf/traffic_helpers/otg_helper.cc @@ -0,0 +1,199 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "tests/integration/system/nsf/traffic_helpers/otg_helper.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "artifacts/otg.grpc.pb.h" +#include "artifacts/otg.pb.h" +#include "grpcpp/client_context.h" +#include "gutil/status.h" +#include "lib/utils/generic_testbed_utils.h" +#include "thinkit/generic_testbed.h" + +namespace pins_test { + +absl::Status OtgHelper::StartTraffic(thinkit::GenericTestbed& testbed) { + ASSIGN_OR_RETURN(std::vector up_links, + GetUpLinks(GetAllTrafficGeneratorLinks, testbed)); + if (up_links.size() <= 2) { + return absl::InvalidArgumentError( + "Test requires at least 2 SUT ports connected to a Software (Host) or " + "Hardware (Ixia) Traffic Generator"); + } + + // Create config. + otg::SetConfigRequest set_config_request; + otg::SetConfigResponse set_config_response; + grpc::ClientContext set_config_context; + auto* config = set_config_request.mutable_config(); + + // Randomly pick source and destination hosts. + const std::string otg_src_port = up_links.front().peer_interface; + const std::string otg_dst_port = up_links.back().peer_interface; + const std::string otg_src_mac = up_links.front().peer_mac_address; + const std::string otg_dst_mac = up_links.back().peer_mac_address; + const std::string otg_src_ip = up_links.front().peer_ipv6_address; + const std::string otg_dst_ip = up_links.back().peer_ipv6_address; + const std::string otg_src_loc = up_links.front().peer_traffic_location; + const std::string otg_dst_loc = up_links.back().peer_traffic_location; + + // Add ports. + auto* src_port = config->add_ports(); + auto* dst_port = config->add_ports(); + src_port->set_name(otg_src_port); + dst_port->set_name(otg_dst_port); + src_port->set_location(otg_src_loc); + dst_port->set_location(otg_dst_loc); + + // Set layer1. + auto* layer1 = config->add_layer1(); + layer1->set_name("ly"); + layer1->add_port_names(otg_src_port); + layer1->add_port_names(otg_dst_port); + + // Set speed. + layer1->set_speed(otg::Layer1::Speed::speed_1_gbps); + + // Set MTU. + layer1->set_mtu(9000); + + // Create flow. + auto* flow = config->add_flows(); + flow->set_name("Host Traffic flow"); + + // Set Tx / Rx ports. + flow->mutable_tx_rx()->set_choice(otg::FlowTxRx::Choice::port); + flow->mutable_tx_rx()->mutable_port()->set_tx_name(otg_src_port); + flow->mutable_tx_rx()->mutable_port()->set_rx_name(otg_dst_port); + + // Set packet size. + flow->mutable_size()->set_choice(otg::FlowSize::Choice::fixed); + flow->mutable_size()->set_fixed(256); + + // Set transmission duration. + flow->mutable_duration()->set_choice(otg::FlowDuration::Choice::continuous); + + // Set transmission rate. + flow->mutable_rate()->set_choice(otg::FlowRate::Choice::percentage); + flow->mutable_rate()->set_percentage(10); + + // Set capture metrics. + flow->mutable_metrics()->set_enable(true); + flow->mutable_metrics()->set_loss(false); + flow->mutable_metrics()->set_timestamps(true); + flow->mutable_metrics()->mutable_latency()->set_enable(true); + flow->mutable_metrics()->mutable_latency()->set_mode( + otg::FlowLatencyMetrics::Mode::cut_through); + + // Set ethernet header. + auto* eth_packet = flow->add_packet(); + eth_packet->set_choice(otg::FlowHeader::Choice::ethernet); + eth_packet->mutable_ethernet()->mutable_src()->set_choice( + otg::PatternFlowEthernetSrc::Choice::value); + eth_packet->mutable_ethernet()->mutable_dst()->set_choice( + otg::PatternFlowEthernetDst::Choice::value); + eth_packet->mutable_ethernet()->mutable_src()->set_value(otg_src_mac); + eth_packet->mutable_ethernet()->mutable_dst()->set_value(otg_dst_mac); + + // Set IP header. + auto* ip_packet = flow->add_packet(); + ip_packet->set_choice(otg::FlowHeader::Choice::ipv6); + ip_packet->mutable_ipv6()->mutable_src()->set_choice( + otg::PatternFlowIpv6Src::Choice::value); + ip_packet->mutable_ipv6()->mutable_dst()->set_choice( + otg::PatternFlowIpv6Dst::Choice::value); + ip_packet->mutable_ipv6()->mutable_src()->set_value(otg_src_ip); + ip_packet->mutable_ipv6()->mutable_dst()->set_value(otg_dst_ip); + + // Set the config. + otg::Openapi::StubInterface* stub = testbed.GetTrafficClient(); + RETURN_IF_ERROR(gutil::GrpcStatusToAbslStatus(stub->SetConfig( + &set_config_context, set_config_request, &set_config_response))); + + // Start the traffic. + otg::SetControlStateRequest request; + otg::SetControlStateResponse response; + grpc::ClientContext context; + request.mutable_control_state()->set_choice( + otg::ControlState::Choice::traffic); + request.mutable_control_state()->mutable_traffic()->set_choice( + otg::StateTraffic::Choice::flow_transmit); + request.mutable_control_state() + ->mutable_traffic() + ->mutable_flow_transmit() + ->set_state(otg::StateTrafficFlowTransmit::State::start); + return gutil::GrpcStatusToAbslStatus( + stub->SetControlState(&context, request, &response)); +} + +absl::Status OtgHelper::StopTraffic(thinkit::GenericTestbed& testbed) { + return absl::UnimplementedError("Stopping traffic is not implemented."); +} + +absl::Status OtgHelper::ValidateTraffic(int error_margin, + thinkit::GenericTestbed& testbed) { + otg::Openapi::StubInterface* stub = testbed.GetTrafficClient(); + otg::GetMetricsRequest metrics_req; + otg::GetMetricsResponse metrics_res; + grpc::ClientContext metrics_ctx; + + metrics_req.mutable_metrics_request()->set_choice( + otg::MetricsRequest::Choice::flow); + RETURN_IF_ERROR(gutil::GrpcStatusToAbslStatus( + stub->GetMetrics(&metrics_ctx, metrics_req, &metrics_res))); + + // Verify flow metrics is not empty. + if (metrics_res.metrics_response().flow_metrics().empty()) { + return absl::InternalError( + "Cannot validate traffic as no flow metrics received."); + } + + // Verify transmission completion and no frames or bytes drops for each flow. + std::vector errors; + for (const auto& flow_metric : + metrics_res.metrics_response().flow_metrics()) { + uint64_t bytes_drop = flow_metric.bytes_tx() - flow_metric.bytes_rx(); + uint64_t frames_drop = flow_metric.frames_tx() - flow_metric.frames_rx(); + uint64_t bytes_drop_percent = (bytes_drop / flow_metric.bytes_tx()) * 100; + float_t frames_drop_percent = (frames_drop / flow_metric.frames_tx()) * 100; + bool transmission_stopped = + flow_metric.transmit() == otg::FlowMetric::Transmit::stopped; + + if (bytes_drop_percent <= error_margin && + frames_drop_percent <= error_margin) + continue; + + errors.push_back(absl::StrCat( + "Flow name:\t\t", flow_metric.name(), "\nBytes dropped:\t\t", + bytes_drop, "\nFrames dropped:\t\t", frames_drop, + transmission_stopped + ? "" + : "\nTransmission not completed within the expected time.")); + } + + if (errors.empty()) return absl::OkStatus(); + return absl::InternalError(absl::StrCat( + "Following errors were observed while validating traffic:\n\n", + absl::StrJoin(errors, "\n\n"))); +} + +} // namespace pins_test diff --git a/tests/integration/system/nsf/traffic_helpers/otg_helper.h b/tests/integration/system/nsf/traffic_helpers/otg_helper.h index 71ee8b43..b1155186 100644 --- a/tests/integration/system/nsf/traffic_helpers/otg_helper.h +++ b/tests/integration/system/nsf/traffic_helpers/otg_helper.h @@ -23,16 +23,10 @@ namespace pins_test { class OtgHelper : public TrafficHelper { public: - absl::Status StartTraffic(thinkit::GenericTestbed& testbed) override { - return absl::OkStatus(); - }; - absl::Status StopTraffic(thinkit::GenericTestbed& testbed) override { - return absl::OkStatus(); - }; + absl::Status StartTraffic(thinkit::GenericTestbed& testbed) override; + absl::Status StopTraffic(thinkit::GenericTestbed& testbed) override; absl::Status ValidateTraffic(int error_margin, - thinkit::GenericTestbed& testbed) override { - return absl::OkStatus(); - }; + thinkit::GenericTestbed& testbed) override; }; } // namespace pins_test diff --git a/tests/integration/system/nsf/upgrade_test.cc b/tests/integration/system/nsf/upgrade_test.cc index 24f7ca8a..9fdde76a 100644 --- a/tests/integration/system/nsf/upgrade_test.cc +++ b/tests/integration/system/nsf/upgrade_test.cc @@ -17,6 +17,7 @@ #include #include +#include "absl/flags/flag.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" #include "glog/logging.h" @@ -30,10 +31,14 @@ #include "tests/integration/system/nsf/interfaces/component_validator.h" #include "tests/integration/system/nsf/interfaces/flow_programmer.h" #include "tests/integration/system/nsf/interfaces/traffic_helper.h" +#include "tests/integration/system/nsf/milestone.h" #include "tests/integration/system/nsf/util.h" #include "thinkit/generic_testbed.h" #include "thinkit/proto/generic_testbed.pb.h" +ABSL_FLAG(pins_test::NsfMilestone, milestone, pins_test::NsfMilestone::kAll, + "The NSF milestone to test."); + namespace pins_test { using ::p4::v1::ReadResponse;