Add a check in reboot test to make sure PORT_INDEX_TABLE is populated before sending traffic.Enhance ValidateSutState to ValidateTestbedState by additionally checking Control Switch for MirrorTestbed.

Author: kishanps

tests/sflow/BUILD.bazel

@@ -90,6 +90,7 @@ cc_library(
         "//lib/validator:validator_lib",
         "//p4_pdpi/netaddr:ipv4_address",
         "//p4_pdpi/netaddr:ipv6_address",
+        "//thinkit:ssh_client",
         "@com_github_gnmi//proto/gnmi:gnmi_cc_grpc_proto",
         "@com_github_google_glog//:glog",
         "@com_github_nlohmann_json//:nlohmann_json",
@@ -100,6 +101,7 @@ cc_library(
         "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/time",
+        "@com_google_absl//absl/types:span",
         "@com_googlesource_code_re2//:re2",
     ],
 )
@@ -140,8 +142,7 @@ cc_test(
         ":sflow_util",
         "//gutil:status_matchers",
         "//gutil:testing",
-        "//p4_pdpi/packetlib",
-        "//p4_pdpi/packetlib:packetlib_cc_proto",
+        "//thinkit:mock_ssh_client",
         "@com_github_gnmi//proto/gnmi:gnmi_cc_grpc_proto",
         "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/status",

tests/sflow/sflow_test.cc

@@ -1223,7 +1223,6 @@ void SflowTestFixture::TearDown() {
 // 2. Collect sFlow samples via sflowtool on SUT.
 // 3. Validate the result is as expected.
 TEST_P(SflowTestFixture, VerifyIngressSamplingForNoMatchPackets) {
-  testbed_->Environment().SetTestCaseID("6c980337-c187-4f19-99a9-e03b6dcf24b5");
 
   const IxiaLink& ingress_link = ready_links_[0];
   Port ingress_port = Port{
@@ -1293,7 +1292,6 @@ TEST_P(SflowTestFixture, VerifyIngressSamplingForNoMatchPackets) {
 
 // Verifies ingress sampling could work when forwarding traffic.
 TEST_P(SflowTestFixture, VerifyIngressSamplingForForwardedPackets) {
-  testbed_->Environment().SetTestCaseID("909e2260-d4ca-4018-8c6a-2bd7891ee686");
 
   const IxiaLink& ingress_link = ready_links_[0];
   Port ingress_port = Port{
@@ -1384,7 +1382,6 @@ TEST_P(SflowTestFixture, VerifyIngressSamplingForForwardedPackets) {
 
 // Verifies ingress sampling could work when dropping packets.
 TEST_P(SflowTestFixture, VerifyIngressSamplesForDropPackets) {
-  testbed_->Environment().SetTestCaseID("6d567c91-d018-4fb3-85fc-c75c5f425d15");
 
   const IxiaLink& ingress_link = ready_links_[0];
   Port ingress_port = Port{
@@ -1460,7 +1457,6 @@ TEST_P(SflowTestFixture, VerifyIngressSamplesForDropPackets) {
 // 3. Send traffic from Ixia.
 // 4. Validate the packets are all get punted and sFlowtool has expected result.
 TEST_P(SflowTestFixture, VerifyIngressSamplesForP4rtPuntTraffic) {
-  testbed_->Environment().SetTestCaseID("70a4e3e1-f3ae-416e-99a7-24f259942981");
 
   const IxiaLink& ingress_link = ready_links_[0];
   Port ingress_port = Port{
@@ -1590,7 +1586,6 @@ TEST_P(SflowTestFixture, VerifyIngressSamplesForP4rtPuntTraffic) {
 // Traffic packet size size_a, sFlow sampling size size_b: expects sample header
 // size equals to min(size_a, size_b).
 TEST_P(SampleSizeTest, VerifySamplingSizeWorks) {
-  testbed_->Environment().SetTestCaseID("d1e8ae2a-0595-4b33-9743-7dbff8eaf48a");
 
   const int packet_size = GetParam().packet_size,
             sample_size = GetParam().sample_size;
@@ -1651,7 +1646,6 @@ TEST_P(SampleSizeTest, VerifySamplingSizeWorks) {
 // send traffic to two interfaces with different sampling rate and verifies
 // samples count.
 TEST_P(SampleRateTest, VerifySamplingRateWorks) {
-  testbed_->Environment().SetTestCaseID("a143121e-c90f-4892-bd6b-897c3d9f9ff3");
 
   const IxiaLink& ingress_link = ready_links_[0];
   const int sample_rate = GetParam().sample_rate;
@@ -1735,7 +1729,6 @@ TEST_P(SampleRateTest, VerifySamplingRateWorks) {
 // 3. Send traffic at a normal rate which would not trigger sFlow backoff.
 // Verify that sample rate is still doubled on all interfaces and samples.
 TEST_P(BackoffTest, VerifyBackoffWorks) {
-  testbed_->Environment().SetTestCaseID("58653895-219a-4b9e-b6af-03f6a33b5959");
 
   const IxiaLink& ingress_link = ready_links_[0];
 
@@ -2470,6 +2463,11 @@ TEST_P(SflowRebootTestFixture, ChangeCollectorConfigOnNsfReboot) {
   pins_test::Testbed testbed_variant;
   testbed_variant.emplace<thinkit::MirrorTestbed*>(&testbed);
   ASSERT_OK(pins_test::NsfReboot(testbed_variant));
+  ASSERT_OK(
+      pins_test::WaitForNsfReboot(testbed_variant, *GetParam().ssh_client));
+  ASSERT_OK(pins_test::ValidateTestbedState(
+      /*version=*/"", testbed_variant, *GetParam().ssh_client,
+      /*gnmi_config=*/std::nullopt));
   // Wait until all sFlow gNMI states are converged.
   ASSERT_OK(pins_test::WaitForCondition(
       VerifySflowStatesConverged, absl::Seconds(60), sut_gnmi_stub_.get(),
@@ -2628,7 +2626,6 @@ TEST_P(SflowRebootTestFixture, ChangeCollectorConfigOnNsfReboot) {
 TEST_P(SflowMirrorTestFixture, TestInbandPathToSflowCollector) {
   thinkit::MirrorTestbed& testbed =
       GetParam().testbed_interface->GetMirrorTestbed();
-  testbed.Environment().SetTestCaseID("3948ef62-67d9-42e2-8cc6-30dc10cb382f");
 
   const int packets_num = 10000;
 
@@ -2812,7 +2809,6 @@ TEST_P(SflowMirrorTestFixture, TestInbandPathToSflowCollector) {
 TEST_P(SflowMirrorTestFixture, TestSflowDscpValue) {
   thinkit::MirrorTestbed& testbed =
       GetParam().testbed_interface->GetMirrorTestbed();
-  testbed.Environment().SetTestCaseID("38694bb8-2a82-4379-8b85-96dcda824a14");
 
   const int packets_num = 10000;
 
@@ -2919,7 +2915,6 @@ TEST_P(SflowMirrorTestFixture, TestSflowDscpValue) {
 TEST_P(SflowMirrorTestFixture, TestSamplingWorksOnAllInterfaces) {
   thinkit::MirrorTestbed& testbed =
       GetParam().testbed_interface->GetMirrorTestbed();
-  testbed.Environment().SetTestCaseID("03bdb9da-387a-4256-9323-60d4503ce3c0");
 
   const int packets_num = 10000;
   ASSERT_OK_AND_ASSIGN(
@@ -3039,7 +3034,6 @@ TEST_P(SflowRebootTestFixture, TestSamplingWorksAfterReboot) {
 
   thinkit::MirrorTestbed& testbed =
       GetParam().testbed_interface->GetMirrorTestbed();
-  testbed.Environment().SetTestCaseID("6cd622a1-2d95-4e07-9c7d-77a2faad4bc0");
 
   ASSERT_OK_AND_ASSIGN(
       auto port_id_per_port_name,
@@ -3192,6 +3186,9 @@ TEST_P(SflowRebootTestFixture, TestSamplingWorksAfterReboot) {
       VerifySflowStatesConverged, absl::Minutes(2), sut_gnmi_stub_.get(),
       agent_address_, kInbandSamplingRate, kSampleHeaderSize,
       collector_address_and_port, sflow_enabled_interfaces));
+  ASSERT_OK(pins_test::WaitForCondition(
+      CheckStateDbPortIndexTableExists, absl::Minutes(2),
+      *GetParam().ssh_client, testbed.Sut().ChassisName(), interface_names));
 
   for (const auto& [interface_name, unused] : sflow_enabled_interfaces) {
     ASSERT_OK_AND_ASSIGN(packets_sampled_per_interface[interface_name],
@@ -3282,7 +3279,6 @@ TEST_P(SflowRebootTestFixture, TestSamplingWorksAfterReboot) {
 TEST_P(SflowMirrorTestFixture, TestIp2MePacketsAreSampledAndPunted) {
   thinkit::MirrorTestbed& testbed =
       GetParam().testbed_interface->GetMirrorTestbed();
-  testbed.Environment().SetTestCaseID("7f765661-840e-4913-b210-8bbf9d5a0e8f");
 
   const absl::string_view kSrcIp6Address = "2001:db8:0:12::1";
   const int packets_num = 3000;

tests/sflow/sflow_util.cc

@@ -16,17 +16,22 @@
 
 #include <cstdint>
 #include <string>
+#include <vector>
 
 #include "absl/container/btree_map.h"
 #include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/ascii.h"
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_cat.h"
+#include "absl/strings/str_join.h"
+#include "absl/strings/str_split.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
 #include "absl/time/time.h"
+#include "absl/types/span.h"
 #include "glog/logging.h"
 #include "gutil/status.h"
 #include "include/nlohmann/json.hpp"
@@ -36,6 +41,7 @@
 #include "p4_pdpi/netaddr/ipv4_address.h"
 #include "p4_pdpi/netaddr/ipv6_address.h"
 #include "re2/re2.h"
+#include "thinkit/ssh_client.h"
 
 namespace pins {
 namespace {
@@ -568,4 +574,38 @@ absl::StatusOr<int> GetSflowCollectorPort(absl::string_view gnmi_config) {
   return collector_json["config"]["port"];
 }
 
+absl::Status CheckStateDbPortIndexTableExists(
+    thinkit::SSHClient& ssh_client, absl::string_view device_name,
+    absl::Span<const std::string> interfaces) {
+  ASSIGN_OR_RETURN(
+      std::string ssh_result,
+      ssh_client.RunCommand(
+          device_name,
+          /*command=*/"/usr/tools/bin/redis-cli -n 6 keys PORT_INDEX_TABLE*",
+          /*timeout=*/absl::Seconds(5)));
+  LOG(INFO) << "ssh_result: " << ssh_result;
+  absl::flat_hash_set<std::string> port_index_table_interfaces;
+  for (absl::string_view interface_str :
+       absl::StrSplit(ssh_result, "PORT_INDEX_TABLE")) {
+    interface_str = absl::StripAsciiWhitespace(interface_str);
+    auto pos = interface_str.find("Ethernet");
+    if (pos == std::string::npos) {
+      continue;
+    }
+    port_index_table_interfaces.insert(std::string(interface_str.substr(pos)));
+  }
+  std::vector<std::string> missing_interfaces;
+  for (const auto& interface : interfaces) {
+    if (!port_index_table_interfaces.contains(interface)) {
+      missing_interfaces.push_back(interface);
+    }
+  }
+  if (missing_interfaces.empty()) {
+    return absl::OkStatus();
+  }
+  return absl::FailedPreconditionError(
+      absl::StrCat("Failed to find PORT_INDEX_TABLE for interface ",
+                   absl::StrJoin(missing_interfaces, ", ")));
+}
+
 }  // namespace pins

tests/sflow/sflow_util.h

@@ -24,7 +24,9 @@
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/string_view.h"
+#include "absl/types/span.h"
 #include "proto/gnmi/gnmi.grpc.pb.h"
+#include "thinkit/ssh_client.h"
 
 namespace pins {
 
@@ -142,5 +144,10 @@ absl::StatusOr<bool> IsSameIpAddressStr(const std::string& ip1,
 // Returns the collector port from `gnmi_config` collector config first
 absl::StatusOr<int> GetSflowCollectorPort(absl::string_view gnmi_config);
 
+// Checks if state db port index table exists for `interfaces`.
+absl::Status CheckStateDbPortIndexTableExists(
+    thinkit::SSHClient& ssh_client, absl::string_view device_name,
+    absl::Span<const std::string> interfaces);
+
 }  // namespace pins
 #endif  // PINS_TESTS_SFLOW_SFLOW_UTIL_H_

tests/sflow/sflow_util_test.cc

@@ -14,6 +14,9 @@
 
 #include "tests/sflow/sflow_util.h"
 
+#include <string>
+#include <vector>
+
 #include "absl/container/flat_hash_set.h"
 #include "absl/status/status.h"
 #include "absl/strings/string_view.h"
@@ -23,11 +26,13 @@
 #include "gutil/status_matchers.h"
 #include "gutil/testing.h"
 #include "proto/gnmi/gnmi_mock.grpc.pb.h"
+#include "thinkit/mock_ssh_client.h"
 
 namespace pins {
 namespace {
 using ::gutil::IsOkAndHolds;
 using ::gutil::StatusIs;
+using ::testing::AllOf;
 using ::testing::DoAll;
 using ::testing::Eq;
 using ::testing::HasSubstr;
@@ -1080,5 +1085,51 @@ TEST(IpAddressTest, SameIpv6AddressDifferentFormat4) {
               IsOkAndHolds(true));
 }
 
+TEST(PortIndexTest, SshCommandFailCheckFail) {
+  thinkit::MockSSHClient mock_ssh_client;
+  ON_CALL(mock_ssh_client, RunCommand)
+      .WillByDefault(Return(absl::InternalError("No ssh client")));
+  const std::vector<std::string> port_names = {"Ethernet1/1/1",
+                                               "Ethernet1/1/2"};
+  EXPECT_THAT(
+      CheckStateDbPortIndexTableExists(mock_ssh_client, "switch_x", port_names),
+      StatusIs(absl::StatusCode::kInternal));
+}
+
+TEST(PortIndexTest, SshCommandNoNilCheckSuccess) {
+  thinkit::MockSSHClient mock_ssh_client;
+  ON_CALL(mock_ssh_client, RunCommand)
+      .WillByDefault(Return(R"(PORT_INDEX_TABLE|Ethernet1/10/1
+PORT_INDEX_TABLE|Ethernet1/9/1
+PORT_INDEX_TABLE|Ethernet1/31/1
+PORT_INDEX_TABLE|Ethernet1/2/5
+PORT_INDEX_TABLE|Ethernet1/13/1)"));
+  const std::vector<std::string> port_names = {
+      "Ethernet1/10/1", "Ethernet1/31/1", "Ethernet1/13/1"};
+  EXPECT_OK(CheckStateDbPortIndexTableExists(mock_ssh_client, "switch_x",
+                                             port_names));
+}
+
+TEST(PortIndexTest, SshCommandNilCheckFail) {
+  thinkit::MockSSHClient mock_ssh_client;
+  ON_CALL(mock_ssh_client, RunCommand).WillByDefault(Return("nil"));
+  const std::vector<std::string> port_names = {"Ethernet1/1/1",
+                                               "Ethernet1/1/2"};
+  EXPECT_THAT(
+      CheckStateDbPortIndexTableExists(mock_ssh_client, "switch_x", port_names),
+      StatusIs(absl::StatusCode::kFailedPrecondition));
+}
+
+TEST(PortIndexTest, SshCommandEmptyCheckFail) {
+  thinkit::MockSSHClient mock_ssh_client;
+  ON_CALL(mock_ssh_client, RunCommand).WillByDefault(Return(""));
+  const std::vector<std::string> port_names = {"Ethernet1/1/1",
+                                               "Ethernet1/1/2"};
+  EXPECT_THAT(
+      CheckStateDbPortIndexTableExists(mock_ssh_client, "switch_x", port_names),
+      StatusIs(absl::StatusCode::kFailedPrecondition,
+               AllOf(HasSubstr("Ethernet1/1/1"), HasSubstr("Ethernet1/1/2"))));
+}
+
 }  // namespace
 }  // namespace pins