refactor(anta.tests): Nicer result failure messages PTP, software tes…

…t module 

anta/tests/ptp.py

@@ -33,7 +33,6 @@ class VerifyPtpModeStatus(AntaTest):
     ```
     """
 
-    description = "Verifies that the device is configured as a PTP Boundary Clock."
     categories: ClassVar[list[str]] = ["ptp"]
     commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show ptp", revision=2)]
 
@@ -48,7 +47,7 @@ def test(self) -> None:
             return
 
         if ptp_mode != "ptpBoundaryClock":
-            self.result.is_failure(f"The device is not configured as a PTP Boundary Clock: '{ptp_mode}'")
+            self.result.is_failure(f"The device is not configured as a PTP Boundary Clock - Actual: {ptp_mode}")
         else:
             self.result.is_success()
 
@@ -79,7 +78,6 @@ class Input(AntaTest.Input):
         gmid: str
         """Identifier of the Grandmaster to which the device should be locked."""
 
-    description = "Verifies that the device is locked to a valid PTP Grandmaster."
     categories: ClassVar[list[str]] = ["ptp"]
     commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show ptp", revision=2)]
 
@@ -118,7 +116,6 @@ class VerifyPtpLockStatus(AntaTest):
     ```
     """
 
-    description = "Verifies that the device was locked to the upstream PTP GM in the last minute."
     categories: ClassVar[list[str]] = ["ptp"]
     commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show ptp", revision=2)]
 
@@ -136,7 +133,7 @@ def test(self) -> None:
         time_difference = ptp_clock_summary["currentPtpSystemTime"] - ptp_clock_summary["lastSyncTime"]
 
         if time_difference >= threshold:
-            self.result.is_failure(f"The device lock is more than {threshold}s old: {time_difference}s")
+            self.result.is_failure(f"The device lock is more than {threshold}s old - Actual: {time_difference}s")
         else:
             self.result.is_success()
 
@@ -158,7 +155,6 @@ class VerifyPtpOffset(AntaTest):
     ```
     """
 
-    description = "Verifies that the PTP timing offset is within +/- 1000ns from the master clock."
     categories: ClassVar[list[str]] = ["ptp"]
     commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show ptp monitor", revision=1)]
 
@@ -167,9 +163,9 @@ class VerifyPtpOffset(AntaTest):
     def test(self) -> None:
         """Main test function for VerifyPtpOffset."""
         threshold = 1000
-        offset_interfaces: dict[str, list[int]] = {}
+        self.result.is_success()
         command_output = self.instance_commands[0].json_output
-
+        offset_interfaces: dict[str, list[int]] = {}
         if not command_output["ptpMonitorData"]:
             self.result.is_skipped("PTP is not configured")
             return
@@ -178,10 +174,10 @@ def test(self) -> None:
             if abs(interface["offsetFromMaster"]) > threshold:
                 offset_interfaces.setdefault(interface["intf"], []).append(interface["offsetFromMaster"])
 
-        if offset_interfaces:
-            self.result.is_failure(f"The device timing offset from master is greater than +/- {threshold}ns: {offset_interfaces}")
-        else:
-            self.result.is_success()
+        for interface, data in offset_interfaces.items():
+            self.result.is_failure(
+                f"Interface: {interface} - The device timing offset from master is greater than +/- {threshold}ns: Actual: {', '.join(map(str, data))}"
+            )
 
 
 class VerifyPtpPortModeStatus(AntaTest):
@@ -202,7 +198,6 @@ class VerifyPtpPortModeStatus(AntaTest):
     ```
     """
 
-    description = "Verifies the PTP interfaces state."
     categories: ClassVar[list[str]] = ["ptp"]
     commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show ptp", revision=2)]
 
@@ -227,4 +222,4 @@ def test(self) -> None:
         if not invalid_interfaces:
             self.result.is_success()
         else:
-            self.result.is_failure(f"The following interface(s) are not in a valid PTP state: '{invalid_interfaces}'")
+            self.result.is_failure(f"The following interface(s) are not in a valid PTP state: {', '.join(invalid_interfaces)}")

anta/tests/software.py

@@ -34,7 +34,6 @@ class VerifyEOSVersion(AntaTest):
     ```
     """
 
-    description = "Verifies the EOS version of the device."
     categories: ClassVar[list[str]] = ["software"]
     commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show version", revision=1)]
 
@@ -48,10 +47,9 @@ class Input(AntaTest.Input):
     def test(self) -> None:
         """Main test function for VerifyEOSVersion."""
         command_output = self.instance_commands[0].json_output
-        if command_output["version"] in self.inputs.versions:
-            self.result.is_success()
-        else:
-            self.result.is_failure(f'device is running version "{command_output["version"]}" not in expected versions: {self.inputs.versions}')
+        self.result.is_success()
+        if command_output["version"] not in self.inputs.versions:
+            self.result.is_failure(f"Device version mismatch - Actual: {command_output['version']} not in Expected: {', '.join(self.inputs.versions)}")
 
 
 class VerifyTerminAttrVersion(AntaTest):
@@ -73,7 +71,6 @@ class VerifyTerminAttrVersion(AntaTest):
     ```
     """
 
-    description = "Verifies the TerminAttr version of the device."
     categories: ClassVar[list[str]] = ["software"]
     commands: ClassVar[list[AntaCommand | AntaTemplate]] = [AntaCommand(command="show version detail", revision=1)]
 
@@ -87,11 +84,10 @@ class Input(AntaTest.Input):
     def test(self) -> None:
         """Main test function for VerifyTerminAttrVersion."""
         command_output = self.instance_commands[0].json_output
+        self.result.is_success()
         command_output_data = command_output["details"]["packages"]["TerminAttr-core"]["version"]
-        if command_output_data in self.inputs.versions:
-            self.result.is_success()
-        else:
-            self.result.is_failure(f"device is running TerminAttr version {command_output_data} and is not in the allowed list: {self.inputs.versions}")
+        if command_output_data not in self.inputs.versions:
+            self.result.is_failure(f"Device TerminAttr version mismatch - Actual: {command_output_data} not in Expected: {', '.join(self.inputs.versions)}")
 
 
 class VerifyEOSExtensions(AntaTest):
@@ -120,6 +116,7 @@ class VerifyEOSExtensions(AntaTest):
     def test(self) -> None:
         """Main test function for VerifyEOSExtensions."""
         boot_extensions = []
+        self.result.is_success()
         show_extensions_command_output = self.instance_commands[0].json_output
         show_boot_extensions_command_output = self.instance_commands[1].json_output
         installed_extensions = [
@@ -131,7 +128,7 @@ def test(self) -> None:
                 boot_extensions.append(formatted_extension)
         installed_extensions.sort()
         boot_extensions.sort()
-        if installed_extensions == boot_extensions:
-            self.result.is_success()
-        else:
-            self.result.is_failure(f"Missing EOS extensions: installed {installed_extensions} / configured: {boot_extensions}")
+        if installed_extensions != boot_extensions:
+            str_installed_extensions = ", ".join(installed_extensions) if installed_extensions else "Not found"
+            str_boot_extensions = ", ".join(boot_extensions) if boot_extensions else "Not found"
+            self.result.is_failure(f"Installed and Boot extensions mismatch - Installed: {str_installed_extensions}, Configured: {str_boot_extensions}")

examples/tests.yaml

@@ -365,16 +365,16 @@ anta.tests.profiles:
       mode: 3
 anta.tests.ptp:
   - VerifyPtpGMStatus:
-      # Verifies that the device is locked to a valid PTP Grandmaster.
+      # Verifies that the device is locked to a valid Precision Time Protocol (PTP) Grandmaster (GM).
       gmid: 0xec:46:70:ff:fe:00:ff:a9
   - VerifyPtpLockStatus:
-      # Verifies that the device was locked to the upstream PTP GM in the last minute.
+      # Verifies that the device was locked to the upstream Precision Time Protocol (PTP) Grandmaster (GM) in the last minute.
   - VerifyPtpModeStatus:
-      # Verifies that the device is configured as a PTP Boundary Clock.
+      # Verifies that the device is configured as a Precision Time Protocol (PTP) Boundary Clock (BC).
   - VerifyPtpOffset:
-      # Verifies that the PTP timing offset is within +/- 1000ns from the master clock.
+      # Verifies that the Precision Time Protocol (PTP) timing offset is within +/- 1000ns from the master clock.
   - VerifyPtpPortModeStatus:
-      # Verifies the PTP interfaces state.
+      # Verifies that all interfaces are in a valid Precision Time Protocol (PTP) state.
 anta.tests.routing.bgp:
   - VerifyBGPAdvCommunities:
       # Verifies that advertised communities are standard, extended and large for BGP IPv4 peer(s).
@@ -860,12 +860,12 @@ anta.tests.software:
   - VerifyEOSExtensions:
       # Verifies that all EOS extensions installed on the device are enabled for boot persistence.
   - VerifyEOSVersion:
-      # Verifies the EOS version of the device.
+      # Verifies that the device is running one of the allowed EOS version.
       versions:
         - 4.25.4M
         - 4.26.1F
   - VerifyTerminAttrVersion:
-      # Verifies the TerminAttr version of the device.
+      # Verifies that he device is running one of the allowed TerminAttr version.
       versions:
         - v1.13.6
         - v1.8.0

tests/units/anta_tests/test_ptp.py

@@ -39,7 +39,7 @@
         "test": VerifyPtpModeStatus,
         "eos_data": [{"ptpMode": "ptpDisabled", "ptpIntfSummaries": {}}],
         "inputs": None,
-        "expected": {"result": "failure", "messages": ["The device is not configured as a PTP Boundary Clock: 'ptpDisabled'"]},
+        "expected": {"result": "failure", "messages": ["The device is not configured as a PTP Boundary Clock - Actual: ptpDisabled"]},
     },
     {
         "name": "skipped",
@@ -158,7 +158,7 @@
             }
         ],
         "inputs": None,
-        "expected": {"result": "failure", "messages": ["The device lock is more than 60s old: 157s"]},
+        "expected": {"result": "failure", "messages": ["The device lock is more than 60s old - Actual: 157s"]},
     },
     {
         "name": "skipped",
@@ -236,7 +236,9 @@
         "inputs": None,
         "expected": {
             "result": "failure",
-            "messages": [("The device timing offset from master is greater than +/- 1000ns: {'Ethernet27/1': [1200, -1300]}")],
+            "messages": [
+                "Interface: Ethernet27/1 - The device timing offset from master is greater than +/- 1000ns: Actual: 1200, -1300",
+            ],
         },
     },
     {
@@ -335,6 +337,6 @@
             }
         ],
         "inputs": None,
-        "expected": {"result": "failure", "messages": ["The following interface(s) are not in a valid PTP state: '['Ethernet53', 'Ethernet1']'"]},
+        "expected": {"result": "failure", "messages": ["The following interface(s) are not in a valid PTP state: Ethernet53, Ethernet1"]},
     },
 ]

tests/units/anta_tests/test_software.py

@@ -35,7 +35,7 @@
             },
         ],
         "inputs": {"versions": ["4.27.1F"]},
-        "expected": {"result": "failure", "messages": ["device is running version \"4.27.0F\" not in expected versions: ['4.27.1F']"]},
+        "expected": {"result": "failure", "messages": ["Device version mismatch - Actual: 4.27.0F not in Expected: 4.27.1F"]},
     },
     {
         "name": "success",
@@ -77,7 +77,7 @@
             },
         ],
         "inputs": {"versions": ["v1.17.1", "v1.18.1"]},
-        "expected": {"result": "failure", "messages": ["device is running TerminAttr version v1.17.0 and is not in the allowed list: ['v1.17.1', 'v1.18.1']"]},
+        "expected": {"result": "failure", "messages": ["Device TerminAttr version mismatch - Actual: v1.17.0 not in Expected: v1.17.1, v1.18.1"]},
     },
     # TODO: add a test with a real extension?
     {
@@ -108,6 +108,6 @@
             {"extensions": ["dummy"]},
         ],
         "inputs": None,
-        "expected": {"result": "failure", "messages": ["Missing EOS extensions: installed [] / configured: ['dummy']"]},
+        "expected": {"result": "failure", "messages": ["Installed and Boot extensions mismatch - Installed: Not found, Configured: dummy"]},
     },
 ]