fix(anta): Fix various issues in CSV and Markdown reporters (#990)

* fix(anta): Fix various issues in CSV and Markdown reporters

* Add sorting in unit test

* Fix per review comments

anta/cli/nrfu/utils.py

@@ -157,7 +157,7 @@ def save_markdown_report(ctx: click.Context, md_output: pathlib.Path) -> None:
         Path to save the markdown report.
     """
     try:
-        MDReportGenerator.generate(results=_get_result_manager(ctx), md_filename=md_output)
+        MDReportGenerator.generate(results=_get_result_manager(ctx).sort(["name", "categories", "test"]), md_filename=md_output)
         console.print(f"Markdown report saved to {md_output} ✅", style="cyan")
     except OSError:
         console.print(f"Failed to save Markdown report to {md_output} ❌", style="cyan")

anta/constants.py

@@ -5,7 +5,26 @@
 
 from __future__ import annotations
 
-ACRONYM_CATEGORIES: set[str] = {"aaa", "mlag", "snmp", "bgp", "ospf", "vxlan", "stp", "igmp", "ip", "lldp", "ntp", "bfd", "ptp", "lanz", "stun", "vlan"}
+ACRONYM_CATEGORIES: set[str] = {
+    "aaa",
+    "avt",
+    "bfd",
+    "bgp",
+    "igmp",
+    "ip",
+    "isis",
+    "lanz",
+    "lldp",
+    "mlag",
+    "ntp",
+    "ospf",
+    "ptp",
+    "snmp",
+    "stp",
+    "stun",
+    "vlan",
+    "vxlan",
+}
 """A set of network protocol or feature acronyms that should be represented in uppercase."""
 
 MD_REPORT_TOC = """**Table of Contents:**

anta/reporter/__init__.py

@@ -168,7 +168,7 @@ def report_summary_tests(
             self.Headers.list_of_error_nodes,
         ]
         table = self._build_headers(headers=headers, table=table)
-        for test, stats in sorted(manager.test_stats.items()):
+        for test, stats in manager.test_stats.items():
             if tests is None or test in tests:
                 table.add_row(
                     test,
@@ -214,7 +214,7 @@ def report_summary_devices(
             self.Headers.list_of_error_tests,
         ]
         table = self._build_headers(headers=headers, table=table)
-        for device, stats in sorted(manager.device_stats.items()):
+        for device, stats in manager.device_stats.items():
             if devices is None or device in devices:
                 table.add_row(
                     device,

anta/reporter/csv_reporter.py

@@ -8,6 +8,7 @@
 
 import csv
 import logging
+import os
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
@@ -111,6 +112,7 @@ def generate(cls, results: ResultManager, csv_filename: pathlib.Path) -> None:
                 csvwriter = csv.writer(
                     csvfile,
                     delimiter=",",
+                    lineterminator=os.linesep,
                 )
                 csvwriter.writerow(headers)
                 for entry in results.results:

anta/reporter/md_reporter.py

@@ -237,7 +237,7 @@ class SummaryTotalsDeviceUnderTest(MDReportBase):
     def generate_rows(self) -> Generator[str, None, None]:
         """Generate the rows of the summary totals device under test table."""
         for device, stat in self.results.device_stats.items():
-            total_tests = stat.tests_success_count + stat.tests_skipped_count + stat.tests_failure_count + stat.tests_error_count
+            total_tests = stat.tests_success_count + stat.tests_skipped_count + stat.tests_failure_count + stat.tests_error_count + stat.tests_unset_count
             categories_skipped = ", ".join(sorted(convert_categories(list(stat.categories_skipped))))
             categories_failed = ", ".join(sorted(convert_categories(list(stat.categories_failed))))
             yield (
@@ -261,10 +261,11 @@ class SummaryTotalsPerCategory(MDReportBase):
 
     def generate_rows(self) -> Generator[str, None, None]:
         """Generate the rows of the summary totals per category table."""
-        for category, stat in self.results.sorted_category_stats.items():
-            total_tests = stat.tests_success_count + stat.tests_skipped_count + stat.tests_failure_count + stat.tests_error_count
+        for category, stat in self.results.category_stats.items():
+            converted_category = convert_categories([category])[0]
+            total_tests = stat.tests_success_count + stat.tests_skipped_count + stat.tests_failure_count + stat.tests_error_count + stat.tests_unset_count
             yield (
-                f"| {category} | {total_tests} | {stat.tests_success_count} | {stat.tests_skipped_count} | {stat.tests_failure_count} "
+                f"| {converted_category} | {total_tests} | {stat.tests_success_count} | {stat.tests_skipped_count} | {stat.tests_failure_count} "
                 f"| {stat.tests_error_count} |\n"
             )
 
@@ -284,9 +285,9 @@ class TestResults(MDReportBase):
 
     def generate_rows(self) -> Generator[str, None, None]:
         """Generate the rows of the all test results table."""
-        for result in self.results.get_results(sort_by=["name", "test"]):
+        for result in self.results.results:
             messages = self.safe_markdown(", ".join(result.messages))
-            categories = ", ".join(convert_categories(result.categories))
+            categories = ", ".join(sorted(convert_categories(result.categories)))
             yield (
                 f"| {result.name or '-'} | {categories or '-'} | {result.test or '-'} "
                 f"| {result.description or '-'} | {self.safe_markdown(result.custom_field) or '-'} | {result.result or '-'} | {messages or '-'} |\n"

anta/result_manager/__init__.py

@@ -7,6 +7,7 @@
 
 import json
 import logging
+import warnings
 from collections import defaultdict
 from functools import cached_property
 from itertools import chain
@@ -143,28 +144,41 @@ def json(self) -> str:
         return json.dumps(self.dump, indent=4)
 
     @property
-    def device_stats(self) -> defaultdict[str, DeviceStats]:
+    def device_stats(self) -> dict[str, DeviceStats]:
         """Get the device statistics."""
         self._ensure_stats_in_sync()
-        return self._device_stats
+        return dict(sorted(self._device_stats.items()))
 
     @property
-    def category_stats(self) -> defaultdict[str, CategoryStats]:
+    def category_stats(self) -> dict[str, CategoryStats]:
         """Get the category statistics."""
         self._ensure_stats_in_sync()
-        return self._category_stats
+        return dict(sorted(self._category_stats.items()))
 
     @property
-    def test_stats(self) -> defaultdict[str, TestStats]:
+    def test_stats(self) -> dict[str, TestStats]:
         """Get the test statistics."""
         self._ensure_stats_in_sync()
-        return self._test_stats
+        return dict(sorted(self._test_stats.items()))
 
     @property
     def sorted_category_stats(self) -> dict[str, CategoryStats]:
-        """A property that returns the category_stats dictionary sorted by key name."""
+        """A property that returns the category_stats dictionary sorted by key name.
+
+        Deprecated
+        ----------
+            This property is deprecated and will be removed in ANTA v2.0.0.
+            Use `category_stats` instead as it is now sorted by default.
+
+        TODO: Remove this property in ANTA v2.0.0.
+        """
+        warnings.warn(
+            "sorted_category_stats is deprecated and will be removed in ANTA v2.0.0. Use category_stats instead as it is now sorted by default.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         self._ensure_stats_in_sync()
-        return dict(sorted(self.category_stats.items()))
+        return self.category_stats
 
     @cached_property
     def results_by_status(self) -> dict[AntaTestStatus, list[TestResult]]:
@@ -316,6 +330,21 @@ def get_status(self, *, ignore_error: bool = False) -> str:
         """Return the current status including error_status if ignore_error is False."""
         return "error" if self.error_status and not ignore_error else self.status
 
+    def sort(self, sort_by: list[str]) -> ResultManager:
+        """Sort the ResultManager results based on TestResult fields.
+
+        Parameters
+        ----------
+        sort_by
+            List of TestResult fields to sort the results.
+        """
+        accepted_fields = TestResult.model_fields.keys()
+        if not set(sort_by).issubset(set(accepted_fields)):
+            msg = f"Invalid sort_by fields: {sort_by}. Accepted fields are: {list(accepted_fields)}"
+            raise ValueError(msg)
+        self._result_entries.sort(key=lambda result: [getattr(result, field) for field in sort_by])
+        return self
+
     def filter(self, hide: set[AntaTestStatus]) -> ResultManager:
         """Get a filtered ResultManager based on test status.
 

tests/data/test_md_report.md

@@ -21,8 +21,8 @@
 
 | Device Under Test | Total Tests | Tests Success | Tests Skipped | Tests Failure | Tests Error | Categories Skipped | Categories Failed |
 | ------------------| ----------- | ------------- | ------------- | ------------- | ----------- | -------------------| ------------------|
-| DC1-SPINE1 | 15 | 2 | 2 | 10 | 1 | MLAG, VXLAN | AAA, BFD, BGP, Connectivity, Routing, SNMP, STP, Services, Software, System |
 | DC1-LEAF1A | 15 | 5 | 0 | 9 | 1 | - | AAA, BFD, BGP, Connectivity, SNMP, STP, Services, Software, System |
+| DC1-SPINE1 | 15 | 2 | 2 | 10 | 1 | MLAG, VXLAN | AAA, BFD, BGP, Connectivity, Routing, SNMP, STP, Services, Software, System |
 
 ### Summary Totals Per Category
 
@@ -47,33 +47,33 @@
 
 | Device Under Test | Categories | Test | Description | Custom Field | Result | Messages |
 | ----------------- | ---------- | ---- | ----------- | ------------ | ------ | -------- |
+| DC1-LEAF1A | AAA | VerifyTacacsSourceIntf | Verifies TACACS source-interface for a specified VRF. | - | failure | Source-interface Management0 is not configured in VRF default |
 | DC1-LEAF1A | BFD | VerifyBFDSpecificPeers | Verifies the IPv4 BFD peer's sessions and remote disc in the specified VRF. | - | failure | Following BFD peers are not configured, status is not up or remote disc is zero: {'192.0.255.8': {'default': 'Not Configured'}, '192.0.255.7': {'default': 'Not Configured'}} |
 | DC1-LEAF1A | BGP | VerifyBGPPeerCount | Verifies the count of BGP peers. | - | failure | Failures: [{'afi': 'ipv4', 'safi': 'unicast', 'vrfs': {'PROD': 'Expected: 2, Actual: 1'}}, {'afi': 'ipv4', 'safi': 'multicast', 'vrfs': {'DEV': 'Expected: 3, Actual: 0'}}] |
-| DC1-LEAF1A | Software | VerifyEOSVersion | Verifies the EOS version of the device. | - | failure | device is running version "4.31.1F-34554157.4311F (engineering build)" not in expected versions: ['4.25.4M', '4.26.1F'] |
-| DC1-LEAF1A | Services | VerifyHostname | Verifies the hostname of a device. | - | failure | Expected 's1-spine1' as the hostname, but found 'DC1-LEAF1A' instead. |
-| DC1-LEAF1A | Interfaces | VerifyInterfaceUtilization | Verifies that the utilization of interfaces is below a certain threshold. | - | success | - |
 | DC1-LEAF1A | Connectivity | VerifyLLDPNeighbors | Verifies that the provided LLDP neighbors are connected properly. | - | failure | Wrong LLDP neighbor(s) on port(s):    Ethernet1       DC1-SPINE1_Ethernet1    Ethernet2       DC1-SPINE2_Ethernet1 Port(s) not configured:    Ethernet7 |
-| DC1-LEAF1A | MLAG | VerifyMlagStatus | Verifies the health status of the MLAG configuration. | - | success | - |
-| DC1-LEAF1A | System | VerifyNTP | Verifies if NTP is synchronised. | - | failure | The device is not synchronized with the configured NTP server(s): 'NTP is disabled.' |
 | DC1-LEAF1A | Connectivity | VerifyReachability | Test the network reachability to one or many destination IP(s). | - | error | ping vrf MGMT 1.1.1.1 source Management1 repeat 2 has failed: No source interface Management1 |
+| DC1-LEAF1A | Interfaces | VerifyInterfaceUtilization | Verifies that the utilization of interfaces is below a certain threshold. | - | success | - |
+| DC1-LEAF1A | MLAG | VerifyMlagStatus | Verifies the health status of the MLAG configuration. | - | success | - |
 | DC1-LEAF1A | Routing | VerifyRoutingTableEntry | Verifies that the provided routes are present in the routing table of a specified VRF. | - | success | - |
-| DC1-LEAF1A | STP | VerifySTPMode | Verifies the configured STP mode for a provided list of VLAN(s). | - | failure | Wrong STP mode configured for the following VLAN(s): [10, 20] |
 | DC1-LEAF1A | SNMP | VerifySnmpStatus | Verifies if the SNMP agent is enabled. | - | failure | SNMP agent disabled in vrf default |
-| DC1-LEAF1A | AAA | VerifyTacacsSourceIntf | Verifies TACACS source-interface for a specified VRF. | - | failure | Source-interface Management0 is not configured in VRF default |
+| DC1-LEAF1A | STP | VerifySTPMode | Verifies the configured STP mode for a provided list of VLAN(s). | - | failure | Wrong STP mode configured for the following VLAN(s): [10, 20] |
 | DC1-LEAF1A | Security | VerifyTelnetStatus | Verifies if Telnet is disabled in the default VRF. | - | success | - |
+| DC1-LEAF1A | Services | VerifyHostname | Verifies the hostname of a device. | - | failure | Expected 's1-spine1' as the hostname, but found 'DC1-LEAF1A' instead. |
+| DC1-LEAF1A | Software | VerifyEOSVersion | Verifies the EOS version of the device. | - | failure | device is running version "4.31.1F-34554157.4311F (engineering build)" not in expected versions: ['4.25.4M', '4.26.1F'] |
+| DC1-LEAF1A | System | VerifyNTP | Verifies if NTP is synchronised. | - | failure | The device is not synchronized with the configured NTP server(s): 'NTP is disabled.' |
 | DC1-LEAF1A | VXLAN | VerifyVxlan1Interface | Verifies the Vxlan1 interface status. | - | success | - |
+| DC1-SPINE1 | AAA | VerifyTacacsSourceIntf | Verifies TACACS source-interface for a specified VRF. | - | failure | Source-interface Management0 is not configured in VRF default |
 | DC1-SPINE1 | BFD | VerifyBFDSpecificPeers | Verifies the IPv4 BFD peer's sessions and remote disc in the specified VRF. | - | failure | Following BFD peers are not configured, status is not up or remote disc is zero: {'192.0.255.8': {'default': 'Not Configured'}, '192.0.255.7': {'default': 'Not Configured'}} |
 | DC1-SPINE1 | BGP | VerifyBGPPeerCount | Verifies the count of BGP peers. | - | failure | Failures: [{'afi': 'ipv4', 'safi': 'unicast', 'vrfs': {'PROD': 'Not Configured', 'default': 'Expected: 3, Actual: 4'}}, {'afi': 'ipv4', 'safi': 'multicast', 'vrfs': {'DEV': 'Not Configured'}}, {'afi': 'evpn', 'vrfs': {'default': 'Expected: 2, Actual: 4'}}] |
-| DC1-SPINE1 | Software | VerifyEOSVersion | Verifies the EOS version of the device. | - | failure | device is running version "4.31.1F-34554157.4311F (engineering build)" not in expected versions: ['4.25.4M', '4.26.1F'] |
-| DC1-SPINE1 | Services | VerifyHostname | Verifies the hostname of a device. | - | failure | Expected 's1-spine1' as the hostname, but found 'DC1-SPINE1' instead. |
-| DC1-SPINE1 | Interfaces | VerifyInterfaceUtilization | Verifies that the utilization of interfaces is below a certain threshold. | - | success | - |
 | DC1-SPINE1 | Connectivity | VerifyLLDPNeighbors | Verifies that the provided LLDP neighbors are connected properly. | - | failure | Wrong LLDP neighbor(s) on port(s):    Ethernet1       DC1-LEAF1A_Ethernet1    Ethernet2       DC1-LEAF1B_Ethernet1 Port(s) not configured:    Ethernet7 |
-| DC1-SPINE1 | MLAG | VerifyMlagStatus | Verifies the health status of the MLAG configuration. | - | skipped | MLAG is disabled |
-| DC1-SPINE1 | System | VerifyNTP | Verifies if NTP is synchronised. | - | failure | The device is not synchronized with the configured NTP server(s): 'NTP is disabled.' |
 | DC1-SPINE1 | Connectivity | VerifyReachability | Test the network reachability to one or many destination IP(s). | - | error | ping vrf MGMT 1.1.1.1 source Management1 repeat 2 has failed: No source interface Management1 |
+| DC1-SPINE1 | Interfaces | VerifyInterfaceUtilization | Verifies that the utilization of interfaces is below a certain threshold. | - | success | - |
+| DC1-SPINE1 | MLAG | VerifyMlagStatus | Verifies the health status of the MLAG configuration. | - | skipped | MLAG is disabled |
 | DC1-SPINE1 | Routing | VerifyRoutingTableEntry | Verifies that the provided routes are present in the routing table of a specified VRF. | - | failure | The following route(s) are missing from the routing table of VRF default: ['10.1.0.2'] |
-| DC1-SPINE1 | STP | VerifySTPMode | Verifies the configured STP mode for a provided list of VLAN(s). | - | failure | STP mode 'rapidPvst' not configured for the following VLAN(s): [10, 20] |
 | DC1-SPINE1 | SNMP | VerifySnmpStatus | Verifies if the SNMP agent is enabled. | - | failure | SNMP agent disabled in vrf default |
-| DC1-SPINE1 | AAA | VerifyTacacsSourceIntf | Verifies TACACS source-interface for a specified VRF. | - | failure | Source-interface Management0 is not configured in VRF default |
+| DC1-SPINE1 | STP | VerifySTPMode | Verifies the configured STP mode for a provided list of VLAN(s). | - | failure | STP mode 'rapidPvst' not configured for the following VLAN(s): [10, 20] |
 | DC1-SPINE1 | Security | VerifyTelnetStatus | Verifies if Telnet is disabled in the default VRF. | - | success | - |
+| DC1-SPINE1 | Services | VerifyHostname | Verifies the hostname of a device. | - | failure | Expected 's1-spine1' as the hostname, but found 'DC1-SPINE1' instead. |
+| DC1-SPINE1 | Software | VerifyEOSVersion | Verifies the EOS version of the device. | - | failure | device is running version "4.31.1F-34554157.4311F (engineering build)" not in expected versions: ['4.25.4M', '4.26.1F'] |
+| DC1-SPINE1 | System | VerifyNTP | Verifies if NTP is synchronised. | - | failure | The device is not synchronized with the configured NTP server(s): 'NTP is disabled.' |
 | DC1-SPINE1 | VXLAN | VerifyVxlan1Interface | Verifies the Vxlan1 interface status. | - | skipped | Vxlan1 interface is not configured |

tests/units/reporter/test_md_reporter.py

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-from io import BytesIO, TextIOWrapper
+from io import StringIO
 from pathlib import Path
 
 import pytest
@@ -22,7 +22,7 @@ def test_md_report_generate(tmp_path: Path, result_manager: ResultManager) -> No
     expected_report = "test_md_report.md"
 
     # Generate the Markdown report
-    MDReportGenerator.generate(result_manager, md_filename)
+    MDReportGenerator.generate(result_manager.sort(sort_by=["name", "categories", "test"]), md_filename)
     assert md_filename.exists()
 
     # Load the existing Markdown report to compare with the generated one
@@ -46,7 +46,7 @@ def generate_section(self) -> None:
 
     results = ResultManager()
 
-    with TextIOWrapper(BytesIO(b"1 2 3")) as mock_file:
+    with StringIO() as mock_file:
         report = FakeMDReportBase(mock_file, results)
         assert report.generate_heading_name() == "Fake MD Report Base"