From 3408217ab6c9345f518b286d3e7444e252505286 Mon Sep 17 00:00:00 2001 From: Guillaume Mulocher Date: Mon, 30 Sep 2024 15:37:10 +0200 Subject: [PATCH] refactor: Remove final-tests-counts from catalog, have dry-run work as expected (#840) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refactor: Remove final-tests-counts from catalog, have dry-run work as expected * refactor: create AntaCatalog.clear_indexes() --------- Co-authored-by: Matthieu Tâche --- anta/catalog.py | 25 +++++++++++++++++-------- anta/runner.py | 21 ++++++++++++--------- tests/benchmark/test_anta.py | 14 ++++++-------- tests/conftest.py | 2 +- tests/units/test_catalog.py | 8 ++++---- 5 files changed, 40 insertions(+), 30 deletions(-) diff --git a/anta/catalog.py b/anta/catalog.py index b5a77ad25..9b752fa05 100644 --- a/anta/catalog.py +++ b/anta/catalog.py @@ -296,11 +296,16 @@ def __init__( else: self._filename = Path(filename) - # Default indexes for faster access - self.tag_to_tests: defaultdict[str | None, set[AntaTestDefinition]] = defaultdict(set) - self.tests_without_tags: set[AntaTestDefinition] = set() - self.indexes_built: bool = False - self.final_tests_count: int = 0 + self.indexes_built: bool + self.tag_to_tests: defaultdict[str | None, set[AntaTestDefinition]] + self._tests_without_tags: set[AntaTestDefinition] + self._init_indexes() + + def _init_indexes(self) -> None: + """Init indexes related variables.""" + self.tag_to_tests = defaultdict(set) + self._tests_without_tags = set() + self.indexes_built = False @property def filename(self) -> Path | None: @@ -485,7 +490,7 @@ def build_indexes(self, filtered_tests: set[str] | None = None) -> None: - tag_to_tests: A dictionary mapping each tag to a set of tests that contain it. - - tests_without_tags: A set of tests that do not have any tags. + - _tests_without_tags: A set of tests that do not have any tags. Once the indexes are built, the `indexes_built` attribute is set to True. """ @@ -499,11 +504,15 @@ def build_indexes(self, filtered_tests: set[str] | None = None) -> None: for tag in test_tags: self.tag_to_tests[tag].add(test) else: - self.tests_without_tags.add(test) + self._tests_without_tags.add(test) - self.tag_to_tests[None] = self.tests_without_tags + self.tag_to_tests[None] = self._tests_without_tags self.indexes_built = True + def clear_indexes(self) -> None: + """Clear this AntaCatalog instance indexes.""" + self._init_indexes() + def get_tests_by_tags(self, tags: set[str], *, strict: bool = False) -> set[AntaTestDefinition]: """Return all tests that match a given set of tags, according to the specified strictness. diff --git a/anta/runner.py b/anta/runner.py index 6e3290267..12f549daa 100644 --- a/anta/runner.py +++ b/anta/runner.py @@ -147,6 +147,7 @@ def prepare_tests( device_to_tests: defaultdict[AntaDevice, set[AntaTestDefinition]] = defaultdict(set) # Create AntaTestRunner tuples from the tags + final_tests_count = 0 for device in inventory.devices: if tags: if not any(tag in device.tags for tag in tags): @@ -159,9 +160,9 @@ def prepare_tests( # Add the tests with matching tags from device tags device_to_tests[device].update(catalog.get_tests_by_tags(device.tags)) - catalog.final_tests_count += len(device_to_tests[device]) + final_tests_count += len(device_to_tests[device]) - if catalog.final_tests_count == 0: + if len(device_to_tests.values()) == 0: msg = ( f"There are no tests{f' matching the tags {tags} ' if tags else ' '}to run in the current test catalog and device inventory, please verify your inputs." ) @@ -171,13 +172,15 @@ def prepare_tests( return device_to_tests -def get_coroutines(selected_tests: defaultdict[AntaDevice, set[AntaTestDefinition]]) -> list[Coroutine[Any, Any, TestResult]]: +def get_coroutines(selected_tests: defaultdict[AntaDevice, set[AntaTestDefinition]], manager: ResultManager) -> list[Coroutine[Any, Any, TestResult]]: """Get the coroutines for the ANTA run. Parameters ---------- selected_tests A mapping of devices to the tests to run. The selected tests are generated by the `prepare_tests` function. + manager + A ResultManager Returns ------- @@ -189,6 +192,7 @@ def get_coroutines(selected_tests: defaultdict[AntaDevice, set[AntaTestDefinitio for test in test_definitions: try: test_instance = test.test(device=device, inputs=test.inputs) + manager.add(test_instance.result) coros.append(test_instance.test()) except Exception as e: # noqa: PERF203, BLE001 # An AntaTest instance is potentially user-defined code. @@ -256,25 +260,26 @@ async def main( # noqa: PLR0913 selected_tests = prepare_tests(selected_inventory, catalog, tests, tags) if selected_tests is None: return + final_tests_count = sum(len(tests) for tests in selected_tests.values()) run_info = ( "--- ANTA NRFU Run Information ---\n" f"Number of devices: {len(inventory)} ({len(selected_inventory)} established)\n" - f"Total number of selected tests: {catalog.final_tests_count}\n" + f"Total number of selected tests: {final_tests_count}\n" f"Maximum number of open file descriptors for the current ANTA process: {limits[0]}\n" "---------------------------------" ) logger.info(run_info) - if catalog.final_tests_count > limits[0]: + if final_tests_count > limits[0]: logger.warning( "The number of concurrent tests is higher than the open file descriptors limit for this ANTA process.\n" "Errors may occur while running the tests.\n" "Please consult the ANTA FAQ." ) - coroutines = get_coroutines(selected_tests) + coroutines = get_coroutines(selected_tests, manager) if dry_run: logger.info("Dry-run mode, exiting before running the tests.") @@ -286,8 +291,6 @@ async def main( # noqa: PLR0913 AntaTest.nrfu_task = AntaTest.progress.add_task("Running NRFU Tests...", total=len(coroutines)) with Catchtime(logger=logger, message="Running ANTA tests"): - test_results = await asyncio.gather(*coroutines) - for r in test_results: - manager.add(r) + await asyncio.gather(*coroutines) log_cache_statistics(selected_inventory.devices) diff --git a/tests/benchmark/test_anta.py b/tests/benchmark/test_anta.py index 82d08cf6e..6885e2e8f 100644 --- a/tests/benchmark/test_anta.py +++ b/tests/benchmark/test_anta.py @@ -38,19 +38,16 @@ def test_anta_dry_run(benchmark: BenchmarkFixture, catalog: AntaCatalog, invento def bench() -> ResultManager: """Need to wrap the ANTA Runner to instantiate a new ResultManger for each benchmark run.""" manager = ResultManager() + catalog.clear_indexes() asyncio.run(main(manager, inventory, catalog, dry_run=True)) return manager manager = benchmark(bench) logging.disable(logging.NOTSET) - if len(manager.results) != 0: - pytest.fail("ANTA Dry-Run mode should not return any result", pytrace=False) - if catalog.final_tests_count != len(inventory) * len(catalog.tests): - pytest.fail(f"Expected {len(inventory) * len(catalog.tests)} selected tests but got {catalog.final_tests_count}", pytrace=False) - bench_info = ( - "\n--- ANTA NRFU Dry-Run Benchmark Information ---\n" f"Selected tests: {catalog.final_tests_count}\n" "-----------------------------------------------" - ) + if len(manager.results) != len(inventory) * len(catalog.tests): + pytest.fail(f"Expected {len(inventory) * len(catalog.tests)} tests but got {len(manager.results)}", pytrace=False) + bench_info = "\n--- ANTA NRFU Dry-Run Benchmark Information ---\n" f"Test count: {len(manager.results)}\n" "-----------------------------------------------" logger.info(bench_info) @@ -73,6 +70,7 @@ def test_anta(benchmark: BenchmarkFixture, catalog: AntaCatalog, inventory: Anta def bench() -> ResultManager: """Need to wrap the ANTA Runner to instantiate a new ResultManger for each benchmark run.""" manager = ResultManager() + catalog.clear_indexes() asyncio.run(main(manager, inventory, catalog)) return manager @@ -94,7 +92,7 @@ def bench() -> ResultManager: for test in dupes: msg = f"Found duplicate in test catalog: {test}" logger.error(msg) - pytest.fail(f"Expected {len(catalog.tests) * len(inventory)} test results but got {len(manager.results)}", pytrace=False) + pytest.fail(f"Expected {len(catalog.tests) * len(inventory)} tests but got {len(manager.results)}", pytrace=False) bench_info = ( "\n--- ANTA NRFU Benchmark Information ---\n" f"Test results: {len(manager.results)}\n" diff --git a/tests/conftest.py b/tests/conftest.py index 7347d4430..dd535c104 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,7 @@ DATA_DIR: Path = Path(__file__).parent.resolve() / "data" -@pytest.fixture(params=[{"count": 1}]) +@pytest.fixture(params=[{"count": 1}], ids=["1-reachable-device-without-cache"]) def inventory(request: pytest.FixtureRequest) -> Iterator[AntaInventory]: """Generate an ANTA inventory.""" user = "admin" diff --git a/tests/units/test_catalog.py b/tests/units/test_catalog.py index c2bb57c93..ca78a870c 100644 --- a/tests/units/test_catalog.py +++ b/tests/units/test_catalog.py @@ -260,10 +260,10 @@ def test_build_indexes_all(self) -> None: """Test AntaCatalog.build_indexes().""" catalog: AntaCatalog = AntaCatalog.parse(DATA_DIR / "test_catalog_with_tags.yml") catalog.build_indexes() - assert len(catalog.tests_without_tags) == 6 + assert len(catalog._tests_without_tags) == 6 assert "leaf" in catalog.tag_to_tests assert len(catalog.tag_to_tests["leaf"]) == 3 - all_unique_tests = catalog.tests_without_tags + all_unique_tests = catalog._tests_without_tags for tests in catalog.tag_to_tests.values(): all_unique_tests.update(tests) assert len(all_unique_tests) == 11 @@ -275,8 +275,8 @@ def test_build_indexes_filtered(self) -> None: catalog.build_indexes({"VerifyUptime", "VerifyCoredump", "VerifyL3MTU"}) assert "leaf" in catalog.tag_to_tests assert len(catalog.tag_to_tests["leaf"]) == 1 - assert len(catalog.tests_without_tags) == 1 - all_unique_tests = catalog.tests_without_tags + assert len(catalog._tests_without_tags) == 1 + all_unique_tests = catalog._tests_without_tags for tests in catalog.tag_to_tests.values(): all_unique_tests.update(tests) assert len(all_unique_tests) == 4