services fixtures depend on load_settings_before_any_test so they mig…

…ht not be automatically loaded

.github/workflows/ci.yml

@@ -15,15 +15,12 @@ concurrency:
 env:
   INFRAHUB_DB_USERNAME: neo4j
   INFRAHUB_DB_PASSWORD: admin
-  INFRAHUB_DB_ADDRESS: localhost
-  INFRAHUB_DB_PORT: 7687
   INFRAHUB_DB_PROTOCOL: bolt
-  INFRAHUB_BROKER_ADDRESS: message-queue
   INFRAHUB_LOG_LEVEL: CRITICAL
   INFRAHUB_IMAGE_NAME: "opsmill/infrahub"
   INFRAHUB_IMAGE_VER: "local"
   PYTEST_XDIST_WORKER_COUNT: 4
-  INFRAHUB_TEST_IN_DOCKER: 1
+  INFRAHUB_USE_TEST_CONTAINERS: 1
   VALE_VERSION: "3.7.1"
   GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
   METRICS_ENDPOINT: ${{ secrets.METRICS_ENDPOINT }}
@@ -247,18 +244,22 @@ jobs:
         uses: "actions/checkout@v4"
         with:
           submodules: true
+      - name: Set up Python latest version
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: "Setup git credentials"
+        run: "git config --global user.name 'Infrahub' && \
+              git config --global user.email 'infrahub@opsmill.com' && \
+              git config --global --add safe.directory '*' && \
+              git config --global credential.usehttppath true && \
+              git config --global credential.helper /usr/local/bin/infrahub-git-credential"
       - name: "Setup Python environment"
-        run: "pip install toml invoke"
-      - name: "Set environment variables"
-        run: echo INFRAHUB_BUILD_NAME=infrahub-${{ runner.name }} >> $GITHUB_ENV
-      - name: "Set environment variables"
-        run: echo INFRAHUB_IMAGE_VER=local-${{ runner.name }}-${{ github.sha }} >> $GITHUB_ENV
-      - name: "Clear docker environment"
-        run: docker compose -p $INFRAHUB_BUILD_NAME down -v --remove-orphans --rmi local
-      - name: "Build Test Image"
-        run: "invoke dev.build"
-      - name: "Pull External Docker Images"
-        run: "invoke dev.pull"
+        run: |
+          poetry config virtualenvs.create false
+          pip install toml invoke
+      - name: "Install dependencies"
+        run: "poetry install --no-interaction --no-ansi"
       - name: "Unit Tests"
         run: "invoke backend.test-unit"
       - name: "Coveralls : Unit Tests"
@@ -296,18 +297,22 @@ jobs:
         uses: "actions/checkout@v4"
         with:
           submodules: true
+      - name: Set up Python latest version
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: "Setup git credentials"
+        run: "git config --global user.name 'Infrahub' && \
+              git config --global user.email 'infrahub@opsmill.com' && \
+              git config --global --add safe.directory '*' && \
+              git config --global credential.usehttppath true && \
+              git config --global credential.helper /usr/local/bin/infrahub-git-credential"
       - name: "Setup Python environment"
-        run: "pip install toml invoke"
-      - name: "Set environment variables"
-        run: echo INFRAHUB_BUILD_NAME=infrahub-${{ runner.name }} >> $GITHUB_ENV
-      - name: "Set environment variables"
-        run: echo INFRAHUB_IMAGE_VER=local-${{ runner.name }}-${{ github.sha }} >> $GITHUB_ENV
-      - name: "Clear docker environment"
-        run: docker compose -p $INFRAHUB_BUILD_NAME down -v --remove-orphans --rmi local
-      - name: "Build Test Image"
-        run: "invoke dev.build"
-      - name: "Pull External Docker Images"
-        run: "invoke dev.pull"
+        run: |
+          poetry config virtualenvs.create false
+          pip install toml invoke
+      - name: "Install dependencies"
+        run: "poetry install --no-interaction --no-ansi"
       - name: "Mypy Tests"
         run: "invoke backend.mypy --docker"
       - name: "Pylint Tests"
@@ -337,37 +342,38 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: backend-tests-neo4j
+          - name: backend-tests-unit-neo4j
             env:
               INFRAHUB_DB_TYPE: neo4j
-          - name: backend-tests-nats
+          - name: backend-tests-unit-nats
             env:
               INFRAHUB_DB_TYPE: memgraph
               INFRAHUB_USE_NATS: 1
               INFRAHUB_BROKER_DRIVER: nats
-              INFRAHUB_BROKER_PORT: 4222
               INFRAHUB_CACHE_DRIVER: nats
-              INFRAHUB_CACHE_ADDRESS: message-queue
-              INFRAHUB_CACHE_PORT: 4222
     name: ${{ matrix.name }}
     env: ${{ matrix.env }}
     steps:
       - name: "Check out repository code"
         uses: "actions/checkout@v4"
         with:
           submodules: true
+      - name: Set up Python latest version
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: "Setup git credentials"
+        run: "git config --global user.name 'Infrahub' && \
+              git config --global user.email 'infrahub@opsmill.com' && \
+              git config --global --add safe.directory '*' && \
+              git config --global credential.usehttppath true && \
+              git config --global credential.helper /usr/local/bin/infrahub-git-credential"
       - name: "Setup Python environment"
-        run: "pip install toml invoke"
-      - name: "Set environment variables"
-        run: echo INFRAHUB_BUILD_NAME=infrahub-${{ runner.name }} >> $GITHUB_ENV
-      - name: "Set environment variables"
-        run: echo INFRAHUB_IMAGE_VER=local-${{ runner.name }}-${{ github.sha }} >> $GITHUB_ENV
-      - name: "Clear docker environment"
-        run: docker compose -p $INFRAHUB_BUILD_NAME down -v --remove-orphans --rmi local
-      - name: "Build Test Image"
-        run: "invoke dev.build"
-      - name: "Pull External Docker Images"
-        run: "invoke dev.pull"
+        run: |
+          poetry config virtualenvs.create false
+          pip install toml invoke
+      - name: "Install dependencies"
+        run: "poetry install --no-interaction --no-ansi"
       - name: "Unit Tests"
         run: "invoke backend.test-unit"
 

backend/infrahub/config.py

@@ -175,6 +175,7 @@ class BrokerSettings(BaseSettings):
     password: str = "infrahub"
     address: str = "localhost"
     port: Optional[int] = Field(default=None, ge=1, le=65535, description="Specified if running on a non default port.")
+    rabbitmq_http_port: Optional[int] = Field(default=None, ge=1, le=65535)
     namespace: str = "infrahub"
     maximum_message_retries: int = Field(
         default=10, description="The maximum number of retries that are attempted for failed messages"

backend/tests/conftest.py

@@ -8,7 +8,8 @@
 
 import pytest
 import ujson
-from infrahub_sdk.utils import str_to_bool
+from testcontainers.core.container import DockerContainer
+from testcontainers.core.waiting_utils import wait_for_logs
 
 from infrahub import config
 from infrahub.core import registry
@@ -34,9 +35,8 @@
 from tests.adapters.log import FakeLogger
 from tests.adapters.message_bus import BusRecorder, BusSimulator
 
-BUILD_NAME = os.environ.get("INFRAHUB_BUILD_NAME", "infrahub")
-TEST_IN_DOCKER = str_to_bool(os.environ.get("INFRAHUB_TEST_IN_DOCKER", "false"))
 ResponseClass = TypeVar("ResponseClass")
+INFRAHUB_USE_TEST_CONTAINERS = os.getenv("INFRAHUB_USE_TEST_CONTAINERS", True)
 
 
 def pytest_addoption(parser):
@@ -133,27 +133,141 @@ async def register_core_models_schema(default_branch: Branch, register_internal_
     return schema_branch
 
 
-@pytest.fixture(scope="module", autouse=True)
-def execute_before_any_test(worker_id, tmpdir_factory):
+@pytest.fixture(scope="session")
+def neo4j(request: pytest.FixtureRequest, load_settings_before_any_test) -> DockerContainer:
+    if not INFRAHUB_USE_TEST_CONTAINERS or os.getenv("INFRAHUB_DB_TYPE") == "memgraph":
+        return None
+
+    container = (
+        DockerContainer(image="neo4j:5.19.0-enterprise")
+        .with_env("NEO4J_AUTH", "neo4j/admin")
+        .with_env("NEO4J_ACCEPT_LICENSE_AGREEMENT", "yes")
+        .with_env("NEO4J_dbms_security_procedures_unrestricted", "apoc.*")
+        .with_env("NEO4J_dbms_security_auth__minimum__password__length", "4")
+        .with_exposed_ports(7687)  # port for bolt protocol
+    )
+
+    def cleanup():
+        container.stop()
+
+    container.start()
+    wait_for_logs(container, "Started.")  # wait_container_is_ready does not seem to be enough
+    request.addfinalizer(cleanup)
+
+    config.SETTINGS.database.address = "localhost"
+    config.SETTINGS.database.port = int(container.get_exposed_port(7687))
+
+    return container
+
+
+@pytest.fixture(scope="session", autouse=True)
+def rabbitmq(request: pytest.FixtureRequest, load_settings_before_any_test) -> DockerContainer:
+    if not INFRAHUB_USE_TEST_CONTAINERS or config.SETTINGS.cache.driver == config.CacheDriver.NATS:
+        return None
+
+    container = (
+        DockerContainer(image="rabbitmq:3.13.1-management")
+        .with_env("RABBITMQ_DEFAULT_USER", "infrahub")
+        .with_env("RABBITMQ_DEFAULT_PASS", "infrahub")
+        .with_exposed_ports(5672, 15672)
+    )
+
+    def cleanup():
+        container.stop()
+
+    container.start()
+    wait_for_logs(container, "Server startup complete;")  # wait_container_is_ready does not seem to be enough
+    request.addfinalizer(cleanup)
+
+    config.SETTINGS.broker.address = "localhost"
+    config.SETTINGS.broker.port = int(container.get_exposed_port(5672))
+    config.SETTINGS.broker.rabbitmq_http_port = int(container.get_exposed_port(15672))
+
+    return container
+
+
+# NOTE: This fixture needs to run before initialize_lock_fixture which is guaranteed to run after as it has a module scope.
+@pytest.fixture(scope="session", autouse=True)
+def redis(request: pytest.FixtureRequest, load_settings_before_any_test) -> DockerContainer:
+    if not INFRAHUB_USE_TEST_CONTAINERS or config.SETTINGS.cache.driver == config.CacheDriver.NATS:
+        raise ValueError(INFRAHUB_USE_TEST_CONTAINERS)
+        return None
+
+    container = DockerContainer(image="redis:latest").with_exposed_ports(6379)
+
+    def cleanup():
+        container.stop()
+
+    container.start()
+    wait_for_logs(container, "Ready to accept connections tcp")  # wait_container_is_ready does not seem to be enough
+    request.addfinalizer(cleanup)
+
+    config.SETTINGS.cache.address = "localhost"
+    config.SETTINGS.cache.port = int(container.get_exposed_port(6379))
+
+    return container
+
+
+@pytest.fixture(scope="session")
+def memgraph(request: pytest.FixtureRequest, load_settings_before_any_test) -> DockerContainer:
+    if not INFRAHUB_USE_TEST_CONTAINERS or os.getenv("INFRAHUB_DB_TYPE") != "memgraph":
+        return None
+
+    container = (
+        DockerContainer(image="memgraph/memgraph-platform:latest", entrypoint=["/usr/bin/supervisord"], init=True)
+        .with_env("MGCONSOLE", "--username neo4j --password admin")
+        .with_env("APP_CYPHER_QUERY_MAX_LEN", 10000)
+        .with_exposed_ports(7687, 7444)
+        # TODO: what to do with 7444 log port?
+    )
+
+    def cleanup():
+        container.stop()
+
+    container.start()
+    wait_for_logs(
+        container, "INFO success: lab entered RUNNING state"
+    )  # wait_container_is_ready does not seem to be enough
+
+    request.addfinalizer(cleanup)
+
+    config.SETTINGS.database.address = "localhost"
+    config.SETTINGS.database.port = int(container.get_exposed_port(7687))
+
+    return container
+
+
+@pytest.fixture(scope="session", autouse=True)
+def nats(request: pytest.FixtureRequest, load_settings_before_any_test) -> DockerContainer:
+    if not INFRAHUB_USE_TEST_CONTAINERS or config.SETTINGS.cache.driver != config.CacheDriver.NATS:
+        return None
+
+    container = DockerContainer(image="nats:alpine").with_command("--jetstream").with_exposed_ports(4222)
+
+    def cleanup():
+        container.stop()
+
+    container.start()
+    wait_for_logs(container, "Server is ready")  # wait_container_is_ready does not seem to be enough
+    request.addfinalizer(cleanup)
+
+    config.SETTINGS.cache.address = "localhost"
+    config.SETTINGS.broker.address = "localhost"
+    config.SETTINGS.broker.port = int(container.get_exposed_port(4222))
+    config.SETTINGS.cache.port = int(container.get_exposed_port(4222))
+
+    return container
+
+
+# TODO: use fixtures only if test does need it.
+@pytest.fixture(scope="session", autouse=True)
+def load_settings_before_any_test(tmpdir_factory):
     config.load_and_exit()
 
     config.SETTINGS.storage.driver = config.StorageDriver.FileSystemStorage
 
-    if TEST_IN_DOCKER:
-        try:
-            db_id = int(worker_id[2]) + 1
-        except (ValueError, IndexError):
-            db_id = 1
-
-        config.SETTINGS.cache.address = f"{BUILD_NAME}-cache-{db_id}"
-        if config.SETTINGS.cache.driver == config.CacheDriver.NATS:
-            config.SETTINGS.cache.address = f"{BUILD_NAME}-message-queue-{db_id}"
-        config.SETTINGS.database.address = f"{BUILD_NAME}-database-{db_id}"
-        config.SETTINGS.broker.address = f"{BUILD_NAME}-message-queue-{db_id}"
-        config.SETTINGS.storage.local = config.FileSystemStorageSettings(path="/opt/infrahub/storage")
-    else:
-        storage_dir = tmpdir_factory.mktemp("storage")
-        config.SETTINGS.storage.local._path = str(storage_dir)
+    storage_dir = tmpdir_factory.mktemp("storage")
+    config.SETTINGS.storage.local._path = str(storage_dir)
 
     config.SETTINGS.broker.enable = False
     config.SETTINGS.cache.enable = True
@@ -162,6 +276,11 @@ def execute_before_any_test(worker_id, tmpdir_factory):
     config.SETTINGS.main.internal_address = "http://mock"
     config.OVERRIDE.message_bus = BusRecorder()
 
+
+# Hacky: We use scope module to make sure this runs after redis fixture that would set config.SETTINGS.cache
+# that are used during init.
+@pytest.fixture(scope="module", autouse=True)
+def initialize_lock_fixture(load_settings_before_any_test):
     initialize_lock()
 
 

backend/tests/helpers/test_app.py

@@ -110,5 +110,4 @@ async def initialize_registry(
             password=config.SETTINGS.initial.admin_password,
             token_value=api_token,
         )
-
         await initialization(db=db)

backend/tests/integration/conftest.py

@@ -6,6 +6,7 @@
 import pytest
 import yaml
 from infrahub_sdk import UUIDT
+from testcontainers.core.container import DockerContainer
 
 from infrahub.core import registry
 from infrahub.core.constants import InfrahubKind
@@ -32,8 +33,9 @@ def event_loop():
     loop.close()
 
 
+# Here load_settings in needed because we want to have parameters already loaded. yes.
 @pytest.fixture(scope="module")
-async def db() -> AsyncGenerator[InfrahubDatabase, None]:
+async def db(neo4j: DockerContainer, memgraph: DockerContainer) -> AsyncGenerator[InfrahubDatabase, None]:
     driver = InfrahubDatabase(driver=await get_db(retry=1))
 
     yield driver

backend/tests/integration/services/adapters/message_bus/test_rabbitmq.py

@@ -62,7 +62,7 @@ class RabbitMQManager:
     @property
     def base_url(self) -> str:
         scheme = "https" if self.settings.tls_enabled else "http"
-        port = f"1{self.settings.service_port}"
+        port = f"{self.settings.rabbitmq_http_port}"
         return f"{scheme}://{self.settings.address}:{port}/api"
 
     async def create_virtual_host(self) -> None: