chore(tests): use long-lived pact broker

The initial implementation of the compatibility suite spun up and down
the Pact Broker for each scenario, which also resulting in flaky tests
in CI.

This refactor uses a session pytest fixture which will spin up the
broker once, and keep re-using it. Functionality to 'reset' the broker
between tests has also been added.

Signed-off-by: JP-Ellis <josh@jpellis.me>

Author: JP-Ellis

.github/workflows/test.yml

@@ -18,17 +18,30 @@ env:
   HATCH_VERBOSE: 1
 
 jobs:
-  test:
+  test-container:
     name: >-
       Tests py${{ matrix.python-version }} on ${{ matrix.os }}
 
     runs-on: ${{ matrix.os }}
     continue-on-error: ${{ matrix.experimental }}
 
+    services:
+      broker:
+        image: pactfoundation/pact-broker:latest@sha256:8f10947f230f661ef21f270a4abcf53214ba27cd68063db81de555fcd93e07dd
+        ports:
+          - "9292:9292"
+        env:
+          # Basic auth credentials for the Broker
+          PACT_BROKER_ALLOW_PUBLIC_READ: "true"
+          PACT_BROKER_BASIC_AUTH_USERNAME: pactbroker
+          PACT_BROKER_BASIC_AUTH_PASSWORD: pactbroker
+          # Database
+          PACT_BROKER_DATABASE_URL: sqlite:////tmp/pact_broker.sqlite
+
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest]
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         experimental: [false]
         include:
@@ -51,8 +64,20 @@ jobs:
       - name: Install Hatch
         run: pip install --upgrade hatch
 
+      - name: Ensure broker is live
+        run: |
+          i=0
+          until curl -sSf http://localhost:9292/diagnostic/status/heartbeat; do
+            i=$((i+1))
+            if [ $i -gt 120 ]; then
+              echo "Broker failed to start"
+              exit 1
+            fi
+            sleep 1
+          done
+
       - name: Run tests
-        run: hatch run test
+        run: hatch run test --broker-url=http://pactbroker:pactbroker@localhost:9292 --container
 
       - name: Upload coverage
         # TODO: Configure code coverage monitoring
@@ -61,12 +86,44 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  test-no-container:
+    name: >-
+      Tests py${{ matrix.python-version }} on ${{ matrix.os }}
+
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, macos-latest]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        experimental: [false]
+
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+        with:
+          submodules: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install Hatch
+        run: pip install --upgrade hatch
+
+      - name: Run tests
+        run: hatch run test
+
   test-conlusion:
     name: Test matrix complete
 
     runs-on: ubuntu-latest
     needs:
-      - test
+      - test-container
+      - test-no-container
 
     steps:
       - run: echo "Test matrix completed successfully."

conftest.py

@@ -19,3 +19,16 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         ),
         type=str,
     )
+    parser.addoption(
+        "--container",
+        action="store_true",
+        help="Run tests using a container",
+    )
+
+
+def pytest_runtest_setup(item: pytest.Item) -> None:
+    """
+    Hook into the setup phase of tests.
+    """
+    if "container" in item.keywords and not item.config.getoption("--container"):
+        pytest.skip("need --container to run this test")

pyproject.toml

@@ -193,6 +193,9 @@ log_format = "%(asctime)s.%(msecs)03d [%(levelname)-8s] %(name)s: %(message)s"
 log_date_format = "%H:%M:%S"
 
 markers = [
+  # Marker for tests that require a container
+  "container",
+
   # Markers for the compatibility suite
   "consumer",
   "provider",

tests/v3/compatibility_suite/test_v1_provider.py

@@ -13,9 +13,10 @@
 import subprocess
 import sys
 import time
+from contextvars import ContextVar
 from pathlib import Path
 from threading import Thread
-from typing import Any, Generator, NoReturn
+from typing import Any, Generator, NoReturn, Union
 
 import pytest
 import requests
@@ -34,13 +35,52 @@
 
 logger = logging.getLogger(__name__)
 
+reset_broker_var = ContextVar("reset_broker", default=True)
+"""
+This context variable is used to determine whether the Pact broker should be
+cleaned up. It is used to ensure that the broker is only cleaned up once, even
+if a step is run multiple times.
+
+All scenarios which make use of the Pact broker should set this to `True` at the
+start of the scenario.
+"""
+
 
 @pytest.fixture()
 def verifier() -> Verifier:
     """Return a new Verifier."""
     return Verifier()
 
 
+@pytest.fixture(scope="session")
+def broker_url(request: pytest.FixtureRequest) -> Generator[URL, Any, None]:
+    """
+    Fixture to run the Pact broker.
+
+    This inspects whether the `--broker-url` option has been given. If it has,
+    it is assumed that the broker is already running and simply returns the
+    given URL.
+
+    Otherwise, the Pact broker is started in a container. The URL of the
+    containerised broker is then returned.
+    """
+    broker_url: Union[str, None] = request.config.getoption("--broker-url")
+
+    # If we have been given a broker URL, there's nothing more to do here and we
+    # can return early.
+    if broker_url:
+        yield URL(broker_url)
+        return
+
+    with DockerCompose(
+        Path(__file__).parent / "util",
+        compose_file_name="pact-broker.yml",
+        pull=True,
+    ) as _:
+        yield URL("http://pactbroker:pactbroker@localhost:9292")
+    return
+
+
 ################################################################################
 ## Scenario
 ################################################################################
@@ -70,36 +110,44 @@ def test_incorrect_request_is_made_to_provider() -> None:
     """Incorrect request is made to provider."""
 
 
+@pytest.mark.container()
 @scenario(
     "definition/features/V1/http_provider.feature",
     "Verifying a simple HTTP request via a Pact broker",
 )
 def test_verifying_a_simple_http_request_via_a_pact_broker() -> None:
     """Verifying a simple HTTP request via a Pact broker."""
+    reset_broker_var.set(True)  # noqa: FBT003
 
 
+@pytest.mark.container()
 @scenario(
     "definition/features/V1/http_provider.feature",
     "Verifying a simple HTTP request via a Pact broker with publishing results enabled",
 )
 def test_verifying_a_simple_http_request_via_a_pact_broker_with_publishing() -> None:
     """Verifying a simple HTTP request via a Pact broker with publishing."""
+    reset_broker_var.set(True)  # noqa: FBT003
 
 
+@pytest.mark.container()
 @scenario(
     "definition/features/V1/http_provider.feature",
     "Verifying multiple Pact files via a Pact broker",
 )
 def test_verifying_multiple_pact_files_via_a_pact_broker() -> None:
     """Verifying multiple Pact files via a Pact broker."""
+    reset_broker_var.set(True)  # noqa: FBT003
 
 
+@pytest.mark.container()
 @scenario(
     "definition/features/V1/http_provider.feature",
     "Incorrect request is made to provider via a Pact broker",
 )
 def test_incorrect_request_is_made_to_provider_via_a_pact_broker() -> None:
     """Incorrect request is made to provider via a Pact broker."""
+    reset_broker_var.set(True)  # noqa: FBT003
 
 
 @scenario(
@@ -475,6 +523,7 @@ def a_pact_file_for_interaction_is_to_be_verified(
 )
 def a_pact_file_for_interaction_is_to_be_verified_from_a_pact_broker(
     interaction_definitions: dict[int, InteractionDefinition],
+    broker_url: URL,
     verifier: Verifier,
     interaction: int,
     temp_dir: Path,
@@ -494,15 +543,14 @@ def a_pact_file_for_interaction_is_to_be_verified_from_a_pact_broker(
     pacts_dir.mkdir(exist_ok=True, parents=True)
     pact.write_file(pacts_dir)
 
-    with DockerCompose(
-        Path(__file__).parent / "util",
-        compose_file_name="pact-broker.yml",
-        pull=True,
-    ) as _:
-        pact_broker = PactBroker(URL("http://pactbroker:pactbroker@localhost:9292"))
-        pact_broker.publish(pacts_dir)
-        verifier.broker_source(pact_broker.url)
-        yield pact_broker
+    pact_broker = PactBroker(broker_url)
+    if reset_broker_var.get():
+        logger.debug("Resetting Pact broker")
+        pact_broker.reset()
+        reset_broker_var.set(False)  # noqa: FBT003
+    pact_broker.publish(pacts_dir)
+    verifier.broker_source(pact_broker.url)
+    yield pact_broker
 
 
 @given("publishing of verification results is enabled")

tests/v3/compatibility_suite/util/provider.py

@@ -299,6 +299,25 @@ def _install(self) -> None:
         msg = "pact-broker not found"
         raise NotImplementedError(msg)
 
+    def reset(self) -> None:
+        """
+        Reset the Pact Broker.
+
+        This function will reset the Pact Broker by deleting all pacts and
+        verification results.
+        """
+        requests.delete(
+            str(
+                self.url
+                / "integrations"
+                / "provider"
+                / self.provider
+                / "consumer"
+                / self.consumer
+            ),
+            timeout=2,
+        )
+
     def publish(self, directory: Path | str, version: str | None = None) -> None:
         """
         Publish the interactions to the Pact Broker.