From be40cbd89d00e42bebb8ca0b81e6b44fcd615dae Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Wed, 20 Mar 2024 11:26:25 +1100 Subject: [PATCH 01/15] fix(v3): allow optional publish options Most of the publish option arguments are optional, but the initial implementation had these as compulsory. Signed-off-by: JP-Ellis --- src/pact/v3/ffi.py | 12 ++++++------ src/pact/v3/verifier.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/pact/v3/ffi.py b/src/pact/v3/ffi.py index e809e173d..16fd47dfb 100644 --- a/src/pact/v3/ffi.py +++ b/src/pact/v3/ffi.py @@ -6638,9 +6638,9 @@ def verifier_set_no_pacts_is_error(handle: VerifierHandle, *, enabled: bool) -> def verifier_set_publish_options( handle: VerifierHandle, provider_version: str, - build_url: str, - provider_tags: List[str], - provider_branch: str, + build_url: str | None, + provider_tags: List[str] | None, + provider_branch: str | None, ) -> None: """ Set the options used when publishing verification results to the Broker. @@ -6667,10 +6667,10 @@ def verifier_set_publish_options( retval: int = lib.pactffi_verifier_set_publish_options( handle._ref, provider_version.encode("utf-8"), - build_url.encode("utf-8"), + build_url.encode("utf-8") if build_url else ffi.NULL, [ffi.new("char[]", t.encode("utf-8")) for t in provider_tags or []], - len(provider_tags), - provider_branch.encode("utf-8"), + len(provider_tags or []), + provider_branch.encode("utf-8") if provider_branch else ffi.NULL, ) if retval != 0: msg = f"Failed to set publish options for {handle}." diff --git a/src/pact/v3/verifier.py b/src/pact/v3/verifier.py index c6cf8f436..deef51907 100644 --- a/src/pact/v3/verifier.py +++ b/src/pact/v3/verifier.py @@ -350,8 +350,8 @@ def set_error_on_empty_pact(self, *, enabled: bool = True) -> Self: def set_publish_options( self, version: str, - url: str, - branch: str, + url: str | None = None, + branch: str | None = None, tags: list[str] | None = None, ) -> Self: """ From b6090f54fb2a55786c891842a10d1dedda60590f Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Wed, 20 Mar 2024 11:29:33 +1100 Subject: [PATCH 02/15] fix(v3): strip embedded user/password from urls The username and password authentication are passed through other arguments. Signed-off-by: JP-Ellis --- src/pact/v3/verifier.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pact/v3/verifier.py b/src/pact/v3/verifier.py index deef51907..2e8ce85b5 100644 --- a/src/pact/v3/verifier.py +++ b/src/pact/v3/verifier.py @@ -589,7 +589,7 @@ def _add_source_remote( pact.v3.ffi.verifier_url_source( self._handle, - str(url), + str(url.with_user(None).with_password(None)), username, password, token, @@ -686,14 +686,14 @@ def broker_source( # noqa: PLR0913 if selector: return BrokerSelectorBuilder( self, - str(url), + str(url.with_user(None).with_password(None)), username, password, token, ) pact.v3.ffi.verifier_broker_source( self._handle, - str(url), + str(url.with_user(None).with_password(None)), username, password, token, From 7417ab8747da3cd038f4622e269e884656c8b910 Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Wed, 20 Mar 2024 11:33:04 +1100 Subject: [PATCH 03/15] chore(tests): update log formatting Signed-off-by: JP-Ellis --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 820218d8a..da309ede8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -189,8 +189,8 @@ filterwarnings = [ ] log_level = "NOTSET" -log_format = "%(asctime)s [%(levelname)-8s] %(name)s: %(message)s" -log_date-format = "%H:%M:%S" +log_format = "%(asctime)s.%(msecs)03d [%(levelname)-8s] %(name)s: %(message)s" +log_date_format = "%H:%M:%S" markers = [ # Markers for the compatibility suite From d24d4cdd056f7aaf1cdd17123707e42dce40a7d5 Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Wed, 20 Mar 2024 11:40:06 +1100 Subject: [PATCH 04/15] refactor(tests): move parse_headers/matching_rules out of class These functions do not really belong to the class, and there's now a need for them to be called directly. Signed-off-by: JP-Ellis --- tests/v3/compatibility_suite/util/__init__.py | 78 +++++++++---------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/tests/v3/compatibility_suite/util/__init__.py b/tests/v3/compatibility_suite/util/__init__.py index 919543db5..0f7fb8ca6 100644 --- a/tests/v3/compatibility_suite/util/__init__.py +++ b/tests/v3/compatibility_suite/util/__init__.py @@ -149,6 +149,41 @@ def parse_markdown_table(content: str) -> list[dict[str, str]]: return [dict(zip(rows[0], row)) for row in rows[1:]] +def parse_headers(headers: str) -> MultiDict[str]: + """ + Parse the headers. + + The headers are in the format: + + ```text + 'X-A: 1', 'X-B: 2', 'X-A: 3' + ``` + + As headers can be repeated, the result is a MultiDict. + """ + kvs: list[tuple[str, str]] = [] + for header in headers.split(", "): + k, _sep, v = header.strip("'").partition(": ") + kvs.append((k, v)) + return MultiDict(kvs) + + +def parse_matching_rules(matching_rules: str) -> str: + """ + Parse the matching rules. + + The matching rules are in one of two formats: + + - An explicit JSON object, prefixed by `JSON: `. + - A fixture file which contains the matching rules. + """ + if matching_rules.startswith("JSON: "): + return matching_rules[6:] + + with (FIXTURES_ROOT / matching_rules).open("r") as file: + return file.read() + + class InteractionDefinition: """ Interaction definition. @@ -300,12 +335,12 @@ def update(self, **kwargs: str) -> None: # noqa: C901, PLR0912 self.query = query if headers := kwargs.pop("headers", None): - self.headers = self.parse_headers(headers) + self.headers = parse_headers(headers) if headers := ( kwargs.pop("raw headers", None) or kwargs.pop("raw_headers", None) ): - self.headers = self.parse_headers(headers) + self.headers = parse_headers(headers) if body := kwargs.pop("body", None): # When updating the body, we _only_ update the body content, not @@ -337,9 +372,7 @@ def update(self, **kwargs: str) -> None: # noqa: C901, PLR0912 if matching_rules := ( kwargs.pop("matching_rules", None) or kwargs.pop("matching rules", None) ): - self.matching_rules = InteractionDefinition.parse_matching_rules( - matching_rules - ) + self.matching_rules = parse_matching_rules(matching_rules) if len(kwargs) > 0: msg = f"Unexpected arguments: {kwargs.keys()}" @@ -353,41 +386,6 @@ def __repr__(self) -> str: ", ".join(f"{k}={v!r}" for k, v in vars(self).items()), ) - @classmethod - def parse_headers(cls, headers: str) -> MultiDict[str]: - """ - Parse the headers. - - The headers are in the format: - - ```text - 'X-A: 1', 'X-B: 2', 'X-A: 3' - ``` - - As headers can be repeated, the result is a MultiDict. - """ - kvs: list[tuple[str, str]] = [] - for header in headers.split(", "): - k, _sep, v = header.strip("'").partition(": ") - kvs.append((k, v)) - return MultiDict(kvs) - - @classmethod - def parse_matching_rules(cls, matching_rules: str) -> str: - """ - Parse the matching rules. - - The matching rules are in one of two formats: - - - An explicit JSON object, prefixed by `JSON: `. - - A fixture file which contains the matching rules. - """ - if matching_rules.startswith("JSON: "): - return matching_rules[6:] - - with (FIXTURES_ROOT / matching_rules).open("r") as file: - return file.read() - def add_to_pact(self, pact: pact.v3.Pact, name: str) -> None: # noqa: PLR0912, C901 """ Add the interaction to the pact. From 55e63ada07f57c8d7572e816f56b8e39913a5749 Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Wed, 20 Mar 2024 11:42:39 +1100 Subject: [PATCH 05/15] chore(test): add state to interaction definition Required for the provider state callbacks. Signed-off-by: JP-Ellis --- tests/v3/compatibility_suite/util/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v3/compatibility_suite/util/__init__.py b/tests/v3/compatibility_suite/util/__init__.py index 0f7fb8ca6..89a74c737 100644 --- a/tests/v3/compatibility_suite/util/__init__.py +++ b/tests/v3/compatibility_suite/util/__init__.py @@ -304,6 +304,7 @@ def parse_file(self, file: Path) -> None: def __init__(self, **kwargs: str) -> None: """Initialise the interaction definition.""" self.id: int | None = None + self.state: str | None = None self.method: str = kwargs.pop("method") self.path: str = kwargs.pop("path") self.response: int = int(kwargs.pop("response", 200)) From 1f5c454acfe71dbd7eb4a0e22ddaca46a548e64e Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Wed, 20 Mar 2024 11:50:07 +1100 Subject: [PATCH 06/15] chore(test): adapt InteractionDefinition for provider This introduces a few important changes to the interaction definition: - Addition of `response_headers` - Support for the response status code through either the `response` or `status` kwargs - Combining the response content type and body to reflect the same logic as with the request body and content type. - A new `add_to_flask` method (akin to `add_to_pact`) which defines the interaction for a Flask app. Signed-off-by: JP-Ellis --- tests/v3/compatibility_suite/util/__init__.py | 99 ++++++++++++++++--- 1 file changed, 84 insertions(+), 15 deletions(-) diff --git a/tests/v3/compatibility_suite/util/__init__.py b/tests/v3/compatibility_suite/util/__init__.py index 89a74c737..19f779965 100644 --- a/tests/v3/compatibility_suite/util/__init__.py +++ b/tests/v3/compatibility_suite/util/__init__.py @@ -28,12 +28,13 @@ def _(): from pathlib import Path from xml.etree import ElementTree +import flask +from flask import request from multidict import MultiDict from yarl import URL if typing.TYPE_CHECKING: - import pact.v3 - import pact.v3.pact + from pact.v3.pact import Pact logger = logging.getLogger(__name__) SUITE_ROOT = Path(__file__).parent.parent / "definition" @@ -311,7 +312,7 @@ def __init__(self, **kwargs: str) -> None: self.query: str | None = None self.headers: MultiDict[str] = MultiDict() self.body: InteractionDefinition.Body | None = None - self.response_content: str | None = None + self.response_headers: MultiDict[str] = MultiDict() self.response_body: InteractionDefinition.Body | None = None self.matching_rules: str | None = None self.update(**kwargs) @@ -357,18 +358,31 @@ def update(self, **kwargs: str) -> None: # noqa: C901, PLR0912 self.body = InteractionDefinition.Body("") self.body.mime_type = content_type - if response := kwargs.pop("response", None): + if response := kwargs.pop("response", None) or kwargs.pop("status", None): self.response = int(response) + if response_headers := ( + kwargs.pop("response_headers", None) or kwargs.pop("response headers", None) + ): + self.response_headers = parse_headers(response_headers) + if response_content := ( kwargs.pop("response_content", None) or kwargs.pop("response content", None) ): - self.response_content = response_content + if self.response_body is None: + self.response_body = InteractionDefinition.Body("") + self.response_body.mime_type = response_content if response_body := ( kwargs.pop("response_body", None) or kwargs.pop("response body", None) ): + orig_content_type = ( + self.response_body.mime_type if self.response_body else None + ) self.response_body = InteractionDefinition.Body(response_body) + self.response_body.mime_type = ( + self.response_body.mime_type or orig_content_type + ) if matching_rules := ( kwargs.pop("matching_rules", None) or kwargs.pop("matching rules", None) @@ -387,7 +401,7 @@ def __repr__(self) -> str: ", ".join(f"{k}={v!r}" for k, v in vars(self).items()), ) - def add_to_pact(self, pact: pact.v3.Pact, name: str) -> None: # noqa: PLR0912, C901 + def add_to_pact(self, pact: Pact, name: str) -> None: # noqa: C901, PLR0912 """ Add the interaction to the pact. @@ -406,6 +420,11 @@ def add_to_pact(self, pact: pact.v3.Pact, name: str) -> None: # noqa: PLR0912, logging.info("with_request(%s, %s)", self.method, self.path) interaction.with_request(self.method, self.path) + # We distinguish between "" and None here. + if self.state is not None: + logging.info("given(%s)", self.state) + interaction.given(self.state) + if self.query: query = URL.build(query_string=self.query).query logging.info("with_query_parameters(%s)", query.items()) @@ -448,31 +467,81 @@ def add_to_pact(self, pact: pact.v3.Pact, name: str) -> None: # noqa: PLR0912, logging.info("will_respond_with(%s)", self.response) interaction.will_respond_with(self.response) - if self.response_content: - if self.response_body is None: - msg = "Expected response body along with response content type" - raise ValueError(msg) - + if self.response_body: if self.response_body.string: logging.info( "with_body(%s, %s)", truncate(self.response_body.string), - self.response_content, + self.response_body.mime_type, ) interaction.with_body( self.response_body.string, - self.response_content, + self.response_body.mime_type, ) elif self.response_body.bytes: logging.info( "with_binary_file(%s, %s)", truncate(self.response_body.bytes), - self.response_content, + self.response_body.mime_type, ) interaction.with_binary_body( self.response_body.bytes, - self.response_content, + self.response_body.mime_type, ) else: msg = "Unexpected body definition" raise RuntimeError(msg) + + def add_to_flask(self, app: flask.Flask) -> None: + """ + Add an interaction to a Flask app. + + Args: + app: + The Flask app to add the interaction to. + """ + + async def route_fn() -> flask.Response: + logger.info("Received request: %s %s", self.method, self.path) + if self.query: + query = URL.build(query_string=self.query).query + # Perform a two-way check to ensure that the query parameters + # are present in the request, and that the request contains no + # unexpected query parameters. + for k, v in query.items(): + assert request.args[k] == v + for k, v in request.args.items(): + assert query[k] == v + + if self.headers: + # Perform a one-way check to ensure that the expected headers + # are present in the request, but don't check for any unexpected + # headers. + for k, v in self.headers.items(): + assert k in request.headers + assert request.headers[k] == v + + if self.body: + assert request.data == self.body.bytes + + return flask.Response( + response=self.response_body.bytes or self.response_body.string or None + if self.response_body + else None, + status=self.response, + headers=self.response_headers, + content_type=self.response_body.mime_type + if self.response_body + else None, + direct_passthrough=True, + ) + + # The route function needs to have a unique name + clean_name = self.path.replace("/", "_").replace("__", "_") + route_fn.__name__ = f"{self.method.lower()}_{clean_name}" + + app.add_url_rule( + self.path, + view_func=route_fn, + methods=[self.method], + ) From 740e03c29e2659d90a0a3680744676a079674134 Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Wed, 20 Mar 2024 11:58:05 +1100 Subject: [PATCH 07/15] chore(test): add serialize function This is a utility function which is required in order to pass certain arguments between the test suite, and the Flask app running in a separate process. Signed-off-by: JP-Ellis --- tests/v3/compatibility_suite/util/__init__.py | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/tests/v3/compatibility_suite/util/__init__.py b/tests/v3/compatibility_suite/util/__init__.py index 19f779965..4c4240d90 100644 --- a/tests/v3/compatibility_suite/util/__init__.py +++ b/tests/v3/compatibility_suite/util/__init__.py @@ -21,11 +21,15 @@ def _(): from __future__ import annotations +import base64 import contextlib import hashlib import logging import typing +from collections.abc import Collection, Mapping +from datetime import date, datetime, time from pathlib import Path +from typing import Any from xml.etree import ElementTree import flask @@ -97,7 +101,7 @@ def truncate(data: str | bytes) -> str: """ if len(data) <= 32: if isinstance(data, str): - return f"{data!r}" + return f"{data}" return data.decode("utf-8", "backslashreplace") length = len(data) @@ -150,6 +154,57 @@ def parse_markdown_table(content: str) -> list[dict[str, str]]: return [dict(zip(rows[0], row)) for row in rows[1:]] +def serialize(obj: Any) -> Any: # noqa: ANN401, PLR0911 + """ + Convert an object to a dictionary. + + This function converts an object to a dictionary by calling `vars` on the + object. This is useful for classes which are not otherwise serializable + using `json.dumps`. + + A few special cases are handled: + + - If the object is a `datetime` object, it is converted to an ISO 8601 + string. + - All forms of [`Mapping`][collections.abc.Mapping] are converted to + dictionaries. + - All forms of [`Collection`][collections.abc.Collection] are converted to + lists. + + All other types are converted to strings using the `repr` function. + """ + if isinstance(obj, datetime | date | time): + return obj.isoformat() + + # Basic types which are already serializable + if isinstance(obj, str | int | float | bool | type(None)): + return obj + + # Bytes + if isinstance(obj, bytes): + return { + "__class__": obj.__class__.__name__, + "data": base64.b64encode(obj).decode("utf-8"), + } + + # Collections + if isinstance(obj, Mapping): + return {k: serialize(v) for k, v in obj.items()} + + if isinstance(obj, Collection): + return [serialize(v) for v in obj] + + # Objects + if hasattr(obj, "__dict__"): + return { + "__class__": obj.__class__.__name__, + "__module__": obj.__class__.__module__, + **{k: serialize(v) for k, v in obj.__dict__.items()}, + } + + return repr(obj) + + def parse_headers(headers: str) -> MultiDict[str]: """ Parse the headers. From 9b35d3b3257d2f6bfcc109ede1f78e72f23ed574 Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Wed, 20 Mar 2024 11:59:37 +1100 Subject: [PATCH 08/15] chore(test): add provider utilities Signed-off-by: JP-Ellis --- tests/v3/compatibility_suite/util/provider.py | 422 ++++++++++++++++++ 1 file changed, 422 insertions(+) create mode 100644 tests/v3/compatibility_suite/util/provider.py diff --git a/tests/v3/compatibility_suite/util/provider.py b/tests/v3/compatibility_suite/util/provider.py new file mode 100644 index 000000000..6ad1d38b3 --- /dev/null +++ b/tests/v3/compatibility_suite/util/provider.py @@ -0,0 +1,422 @@ +""" +Provider utilities for compatibility suite tests. + +The main functionality provided by this module is the ability to start a +provider application with a set of interactions. Since this is done +in a subprocess, any configuration must be passed in through files. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).parent.parent.parent.parent.parent)) + + +import contextlib +import json +import logging +import os +import pickle +import shutil +import socket +import subprocess +from contextvars import ContextVar +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING + +import flask +import requests +from flask import request +from yarl import URL + +import pact.constants # type: ignore[import-untyped] +from tests.v3.compatibility_suite.util import serialize + +if TYPE_CHECKING: + from tests.v3.compatibility_suite.util import InteractionDefinition + + +logger = logging.getLogger(__name__) + +version_var = ContextVar("version_var", default="0") + + +def next_version() -> str: + """ + Get the next version for the consumer. + + This is used to generate a new version for the consumer application to use + when publishing the interactions to the Pact Broker. + + Returns: + The next version. + """ + version = version_var.get() + version_var.set(str(int(version) + 1)) + return version + + +def _find_free_port() -> int: + """ + Find a free port. + + This is used to find a free port to host the API on when running locally. It + is allocated, and then released immediately so that it can be used by the + API. + + Returns: + The port number. + """ + with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + + +class Provider: + """ + HTTP Provider. + """ + + def __init__(self, provider_dir: Path | str) -> None: + """ + Instantiate a new provider. + + Args: + provider_dir: + The directory containing various files used to configure the + provider. At a minimum, this directory must contain a file + called `interactions.pkl`. This file must contain a list of + [`InteractionDefinition`] objects. + """ + self.provider_dir = Path(provider_dir) + if not self.provider_dir.is_dir(): + msg = f"Directory {self.provider_dir} does not exist" + raise ValueError(msg) + + self.app: flask.Flask = flask.Flask("provider") + self._add_ping(self.app) + self._add_callback(self.app) + self._add_after_request(self.app) + self._add_interactions(self.app) + + def _add_ping(self, app: flask.Flask) -> None: + """ + Add a ping endpoint to the provider. + + This is used to check that the provider is running. + """ + + @app.get("/_test/ping") + def ping() -> str: + """Simple ping endpoint for testing.""" + return "pong" + + def _add_callback(self, app: flask.Flask) -> None: + """ + Add a callback endpoint to the provider. + + This is used to receive any callbacks from Pact to configure any + internal state (e.g., "given a user exists"). As far as the testing + is concerned, this is just a simple endpoint that records the request + and returns an empty response. + + If the provider directory contains a file called `fail_callback`, then + the callback will return a 404 response. + + If the provider directory contains a file called `provider_state`, then + the callback will check that the `state` query parameter matches the + contents of the file. + """ + + @app.route("/_test/callback", methods=["GET", "POST"]) + def callback() -> tuple[str, int] | str: + if (self.provider_dir / "fail_callback").exists(): + return "Provider state not found", 404 + + provider_state_path = self.provider_dir / "provider_state" + if provider_state_path.exists(): + state = provider_state_path.read_text() + assert request.args["state"] == state + + json_file = ( + self.provider_dir + / f"callback.{datetime.now(tz=UTC).strftime('%H:%M:%S.%f')}.json" + ) + with json_file.open("w") as f: + json.dump( + { + "method": request.method, + "path": request.path, + "query_string": request.query_string.decode("utf-8"), + "query_params": serialize(request.args), + "headers_list": serialize(request.headers), + "headers_dict": serialize(dict(request.headers)), + "body": request.data.decode("utf-8"), + "form": serialize(request.form), + }, + f, + ) + + return "" + + def _add_after_request(self, app: flask.Flask) -> None: + """ + Add a handler to log requests and responses. + + This is used to log the requests and responses to the provider + application (both to the logger as well as to files). + """ + + @app.after_request + def log_request(response: flask.Response) -> flask.Response: + logger.debug("Request: %s %s", request.method, request.path) + logger.debug( + "Request query string: %s", request.query_string.decode("utf-8") + ) + logger.debug("Request query params: %s", serialize(request.args)) + logger.debug("Request headers: %s", serialize(request.headers)) + logger.debug("Request body: %s", request.data.decode("utf-8")) + logger.debug("Request form: %s", serialize(request.form)) + + with ( + self.provider_dir + / f"request.{datetime.now(tz=UTC).strftime('%H:%M:%S.%f')}.json" + ).open("w") as f: + json.dump( + { + "method": request.method, + "path": request.path, + "query_string": request.query_string.decode("utf-8"), + "query_params": serialize(request.args), + "headers_list": serialize(request.headers), + "headers_dict": serialize(dict(request.headers)), + "body": request.data.decode("utf-8"), + "form": serialize(request.form), + }, + f, + ) + return response + + @app.after_request + def log_response(response: flask.Response) -> flask.Response: + try: + body = response.get_data(as_text=True) + except UnicodeDecodeError: + body = "" + logger.debug("Response: %s", response.status_code) + logger.debug("Response headers: %s", serialize(response.headers)) + logger.debug("Response body: %s", body) + with ( + self.provider_dir + / f"response.{datetime.now(tz=UTC).strftime('%H:%M:%S.%f')}.json" + ).open("w") as f: + json.dump( + { + "status_code": response.status_code, + "headers_list": serialize(response.headers), + "headers_dict": serialize(dict(response.headers)), + "body": body, + }, + f, + ) + return response + + def _add_interactions(self, app: flask.Flask) -> None: + """ + Add the interactions to the provider. + """ + with (self.provider_dir / "interactions.pkl").open("rb") as f: + interactions: list[InteractionDefinition] = pickle.load(f) # noqa: S301 + + for interaction in interactions: + interaction.add_to_flask(app) + + def run(self) -> None: + """ + Start the provider. + """ + url = URL(f"http://localhost:{_find_free_port()}") + self.app.run( + host=url.host, + port=url.port, + debug=True, + ) + + +class PactBroker: + """ + Interface to the Pact Broker. + """ + + def __init__( # noqa: PLR0913 + self, + broker_url: URL, + *, + username: str | None = None, + password: str | None = None, + provider: str = "provider", + consumer: str = "consumer", + ) -> None: + """ + Instantiate a new Pact Broker interface. + """ + self.url = broker_url + self.username = broker_url.user or username + self.password = broker_url.password or password + self.provider = provider + self.consumer = consumer + + self.broker_bin: str = ( + shutil.which("pact-broker") or pact.constants.BROKER_CLIENT_PATH + ) + if not self.broker_bin: + if "CI" in os.environ: + self._install() + bin_path = shutil.which("pact-broker") + assert bin_path, "pact-broker not found" + self.broker_bin = bin_path + else: + msg = "pact-broker not found" + raise RuntimeError(msg) + + def _install(self) -> None: + """ + Install the Pact Broker CLI tool. + + This function is intended to be run in CI environments, where the pact-broker + CLI tool may not be installed already. This will download and extract + the tool + """ + msg = "pact-broker not found" + raise NotImplementedError(msg) + + def publish(self, directory: Path | str, version: str | None = None) -> None: + """ + Publish the interactions to the Pact Broker. + + Args: + directory: + The directory containing the pact files. + + version: + The version of the consumer application. + """ + cmd = [ + self.broker_bin, + "publish", + str(directory), + "--broker-base-url", + str(self.url), + ] + if self.username: + cmd.extend(["--broker-username", self.username]) + if self.password: + cmd.extend(["--broker-password", self.password]) + + cmd.extend(["--consumer-app-version", version or next_version()]) + + subprocess.run( + cmd, # noqa: S603 + encoding="utf-8", + check=True, + ) + + def interaction_id(self, num: int) -> str: + """ + Find the interaction ID for the given interaction. + + This function is used to find the Pact Broker interaction ID for the given + interaction. It does this by looking for the interaction with the + description `f"interaction {num}"`. + + Args: + num: + The ID of the interaction. + """ + response = requests.get( + str( + self.url + / "pacts" + / "provider" + / self.provider + / "consumer" + / self.consumer + / "latest" + ), + timeout=2, + ) + response.raise_for_status() + for interaction in response.json()["interactions"]: + if interaction["description"] == f"interaction {num}": + return interaction["_id"] + msg = f"Interaction {num} not found" + raise ValueError(msg) + + def verification_results(self, num: int) -> requests.Response: + """ + Fetch the verification results for the given interaction. + + Args: + num: + The ID of the interaction. + """ + interaction_id = self.interaction_id(num) + response = requests.get( + str( + self.url + / "pacts" + / "provider" + / self.provider + / "consumer" + / self.consumer + / "latest" + / "verification-results" + / interaction_id + ), + timeout=2, + ) + response.raise_for_status() + return response + + def latest_verification_results(self) -> requests.Response | None: + """ + Fetch the latest verification results for the provider. + + If there are no verification results, then this function will return + `None`. + """ + response = requests.get( + str( + self.url + / "pacts" + / "provider" + / self.provider + / "consumer" + / self.consumer + / "latest" + ), + timeout=2, + ) + response.raise_for_status() + links = response.json()["_links"] + response = requests.get( + links["pb:latest-verification-results"]["href"], timeout=2 + ) + if response.status_code == 404: + return None + response.raise_for_status() + return response + + +if __name__ == "__main__": + import sys + + if len(sys.argv) != 2: + sys.stderr.write(f"Usage: {sys.argv[0]} ") + sys.exit(1) + + Provider(sys.argv[1]).run() From c2a630351572366c364f96ceeba6816a77c8cef0 Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Wed, 20 Mar 2024 12:27:11 +1100 Subject: [PATCH 09/15] chore(tests): add v1 provider compatibility suite Signed-off-by: JP-Ellis --- pyproject.toml | 1 + .../compatibility_suite/test_v1_provider.py | 904 ++++++++++++++++++ .../compatibility_suite/util/pact-broker.yml | 43 + 3 files changed, 948 insertions(+) create mode 100644 tests/v3/compatibility_suite/test_v1_provider.py create mode 100644 tests/v3/compatibility_suite/util/pact-broker.yml diff --git a/pyproject.toml b/pyproject.toml index da309ede8..5bca97911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -195,6 +195,7 @@ log_date_format = "%H:%M:%S" markers = [ # Markers for the compatibility suite "consumer", + "provider", ] ################################################################################ diff --git a/tests/v3/compatibility_suite/test_v1_provider.py b/tests/v3/compatibility_suite/test_v1_provider.py new file mode 100644 index 000000000..f795f3ce0 --- /dev/null +++ b/tests/v3/compatibility_suite/test_v1_provider.py @@ -0,0 +1,904 @@ +""" +Basic HTTP provider feature test. +""" + +from __future__ import annotations + +import copy +import json +import logging +import pickle +import re +import signal +import subprocess +import sys +import time +from pathlib import Path +from threading import Thread +from typing import Any, Generator, NoReturn + +import pytest +import requests +from pytest_bdd import given, parsers, scenario, then, when +from testcontainers.compose import DockerCompose # type: ignore[import-untyped] +from yarl import URL + +from pact.v3.pact import Pact +from pact.v3.verifier import Verifier +from tests.v3.compatibility_suite.util import ( + InteractionDefinition, + parse_headers, + parse_markdown_table, +) +from tests.v3.compatibility_suite.util.provider import PactBroker + +logger = logging.getLogger(__name__) + + +@pytest.fixture() +def verifier() -> Verifier: + """Return a new Verifier.""" + return Verifier() + + +################################################################################ +## Scenario +################################################################################ + + +@scenario( + "definition/features/V1/http_provider.feature", + "Verifying a simple HTTP request", +) +def test_verifying_a_simple_http_request() -> None: + """Verifying a simple HTTP request.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Verifying multiple Pact files", +) +def test_verifying_multiple_pact_files() -> None: + """Verifying multiple Pact files.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Incorrect request is made to provider", +) +def test_incorrect_request_is_made_to_provider() -> None: + """Incorrect request is made to provider.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Verifying a simple HTTP request via a Pact broker", +) +def test_verifying_a_simple_http_request_via_a_pact_broker() -> None: + """Verifying a simple HTTP request via a Pact broker.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Verifying a simple HTTP request via a Pact broker with publishing results enabled", +) +def test_verifying_a_simple_http_request_via_a_pact_broker_with_publishing() -> None: + """Verifying a simple HTTP request via a Pact broker with publishing.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Verifying multiple Pact files via a Pact broker", +) +def test_verifying_multiple_pact_files_via_a_pact_broker() -> None: + """Verifying multiple Pact files via a Pact broker.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Incorrect request is made to provider via a Pact broker", +) +def test_incorrect_request_is_made_to_provider_via_a_pact_broker() -> None: + """Incorrect request is made to provider via a Pact broker.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Verifying an interaction with a defined provider state", +) +def test_verifying_an_interaction_with_a_defined_provider_state() -> None: + """Verifying an interaction with a defined provider state.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Verifying an interaction with no defined provider state", +) +def test_verifying_an_interaction_with_no_defined_provider_state() -> None: + """Verifying an interaction with no defined provider state.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Verifying an interaction where the provider state callback fails", +) +def test_verifying_an_interaction_where_the_provider_state_callback_fails() -> None: + """Verifying an interaction where the provider state callback fails.""" + + +# TODO: Enable this test once we can capture warnings +# https://github.com/pact-foundation/pact-reference/issues/404 +@pytest.mark.skip("Unable to get warnings to be captured") +@scenario( + "definition/features/V1/http_provider.feature", + "Verifying an interaction where a provider state callback is not configured", +) +def test_verifying_an_interaction_where_no_provider_state_callback_configured() -> None: + """Verifying an interaction where a provider state callback is not configured.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Verifying a HTTP request with a request filter configured", +) +def test_verifying_a_http_request_with_a_request_filter_configured() -> None: + """Verifying a HTTP request with a request filter configured.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Verifies the response status code", +) +def test_verifies_the_response_status_code() -> None: + """Verifies the response status code.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Verifies the response headers", +) +def test_verifies_the_response_headers() -> None: + """Verifies the response headers.""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with plain text body (positive case)", +) +def test_response_with_plain_text_body_positive_case() -> None: + """Response with plain text body (positive case).""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with plain text body (negative case)", +) +def test_response_with_plain_text_body_negative_case() -> None: + """Response with plain text body (negative case).""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with JSON body (positive case)", +) +def test_response_with_json_body_positive_case() -> None: + """Response with JSON body (positive case).""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with JSON body (negative case)", +) +def test_response_with_json_body_negative_case() -> None: + """Response with JSON body (negative case).""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with XML body (positive case)", +) +def test_response_with_xml_body_positive_case() -> None: + """Response with XML body (positive case).""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with XML body (negative case)", +) +def test_response_with_xml_body_negative_case() -> None: + """Response with XML body (negative case).""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with binary body (positive case)", +) +def test_response_with_binary_body_positive_case() -> None: + """Response with binary body (positive case).""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with binary body (negative case)", +) +def test_response_with_binary_body_negative_case() -> None: + """Response with binary body (negative case).""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with form post body (positive case)", +) +def test_response_with_form_post_body_positive_case() -> None: + """Response with form post body (positive case).""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with form post body (negative case)", +) +def test_response_with_form_post_body_negative_case() -> None: + """Response with form post body (negative case).""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with multipart body (positive case)", +) +def test_response_with_multipart_body_positive_case() -> None: + """Response with multipart body (positive case).""" + + +@scenario( + "definition/features/V1/http_provider.feature", + "Response with multipart body (negative case)", +) +def test_response_with_multipart_body_negative_case() -> None: + """Response with multipart body (negative case).""" + + +################################################################################ +## Given +################################################################################ + + +@given( + parsers.parse("the following HTTP interactions have been defined:\n{content}"), + target_fixture="interaction_definitions", + converters={"content": parse_markdown_table}, +) +def the_following_http_interactions_have_been_defined( + content: list[dict[str, str]], +) -> dict[int, InteractionDefinition]: + """ + Parse the HTTP interactions table into a dictionary. + + The table columns are expected to be: + + - No + - method + - path + - query + - headers + - body + - response + - response headers + - response content + - response body + + The first row is ignored, as it is assumed to be the column headers. The + order of the columns is similarly ignored. + """ + logger.debug("Parsing interaction definitions") + + # Check that the table is well-formed + assert len(content[0]) == 10, f"Expected 10 columns, got {len(content[0])}" + assert "No" in content[0], "'No' column not found" + + # Parse the table into a more useful format + interactions: dict[int, InteractionDefinition] = {} + for row in content: + interactions[int(row["No"])] = InteractionDefinition(**row) + return interactions + + +@given( + parsers.re( + r"a provider is started that returns the responses? " + r'from interactions? "?(?P[0-9, ]+)"?', + ), + converters={"interactions": lambda x: [int(i) for i in x.split(",") if i]}, + target_fixture="provider_url", +) +def a_provider_is_started_that_returns_the_responses_from_interactions( + interaction_definitions: dict[int, InteractionDefinition], + interactions: list[int], + temp_dir: Path, +) -> Generator[URL, None, None]: + """ + Start a provider that returns the responses from the given interactions. + """ + logger.debug("Starting provider for interactions %s", interactions) + + for i in interactions: + logger.debug("Interaction %d: %s", i, interaction_definitions[i]) + + with (temp_dir / "interactions.pkl").open("wb") as pkl_file: + pickle.dump([interaction_definitions[i] for i in interactions], pkl_file) + + yield from start_provider(temp_dir) + + +@given( + parsers.re( + r"a provider is started that returns the responses?" + r' from interactions? "?(?P[0-9, ]+)"?' + r" with the following changes:\n(?P.+)", + re.DOTALL, + ), + converters={ + "interactions": lambda x: [int(i) for i in x.split(",") if i], + "changes": parse_markdown_table, + }, + target_fixture="provider_url", +) +def a_provider_is_started_that_returns_the_responses_from_interactions_with_changes( + interaction_definitions: dict[int, InteractionDefinition], + interactions: list[int], + changes: list[dict[str, str]], + temp_dir: Path, +) -> Generator[URL, None, None]: + """ + Start a provider that returns the responses from the given interactions. + """ + logger.debug("Starting provider for interactions %s", interactions) + + assert len(changes) == 1, "Only one set of changes is supported" + defns: list[InteractionDefinition] = [] + for interaction in interactions: + defn = copy.deepcopy(interaction_definitions[interaction]) + defn.update(**changes[0]) + defns.append(defn) + logger.debug( + "Update interaction %d: %s", + interaction, + defn, + ) + + with (temp_dir / "interactions.pkl").open("wb") as pkl_file: + pickle.dump(defns, pkl_file) + + yield from start_provider(temp_dir) + + +def start_provider(provider_dir: str | Path) -> Generator[URL, None, None]: # noqa: C901 + """Start the provider app with the given interactions.""" + process = subprocess.Popen( + [ # noqa: S603 + sys.executable, + Path(__file__).parent / "util" / "provider.py", + str(provider_dir), + ], + cwd=Path.cwd(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + ) + + pattern = re.compile(r" \* Running on (?P[^ ]+)") + while True: + if process.poll() is not None: + logger.error("Provider process exited with code %d", process.returncode) + logger.error( + "Provider stdout: %s", process.stdout.read() if process.stdout else "" + ) + logger.error( + "Provider stderr: %s", process.stderr.read() if process.stderr else "" + ) + msg = f"Provider process exited with code {process.returncode}" + raise RuntimeError(msg) + if ( + process.stderr + and (line := process.stderr.readline()) + and (match := pattern.match(line)) + ): + break + time.sleep(0.1) + + url = URL(match.group("url")) + logger.debug("Provider started on %s", url) + for _ in range(50): + try: + response = requests.get(str(url / "_test" / "ping"), timeout=1) + assert response.text == "pong" + break + except (requests.RequestException, AssertionError): + time.sleep(0.1) + continue + else: + msg = "Failed to ping provider" + raise RuntimeError(msg) + + def redirect() -> NoReturn: + while True: + if process.stdout: + while line := process.stdout.readline(): + logger.debug("Provider stdout: %s", line) + if process.stderr: + while line := process.stderr.readline(): + logger.debug("Provider stderr: %s", line) + + thread = Thread(target=redirect, daemon=True) + thread.start() + + yield url + + process.send_signal(signal.SIGINT) + + +@given( + parsers.re( + r"a Pact file for interaction (?P\d+) is to be verified", + ), + converters={"interaction": int}, +) +def a_pact_file_for_interaction_is_to_be_verified( + interaction_definitions: dict[int, InteractionDefinition], + verifier: Verifier, + interaction: int, + temp_dir: Path, +) -> None: + """ + Verify the Pact file for the given interaction. + """ + logger.debug( + "Adding interaction %d to be verified: %s", + interaction, + interaction_definitions[interaction], + ) + + defn = interaction_definitions[interaction] + + pact = Pact("consumer", "provider") + pact.with_specification("V1") + defn.add_to_pact(pact, f"interaction {interaction}") + (temp_dir / "pacts").mkdir(exist_ok=True, parents=True) + pact.write_file(temp_dir / "pacts") + + verifier.add_source(temp_dir / "pacts") + + +@given( + parsers.re( + r"a Pact file for interaction (?P\d+)" + r" is to be verified from a Pact broker", + ), + converters={"interaction": int}, + target_fixture="pact_broker", +) +def a_pact_file_for_interaction_is_to_be_verified_from_a_pact_broker( + interaction_definitions: dict[int, InteractionDefinition], + verifier: Verifier, + interaction: int, + temp_dir: Path, +) -> Generator[PactBroker, None, None]: + """ + Verify the Pact file for the given interaction from a Pact broker. + """ + logger.debug("Adding interaction %d to be verified from a Pact broker", interaction) + + defn = interaction_definitions[interaction] + + pact = Pact("consumer", "provider") + pact.with_specification("V1") + defn.add_to_pact(pact, f"interaction {interaction}") + + pacts_dir = temp_dir / "pacts" + pacts_dir.mkdir(exist_ok=True, parents=True) + pact.write_file(pacts_dir) + + with DockerCompose( + Path(__file__).parent / "util", + compose_file_name="pact-broker.yml", + pull=True, + ) as _: + pact_broker = PactBroker(URL("http://pactbroker:pactbroker@localhost:9292")) + pact_broker.publish(pacts_dir) + verifier.broker_source(pact_broker.url) + yield pact_broker + + +@given("publishing of verification results is enabled") +def publishing_of_verification_results_is_enabled(verifier: Verifier) -> None: + """ + Enable publishing of verification results. + """ + logger.debug("Publishing verification results") + + verifier.set_publish_options( + "0.0.0", + ) + + +@given( + parsers.re( + r"a provider state callback is configured" + r"(?P(, but will return a failure)?)", + ), + converters={"failure": lambda x: x != ""}, +) +def a_provider_state_callback_is_configured( + verifier: Verifier, + provider_url: URL, + temp_dir: Path, + failure: bool, # noqa: FBT001 +) -> None: + """ + Configure a provider state callback. + """ + logger.debug("Configuring provider state callback") + + if failure: + with (temp_dir / "fail_callback").open("w") as f: + f.write("true") + + verifier.set_state( + provider_url / "_test" / "callback", + teardown=True, + ) + + +@given( + parsers.re( + r"a Pact file for interaction (?P\d+) is to be verified" + r' with a provider state "(?P[^"]+)" defined', + ), + converters={"interaction": int}, +) +def a_pact_file_for_interaction_is_to_be_verified_with_a_provider_state_define( + interaction_definitions: dict[int, InteractionDefinition], + verifier: Verifier, + interaction: int, + state: str, + temp_dir: Path, +) -> None: + """ + Verify the Pact file for the given interaction with a provider state defined. + """ + logger.debug( + "Adding interaction %d to be verified with provider state %s", + interaction, + state, + ) + + defn = interaction_definitions[interaction] + defn.state = state + + pact = Pact("consumer", "provider") + pact.with_specification("V1") + defn.add_to_pact(pact, f"interaction {interaction}") + (temp_dir / "pacts").mkdir(exist_ok=True, parents=True) + pact.write_file(temp_dir / "pacts") + + verifier.add_source(temp_dir / "pacts") + + with (temp_dir / "provider_state").open("w") as f: + logger.debug("Writing provider state to %s", temp_dir / "provider_state") + f.write(state) + + +@given( + parsers.parse( + "a request filter is configured to make the following changes:\n{content}" + ), + converters={"content": parse_markdown_table}, +) +def a_request_filter_is_configured_to_make_the_following_changes( + content: list[dict[str, str]], + verifier: Verifier, +) -> None: + """ + Configure a request filter to make the given changes. + """ + logger.debug("Configuring request filter") + + if "headers" in content[0]: + verifier.add_custom_headers(parse_headers(content[0]["headers"]).items()) + else: + msg = "Unsupported filter type" + raise RuntimeError(msg) + + +################################################################################ +## When +################################################################################ + + +@when("the verification is run", target_fixture="verifier_result") +def the_verification_is_run( + verifier: Verifier, + provider_url: URL, +) -> tuple[Verifier, Exception | None]: + """ + Run the verification. + """ + logger.debug("Running verification on %r", verifier) + + verifier.set_info("provider", url=provider_url) + try: + verifier.verify() + except Exception as e: # noqa: BLE001 + return verifier, e + return verifier, None + + +################################################################################ +## Then +################################################################################ + + +@then( + parsers.re(r"the verification will(?P( NOT)?) be successful"), + converters={"negated": lambda x: x == " NOT"}, +) +def the_verification_will_be_successful( + verifier_result: tuple[Verifier, Exception | None], + negated: bool, # noqa: FBT001 +) -> None: + """ + Check that the verification was successful. + """ + logger.debug("Checking verification result") + logger.debug("Verifier result: %s", verifier_result) + + if negated: + assert verifier_result[1] is not None + else: + assert verifier_result[1] is None + + +@then( + parsers.re(r'the verification results will contain a "(?P[^"]+)" error'), +) +def the_verification_results_will_contain_a_error( + verifier_result: tuple[Verifier, Exception | None], error: str +) -> None: + """ + Check that the verification results contain the given error. + """ + logger.debug("Checking that verification results contain error %s", error) + + verifier = verifier_result[0] + logger.debug("Verification results: %s", json.dumps(verifier.results, indent=2)) + + if error == "Response status did not match": + mismatch_type = "StatusMismatch" + elif error == "Headers had differences": + mismatch_type = "HeaderMismatch" + elif error == "Body had differences": + mismatch_type = "BodyMismatch" + elif error == "State change request failed": + assert "One or more of the setup state change handlers has failed" in [ + error["mismatch"]["message"] for error in verifier.results["errors"] + ] + return + else: + msg = f"Unknown error type: {error}" + raise ValueError(msg) + + assert mismatch_type in [ + mismatch["type"] + for error in verifier.results["errors"] + for mismatch in error["mismatch"]["mismatches"] + ] + + +@then( + parsers.re(r"a verification result will NOT be published back"), +) +def a_verification_result_will_not_be_published_back(pact_broker: PactBroker) -> None: + """ + Check that the verification result was published back to the Pact broker. + """ + logger.debug("Checking that verification result was not published back") + + response = pact_broker.latest_verification_results() + if response: + with pytest.raises(requests.HTTPError, match="404 Client Error"): + response.raise_for_status() + + +@then( + parsers.re( + "a successful verification result " + "will be published back " + r"for interaction \{(?P\d+)\}", + ), + converters={"interaction": int}, +) +def a_successful_verification_result_will_be_published_back( + pact_broker: PactBroker, + interaction: int, +) -> None: + """ + Check that the verification result was published back to the Pact broker. + """ + logger.debug( + "Checking that verification result was published back for interaction %d", + interaction, + ) + + interaction_id = pact_broker.interaction_id(interaction) + response = pact_broker.latest_verification_results() + assert response is not None + assert response.ok + data: dict[str, Any] = response.json() + assert data["success"] + + for test_result in data["testResults"]: + if test_result["interactionId"] == interaction_id: + assert test_result["success"] + break + else: + msg = f"Interaction {interaction} not found in verification results" + raise ValueError(msg) + + +@then( + parsers.re( + "a failed verification result " + "will be published back " + r"for the interaction \{(?P\d+)\}", + ), + converters={"interaction": int}, +) +def a_failed_verification_result_will_be_published_back( + pact_broker: PactBroker, + interaction: int, +) -> None: + """ + Check that the verification result was published back to the Pact broker. + """ + logger.debug( + "Checking that failed verification result" + " was published back for interaction %d", + interaction, + ) + + interaction_id = pact_broker.interaction_id(interaction) + response = pact_broker.latest_verification_results() + assert response is not None + assert response.ok + data: dict[str, Any] = response.json() + assert not data["success"] + + for test_result in data["testResults"]: + if test_result["interactionId"] == interaction_id: + assert not test_result["success"] + break + else: + msg = f"Interaction {interaction} not found in verification results" + raise ValueError(msg) + + +@then("the provider state callback will be called before the verification is run") +def the_provider_state_callback_will_be_called_before_the_verification_is_run() -> None: + """ + Check that the provider state callback was called before the verification was run. + """ + logger.debug("Checking provider state callback was called before verification") + + +@then( + parsers.re( + r"the provider state callback will receive a (?Psetup|teardown) call" + r' (with )?"(?P[^"]*)" as the provider state parameter', + ), +) +def the_provider_state_callback_will_receive_a_setup_call( + temp_dir: Path, + action: str, + state: str, +) -> None: + """ + Check that the provider state callback received a setup call. + """ + logger.info("Checking provider state callback received a %s call", action) + logger.info("Callback files: %s", list(temp_dir.glob("callback.*.json"))) + for file in temp_dir.glob("callback.*.json"): + with file.open("r") as f: + data: dict[str, Any] = json.load(f) + logger.debug("Checking callback data: %s", data) + if ( + "action" in data["query_params"] + and data["query_params"]["action"] == action + and data["query_params"]["state"] == state + ): + break + else: + msg = f"No {action} call found" + raise AssertionError(msg) + + +@then( + parsers.re( + r"the provider state callback will " + r"NOT receive a (?Psetup|teardown) call" + ) +) +def the_provider_state_callback_will_not_receive_a_setup_call( + temp_dir: Path, + action: str, +) -> None: + """ + Check that the provider state callback did not receive a setup call. + """ + for file in temp_dir.glob("callback.*.json"): + with file.open("r") as f: + data: dict[str, Any] = json.load(f) + logger.debug("Checking callback data: %s", data) + if ( + "action" in data["query_params"] + and data["query_params"]["action"] == action + ): + msg = f"Unexpected {action} call found" + raise AssertionError(msg) + + +@then("the provider state callback will be called after the verification is run") +def the_provider_state_callback_will_be_called_after_the_verification_is_run() -> None: + """ + Check that the provider state callback was called after the verification was run. + """ + + +@then( + parsers.re( + r"a warning will be displayed " + r"that there was no provider state callback configured " + r'for provider state "(?P[^"]*)"', + ) +) +def a_warning_will_be_displayed_that_there_was_no_callback_configured( + verifier_result: tuple[Verifier, Exception | None], + state: str, # noqa: ARG001 +) -> None: + """ + Check that a warning was displayed that there was no callback configured. + """ + logger.debug("Checking for warning about missing provider state callback") + verifier = verifier_result[0] + logger.debug("verifier output: %s", verifier.output(strip_ansi=True)) + logger.debug("verifier results: %s", json.dumps(verifier.results, indent=2)) + msg = "Not implemented" + raise NotImplementedError(msg) + + +@then( + parsers.re( + r'the request to the provider will contain the header "(?P
[^"]+)"', + ), + converters={"header": lambda x: parse_headers(f"'{x}'")}, +) +def the_request_to_the_provider_will_contain_the_header( + verifier_result: tuple[Verifier, Exception | None], + header: dict[str, str], + temp_dir: Path, +) -> None: + """ + Check that the request to the provider contained the given header. + """ + verifier = verifier_result[0] + logger.debug("verifier output: %s", verifier.output(strip_ansi=True)) + logger.debug("verifier results: %s", json.dumps(verifier.results, indent=2)) + for request in temp_dir.glob("request.*.json"): + with request.open("r") as f: + data: dict[str, Any] = json.load(f) + if data["path"].startswith("/_test"): + continue + logger.debug("Checking request data: %s", data) + assert all([k, v] in data["headers_list"] for k, v in header.items()) + break + else: + msg = "No request found" + raise AssertionError(msg) diff --git a/tests/v3/compatibility_suite/util/pact-broker.yml b/tests/v3/compatibility_suite/util/pact-broker.yml new file mode 100644 index 000000000..53b3f6d72 --- /dev/null +++ b/tests/v3/compatibility_suite/util/pact-broker.yml @@ -0,0 +1,43 @@ +version: "3.9" + +services: + postgres: + image: postgres + ports: + - "5432:5432" + healthcheck: + test: psql postgres -U postgres --command 'SELECT 1' + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + + broker: + image: pactfoundation/pact-broker:latest + depends_on: + - postgres + ports: + - "9292:9292" + restart: always + environment: + # Basic auth credentials for the Broker + PACT_BROKER_ALLOW_PUBLIC_READ: "true" + PACT_BROKER_BASIC_AUTH_USERNAME: pactbroker + PACT_BROKER_BASIC_AUTH_PASSWORD: pactbroker + # Database + PACT_BROKER_DATABASE_URL: "postgres://postgres:postgres@postgres/postgres" + # PACT_BROKER_DATABASE_URL: sqlite:////tmp/pact_broker.sqlite # Pending pact-foundation/pact-broker-docker#148 + + healthcheck: + test: + [ + "CMD", + "curl", + "--silent", + "--show-error", + "--fail", + "http://pactbroker:pactbroker@localhost:9292/diagnostic/status/heartbeat", + ] + interval: 1s + timeout: 2s + retries: 5 From 8da7d3d9d58d9ff021cab6b081a9738d91db6f01 Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Wed, 20 Mar 2024 13:57:37 +1100 Subject: [PATCH 10/15] chore(tests): fixes for lower python versions Signed-off-by: JP-Ellis --- tests/v3/compatibility_suite/util/__init__.py | 4 ++-- tests/v3/compatibility_suite/util/provider.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/v3/compatibility_suite/util/__init__.py b/tests/v3/compatibility_suite/util/__init__.py index 4c4240d90..50e149c73 100644 --- a/tests/v3/compatibility_suite/util/__init__.py +++ b/tests/v3/compatibility_suite/util/__init__.py @@ -173,11 +173,11 @@ def serialize(obj: Any) -> Any: # noqa: ANN401, PLR0911 All other types are converted to strings using the `repr` function. """ - if isinstance(obj, datetime | date | time): + if isinstance(obj, (datetime, date, time)): return obj.isoformat() # Basic types which are already serializable - if isinstance(obj, str | int | float | bool | type(None)): + if isinstance(obj, (str, int, float, bool, type(None))): return obj # Bytes diff --git a/tests/v3/compatibility_suite/util/provider.py b/tests/v3/compatibility_suite/util/provider.py index 6ad1d38b3..1f7b86c09 100644 --- a/tests/v3/compatibility_suite/util/provider.py +++ b/tests/v3/compatibility_suite/util/provider.py @@ -23,7 +23,7 @@ import socket import subprocess from contextvars import ContextVar -from datetime import UTC, datetime +from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING @@ -38,6 +38,13 @@ if TYPE_CHECKING: from tests.v3.compatibility_suite.util import InteractionDefinition +if sys.version_info < (3, 11): + from datetime import timezone + + UTC = timezone.utc +else: + from datetime import UTC + logger = logging.getLogger(__name__) From 388242dafd95fd25d416bc7848107e09c5056fc7 Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Wed, 20 Mar 2024 14:29:16 +1100 Subject: [PATCH 11/15] chore(tests): re-enable warning check Following recommendations from Ron, making the check a no-op. Ref: pact-foundation/pact-reference#404 Signed-off-by: JP-Ellis --- tests/v3/compatibility_suite/test_v1_provider.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/v3/compatibility_suite/test_v1_provider.py b/tests/v3/compatibility_suite/test_v1_provider.py index f795f3ce0..359d135f9 100644 --- a/tests/v3/compatibility_suite/test_v1_provider.py +++ b/tests/v3/compatibility_suite/test_v1_provider.py @@ -126,9 +126,6 @@ def test_verifying_an_interaction_where_the_provider_state_callback_fails() -> N """Verifying an interaction where the provider state callback fails.""" -# TODO: Enable this test once we can capture warnings -# https://github.com/pact-foundation/pact-reference/issues/404 -@pytest.mark.skip("Unable to get warnings to be captured") @scenario( "definition/features/V1/http_provider.feature", "Verifying an interaction where a provider state callback is not configured", @@ -860,18 +857,13 @@ def the_provider_state_callback_will_be_called_after_the_verification_is_run() - ) ) def a_warning_will_be_displayed_that_there_was_no_callback_configured( - verifier_result: tuple[Verifier, Exception | None], - state: str, # noqa: ARG001 + state: str, ) -> None: """ Check that a warning was displayed that there was no callback configured. """ logger.debug("Checking for warning about missing provider state callback") - verifier = verifier_result[0] - logger.debug("verifier output: %s", verifier.output(strip_ansi=True)) - logger.debug("verifier results: %s", json.dumps(verifier.results, indent=2)) - msg = "Not implemented" - raise NotImplementedError(msg) + assert state @then( From 20d96a0773ff3ca6207dbbf63e7556cdc571ff96 Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Thu, 21 Mar 2024 15:07:07 +1100 Subject: [PATCH 12/15] chore(tests): improve logging from provider As the provider is launched in its own Python process, logging is not configured. So instead of using various `logging` methods, directly write to `stderr`. Signed-off-by: JP-Ellis --- .../compatibility_suite/test_v1_provider.py | 4 +-- tests/v3/compatibility_suite/util/__init__.py | 16 ++++++++-- tests/v3/compatibility_suite/util/provider.py | 30 +++++++++---------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/tests/v3/compatibility_suite/test_v1_provider.py b/tests/v3/compatibility_suite/test_v1_provider.py index 359d135f9..9b216962d 100644 --- a/tests/v3/compatibility_suite/test_v1_provider.py +++ b/tests/v3/compatibility_suite/test_v1_provider.py @@ -420,10 +420,10 @@ def redirect() -> NoReturn: while True: if process.stdout: while line := process.stdout.readline(): - logger.debug("Provider stdout: %s", line) + logger.debug("Provider stdout: %s", line.strip()) if process.stderr: while line := process.stderr.readline(): - logger.debug("Provider stderr: %s", line) + logger.debug("Provider stderr: %s", line.strip()) thread = Thread(target=redirect, daemon=True) thread.start() diff --git a/tests/v3/compatibility_suite/util/__init__.py b/tests/v3/compatibility_suite/util/__init__.py index 50e149c73..1f47080b2 100644 --- a/tests/v3/compatibility_suite/util/__init__.py +++ b/tests/v3/compatibility_suite/util/__init__.py @@ -25,6 +25,7 @@ def _(): import contextlib import hashlib import logging +import sys import typing from collections.abc import Collection, Mapping from datetime import date, datetime, time @@ -555,9 +556,18 @@ def add_to_flask(self, app: flask.Flask) -> None: app: The Flask app to add the interaction to. """ - - async def route_fn() -> flask.Response: - logger.info("Received request: %s %s", self.method, self.path) + sys.stderr.write( + f"Adding interaction to Flask app: {self.method} {self.path}\n" + ) + sys.stderr.write(f" Query: {self.query}\n") + sys.stderr.write(f" Headers: {self.headers}\n") + sys.stderr.write(f" Body: {self.body}\n") + sys.stderr.write(f" Response: {self.response}\n") + sys.stderr.write(f" Response headers: {self.response_headers}\n") + sys.stderr.write(f" Response body: {self.response_body}\n") + + def route_fn() -> flask.Response: + sys.stderr.write(f"Received request: {self.method} {self.path}\n") if self.query: query = URL.build(query_string=self.query).query # Perform a two-way check to ensure that the query parameters diff --git a/tests/v3/compatibility_suite/util/provider.py b/tests/v3/compatibility_suite/util/provider.py index 1f7b86c09..f90c35902 100644 --- a/tests/v3/compatibility_suite/util/provider.py +++ b/tests/v3/compatibility_suite/util/provider.py @@ -180,14 +180,12 @@ def _add_after_request(self, app: flask.Flask) -> None: @app.after_request def log_request(response: flask.Response) -> flask.Response: - logger.debug("Request: %s %s", request.method, request.path) - logger.debug( - "Request query string: %s", request.query_string.decode("utf-8") - ) - logger.debug("Request query params: %s", serialize(request.args)) - logger.debug("Request headers: %s", serialize(request.headers)) - logger.debug("Request body: %s", request.data.decode("utf-8")) - logger.debug("Request form: %s", serialize(request.form)) + sys.stderr.write(f"START REQUEST: {request.method} {request.path}\n") + sys.stderr.write(f"Query string: {request.query_string.decode('utf-8')}\n") + sys.stderr.write(f"Header: {serialize(request.headers)}\n") + sys.stderr.write(f"Body: {request.data.decode('utf-8')}\n") + sys.stderr.write(f"Form: {serialize(request.form)}\n") + sys.stderr.write("END REQUEST\n") with ( self.provider_dir @@ -210,13 +208,13 @@ def log_request(response: flask.Response) -> flask.Response: @app.after_request def log_response(response: flask.Response) -> flask.Response: - try: - body = response.get_data(as_text=True) - except UnicodeDecodeError: - body = "" - logger.debug("Response: %s", response.status_code) - logger.debug("Response headers: %s", serialize(response.headers)) - logger.debug("Response body: %s", body) + sys.stderr.write(f"START RESPONSE: {response.status_code}\n") + sys.stderr.write(f"Headers: {serialize(response.headers)}\n") + sys.stderr.write( + f"Body: {response.get_data().decode('utf-8', errors='replace')}\n" + ) + sys.stderr.write("END RESPONSE\n") + with ( self.provider_dir / f"response.{datetime.now(tz=UTC).strftime('%H:%M:%S.%f')}.json" @@ -226,7 +224,7 @@ def log_response(response: flask.Response) -> flask.Response: "status_code": response.status_code, "headers_list": serialize(response.headers), "headers_dict": serialize(dict(response.headers)), - "body": body, + "body": response.get_data().decode("utf-8", errors="replace"), }, f, ) From 63b30f9259cf017af1b2497600b6baf084734a5b Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Thu, 21 Mar 2024 17:17:58 +1100 Subject: [PATCH 13/15] chore(test): strip authentication from url Signed-off-by: JP-Ellis --- tests/v3/compatibility_suite/util/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/v3/compatibility_suite/util/provider.py b/tests/v3/compatibility_suite/util/provider.py index f90c35902..0f4d4be62 100644 --- a/tests/v3/compatibility_suite/util/provider.py +++ b/tests/v3/compatibility_suite/util/provider.py @@ -315,7 +315,7 @@ def publish(self, directory: Path | str, version: str | None = None) -> None: "publish", str(directory), "--broker-base-url", - str(self.url), + str(self.url.with_user(None).with_password(None)), ] if self.username: cmd.extend(["--broker-username", self.username]) From edcf260bb94c8bd54d80f1ed82479f82c3e39676 Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Fri, 22 Mar 2024 11:05:19 +1100 Subject: [PATCH 14/15] chore(tests): use long-lived pact broker The initial implementation of the compatibility suite spun up and down the Pact Broker for each scenario, which also resulting in flaky tests in CI. This refactor uses a session pytest fixture which will spin up the broker once, and keep re-using it. Functionality to 'reset' the broker between tests has also been added. Signed-off-by: JP-Ellis --- .github/workflows/test.yml | 65 ++++++++++++++++-- conftest.py | 13 ++++ pyproject.toml | 3 + .../compatibility_suite/test_v1_provider.py | 68 ++++++++++++++++--- tests/v3/compatibility_suite/util/provider.py | 19 ++++++ 5 files changed, 154 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0695b7ff2..16213fb08 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,17 +18,30 @@ env: HATCH_VERBOSE: 1 jobs: - test: + test-container: name: >- Tests py${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} continue-on-error: ${{ matrix.experimental }} + services: + broker: + image: pactfoundation/pact-broker:latest@sha256:8f10947f230f661ef21f270a4abcf53214ba27cd68063db81de555fcd93e07dd + ports: + - "9292:9292" + env: + # Basic auth credentials for the Broker + PACT_BROKER_ALLOW_PUBLIC_READ: "true" + PACT_BROKER_BASIC_AUTH_USERNAME: pactbroker + PACT_BROKER_BASIC_AUTH_PASSWORD: pactbroker + # Database + PACT_BROKER_DATABASE_URL: sqlite:////tmp/pact_broker.sqlite + strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] experimental: [false] include: @@ -51,8 +64,20 @@ jobs: - name: Install Hatch run: pip install --upgrade hatch + - name: Ensure broker is live + run: | + i=0 + until curl -sSf http://localhost:9292/diagnostic/status/heartbeat; do + i=$((i+1)) + if [ $i -gt 120 ]; then + echo "Broker failed to start" + exit 1 + fi + sleep 1 + done + - name: Run tests - run: hatch run test + run: hatch run test --broker-url=http://pactbroker:pactbroker@localhost:9292 --container - name: Upload coverage # TODO: Configure code coverage monitoring @@ -61,12 +86,44 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} + test-no-container: + name: >- + Tests py${{ matrix.python-version }} on ${{ matrix.os }} + + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.experimental }} + + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + experimental: [false] + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + with: + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install Hatch + run: pip install --upgrade hatch + + - name: Run tests + run: hatch run test + test-conlusion: name: Test matrix complete runs-on: ubuntu-latest needs: - - test + - test-container + - test-no-container steps: - run: echo "Test matrix completed successfully." diff --git a/conftest.py b/conftest.py index f6b59a455..359b5e916 100644 --- a/conftest.py +++ b/conftest.py @@ -19,3 +19,16 @@ def pytest_addoption(parser: pytest.Parser) -> None: ), type=str, ) + parser.addoption( + "--container", + action="store_true", + help="Run tests using a container", + ) + + +def pytest_runtest_setup(item: pytest.Item) -> None: + """ + Hook into the setup phase of tests. + """ + if "container" in item.keywords and not item.config.getoption("--container"): + pytest.skip("need --container to run this test") diff --git a/pyproject.toml b/pyproject.toml index 5bca97911..0bc02ad05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -193,6 +193,9 @@ log_format = "%(asctime)s.%(msecs)03d [%(levelname)-8s] %(name)s: %(message)s" log_date_format = "%H:%M:%S" markers = [ + # Marker for tests that require a container + "container", + # Markers for the compatibility suite "consumer", "provider", diff --git a/tests/v3/compatibility_suite/test_v1_provider.py b/tests/v3/compatibility_suite/test_v1_provider.py index 9b216962d..880847d1a 100644 --- a/tests/v3/compatibility_suite/test_v1_provider.py +++ b/tests/v3/compatibility_suite/test_v1_provider.py @@ -13,9 +13,10 @@ import subprocess import sys import time +from contextvars import ContextVar from pathlib import Path from threading import Thread -from typing import Any, Generator, NoReturn +from typing import Any, Generator, NoReturn, Union import pytest import requests @@ -34,6 +35,16 @@ logger = logging.getLogger(__name__) +reset_broker_var = ContextVar("reset_broker", default=True) +""" +This context variable is used to determine whether the Pact broker should be +cleaned up. It is used to ensure that the broker is only cleaned up once, even +if a step is run multiple times. + +All scenarios which make use of the Pact broker should set this to `True` at the +start of the scenario. +""" + @pytest.fixture() def verifier() -> Verifier: @@ -41,6 +52,35 @@ def verifier() -> Verifier: return Verifier() +@pytest.fixture(scope="session") +def broker_url(request: pytest.FixtureRequest) -> Generator[URL, Any, None]: + """ + Fixture to run the Pact broker. + + This inspects whether the `--broker-url` option has been given. If it has, + it is assumed that the broker is already running and simply returns the + given URL. + + Otherwise, the Pact broker is started in a container. The URL of the + containerised broker is then returned. + """ + broker_url: Union[str, None] = request.config.getoption("--broker-url") + + # If we have been given a broker URL, there's nothing more to do here and we + # can return early. + if broker_url: + yield URL(broker_url) + return + + with DockerCompose( + Path(__file__).parent / "util", + compose_file_name="pact-broker.yml", + pull=True, + ) as _: + yield URL("http://pactbroker:pactbroker@localhost:9292") + return + + ################################################################################ ## Scenario ################################################################################ @@ -70,36 +110,44 @@ def test_incorrect_request_is_made_to_provider() -> None: """Incorrect request is made to provider.""" +@pytest.mark.container() @scenario( "definition/features/V1/http_provider.feature", "Verifying a simple HTTP request via a Pact broker", ) def test_verifying_a_simple_http_request_via_a_pact_broker() -> None: """Verifying a simple HTTP request via a Pact broker.""" + reset_broker_var.set(True) # noqa: FBT003 +@pytest.mark.container() @scenario( "definition/features/V1/http_provider.feature", "Verifying a simple HTTP request via a Pact broker with publishing results enabled", ) def test_verifying_a_simple_http_request_via_a_pact_broker_with_publishing() -> None: """Verifying a simple HTTP request via a Pact broker with publishing.""" + reset_broker_var.set(True) # noqa: FBT003 +@pytest.mark.container() @scenario( "definition/features/V1/http_provider.feature", "Verifying multiple Pact files via a Pact broker", ) def test_verifying_multiple_pact_files_via_a_pact_broker() -> None: """Verifying multiple Pact files via a Pact broker.""" + reset_broker_var.set(True) # noqa: FBT003 +@pytest.mark.container() @scenario( "definition/features/V1/http_provider.feature", "Incorrect request is made to provider via a Pact broker", ) def test_incorrect_request_is_made_to_provider_via_a_pact_broker() -> None: """Incorrect request is made to provider via a Pact broker.""" + reset_broker_var.set(True) # noqa: FBT003 @scenario( @@ -475,6 +523,7 @@ def a_pact_file_for_interaction_is_to_be_verified( ) def a_pact_file_for_interaction_is_to_be_verified_from_a_pact_broker( interaction_definitions: dict[int, InteractionDefinition], + broker_url: URL, verifier: Verifier, interaction: int, temp_dir: Path, @@ -494,15 +543,14 @@ def a_pact_file_for_interaction_is_to_be_verified_from_a_pact_broker( pacts_dir.mkdir(exist_ok=True, parents=True) pact.write_file(pacts_dir) - with DockerCompose( - Path(__file__).parent / "util", - compose_file_name="pact-broker.yml", - pull=True, - ) as _: - pact_broker = PactBroker(URL("http://pactbroker:pactbroker@localhost:9292")) - pact_broker.publish(pacts_dir) - verifier.broker_source(pact_broker.url) - yield pact_broker + pact_broker = PactBroker(broker_url) + if reset_broker_var.get(): + logger.debug("Resetting Pact broker") + pact_broker.reset() + reset_broker_var.set(False) # noqa: FBT003 + pact_broker.publish(pacts_dir) + verifier.broker_source(pact_broker.url) + yield pact_broker @given("publishing of verification results is enabled") diff --git a/tests/v3/compatibility_suite/util/provider.py b/tests/v3/compatibility_suite/util/provider.py index 0f4d4be62..2968152b0 100644 --- a/tests/v3/compatibility_suite/util/provider.py +++ b/tests/v3/compatibility_suite/util/provider.py @@ -299,6 +299,25 @@ def _install(self) -> None: msg = "pact-broker not found" raise NotImplementedError(msg) + def reset(self) -> None: + """ + Reset the Pact Broker. + + This function will reset the Pact Broker by deleting all pacts and + verification results. + """ + requests.delete( + str( + self.url + / "integrations" + / "provider" + / self.provider + / "consumer" + / self.consumer + ), + timeout=2, + ) + def publish(self, directory: Path | str, version: str | None = None) -> None: """ Publish the interactions to the Pact Broker. From 5addbc1a1096cb44a52159b61e6273b9f83b0aee Mon Sep 17 00:00:00 2001 From: JP-Ellis Date: Fri, 22 Mar 2024 12:12:37 +1100 Subject: [PATCH 15/15] chore(test): apply a temporary diff to compatibility suite Signed-off-by: JP-Ellis --- .github/workflows/test.yml | 12 +++ .../definition-update.diff | 79 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 tests/v3/compatibility_suite/definition-update.diff diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 16213fb08..7e56120ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,6 +55,12 @@ jobs: with: submodules: true + - name: Apply temporary definitions update + shell: bash + run: | + cd tests/v3/compatibility_suite + patch -p1 -d definition < definition-update.diff + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: @@ -105,6 +111,12 @@ jobs: with: submodules: true + - name: Apply temporary definitions update + shell: bash + run: | + cd tests/v3/compatibility_suite + patch -p1 -d definition < definition-update.diff + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: diff --git a/tests/v3/compatibility_suite/definition-update.diff b/tests/v3/compatibility_suite/definition-update.diff new file mode 100644 index 000000000..23538b1ab --- /dev/null +++ b/tests/v3/compatibility_suite/definition-update.diff @@ -0,0 +1,79 @@ +diff --git a/features/V1/http_provider.feature b/features/V1/http_provider.feature +index 94fda44..2838116 100644 +--- a/features/V1/http_provider.feature ++++ b/features/V1/http_provider.feature +@@ -118,16 +118,16 @@ Feature: Basic HTTP provider + + Scenario: Verifies the response status code + Given a provider is started that returns the response from interaction 1, with the following changes: +- | status | +- | 400 | ++ | response | ++ | 400 | + And a Pact file for interaction 1 is to be verified + When the verification is run + Then the verification will NOT be successful + And the verification results will contain a "Response status did not match" error + + Scenario: Verifies the response headers +- Given a provider is started that returns the response from interaction 1, with the following changes: +- | headers | ++ Given a provider is started that returns the response from interaction 5, with the following changes: ++ | response headers | + | 'X-TEST: Compatibility' | + And a Pact file for interaction 5 is to be verified + When the verification is run +@@ -142,7 +142,7 @@ Feature: Basic HTTP provider + + Scenario: Response with plain text body (negative case) + Given a provider is started that returns the response from interaction 6, with the following changes: +- | body | ++ | response body | + | Hello Compatibility Suite! | + And a Pact file for interaction 6 is to be verified + When the verification is run +@@ -157,7 +157,7 @@ Feature: Basic HTTP provider + + Scenario: Response with JSON body (negative case) + Given a provider is started that returns the response from interaction 1, with the following changes: +- | body | ++ | response body | + | JSON: { "one": 100, "two": "b" } | + And a Pact file for interaction 1 is to be verified + When the verification is run +@@ -172,7 +172,7 @@ Feature: Basic HTTP provider + + Scenario: Response with XML body (negative case) + Given a provider is started that returns the response from interaction 7, with the following changes: +- | body | ++ | response body | + | XML: A | + And a Pact file for interaction 7 is to be verified + When the verification is run +@@ -187,7 +187,7 @@ Feature: Basic HTTP provider + + Scenario: Response with binary body (negative case) + Given a provider is started that returns the response from interaction 8, with the following changes: +- | body | ++ | response body | + | file: spider.jpg | + And a Pact file for interaction 8 is to be verified + When the verification is run +@@ -202,7 +202,7 @@ Feature: Basic HTTP provider + + Scenario: Response with form post body (negative case) + Given a provider is started that returns the response from interaction 9, with the following changes: +- | body | ++ | response body | + | a=1&b=2&c=33&d=4 | + And a Pact file for interaction 9 is to be verified + When the verification is run +@@ -217,7 +217,7 @@ Feature: Basic HTTP provider + + Scenario: Response with multipart body (negative case) + Given a provider is started that returns the response from interaction 10, with the following changes: +- | body | ++ | response body | + | file: multipart2-body.xml | + And a Pact file for interaction 10 is to be verified + When the verification is run