WIP

Makefile

@@ -57,6 +57,14 @@ fast_fix: $(VIRTUAL_ENV)
 fix: fast_fix
 	djlint --reformat pilotage
 
+# Tests.
+# =============================================================================
+
+.PHONY: test
+
+test: $(VIRTUAL_ENV)
+	pytest --create-db $(TARGET)
+
 # Deployment
 # =============================================================================
 .PHONY: deploy_prod

config/settings/base.py

@@ -48,9 +48,6 @@
 
 SECURE_HSTS_PRELOAD = True
 
-# Custom variables
-METABASE_SECRET_KEY = os.getenv("METABASE_SECRET_KEY")
-
 # Application definition
 
 INSTALLED_APPS = [
@@ -67,7 +64,9 @@
 
 MIDDLEWARE = [
     "django.middleware.security.SecurityMiddleware",
+    "django.middleware.gzip.GZipMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.middleware.http.ConditionalGetMiddleware",
     "django.middleware.common.CommonMiddleware",
     "django.middleware.csrf.CsrfViewMiddleware",
     "django.contrib.auth.middleware.AuthenticationMiddleware",
@@ -173,3 +172,8 @@
 # media files
 MEDIA_ROOT = os.path.join(APPS_DIR, "media")
 MEDIA_URL = "/media/"
+
+# Custom variables
+METABASE_URL = os.getenv("METABASE_URL")
+METABASE_SECRET_KEY = os.getenv("METABASE_SECRET_KEY")
+METABASE_API_KEY = os.getenv("METABASE_API_KEY")

config/urls.py

@@ -2,11 +2,13 @@
 from django.contrib import admin
 from django.urls import include, path
 
+from pilotage.api import urls as api_urls
 from pilotage.dashboards import urls as dashboards_urls
 from pilotage.pilotage import urls as pilotage_urls
 
 urlpatterns = [
     path("admin/", admin.site.urls),
+    path("api/", include(api_urls)),
     path("", include(pilotage_urls)),
     path("", include(dashboards_urls)),
 ]

itoutils/metabase.py

@@ -0,0 +1,92 @@
+import copy
+import json
+from urllib.parse import urljoin
+
+import httpx
+from django.conf import settings
+
+
+# Metabase API client
+# See: https://www.metabase.com/docs/latest/api/
+class Client:
+    def __init__(self, base_url):
+        self._client = httpx.Client(
+            base_url=urljoin(base_url, "/api"),
+            headers={
+                "X-API-KEY": settings.METABASE_API_KEY,
+            },
+            timeout=httpx.Timeout(5, read=60),  # Use a not-so-long but not not-so-short read timeout
+        )
+
+    @staticmethod
+    def _build_metabase_field(field, base_type="type/Text"):
+        return ["field", field, {"base-type": base_type}]
+
+    @staticmethod
+    def _build_metabase_filter(field, values, base_type="type/Text"):
+        return [
+            "=",
+            Client._build_metabase_field(field, base_type),
+            *values,
+        ]
+
+    @staticmethod
+    def _join_metabase_filters(*filters):
+        return [
+            "and",
+            *filters,
+        ]
+
+    def build_query(self, *, select=None, table=None, where=None, group_by=None, limit=None):
+        query = {}
+        if select:
+            query["fields"] = [self._build_metabase_field(field) for field in select]
+        if table:
+            query["source-table"] = table
+        if where:
+            query["filter"] = [self._build_metabase_filter(field, values) for field, values in where.items()]
+        if group_by:
+            query["breakout"] = [self._build_metabase_field(field) for field in group_by]
+        if limit:
+            query["limit"] = limit
+
+        return query
+
+    def merge_query(self, into, query):
+        into = copy.deepcopy(into)
+
+        if "fields" in query:
+            into.setdefault("fields", [])
+            into["fields"].extend(query["fields"])
+        if "filter" in query:
+            into.setdefault("filter", [])
+            into["filter"] = self._join_metabase_filters(into["filter"], query["filter"])
+        if "breakout" in query:
+            into.setdefault("breakout", [])
+            into["breakout"].extend(query["breakout"])
+        if "limit" in query:
+            into["limit"] = query["limit"]
+
+        return into
+
+    def build_dataset_query(self, *, database, query):
+        return {"database": database, "type": "query", "query": query, "parameters": []}
+
+    def fetch_dataset_results(self, query):
+        # /!\ MB (hardcoded) limit to the first 2000 rows when "viewing", "download" has a 1_000_000 rows limits
+        return (
+            self._client.post("/dataset/json", data={"query": json.dumps(query)})
+            .raise_for_status()
+            .json()
+        )
+
+    def fetch_card_results(self, card, filters=None, group_by=None):
+        if not any([filters, group_by]):
+            return self._client.post(f"/card/{card}/query/json").raise_for_status().json()
+
+        dataset_query = self._client.get(f"/card/{card}").raise_for_status().json()["dataset_query"]
+        dataset_query["query"] = self.merge_query(
+            dataset_query["query"],
+            self.build_query(where=filters, group_by=group_by),
+        )
+        return self.fetch_dataset_results(dataset_query)

pilotage/api/urls.py

@@ -0,0 +1,8 @@
+from django.urls import path
+
+from . import views
+
+app_name = "api"
+urlpatterns = [
+    path("dataset/<slug:name>", views.dataset, name="dataset"),
+]

pilotage/api/views.py

@@ -0,0 +1,57 @@
+from django.conf import settings
+from django.db.models.enums import TextChoices
+from django.http import (
+    HttpResponseNotFound,
+    JsonResponse,
+)
+
+from itoutils import metabase
+
+
+class DataSetName(TextChoices):
+    DI_SERVICES = "di_services", "data·inclusion - Services"
+    DI_STRUCTURES = "di_structures", "data·inclusion - Structures"
+
+
+QUERIES = {
+    DataSetName.DI_SERVICES: {
+        "database": 2,
+        "type": "query",
+        "query": {
+            "source-table": 2052,
+            "filter": [
+                "starts-with",
+                ["field", 59654, {"base-type": "type/Text"}],
+                "000",
+                {"case-sensitive": False},
+            ],
+        },
+    },
+    DataSetName.DI_STRUCTURES: {
+        "database": 2,
+        "type": "query",
+        "query": {
+            "source-table": 2051,
+            "filter": [
+                "starts-with",
+                ["field", 59588, {"base-type": "type/Text"}],
+                "000",
+                {"case-sensitive": False},
+            ],
+        },
+    },
+}
+
+
+def dataset(request, name):
+    if name not in DataSetName:
+        return HttpResponseNotFound()
+
+    query = QUERIES[DataSetName(name)]
+    if department := request.GET.get("department"):
+        # TODO: Check is_digit() + 2A/2B if applicable, and length
+        query["query"]["filter"][2] = str(department)[:3]
+
+    # TODO: handle errors and whatnot
+    data = metabase.Client(settings.METABASE_URL).fetch_dataset_results(query)
+    return JsonResponse(data, safe=False)  # Metabase return a list

pyproject.toml

@@ -10,3 +10,11 @@ preserve_blank_lines=true
 no_function_formatting=true
 format_css=true
 format_js=true
+
+[tool.pytest.ini_options]
+DJANGO_SETTINGS_MODULE = "config.settings.dev"
+python_files = ["test_*.py"]
+addopts = [
+    "--reuse-db",
+    "--strict-markers",
+]

requirements/base.in

@@ -9,7 +9,8 @@ django-libsass  # https://github.com/torchbox/django-libsass
 
 # Third-party applications
 # ------------------------------------------------------------------------------
+# Requests alternative including a default time out
+httpx  # https://github.com/encode/httpx/
 Markdown  # https://python-markdown.github.io/
-
 # Embedding Metabase signed dashboards
 PyJWT  # https://github.com/jpadilla/pyjwt

requirements/base.txt

@@ -1,9 +1,19 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile --generate-hashes -o requirements/base.txt requirements/base.in
+anyio==4.8.0 \
+    --hash=sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a \
+    --hash=sha256:b5011f270ab5eb0abf13385f851315585cc37ef330dd88e27ec3d34d651fd47a
+    # via httpx
 asgiref==3.8.1 \
     --hash=sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47 \
     --hash=sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590
     # via django
+certifi==2025.1.31 \
+    --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \
+    --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe
+    # via
+    #   httpcore
+    #   httpx
 django==5.0.12 \
     --hash=sha256:05097ea026cceb2db4db0655ecf77cc96b0753ac6a367280e458e603f6556f53 \
     --hash=sha256:3566604af111f586a1c9d49cb14ba6c607a0ccbbf87f57d98872cd8aae7d48ad
@@ -25,6 +35,24 @@ django-libsass==0.9 \
     --hash=sha256:5234d29100889cac79e36a0f44207ec6d275adfd2da1acb6a94b55c89fe2bd97 \
     --hash=sha256:bfbbb55a8950bb40fa04dd416605f92da34ad1f303b10a41abc3232386ec27b5
     # via -r requirements/base.in
+h11==0.14.0 \
+    --hash=sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d \
+    --hash=sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761
+    # via httpcore
+httpcore==1.0.7 \
+    --hash=sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c \
+    --hash=sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd
+    # via httpx
+httpx==0.28.1 \
+    --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
+    --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
+    # via -r requirements/base.in
+idna==3.10 \
+    --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
+    --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
+    # via
+    #   anyio
+    #   httpx
 libsass==0.23.0 \
     --hash=sha256:31e86d92a5c7a551df844b72d83fc2b5e50abc6fbbb31e296f7bebd6489ed1b4 \
     --hash=sha256:34cae047cbbfc4ffa832a61cbb110f3c95f5471c6170c842d3fed161e40814dc \
@@ -209,11 +237,17 @@ rjsmin==1.2.2 \
     --hash=sha256:e0e009f6f8460901f5144b34ac2948f94af2f9b8c9b5425da705dbc8152c36c2 \
     --hash=sha256:e733fea039a7b5ad7c06cc8bf215ee7afac81d462e273b3ab55c1ccc906cf127
     # via django-compressor
+sniffio==1.3.1 \
+    --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
+    --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
+    # via anyio
 sqlparse==0.5.3 \
     --hash=sha256:09f67787f56a0b16ecdbde1bfc7f5d9c3371ca683cfeaa8e6ff60b4807ec9272 \
     --hash=sha256:cf2196ed3418f3ba5de6af7e82c694a9fbdbfecccdfc72e281548517081f16ca
     # via django
 typing-extensions==4.12.2 \
     --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
     --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8
-    # via psycopg
+    # via
+    #   anyio
+    #   psycopg

requirements/dev.in

@@ -8,3 +8,11 @@ ruff  # https://github.com/charliermarsh/ruff
 # Django
 # ------------------------------------------------------------------------------
 django-debug-toolbar  # https://github.com/jazzband/django-debug-toolbar
+
+# Test & Mock
+# ------------------------------------------------------------------------------
+faker  # https://github.com/joke2k/faker
+pytest  # https://github.com/pytest-dev/pytest
+pytest-django # https://github.com/pytest-dev/pytest-django/
+pytest-mock  # https://github.com/pytest-dev/pytest-mock/
+pytest-randomly  # https://github.com/pytest-dev/pytest-randomly