Merge pull request #149 from inab/full_circle

Workflow discovery code has been revamped
inab · Feb 18, 2025 · c8f794a · c8f794a
2 parents 784c90c + 115f9a5
commit c8f794a
Show file tree

Hide file tree

Showing 33 changed files with 3,306 additions and 1,401 deletions.
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
@@ -18,7 +18,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 100
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
@@ -35,7 +35,7 @@ jobs:
 
       - name: 'Install requirements (standard or constraints ${{ matrix.python-version }})'
         run: |
-          pip install --upgrade pip wheel
+          pip install --upgrade pip wheel setuptools
           if [ ${{ steps.changed-requirements-txt.outputs.any_changed }} != 'true' ] && [ -f constraints-${{ matrix.python-version }}.txt ] ; then
             pip install -r requirements.txt -c constraints-${{ matrix.python-version }}.txt
           else
@@ -73,8 +73,9 @@ jobs:
       - name: Print licences report
         if: ${{ always() }}
         run: echo "${{ steps.license_check_report.outputs.report }}"
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
+          name: pre-commit-${{ matrix.python-version }}
           retention-days: 2
           path: constraints-${{ matrix.python-version }}.txt
 
@@ -88,7 +89,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 100
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
@@ -105,7 +106,7 @@ jobs:
 
       - name: 'Install requirements (standard or constraints ${{ matrix.python-version }})'
         run: |
-          pip install --upgrade pip wheel
+          pip install --upgrade pip wheel setuptools
           if [ ${{ steps.changed-requirements-txt.outputs.any_changed }} != 'true' ] && [ -f constraints-${{ matrix.python-version }}.txt ] ; then
             pip install -r requirements.txt -c constraints-${{ matrix.python-version }}.txt
           else
@@ -143,8 +144,9 @@ jobs:
       - name: Print licences report
         if: ${{ always() }}
         run: echo "${{ steps.license_check_report.outputs.report }}"
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
+          name: pre-commit-${{ matrix.python-version }}
           retention-days: 2
           path: constraints-${{ matrix.python-version }}.txt
 
@@ -157,24 +159,46 @@ jobs:
       - pre-commit
       - pre-commit-22_04
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
+      - name: Get analysis timestamp
+        id: timestamp
+        run: echo "timestamp=$(date -Is)" >> "$GITHUB_OUTPUT"
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        id: download
         with:
+          pattern: pre-commit-*
+          merge-multiple: true
           path: changes-dir
       - name: Move artifacts to their right place
+        id: move
         run: |
-          cp -dpr changes-dir/artifact/* .
-          rm -r changes-dir/artifact
+          skip=true
+          if [ -d "${{steps.download.outputs.download-path}}" ] ; then
+            for con in "${{steps.download.outputs.download-path}}"/constraints-*.txt ; do
+              case "$con" in
+                */constraints-\*.txt)
+                  break
+                  ;;
+                *)
+                  cp -p "$con" .
+                  skip=false
+                  ;;
+              esac
+            done
+          fi
+          echo "skip=$skip" >> "$GITHUB_OUTPUT"
       - name: Create Pull Request
         id: cpr
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v7
+        if: steps.move.outputs.skip == 'false'
         with:
-          title: Updated constraints (triggered by ${{ github.sha }})
+          title: Updated constraints (triggered on ${{ steps.timestamp.outputs.timestamp }} by ${{ github.sha }})
           branch: create-pull-request/patch-constraints
+          add-paths: constraints-*.txt
           delete-branch: true
           commit-message: "[create-pull-request] Automatically commit updated contents (constraints)"
       - name: Check outputs
         if: ${{ steps.cpr.outputs.pull-request-number }}
         run: |
-          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
-          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" >> "$GITHUB_STEP_SUMMARY"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" >> "$GITHUB_STEP_SUMMARY"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -107,6 +107,10 @@ repos:
   - repo: https://github.com/ambv/black.git
     rev: 23.3.0
     hooks:
+      - id: black
+        name: black_apply
+        exclude: "^[^/]*env/|development-[^/]*/|docs/"
+        stages: [manual]
       - id: black
         exclude: "^[^/]*env/|development-[^/]*/|docs/"
         args: [--diff, --check]

diff --git a/requirements.txt b/requirements.txt
@@ -26,7 +26,7 @@ wiktionary-term-fetcher >= 0.1.1
 funny-passphrase >= 0.2.3
 pyxdg
 groovy-parser == 0.1.1
-data-url
+data-url >= 1.1.1
 pgzip
 defusedxml
 # This is needed for exception groups

diff --git a/tests/fetchers/test_git.py b/tests/fetchers/test_git.py
@@ -29,6 +29,8 @@
 if TYPE_CHECKING:
     from typing import (
         Optional,
+        Type,
+        Union,
     )
 
     from wfexs_backend.common import (
@@ -38,14 +40,24 @@
         URIType,
     )
 
+from wfexs_backend.scheme_catalog import (
+    SchemeCatalog,
+)
+
 from wfexs_backend.fetchers import (
     RemoteRepo,
+    RepoGuessException,
     RepoGuessFlavor,
     RepoType,
 )
+
+from wfexs_backend.fetchers.http import HTTPFetcher
+
 from wfexs_backend.fetchers.git import GitFetcher
 
-WfExS_basedir = Path(__file__).parent.parent
+import wfexs_backend
+
+WfExS_basedir = Path(wfexs_backend.__file__).parent.parent
 WfExS_basedir_file_uri = WfExS_basedir.as_uri()
 WfExS_git_basedir = WfExS_basedir / ".git"
 WfExS_git_basedir_file_uri = WfExS_git_basedir.as_uri()
@@ -54,7 +66,7 @@
 logger.setLevel(logging.INFO)
 
 GIT_TESTBED = pytest.mark.parametrize(
-    ["url", "remote_repo", "repo_pid"],
+    ["url", "remote_repo_or_exception_class", "repo_pid"],
     [
         (
             "https://github.com/inab/WfExS-backend.git",
@@ -116,12 +128,9 @@
             "git+ssh://git@github.com:inab/WfExS-backend.git@main",
         ),
         (
+            # This tag does not exists!
             "ssh://git@github.com:inab/WfExS-backend.git@0.1.2",
-            RemoteRepo(
-                repo_url=cast("RepoURL", "ssh://git@github.com/inab/WfExS-backend.git"),
-                repo_type=RepoType.Git,
-                tag=cast("RepoTag", "0.1.2"),
-            ),
+            RepoGuessException,
             "git+ssh://git@github.com:inab/WfExS-backend.git@0.1.2",
         ),
         (
@@ -153,12 +162,9 @@
             "git+" + WfExS_git_basedir_file_uri,
         ),
         (
+            # This tag does not exists!
             WfExS_git_basedir_file_uri + "@0.1.2",
-            RemoteRepo(
-                repo_url=cast("RepoURL", WfExS_git_basedir_file_uri),
-                repo_type=RepoType.Git,
-                tag=cast("RepoTag", "0.1.2"),
-            ),
+            RepoGuessException,
             "git+" + WfExS_git_basedir_file_uri + "@0.1.2",
         ),
         (
@@ -233,28 +239,48 @@
 
 @GIT_TESTBED
 def test_guess_git_repo_params(
-    url: "str", remote_repo: "Optional[RemoteRepo]", repo_pid: "Optional[str]"
+    url: "str",
+    remote_repo_or_exception_class: "Optional[Union[RemoteRepo, Type[Exception]]]",
+    repo_pid: "Optional[str]",
 ) -> "None":
-    output = GitFetcher.GuessRepoParams(cast("URIType", url), logger=logger)
+    if (
+        isinstance(remote_repo_or_exception_class, RemoteRepo)
+        or remote_repo_or_exception_class is None
+    ):
+        output = GitFetcher.GuessRepoParams(cast("URIType", url), logger=logger)
+
+        # When no tag is given, ignore what it was discovered
+        if output is not None and remote_repo_or_exception_class is not None:
+            if remote_repo_or_exception_class.tag is None:
+                output = output._replace(tag=None)
+            # For now, patch this
+            if remote_repo_or_exception_class.checkout is not None:
+                output = output._replace(
+                    checkout=remote_repo_or_exception_class.checkout
+                )
+        assert output == remote_repo_or_exception_class
 
-    # When no tag is given, ignore what it was discovered
-    if output is not None and remote_repo is not None:
-        if remote_repo.tag is None:
-            output = output._replace(tag=None)
-        # For now, patch this
-        if remote_repo.checkout is not None:
-            output = output._replace(checkout=remote_repo.checkout)
-    assert output == remote_repo
+    else:
+        with pytest.raises(remote_repo_or_exception_class):
+            output = GitFetcher.GuessRepoParams(cast("URIType", url), logger=logger)
 
 
 @GIT_TESTBED
 def test_build_git_pid_from_repo(
-    url: "str", remote_repo: "Optional[RemoteRepo]", repo_pid: "Optional[str]"
+    url: "str",
+    remote_repo_or_exception_class: "Optional[Union[RemoteRepo, Type[Exception]]]",
+    repo_pid: "Optional[str]",
 ) -> "None":
-    if remote_repo is None:
+    if remote_repo_or_exception_class is None or not isinstance(
+        remote_repo_or_exception_class, RemoteRepo
+    ):
         pytest.skip("Skipped test because no remote repo was provided")
     else:
-        fetcher = GitFetcher({})
-        output = fetcher.build_pid_from_repo(remote_repo)
+        scheme_catalog = SchemeCatalog(
+            scheme_handlers=HTTPFetcher.GetSchemeHandlers(),
+        )
+
+        fetcher = GitFetcher(scheme_catalog, progs={})
+        output = fetcher.build_pid_from_repo(remote_repo_or_exception_class)
 
         assert output == repo_pid
diff --git a/tests/fetchers/test_swh.py b/tests/fetchers/test_swh.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # SPDX-License-Identifier: Apache-2.0
-# Copyright 2020-2024 Barcelona Supercomputing Center (BSC), Spain
+# Copyright 2020-2025 Barcelona Supercomputing Center (BSC), Spain
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -38,17 +38,19 @@
         URIType,
     )
 
+from wfexs_backend.scheme_catalog import (
+    SchemeCatalog,
+)
+
 from wfexs_backend.fetchers import (
     RemoteRepo,
     RepoGuessFlavor,
     RepoType,
 )
-from wfexs_backend.fetchers.swh import SoftwareHeritageFetcher
 
-WfExS_basedir = Path(__file__).parent.parent
-WfExS_basedir_file_uri = WfExS_basedir.as_uri()
-WfExS_git_basedir = WfExS_basedir / ".git"
-WfExS_git_basedir_file_uri = WfExS_git_basedir.as_uri()
+from wfexs_backend.fetchers.http import HTTPFetcher
+
+from wfexs_backend.fetchers.swh import SoftwareHeritageFetcher
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -139,7 +141,11 @@ def test_build_swh_pid_from_repo(
     if remote_repo is None:
         pytest.skip("Skipped test because no remote repo was provided")
     else:
-        fetcher = SoftwareHeritageFetcher({})
+        scheme_catalog = SchemeCatalog(
+            scheme_handlers=HTTPFetcher.GetSchemeHandlers(),
+        )
+
+        fetcher = SoftwareHeritageFetcher(scheme_catalog, progs={})
         output = fetcher.build_pid_from_repo(remote_repo)
 
         assert output == repo_pid