Build pipeline - automate ci/release/publish process and periodic ima…

…ges rebuild
OKDP · May 14, 2024 · ce537f7 · ce537f7
1 parent 3780fc5
commit ce537f7
Show file tree

Hide file tree

Showing 47 changed files with 794 additions and 529 deletions.
diff --git a/.build/.versions.yml b/.build/.versions.yml
@@ -30,7 +30,7 @@ compatibility-matrix:
   # https://spark.apache.org/releases/spark-release-3-5-0.html
   # Minimum supported java version: 17/21
   - python_version: 3.11
-    spark_version: [3.5.0]
+    spark_version: [3.5.1]
     java_version: [17]
     scala_version: [2.12, 2.13]
     hadoop_version: 3
@@ -41,7 +41,7 @@ compatibility-matrix:
 ### 3- The versions not present on compatibility-matrix are ignored
 build-matrix:
   python_version: ['3.9', '3.10', '3.11']
-  spark_version: [3.2.4, 3.3.2, 3.3.4, 3.4.2, 3.5.0]
+  spark_version: [3.2.4, 3.3.2, 3.3.4, 3.4.2, 3.5.1]
   java_version: [11, 17]
   scala_version: [2.12]
 
diff --git a/python/.flake8 → .build/python/.flake8 b/python/.flake8 → .build/python/.flake8
diff --git a/python/okdp/__init__.py → .build/python/okdp/__init__.py b/python/okdp/__init__.py → .build/python/okdp/__init__.py
diff --git a/python/okdp/extension/__init__.py → .build/python/okdp/extension/__init__.py b/python/okdp/extension/__init__.py → .build/python/okdp/extension/__init__.py
diff --git a/python/okdp/extension/matrix/__init__.py → .../python/okdp/extension/matrix/__init__.py b/python/okdp/extension/matrix/__init__.py → .../python/okdp/extension/matrix/__init__.py
diff --git a/python/okdp/extension/matrix/constants.py → ...python/okdp/extension/matrix/constants.py b/python/okdp/extension/matrix/constants.py → ...python/okdp/extension/matrix/constants.py
diff --git a/...n/okdp/extension/matrix/utils/__init__.py → ...n/okdp/extension/matrix/utils/__init__.py b/...n/okdp/extension/matrix/utils/__init__.py → ...n/okdp/extension/matrix/utils/__init__.py
diff --git a/...dp/extension/matrix/utils/matrix_utils.py → ...dp/extension/matrix/utils/matrix_utils.py b/...dp/extension/matrix/utils/matrix_utils.py → ...dp/extension/matrix/utils/matrix_utils.py
diff --git a/...on/matrix/version_compatibility_matrix.py → ...on/matrix/version_compatibility_matrix.py b/...on/matrix/version_compatibility_matrix.py → ...on/matrix/version_compatibility_matrix.py
diff --git a/python/okdp/extension/tagging/__init__.py → ...python/okdp/extension/tagging/__init__.py b/python/okdp/extension/tagging/__init__.py → ...python/okdp/extension/tagging/__init__.py
diff --git a/python/okdp/extension/tagging/apply_tags.py → ...thon/okdp/extension/tagging/apply_tags.py b/python/okdp/extension/tagging/apply_tags.py → ...thon/okdp/extension/tagging/apply_tags.py
@@ -75,7 +75,7 @@ def generate_tags(self) -> [str]:
         "--registry",
         required=True,
         type=str,
-        choices=["ghcr.io"],
+        choices=["quay.io", "ghcr.io"],
         help="Image registry",
     )
     arg_parser.add_argument(

diff --git a/...sion/tagging/get_taggers_and_manifests.py → ...sion/tagging/get_taggers_and_manifests.py b/...sion/tagging/get_taggers_and_manifests.py → ...sion/tagging/get_taggers_and_manifests.py
diff --git a/...kdp/extension/tagging/images_hierarchy.py → ...kdp/extension/tagging/images_hierarchy.py b/...kdp/extension/tagging/images_hierarchy.py → ...kdp/extension/tagging/images_hierarchy.py
diff --git a/python/okdp/extension/tagging/taggers.py → .../python/okdp/extension/tagging/taggers.py b/python/okdp/extension/tagging/taggers.py → .../python/okdp/extension/tagging/taggers.py
diff --git a/python/okdp/patch/README.md → .build/python/okdp/patch/README.md b/python/okdp/patch/README.md → .build/python/okdp/patch/README.md
diff --git a/...-stacks-foundation/test_python_version.py → ...-stacks-foundation/test_python_version.py b/...-stacks-foundation/test_python_version.py → ...-stacks-foundation/test_python_version.py
diff --git a/python/okdp/patch/tests/run_tests.py → .build/python/okdp/patch/tests/run_tests.py b/python/okdp/patch/tests/run_tests.py → .build/python/okdp/patch/tests/run_tests.py
diff --git a/python/requirements-extended.txt → .build/python/requirements-dev.txt b/python/requirements-extended.txt → .build/python/requirements-dev.txt
diff --git a/python/tests/__init__.py → .build/python/tests/__init__.py b/python/tests/__init__.py → .build/python/tests/__init__.py
diff --git a/python/tests/extension/__init__.py → .build/python/tests/extension/__init__.py b/python/tests/extension/__init__.py → .build/python/tests/extension/__init__.py
diff --git a/python/tests/extension/matrix/__init__.py → ...python/tests/extension/matrix/__init__.py b/python/tests/extension/matrix/__init__.py → ...python/tests/extension/matrix/__init__.py
diff --git a/python/tests/extension/matrix/conftest.py → ...python/tests/extension/matrix/conftest.py b/python/tests/extension/matrix/conftest.py → ...python/tests/extension/matrix/conftest.py
diff --git a/...esources/expected_build_matrix_empty.json → ...esources/expected_build_matrix_empty.json b/...esources/expected_build_matrix_empty.json → ...esources/expected_build_matrix_empty.json
diff --git a/...trix/test_version_compatibility_matrix.py → ...trix/test_version_compatibility_matrix.py b/...trix/test_version_compatibility_matrix.py → ...trix/test_version_compatibility_matrix.py
diff --git a/python/tests/pytest.ini → .build/python/tests/pytest.ini b/python/tests/pytest.ini → .build/python/tests/pytest.ini
diff --git a/.github/actions/free-disk-space/action.yml b/.github/actions/free-disk-space/action.yml
@@ -0,0 +1,40 @@
+#
+# Copyright 2024 tosit.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Free disk space
+description: Free Github runnner disk space
+
+runs:
+  using: composite
+  steps:
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tool-cache: false
+
+        # all of these default to true, but feel free to set to
+        # "false" if necessary for your workflow
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        docker-images: true
+        swap-storage: true
+
+
+
diff --git a/.github/actions/install-patchs-and-extension/action.yml b/.github/actions/install-patchs-and-extension/action.yml
@@ -6,15 +6,15 @@ runs:
   steps:
   - name: Setup dev env requirement 📦
     run: |
-      cp -f requirements-dev.txt ../
-      cat ../python/requirements-extended.txt >> ../requirements-dev.txt
+      cp -f requirements-dev.txt ../requirements-dev.txt
+      cat ../.build/python/requirements-dev.txt >> ../requirements-dev.txt
     working-directory: ./docker-stacks
     shell: bash
 
   - name: Copy patchs 📦
     run: |
-      cp -fr ../python/okdp/patch/* ./
-      cp -fr ../python/okdp ./
+      cp -fr ../.build/python/okdp/patch/* ./
+      cp -fr ../.build/python/okdp ./
     working-directory: ./docker-stacks
     shell: bash
 
diff --git a/.github/actions/setup-buildx/action.yaml b/.github/actions/setup-buildx/action.yaml
@@ -0,0 +1,29 @@
+#
+# Copyright 2024 tosit.io
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Set up QEMU and Docker Buildx
+description: Set up Docker Buildx
+
+runs:
+  using: composite
+  steps:
+    - name: Set up QEMU 📦
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx 📦
+      uses: docker/setup-buildx-action@v3
+      with:
+        driver-opts: network=host
diff --git a/.github/workflows/auto-rerun.yml b/.github/workflows/auto-rerun.yml
@@ -1,4 +1,4 @@
-# Credit for the following workraound: https://github.com/orgs/community/discussions/67654#discussioncomment-8038649
+# https://github.com/orgs/community/discussions/67654#discussioncomment-8038649
 name: Automatic partital rerun on workflow failure
 on:
   workflow_dispatch:
@@ -23,5 +23,4 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           GH_DEBUG: api
         run: |
-           gh run watch ${{ inputs.run_id }} > /dev/null 2>&1
            gh run rerun ${{ inputs.run_id }} --failed
diff --git a/.github/workflows/build-test-base.yml → .../workflows/build-base-images-template.yml b/.github/workflows/build-test-base.yml → .../workflows/build-base-images-template.yml
@@ -1,4 +1,4 @@
-name: Build, test, and push jupyter base images
+name: Jupyter base build images template
 
 on:
   workflow_call:
@@ -11,72 +11,73 @@ on:
         description: Tag to use for latest base images (foundation, minimal, etc)
         required: true
         type: string
+      publish_to_registry:
+        description: Wheter to push to the registry
+        required: false
+        type: string
+        default: "false"
       registry:
         description: The list of tags space separated values
-        required: true
+        required: false
         type: string
+      git_latest_release_tag:
+        description: The latest remote release tag 
+        required: false
+        type: string
+        default: ""
       runs-on:
         description: GitHub Actions Runner image
         required: true
         type: string
-    secrets:
-      registry_username:
-        required: true
-      registry_token:
-        required: true
-
-defaults:
-  run:
-    working-directory: ./docker-stacks
 
 jobs:
 
   docker-stacks-foundation:
-    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    uses: ./.github/workflows/build-image-template.yml
     with:
       parent-image: ""
       image: docker-stacks-foundation:${{ inputs.python_dev_tag }}
-      registry: ${{ inputs.registry }}
       build-args:
         PYTHON_VERSION=${{ inputs.python_version }}
+      registry: ${{ inputs.registry }}
+      publish_to_registry: ${{ inputs.publish_to_registry }}
+      git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
       runs-on: ubuntu-latest
-    secrets:
-      registry_username: ${{ secrets.registry_username }}
-      registry_token: ${{ secrets.registry_token }}
+    secrets: inherit
 
   base-notebook:
-    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    uses: ./.github/workflows/build-image-template.yml
     with:
       parent-image: docker-stacks-foundation:${{ inputs.python_dev_tag }}
       image: base-notebook:${{ inputs.python_dev_tag }}
       registry: ${{ inputs.registry }}
+      publish_to_registry: ${{ inputs.publish_to_registry }}
+      git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
       runs-on: ubuntu-latest
-    secrets:
-      registry_username: ${{ secrets.registry_username }}
-      registry_token: ${{ secrets.registry_token }}
+    secrets: inherit
     needs: [docker-stacks-foundation]
 
   minimal-notebook:
-    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    uses: ./.github/workflows/build-image-template.yml
     with:
       parent-image: base-notebook:${{ inputs.python_dev_tag }}
       image: minimal-notebook:${{ inputs.python_dev_tag }}
       registry: ${{ inputs.registry }}
+      publish_to_registry: ${{ inputs.publish_to_registry }}
+      git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
       runs-on: ubuntu-latest
-    secrets:
-      registry_username: ${{ secrets.registry_username }}
-      registry_token: ${{ secrets.registry_token }}
+    secrets: inherit
     needs: [base-notebook]
 
   scipy-notebook:
-    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    uses: ./.github/workflows/build-image-template.yml
     with:
       parent-image: minimal-notebook:${{ inputs.python_dev_tag }}
       image: scipy-notebook:${{ inputs.python_dev_tag }}
       registry: ${{ inputs.registry }}
+      publish_to_registry: ${{ inputs.publish_to_registry }}
+      git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
       runs-on: ubuntu-latest
-    secrets:
-      registry_username: ${{ secrets.registry_username }}
-      registry_token: ${{ secrets.registry_token }}
+    secrets: inherit
     needs: [minimal-notebook]
 
diff --git a/.github/workflows/build-test-datascience.yml → ...ows/build-datascience-images-template.yml b/.github/workflows/build-test-datascience.yml → ...ows/build-datascience-images-template.yml
@@ -1,4 +1,4 @@
-name: Build, test, and push jupyter datascience images
+name: Jupyter datascience build images template
 
 on:
   workflow_call:
@@ -7,79 +7,80 @@ on:
         description: Tag to use for latest base images (foundation, minimal, etc)
         required: true
         type: string
+      publish_to_registry:
+        description: Wheter to push to the registry
+        required: false
+        type: string
+        default: "false"
       registry:
         description: The list of tags space separated values
-        required: true
+        required: false
+        type: string
+      git_latest_release_tag:
+        description: The latest remote release tag 
+        required: false
         type: string
+        default: ""
       runs-on:
         description: GitHub Actions Runner image
         required: true
         type: string
-    secrets:
-      registry_username:
-        required: true
-      registry_token:
-        required: true
-
-defaults:
-  run:
-    working-directory: ./docker-stacks
 
 jobs:
   r:
-    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    uses: ./.github/workflows/build-image-template.yml
     with:
       parent-image: minimal-notebook:${{ inputs.python_dev_tag }}
       image: r-notebook:${{ inputs.python_dev_tag }}
       registry: ${{ inputs.registry }}
+      publish_to_registry: ${{ inputs.publish_to_registry }}
+      git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
       runs-on: ${{ inputs.runs-on }}
-    secrets:
-      registry_username: ${{ secrets.registry_username }}
-      registry_token: ${{ secrets.registry_token }}
+    secrets: inherit
 
   datascience:
-    uses: ./.github/workflows/docker-build-test-push-latest.yml
+    uses: ./.github/workflows/build-image-template.yml
     with:
       parent-image: scipy-notebook:${{ inputs.python_dev_tag }}
       image: datascience-notebook:${{ inputs.python_dev_tag }}
       registry: ${{ inputs.registry }}
+      publish_to_registry: ${{ inputs.publish_to_registry }}
+      git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
       runs-on: ${{ inputs.runs-on }}
-    secrets:
-      registry_username: ${{ secrets.registry_username }}
-      registry_token: ${{ secrets.registry_token }}
+    secrets: inherit
 
   # julia:
-  #   uses: ./.github/workflows/docker-build-test-push-latest.yml
+  #   uses: ./.github/workflows/build-image-template.yml
   #   with:
   #     parent-image: minimal-notebook:${{ inputs.python_dev_tag }}
   #     image: julia-notebook
   #     registry: ${{ inputs.registry }}
+  #     publish_to_registry: ${{ inputs.publish_to_registry }}
+  #     git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
   #     runs-on: ${{ inputs.runs-on }}
-  #   secrets:
-  #     registry_username: ${{ secrets.registry_username }}
-  #     registry_token: ${{ secrets.registry_token }}
+  #   secrets: inherit
 
   # tensorflow:
-  #   uses: ./.github/workflows/docker-build-test-push-latest.yml
+  #   uses: ./.github/workflows/build-image-template.yml
   #   with:
   #     parent-image: scipy-notebook:${{ inputs.python_dev_tag }}
   #     image: tensorflow-notebook
   #     registry: ${{ inputs.registry }}
+  #     publish_to_registry: ${{ inputs.publish_to_registry }}
+  #     git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
   #     runs-on: ${{ inputs.runs-on }}
-  #   secrets:
-  #     registry_username: ${{ secrets.registry_username }}
-  #     registry_token: ${{ secrets.registry_token }}
+  #   secrets: inherit
 
   # pytorch:
-  #   uses: ./.github/workflows/docker-build-test-push-latest.yml
+  #   uses: ./.github/workflows/build-image-template.yml
   #   with:
   #     parent-image: scipy-notebook:${{ inputs.python_dev_tag }}
   #     image: pytorch-notebook
   #     registry: ${{ inputs.registry }}
+  #     publish_to_registry: ${{ inputs.publish_to_registry }}
+  #     git_latest_release_tag: ${{ inputs.git_latest_release_tag }}
   #     runs-on: ${{ inputs.runs-on }}
-  #   secrets:
-  #     registry_username: ${{ secrets.registry_username }}
-  #     registry_token: ${{ secrets.registry_token }}
+  #   secrets: inherit