diff --git a/.build/.versions.yml b/.build/.versions.yml index b7282a0..3a4b072 100644 --- a/.build/.versions.yml +++ b/.build/.versions.yml @@ -30,7 +30,7 @@ compatibility-matrix: # https://spark.apache.org/releases/spark-release-3-5-0.html # Minimum supported java version: 17/21 - python_version: 3.11 - spark_version: [3.5.0] + spark_version: [3.5.1] java_version: [17] scala_version: [2.12, 2.13] hadoop_version: 3 @@ -41,7 +41,7 @@ compatibility-matrix: ### 3- The versions not present on compatibility-matrix are ignored build-matrix: python_version: ['3.9', '3.10', '3.11'] - spark_version: [3.2.4, 3.3.2, 3.3.4, 3.4.2, 3.5.0] + spark_version: [3.2.4, 3.3.2, 3.3.4, 3.4.2, 3.5.1] java_version: [11, 17] scala_version: [2.12] diff --git a/python/.flake8 b/.build/python/.flake8 similarity index 100% rename from python/.flake8 rename to .build/python/.flake8 diff --git a/python/okdp/__init__.py b/.build/python/okdp/__init__.py similarity index 100% rename from python/okdp/__init__.py rename to .build/python/okdp/__init__.py diff --git a/python/okdp/extension/__init__.py b/.build/python/okdp/extension/__init__.py similarity index 100% rename from python/okdp/extension/__init__.py rename to .build/python/okdp/extension/__init__.py diff --git a/python/okdp/extension/matrix/__init__.py b/.build/python/okdp/extension/matrix/__init__.py similarity index 100% rename from python/okdp/extension/matrix/__init__.py rename to .build/python/okdp/extension/matrix/__init__.py diff --git a/python/okdp/extension/matrix/constants.py b/.build/python/okdp/extension/matrix/constants.py similarity index 100% rename from python/okdp/extension/matrix/constants.py rename to .build/python/okdp/extension/matrix/constants.py diff --git a/python/okdp/extension/matrix/utils/__init__.py b/.build/python/okdp/extension/matrix/utils/__init__.py similarity index 100% rename from python/okdp/extension/matrix/utils/__init__.py rename to .build/python/okdp/extension/matrix/utils/__init__.py diff --git a/python/okdp/extension/matrix/utils/matrix_utils.py b/.build/python/okdp/extension/matrix/utils/matrix_utils.py similarity index 100% rename from python/okdp/extension/matrix/utils/matrix_utils.py rename to .build/python/okdp/extension/matrix/utils/matrix_utils.py diff --git a/python/okdp/extension/matrix/version_compatibility_matrix.py b/.build/python/okdp/extension/matrix/version_compatibility_matrix.py similarity index 100% rename from python/okdp/extension/matrix/version_compatibility_matrix.py rename to .build/python/okdp/extension/matrix/version_compatibility_matrix.py diff --git a/python/okdp/extension/tagging/__init__.py b/.build/python/okdp/extension/tagging/__init__.py similarity index 100% rename from python/okdp/extension/tagging/__init__.py rename to .build/python/okdp/extension/tagging/__init__.py diff --git a/python/okdp/extension/tagging/apply_tags.py b/.build/python/okdp/extension/tagging/apply_tags.py similarity index 98% rename from python/okdp/extension/tagging/apply_tags.py rename to .build/python/okdp/extension/tagging/apply_tags.py index 524c238..c562087 100755 --- a/python/okdp/extension/tagging/apply_tags.py +++ b/.build/python/okdp/extension/tagging/apply_tags.py @@ -75,7 +75,7 @@ def generate_tags(self) -> [str]: "--registry", required=True, type=str, - choices=["ghcr.io"], + choices=["quay.io", "ghcr.io"], help="Image registry", ) arg_parser.add_argument( diff --git a/python/okdp/extension/tagging/get_taggers_and_manifests.py b/.build/python/okdp/extension/tagging/get_taggers_and_manifests.py similarity index 100% rename from python/okdp/extension/tagging/get_taggers_and_manifests.py rename to .build/python/okdp/extension/tagging/get_taggers_and_manifests.py diff --git a/python/okdp/extension/tagging/images_hierarchy.py b/.build/python/okdp/extension/tagging/images_hierarchy.py similarity index 100% rename from python/okdp/extension/tagging/images_hierarchy.py rename to .build/python/okdp/extension/tagging/images_hierarchy.py diff --git a/python/okdp/extension/tagging/taggers.py b/.build/python/okdp/extension/tagging/taggers.py similarity index 100% rename from python/okdp/extension/tagging/taggers.py rename to .build/python/okdp/extension/tagging/taggers.py diff --git a/python/okdp/patch/README.md b/.build/python/okdp/patch/README.md similarity index 100% rename from python/okdp/patch/README.md rename to .build/python/okdp/patch/README.md diff --git a/python/okdp/patch/tests/docker-stacks-foundation/test_python_version.py b/.build/python/okdp/patch/tests/docker-stacks-foundation/test_python_version.py similarity index 100% rename from python/okdp/patch/tests/docker-stacks-foundation/test_python_version.py rename to .build/python/okdp/patch/tests/docker-stacks-foundation/test_python_version.py diff --git a/python/okdp/patch/tests/run_tests.py b/.build/python/okdp/patch/tests/run_tests.py similarity index 100% rename from python/okdp/patch/tests/run_tests.py rename to .build/python/okdp/patch/tests/run_tests.py diff --git a/python/requirements-extended.txt b/.build/python/requirements-dev.txt similarity index 100% rename from python/requirements-extended.txt rename to .build/python/requirements-dev.txt diff --git a/python/tests/__init__.py b/.build/python/tests/__init__.py similarity index 100% rename from python/tests/__init__.py rename to .build/python/tests/__init__.py diff --git a/python/tests/extension/__init__.py b/.build/python/tests/extension/__init__.py similarity index 100% rename from python/tests/extension/__init__.py rename to .build/python/tests/extension/__init__.py diff --git a/python/tests/extension/matrix/__init__.py b/.build/python/tests/extension/matrix/__init__.py similarity index 100% rename from python/tests/extension/matrix/__init__.py rename to .build/python/tests/extension/matrix/__init__.py diff --git a/python/tests/extension/matrix/conftest.py b/.build/python/tests/extension/matrix/conftest.py similarity index 100% rename from python/tests/extension/matrix/conftest.py rename to .build/python/tests/extension/matrix/conftest.py diff --git a/python/tests/extension/matrix/resources/expected_build_matrix_empty.json b/.build/python/tests/extension/matrix/resources/expected_build_matrix_empty.json similarity index 100% rename from python/tests/extension/matrix/resources/expected_build_matrix_empty.json rename to .build/python/tests/extension/matrix/resources/expected_build_matrix_empty.json diff --git a/python/tests/extension/matrix/test_version_compatibility_matrix.py b/.build/python/tests/extension/matrix/test_version_compatibility_matrix.py similarity index 100% rename from python/tests/extension/matrix/test_version_compatibility_matrix.py rename to .build/python/tests/extension/matrix/test_version_compatibility_matrix.py diff --git a/python/tests/pytest.ini b/.build/python/tests/pytest.ini similarity index 100% rename from python/tests/pytest.ini rename to .build/python/tests/pytest.ini diff --git a/.github/actions/free-disk-space/action.yml b/.github/actions/free-disk-space/action.yml new file mode 100644 index 0000000..4f661da --- /dev/null +++ b/.github/actions/free-disk-space/action.yml @@ -0,0 +1,40 @@ +# +# Copyright 2024 tosit.io +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Free disk space +description: Free Github runnner disk space + +runs: + using: composite + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + + + diff --git a/.github/actions/install-patchs-and-extension/action.yml b/.github/actions/install-patchs-and-extension/action.yml index 3ceed4f..c236734 100644 --- a/.github/actions/install-patchs-and-extension/action.yml +++ b/.github/actions/install-patchs-and-extension/action.yml @@ -6,15 +6,15 @@ runs: steps: - name: Setup dev env requirement 📦 run: | - cp -f requirements-dev.txt ../ - cat ../python/requirements-extended.txt >> ../requirements-dev.txt + cp -f requirements-dev.txt ../requirements-dev.txt + cat ../.build/python/requirements-dev.txt >> ../requirements-dev.txt working-directory: ./docker-stacks shell: bash - name: Copy patchs 📦 run: | - cp -fr ../python/okdp/patch/* ./ - cp -fr ../python/okdp ./ + cp -fr ../.build/python/okdp/patch/* ./ + cp -fr ../.build/python/okdp ./ working-directory: ./docker-stacks shell: bash diff --git a/.github/actions/setup-buildx/action.yaml b/.github/actions/setup-buildx/action.yaml new file mode 100644 index 0000000..6de6665 --- /dev/null +++ b/.github/actions/setup-buildx/action.yaml @@ -0,0 +1,29 @@ +# +# Copyright 2024 tosit.io +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Set up QEMU and Docker Buildx +description: Set up Docker Buildx + +runs: + using: composite + steps: + - name: Set up QEMU 📦 + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx 📦 + uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host \ No newline at end of file diff --git a/.github/workflows/auto-rerun.yml b/.github/workflows/auto-rerun.yml index 6ecc3b4..a809490 100644 --- a/.github/workflows/auto-rerun.yml +++ b/.github/workflows/auto-rerun.yml @@ -1,4 +1,4 @@ -# Credit for the following workraound: https://github.com/orgs/community/discussions/67654#discussioncomment-8038649 +# https://github.com/orgs/community/discussions/67654#discussioncomment-8038649 name: Automatic partital rerun on workflow failure on: workflow_dispatch: @@ -23,5 +23,4 @@ jobs: GH_TOKEN: ${{ github.token }} GH_DEBUG: api run: | - gh run watch ${{ inputs.run_id }} > /dev/null 2>&1 gh run rerun ${{ inputs.run_id }} --failed diff --git a/.github/workflows/build-test-base.yml b/.github/workflows/build-base-images-template.yml similarity index 56% rename from .github/workflows/build-test-base.yml rename to .github/workflows/build-base-images-template.yml index 71c8023..342674f 100644 --- a/.github/workflows/build-test-base.yml +++ b/.github/workflows/build-base-images-template.yml @@ -1,4 +1,4 @@ -name: Build, test, and push jupyter base images +name: Jupyter base build images template on: workflow_call: @@ -11,72 +11,73 @@ on: description: Tag to use for latest base images (foundation, minimal, etc) required: true type: string + publish_to_registry: + description: Wheter to push to the registry + required: false + type: string + default: "false" registry: description: The list of tags space separated values - required: true + required: false type: string + git_latest_release_tag: + description: The latest remote release tag + required: false + type: string + default: "" runs-on: description: GitHub Actions Runner image required: true type: string - secrets: - registry_username: - required: true - registry_token: - required: true - -defaults: - run: - working-directory: ./docker-stacks jobs: docker-stacks-foundation: - uses: ./.github/workflows/docker-build-test-push-latest.yml + uses: ./.github/workflows/build-image-template.yml with: parent-image: "" image: docker-stacks-foundation:${{ inputs.python_dev_tag }} - registry: ${{ inputs.registry }} build-args: PYTHON_VERSION=${{ inputs.python_version }} + registry: ${{ inputs.registry }} + publish_to_registry: ${{ inputs.publish_to_registry }} + git_latest_release_tag: ${{ inputs.git_latest_release_tag }} runs-on: ubuntu-latest - secrets: - registry_username: ${{ secrets.registry_username }} - registry_token: ${{ secrets.registry_token }} + secrets: inherit base-notebook: - uses: ./.github/workflows/docker-build-test-push-latest.yml + uses: ./.github/workflows/build-image-template.yml with: parent-image: docker-stacks-foundation:${{ inputs.python_dev_tag }} image: base-notebook:${{ inputs.python_dev_tag }} registry: ${{ inputs.registry }} + publish_to_registry: ${{ inputs.publish_to_registry }} + git_latest_release_tag: ${{ inputs.git_latest_release_tag }} runs-on: ubuntu-latest - secrets: - registry_username: ${{ secrets.registry_username }} - registry_token: ${{ secrets.registry_token }} + secrets: inherit needs: [docker-stacks-foundation] minimal-notebook: - uses: ./.github/workflows/docker-build-test-push-latest.yml + uses: ./.github/workflows/build-image-template.yml with: parent-image: base-notebook:${{ inputs.python_dev_tag }} image: minimal-notebook:${{ inputs.python_dev_tag }} registry: ${{ inputs.registry }} + publish_to_registry: ${{ inputs.publish_to_registry }} + git_latest_release_tag: ${{ inputs.git_latest_release_tag }} runs-on: ubuntu-latest - secrets: - registry_username: ${{ secrets.registry_username }} - registry_token: ${{ secrets.registry_token }} + secrets: inherit needs: [base-notebook] scipy-notebook: - uses: ./.github/workflows/docker-build-test-push-latest.yml + uses: ./.github/workflows/build-image-template.yml with: parent-image: minimal-notebook:${{ inputs.python_dev_tag }} image: scipy-notebook:${{ inputs.python_dev_tag }} registry: ${{ inputs.registry }} + publish_to_registry: ${{ inputs.publish_to_registry }} + git_latest_release_tag: ${{ inputs.git_latest_release_tag }} runs-on: ubuntu-latest - secrets: - registry_username: ${{ secrets.registry_username }} - registry_token: ${{ secrets.registry_token }} + secrets: inherit needs: [minimal-notebook] diff --git a/.github/workflows/build-test-datascience.yml b/.github/workflows/build-datascience-images-template.yml similarity index 50% rename from .github/workflows/build-test-datascience.yml rename to .github/workflows/build-datascience-images-template.yml index 0bffa9b..a57abbd 100644 --- a/.github/workflows/build-test-datascience.yml +++ b/.github/workflows/build-datascience-images-template.yml @@ -1,4 +1,4 @@ -name: Build, test, and push jupyter datascience images +name: Jupyter datascience build images template on: workflow_call: @@ -7,79 +7,80 @@ on: description: Tag to use for latest base images (foundation, minimal, etc) required: true type: string + publish_to_registry: + description: Wheter to push to the registry + required: false + type: string + default: "false" registry: description: The list of tags space separated values - required: true + required: false + type: string + git_latest_release_tag: + description: The latest remote release tag + required: false type: string + default: "" runs-on: description: GitHub Actions Runner image required: true type: string - secrets: - registry_username: - required: true - registry_token: - required: true - -defaults: - run: - working-directory: ./docker-stacks jobs: r: - uses: ./.github/workflows/docker-build-test-push-latest.yml + uses: ./.github/workflows/build-image-template.yml with: parent-image: minimal-notebook:${{ inputs.python_dev_tag }} image: r-notebook:${{ inputs.python_dev_tag }} registry: ${{ inputs.registry }} + publish_to_registry: ${{ inputs.publish_to_registry }} + git_latest_release_tag: ${{ inputs.git_latest_release_tag }} runs-on: ${{ inputs.runs-on }} - secrets: - registry_username: ${{ secrets.registry_username }} - registry_token: ${{ secrets.registry_token }} + secrets: inherit datascience: - uses: ./.github/workflows/docker-build-test-push-latest.yml + uses: ./.github/workflows/build-image-template.yml with: parent-image: scipy-notebook:${{ inputs.python_dev_tag }} image: datascience-notebook:${{ inputs.python_dev_tag }} registry: ${{ inputs.registry }} + publish_to_registry: ${{ inputs.publish_to_registry }} + git_latest_release_tag: ${{ inputs.git_latest_release_tag }} runs-on: ${{ inputs.runs-on }} - secrets: - registry_username: ${{ secrets.registry_username }} - registry_token: ${{ secrets.registry_token }} + secrets: inherit # julia: - # uses: ./.github/workflows/docker-build-test-push-latest.yml + # uses: ./.github/workflows/build-image-template.yml # with: # parent-image: minimal-notebook:${{ inputs.python_dev_tag }} # image: julia-notebook # registry: ${{ inputs.registry }} + # publish_to_registry: ${{ inputs.publish_to_registry }} + # git_latest_release_tag: ${{ inputs.git_latest_release_tag }} # runs-on: ${{ inputs.runs-on }} - # secrets: - # registry_username: ${{ secrets.registry_username }} - # registry_token: ${{ secrets.registry_token }} + # secrets: inherit # tensorflow: - # uses: ./.github/workflows/docker-build-test-push-latest.yml + # uses: ./.github/workflows/build-image-template.yml # with: # parent-image: scipy-notebook:${{ inputs.python_dev_tag }} # image: tensorflow-notebook # registry: ${{ inputs.registry }} + # publish_to_registry: ${{ inputs.publish_to_registry }} + # git_latest_release_tag: ${{ inputs.git_latest_release_tag }} # runs-on: ${{ inputs.runs-on }} - # secrets: - # registry_username: ${{ secrets.registry_username }} - # registry_token: ${{ secrets.registry_token }} + # secrets: inherit # pytorch: - # uses: ./.github/workflows/docker-build-test-push-latest.yml + # uses: ./.github/workflows/build-image-template.yml # with: # parent-image: scipy-notebook:${{ inputs.python_dev_tag }} # image: pytorch-notebook # registry: ${{ inputs.registry }} + # publish_to_registry: ${{ inputs.publish_to_registry }} + # git_latest_release_tag: ${{ inputs.git_latest_release_tag }} # runs-on: ${{ inputs.runs-on }} - # secrets: - # registry_username: ${{ secrets.registry_username }} - # registry_token: ${{ secrets.registry_token }} + # secrets: inherit diff --git a/.github/workflows/build-image-template.yml b/.github/workflows/build-image-template.yml new file mode 100644 index 0000000..43fe89f --- /dev/null +++ b/.github/workflows/build-image-template.yml @@ -0,0 +1,197 @@ +name: Jupyter build single image template + +on: + workflow_call: + inputs: + parent-image: + description: Parent image name + required: true + type: string + image: + description: Image name + required: true + type: string + publish_to_registry: + description: Wheter to push to the registry + required: false + type: string + default: "false" + ci_registry: + description: "The registry used to push ci images" + required: false + type: string + default: "ghcr.io" + registry: + description: The container registry + required: false + type: string + default: "quay.io" + build-args: + description: Build args comma separated list, ex. PYTHON_VERSION=3.11, ... + required: false + type: string + git_latest_release_tag: + description: The latest remote release tag + required: false + type: string + default: "" + runs-on: + description: GitHub Actions Runner image + required: true + type: string + +defaults: + run: + working-directory: ./docker-stacks + +jobs: + build-test-push: + name: ${{ inputs.image }} + runs-on: ${{ inputs.runs-on }} + + steps: + ### The publish and periodic rebuilds are based on the latest stable github release tag + - name: Checkout latest Github Release tag (${{ inputs.git_latest_release_tag }}) ⚡️ + if: inputs.publish_to_registry == 'true' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.git_latest_release_tag }} + + - name: Checkout Repo ⚡️ + if: inputs.publish_to_registry == 'false' + uses: actions/checkout@v4 + + - name: Free up disk space 📦 + uses: ./.github/actions/free-disk-space + + - name: Set up QEMU and Docker Buildx 📦 + uses: ./.github/actions/setup-buildx + + - name: Setup dev env patchs 📦 + uses: ./.github/actions/install-patchs-and-extension + + - name: Create dev environment 📦 + uses: ./docker-stacks/.github/actions/create-dev-env + + - name: Expose git commit sha as env variable + uses: rlespinasse/git-commit-data-action@v1.5.0 + + - name: Login to the CI registry 🔐 + if: inputs.publish_to_registry == 'false' + uses: docker/login-action@v3 + with: + registry: ${{ inputs.ci_registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login into official registry 🔐 + if: inputs.publish_to_registry == 'true' + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_ROBOT_TOKEN }} + + - name: Set CI Github container registry namespace + if: inputs.publish_to_registry == 'false' + run: | + echo "OWNER=${GITHUB_REPOSITORY_OWNER@L}" >> $GITHUB_ENV + echo "REGISTRY=${{ inputs.ci_registry }}" >> $GITHUB_ENV + shell: bash + + - name: Set official publish container registry namespace + if: inputs.publish_to_registry == 'true' + run: | + echo "OWNER=${GITHUB_REPOSITORY_OWNER@L}/jupyter" >> $GITHUB_ENV + echo "REGISTRY=${{ inputs.registry }}" >> $GITHUB_ENV + shell: bash + + - name: Pull parent image 📥 + if: inputs.parent-image != '' + run: docker pull $REGISTRY/$OWNER/${{ inputs.parent-image }} + shell: bash + + - name: Prepare image build (build-args) 📦 + run: | + for build_arg in ${{ inputs.build-args }} + do + BUILD_ARGS+="--build-arg $build_arg " + done + if [[ "${{ inputs.parent-image }}" ]] + then + BUILD_ARGS+="--build-arg BASE_CONTAINER=$REGISTRY/$OWNER/${{ inputs.parent-image }} " + fi + echo "BUILD_ARGS=$BUILD_ARGS" >> $GITHUB_ENV + # The short image name is necessary to run the tests (not pushed, local to jobs only) + echo "SHORT_IMAGE_NAME=${{ inputs.image }}" | awk -F: '{print $1}' >> $GITHUB_ENV + shell: bash + + - name: Patch PySpark Dockerfile to be compatible with java +11 📦 + if: contains(inputs.image, 'pyspark-notebook:') && contains(inputs.build-args, 'spark_version=3.2.') && ! contains(inputs.build-args, 'openjdk_version=8') + run: | + cat pyspark-notebook/Dockerfile.spark3.2.x >> docker-stacks/images/$SHORT_IMAGE_NAME/Dockerfile + working-directory: ./ + shell: bash + + - name: Add additional librairies 📦 + run: | + if test -f "$SHORT_IMAGE_NAME/Dockerfile" + then + cat $SHORT_IMAGE_NAME/Dockerfile >> docker-stacks/images/$SHORT_IMAGE_NAME/Dockerfile + cd $SHORT_IMAGE_NAME + cp $(ls | grep -v Dockerfile) ../docker-stacks/images/$SHORT_IMAGE_NAME/ + fi + working-directory: ./ + shell: bash + + - name: Build image 🛠 + run: | + #docker buildx build --platform=linux/amd64 --rm --force-rm --tag ... + docker build --rm --force-rm --tag $REGISTRY/$OWNER/${{ inputs.image }} \ + --tag $REGISTRY/$OWNER/$SHORT_IMAGE_NAME:latest images/$SHORT_IMAGE_NAME/ \ + --build-arg REGISTRY=$REGISTRY \ + --build-arg OWNER=$OWNER $BUILD_ARGS \ + --label "org.opencontainers.image.source=https://github.com/${{ github.repository }}" \ + --label "org.opencontainers.image.description=$SHORT_IMAGE_NAME" + env: + DOCKER_BUILDKIT: 1 + # Full logs for CI build + BUILDKIT_PROGRESS: plain + shell: bash + + # Run docker-stacks tests (docker-stacks/tests) + - name: Run tests ✅ + # Skip tests when running with ACT + if: inputs.publish_to_registry == 'false' && env.ACT_SKIP_TESTS == '' + run: | + python3 -m tests.run_tests --short-image-name $SHORT_IMAGE_NAME --registry ${{ inputs.ci_registry }} --owner $OWNER + shell: bash + + - name: Push to ci registry 📤 + if: inputs.publish_to_registry == 'false' + run: docker push ${{ inputs.ci_registry }}/$OWNER/${{ inputs.image }} + shell: bash + + ### Publish steps + ### The publish and periodic rebuilds are based on the latest stable github release tag + - name: Apply tags to the loaded image 🏷 + if: inputs.publish_to_registry == 'true' + run: | + python3 -m okdp.extension.tagging.apply_tags --short-image-name ${{ inputs.image }} \ + --registry ${{ inputs.registry }} \ + --owner $OWNER + + - name: Prepare image push 📦 + if: inputs.publish_to_registry == 'true' + run: | + # The short image name (without tag) is necessary to push to the registry + echo "SHORT_IMAGE_NAME=${{ inputs.image }}" | awk -F: '{print $1}' >> $GITHUB_ENV + shell: bash + + - name: Push to official registry 📤 + if: inputs.publish_to_registry == 'true' + run: docker push --all-tags ${{ inputs.registry }}/$OWNER/$SHORT_IMAGE_NAME + shell: bash + + + \ No newline at end of file diff --git a/.github/workflows/build-test-spark.yml b/.github/workflows/build-spark-images-template.yml similarity index 71% rename from .github/workflows/build-test-spark.yml rename to .github/workflows/build-spark-images-template.yml index 255e5d1..d378e13 100644 --- a/.github/workflows/build-test-spark.yml +++ b/.github/workflows/build-spark-images-template.yml @@ -1,4 +1,4 @@ -name: Build, test, and push jupyter Spark images +name: Jupyter pyspark build images template on: workflow_call: @@ -35,31 +35,34 @@ on: description: Tag to use for latest pyspark images (pyspark, all-spark, etc) required: true type: string + publish_to_registry: + description: Wheter to push to the registry + required: false + type: string + default: "false" registry: description: The list of tags space separated values - required: true + required: false + type: string + git_latest_release_tag: + description: The latest remote release tag + required: false type: string + default: "" runs-on: description: GitHub Actions Runner image required: true type: string - secrets: - registry_username: - required: true - registry_token: - required: true - -defaults: - run: - working-directory: ./docker-stacks jobs: pyspark: - uses: ./.github/workflows/docker-build-test-push-latest.yml + uses: ./.github/workflows/build-image-template.yml with: parent-image: scipy-notebook:${{ inputs.python_dev_tag }} image: pyspark-notebook:${{ inputs.spark_dev_tag }} registry: ${{ inputs.registry }} + publish_to_registry: ${{ inputs.publish_to_registry }} + git_latest_release_tag: ${{ inputs.git_latest_release_tag }} runs-on: ${{ inputs.runs-on }} build-args: spark_download_url=${{ inputs.spark_download_url }} @@ -67,19 +70,17 @@ jobs: openjdk_version=${{ inputs.java_version }} scala_version=${{ inputs.scala_version }} hadoop_version=${{ inputs.hadoop_version }} - secrets: - registry_username: ${{ secrets.registry_username }} - registry_token: ${{ secrets.registry_token }} + secrets: inherit all-spark: - uses: ./.github/workflows/docker-build-test-push-latest.yml + uses: ./.github/workflows/build-image-template.yml with: parent-image: pyspark-notebook:${{ inputs.spark_dev_tag }} image: all-spark-notebook:${{ inputs.spark_dev_tag }} registry: ${{ inputs.registry }} + publish_to_registry: ${{ inputs.publish_to_registry }} + git_latest_release_tag: ${{ inputs.git_latest_release_tag }} runs-on: ${{ inputs.runs-on }} - secrets: - registry_username: ${{ secrets.registry_username }} - registry_token: ${{ secrets.registry_token }} + secrets: inherit needs: [pyspark] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..807a654 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,166 @@ +name: ci + +on: + pull_request: + branches: + - main + paths: + - ".github/workflows/ci.yml" + - ".github/workflows/publish.yml" + - ".github/workflows/build-image-template.yml" + - ".github/workflows/build-base-images-template.yml" + - ".github/workflows/build-datascience-images-template.yml" + - ".github/workflows/build-spark-images-template.yml" + - ".github/actions/install-patchs-and-extension/action.yml" + - ".github/actions/setup-buildx/action.yml" + - ".github/actions/free-disk-space/action.yml" + + - ".build/.versions.yml" + - ".build/python/okdp/**" + + - "docker-stacks/images/**" + - "docker-stacks/tests/**" + - "docker-stacks/tagging/**" + + - "all-spark-notebook/**" + - "base-notebook/**" + - "datascience-notebook/**" + - "docker-stacks-foundation/**" + - "julia-notebook/**" + - "minimal-notebook/**" + - "pytorch-notebook/**" + - "r-notebook/**" + - "scipy-notebook/**" + - "tensorflow-notebook/**" + - "pyspark-notebook/**" + + - "!.build/python/okdp/patch/README.md" + - "!doc/**" + - "!README.md" + + push: + paths: + - ".github/workflows/ci.yml" + - ".github/workflows/publish.yml" + - ".github/workflows/build-image-template.yml" + - ".github/workflows/build-base-images-template.yml" + - ".github/workflows/build-datascience-images-template.yml" + - ".github/workflows/build-spark-images-template.yml" + - ".github/actions/install-patchs-and-extension/action.yml" + - ".github/actions/setup-buildx/action.yml" + - ".github/actions/free-disk-space/action.yml" + + - ".build/.versions.yml" + - ".build/python/okdp/**" + + - "docker-stacks/images/**" + - "docker-stacks/tests/**" + - "docker-stacks/tagging/**" + + - "all-spark-notebook/**" + - "base-notebook/**" + - "datascience-notebook/**" + - "docker-stacks-foundation/**" + - "julia-notebook/**" + - "minimal-notebook/**" + - "pytorch-notebook/**" + - "r-notebook/**" + - "scipy-notebook/**" + - "tensorflow-notebook/**" + - "pyspark-notebook/**" + + - "!.build/python/okdp/patch/README.md" + - "!doc/**" + - "!README.md" + + workflow_dispatch: + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + working-directory: ./docker-stacks + +permissions: + packages: write + +jobs: + + build-version-compatibility-matrix: + runs-on: ubuntu-latest + outputs: + spark: ${{ steps.set-matrix.outputs.spark }} + python: ${{ steps.set-matrix.outputs.python }} + steps: + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Setup dev env patchs 📦 + uses: ./.github/actions/install-patchs-and-extension + + - name: Create dev environment 📦 + uses: ./docker-stacks/.github/actions/create-dev-env + + - name: Run unit tests (versions compatibility matrix) + run: pytest python/tests -v --color=yes + working-directory: ./.build + shell: bash + + - name: Get current branch 📦 + id: branch + uses: tj-actions/branch-names@v8 + + - name: Build version compatibility matrix 🛠 + id: set-matrix + run: | + python3 -m okdp.extension.matrix.version_compatibility_matrix \ + --versions-matrix-path ../.build/.versions.yml \ + --git-branch ${{ steps.branch.outputs.current_branch || steps.branch.outputs.tag}} >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT + shell: bash + + build-base: + name: build-base (python-${{ matrix.python.python_version }}) + strategy: + matrix: + python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} + uses: ./.github/workflows/build-base-images-template.yml + with: + python_version: ${{ matrix.python.python_version }} + python_dev_tag: ${{ matrix.python.python_dev_tag }} + runs-on: ubuntu-latest + needs: [build-version-compatibility-matrix] + + build-datascience: + name: build-datascience (python-${{ matrix.python.python_version }}) + strategy: + matrix: + python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} + uses: ./.github/workflows/build-datascience-images-template.yml + with: + python_dev_tag: ${{ matrix.python.python_dev_tag }} + runs-on: ubuntu-latest + needs: [build-version-compatibility-matrix, build-base] + + build-spark: + name: build-spark (python-${{ matrix.spark.python_version }}) + strategy: + matrix: + spark: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.spark) }} + uses: ./.github/workflows/build-spark-images-template.yml + with: + spark_download_url: ${{ matrix.spark.spark_download_url }} + python_version: ${{ matrix.spark.python_version }} + spark_version: ${{ matrix.spark.spark_version }} + java_version: ${{ matrix.spark.java_version }} + scala_version: ${{ matrix.spark.scala_version }} + hadoop_version: ${{ matrix.spark.hadoop_version }} + python_dev_tag: ${{ matrix.spark.python_dev_tag }} + spark_dev_tag: ${{ matrix.spark.spark_dev_tag }} + runs-on: ubuntu-latest + needs: [build-version-compatibility-matrix, build-base] + diff --git a/.github/workflows/docker-build-test-push-latest.yml b/.github/workflows/docker-build-test-push-latest.yml deleted file mode 100644 index 59f3436..0000000 --- a/.github/workflows/docker-build-test-push-latest.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: Build, test and tag the image as --latest; then push to the container registry - -on: - workflow_call: - inputs: - parent-image: - description: Parent image name - required: true - type: string - image: - description: Image name - required: true - type: string - registry: - description: The list of tags space separated values - required: true - type: string - build-args: - description: Build args comma separated list, ex. PYTHON_VERSION=3.11, ... - required: false - type: string - runs-on: - description: GitHub Actions Runner image - required: true - type: string - secrets: - registry_username: - required: true - registry_token: - required: true - -defaults: - run: - working-directory: ./docker-stacks -jobs: - build-test-upload: - name: ${{ inputs.image }} - runs-on: ${{ inputs.runs-on }} - - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - - name: Setup dev env patchs 📦 - uses: ./.github/actions/install-patchs-and-extension - - - name: Create dev environment 📦 - uses: ./docker-stacks/.github/actions/create-dev-env - - - name: Expose git commit sha as env variable - uses: rlespinasse/git-commit-data-action@v1.5.0 - - - name: Login to Registry 🔐 - uses: docker/login-action@v3 - with: - registry: ${{ inputs.registry }} - username: ${{ secrets.registry_username }} - password: ${{ secrets.registry_token }} - - - name: Lower github repository owner - run: echo "OWNER=${GITHUB_REPOSITORY_OWNER@L}" >> $GITHUB_ENV - shell: bash - - - name: Pull parent image 📥 - if: inputs.parent-image != '' - run: docker pull ${{ inputs.registry }}/$OWNER/${{ inputs.parent-image }} - shell: bash - - - name: Prepare image build (build args) 📦 - run: | - for build_arg in ${{ inputs.build-args }} - do - BUILD_ARGS+="--build-arg $build_arg " - done - if [[ "${{ inputs.parent-image }}" ]] - then - BUILD_ARGS+="--build-arg BASE_CONTAINER=${{ inputs.registry }}/$OWNER/${{ inputs.parent-image }} " - fi - echo "BUILD_ARGS=$BUILD_ARGS" >> $GITHUB_ENV - # The short image name is necessary to run the tests (not pushed, local to jobs only) - echo "SHORT_IMAGE_NAME=${{ inputs.image }}" | awk -F: '{print $1}' >> $GITHUB_ENV - shell: bash - - - name: Patch PySpark Dockerfile to be compatible with java +11 📦 - if: contains(inputs.image, 'pyspark-notebook:') && contains(inputs.build-args, 'spark_version=3.2.') && ! contains(inputs.build-args, 'openjdk_version=8') - run: | - cat ../images/patch/pyspark-notebook/Dockerfile.spark3.2.x >> images/$SHORT_IMAGE_NAME/Dockerfile - shell: bash - - - name: Build image 🛠 - run: | - #docker buildx build --platform=linux/amd64 --rm --force-rm --tag ... - docker build --rm --force-rm --tag ${{ inputs.registry }}/$OWNER/${{ inputs.image }} \ - --tag ${{ inputs.registry }}/$OWNER/$SHORT_IMAGE_NAME:latest images/$SHORT_IMAGE_NAME/ \ - --build-arg REGISTRY=${{ inputs.registry }} \ - --build-arg OWNER=$OWNER $BUILD_ARGS \ - --label "org.opencontainers.image.source=https://github.com/${{ github.repository }}" \ - --label "org.opencontainers.image.description=$SHORT_IMAGE_NAME" - env: - DOCKER_BUILDKIT: 1 - # Full logs for CI build - BUILDKIT_PROGRESS: plain - shell: bash - - # Run docker-stacks tests (docker-stacks/tests) - - name: Run tests ✅ - # Skip tests when running with ACT - if: env.ACT_SKIP_TESTS == '' - run: | - python3 -m tests.run_tests --short-image-name $SHORT_IMAGE_NAME --registry ${{ inputs.registry }} --owner $OWNER - shell: bash - - - name: Push latest tag image to registry 📤 - run: docker push ${{ inputs.registry }}/$OWNER/${{ inputs.image }} - shell: bash - - - \ No newline at end of file diff --git a/.github/workflows/docker-tag-push.yml b/.github/workflows/docker-tag-push.yml deleted file mode 100644 index f7da407..0000000 --- a/.github/workflows/docker-tag-push.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Pull the image with it's latest tag, apply new tags and push back to container registry - -on: - workflow_call: - inputs: - image: - description: Image name - required: true - type: string - registry: - description: The list of tags space separated values - required: true - type: string - runs-on: - description: GitHub Actions Runner image - required: true - type: string - secrets: - registry_username: - required: true - registry_token: - required: true - -defaults: - run: - working-directory: ./docker-stacks - -jobs: - tag-push: - runs-on: ${{ inputs.runs-on }} - name: ${{ inputs.image }} - - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - - name: Setup dev env patchs 📦 - uses: ./.github/actions/install-patchs-and-extension - - - name: Create dev environment 📦 - uses: ./docker-stacks/.github/actions/create-dev-env - - - name: Login to Registry 🔐 - uses: docker/login-action@v3 - with: - registry: ${{ inputs.registry }} - username: ${{ secrets.registry_username }} - password: ${{ secrets.registry_token }} - - - name: Lower github repository owner - run: echo "OWNER=${GITHUB_REPOSITORY_OWNER@L}" >> $GITHUB_ENV - shell: bash - - - name: Pull latest tag image 📥 - run: docker pull ${{ inputs.registry }}/$OWNER/${{ inputs.image }} - shell: bash - - - name: Apply tags to the loaded image 🏷 - run: python3 -m okdp.extension.tagging.apply_tags --short-image-name ${{ inputs.image }} --registry ${{ inputs.registry }} --owner $OWNER - - - name: Prepare image push 📦 - run: | - # The short image name (without tag) is necessary to push to the registry - echo "SHORT_IMAGE_NAME=${{ inputs.image }}" | awk -F: '{print $1}' >> $GITHUB_ENV - shell: bash - - - name: Push Images to Registry 📤 - run: docker push --all-tags ${{ inputs.registry }}/$OWNER/$SHORT_IMAGE_NAME - shell: bash diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index aa99e38..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,225 +0,0 @@ -name: "[Main] Build, test, and push jupyter images" - -on: - pull_request: - paths: - - ".github/workflows/main.yml" - - ".github/workflows/build-base.yml" - - ".github/workflows/build-datascience.yml" - - ".github/workflows/build-spark.yml" - - ".github/workflows/docker-build-test-push-latest.yml" - - ".github/workflows/docker-tag-push.yml" - - ".github/actions/generate-build-matrix/action.yml" - - ".github/actions/install-patchs-and-extension/action.yml" - - - ".build/.versions.yml" - - - "python/okdp/**" - - "docker-stacks/images/**" - - "docker-stacks/tests/**" - - "docker-stacks/tagging/**" - - "images/**" - - - "!python/okdp/patch/README.md" - - "!images/README.md" - - push: - branches: - - main - paths: - - ".github/workflows/main.yml" - - ".github/workflows/build-base.yml" - - ".github/workflows/build-datascience.yml" - - ".github/workflows/build-spark.yml" - - ".github/workflows/docker-build-test-push-latest.yml" - - ".github/workflows/docker-tag-push.yml" - - ".github/actions/generate-build-matrix/action.yml" - - ".github/actions/install-patchs-and-extension/action.yml" - - - ".build/.versions.yml" - - - "python/okdp/**" - - "docker-stacks/images/**" - - "docker-stacks/tests/**" - - "docker-stacks/tagging/**" - - "images/**" - - - "!python/okdp/patch/README.md" - - "!images/README.md" - - workflow_dispatch: - -# https://docs.github.com/en/actions/using-jobs/using-concurrency -concurrency: - # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -defaults: - run: - working-directory: ./docker-stacks - -jobs: - run-unit-tests: - uses: ./.github/workflows/unit-tests.yml - with: - runs-on: ubuntu-latest - - build-version-compatibility-matrix: - runs-on: ubuntu-latest - outputs: - spark: ${{ steps.set-matrix.outputs.spark }} - python: ${{ steps.set-matrix.outputs.python }} - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - - name: Setup dev env patchs 📦 - uses: ./.github/actions/install-patchs-and-extension - - - name: Create dev environment 📦 - uses: ./docker-stacks/.github/actions/create-dev-env - - - name: Get current branch 📦 - id: branch - uses: tj-actions/branch-names@v8 - - - name: Build version compatibility matrix 🛠 - id: set-matrix - run: | - python3 -m okdp.extension.matrix.version_compatibility_matrix \ - --versions-matrix-path ../.build/.versions.yml \ - --git-branch ${{ steps.branch.outputs.current_branch || steps.branch.outputs.tag}} >> $GITHUB_OUTPUT - cat $GITHUB_OUTPUT - shell: bash - needs: [run-unit-tests] - - build-base: - name: build-test-base (python-${{ matrix.python.python_version }}) - strategy: - matrix: - python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} - uses: ./.github/workflows/build-test-base.yml - with: - python_version: ${{ matrix.python.python_version }} - python_dev_tag: ${{ matrix.python.python_dev_tag }} - registry: ${{ vars.REGISTRY || 'ghcr.io' }} - runs-on: ubuntu-latest - secrets: - registry_username: ${{ secrets.REGISTRY_USERNAME }} - registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} - needs: [build-version-compatibility-matrix] - - build-datascience: - name: build-test-datascience (python-${{ matrix.python.python_version }}) - strategy: - matrix: - python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} - uses: ./.github/workflows/build-test-datascience.yml - with: - python_dev_tag: ${{ matrix.python.python_dev_tag }} - registry: ${{ vars.REGISTRY || 'ghcr.io' }} - runs-on: ubuntu-latest - secrets: - registry_username: ${{ secrets.REGISTRY_USERNAME }} - registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} - needs: [build-version-compatibility-matrix, build-base] - - build-spark: - name: build-test-spark (python-${{ matrix.spark.python_version }}) - strategy: - matrix: - spark: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.spark) }} - uses: ./.github/workflows/build-test-spark.yml - with: - spark_download_url: ${{ matrix.spark.spark_download_url }} - python_version: ${{ matrix.spark.python_version }} - spark_version: ${{ matrix.spark.spark_version }} - java_version: ${{ matrix.spark.java_version }} - scala_version: ${{ matrix.spark.scala_version }} - hadoop_version: ${{ matrix.spark.hadoop_version }} - python_dev_tag: ${{ matrix.spark.python_dev_tag }} - spark_dev_tag: ${{ matrix.spark.spark_dev_tag }} - registry: ${{ vars.REGISTRY || 'ghcr.io' }} - runs-on: ubuntu-latest - secrets: - registry_username: ${{ secrets.REGISTRY_USERNAME }} - registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} - needs: [build-version-compatibility-matrix, build-base] - - ### 3 push in // - push-base: - if: github.ref == 'refs/heads/main' - name: push-base (python-${{ matrix.python.python_version }}) - strategy: - matrix: - image: [ docker-stacks-foundation, base-notebook, minimal-notebook, scipy-notebook,] - python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} - uses: ./.github/workflows/docker-tag-push.yml - with: - image: "${{ matrix.image }}:${{ matrix.python.python_dev_tag }}" - registry: ${{ vars.REGISTRY || 'ghcr.io' }} - runs-on: ubuntu-latest - secrets: - registry_username: ${{ secrets.REGISTRY_USERNAME }} - registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} - needs: [build-version-compatibility-matrix, build-datascience, build-spark,] - - push-datascience: - if: github.ref == 'refs/heads/main' - name: push-datascience (python-${{ matrix.python.python_version }}) - strategy: - matrix: - image: [r-notebook, datascience-notebook, ] - python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} - uses: ./.github/workflows/docker-tag-push.yml - with: - image: "${{ matrix.image }}:${{ matrix.python.python_dev_tag }}" - registry: ${{ vars.REGISTRY || 'ghcr.io' }} - runs-on: ubuntu-latest - secrets: - registry_username: ${{ secrets.REGISTRY_USERNAME }} - registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} - needs: [build-version-compatibility-matrix, build-datascience, build-spark] - - push-spark: - if: github.ref == 'refs/heads/main' - strategy: - matrix: - image: [ pyspark-notebook, all-spark-notebook,] - spark: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.spark) }} - uses: ./.github/workflows/docker-tag-push.yml - with: - image: "${{ matrix.image }}:${{ matrix.spark.spark_dev_tag }}" - registry: ${{ vars.REGISTRY || 'ghcr.io' }} - runs-on: ubuntu-latest - secrets: - registry_username: ${{ secrets.REGISTRY_USERNAME }} - registry_token: ${{ secrets.REGISTRY_ROBOT_TOKEN }} - needs: [build-version-compatibility-matrix, build-datascience, build-spark] - - # Credit for the following workraound: https://github.com/orgs/community/discussions/67654#discussioncomment-8038649 - partial-rerun-on-failure: - # Allow re-run the main branch only - if: failure() && github.ref == 'refs/heads/main' && fromJSON(github.run_attempt) < 3 - runs-on: ubuntu-latest - permissions: - contents: write - actions: write - packages: write - steps: - - name: "Re-run failed jobs" - env: - GH_REPO: ${{ github.repository }} - GH_TOKEN: ${{ github.token }} - GH_DEBUG: api - run: | - gh workflow run auto-rerun.yml \ - -F run_id=${{ github.run_id }} \ - -F workflow_name=${{ github.workflow }} \ - -F workflow_sha=${{ github.workflow_sha }} - - shell: bash - - needs: [push-base, push-datascience, push-spark] - \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..37f5d5e --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,175 @@ +name: publish + +on: + ### Periodically rebuild all the images to fix os security vulnerabilities + schedule: + # At 05:00 AM, only on Monday + #- cron: "0 5 * * 1" + # https://crontab.cronhub.io/ + # At 05:"0 AM, only on Monday + - cron: "0 5 * * 1" + # The release should be created manually (or with user token=pr approval/merge) in order to trigger the event + ### https://github.com/orgs/community/discussions/25281 + ### Instead of using the event, we call the workflow from release-please workflow (more secure) + #release: + # types: [published] + + workflow_dispatch: + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + working-directory: ./docker-stacks + +permissions: + packages: write + +jobs: + + latest-github-release: + if: github.repository_owner == 'OKDP' + runs-on: "ubuntu-latest" + outputs: + tag_name: ${{ steps.git-release-tag.outputs.tag_name }} + steps: + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Get latest GitHub Release tag name 📥 + id: git-release-tag + uses: InsonusK/get-latest-release@v1.0.1 + with: + myToken: ${{ github.token }} + exclude_types: "draft" + view_top: 1 + + - name: Info - Found latest release tag + run: | + echo "id: ${{ steps.git-release-tag.outputs.id }}" + echo "name: ${{ steps.git-release-tag.outputs.name }}" + echo "tag_name: ${{ steps.git-release-tag.outputs.tag_name }}" + echo "created_at: ${{ steps.git-release-tag.outputs.created_at }}" + echo "draft: ${{ steps.git-release-tag.outputs.draft }}" + echo "prerelease: ${{ steps.git-release-tag.outputs.prerelease }}" + shell: bash + + build-version-compatibility-matrix: + runs-on: ubuntu-latest + outputs: + spark: ${{ steps.set-matrix.outputs.spark }} + python: ${{ steps.set-matrix.outputs.python }} + steps: + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Setup dev env patchs 📦 + uses: ./.github/actions/install-patchs-and-extension + + - name: Create dev environment 📦 + uses: ./docker-stacks/.github/actions/create-dev-env + + - name: Run unit tests (versions compatibility matrix) + run: pytest python/tests -v --color=yes + working-directory: ./.build + shell: bash + + - name: Get current branch 📦 + id: branch + uses: tj-actions/branch-names@v8 + + - name: Build version compatibility matrix 🛠 + id: set-matrix + run: | + python3 -m okdp.extension.matrix.version_compatibility_matrix \ + --versions-matrix-path ../.build/.versions.yml \ + --git-branch ${{ steps.branch.outputs.current_branch || steps.branch.outputs.tag}} >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT + shell: bash + + publish-base: + if: github.repository_owner == 'OKDP' && needs.latest-github-release.outputs.tag_name != '' + name: publish-base (python-${{ matrix.python.python_version }}) + strategy: + matrix: + python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} + uses: ./.github/workflows/build-base-images-template.yml + with: + python_version: ${{ matrix.python.python_version }} + python_dev_tag: ${{ matrix.python.python_dev_tag }} + publish_to_registry: "true" + registry: ${{ vars.REGISTRY || 'quay.io' }} + git_latest_release_tag: ${{ needs.latest-github-release.outputs.tag_name }} + runs-on: ubuntu-latest + secrets: inherit + needs: [latest-github-release,build-version-compatibility-matrix] + + publish-datascience: + if: github.repository_owner == 'OKDP' && needs.latest-github-release.outputs.tag_name != '' + name: publish-datascience (python-${{ matrix.python.python_version }}) + strategy: + matrix: + python: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.python) }} + uses: ./.github/workflows/build-datascience-images-template.yml + with: + python_dev_tag: ${{ matrix.python.python_dev_tag }} + publish_to_registry: "true" + registry: ${{ vars.REGISTRY || 'quay.io' }} + git_latest_release_tag: ${{ needs.latest-github-release.outputs.tag_name }} + runs-on: ubuntu-latest + secrets: inherit + needs: [publish-base] + + publish-spark: + if: github.repository_owner == 'OKDP' && needs.latest-github-release.outputs.tag_name != '' + strategy: + matrix: + spark: ${{ fromJson(needs.build-version-compatibility-matrix.outputs.spark) }} + uses: ./.github/workflows/build-spark-images-template.yml + with: + spark_download_url: ${{ matrix.spark.spark_download_url }} + python_version: ${{ matrix.spark.python_version }} + spark_version: ${{ matrix.spark.spark_version }} + java_version: ${{ matrix.spark.java_version }} + scala_version: ${{ matrix.spark.scala_version }} + hadoop_version: ${{ matrix.spark.hadoop_version }} + python_dev_tag: ${{ matrix.spark.python_dev_tag }} + spark_dev_tag: ${{ matrix.spark.spark_dev_tag }} + publish_to_registry: "true" + registry: ${{ vars.REGISTRY || 'quay.io' }} + git_latest_release_tag: ${{ needs.latest-github-release.outputs.tag_name }} + runs-on: ubuntu-latest + secrets: inherit + needs: [publish-base] + + ## https://github.com/orgs/community/discussions/67654#discussioncomment-8038649 + partial-rerun-on-failure: + # Allow re-run the main branch only + if: failure() && github.ref == 'refs/heads/main' && fromJSON(github.run_attempt) < 3 + runs-on: ubuntu-latest + permissions: + contents: write + actions: write + packages: write + steps: + - name: "Re-run failed jobs" + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + GH_DEBUG: api + run: | + gh workflow run auto-rerun.yml \ + -F run_id=${{ github.run_id }} \ + -F workflow_name=${{ github.workflow }} \ + -F workflow_sha=${{ github.workflow_sha }} + + working-directory: ./ + shell: bash + + needs: [publish-base, publish-datascience, publish-spark] + + diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 0000000..f73f3c2 --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,70 @@ +# +# Copyright 2024 tosit.io +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: release-please + +on: + pull_request: + types: + - closed + branches: + - main + +permissions: + contents: write + pull-requests: write + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + release-please: + runs-on: ubuntu-latest + outputs: + release_created: ${{ steps.release-please.outputs.release_created }} + tag_name: ${{ steps.release-please.outputs.tag_name }} + # Skip the release process in the fork + # The pull request should come from the same repo (github_token from the fork does not have write permissions) + if: github.repository_owner == 'OKDP' && github.event.pull_request.merged == true && github.event.pull_request.head.repo.full_name == github.repository + steps: + - uses: google-github-actions/release-please-action@v4 + id: release-please + + publish: + runs-on: ubuntu-latest + needs: [release-please] + if: needs.release-please.outputs.release_created == 'true' + permissions: + contents: write + actions: write + packages: write + steps: + - name: "Publish images to official registry" + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + GH_DEBUG: api + run: | + gh workflow run publish.yml + shell: bash + \ No newline at end of file diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 8c07a70..0000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,28 +0,0 @@ -on: - workflow_call: - inputs: - runs-on: - description: GitHub Actions Runner image - required: true - type: string - -jobs: - - unit-tests: - runs-on: ${{ inputs.runs-on }} - - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - - name: Setup dev env patchs 📦 - uses: ./.github/actions/install-patchs-and-extension - - - name: Create dev environment 📦 - uses: ./docker-stacks/.github/actions/create-dev-env - - - name: Run unit tests - run: pytest python/tests -v --color=yes - shell: bash - - diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1 @@ +{} diff --git a/README.md b/README.md index 80195dd..2e08fe0 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Development images with tags ```--latest``` suffix (ex.: spark3.2.4- The [official images](#tagging) are publiqhed to the [okdp quay.io registry](https://quay.io/organization/okdp): 1. At every release, and, -2. Periodically, every mondat at 05H00 GMT +2. Periodically, every monday at 05H00 GMT ## Tagging diff --git a/pyspark-notebook/Dockerfile b/pyspark-notebook/Dockerfile new file mode 100644 index 0000000..87dd3ea --- /dev/null +++ b/pyspark-notebook/Dockerfile @@ -0,0 +1,6 @@ +#### Add additional libraries + +COPY requirements.txt . +RUN pip install -r requirements.txt;\ + rm -f requirements.txt + diff --git a/images/patch/pyspark-notebook/Dockerfile.spark3.2.x b/pyspark-notebook/Dockerfile.spark3.2.x similarity index 100% rename from images/patch/pyspark-notebook/Dockerfile.spark3.2.x rename to pyspark-notebook/Dockerfile.spark3.2.x diff --git a/images/README.md b/pyspark-notebook/README.md similarity index 78% rename from images/README.md rename to pyspark-notebook/README.md index f8674f0..7332691 100644 --- a/images/README.md +++ b/pyspark-notebook/README.md @@ -4,7 +4,7 @@ The directory contains a list of patchs for original dockerfiles: ## List of patchs: -[PySpark 3.2.x java 11 support](patch/pyspark-notebook/Dockerfile.spark3.2.x#L6): Add "--add-opens options" to be compatible with java 11 (<=3.2.x) +[PySpark 3.2.x java 11 support](Dockerfile.spark3.2.x#L6): Add "--add-opens options" to be compatible with java 11 (<=3.2.x) The options are picked from the java module: [JavaModuleOptions.java](https://github.com/apache/spark/blob/8706ccdf461c3b7f82b94b9e953ca4547f551ab1/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java) diff --git a/pyspark-notebook/requirements.txt b/pyspark-notebook/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..c7275fa --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,19 @@ +{ + "extra-files": [ + "README.md" + ], + "packages": { + ".": { + "changelog-path": "CHANGELOG.md", + "release-type": "simple", + "changelog-type": "default", + "bump-minor-pre-major": false, + "bump-patch-for-minor-pre-major": false, + "draft": false, + "prerelease": false, + "skip-snapshot": false + } + }, + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json" +} +