container-ci.yaml

# Copyright (c) 2024 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Container CI
permissions: read-all
on:
  workflow_dispatch:
    inputs:
      group_dir:
        required: true
        description: 'Enter Container Group Directory:'
        type: string
      env_overrides:
        description: 'Enter Bash Env Variable Overrides in `KEY=VAL KEY2=VAL2` format:'
        required: false
        type: string
      runner_label:
        description: 'Enter Validation Runner Label:'
        default: test-runner
        required: true
        type: string
      no_build:
        description: No build
        default: false
        required: true
        type: boolean
      no_push:
        description: Do not push to Registry
        required: true
        default: false
        type: boolean
      no_start:
        description: No Start
        default: false
        required: true
        type: boolean
  workflow_call:
    inputs:
      group_dir:
        required: true
        type: string
      no_push:
        required: false
        type: boolean
      no_start:
        required: false
        type: boolean
jobs:
  ####################################################################################################
  # Compose Build
  ####################################################################################################
  setup-build:
    outputs:
      matrix: ${{ steps.build-matrix.outputs.matrix }}
    runs-on: ubuntu-latest
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
      with:
        egress-policy: audit
    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
    - name: Set Matrix
      id: build-matrix
      run: echo "matrix=$(jq -c . < ${{ inputs.group_dir }}/.actions.json)" >> $GITHUB_OUTPUT
    - name: Print Inputs
      if: ${{ inputs.env_overrides }}
      run: echo "Overrides - ${{ inputs.env_overrides }}" >> $GITHUB_STEP_SUMMARY
  build-containers:
    needs: [setup-build]
    env: ${{ matrix }}
    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
    strategy:
      matrix: ${{ fromJson(needs.setup-build.outputs.matrix) }}
      fail-fast: false
    outputs:
      group: ${{ steps.build-group.outputs.container-group }}
    steps:
    - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      if: ${{ !inputs.no_build }}
    - uses: azure/docker-login@15c4aadf093404726ab2ff205b2cdd33fa6d054c # v2
      with:
        login-server: ${{ secrets.REGISTRY }}
        username: ${{ secrets.REGISTRY_USER }}
        password: ${{ secrets.REGISTRY_TOKEN }}
    # - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
    #   with:
    #     registry: ${{ secrets.REGISTRY }}
    #     username: ${{ secrets.REGISTRY_USER }}
    #     password: ${{ secrets.REGISTRY_TOKEN }}
      if: ${{ !inputs.no_build }}
    - name: Build Container Group
      if: ${{ !inputs.no_build }}
      id: build-group
      uses: intel/ai-containers/.github@main
      with:
        group_dir: ${{ inputs.group_dir }}
        env_overrides: ${{ inputs.env_overrides || env.env_overrides || '' }}
        registry: ${{ secrets.REGISTRY }}
        repo: ${{ secrets.REPO }}
        no-push: ${{ inputs.no_push }}
        no-start: ${{ inputs.no_start }}
        ####################################################################################################
        # Trivy Scan
        ####################################################################################################
  setup-scan:
    needs: [build-containers]
#    if: ${{ github.event_name == 'pull_request' }}
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.scan-matrix.outputs.matrix }}
    steps:
    - name: Set up date for cache key
      run: echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
    - name: Set up Trivy cache directory
      run: echo "TRIVY_CACHE_DIR=$GITHUB_WORKSPACE/trivy-cache" >> $GITHUB_ENV
    - name: Download Trivy database
      run: |
          trivy --download-db-only
          mkdir -p $GITHUB_WORKSPACE/trivy-cache
          mv ~/.cache/trivy/db $GITHUB_WORKSPACE/trivy-cache
    - name: Save Trivy cache
      uses: actions/cache@v4.1.2
      with:
        path: ${{ github.workspace }}/trivy-cache
        key: ${{ runner.os }}-trivy-db-${{ env.DATE }}
    - name: Harden Runner
      uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
      with:
        egress-policy: audit
    - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
      with:
        path: matrix
    - name: Set Matrix
      id: scan-matrix
      run: echo "matrix=$(cat matrix/*-${{ needs.build-containers.outputs.group }}/*.txt | jq -R '.' | jq -sc '. | unique')" >> $GITHUB_OUTPUT
  scan-containers:
    needs: [setup-scan]
#    if: ${{ !inputs.no_build && github.event_name == 'pull_request' }}
    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
    strategy:
      matrix:
        container: ${{ fromJSON(needs.setup-scan.outputs.matrix) }}
      fail-fast: false
    steps:
    - name: Set up Trivy cache directory
      run: echo "TRIVY_CACHE_DIR=$GITHUB_WORKSPACE/trivy-cache" >> $GITHUB_ENV
    - name: Restore Trivy cache
      uses: actions/cache@v4.1.2
      with:
        path: ${{ github.workspace }}/trivy-cache
        key: ${{ runner.os }}-trivy-db-${{ env.DATE }}
    - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
    - name: Harden Runner
      uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
      with:
        egress-policy: audit
    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
    - uses: azure/docker-login@15c4aadf093404726ab2ff205b2cdd33fa6d054c # v2
      with:
        login-server: ${{ secrets.REGISTRY }}
        username: ${{ secrets.REGISTRY_USER }}
        password: ${{ secrets.REGISTRY_TOKEN }}
    # - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
    #   with:
    #     registry: ${{ secrets.REGISTRY }}
    #     username: ${{ secrets.REGISTRY_USER }}
    #     password: ${{ secrets.REGISTRY_TOKEN }}
    - name: Pull Image
      run: docker pull ${{ secrets.REGISTRY }}/${{ secrets.REPO }}:${{ matrix.container }}
    - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
      with:
        path: ${{ github.workspace }}/.cache/trivy
        key: ${{ github.head_ref || github.ref_name }}-trivy
    - name: Scan Container
      uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # v0.28.0
      with:
        cache: true
        format: sarif
        github-pat: ${{ secrets.GITHUB_TOKEN }}
        image-ref: ${{ secrets.REGISTRY }}/${{ secrets.REPO }}:${{ matrix.container }}
        output: ${{ matrix.container }}-scan.sarif
        timeout: 30m0s
    - name: Cleanup
      if: always()
      run: docker rmi -f ${{ secrets.REGISTRY }}/${{ secrets.REPO }}:${{ matrix.container }}
    - uses: github/codeql-action/upload-sarif@4f3212b61783c3c68e8309a0f18a699764811cda # v3.27.1
      with:
        sarif_file: '${{ matrix.container }}-scan.sarif'
        category: '${{ matrix.container }}'
      continue-on-error: true
      ####################################################################################################
      # Generic Test Runner
      ####################################################################################################
  setup-test:
    needs: [build-containers]
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.test-matrix.outputs.matrix }}
    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
      with:
        egress-policy: audit
    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
    - name: Get Recipes
      id: test-matrix
      run: echo "matrix=$(find ${{ inputs.group_dir }} -type f -name 'tests.yaml' -exec dirname {} \; | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
  test-containers:
    needs: [setup-build, setup-test]
    if: ${{ needs.setup-test.outputs.matrix != '[]' }}
    runs-on: [self-hosted, Linux, "${{ inputs.runner_label || fromJson(needs.setup-build.outputs.matrix).runner_label }}"]
    strategy:
      matrix:
        tests: ${{ fromJson(needs.setup-test.outputs.matrix) }}
        experimental: [true]
      fail-fast: false
    steps:
    - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
      with:
        egress-policy: audit
    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
    - uses: azure/docker-login@15c4aadf093404726ab2ff205b2cdd33fa6d054c # v2
      with:
        login-server: ${{ secrets.REGISTRY }}
        username: ${{ secrets.REGISTRY_USER }}
        password: ${{ secrets.REGISTRY_TOKEN }}
    # - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
    #   with:
    #     registry: ${{ secrets.REGISTRY }}
    #     username: ${{ secrets.REGISTRY_USER }}
    #     password: ${{ secrets.REGISTRY_TOKEN }}
    - name: Test Container Group
      uses: intel/ai-containers/test-runner@main
      with:
        cache_registry: ${{ secrets.CACHE_REGISTRY }}
        recipe_dir: ${{ inputs.group_dir }}
        registry: ${{ secrets.REGISTRY }}
        repo: ${{ secrets.REPO }}
        test_dir: ${{ matrix.tests }}
        token: ${{ secrets.GITHUB_TOKEN }}