Skip to content

Commit

Permalink
Feat/ci as a state of the art part 4 (#30)
Browse files Browse the repository at this point in the history
* use go_test as separate workflow

* add exit_code for conclusion

* make build workflow only build

* pack to docker only if we build it

* dont retag if changes in go.mod or common libs

* skip tests if no changes in code nor in tests
  • Loading branch information
fedordikarev authored Feb 28, 2025
1 parent e01b5cd commit 719d57b
Show file tree
Hide file tree
Showing 8 changed files with 388 additions and 349 deletions.
45 changes: 37 additions & 8 deletions .github/workflows/00_pull-request-entry-point.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ jobs:
go-projects-changes: ${{ steps.go-projects-changes.outputs.changes }}
go-mod-changes: ${{ steps.go-projects-changes.outputs.go-mod }}
common-libs-changes: ${{ steps.go-projects-changes.outputs.common-libs }}
any-common-changes: ${{ fromJson(steps.go-projects-changes.outputs.go-mod) || fromJson(steps.go-projects-changes.outputs.common-libs) }}
cmd-changes: ${{ steps.per-component-changes.outputs.cmd-changes }}
tests-changes: ${{ steps.per-component-changes.outputs.tests-changes }}
cmd-unchanged: ${{ steps.per-component-changes.outputs.cmd-unchanged }}
Expand Down Expand Up @@ -68,37 +69,55 @@ jobs:
echo "cmd-unchanged=${cmd_unchanged}" >> "${GITHUB_OUTPUT}"
debug-projects-changed:
needs: [ projects-changed ]
runs-on: [ self-hosted, small ]
steps:
- name: show outputs
env:
CHANGES_GO_PROJECTS: ${{ needs.projects-changed.outputs.go-projects-changes }}
CHANGES_GO_MOD: ${{ needs.projects-changed.outputs.go-mod-changes }}
CHANGES_COMMON_LIB: ${{ needs.projects-changed.outputs.common-libs-changes }}
CHANGES_ANY_COMMON: ${{ needs.projects-changed.outputs.any-common-changes }}
CHANGES_CMD_CHANGES: ${{ needs.projects-changed.outputs.cmd-changes }}
CHANGES_TEST_CHANGES: ${{ needs.projects-changed.outputs.tests-changes }}
CHANGES_CMD_UNCHANGED: ${{ needs.projects-changed.outputs.cmd-unchanged }}
run: |
env | grep -E "^CHANGES"
build-or-retag-go-projects:
needs: [ check-permissions, projects-changed ]
strategy:
matrix:
go-component: ${{ fromJson('["gh-action", "history-exporter"]') }}
uses: ./.github/workflows/50_go_build-one-component.yml
uses: ./.github/workflows/50_go_build-component.yml
secrets: inherit
name: 'Build: ${{ matrix.go-component }}'
with:
component-name: ${{ matrix.go-component }}
cmd-path: './cmd/${{ matrix.go-component }}'
base-sha: ${{ github.event.pull_request.base.sha }}
head-sha: ${{ github.event.pull_request.head.sha }}
docker-image-base: neondatabase/${{ matrix.go-component }}
retag-base-image: ${{ contains(needs.projects-changed.outputs.cmd-unchanged, matrix.go-component) }}
run-build: ${{ contains(needs.projects-changed.outputs.cmd-changes, matrix.go-component) }}
retag-base-image: ${{ contains(needs.projects-changed.outputs.cmd-unchanged, matrix.go-component) && (needs.projects-changed.outputs.any-common-changes != 'true') }}
pack-to-docker-image: true
goos: linux
goarch: amd64

test-go-projects:
needs: [ check-permissions, projects-changed ]
strategy:
matrix:
go-component: ${{ fromJson('["gh-action", "history-exporter"]') }}
uses: ./.github/workflows/50_go_build-one-component.yml
uses: ./.github/workflows/50_go_test-component.yml
secrets: inherit
name: 'Unit Tests: ${{ matrix.go-component }}'
with:
component-name: ${{ matrix.go-component }}
cmd-path: './cmd/${{ matrix.go-component }}'
base-sha: ${{ github.event.pull_request.base.sha }}
head-sha: ${{ github.event.pull_request.head.sha }}
docker-image-base: neondatabase/${{ matrix.go-component }}
run-tests: true
skip-tests: ${{ contains(needs.projects-changed.outputs.cmd-unchanged, matrix.go-component) && (! contains(needs.projects-changed.outputs.tests-changes, format('{0}-tests', matrix.go-component))) && (needs.projects-changed.outputs.any-common-changes != 'true') }}
goos: linux
goarch: amd64

e2e-tests:
needs: [ build-or-retag-go-projects ]
Expand All @@ -111,7 +130,17 @@ jobs:
needs: [ test-go-projects, e2e-tests ]
if: ${{ always() }}
runs-on: [ self-hosted, small ]
env:
GO_TESTS_RESULT: ${{ needs.test-go-projects.result }}
E2E_TESTS_RESULT: ${{ needs.e2e-tests.result }}
steps:
- name: Print final message
run: |
echo Thats it.
echo "We can add here labels to PR to make it visible if any parts are failing"
- name: Conclusion
run: |
exit_code=0
if [ "${GO_TESTS_RESULT}" != "success" ]; then exit_code=$(( exit_code + 1 )); fi
if [ "${E2E_TESTS_RESULT}" != "success" ]; then exit_code=$(( exit_code + 2 )); fi
exit "${exit_code}"
14 changes: 8 additions & 6 deletions .github/workflows/00_push-entry-point.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,34 @@ jobs:
strategy:
matrix:
go-component: [ 'gh-action', 'history-exporter' ]
uses: ./.github/workflows/50_go_build-one-component.yml
uses: ./.github/workflows/50_go_build-component.yml
secrets: inherit
name: 'Golang: ${{ matrix.go-component }}'
with:
component-name: ${{ matrix.go-component }}
cmd-path: './cmd/${{ matrix.go-component}}'
base-sha: ${{ github.event.after }}
head-sha: ${{ github.event.after }}
docker-image-base: neondatabase/${{ matrix.go-component }}
run-build: true
pack-to-docker-image: true
go-build-cache-restore-or-update: 'update'
go-mod-cache-restore-or-update: 'update'
goos: linux
goarch: amd64

test-go-projects:
strategy:
matrix:
go-component: ${{ fromJson('["gh-action", "history-exporter"]') }}
uses: ./.github/workflows/50_go_build-one-component.yml
uses: ./.github/workflows/50_go_test-component.yml
secrets: inherit
name: 'Unit Tests: ${{ matrix.go-component }}'
with:
component-name: ${{ matrix.go-component }}
cmd-path: './cmd/${{ matrix.go-component}}'
base-sha: ${{ github.event.after }}
head-sha: ${{ github.event.after }}
docker-image-base: neondatabase/${{ matrix.go-component }}
run-tests: true
goos: linux
goarch: amd64

e2e-tests:
needs: [ build-go-projects ]
Expand Down
195 changes: 195 additions & 0 deletions .github/workflows/50_go_build-component.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
name: Build Go cmd

on:
workflow_call:
inputs:
cmd-path:
required: true
type: string
go-build-cache-restore-or-update:
required: false
type: string
default: 'restore'
go-mod-path:
required: false
type: string
default: 'go.mod'
go-mod-cache-restore-or-update:
required: false
type: string
default: 'restore'
goarch:
required: false
type: string
default: '' # default is to take it from ${{ runner.arch }}
goos:
required: false
type: string
default: '' # default is to take it from ${{ runner.os }}
docker-image-base:
required: false
type: string
default: ''
retag-base-image:
required: false
type: boolean
default: false
run-build:
required: false
type: boolean
default: false
run-tests:
required: false
type: boolean
default: false
pack-to-docker-image:
required: false
type: boolean
default: false
base-sha:
required: false
type: string
default: 'anysha'
head-sha:
required: false
type: string
default: 'anysha'


jobs:
prepare-running-env:
name: Build ${{ inputs.cmd-path}} for "${{ inputs.goos }}/${{ inputs.goarch }}"
runs-on: [ self-hosted, small ]
outputs:
goarch: ${{ steps.goarch-goos.outputs.goarch }}
goos: ${{ steps.goarch-goos.outputs.goos }}
go-mod-cache-key: ${{ steps.set-cache-keys.outputs.go-mod-cache-key }}
go-build-cache-key-cmd: ${{ steps.set-cache-keys.outputs.go-build-cache-key-cmd }}
go-build-cache-key-test: ${{ steps.set-cache-keys.outputs.go-build-cache-key-test }}
prepare-runner-os: ${{ runner.os }}
prepare-runner-arch: ${{ runner.arch }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
id: setup-go
with:
cache: false
go-version-file: ${{ inputs.go-mod-path }}
- name: Set GOARCH and GOOS
id: goarch-goos
run: |
GOARCH_INPUT="${{ inputs.goarch }}"
GOOS_INPUT="${{ inputs.goos }}"
declare -A runner_to_go_os=( ["Linux"]="linux" ["Windows"]="windows" ["macOS"]="darwin" )
declare -A runner_to_go_arch=( ["X86"]="386" ["X64"]="amd64" ["ARM"]="arm" ["ARM64"]="arm64" )
ARCH_FROM_RUNNER="${runner_to_go_arch[${{ runner.arch }}]}"
OS_FROM_RUNNER="${runner_to_go_os[${{ runner.os }}]}"
echo "goarch=${GOARCH_INPUT:-$ARCH_FROM_RUNNER}" >> "$GITHUB_OUTPUT"
echo "goos=${GOOS_INPUT:-$OS_FROM_RUNNER}" >> "$GITHUB_OUTPUT"
- id: set-cache-keys
name: Set keys needed for caching
env:
OS: "${{ steps.goarch-goos.outputs.goos }}"
ARCH: "${{ steps.goarch-goos.outputs.goarch }}"
GO_VER: "${{ steps.setup-go.outputs.go-version }}"
GO_SUM_HASH: "${{ hashFiles('**/go.sum') }}"
GH_REPO: "${{ github.event.repository.full_name }}"
BASE_SHA: "${{ inputs.base-sha }}"
HEAD_SHA: ${{ inputs.head-sha }}
CMD_PATH: ${{ inputs.cmd-path }}
run: |
GH_REPO="${GH_REPO//\//_}" # '//\//_' wat?
# here how it works:
# ${var//pattern/replace}, -- to replace all the occurencise of 'pattern' with 'replace'
# and pattern is '/', so escape it: '\/'.
# and replace is just '_', easy.
CMD_PATH="${CMD_PATH//\//_}" # well, now you know what '//\//_' means and how it works, right? :)
echo "go-mod-cache-key=go-mod-cache-${OS}-${ARCH}-go-${GO_VER}-${GO_SUM_HASH}" >> "$GITHUB_OUTPUT"
echo "go-build-cache-key-base=go-build-cache-${OS}-${ARCH}-go-${GO_VER}-gh-${GH_REPO}-git-${BASE_SHA}-build-${CMD_PATH}" >> "$GITHUB_OUTPUT"
echo "go-build-cache-key-head=go-build-cache-${OS}-${ARCH}-go-${GO_VER}-gh-${GH_REPO}-git-${HEAD_SHA}-build-${CMD_PATH}" >> "$GITHUB_OUTPUT"
- id: docker-login
if: ${{ inputs.retag-base-image || inputs.pack-to-docker-image }}
uses: docker/login-action@v3
with:
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
- id: docker-retag
if: ${{ inputs.retag-base-image }}
env:
DOCKER_IMAGE_BASE: ${{ inputs.docker-image-base }}
BASE_SHA: ${{ inputs.base-sha }}
HEAD_SHA: ${{ inputs.head-sha }}
BASE_TAG: arch-${{ steps.goarch-goos.outputs.goarch }}-commit-${{ inputs.base-sha }}
HEAD_TAG: arch-${{ steps.goarch-goos.outputs.goarch }}-commit-${{ inputs.head-sha }}
PLATFORM: ${{ steps.goarch-goos.outputs.goarch }}
continue-on-error: true
run: |
docker buildx imagetools create \
--tag "${DOCKER_IMAGE_BASE}:${HEAD_TAG}" \
"${DOCKER_IMAGE_BASE}:${BASE_TAG}"
- name: Go Mod Cache
uses: tespkg/actions-cache@v1
if: ${{ (inputs.go-build-cache-restore-or-update == 'update') && (steps.docker-retag.outcome != 'success') }}
with:
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
use-fallback: false
path: |
${{ steps.set-cache-keys.outputs.go-mod-cache }}
key: ${{ steps.set-cache-keys.outputs.go-mod-cache-key }}
- name: Go Mod Cache Restore
uses: tespkg/actions-cache/restore@v1
if: ${{ (inputs.go-build-cache-restore-or-update == 'restore') && (steps.docker-retag.outcome != 'success') }}
with:
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
use-fallback: false
path: |
${{ steps.set-cache-keys.outputs.go-mod-cache }}
key: ${{ steps.set-cache-keys.outputs.go-mod-cache-key }}
- name: Go Build Cache
uses: tespkg/actions-cache@v1
if: ${{ (inputs.go-build-cache-restore-or-update == 'update') && (steps.docker-retag.outcome != 'success') }}
with:
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
use-fallback: false
path: |
${{ steps.set-cache-keys.outputs.go-build }}
key: ${{ steps.set-cache-keys.outputs.go-build-cache-key-head }}
- name: Go Build
if: ${{ steps.docker-retag.outcome != 'success' }}
env:
GOARCH: ${{ steps.goarch-goos.outputs.goarch }}
GOOS: ${{ steps.goarch-goos.outputs.goos }}
CMD_PATH: ${{ inputs.cmd-path }}
run: |
go build -v -o "build/$(basename "${CMD_PATH}")" "${CMD_PATH}"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Pack as Docker image
if: ${{ steps.docker-retag.outcome != 'success' }}
env:
DOCKER_IMAGE_BASE: ${{ inputs.docker-image-base }}
BASE_TAG: arch-${{ steps.goarch-goos.outputs.goarch }}-commit-${{ inputs.base-sha }}
HEAD_TAG: arch-${{ steps.goarch-goos.outputs.goarch }}-commit-${{ inputs.head-sha }}
PLATFORM: ${{ steps.goarch-goos.outputs.goarch }}
CMD_PATH: ${{ inputs.cmd-path }}
run: |
echo "+++ BUILD:"
ls -laht build/
docker build \
--tag "${DOCKER_IMAGE_BASE}:${HEAD_TAG}" \
--platform "${PLATFORM}" \
--build-arg BINARY_TO_ADD="$(basename "${CMD_PATH}")" \
--file Dockerfiles/go-app-common.Dockerfile \
build/
docker push \
"${DOCKER_IMAGE_BASE}:${HEAD_TAG}"
Loading

0 comments on commit 719d57b

Please sign in to comment.