From 1f73526c9ebde652f92849fb853ace7c06fd6381 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Mon, 22 Jan 2024 15:02:54 +0000 Subject: [PATCH 01/27] Moved GH workflow reporting into separate job --- .github/workflows/PR-validation.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index 69e1ada3..b94fc734 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -5,9 +5,7 @@ on: branches: - master jobs: - test-api: - permissions: - id-token: write # Required for authentication through OIDC to AWS + reporting: runs-on: ubuntu-22.04 steps: - name: Report workflow details @@ -20,6 +18,13 @@ jobs: run: | git fetch -q origin ${{ github.base_ref }} ${{ github.head_ref }} git diff --name-only origin/${{ github.base_ref }} origin/${{ github.head_ref }} + test-api: + permissions: + id-token: write # Required for authentication through OIDC to AWS + runs-on: ubuntu-22.04 + steps: + - name: Check out repository code + uses: actions/checkout@v3 - name: Set up JDK 8 uses: actions/setup-java@v3 with: From b6dd3099ead7192f52ef6adf8336473ced38c7f2 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Mon, 22 Jan 2024 17:32:36 +0000 Subject: [PATCH 02/27] Moved Dockerfile to repo root and API jar build artefact to dedicated build directory --- .gitignore | 2 +- docker/Dockerfile => Dockerfile | 0 Makefile | 14 ++++++++------ scripts/build-appjar.sh | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) rename docker/Dockerfile => Dockerfile (100%) diff --git a/.gitignore b/.gitignore index 30698a5b..8386fb7f 100644 --- a/.gitignore +++ b/.gitignore @@ -32,7 +32,7 @@ target/ !.elasticbeanstalk/*.cfg.yml !.elasticbeanstalk/*.global.yml Dockerrun.aws.json -/docker/app.jar +build/* /.dockerignore #Local docker-compose env-variable files diff --git a/docker/Dockerfile b/Dockerfile similarity index 100% rename from docker/Dockerfile rename to Dockerfile diff --git a/Makefile b/Makefile index 27c3ae62..08e0bf41 100644 --- a/Makefile +++ b/Makefile @@ -67,13 +67,15 @@ show-version: \ Show the current application version.) @echo "${VERSION_TAG}" +build/: + mkdir build/ + .PHONY: build -build: clean ui-build docker/${DEPLOY_JAR} \ +build: clean build/ ui-build build/${DEPLOY_JAR} \ $(call print-help,build,\ Build the docker images from using the current git revision.) @docker build -t ${ECR_REPO_NAME}:${VERSION_TAG} \ - --build-arg uberjar_path=${DEPLOY_JAR} \ - ./docker/ + --build-arg uberjar_path=build/${DEPLOY_JAR} . .PHONY: ui-build ui-build: google-oauth2-secrets \ @@ -87,10 +89,10 @@ ui-build: google-oauth2-secrets \ clean: \ $(call print-help,clean,\ Remove the locally built JAR file.) - @rm -f ./docker/${DEPLOY_JAR} + @rm -f ./build/${DEPLOY_JAR} -docker/${DEPLOY_JAR}: \ - $(call print-help,docker/${DEPLOY_JAR},\ +build/${DEPLOY_JAR}: \ + $(call print-help,build/${DEPLOY_JAR},\ Build the jar file.) @./scripts/build-appjar.sh diff --git a/scripts/build-appjar.sh b/scripts/build-appjar.sh index 8fbdd2de..d4576157 100755 --- a/scripts/build-appjar.sh +++ b/scripts/build-appjar.sh @@ -1,6 +1,6 @@ #!/bin/bash -ARTEFACT="docker/app.jar" +ARTEFACT="build/app.jar" LATEST_TAG=$(git describe --abbrev=0) RELEASE_NAME="wormbase-names-${LATEST_TAG}" DEPLOY_JAR="target/${RELEASE_NAME}.jar" From 68744da50f14a83a5d53f3d53ec391d1f27ba542 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Tue, 23 Jan 2024 15:15:25 +0000 Subject: [PATCH 03/27] Moved git version retrieval to separate target (to enable target calling without) --- Makefile | 68 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 08e0bf41..7d299ece 100644 --- a/Makefile +++ b/Makefile @@ -21,21 +21,9 @@ DEPLOY_JAR := app.jar PORT := 3000 WB_ACC_NUM := 357210185381 -REVISION_NAME_CMD := git describe --tags -# If REF_NAME is defined, convert provided ref in matching name -# (ref could be a tagname, a reference like "HEAD", branch-names...) -ifneq (${REF_NAME},) - REVISION_NAME := $(shell ${REVISION_NAME_CMD} ${REF_NAME}) -# Else, assume working-directory deployment -else - REVISION_NAME := $(shell ${REVISION_NAME_CMD} --dirty=-DIRTY) -endif - -VERSION_TAG ?= $(shell echo "${REVISION_NAME}" | sed 's/^wormbase-names-//') - ECR_URI := ${WB_ACC_NUM}.dkr.ecr.us-east-1.amazonaws.com ECR_REPO_URI := ${ECR_URI}/${ECR_REPO_NAME} -ECR_IMAGE_URI := ${ECR_REPO_URI}:${VERSION_TAG} +ECR_IMAGE_URI = ${ECR_REPO_URI}:${VERSION_TAG} # Set AWS (EB) profile env vars if undefined ifneq (${AWS_PROFILE},) AWS_EB_PROFILE ?= ${AWS_PROFILE} @@ -61,8 +49,30 @@ need-help := $(filter help,$(MAKECMDGOALS)) help: ; @echo $(if $(need-help),,\ Type \'$(MAKE)$(dash-f) help\' to get help) +.PHONY: ENV.VERSION_TAG +ENV.VERSION_TAG: \ + $(call print-help,ENV.VERSION_TAG,\ + Extract the VERSION_TAG env variables for make targets from git tags if undefined.) + $(eval REVISION_NAME_CMD = git describe --tags) +ifeq (${VERSION_TAG},) +# If REF_NAME is defined, convert provided ref in matching name +# (ref could be a tagname, a reference like "HEAD", branch-names...) +ifneq (${REF_NAME},) + $(eval REVISION_NAME_CMD = ${REVISION_NAME_CMD} ${REF_NAME}) +# Else, assume working-directory deployment +else + $(eval REVISION_NAME_CMD = ${REVISION_NAME_CMD} --dirty=-DIRTY) +endif + $(eval VERSION_TAG = $(shell ${REVISION_NAME_CMD} | sed 's/^wormbase-names-//')) + $(call check_defined, VERSION_TAG, Ensure your working directory is a\ + git clone of the repository) + @echo "Retrieved VERSION_TAG '${VERSION_TAG}' from git tags." +else + @echo "Using predefined VERSION_TAG '${VERSION_TAG}'." +endif + .PHONY: show-version -show-version: \ +show-version: ENV.VERSION_TAG \ $(call print-help,show-version,\ Show the current application version.) @echo "${VERSION_TAG}" @@ -71,7 +81,7 @@ build/: mkdir build/ .PHONY: build -build: clean build/ ui-build build/${DEPLOY_JAR} \ +build: ENV.VERSION_TAG clean build/ ui-build build/${DEPLOY_JAR} \ $(call print-help,build,\ Build the docker images from using the current git revision.) @docker build -t ${ECR_REPO_NAME}:${VERSION_TAG} \ @@ -114,14 +124,14 @@ docker-push-ecr: docker-ecr-login \ @docker push ${ECR_IMAGE_URI} .PHONY: docker-tag -docker-tag: \ +docker-tag: ENV.VERSION_TAG \ $(call print-help,docker-tag,\ Tag the image with current git revision and ':latest' alias.) @docker tag ${ECR_REPO_NAME}:${VERSION_TAG} ${ECR_IMAGE_URI} @docker tag ${ECR_REPO_NAME}:${VERSION_TAG} ${ECR_REPO_URI} .PHONY: eb-def-app-env -eb-def-app-env: google-oauth2-secrets \ +eb-def-app-env: google-oauth2-secrets ENV.VERSION_TAG \ $(call print-help,eb-def-app-env \ [WB_DB_URI=] [GOOGLE_REDIRECT_URI=],\ Define the ElasticBeanStalk app-environment config file.) @@ -188,7 +198,7 @@ eb-local: docker-ecr-login \ @eb local run --envvars PORT=${PORT},WB_DB_URI=${WB_DB_URI},GOOGLE_REDIRECT_URI=${GOOGLE_REDIRECT_URI} .PHONY: run -run: \ +run: ENV.VERSION_TAG \ $(call print-help,run [PORT=] [PROJ_NAME=] \ [WB_DB_URI=] [GOOGLE_REDIRECT_URI=],\ Run the application in docker (locally).) @@ -217,16 +227,15 @@ deploy-ecr: docker-build docker-tag docker-push-ecr Deploy the application to the AWS container registry.) .PHONY: vc-release -vc-release: export VERSION_TAG := ${VERSION_TAG} -vc-release: $(call print-help,vc-release LEVEL=,\ +vc-release: ENV.VERSION_TAG \ + $(call print-help,vc-release LEVEL=,\ Perform the Version Control tasks to release the applicaton.) clj -A:release --without-sign ${LEVEL} @echo "Edit version of application in pom.xml to match wormbase-names-* version reported above (version number only)." .PHONY: release -release: export VERSION_TAG := ${VERSION_TAG} -release: deploy-ecr \ +release: ENV.VERSION_TAG deploy-ecr \ $(call print-help,release [AWS_PROFILE=] [REF_NAME=],\ Release the applicaton.) @@ -266,15 +275,22 @@ google-oauth2-secrets: \ $(call print-help,google-oauth2-secrets,\ Store the Google oauth2 client details as env variables.) $(eval GOOGLE_OAUTH_CLIENT_ID = $(shell aws ssm get-parameter --name "/name-service/${GOOGLE_APP_PROFILE}/google-oauth2-app-config/client-id" --query "Parameter.Value" --output text --with-decryption)) - $(call check_defined, GOOGLE_OAUTH_CLIENT_ID) + $(call check_defined, GOOGLE_OAUTH_CLIENT_ID, Check the defined GOOGLE_APP_PROFILE value\ + and ensure the AWS_PROFILE variable is appropriately defined) $(eval GOOGLE_OAUTH_CLIENT_SECRET = $(shell aws ssm get-parameter --name "/name-service/${GOOGLE_APP_PROFILE}/google-oauth2-app-config/client-secret" --query "Parameter.Value" --output text --with-decryption)) - $(call check_defined, GOOGLE_OAUTH_CLIENT_SECRET) + $(call check_defined, GOOGLE_OAUTH_CLIENT_SECRET, Check the defined GOOGLE_APP_PROFILE value\ + and ensure the AWS_PROFILE variable is appropriately defined) @echo "Retrieved google-oauth2-secrets for GOOGLE_APP_PROFILE '${GOOGLE_APP_PROFILE}'." +# Check that given variables are set and all have non-empty values, +# die with an error otherwise. +# +# Params: +# 1. Variable name(s) to test. +# 2. (optional) Error message to print. check_defined = \ $(strip $(foreach 1,$1, \ $(call __check_defined,$1,$(strip $(value 2))))) __check_defined = \ $(if $(value $1),, \ - $(error Failed to retrieve $1. Check the defined GOOGLE_APP_PROFILE value\ - and ensure the AWS_PROFILE variable is appropriately defined) ) + $(error Failed to define $1. $(if $2, $2))) From 593601c7a698f307ba51cfe7381e1b671a8f8cca Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Wed, 24 Jan 2024 10:07:26 +0000 Subject: [PATCH 04/27] Split google secret retrieval into independent ENV targets --- Makefile | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 7d299ece..d24d18e3 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ build: ENV.VERSION_TAG clean build/ ui-build build/${DEPLOY_JAR} \ --build-arg uberjar_path=build/${DEPLOY_JAR} . .PHONY: ui-build -ui-build: google-oauth2-secrets \ +ui-build: ENV.GOOGLE_OAUTH_CLIENT_ID \ $(call print-help,ui-build,\ Build JS and CSS file for release.) @ export REACT_APP_GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID} && \ @@ -270,17 +270,36 @@ run-dev-ui: google-oauth2-secrets\ npm install && \ npm run start -.PHONY: google-oauth2-secrets -google-oauth2-secrets: \ - $(call print-help,google-oauth2-secrets,\ - Store the Google oauth2 client details as env variables.) +.PHONY: ENV.GOOGLE_OAUTH_CLIENT_ID +ENV.GOOGLE_OAUTH_CLIENT_ID: \ + $(call print-help,ENV.GOOGLE_OAUTH_CLIENT_ID,\ + Retrieve the GOOGLE_OAUTH_CLIENT_ID env variable for make targets from aws ssm if undefined.) +ifeq (${GOOGLE_OAUTH_CLIENT_ID},) $(eval GOOGLE_OAUTH_CLIENT_ID = $(shell aws ssm get-parameter --name "/name-service/${GOOGLE_APP_PROFILE}/google-oauth2-app-config/client-id" --query "Parameter.Value" --output text --with-decryption)) $(call check_defined, GOOGLE_OAUTH_CLIENT_ID, Check the defined GOOGLE_APP_PROFILE value\ and ensure the AWS_PROFILE variable is appropriately defined) + @echo "Retrieved GOOGLE_OAUTH_CLIENT_ID from AWS SSM (GOOGLE_APP_PROFILE '${GOOGLE_APP_PROFILE}')." +else + @echo "Using predefined GOOGLE_OAUTH_CLIENT_ID." +endif + +.PHONY: ENV.GOOGLE_OAUTH_CLIENT_SECRET +ENV.GOOGLE_OAUTH_CLIENT_SECRET: \ + $(call print-help,ENV.GOOGLE_OAUTH_CLIENT_SECRET,\ + Retrieve the GOOGLE_OAUTH_CLIENT_SECRET env variable for make targets from aws ssm if undefined.) +ifeq (${GOOGLE_OAUTH_CLIENT_SECRET},) $(eval GOOGLE_OAUTH_CLIENT_SECRET = $(shell aws ssm get-parameter --name "/name-service/${GOOGLE_APP_PROFILE}/google-oauth2-app-config/client-secret" --query "Parameter.Value" --output text --with-decryption)) $(call check_defined, GOOGLE_OAUTH_CLIENT_SECRET, Check the defined GOOGLE_APP_PROFILE value\ and ensure the AWS_PROFILE variable is appropriately defined) - @echo "Retrieved google-oauth2-secrets for GOOGLE_APP_PROFILE '${GOOGLE_APP_PROFILE}'." + @echo "Retrieved GOOGLE_OAUTH_CLIENT_SECRET from AWS SSM (GOOGLE_APP_PROFILE '${GOOGLE_APP_PROFILE}')." +else + @echo "Using predefined GOOGLE_OAUTH_CLIENT_SECRET." +endif + +.PHONY: google-oauth2-secrets +google-oauth2-secrets: ENV.GOOGLE_OAUTH_CLIENT_ID ENV.GOOGLE_OAUTH_CLIENT_SECRET \ + $(call print-help,google-oauth2-secrets,\ + Store the Google oauth2 client details as env variables.) # Check that given variables are set and all have non-empty values, # die with an error otherwise. From f241ca09a6a58052ecb3141fc524ec2efc93a540 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Wed, 24 Jan 2024 16:16:16 +0000 Subject: [PATCH 05/27] Enabled (google) secrets storage and sourcing in Makefile --- Makefile | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index d24d18e3..e36d7d62 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,9 @@ else GOOGLE_REDIRECT_URI ?= ${LOCAL_GOOGLE_REDIRECT_URI} GOOGLE_APP_PROFILE ?= "dev" endif + +STORE_SECRETS_FILE = secrets.makedef + DEPLOY_JAR := app.jar PORT := 3000 WB_ACC_NUM := 357210185381 @@ -49,6 +52,13 @@ need-help := $(filter help,$(MAKECMDGOALS)) help: ; @echo $(if $(need-help),,\ Type \'$(MAKE)$(dash-f) help\' to get help) +source-secrets: +ifneq ($(SECRETS_SRC), ) + @echo "Sourcing secrets file ${SECRETS_SRC}" + $(eval SECRETS_CONTENT := $(shell cat "${SECRETS_SRC}")) + $(foreach secret,${SECRETS_CONTENT},$(eval ${secret})) +endif + .PHONY: ENV.VERSION_TAG ENV.VERSION_TAG: \ $(call print-help,ENV.VERSION_TAG,\ @@ -271,36 +281,35 @@ run-dev-ui: google-oauth2-secrets\ npm run start .PHONY: ENV.GOOGLE_OAUTH_CLIENT_ID -ENV.GOOGLE_OAUTH_CLIENT_ID: \ +ENV.GOOGLE_OAUTH_CLIENT_ID: source-secrets \ $(call print-help,ENV.GOOGLE_OAUTH_CLIENT_ID,\ Retrieve the GOOGLE_OAUTH_CLIENT_ID env variable for make targets from aws ssm if undefined.) -ifeq (${GOOGLE_OAUTH_CLIENT_ID},) - $(eval GOOGLE_OAUTH_CLIENT_ID = $(shell aws ssm get-parameter --name "/name-service/${GOOGLE_APP_PROFILE}/google-oauth2-app-config/client-id" --query "Parameter.Value" --output text --with-decryption)) + $(eval ACTION_MSG := $(if ${GOOGLE_OAUTH_CLIENT_ID},"Using predefined GOOGLE_OAUTH_CLIENT_ID.","Retrieving GOOGLE_OAUTH_CLIENT_ID from AWS SSM (GOOGLE_APP_PROFILE '${GOOGLE_APP_PROFILE}').")) + @echo ${ACTION_MSG} + $(if ${GOOGLE_OAUTH_CLIENT_ID},,$(eval GOOGLE_OAUTH_CLIENT_ID := $(shell aws ssm get-parameter --name "/name-service/${GOOGLE_APP_PROFILE}/google-oauth2-app-config/client-id" --query "Parameter.Value" --output text --with-decryption))) $(call check_defined, GOOGLE_OAUTH_CLIENT_ID, Check the defined GOOGLE_APP_PROFILE value\ and ensure the AWS_PROFILE variable is appropriately defined) - @echo "Retrieved GOOGLE_OAUTH_CLIENT_ID from AWS SSM (GOOGLE_APP_PROFILE '${GOOGLE_APP_PROFILE}')." -else - @echo "Using predefined GOOGLE_OAUTH_CLIENT_ID." -endif .PHONY: ENV.GOOGLE_OAUTH_CLIENT_SECRET -ENV.GOOGLE_OAUTH_CLIENT_SECRET: \ +ENV.GOOGLE_OAUTH_CLIENT_SECRET: source-secrets \ $(call print-help,ENV.GOOGLE_OAUTH_CLIENT_SECRET,\ Retrieve the GOOGLE_OAUTH_CLIENT_SECRET env variable for make targets from aws ssm if undefined.) -ifeq (${GOOGLE_OAUTH_CLIENT_SECRET},) - $(eval GOOGLE_OAUTH_CLIENT_SECRET = $(shell aws ssm get-parameter --name "/name-service/${GOOGLE_APP_PROFILE}/google-oauth2-app-config/client-secret" --query "Parameter.Value" --output text --with-decryption)) + $(eval ACTION_MSG := $(if ${GOOGLE_OAUTH_CLIENT_SECRET},"Using predefined GOOGLE_OAUTH_CLIENT_SECRET.","Retrieving GOOGLE_OAUTH_CLIENT_SECRET from AWS SSM (GOOGLE_APP_PROFILE '${GOOGLE_APP_PROFILE}').")) + @echo ${ACTION_MSG} + $(if ${GOOGLE_OAUTH_CLIENT_SECRET},,$(eval GOOGLE_OAUTH_CLIENT_SECRET := $(shell aws ssm get-parameter --name "/name-service/${GOOGLE_APP_PROFILE}/google-oauth2-app-config/client-secret" --query "Parameter.Value" --output text --with-decryption))) $(call check_defined, GOOGLE_OAUTH_CLIENT_SECRET, Check the defined GOOGLE_APP_PROFILE value\ and ensure the AWS_PROFILE variable is appropriately defined) - @echo "Retrieved GOOGLE_OAUTH_CLIENT_SECRET from AWS SSM (GOOGLE_APP_PROFILE '${GOOGLE_APP_PROFILE}')." -else - @echo "Using predefined GOOGLE_OAUTH_CLIENT_SECRET." -endif .PHONY: google-oauth2-secrets -google-oauth2-secrets: ENV.GOOGLE_OAUTH_CLIENT_ID ENV.GOOGLE_OAUTH_CLIENT_SECRET \ +google-oauth2-secrets: source-secrets ENV.GOOGLE_OAUTH_CLIENT_ID ENV.GOOGLE_OAUTH_CLIENT_SECRET \ $(call print-help,google-oauth2-secrets,\ Store the Google oauth2 client details as env variables.) +${STORE_SECRETS_FILE}: google-oauth2-secrets + @install -m 600 /dev/null ${STORE_SECRETS_FILE} + @echo "GOOGLE_OAUTH_CLIENT_ID:=${GOOGLE_OAUTH_CLIENT_ID}" >> ${STORE_SECRETS_FILE} + @echo "GOOGLE_OAUTH_CLIENT_SECRET:=${GOOGLE_OAUTH_CLIENT_SECRET}" >> ${STORE_SECRETS_FILE} + # Check that given variables are set and all have non-empty values, # die with an error otherwise. # From 4ea0c50afd431d31e964b5c9dd057c965cd8aafd Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Thu, 25 Jan 2024 11:42:47 +0000 Subject: [PATCH 06/27] Enabled building ui without nvm usage (for CI/docker context) --- .github/workflows/PR-validation.yml | 2 +- Makefile | 22 +++++++++++----------- README.md | 4 ++-- scripts/build-ui.sh | 19 +++++++++++++------ 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index 69e1ada3..c09227a2 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -50,5 +50,5 @@ jobs: clojure -Spom - name: Run Integration tests run: | - make run-tests GOOGLE_APP_PROFILE=dev + make run-tests APP_PROFILE=dev #TODO: add UI and API build and container packaging test diff --git a/Makefile b/Makefile index e36d7d62..0439c98b 100644 --- a/Makefile +++ b/Makefile @@ -5,17 +5,17 @@ LOCAL_GOOGLE_REDIRECT_URI := "http://lvh.me:3000" ifeq ($(PROJ_NAME), wormbase-names) WB_DB_URI ?= "datomic:ddb://us-east-1/WSNames/wormbase" GOOGLE_REDIRECT_URI ?= "https://names.wormbase.org" - GOOGLE_APP_PROFILE ?= "prod" + APP_PROFILE ?= "prod" else ifeq ($(PROJ_NAME), wormbase-names-test) WB_DB_URI ?= "datomic:ddb://us-east-1/WSNames-test-14/wormbase" GOOGLE_REDIRECT_URI ?= "https://test-names.wormbase.org" - GOOGLE_APP_PROFILE ?= "prod" + APP_PROFILE ?= "prod" else WB_DB_URI ?= "datomic:ddb-local://localhost:8000/WBNames_local/wormbase" # Ensure GOOGLE_REDIRECT_URI is defined appropriately as an env variable or CLI argument # if intended for AWS deployment (default is set for local execution) GOOGLE_REDIRECT_URI ?= ${LOCAL_GOOGLE_REDIRECT_URI} - GOOGLE_APP_PROFILE ?= "dev" + APP_PROFILE ?= "dev" endif STORE_SECRETS_FILE = secrets.makedef @@ -102,8 +102,8 @@ ui-build: ENV.GOOGLE_OAUTH_CLIENT_ID \ $(call print-help,ui-build,\ Build JS and CSS file for release.) @ export REACT_APP_GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID} && \ - echo "Building UI using GOOGLE_APP_PROFILE: '${GOOGLE_APP_PROFILE}'" && \ - ./scripts/build-ui.sh + echo "Building UI using APP_PROFILE: '${APP_PROFILE}'" && \ + ./scripts/build-ui.sh ${APP_PROFILE} .PHONY: clean clean: \ @@ -284,20 +284,20 @@ run-dev-ui: google-oauth2-secrets\ ENV.GOOGLE_OAUTH_CLIENT_ID: source-secrets \ $(call print-help,ENV.GOOGLE_OAUTH_CLIENT_ID,\ Retrieve the GOOGLE_OAUTH_CLIENT_ID env variable for make targets from aws ssm if undefined.) - $(eval ACTION_MSG := $(if ${GOOGLE_OAUTH_CLIENT_ID},"Using predefined GOOGLE_OAUTH_CLIENT_ID.","Retrieving GOOGLE_OAUTH_CLIENT_ID from AWS SSM (GOOGLE_APP_PROFILE '${GOOGLE_APP_PROFILE}').")) + $(eval ACTION_MSG := $(if ${GOOGLE_OAUTH_CLIENT_ID},"Using predefined GOOGLE_OAUTH_CLIENT_ID.","Retrieving GOOGLE_OAUTH_CLIENT_ID from AWS SSM (APP_PROFILE '${APP_PROFILE}').")) @echo ${ACTION_MSG} - $(if ${GOOGLE_OAUTH_CLIENT_ID},,$(eval GOOGLE_OAUTH_CLIENT_ID := $(shell aws ssm get-parameter --name "/name-service/${GOOGLE_APP_PROFILE}/google-oauth2-app-config/client-id" --query "Parameter.Value" --output text --with-decryption))) - $(call check_defined, GOOGLE_OAUTH_CLIENT_ID, Check the defined GOOGLE_APP_PROFILE value\ + $(if ${GOOGLE_OAUTH_CLIENT_ID},,$(eval GOOGLE_OAUTH_CLIENT_ID := $(shell aws ssm get-parameter --name "/name-service/${APP_PROFILE}/google-oauth2-app-config/client-id" --query "Parameter.Value" --output text --with-decryption))) + $(call check_defined, GOOGLE_OAUTH_CLIENT_ID, Check the defined APP_PROFILE value\ and ensure the AWS_PROFILE variable is appropriately defined) .PHONY: ENV.GOOGLE_OAUTH_CLIENT_SECRET ENV.GOOGLE_OAUTH_CLIENT_SECRET: source-secrets \ $(call print-help,ENV.GOOGLE_OAUTH_CLIENT_SECRET,\ Retrieve the GOOGLE_OAUTH_CLIENT_SECRET env variable for make targets from aws ssm if undefined.) - $(eval ACTION_MSG := $(if ${GOOGLE_OAUTH_CLIENT_SECRET},"Using predefined GOOGLE_OAUTH_CLIENT_SECRET.","Retrieving GOOGLE_OAUTH_CLIENT_SECRET from AWS SSM (GOOGLE_APP_PROFILE '${GOOGLE_APP_PROFILE}').")) + $(eval ACTION_MSG := $(if ${GOOGLE_OAUTH_CLIENT_SECRET},"Using predefined GOOGLE_OAUTH_CLIENT_SECRET.","Retrieving GOOGLE_OAUTH_CLIENT_SECRET from AWS SSM (APP_PROFILE '${APP_PROFILE}').")) @echo ${ACTION_MSG} - $(if ${GOOGLE_OAUTH_CLIENT_SECRET},,$(eval GOOGLE_OAUTH_CLIENT_SECRET := $(shell aws ssm get-parameter --name "/name-service/${GOOGLE_APP_PROFILE}/google-oauth2-app-config/client-secret" --query "Parameter.Value" --output text --with-decryption))) - $(call check_defined, GOOGLE_OAUTH_CLIENT_SECRET, Check the defined GOOGLE_APP_PROFILE value\ + $(if ${GOOGLE_OAUTH_CLIENT_SECRET},,$(eval GOOGLE_OAUTH_CLIENT_SECRET := $(shell aws ssm get-parameter --name "/name-service/${APP_PROFILE}/google-oauth2-app-config/client-secret" --query "Parameter.Value" --output text --with-decryption))) + $(call check_defined, GOOGLE_OAUTH_CLIENT_SECRET, Check the defined APP_PROFILE value\ and ensure the AWS_PROFILE variable is appropriately defined) .PHONY: google-oauth2-secrets diff --git a/README.md b/README.md index c3aa6a57..efb013a1 100644 --- a/README.md +++ b/README.md @@ -299,7 +299,7 @@ To deploy an update for the main application, change your working dir to the repository root dir and execute the following commands (bash): ```bash # Build the client application to ensure no errors occur. -make ui-build GOOGLE_APP_PROFILE=prod +make ui-build APP_PROFILE=prod # Generate the pom.xml file (not version-controlled) # to ensure no errors occur (in API code) @@ -332,7 +332,7 @@ sudo service docker start # NOTE: To deploy a tagged or branched codeversion that does not equal your (potentially dirty) working-dir content, # use the additional argument REF_NAME= # E.g. make release AWS_PROFILE=wormbase REF_NAME=wormbase-names-1.4.7 -make release [AWS_PROFILE=] GOOGLE_APP_PROFILE=prod +make release [AWS_PROFILE=] APP_PROFILE=prod # Deploy the application to an EB environmnent. # Before execution: diff --git a/scripts/build-ui.sh b/scripts/build-ui.sh index 5f4401c2..b1fc439b 100755 --- a/scripts/build-ui.sh +++ b/scripts/build-ui.sh @@ -1,9 +1,16 @@ #!/bin/bash +APP_PROFILE=$1 -export NVM_DIR="$HOME/.nvm" - -[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm -[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion cd client/ -nvm exec npm ci -nvm exec npm run build + +if [ "${APP_PROFILE}" == "dev" ]; then + echo "Activating nvm" + export NVM_DIR="$HOME/.nvm" + + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm + [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion + nvm use +fi + +npm ci +npm run build From 069ff778c49f98e43d438401a611e05f5c2ca65e Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Thu, 25 Jan 2024 17:45:24 +0000 Subject: [PATCH 07/27] Support building app jar to custom path --- Makefile | 14 +++++++++----- scripts/build-appjar.sh | 12 +++++------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 0439c98b..a9522c8b 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ endif STORE_SECRETS_FILE = secrets.makedef -DEPLOY_JAR := app.jar +APP_JAR_PATH ?= build/app.jar PORT := 3000 WB_ACC_NUM := 357210185381 @@ -109,12 +109,16 @@ ui-build: ENV.GOOGLE_OAUTH_CLIENT_ID \ clean: \ $(call print-help,clean,\ Remove the locally built JAR file.) - @rm -f ./build/${DEPLOY_JAR} + @rm -f ${APP_JAR_PATH} -build/${DEPLOY_JAR}: \ - $(call print-help,build/${DEPLOY_JAR},\ +${APP_JAR_PATH}: build/ \ + $(call print-help,${APP_JAR_PATH},\ Build the jar file.) - @./scripts/build-appjar.sh + @./scripts/build-appjar.sh ${APP_JAR_PATH} + +build-app-jar: ${APP_JAR_PATH} \ + $(call print-help,build-app-jar,\ + Build the jar file.) .PHONY: docker-build docker-build: clean build \ diff --git a/scripts/build-appjar.sh b/scripts/build-appjar.sh index d4576157..af79b3c0 100755 --- a/scripts/build-appjar.sh +++ b/scripts/build-appjar.sh @@ -1,19 +1,17 @@ #!/bin/bash -ARTEFACT="build/app.jar" -LATEST_TAG=$(git describe --abbrev=0) -RELEASE_NAME="wormbase-names-${LATEST_TAG}" -DEPLOY_JAR="target/${RELEASE_NAME}.jar" +ARTEFACT=$1 +TMP_JAR=$(mktemp --dry-run target/wb-names-app-XXXXXXXXXX.jar) rm -rf target mkdir -p target clj -Spom -clj -A:logging:prod:datomic-pro:webassets:depstar -m hf.depstar.uberjar "${DEPLOY_JAR}" +clj -A:logging:prod:datomic-pro:webassets:depstar -m hf.depstar.uberjar "${TMP_JAR}" if [ $? -eq 0 ]; then - mv "${DEPLOY_JAR}" "${ARTEFACT}" - echo "${ARTEFACT}" + mv "${TMP_JAR}" "${ARTEFACT}" exit 0 else + echo >&2 "Failed to build jar '${TMP_JAR}'" exit 1 fi From 8a7d6b55590eef8bf6e31fdc7b2349b8d4f1b27a Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Thu, 25 Jan 2024 17:49:11 +0000 Subject: [PATCH 08/27] Make build target naming consistent --- Makefile | 6 +++--- README.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index a9522c8b..eb8bf0c6 100644 --- a/Makefile +++ b/Makefile @@ -97,9 +97,9 @@ build: ENV.VERSION_TAG clean build/ ui-build build/${DEPLOY_JAR} \ @docker build -t ${ECR_REPO_NAME}:${VERSION_TAG} \ --build-arg uberjar_path=build/${DEPLOY_JAR} . -.PHONY: ui-build -ui-build: ENV.GOOGLE_OAUTH_CLIENT_ID \ - $(call print-help,ui-build,\ +.PHONY: build-ui +build-ui: ENV.GOOGLE_OAUTH_CLIENT_ID \ + $(call print-help,build-ui,\ Build JS and CSS file for release.) @ export REACT_APP_GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID} && \ echo "Building UI using APP_PROFILE: '${APP_PROFILE}'" && \ diff --git a/README.md b/README.md index efb013a1..77070453 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ Correct functionality of the client app can be tested in two ways: - Making a production build of the client app. Failure during this process means fixes will be needed before deployment. ```bash # performs a npm clean install of dependencies based on package-lock.json -make ui-build +make build-ui ``` To start up a local client development server: @@ -299,7 +299,7 @@ To deploy an update for the main application, change your working dir to the repository root dir and execute the following commands (bash): ```bash # Build the client application to ensure no errors occur. -make ui-build APP_PROFILE=prod +make build-ui APP_PROFILE=prod # Generate the pom.xml file (not version-controlled) # to ensure no errors occur (in API code) From bb678e9d8621ba74b84c187fcac9b4c863f97c80 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 15:00:22 +0000 Subject: [PATCH 09/27] Build complete application in Dockerfile. This ensures container building process is build-environment agnostic. --- Dockerfile | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++---- Makefile | 9 ++++++--- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index cc6732dd..60f299c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,55 @@ -FROM openjdk:8-jre-alpine +ARG WORKDIR=/wb_names +ARG APPJAR_BUILDPATH=build/wb-names-app.jar -ARG uberjar_path= -ADD $uberjar_path /srv/app.jar +### Stage 1: build UI +FROM node:12 AS BUILD_UI_STAGE + +ARG WORKDIR + +WORKDIR $WORKDIR +COPY Makefile ./ +COPY client/ ./client/ +COPY scripts/ ./scripts/ + +RUN --mount=type=secret,id=make-secrets-file,required=true \ + make build-ui SECRETS_SRC=/run/secrets/make-secrets-file APP_PROFILE=prod + +### Stage 2: build API (and include UI components) +FROM clojure:temurin-8-tools-deps-jammy as BUILD_API_STAGE + +ARG WORKDIR +ARG APPJAR_BUILDPATH + +RUN apt update && apt upgrade -y && apt install -y maven unzip + +#Install clojure manually (com.datomic/datomic-pro 1.0.6165 is not stored on maven central) +COPY build/datomic-pro-1.0.6165.zip datomic-pro-1.0.6165.zip +RUN unzip datomic-pro-1.0.6165.zip \ + && cd datomic-pro-1.0.6165/ \ + && bin/maven-install + +WORKDIR $WORKDIR +COPY Makefile ./ +COPY src/wormbase/ src/wormbase/ +COPY resources/ ./resources/ +COPY deps.edn ./ +COPY project.clj ./ +COPY scripts/ ./scripts/ +COPY --from=BUILD_UI_STAGE $WORKDIR/client/ $WORKDIR/client/ + +RUN make build-app-jar APP_JAR_PATH=$APPJAR_BUILDPATH + +### Stage 3: build final application image +FROM openjdk:8-jre-alpine as APPLICATION_IMAGE_STAGE + +ARG WORKDIR +ARG APPJAR_BUILDPATH + +RUN apk update && apk upgrade + +COPY --from=BUILD_API_STAGE $WORKDIR/$APPJAR_BUILDPATH /srv/wb-names-app.jar # Expose necessary ports EXPOSE 3000 -CMD ["java", "-cp", "/srv/app.jar", "clojure.main", "-m", "wormbase.names.service"] +CMD ["java", "-cp", "/srv/wb-names-app.jar", "clojure.main", "-m", "wormbase.names.service"] diff --git a/Makefile b/Makefile index eb8bf0c6..11759b5e 100644 --- a/Makefile +++ b/Makefile @@ -91,11 +91,14 @@ build/: mkdir build/ .PHONY: build -build: ENV.VERSION_TAG clean build/ ui-build build/${DEPLOY_JAR} \ +build: build/ ENV.VERSION_TAG ${STORE_SECRETS_FILE} \ $(call print-help,build,\ - Build the docker images from using the current git revision.) + Build the docker images from the current git revision.) + @aws s3 cp s3://wormbase/datomic-pro/distro/datomic-pro-1.0.6165.zip build/ @docker build -t ${ECR_REPO_NAME}:${VERSION_TAG} \ - --build-arg uberjar_path=build/${DEPLOY_JAR} . + --secret id=make-secrets-file,src=${STORE_SECRETS_FILE} \ + . + @rm ${STORE_SECRETS_FILE} .PHONY: build-ui build-ui: ENV.GOOGLE_OAUTH_CLIENT_ID \ From 1a0ca06eb649df6b69b882392547b9dab9a726e4 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 15:04:24 +0000 Subject: [PATCH 10/27] Enable make run using AWS_PROFILE for AWS authentication (DB access) --- Makefile | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 11759b5e..e855073a 100644 --- a/Makefile +++ b/Makefile @@ -219,17 +219,30 @@ run: ENV.VERSION_TAG \ $(call print-help,run [PORT=] [PROJ_NAME=] \ [WB_DB_URI=] [GOOGLE_REDIRECT_URI=],\ Run the application in docker (locally).) - @docker run \ + + $(eval RUN_CMD = docker run \ --name ${PROJ_NAME} \ --publish-all=true \ --publish ${PORT}:${PORT} \ --detach \ - -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \ - -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \ -e WB_DB_URI=${WB_DB_URI} \ -e GOOGLE_REDIRECT_URI=${GOOGLE_REDIRECT_URI} \ - -e PORT=${PORT} \ - ${ECR_REPO_NAME}:${VERSION_TAG} + -e PORT=${PORT}) +ifneq (${AWS_ACCESS_KEY_ID},) +ifneq (${AWS_SECRET_ACCESS_KEY},) + $(eval RUN_CMD = ${RUN_CMD} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}) + $(eval RUN_CMD = ${RUN_CMD} -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}) +else + @echo 'ENV var "AWS_ACCESS_KEY_ID" is defined but "AWS_SECRET_ACCESS_KEY" is not. Either define both or none.' >&2 + @exit 1 +endif +else +ifneq (${AWS_PROFILE},) + $(eval RUN_CMD = ${RUN_CMD} -e AWS_PROFILE=${AWS_PROFILE} -v ~/.aws:/root/.aws) +endif +endif + + ${RUN_CMD} ${ECR_REPO_NAME}:${VERSION_TAG} .PHONY: docker-clean docker-clean: \ From 1ecec25b0f6c5a25f69a05f9c467ad787bb1bae3 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 15:04:47 +0000 Subject: [PATCH 11/27] Minor cleanup --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e855073a..70271548 100644 --- a/Makefile +++ b/Makefile @@ -321,7 +321,7 @@ ENV.GOOGLE_OAUTH_CLIENT_SECRET: source-secrets \ and ensure the AWS_PROFILE variable is appropriately defined) .PHONY: google-oauth2-secrets -google-oauth2-secrets: source-secrets ENV.GOOGLE_OAUTH_CLIENT_ID ENV.GOOGLE_OAUTH_CLIENT_SECRET \ +google-oauth2-secrets: ENV.GOOGLE_OAUTH_CLIENT_ID ENV.GOOGLE_OAUTH_CLIENT_SECRET \ $(call print-help,google-oauth2-secrets,\ Store the Google oauth2 client details as env variables.) From 9037191a7380ac4710ec6161b20110e2422b6073 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 16:22:22 +0000 Subject: [PATCH 12/27] Run target naming cleanup --- Makefile | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 70271548..dc675855 100644 --- a/Makefile +++ b/Makefile @@ -90,10 +90,10 @@ show-version: ENV.VERSION_TAG \ build/: mkdir build/ -.PHONY: build -build: build/ ENV.VERSION_TAG ${STORE_SECRETS_FILE} \ - $(call print-help,build,\ - Build the docker images from the current git revision.) +.PHONY: build-docker-image +build-docker-image: build/ ENV.VERSION_TAG ${STORE_SECRETS_FILE} \ + $(call print-help,build,\ + Build the docker image from the current git revision.) @aws s3 cp s3://wormbase/datomic-pro/distro/datomic-pro-1.0.6165.zip build/ @docker build -t ${ECR_REPO_NAME}:${VERSION_TAG} \ --secret id=make-secrets-file,src=${STORE_SECRETS_FILE} \ @@ -111,8 +111,9 @@ build-ui: ENV.GOOGLE_OAUTH_CLIENT_ID \ .PHONY: clean clean: \ $(call print-help,clean,\ - Remove the locally built JAR file.) + Remove the locally built UI and API artefacts.) @rm -f ${APP_JAR_PATH} + @rm -rf client/build/* ${APP_JAR_PATH}: build/ \ $(call print-help,${APP_JAR_PATH},\ @@ -123,10 +124,10 @@ build-app-jar: ${APP_JAR_PATH} \ $(call print-help,build-app-jar,\ Build the jar file.) -.PHONY: docker-build -docker-build: clean build \ - $(call print-help,docker-build,\ - Create docker container.) +.PHONY: build-local +build-local: clean build-ui build-app-jar \ + $(call print-help,build-local,\ + Build UI and app JAR (locally).) .PHONY: docker-ecr-login docker-ecr-login: \ @@ -214,11 +215,11 @@ eb-local: docker-ecr-login \ Runs the ElasticBeanStalk/docker build and run locally.) @eb local run --envvars PORT=${PORT},WB_DB_URI=${WB_DB_URI},GOOGLE_REDIRECT_URI=${GOOGLE_REDIRECT_URI} -.PHONY: run -run: ENV.VERSION_TAG \ - $(call print-help,run [PORT=] [PROJ_NAME=] \ - [WB_DB_URI=] [GOOGLE_REDIRECT_URI=],\ - Run the application in docker (locally).) +.PHONY: run-docker +run-docker: ENV.VERSION_TAG clean-docker-run \ + $(call print-help,run [PORT=] [PROJ_NAME=] \ + [WB_DB_URI=] [GOOGLE_REDIRECT_URI=],\ + Run the application docker container (locally).) $(eval RUN_CMD = docker run \ --name ${PROJ_NAME} \ @@ -244,9 +245,9 @@ endif ${RUN_CMD} ${ECR_REPO_NAME}:${VERSION_TAG} -.PHONY: docker-clean -docker-clean: \ - $(call print-help,docker-clean [PROJ_NAME=],\ +.PHONY: clean-docker-run +clean-docker-run: \ + $(call print-help,clean-docker-run [PROJ_NAME=],\ Stop and remove the docker container (if running).) @docker stop ${PROJ_NAME} @docker rm ${PROJ_NAME} @@ -278,7 +279,7 @@ run-tests: google-oauth2-secrets \ export GOOGLE_REDIRECT_URI=${LOCAL_GOOGLE_REDIRECT_URI} && \ clojure -A:datomic-pro:logging:webassets:dev:test:run-tests -.PHONY: run-dev-server +.PHONY: run-dev-webserver run-dev-webserver: PORT := 4010 run-dev-webserver: google-oauth2-secrets \ $(call print-help,run-dev-webserver PORT= WB_DB_URI= \ From c2fe52af6c2b23380022fc812c1813c44da56e2a Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 16:26:25 +0000 Subject: [PATCH 13/27] Added .dockerIgnore to reduce docker image build-time and size --- .dockerignore | 6 ++++++ .gitignore | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..a7bfaf60 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +client/node_modules/ +client/build/ +target/ +.github/ +.ebextensions/ +.elasticbeanstalk/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 8386fb7f..9d9861ae 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,6 @@ target/ !.elasticbeanstalk/*.global.yml Dockerrun.aws.json build/* -/.dockerignore #Local docker-compose env-variable files .env From ffee94ec8755cb1a36f3e3b0998c51fe0eb18ea5 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 16:32:45 +0000 Subject: [PATCH 14/27] Moved datomic bundle download into separate target (for improved repeat-performance) --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dc675855..79c5262c 100644 --- a/Makefile +++ b/Makefile @@ -90,11 +90,14 @@ show-version: ENV.VERSION_TAG \ build/: mkdir build/ +build/datomic-pro-1.0.6165.zip: + @echo "Downloading datomic bundle from S3." + @aws s3 cp s3://wormbase/datomic-pro/distro/datomic-pro-1.0.6165.zip build/ + .PHONY: build-docker-image -build-docker-image: build/ ENV.VERSION_TAG ${STORE_SECRETS_FILE} \ +build-docker-image: build/ ENV.VERSION_TAG ${STORE_SECRETS_FILE} build/datomic-pro-1.0.6165.zip \ $(call print-help,build,\ Build the docker image from the current git revision.) - @aws s3 cp s3://wormbase/datomic-pro/distro/datomic-pro-1.0.6165.zip build/ @docker build -t ${ECR_REPO_NAME}:${VERSION_TAG} \ --secret id=make-secrets-file,src=${STORE_SECRETS_FILE} \ . From a7652b424d14c9ceebf9e0e55cabfbb7bd937395 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 16:33:59 +0000 Subject: [PATCH 15/27] Prevent accidental secrets leakage --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9d9861ae..fc9aa6bf 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ build/* # secrets /resources/secrets/ +secrets.makedef # Generated assets /resources/client_build From a4945219b4cb10b4a5e42b4c6cf8db35d52ce3a2 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 16:43:52 +0000 Subject: [PATCH 16/27] Added note to document current issues running run-docker using local DB --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index 79c5262c..6bf0d9bd 100644 --- a/Makefile +++ b/Makefile @@ -218,6 +218,14 @@ eb-local: docker-ecr-login \ Runs the ElasticBeanStalk/docker build and run locally.) @eb local run --envvars PORT=${PORT},WB_DB_URI=${WB_DB_URI},GOOGLE_REDIRECT_URI=${GOOGLE_REDIRECT_URI} +#Note: the run-docker command can currently only be used with non-local WB_DB_URI value. +# Current setup fails to connect to local datomic DB (on host, outside of container) +# from within the application container. +# Making the host's "localhost" accessible within the container as "host.docker.internal" +# through the following options in the `docker run` command makes the DB URI accessible +# from within the container, but the transactor still fails to be accessible. +# -e WB_DB_URI=datomic:ddb-local://host.docker.internal:8000/WBNames_local/wormbase +# --add-host=host.docker.internal:host-gateway .PHONY: run-docker run-docker: ENV.VERSION_TAG clean-docker-run \ $(call print-help,run [PORT=] [PROJ_NAME=] \ From 90e15d9aa00e5ab9d57520a8f604fa5c2ec79bca Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 17:00:52 +0000 Subject: [PATCH 17/27] Added container build and run testing to PR validation --- .github/workflows/PR-validation.yml | 36 ++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index f107cd19..97516ffc 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -56,4 +56,38 @@ jobs: - name: Run Integration tests run: | make run-tests APP_PROFILE=dev - #TODO: add UI and API build and container packaging test + build-and-run-container: + permissions: + id-token: write # Required for authentication through OIDC to AWS + runs-on: ubuntu-22.04 + steps: + - name: Check out repository code + uses: actions/checkout@v3 + - name: AWS credentials configuration + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}} + role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-test-api + aws-region: us-east-1 + mask-aws-account-id: true + - name: Build container image + run: | + make build-docker-imagedock + - name: Launch test instance of container (connecting to test env DB) + run: | + make run-docker PROJ_NAME=wormbase-names-test + sleep 60 + - name: Test API accessability + id: test-api-accessability + continue-on-error: true + run: | + curl -I http://localhost:3000/api/auth/identity + - name: Test UI accessability + id: test-ui-accessability + continue-on-error: true + run: | + curl -I http://localhost:3000/ + - name: Report container logs if either accessibility test fails + if: ${{ steps.test-ui-accessability.outcome == 'failure' || steps.test-api-accessability.outcome == 'failure' }} + run: | + docker logs wormbase-names-test From b7d1f797193cfffcf1a30c81bdb0560df99f345c Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 17:06:59 +0000 Subject: [PATCH 18/27] Fixed editor mishap --- .github/workflows/PR-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index 97516ffc..15fb312e 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -72,7 +72,7 @@ jobs: mask-aws-account-id: true - name: Build container image run: | - make build-docker-imagedock + make build-docker-image - name: Launch test instance of container (connecting to test env DB) run: | make run-docker PROJ_NAME=wormbase-names-test From 70e0451a1bcf7811caf38b5a0139d660832a3b8c Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 17:10:27 +0000 Subject: [PATCH 19/27] Allow retrieval of tags for image building in GH actions --- .github/workflows/PR-validation.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index 15fb312e..0ea1f719 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -63,6 +63,8 @@ jobs: steps: - name: Check out repository code uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: AWS credentials configuration uses: aws-actions/configure-aws-credentials@v4 with: From d7ccfd7ac40ad0672e78708b827197af4dbfc93f Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 19:36:54 +0000 Subject: [PATCH 20/27] Prevent errors on cleanup of non-existing containers --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6bf0d9bd..1d996d88 100644 --- a/Makefile +++ b/Makefile @@ -260,8 +260,10 @@ endif clean-docker-run: \ $(call print-help,clean-docker-run [PROJ_NAME=],\ Stop and remove the docker container (if running).) - @docker stop ${PROJ_NAME} - @docker rm ${PROJ_NAME} +# Stop if running container found + @echo $(if $(shell docker ps -q --filter name=${PROJ_NAME}),$(shell docker stop ${PROJ_NAME})) > /dev/null +# Remove if stopped container found + @echo $(if $(shell docker ps -a -q --filter name=${PROJ_NAME}),$(shell docker rm ${PROJ_NAME})) > /dev/null .PHONY: deploy-ecr deploy-ecr: docker-build docker-tag docker-push-ecr From 8d5eac98e7d4fbe743f8c3cdcaf575032ea3c2c9 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 19:52:31 +0000 Subject: [PATCH 21/27] Ensure container run tests cause correct job failure reporting on step failure --- .github/workflows/PR-validation.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index 0ea1f719..02e14bae 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -81,12 +81,10 @@ jobs: sleep 60 - name: Test API accessability id: test-api-accessability - continue-on-error: true run: | curl -I http://localhost:3000/api/auth/identity - name: Test UI accessability id: test-ui-accessability - continue-on-error: true run: | curl -I http://localhost:3000/ - name: Report container logs if either accessibility test fails From 4ea84b138ee69f93bffd9c5cbd387c5b4f38fb8a Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 20:12:21 +0000 Subject: [PATCH 22/27] Fix 2 to ensure container accessability tests cause correct job failure reporting on step failure --- .github/workflows/PR-validation.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index 02e14bae..d588d409 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -81,13 +81,19 @@ jobs: sleep 60 - name: Test API accessability id: test-api-accessability + continue-on-error: true run: | curl -I http://localhost:3000/api/auth/identity - name: Test UI accessability id: test-ui-accessability + continue-on-error: true run: | curl -I http://localhost:3000/ - name: Report container logs if either accessibility test fails if: ${{ steps.test-ui-accessability.outcome == 'failure' || steps.test-api-accessability.outcome == 'failure' }} run: | docker logs wormbase-names-test + - name: Report accessibility test failures + if: ${{ steps.test-ui-accessability.outcome == 'failure' || steps.test-api-accessability.outcome == 'failure' }} + run: | + echo "::setFailed title=Accessability tests failed::UI and/or API accessability tests failed." From 9522c4927337073a7246643e0b6dad7dd55dc95a Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 20:18:52 +0000 Subject: [PATCH 23/27] Attempt 3 to fix correct job failure reporting on run-accessability testing step failures --- .github/workflows/PR-validation.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index d588d409..74d5f080 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -95,5 +95,4 @@ jobs: docker logs wormbase-names-test - name: Report accessibility test failures if: ${{ steps.test-ui-accessability.outcome == 'failure' || steps.test-api-accessability.outcome == 'failure' }} - run: | - echo "::setFailed title=Accessability tests failed::UI and/or API accessability tests failed." + run: echo "::setFailed title=Accessability tests failed::UI and/or API accessability tests failed." From ae20125be17beadc260a3651d3d4c1295c54498f Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 20:30:11 +0000 Subject: [PATCH 24/27] Attempt 4 to fix correct job failure reporting on run-accessability testing step failures --- .github/workflows/PR-validation.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index 74d5f080..3a387cd9 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -93,6 +93,15 @@ jobs: if: ${{ steps.test-ui-accessability.outcome == 'failure' || steps.test-api-accessability.outcome == 'failure' }} run: | docker logs wormbase-names-test - - name: Report accessibility test failures + - name: Report UI accessibility test failures + if: ${{ steps.test-ui-accessability.outcome == 'failure' }} + run: | + echo "UI accessability test step failed." + - name: Report API accessibility test failures + if: ${{ steps.test-api-accessability.outcome == 'failure' }} + run: | + echo "API accessability test step failed." + - name: Fail if either accessibility test fails if: ${{ steps.test-ui-accessability.outcome == 'failure' || steps.test-api-accessability.outcome == 'failure' }} - run: echo "::setFailed title=Accessability tests failed::UI and/or API accessability tests failed." + run: | + exit 1 From cdf92ab6ea7e6d964a1a15dba44e7c358af57327 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 21:03:12 +0000 Subject: [PATCH 25/27] Define default AWS_DEFAULT_REGION value and pass through to container at runtime --- Makefile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 1d996d88..9c465b2f 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,13 @@ ECR_REPO_NAME := wormbase/names EB_APP_ENV_FILE := app-env.config PROJ_NAME ?= wormbase-names-dev LOCAL_GOOGLE_REDIRECT_URI := "http://lvh.me:3000" +AWS_DEFAULT_REGION ?= us-east-1 ifeq ($(PROJ_NAME), wormbase-names) - WB_DB_URI ?= "datomic:ddb://us-east-1/WSNames/wormbase" + WB_DB_URI ?= "datomic:ddb://${AWS_DEFAULT_REGION}/WSNames/wormbase" GOOGLE_REDIRECT_URI ?= "https://names.wormbase.org" APP_PROFILE ?= "prod" else ifeq ($(PROJ_NAME), wormbase-names-test) - WB_DB_URI ?= "datomic:ddb://us-east-1/WSNames-test-14/wormbase" + WB_DB_URI ?= "datomic:ddb://${AWS_DEFAULT_REGION}/WSNames-test-14/wormbase" GOOGLE_REDIRECT_URI ?= "https://test-names.wormbase.org" APP_PROFILE ?= "prod" else @@ -24,7 +25,7 @@ APP_JAR_PATH ?= build/app.jar PORT := 3000 WB_ACC_NUM := 357210185381 -ECR_URI := ${WB_ACC_NUM}.dkr.ecr.us-east-1.amazonaws.com +ECR_URI := ${WB_ACC_NUM}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com ECR_REPO_URI := ${ECR_URI}/${ECR_REPO_NAME} ECR_IMAGE_URI = ${ECR_REPO_URI}:${VERSION_TAG} # Set AWS (EB) profile env vars if undefined @@ -182,7 +183,7 @@ eb-create: eb-def-app-env \ && exit 1 \ ) @eb create ${PROJ_NAME} \ - --region=us-east-1 \ + --region=${AWS_DEFAULT_REGION} \ --tags="CreatedBy=${AWS_IAM_UNAME},Role=RestAPI" \ --cname="${PROJ_NAME}" \ -p docker \ @@ -209,6 +210,7 @@ eb-setenv: \ _JAVA_OPTIONS="-Xmx14g" \ AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ + AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" \ -e "${PROJ_NAME}" .PHONY: eb-local @@ -239,7 +241,8 @@ run-docker: ENV.VERSION_TAG clean-docker-run \ --detach \ -e WB_DB_URI=${WB_DB_URI} \ -e GOOGLE_REDIRECT_URI=${GOOGLE_REDIRECT_URI} \ - -e PORT=${PORT}) + -e PORT=${PORT} \ + -e AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}) ifneq (${AWS_ACCESS_KEY_ID},) ifneq (${AWS_SECRET_ACCESS_KEY},) $(eval RUN_CMD = ${RUN_CMD} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}) From af9a7ac4210fb9fa01a7167cff145769bd33b2cf Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 21:25:40 +0000 Subject: [PATCH 26/27] Attempt to pass through temporary AWS credentials retrieved through roles to application container --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 9c465b2f..8f6653dd 100644 --- a/Makefile +++ b/Makefile @@ -247,6 +247,9 @@ ifneq (${AWS_ACCESS_KEY_ID},) ifneq (${AWS_SECRET_ACCESS_KEY},) $(eval RUN_CMD = ${RUN_CMD} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}) $(eval RUN_CMD = ${RUN_CMD} -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}) +ifneq (${AWS_SESSION_TOKEN},) + $(eval RUN_CMD = ${RUN_CMD} -e AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}) +endif else @echo 'ENV var "AWS_ACCESS_KEY_ID" is defined but "AWS_SECRET_ACCESS_KEY" is not. Either define both or none.' >&2 @exit 1 From 82b7ca3cea8bb1765a79ce73df8e93c4067fd3a8 Mon Sep 17 00:00:00 2001 From: Manuel Luypaert Date: Fri, 26 Jan 2024 22:10:33 +0000 Subject: [PATCH 27/27] Allow more time for application to start up + silenced curl progress reporting --- .github/workflows/PR-validation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index 3a387cd9..e61d1b1a 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -78,17 +78,17 @@ jobs: - name: Launch test instance of container (connecting to test env DB) run: | make run-docker PROJ_NAME=wormbase-names-test - sleep 60 + sleep 120 - name: Test API accessability id: test-api-accessability continue-on-error: true run: | - curl -I http://localhost:3000/api/auth/identity + curl --no-progress-meter -I http://localhost:3000/api/auth/identity - name: Test UI accessability id: test-ui-accessability continue-on-error: true run: | - curl -I http://localhost:3000/ + curl --no-progress-meter -I http://localhost:3000/ - name: Report container logs if either accessibility test fails if: ${{ steps.test-ui-accessability.outcome == 'failure' || steps.test-api-accessability.outcome == 'failure' }} run: |