V2

.gitignore

@@ -182,5 +182,9 @@ dmypy.json
 # pytype static role analyzer
 .pytype/
 
+# vscode
+.vscode/
+
 ## Misc
 sample_data/**
+/aws_config.json

.gitlab-ci.yml

@@ -2,80 +2,43 @@ stages:
   - Build
   - Deploy
 
+include:
+  - project: "tethys/kubernetes/gl-k8s-integration"
+    ref: 1.6.1
+    file: "/helm-check.yml"
+  - project: "tethys/kubernetes/gl-k8s-integration"
+    ref: 1.6.1
+    file: "/kaniko-build.yml"
+
 ### Build ###
 "Check Helm":
   stage: Build
-  image: docker.aquaveo.com/tethys/kubernetes/helm-docker:0.7.1
   variables:
-    GIT_SUBMODULE_STRATEGY: none
+    HELM_REPO_URL: https://eccr.ecmwf.int/chartrepo
+    HELM_REPO_CHANNEL: geoglows_api
+    HELM_REPO_USERNAME: $ECCR_USER
+    HELM_REPO_PASSWORD: $ECCR_PASSWORD
     HELM_CHART: $CI_PROJECT_DIR/helm/gsprestapi
   script:
     # Ensure all required variables are set
     - ": \"${HELM_CHART:?must be set}\""
-    - helm lint $HELM_CHART
     - chmod +x ./check_helm_chart
     - ./check_helm_chart $HELM_CHART
-  except:
-   - tags
-   - master
 
 "Update Helm":
   stage: Build
-  image: docker.aquaveo.com/tethys/kubernetes/helm-docker:0.7.1
   variables:
-    GIT_SUBMODULE_STRATEGY: none
-    HELM_REPO_USERNAME: aquaveo
+    HELM_REPO_URL: https://eccr.ecmwf.int/chartrepo
+    HELM_REPO_CHANNEL: geoglows_api
+    HELM_REPO_USERNAME: $ECCR_USER
+    HELM_REPO_PASSWORD: $ECCR_PASSWORD
     HELM_CHART: $CI_PROJECT_DIR/helm/gsprestapi
   script:
     # Ensure all required variables are set
     - ": \"${HELM_CHART:?must be set}\""
-    - ": \"${HELM_KEY_PASSPHRASE:?must be set}\""
-    - ": \"${HELM_REPO_PASSWORD:?must be set}\""
     - helm lint $HELM_CHART
     - chmod +x ./check_helm_chart
     - ./check_helm_chart $HELM_CHART --upload
-  only:
-    - master
-
-.kaniko_build:
-  stage: Build
-  image:
-    name: gcr.io/kaniko-project/executor:debug-v0.23.0
-    entrypoint: [""]
-  variables:
-    GIT_SUBMODULE_STRATEGY: recursive
-  before_script:
-    - >
-      echo "{\"auths\":{
-        \"$CI_REGISTRY\": {\"username\":\"$CI_REGISTRY_USER\", \"password\":\"$CI_REGISTRY_PASSWORD\"},
-        \"eccr.ecmwf.int/harbor\": {\"username\":\"$ECCR_USER\", \"password\":\"$ECCR_PASSWORD\"}
-      }}" > /kaniko/.docker/config.json
-  script:
-    # Ensure all required variables are set
-    - ": \"${KANIKO_DOCKERFILE:?must be set}\""
-    - ": \"${KANIKO_CONTEXT:?must be set}\""
-    - ": \"${KANIKO_DESTINATIONS:?must be set}\""
-    # Run Kaniko
-    - export KANIKO_CMD="/kaniko/executor
-        --context $KANIKO_CONTEXT
-        --dockerfile $KANIKO_DOCKERFILE
-        $(for DEST in $KANIKO_DESTINATIONS; do echo "--destination $DEST "; done)
-        --cache=true
-        --cache-repo $CI_REGISTRY_IMAGE/cache
-        --force"
-    - echo $KANIKO_CMD
-    - $KANIKO_CMD
-
-"Build Commit":
-  extends: .kaniko_build
-  variables:
-    GIT_SUBMODULE_STRATEGY: recursive
-    KANIKO_CONTEXT: $CI_PROJECT_DIR
-    KANIKO_DOCKERFILE: $CI_PROJECT_DIR/Dockerfile
-    KANIKO_DESTINATIONS: $CI_REGISTRY_IMAGE/dev:$CI_COMMIT_SHORT_SHA
-
-  except:
-    - tags
 
 "Build Tag":
   extends: .kaniko_build
@@ -86,6 +49,12 @@ stages:
     KANIKO_DESTINATIONS: >
       eccr.ecmwf.int/geoglows_api/geoglows_api:$CI_COMMIT_TAG
       eccr.ecmwf.int/geoglows_api/geoglows_api:latest
+  before_script:
+    - >
+      echo "{\"auths\":{
+        \"$CI_REGISTRY\": {\"username\":\"$CI_REGISTRY_USER\", \"password\":\"$CI_REGISTRY_PASSWORD\"},
+        \"eccr.ecmwf.int/harbor\": {\"username\":\"$ECCR_USER\", \"password\":\"$ECCR_PASSWORD\"}
+      }}" > /kaniko/.docker/config.json
   only:
     - tags
 
@@ -95,13 +64,19 @@ stages:
     KANIKO_CONTEXT: $CI_PROJECT_DIR
     KANIKO_DOCKERFILE: $CI_PROJECT_DIR/Dockerfile
     KANIKO_DESTINATIONS: eccr.ecmwf.int/geoglows_api/geoglows_api:stable
+  before_script:
+    - >
+      echo "{\"auths\":{
+        \"$CI_REGISTRY\": {\"username\":\"$CI_REGISTRY_USER\", \"password\":\"$CI_REGISTRY_PASSWORD\"},
+        \"eccr.ecmwf.int/harbor\": {\"username\":\"$ECCR_USER\", \"password\":\"$ECCR_PASSWORD\"}
+      }}" > /kaniko/.docker/config.json
   only:
     - /^stable$/
 
 ### DEPLOY ###
 "Deploy to Staging":
   stage: Deploy
-  image: docker.aquaveo.com/tethys/kubernetes/helm-docker:2.0.0-rc1
+  image: aquaveollc/helm-docker:0.3.1
   dependencies: [] # Deploys do not depend on the Build phase artifacts
   variables:
     GIT_SUBMODULE_STRATEGY: none
@@ -115,7 +90,7 @@ stages:
 
 "Deploy to Production":
   stage: Deploy
-  image: docker.aquaveo.com/tethys/kubernetes/helm-docker:2.0.0-rc1
+  image: aquaveollc/helm-docker:0.3.1
   dependencies: [] # Deploys do not depend on the Build phase artifacts
   variables:
     GIT_SUBMODULE_STRATEGY: none
@@ -126,5 +101,3 @@ stages:
   when: manual
   only:
     - tags
-
-

Dockerfile

@@ -2,36 +2,23 @@ FROM continuumio/miniconda3:latest
 
 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PATH=/opt/conda/envs/gsp_api/bin:$PATH API_PREFIX=/api
 
-# For Development On Analytics Only:
-# ENV AWS_ACCESS_KEY_ID=
-# ENV AWS_SECRET_ACCESS_KEY=
-# ENV AWS_LOG_GROUP_NAME=
-# ENV AWS_LOG_STREAM_NAME=
-# ENV AWS_REGION=
-
 RUN mkdir /var/uwsgi
-
 RUN apt-get update -qq && apt-get install -yqq supervisor vim
 
-COPY ./environment.yml ./startup.sh ./
+COPY environment.yaml ./startup.sh ./
 
 RUN conda config --set channel_priority strict && \
     conda config --add channels conda-forge && \
-    conda env create -f environment.yml && \
-    echo "conda activate gsp_api" >> ~/.bashrc
+    conda env create -f environment.yaml && \
+    echo "conda activate app-env" >> ~/.bashrc
 
 RUN mkdir -p /mnt/output/forecasts && \
     mkdir -p /mnt/output/era-interim && \
     mkdir -p /mnt/output/era-5 && \
     mkdir -p /mnt/output/forecast-records
 
-# COPY ./sample_data/forecasts /mnt/output/forecasts
-# COPY ./sample_data/era-interim /mnt/output/era-interim
-# COPY ./sample_data/era-5 /mnt/output/era-5
-# COPY ./sample_data/forecast-records /mnt/output/forecast-records
-
 # Copy API code
-COPY ./GSP_API /app/GSP_API/
+COPY app /app
 COPY ./supervisord.conf /etc/supervisor/conf.d/uwsgi.conf
 
 # startup.sh is a helper script