molgenis · DickPostma · Feb 25, 2025 · Feb 26, 2025 · Feb 26, 2025 · Feb 27, 2025
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -26,20 +26,19 @@ orbs:
 definitions:
   build_config: &build_config
     docker:
-    - image: molgenis/ci-build:1.2.3
+      - image: molgenis/ci-build-armadillo:1.1.4
     working_directory: ~/repo
     resource_class: large
     environment:
       JVM_OPTS: -Xmx3200m
       GRADLE_OPTS: "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=2"
       TERM: dumb
 
-
 
 jobs:
   build:
     docker:
-      - image: molgenis/ci-build:1.2.3 
+      - image: molgenis/ci-build-armadillo:1.1.4
 
     working_directory: ~/repo
     resource_class: large
@@ -53,29 +52,6 @@ jobs:
     steps:
       - checkout
 
-      # https://support.circleci.com/hc/en-us/articles/16164465307931-Using-an-environment-variable-to-define-a-path-when-using-the-Docker-executor
-      - run:
-          name: "Fix CIRCLE_WORKING_DIRECTORY"
-          # replace ~ by $HOME
-          command: echo 'CIRCLE_WORKING_DIRECTORY="${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}"' >> $BASH_ENV
-
-      - run:
-          name: Test env
-          command: |
-            echo "id: `id`"
-            cd
-            cd repo
-            pwd
-            echo "$CIRCLE_WORKING_DIRECTORY"
-
-      - run:
-          name: Install Docker client
-          command: apt-get -y install docker-ce docker-ce-cli containerd.io
-
-      - run:
-          name: Install zip
-          command: apt-get --yes install zip
-
       - setup_remote_docker:
           version: previous
           docker_layer_caching: true
@@ -96,25 +72,6 @@ jobs:
           name: Build Armadillo and R CICD image
           command: |
             ./gradlew docker ci
-            ./docker/bin/prepare.bash ci
-
-      - store_artifacts:
-          path: build/libs
-
-      - run:
-          name: Zip armadillo-compose
-          command: |
-            echo "Working directory: $CIRCLE_WORKING_DIRECTORY"
-            cd $CIRCLE_WORKING_DIRECTORY
-            mkdir build/artifacts
-
-            # Workaround for ~ expansion not working
-            K=`pwd`
-            cd build/docker/
-            zip -r $K/build/artifacts/armadillo-compose.zip armadillo-compose
-
-      - store_artifacts:
-          path: build/artifacts/armadillo-compose.zip
 
       - run:
           name: build, test, push docker
@@ -155,74 +112,47 @@ jobs:
             fi
 
       - run:
-          name: Make sure all images declared in docker-compose.yml are available and ready
-          command: |
-            echo "Working directory: $CIRCLE_WORKING_DIRECTORY"
-            cd $CIRCLE_WORKING_DIRECTORY
-
-            # Somehow build/docker/armadillo-compose is deleted so redo
-            ./docker/bin/prepare.bash ci
-
-            cd build/docker/armadillo-compose
-
-            docker images ls
-            docker compose up -d
-            docker images ls
+          name: Wait 10s for deployment to get ingress configuration to come up
+          command: sleep 10;
 
       - run:
-          name: Start docker-compose and wait for `release-test.R` to finish
+          name: Wait for preview to be up, poll every 10 seconds
           command: |
-            # FIXME: make this name not directory dependent
-            ARMADILLO="armadillo-compose-armadillo-1"
-
-            cd $CIRCLE_WORKING_DIRECTORY
-            cd build/docker/armadillo-compose
-
-            docker compose up -d
-            docker ps
-
-
-            # 1 CircleCI docker in docker workaround
-            docker cp ./armadillo/config $ARMADILLO:/config
-            docker cp ./armadillo/data   $ARMADILLO:/data
-            docker cp ./armadillo/logs   $ARMADILLO:/logs
-            # .1 CircleCI docker in docker workaround
+            GETURL="https://preview-armadillo-pr-${CIRCLE_PULL_REQUEST##*/}.dev.molgenis.org/"
+            while true;
+                do
+                    STATUS=$(curl --silent --head $GETURL | awk '/^HTTP/{print $2}')
+                    echo ${STATUS}
+                    if [[ "$STATUS" == "200" ]]; then
+                        echo "preview is up";
+                        break;
+                    else
+                        echo "preview is not ready yet, waiting 10 seconds...";
+                    fi;
+                    sleep 10;
+                done
 
-            # Poll to see Armadillo is up
-            docker container run --network container:$ARMADILLO \
-              docker.io/jwilder/dockerize \
-              -wait http://localhost:8080/ \
-              -wait-retry-interval 20s \
-              -timeout 30s || echo "Timed out"
-
-            cd $CIRCLE_WORKING_DIRECTORY
-            cd build/docker/cicd
-
-            # Run release-test.R
-            docker container run \
-              --network container:$ARMADILLO \
-              --interactive --tty \
-              --entrypoint /bin/bash molgenis/r-cicd -c "cd /cicd/scripts/release ; ./armadillo-ready.bash"
-
-            cd $CIRCLE_WORKING_DIRECTORY
-            cd build/docker/armadillo-compose
+      - run:
+          name: Settting enviroment for testing
+          command: |
+            bash ci/set_test_environment.sh "preview-armadillo-pr-${CIRCLE_PULL_REQUEST##*/}"
 
-            # 2 CircleCI docker in docker workaround
-            # See what changed within Armadillo
-            docker cp $ARMADILLO:/logs   ./armadillo/logs
-            # .2 CircleCI docker in docker workaround
+      - run:
+          name: Install dependencies for release testing
+          command: |
+            ./scripts/release/install_release_script_dependencies.R
 
-            docker container ls
-            docker compose down
-            docker images ls
-            docker compose rm
+      - run:
+          name: Release testing
+          command: |
+             cd scripts/release && ./release-test.R
 
   delete-helm-preview:
     environment:
       GHA_PREVIEW_NR: << pipeline.parameters.GHA_Meta >>
-    
+
     docker:
-      - image: molgenis/ci-build:1.2.3 
+      - image: molgenis/ci-build-armadillo:1.1.4
 
     working_directory: ~/repo
     resource_class: large
@@ -242,16 +172,13 @@ workflows:
     unless:
       equal: ["delete-pr-preview", << pipeline.parameters.GHA_Action >>]
     jobs:
-    - build:
-      filters:
-        branches:
-          ignore: master
-  
+      - build:
+        filters:
+          branches:
+            ignore: master
+
   delete_preview:
     when:
       equal: [ "delete-pr-preview", << pipeline.parameters.GHA_Action >> ]
     jobs:
-    - delete-helm-preview
-
-
-
+      - delete-helm-preview
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,5 @@
-FROM --platform=linux/amd64 eclipse-temurin:17.0.12_7-jdk-focal
+#FROM --platform=linux/amd64 eclipse-temurin:17.0.12_7-jdk-focal
+FROM --platform=linux/amd64 eclipse-temurin:21
 VOLUME /data
 VOLUME /config
 VOLUME /logs

diff --git a/ci/ci-build-images/Dockerfile b/ci/ci-build-images/Dockerfile
@@ -0,0 +1,59 @@
+FROM molgenis/ci-build:1.2.4 
+
+
+RUN echo "deb https://cran.r-project.org/bin/linux/ubuntu jammy-cran40/" >> /etc/apt/sources.list
+RUN wget -qO- https://cran.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
+
+#Install R
+RUN apt update && apt -y install r-base r-base-dev 
+
+#install deps for building 
+RUN apt -y install libpcre2-dev libdeflate-dev liblzma-dev libbz2-dev zlib1g-dev libzstd-dev libicu-dev libfftw3-dev curl ssh libcurl4-openssl-dev libssl-dev build-essential ninja-build cmake jq  
+RUN apt install -y \
+    fonts-roboto \
+    ghostscript \
+    hugo \
+    less \
+    libglpk-dev \
+    libgmp3-dev \
+    libfribidi-dev \
+    libharfbuzz-dev \
+    libhunspell-dev \
+    libicu-dev \
+    liblzma-dev \
+    libopenmpi-dev \
+    libpcre2-dev \
+    libssl-dev \
+    libv8-dev \
+    libxml2-dev \
+    libxslt1-dev \
+    libzmq3-dev \
+    libfontconfig-dev \
+    libfontconfig1-dev \
+    libtiff5 \ 
+    libtiff5-dev \
+    libudunits2-dev \
+    libgdal-dev \
+    qpdf \
+    texinfo \
+    software-properties-common \
+    vim \
+    wget
+
+#Install littler
+RUN Rscript -e "install.packages(c('littler', 'docopt'), repos='https://cran.r-project.org')"
+
+RUN ln -sf /usr/local/lib/R/site-library/littler/bin/r /usr/local/bin
+RUN ln -sf /usr/local/lib/R/site-library/littler/examples/install2.r /usr/local/bin
+
+RUN install2.r --skipinstalled --repo https://cran.r-project.org devtools diffobj getPass getopt triebeard
+RUN install2.r --skipinstalled --repo https://cran.r-project.org qqconf mutoss fftw 
+RUN install2.r --skipinstalled --repo https://cran.r-project.org metap jsonlite urltools vroom
+RUN install2.r --skipinstalled --repo https://cran.r-project.org arrow future RCurl cli sf
+RUN install2.r --skipinstalled --repo https://cran.r-project.org DSI resourcer MolgenisArmadillo DSMolgenisArmadillo
+
+RUN mkdir -p /cicd/armadillo/scripts/release
+COPY scripts/release/install_release_script_dependencies.R /cicd/scripts/release/
+
+
+RUN /cicd/scripts/release/install_release_script_dependencies.R
diff --git a/ci/ci.env b/ci/ci.env
@@ -0,0 +1,18 @@
+GIT_CLONE_PATH=
+
+TEST_FILE_PATH=
+
+INTERACTIVE=N
+
+AS_DOCKER_CONTAINER=N
+
+SKIP_TESTS=donkey-tidyverse
+DO_TESTS=
+
+PROFILE=xenon
+
+ADMIN_PASSWORD=admin
+ARMADILLO_URL=localhost:8080
+OIDC_EMAIL=
+
+TOKEN=
diff --git a/ci/set_test_environment.sh b/ci/set_test_environment.sh
@@ -0,0 +1,9 @@
+#this scripts takes arguments
+NAME=$1
+
+cp ci/ci.env scripts/release/.env
+echo "Using name $NAME"
+
+sed -i "s|^ARMADILLO_URL=.*|ARMADILLO_URL=${NAME}.dev.molgenis.org|" scripts/release/.env
+sed -i "s|^ARMADILLO_PASSWORD=.*|ARMADILLO_PASSWORD=${ADMINPASS}|" scripts/release/.env
+
diff --git a/docker/ci/ci.env b/docker/ci/ci.env
@@ -12,8 +12,7 @@ DO_TESTS=
 PROFILE=xenon
 
 ADMIN_PASSWORD=admin
-ARMADILLO_URL=http://armadillo:8080
-
+ARMADILLO_URL={PREVIEW_SERVER}
 OIDC_EMAIL=
 
 TOKEN=
diff --git a/helm-chart/templates/ingress.yaml b/helm-chart/templates/ingress.yaml
@@ -20,6 +20,7 @@ metadata:
     {{- include "armadillo.labels" . | nindent 4 }}
   {{- with .Values.ingress.annotations }}
   annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:

diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml
@@ -12,9 +12,11 @@ config:
     container-prefix: 'dev-'
     profiles:
       - name: default 
-        image: datashield/rock-base:latest
-        #host: rock-molgenis-rock.armadillo-rock.svc
-        host: rock-molgenis-rock.armadillo-rock.svc
+      #  image: datashield/rock-base:latest
+      #  #host: rock-molgenis-rock.armadillo-rock.svc
+      #  host: rock-molgenis-rock.armadillo-rock.svc
+        image: datashield/rock-dolomite-xenon:latest
+        host: xenon-molgenis-xenon.armadillo-xenon.svc
         port: 8085 
         package-whitelist:
           - dsBase
@@ -33,6 +35,8 @@ config:
           - resourcer
           - dsMediation
           - dsMTLBase
+          - dsSurvival
+          - dsOmics
         function-blacklist: [ ]
         options:
           datashield:

diff --git a/scripts/release/install_release_script_dependencies.R b/scripts/release/install_release_script_dependencies.R
@@ -31,11 +31,11 @@ packages <- c(
   "devtools",
   "DSI",
   "resourcer",
-  "MolgenisArmadillo",
-  "DSMolgenisArmadillo", 
   "purrr", 
   "stringr",
   "tibble", 
+  "MolgenisArmadillo",
+  "DSMolgenisArmadillo",
   "dsTidyverseClient"
 )