[IA-4703] New Bioconductor and GATK images (with samtools 1.18) (#3981)

automation/src/test/resources/reference.conf

@@ -1,12 +1,12 @@
-leonardo.rImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.3"
-leonardo.pythonImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.4"
-leonardo.hailImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.7"
-leonardo.gatkImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.4"
-leonardo.aouImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.7"
+leonardo.rImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.4"
+leonardo.pythonImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.5"
+leonardo.hailImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.8"
+leonardo.gatkImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.6"
+leonardo.aouImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.8"
 leonardo.baseImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.3"
 leonardo.gcrWelderUri = "us.gcr.io/broad-dsp-gcr-public/welder-server"
 leonardo.dockerHubWelderUri = "broadinstitute/welder-server"
-leonardo.rstudioBioconductorImageUrl = "us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.17.1"
+leonardo.rstudioBioconductorImageUrl = "us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.18.0"
 //each fiab will have a unique topic name sourced from leonardo.conf, this is never published to in automation tests to ensure pub/sub components can auth properly, but never read from to avoid conflicts.
 leonardo.topicName = "leonardo-pubsub-test"
 leonardo.location = "us-central1-a"

http/src/main/resources/reference.conf

@@ -69,9 +69,9 @@ dataproc {
   }
 
   # Cached dataproc image used by Terra
-  customDataprocImage = "projects/broad-dsp-gcr-public/global/images/leo-dataproc-image-2-1-11-debian11-2023-11-01-17-44-04"
+  customDataprocImage = "projects/broad-dsp-gcr-public/global/images/leo-dataproc-image-2-1-11-debian11-2023-12-01-22-47-51"
   # Cached dataproc image used by AOU
-  legacyAouCustomDataprocImage = "projects/broad-dsp-gcr-public/global/images/leo-dataproc-image-2-0-51-debian10-2023-10-25-14-46-44"
+  legacyAouCustomDataprocImage = "projects/broad-dsp-gcr-public/global/images/leo-dataproc-image-2-0-51-debian10-2023-12-05-21-47-12"
 
   # The ratio of memory allocated to spark. 0.8 = 80%.
   # Hail/Spark users generally allocate 80% of the ram to the JVM.
@@ -112,7 +112,7 @@ dataproc {
 }
 
 gce {
-  customGceImage = "projects/broad-dsp-gcr-public/global/images/leo-gce-image-2023-11-01-17-44-08"
+  customGceImage = "projects/broad-dsp-gcr-public/global/images/leo-gce-image-2023-12-01-17-29-26"
   userDiskDeviceName = "user-disk"
   defaultScopes = [
     "https://www.googleapis.com/auth/userinfo.email",
@@ -792,7 +792,7 @@ image {
   welderGcrUri = "us.gcr.io/broad-dsp-gcr-public/welder-server"
   welderDockerHubUri = "broadinstitute/welder-server"
   welderHash = "6648f5c"
-  jupyterImage =  "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.4"
+  jupyterImage =  "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.6"
   proxyImage = "broadinstitute/openidc-proxy:2.3.1_2"
   # Note: If you update this, please also update prepare_gce_image.sh and
   # prepare-custom-leonardo-jupyter-dataproc-image.sh scripts.

jenkins/dataproc-custom-images/prepare-custom-leonardo-jupyter-dataproc-image.sh

@@ -16,15 +16,15 @@ set -e -x
 # the image tags are set via jenkins automation
 #
 
-terra_jupyter_python="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.4"
-terra_jupyter_r="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.3"
-terra_jupyter_bioconductor="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.3"
-terra_jupyter_hail="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.7"
-terra_jupyter_gatk="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.4"
-terra_jupyter_aou="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.7"
+terra_jupyter_python="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.5"
+terra_jupyter_r="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.4"
+terra_jupyter_bioconductor="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.4"
+terra_jupyter_hail="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.8"
+terra_jupyter_gatk="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.6"
+terra_jupyter_aou="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.8"
 welder_server="us.gcr.io/broad-dsp-gcr-public/welder-server:8667bfe"
 openidc_proxy="broadinstitute/openidc-proxy:2.3.1_2"
-anvil_rstudio_bioconductor="us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.17.1"
+anvil_rstudio_bioconductor="us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.18.0"
 
 # Note that this is the version used currently by AOU in production!
 terra_jupyter_aou_old="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.15"
@@ -34,7 +34,11 @@ cryptomining_detector="us.gcr.io/broad-dsp-gcr-public/cryptomining-detector:0.0.
 
 # this array determines which of the above images are baked into the custom image
 # the entry must match the var name above, which must correspond to a valid docker URI
-docker_image_var_names="welder_server terra_jupyter_python terra_jupyter_r terra_jupyter_bioconductor terra_jupyter_hail terra_jupyter_gatk terra_jupyter_aou terra_jupyter_aou_old openidc_proxy anvil_rstudio_bioconductor cryptomining_detector"
+docker_image_var_names="welder_server terra_jupyter_python terra_jupyter_r terra_jupyter_bioconductor terra_jupyter_hail terra_jupyter_gatk terra_jupyter_aou openidc_proxy anvil_rstudio_bioconductor cryptomining_detector"
+
+# NOTE - UNCOMMENT TO REGENERATE THE AOU LEGACY DATAPROC IMAGE
+# You would also need to change the debian version, see https://github.com/DataBiosphere/leonardo/pull/3871
+#docker_image_var_names="welder_server terra_jupyter_python terra_jupyter_r terra_jupyter_bioconductor terra_jupyter_hail terra_jupyter_gatk terra_jupyter_aou_old openidc_proxy anvil_rstudio_bioconductor cryptomining_detector"
 
 # The version of python to install
 # Note: this should match the version of python in the terra-jupyter-hail image.

jenkins/gce-custom-images/prepare_gce_image.sh

@@ -16,14 +16,14 @@ set -e -x
 # Constants and Global Vars
 #
 
-terra_jupyter_python="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.4"
-terra_jupyter_r="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.3"
-terra_jupyter_bioconductor="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.3"
-terra_jupyter_gatk="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.4"
-terra_jupyter_aou="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.7"
+terra_jupyter_python="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.5"
+terra_jupyter_r="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.4"
+terra_jupyter_bioconductor="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-bioconductor:2.2.4"
+terra_jupyter_gatk="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.6"
+terra_jupyter_aou="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.8"
 welder_server="us.gcr.io/broad-dsp-gcr-public/welder-server:8667bfe"
 openidc_proxy="broadinstitute/openidc-proxy:2.3.1_2"
-anvil_rstudio_bioconductor="us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.17.1"
+anvil_rstudio_bioconductor="us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.18.0"
 
 # Note that this is the version used currently by AOU in production!
 terra_jupyter_aou_old="us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.15"