Merge remote-tracking branch 'origin/develop' into grantaar-azuregov-…

…storageurls

.github/workflows/leo-build-tag-publish-and-run-tests.yml

@@ -93,7 +93,7 @@ jobs:
       - name: Extract branch
         id: extract-branch
         run: |
-          if [[ '${{ github.event_name }}' == 'push' ]]; then
+          if [[ '${{ github.event_name }}' == 'push' || '${{ github.event_name }}' == 'workflow_run' ]]; then
             BRANCH_NAME=${{ github.ref_name }}
           elif [[ '${{ github.event_name }}' == 'pull_request' ]]; then
             BRANCH_NAME=${{ github.head_ref }}

Dockerfile

@@ -30,7 +30,7 @@ ENV TERRA_APP_VERSION 0.5.0
 ENV GALAXY_VERSION 2.9.0
 ENV NGINX_VERSION 4.3.0
 # If you update this here, make sure to also update reference.conf:
-ENV CROMWELL_CHART_VERSION 0.2.506
+ENV CROMWELL_CHART_VERSION 0.2.523
 ENV HAIL_BATCH_CHART_VERSION 0.2.0
 ENV RSTUDIO_CHART_VERSION 0.12.0
 ENV SAS_CHART_VERSION 0.17.0

automation/src/test/resources/reference.conf

@@ -1,9 +1,9 @@
 leonardo {
     rImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.5"
     pythonImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.5"
-    hailImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.10"
+    hailImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.12"
     gatkImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.7"
-    aouImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.12"
+    aouImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.13"
     baseImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.3"
     gcrWelderUri = "us.gcr.io/broad-dsp-gcr-public/welder-server"
     dockerHubWelderUri = "broadinstitute/welder-server"

automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/LeonardoConfig.scala

@@ -48,6 +48,10 @@ object LeonardoConfig extends CommonConfig {
     val wsmUri: String = "https://workspace.dsde-dev.broadinstitute.org"
   }
 
+  object BPM {
+    val bpmUri: String = "https://bpm.dsde-dev.broadinstitute.org"
+  }
+
   object LeonardoClient {
     val writeTimeout = leonardoClient.getInt("writeTimeout")
     val readTimeout = leonardoClient.getInt("readTimeout")

core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/JsonCodec.scala

@@ -571,6 +571,14 @@ object JsonCodec {
     "blockSize"
   )(x => (x.name, x.size, x.diskType, x.blockSize))
 
+  implicit val computeClassEncoder: Encoder[ComputeClass] = Encoder.encodeString.contramap(_.toString)
+  implicit val autopilotEncoder: Encoder[Autopilot] = Encoder.forProduct4(
+    "computeClass",
+    "cpuInMillicores",
+    "memoryInGb",
+    "ephemeralStorageInGb"
+  )(x => Autopilot.unapply(x).get)
+
   // can't use Encoder.forProductX because there are 23 fields
   implicit val getRuntimeResponseEncoder: Encoder[GetRuntimeResponse] = Encoder.instance { x =>
     Json.obj(

core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/appRoutesModels.scala

@@ -52,6 +52,7 @@ final case class GetAppResponse(
   cloudContext: CloudContext,
   region: RegionName,
   kubernetesRuntimeConfig: KubernetesRuntimeConfig,
+  autopilot: Option[Autopilot],
   errors: List[AppError],
   status: AppStatus, // TODO: do we need some sort of aggregate status?
   proxyUrls: Map[ServiceName, URL],
@@ -70,6 +71,7 @@ final case class ListAppResponse(workspaceId: Option[WorkspaceId],
                                  cloudContext: CloudContext,
                                  region: RegionName,
                                  kubernetesRuntimeConfig: KubernetesRuntimeConfig,
+                                 autopilot: Option[Autopilot],
                                  errors: List[AppError],
                                  status: AppStatus, // TODO: do we need some sort of aggregate status?
                                  proxyUrls: Map[ServiceName, URL],
@@ -99,6 +101,7 @@ object ListAppResponse {
             n.machineType,
             n.autoscalingEnabled
           ),
+          a.autopilot,
           a.errors,
           a.status,
           a.getProxyUrls(c, proxyUrlBase),
@@ -128,6 +131,7 @@ object GetAppResponse {
         appResult.nodepool.machineType,
         appResult.nodepool.autoscalingEnabled
       ),
+      appResult.app.autopilot,
       appResult.app.errors,
       appResult.app.status,
       appResult.app.getProxyUrls(appResult.cluster, proxyUrlBase),

core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/runtimeModels.scala

@@ -525,17 +525,26 @@ object WelderAction extends Enum[WelderAction] {
   case object DisableDelocalization extends WelderAction
 }
 
-final case class MemorySize(bytes: Long) extends AnyVal {
+final case class MemorySizeBytes(bytes: Long) extends AnyVal {
   override def toString: String = bytes.toString + "b"
 }
-object MemorySize {
+object MemorySizeBytes {
   val kbInBytes = 1024
   val mbInBytes = 1048576
   val gbInBytes = 1073741824
 
-  def fromKb(kb: Double): MemorySize = MemorySize((kb * kbInBytes).toLong)
-  def fromMb(mb: Double): MemorySize = MemorySize((mb * mbInBytes).toLong)
-  def fromGb(gb: Double): MemorySize = MemorySize((gb * gbInBytes).toLong)
+  def fromKb(kb: Double): MemorySizeBytes = MemorySizeBytes((kb * kbInBytes).toLong)
+  def fromMb(mb: Double): MemorySizeBytes = MemorySizeBytes((mb * mbInBytes).toLong)
+  def fromGb(gb: Double): MemorySizeBytes = MemorySizeBytes((gb * gbInBytes).toLong)
+}
+
+final case class MemorySizeMegaBytes(megabytes: Long) extends AnyVal {
+  override def toString: String = megabytes.toString + "m"
+}
+object MemorySizeMegaBytes {
+  val mbInBytes = 1048576
+
+  def fromB(b: Double): MemorySizeMegaBytes = MemorySizeMegaBytes((b / mbInBytes).toLong)
 }
 
 /**
@@ -548,9 +557,10 @@ object MemorySize {
  * Note that the memory limit includes all the sub-procesess of the Notebook server including the
  * Notebook kernel and the Spark driver process, if any.
  */
-final case class RuntimeResourceConstraints(memoryLimit: MemorySize,
-                                            totalMachineMemory: MemorySize,
-                                            driverMemory: Option[MemorySize]
+final case class RuntimeResourceConstraints(memoryLimit: MemorySizeBytes,
+                                            shmSize: MemorySizeMegaBytes,
+                                            totalMachineMemory: MemorySizeBytes,
+                                            driverMemory: Option[MemorySizeBytes]
 )
 
 final case class RuntimeMetrics(cloudContext: CloudContext,

core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/samModels.scala

@@ -223,6 +223,9 @@ object WorkspaceAction {
   final case object CreateControlledUserResource extends WorkspaceAction {
     val asString = "create_controlled_user_private"
   }
+  final case object Delete extends WorkspaceAction {
+    val asString = "delete"
+  }
 
   val allActions = sealerate.values[WorkspaceAction]
   val stringToAction: Map[String, WorkspaceAction] =

core/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/AppRoutesTestJsonCodec.scala

@@ -39,12 +39,13 @@ object AppRoutesTestJsonCodec {
     Decoder.decodeMap[ServiceName, URL](KeyDecoder.decodeKeyString.map(ServiceName), urlDecoder)
 
   implicit val getAppResponseDecoder: Decoder[GetAppResponse] =
-    Decoder.forProduct17(
+    Decoder.forProduct18(
       "workspaceId",
       "appName",
       "cloudContext",
       "region",
       "kubernetesRuntimeConfig",
+      "autopilot",
       "errors",
       "status",
       "proxyUrls",
@@ -60,11 +61,12 @@ object AppRoutesTestJsonCodec {
     )(GetAppResponse.apply)
 
   implicit val listAppResponseDecoder: Decoder[ListAppResponse] =
-    Decoder.forProduct16(
+    Decoder.forProduct17(
       "workspaceId",
       "cloudContext",
       "region",
       "kubernetesRuntimeConfig",
+      "autopilot",
       "errors",
       "status",
       "proxyUrls",

http/src/main/resources/init-resources/cluster-site-gce.conf

@@ -34,6 +34,8 @@
     # Append SameSite=None to cookies set by RStudio. This is required by some browsers because we
     # render RStudio in an iframe. There does not appear to be a way within RStudio to do this, hence
     # doing it in the proxy.
+    # [IA-4997] to support CHIPS by setting partitioned cookies
+    # Header edit Set-Cookie ^(.*)$ $1;Secure;SameSite=None;HttpOnly;Partitioned "expr=%{REQUEST_URI} =~ m#/proxy/[^/]*/[^/]*/rstudio/.*#"
     Header edit Set-Cookie ^(.*)$ $1;Secure;SameSite=None;HttpOnly "expr=%{REQUEST_URI} =~ m#/proxy/[^/]*/[^/]*/rstudio/.*#"
 
     ####################

http/src/main/resources/init-resources/cluster-site.conf

@@ -55,6 +55,8 @@
     # Append SameSite=None to cookies set by RStudio. This is required by some browsers because we
     # render RStudio in an iframe. There does not appear to be a way within RStudio to do this, hence
     # doing it in the proxy.
+    # [IA-4997] to support CHIPS by setting partitioned cookies
+    # Header edit Set-Cookie ^(.*)$ $1;Secure;SameSite=None;HttpOnly;Partitioned "expr=%{REQUEST_URI} =~ m#/proxy/[^/]*/[^/]*/rstudio/.*#"
     Header edit Set-Cookie ^(.*)$ $1;Secure;SameSite=None;HttpOnly "expr=%{REQUEST_URI} =~ m#/proxy/[^/]*/[^/]*/rstudio/.*#"
 
     ####################

http/src/main/resources/init-resources/gce-init.sh

@@ -103,6 +103,7 @@ export PROXY_DOCKER_IMAGE=$(proxyDockerImage)
 export CRYPTO_DETECTOR_SERVER_NAME=$(cryptoDetectorServerName)
 export CRYPTO_DETECTOR_DOCKER_IMAGE=$(cryptoDetectorDockerImage)
 export MEM_LIMIT=$(memLimit)
+export SHM_SIZE=$(shmSize)
 export WELDER_MEM_LIMIT=$(welderMemLimit)
 export PROXY_SERVER_HOST_NAME=$(proxyServerHostName)
 export WELDER_ENABLED=$(welderEnabled)
@@ -323,6 +324,7 @@ OWNER_EMAIL=${OWNER_EMAIL}
 PET_SA_EMAIL=${PET_SA_EMAIL}
 WELDER_ENABLED=${WELDER_ENABLED}
 MEM_LIMIT=${MEM_LIMIT}
+SHM_SIZE=${SHM_SIZE}
 WELDER_SERVER_NAME=${WELDER_SERVER_NAME}
 WELDER_DOCKER_IMAGE=${WELDER_DOCKER_IMAGE}
 STAGING_BUCKET=${STAGING_BUCKET}

http/src/main/resources/init-resources/init-actions.sh

@@ -176,6 +176,7 @@ if [[ "${ROLE}" == 'Master' ]]; then
     export WELDER_ENABLED=$(welderEnabled)
     export NOTEBOOKS_DIR=$(notebooksDir)
     export MEM_LIMIT=$(memLimit)
+    export SHM_SIZE=$(shmSize)
     export WELDER_MEM_LIMIT=$(welderMemLimit)
     export PROXY_SERVER_HOST_NAME=$(proxyServerHostName)
     export CERT_DIRECTORY='/certs'

http/src/main/resources/init-resources/jupyter-docker-compose-gce.yaml

@@ -41,6 +41,7 @@ services:
     # See https://docs.docker.com/engine/reference/run/#user-memory-constraints
     mem_limit: ${MEM_LIMIT}   # hard limit on memory consumption by the container
     memswap_limit: ${MEM_LIMIT}
+    shm_size: ${SHM_SIZE}
 networks:
   app_network:
     external: true

http/src/main/resources/init-resources/jupyter-docker-compose.yaml

@@ -59,3 +59,4 @@ services:
     # See https://docs.docker.com/engine/reference/run/#user-memory-constraints
     mem_limit: ${MEM_LIMIT}   # hard limit on memory consumption by the container
     memswap_limit: ${MEM_LIMIT}
+    shm_size: ${SHM_SIZE}

http/src/main/resources/init-resources/rstudio-docker-compose-gce.yaml

@@ -27,8 +27,9 @@ services:
     env_file:
       - /var/custom_env_vars.env
     # See https://docs.docker.com/engine/reference/run/#user-memory-constraints
-    mem_limit: ${MEM_LIMIT}   # hard limit on memory consumption by the container
+    mem_limit: ${MEM_LIMIT}   # hard limit in byte on memory consumption by the container
     memswap_limit: ${MEM_LIMIT}
+    shm_size: ${SHM_SIZE}
 networks:
   app_network:
     external: true

http/src/main/resources/init-resources/rstudio-docker-compose.yaml

@@ -25,3 +25,4 @@ services:
     # See https://docs.docker.com/engine/reference/run/#user-memory-constraints
     mem_limit: ${MEM_LIMIT}   # hard limit on memory consumption by the container
     memswap_limit: ${MEM_LIMIT}
+    shm_size: ${SHM_SIZE}

http/src/main/resources/init-resources/startup.sh

@@ -68,6 +68,7 @@ export START_USER_SCRIPT_URI=$(startUserScriptUri)
 export START_USER_SCRIPT_OUTPUT_URI=$(startUserScriptOutputUri)
 export WELDER_MEM_LIMIT=$(welderMemLimit)
 export MEM_LIMIT=$(memLimit)
+export SHM_SIZE=$(shmSize)
 export INIT_BUCKET_NAME=$(initBucketName)
 export USE_GCE_STARTUP_SCRIPT=$(useGceStartupScript)
 export PROXY_DOCKER_COMPOSE=$(proxyDockerCompose)
@@ -127,6 +128,26 @@ then
     if [ ! -z "$JUPYTER_DOCKER_IMAGE" ] ; then
         echo "Restarting Jupyter Container $GOOGLE_PROJECT / $CLUSTER_NAME..."
 
+        # Make sure when runtimes restarts, they'll get a new version of jupyter docker compose file
+        $GSUTIL_CMD cp gs://${INIT_BUCKET_NAME}/`basename ${JUPYTER_DOCKER_COMPOSE}` $JUPYTER_DOCKER_COMPOSE
+
+tee /var/variables.env << END
+JUPYTER_SERVER_NAME=${JUPYTER_SERVER_NAME}
+JUPYTER_DOCKER_IMAGE=${JUPYTER_DOCKER_IMAGE}
+NOTEBOOKS_DIR=${NOTEBOOKS_DIR}
+GOOGLE_PROJECT=${GOOGLE_PROJECT}
+RUNTIME_NAME=${RUNTIME_NAME}
+OWNER_EMAIL=${OWNER_EMAIL}
+PET_SA_EMAIL=${PET_SA_EMAIL}
+WELDER_ENABLED=${WELDER_ENABLED}
+MEM_LIMIT=${MEM_LIMIT}
+SHM_SIZE=${SHM_SIZE}
+END
+
+        ${DOCKER_COMPOSE} -f ${JUPYTER_DOCKER_COMPOSE} stop
+        ${DOCKER_COMPOSE} -f ${JUPYTER_DOCKER_COMPOSE} rm -f
+        ${DOCKER_COMPOSE} --env-file=/var/variables.env -f ${JUPYTER_DOCKER_COMPOSE} up -d
+
         if [ "${GPU_ENABLED}" == "true" ] ; then
           # Containers will usually restart just fine. But when gpu is enabled,
           # jupyter container will fail to start until the appropriate volume/device exists.
@@ -144,32 +165,13 @@ then
 
         if [ "$NEED_MIGRATE" == "true" ] ; then
           docker exec $JUPYTER_SERVER_NAME /bin/bash -c "[ ! -d $JUPYTER_USER_HOME/notebooks/.jupyter ] && rsync -av --progress --exclude notebooks . $JUPYTER_USER_HOME/notebooks || true"
-
-          # Make sure when runtimes restarts, they'll get a new version of jupyter docker compose file
-          $GSUTIL_CMD cp gs://${INIT_BUCKET_NAME}/`basename ${JUPYTER_DOCKER_COMPOSE}` $JUPYTER_DOCKER_COMPOSE
-
-          tee /var/variables.env << END
-JUPYTER_SERVER_NAME=${JUPYTER_SERVER_NAME}
-JUPYTER_DOCKER_IMAGE=${JUPYTER_DOCKER_IMAGE}
-NOTEBOOKS_DIR=${NOTEBOOKS_DIR}
-GOOGLE_PROJECT=${GOOGLE_PROJECT}
-RUNTIME_NAME=${RUNTIME_NAME}
-OWNER_EMAIL=${OWNER_EMAIL}
-PET_SA_EMAIL=${PET_SA_EMAIL}
-WELDER_ENABLED=${WELDER_ENABLED}
-MEM_LIMIT=${MEM_LIMIT}
-END
-
-          ${DOCKER_COMPOSE} -f ${JUPYTER_DOCKER_COMPOSE} stop
-          ${DOCKER_COMPOSE} -f ${JUPYTER_DOCKER_COMPOSE} rm -f
-          ${DOCKER_COMPOSE} --env-file=/var/variables.env -f ${JUPYTER_DOCKER_COMPOSE} up -d
-
-          log 'Copy Jupyter frontend notebook config...'
-          $GSUTIL_CMD cp ${JUPYTER_NOTEBOOK_FRONTEND_CONFIG_URI} /var
-          JUPYTER_NOTEBOOK_FRONTEND_CONFIG=`basename ${JUPYTER_NOTEBOOK_FRONTEND_CONFIG_URI}`
-          retry 3 docker exec -u root ${JUPYTER_SERVER_NAME} /bin/bash -c "mkdir -p $JUPYTER_HOME/nbconfig"
-          docker cp /var/${JUPYTER_NOTEBOOK_FRONTEND_CONFIG} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/nbconfig/
         fi
+
+        log 'Copy Jupyter frontend notebook config...'
+        $GSUTIL_CMD cp ${JUPYTER_NOTEBOOK_FRONTEND_CONFIG_URI} /var
+        JUPYTER_NOTEBOOK_FRONTEND_CONFIG=`basename ${JUPYTER_NOTEBOOK_FRONTEND_CONFIG_URI}`
+        retry 3 docker exec -u root ${JUPYTER_SERVER_NAME} /bin/bash -c "mkdir -p $JUPYTER_HOME/nbconfig"
+        docker cp /var/${JUPYTER_NOTEBOOK_FRONTEND_CONFIG} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/nbconfig/
     fi
 
     if [ "$UPDATE_WELDER" == "true" ] ; then
@@ -226,13 +228,14 @@ else
           ${DOCKER_COMPOSE} -f ${JUPYTER_DOCKER_COMPOSE} stop
           ${DOCKER_COMPOSE} -f ${JUPYTER_DOCKER_COMPOSE} rm -f
           ${DOCKER_COMPOSE} -f ${JUPYTER_DOCKER_COMPOSE} up -d
-
-          log 'Copy Jupyter frontend notebook config...'
-          $GSUTIL_CMD cp ${JUPYTER_NOTEBOOK_FRONTEND_CONFIG_URI} /var
-          JUPYTER_NOTEBOOK_FRONTEND_CONFIG=`basename ${JUPYTER_NOTEBOOK_FRONTEND_CONFIG_URI}`
-          retry 3 docker exec -u root ${JUPYTER_SERVER_NAME} /bin/bash -c "mkdir -p $JUPYTER_HOME/nbconfig"
-          docker cp /var/${JUPYTER_NOTEBOOK_FRONTEND_CONFIG} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/nbconfig/
         fi
+
+        log 'Copy Jupyter frontend notebook config...'
+        $GSUTIL_CMD cp ${JUPYTER_NOTEBOOK_FRONTEND_CONFIG_URI} /var
+        JUPYTER_NOTEBOOK_FRONTEND_CONFIG=`basename ${JUPYTER_NOTEBOOK_FRONTEND_CONFIG_URI}`
+        retry 3 docker exec -u root ${JUPYTER_SERVER_NAME} /bin/bash -c "mkdir -p $JUPYTER_HOME/nbconfig"
+        docker cp /var/${JUPYTER_NOTEBOOK_FRONTEND_CONFIG} ${JUPYTER_SERVER_NAME}:${JUPYTER_HOME}/nbconfig/
+
         # jupyter_delocalize.py now assumes welder's url is `http://welder:8080`, but on dataproc, we're still using host network
         # A better to do this might be to take welder host as an argument to the script
         docker exec $JUPYTER_SERVER_NAME /bin/bash -c "sed -i 's/http:\/\/welder/http:\/\/127.0.0.1/g' /etc/jupyter/custom/jupyter_delocalize.py"

http/src/main/resources/leo.conf

@@ -65,6 +65,7 @@ proxy {
   # Should match the jupyter wildcard cert
   proxyDomain = ${?PROXY_DOMAIN}
   proxyUrlBase = ${?PROXY_URL_BASE}
+  isProxyCookiePartitioned = ${?IS_PROXY_COOKIE_PARTITIONED}
 }
 
 app-service.enable-custom-app-check = ${?CUSTOM_APP_GROUP_PERMISSION_CHECK}
@@ -217,6 +218,10 @@ azure {
     uri = ${?WSM_URL}
   }
 
+  bpm {
+      uri = ${?BPM_URL}
+    }
+
   tdr  {
     url = ${?DATA_REPO_URL}
   }
@@ -256,4 +261,4 @@ clusterFiles {
 
 drs {
   url = ${?DRS_URL}
-}
+}