Migrate to RockyLinux9 with latest docker engine and compose plugin (#16

)

Additional `dockerd` options are required for RockyLinux9, see
documentation in `docker-config/dockerd.md`.

`docker-compose.yml` syntax updated for latest compose plugin version.

Switched to use compose plugin instead of `docker-compose` standalone
binary but to keep backward-compatibility, `docker-compose` standalone
library can still be used via a new config in `env.local`.

@fmigneault FYI.

docker-compose.override.yml.example

@@ -1,4 +1,3 @@
-version: '2'
 services:
   master:
     environment:

docker-compose.yml

@@ -1,4 +1,16 @@
-version: '2'
+x-logging: &default-logging
+  driver: "json-file"
+  options:
+    max-size: "50m"
+    max-file: "10"
+
+x-common-env: &common-env
+  # For error "process apparently never started in
+  # /home/jenkins/agent/workspace/_workflow-tests_new-docker-build@tmp/durable-12eaff4f"
+  # on RockyLinux9 (RL8 was fine)
+  JAVA_OPTS: -Dorg.jenkinsci.plugins.durabletask.BourneShellScript.LAUNCH_DIAGNOSTICS=true
+
+
 services:
   master:
     image: pavics/jenkins-master:2.190.3.191209
@@ -9,25 +21,28 @@ services:
       - master_home:/var/jenkins_home
       - ./buildout/jcasc_config:/jcasc:ro
     environment:
-      - CASC_JENKINS_CONFIG=/jcasc/
-      - SSH_PRIVATE_KEY=${SSH_PRIVATE_KEY}
-      - ADMIN_PASSWD=$JENKINS_ADMIN_PASSWD
-      - HTTP_PROTO=$JENKINS_HTTP_PROTO
-      - HOSTNAME=$JENKINS_HOSTNAME
-      - HOSTPORT=$JENKINS_MASTER_PORT
-      - NUM_EXECUTORS=$JENKINS_NUM_EXECUTORS_LOCAL
-      - ESGF_AUTH_USERNAME=$JENKINS_ESGF_AUTH_USERNAME
-      - ESGF_AUTH_PASSWORD=$JENKINS_ESGF_AUTH_PASSWORD
-      - ESGF_AUTH_TOKEN=$JENKINS_ESGF_AUTH_TOKEN
-      - HYDROSHARE_AUTH_CLIENT_ID=$JENKINS_HYDROSHARE_AUTH_CLIENT_ID
-      - HYDROSHARE_AUTH_TOKEN=$JENKINS_HYDROSHARE_AUTH_TOKEN
-      - IS_PROD=$JENKINS_IS_PROD
-      - ENABLE_SCHEDULED_TRIGGER=$JENKINS_ENABLE_SCHEDULED_TRIGGER
-      - DEFAULT_PAVICS_HOST=$JENKINS_DEFAULT_PAVICS_HOST
-      - SMTP_HOST=$JENKINS_SMTP_HOST
+      CASC_JENKINS_CONFIG: /jcasc/
+      SSH_PRIVATE_KEY: ${SSH_PRIVATE_KEY}
+      ADMIN_PASSWD: $JENKINS_ADMIN_PASSWD
+      HTTP_PROTO: $JENKINS_HTTP_PROTO
+      HOSTNAME: $JENKINS_HOSTNAME
+      HOSTPORT: $JENKINS_MASTER_PORT
+      NUM_EXECUTORS: $JENKINS_NUM_EXECUTORS_LOCAL
+      ESGF_AUTH_USERNAME: $JENKINS_ESGF_AUTH_USERNAME
+      ESGF_AUTH_PASSWORD: $JENKINS_ESGF_AUTH_PASSWORD
+      ESGF_AUTH_TOKEN: $JENKINS_ESGF_AUTH_TOKEN
+      HYDROSHARE_AUTH_CLIENT_ID: $JENKINS_HYDROSHARE_AUTH_CLIENT_ID
+      HYDROSHARE_AUTH_TOKEN: $JENKINS_HYDROSHARE_AUTH_TOKEN
+      IS_PROD: $JENKINS_IS_PROD
+      ENABLE_SCHEDULED_TRIGGER: $JENKINS_ENABLE_SCHEDULED_TRIGGER
+      DEFAULT_PAVICS_HOST: $JENKINS_DEFAULT_PAVICS_HOST
+      SMTP_HOST: $JENKINS_SMTP_HOST
+      <<: *common-env
     links:
       - slave
     restart: always
+    logging:
+      <<: *default-logging
 
   slave:
     image: pavics/jenkins-ssh-slave:190226
@@ -37,16 +52,20 @@ services:
       - ./jenkins-slave/entrypoint_slave:/entrypoint:ro
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
-      - JENKINS_SLAVE_SSH_PUBKEY=$JENKINS_SLAVE_SSH_PUBKEY
-      - DOCKER_GROUP_ON_HOST=$DOCKER_GROUP_ON_HOST
+      JENKINS_SLAVE_SSH_PUBKEY: $JENKINS_SLAVE_SSH_PUBKEY
+      DOCKER_GROUP_ON_HOST: $DOCKER_GROUP_ON_HOST
+      <<: *common-env
     restart: always
+    logging:
+      <<: *default-logging
+
 
 volumes:
   master_home:
-    external:
-      name: jenkins_master_home_$JENKINS_MASTER_PORT
+    external: true
+    name: jenkins_master_home_$JENKINS_MASTER_PORT
   slave_home:
-    external:
-      name: jenkins_slave_home_$JENKINS_MASTER_PORT
+    external: true
+    name: jenkins_slave_home_$JENKINS_MASTER_PORT
 
 # vi: tabstop=8 expandtab shiftwidth=2 softtabstop=2

docker-config/dockerd.md

@@ -0,0 +1,19 @@
+Need to start `dockerd` with `--default-ulimit nofile=65536:524288
+--default-cgroupns-mode host`.
+
+Need to set `default-ulimit` because the default changed with RL9 and
+the default value is too low.
+
+```
+# RL9
+$ cat /etc/systemd/system.conf |grep DefaultLimitNOFILE
+#DefaultLimitNOFILE=1024:524288
+
+# RL8
+$ cat /etc/systemd/system.conf |grep DefaultLimitNOFILE
+#DefaultLimitNOFILE=
+```
+
+Need to set `default-cgroupns-mode` to `host` because
+`default-cgroupns-mode` default changed to `private` in RL9
+because the switch to Cgroup v2.

env.local.example

@@ -61,3 +61,9 @@ export JCASC_EXTRA_DIRS="./jcasc_extra_example"
 # sample config.
 #
 #export JENKINS_SMTP_HOST="stmp.example.com"
+
+# Can be used to revert to previous `docker-compose` standalone binary.
+#
+# Default: `docker compose` (docker plugin instead of standalone binary).
+#
+#export JENKINS_COMPOSE_BIN="docker-compose"

jenkins-compose.sh

@@ -69,6 +69,7 @@ fi
 #    cat $FILE | envsubst "$VARS" > $DEST
 #  done
 
+set -x  # Log command
 # create volume outside of docker-compose.yml to prevent
 # `docker-compose down -v` to delete the volume.
 # if volume already exist, this call will do nothing
@@ -77,8 +78,9 @@ if [[ $1 == "up" ]]; then
   docker volume create jenkins_slave_home_$JENKINS_MASTER_PORT
 fi
 
-docker-compose "$@"
+${JENKINS_COMPOSE_BIN:=docker compose} "$@"
 ERR=$?
+set +x
 
 # execute post-compose function if exists and no error occurred
 type post-compose 2>&1 | grep 'post-compose is a function' > /dev/null