From c70bcc91a6762c0832eaa684ebab69c2eb66e1ff Mon Sep 17 00:00:00 2001
From: Viktoria Spisakova <spisakova@ics.muni.cz>
Date: Fri, 5 Jun 2020 13:38:00 +0200
Subject: [PATCH 1/9] ci: modify deployment files for k8s ci: modify deployment
 files for k8s

---
 deployment/README.md                          | 44 +++++++++++++++--
 .../templates/mongodb/mongodb-deployment.yaml | 17 +++++++
 deployment/templates/mongodb/mongodb-pvc.yaml |  2 +-
 .../templates/rabbitmq/rabbitmq-pvc.yaml      |  2 +-
 deployment/templates/wes/wes-configmap.yaml   |  2 +-
 deployment/templates/wes/wes-ingress.yaml     | 47 ++-----------------
 deployment/templates/wes/wes-volume.yaml      |  4 ++
 deployment/values.yaml                        | 31 +++++++++---
 8 files changed, 92 insertions(+), 57 deletions(-)

diff --git a/deployment/README.md b/deployment/README.md
index 4e4d5ca..8be4aa6 100644
--- a/deployment/README.md
+++ b/deployment/README.md
@@ -5,8 +5,30 @@ Flower and Autocert. This was tested with Helm v3.0.0.
 
 ## Prerequisites
 1. A working kubernetes cluster and access to the ```kubectl``` command.
-2. A dynamic storage provisioner ([StorageClass](https://kubernetes.io/docs/concepts/storage/storage-classes/)) that can provide volumes in ReadWriteMany (RWX) access mode. You can find [a list of internal provisioners](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) that support this. We deployed cwl-WES successfully with an [external NFS volume provisioner](https://github.com/kubernetes-incubator/external-storage/tree/master/nfs).
-3. If you are planning to use cwl-WES in FTP mode you need an FTP server that supports TLS encryption. 
+2. A dynamic storage provisioner ([StorageClass](https://kubernetes.io/docs/concepts/storage/storage-classes/)) that can provide volumes in ReadWriteMany (RWM) access mode. You can find [a list of internal provisioners](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) that support this. We deployed cwl-WES successfully with an [external NFS volume provisioner](https://github.com/kubernetes-incubator/external-storage/tree/master/nfs).
+3. If you are planning to use cwl-WES in FTP mode you need an FTP server that supports TLS encryption. Choose from options:
+    1. Use system wide certificate manager, see [Jetstack](https://hub.helm.sh/charts/jetstack/cert-manager) for install. Instance of ClusterIssuer is needed, YAML could look like:
+        ```yaml
+        apiVersion: cert-manager.io/v1alpha2
+        kind: ClusterIssuer
+        metadata:
+          name: [name]
+          labels:
+            name: [name]
+        spec:
+          acme:
+            email: email@example.com
+            privateKeySecretRef:
+              name: [name]
+            server: https://acme-v02.api.letsencrypt.org/directory
+            solvers:
+            - http01:
+                ingress:
+                  class: nginx
+        ```
+        Also you need system wide ingress and load balancer configuration, see [Rancher Nginx](https://rancher.com/docs/rancher/v2.x/en/installation/options/nginx/) and [K8S RKE](https://rancher.com/docs/rancher/v2.x/en/installation/k8s-install/kubernetes-rke/).
+        If you choose this option, in `values.yaml` set `autocert.createJob: "false"` and `ingress.letsencryptSystem: "true"`
+    2. Install ingress and autocert from WES (set `autocert.createJob: "true"` and `ingress.letsencryptSystem: "false"`). For Autocert, see section below.
 4. A working TES installation like [TESK](https://github.com/EMBL-EBI-TSI/TESK) or [Funnel](https://github.com/ohsu-comp-bio/funnel) exposed via an endpoint. If you are planning to use cwl-WES in FTP mode, then your TES endpoint must also support FTP.
 
 ## Deploying in FTP mode
@@ -24,7 +46,7 @@ kubectl create namespace <new-namespace-name>
 	1. Change the following values in [/cwl_wes/config/app_config.yaml](/cwl_wes/config/app_config.yaml):
 		1. ```storage.remote_storage_url```: The endpoint and folder of the FTP service that will be used for remote storage:
 		```ftp://endpoint//path```
-		2. ```tes.url```: The endpoint of your TES Service.
+		2. ```tesk.url```: The endpoint of your TES Service.
 4. Navigate into the **[`deployment/`](/deployment) directory** and issue the following command:
 ```bash
 helm install <name-of-your-deployment> . -f values.yaml -n <new-namespace-name>
@@ -76,23 +98,35 @@ See [`values.yaml`](values.yaml) for default values.
 | autocert.schedule | string | schedule for certificate refreshment |
 | autocert.testCert | string | whether to use Let's Encrypt staging so as not to exceed quota |
 | celeryWorker.appName | string | name of the Celery app on Kubernetes cluster |
-| celeryWorker.image | string | containger image to be used for the Celery application |
+| celeryWorker.image | string | container image to be used for the Celery application |
 | clusterType | string | type of Kubernetes cluster; either 'kubernetes' or 'openshift' |
+| ingress.letsencryptSystem | string | for K8S, whether use system LetsEncrypt or not |
+| ingress.nginx_image | string | for K8S, container image to be used to run nginx |
+| ingress.scope.annotations.clusterissuer | string | for K8S, name of instance of letsencrypt cert manager |
+| ingress.scope.annotations.ingressclass | string | for K8S, name of class that takes care of ingress |
+| ingress.scope.annotations.tlsacme | string | for K8S, true if letsencrypt should be used |
 | mongodb.appName | string | name of MongoDB app on Kubernetes cluster |
 | mongodb.databaseAdminPassword | string | admin password for MongoDB |
 | mongodb.databaseName | string | name of MongoDB database to be used in application |
 | mongodb.databasePassword | string | user password for MongoDB |
 | mongodb.databaseUser | string | username for MongoDB |
 | mongodb.image | string | container image to be used to run MongoDB |
+| mongodb.mountPath| string | for K8S, where to mount the PVC |
+| mongodb.pullPolicy | string | pull Policy for container image |
+| mongodb.securityContext.enabled | string | for K8S, whether security is enabled (to solve issues with newly created PVC) |
+| mongodb.securityContext.fsGroup  | string | for K8S, fsGroup that can access the PVC |
+| mongodb.securityContext.runAsUser  | string | for K8S, user that can access the PVC |
+| mongodb.securityContext.runAsNonRoot  | string | for K8S, run as non root |
 | mongodb.volumeSize | string | size of volume reserved for MongoDB database |
 | rabbitmq.appName | string | name of RabbitMQ app on Kubernetes cluster |
 | rabbitmq.image | string | container image to be used to run RabbitMQ |
 | rabbitmq.volumeSize | string | size of volume reserved for RabbitMQ broker |
+| storageAccessMode | string | access mode for MongoDB and RabbitMQ PVC |
 | tlsSecret | string | secret for TLS encryption |
 | wes.appName | string | name of the main application on Kubernetes cluster |
 | wes.image | string | containger image to be used for the main application |
 | wes.netrcLogin | string | login name for accessing the sFTP server |
 | wes.netrcMachine | string | host name of sFTP server |
 | wes.netrcPassword | string | password for accessing the sFTP server |
+| wes.storageClass | string | type of storageClass for WES, must have RWX capability |
 | wes.volumeSize | string | size of volume reserved for the main application |
-
diff --git a/deployment/templates/mongodb/mongodb-deployment.yaml b/deployment/templates/mongodb/mongodb-deployment.yaml
index 5a09179..75c01fc 100644
--- a/deployment/templates/mongodb/mongodb-deployment.yaml
+++ b/deployment/templates/mongodb/mongodb-deployment.yaml
@@ -14,6 +14,18 @@ spec:
       labels:
         app: {{ .Values.mongodb.appName }}
     spec:
+      {{ if eq .Values.clusterType "kubernetes" }}
+      initContainers:
+        - name: volume-permissions
+          image: busybox
+          imagePullPolicy: {{ .Values.mongodb.pullPolicy | quote }}
+          command: ["chown", "-R", "{{ .Values.mongodb.securityContext.runAsUser }}:{{ .Values.mongodb.securityContext.fsGroup }}", "{{ .Values.mongodb.mountPath }}"]
+          securityContext:
+            runAsUser: 0
+          volumeMounts:
+            - name: mongodb-data
+              mountPath: {{ .Values.mongodb.mountPath }}
+      {{ end }}
       containers:
         - env:
           - name: MONGODB_USER
@@ -67,6 +79,11 @@ spec:
           resources:
             limits:
               memory: 512Mi
+          {{ if eq .Values.clusterType "kubernetes" }}
+          securityContext:
+            runAsNonRoot: {{ .Values.mongodb.securityContext.runAsNonRoot }}
+            runAsUser: {{ .Values.mongodb.securityContext.runAsUser }}
+          {{ end }}
           volumeMounts:
             - mountPath: /var/lib/mongodb/data
               name: mongodb-data
diff --git a/deployment/templates/mongodb/mongodb-pvc.yaml b/deployment/templates/mongodb/mongodb-pvc.yaml
index 1be0856..ce124bb 100644
--- a/deployment/templates/mongodb/mongodb-pvc.yaml
+++ b/deployment/templates/mongodb/mongodb-pvc.yaml
@@ -4,7 +4,7 @@ metadata:
   name: {{ .Values.mongodb.appName }}-volume
 spec:
   accessModes:
-    - ReadWriteMany
+    - {{ .Values.storageAccessMode }}
   resources:
     requests:
       storage: {{ .Values.mongodb.volumeSize }}
diff --git a/deployment/templates/rabbitmq/rabbitmq-pvc.yaml b/deployment/templates/rabbitmq/rabbitmq-pvc.yaml
index f70db48..2ccd99f 100644
--- a/deployment/templates/rabbitmq/rabbitmq-pvc.yaml
+++ b/deployment/templates/rabbitmq/rabbitmq-pvc.yaml
@@ -5,7 +5,7 @@ metadata:
   name: {{ .Values.rabbitmq.appName }}-volume
 spec:
   accessModes:
-    - ReadWriteMany
+    - {{ .Values.storageAccessMode }}
   resources:
     requests:
       storage: {{ .Values.rabbitmq.volumeSize }}
diff --git a/deployment/templates/wes/wes-configmap.yaml b/deployment/templates/wes/wes-configmap.yaml
index 98d0730..a38212e 100644
--- a/deployment/templates/wes/wes-configmap.yaml
+++ b/deployment/templates/wes/wes-configmap.yaml
@@ -21,6 +21,6 @@ spec:
         - name: WES_APP_NAME
           value: {{ .Values.wes.appName }}
       restartPolicy: Never
-      serviceAccountName: {{ .Values.wes.appName }}-autocert
+      serviceAccountName: {{ .Values.wes.appName }}-autoadmin
 status: {}
 
diff --git a/deployment/templates/wes/wes-ingress.yaml b/deployment/templates/wes/wes-ingress.yaml
index b4783c5..cc30600 100644
--- a/deployment/templates/wes/wes-ingress.yaml
+++ b/deployment/templates/wes/wes-ingress.yaml
@@ -1,31 +1,5 @@
 ---
-{{ if eq .Values.clusterType "kubernetes" }}
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  name: {{ .Values.wes.appName }}
-  labels:
-    app: {{ .Values.wes.appName }}
-spec:
-  rules:
-  - host: {{ .Values.wes.appName }}.{{ .Values.applicationDomain }}
-    http:
-      paths:
-      - path: "/.well-known"
-        backend:
-          serviceName: {{ .Values.wes.appName }}-certbot
-          servicePort: 8080
-      - path: "/ga4gh/wes/v1"
-        backend:
-          serviceName: {{ .Values.wes.appName }}
-          servicePort: 8080
-  {{ if ne .Values.tlsSecret "" }}
-  tls:
-  - secretName: {{ .Values.tlsSecret }}
-    hosts:
-    - {{ .Values.wes.appName }}.{{ .Values.applicationDomain }}
-  {{ end }}
-{{ else }} # assume OpenShift by default
+{{ if ne .Values.clusterType "kubernetes" }} # assume OpenShift, for k8s is wes-ingress-kubernetes.yaml
 apiVersion: route.openshift.io/v1
 kind: Route
 metadata:
@@ -122,7 +96,7 @@ spec:
             - name: SECRET
               value: {{ .Values.tlsSecret }}
           restartPolicy: OnFailure
-          serviceAccountName: {{ .Values.wes.appName }}-autocert
+          serviceAccountName: {{ .Values.wes.appName }}-autoadmin
 status: {}
 
 ---
@@ -158,7 +132,7 @@ spec:
         - name: SECRET
           value: {{ .Values.tlsSecret }}
       restartPolicy: OnFailure
-      serviceAccountName: {{ .Values.wes.appName }}-autocert
+      serviceAccountName: {{ .Values.wes.appName }}-autoadmin
 status: {}
 
 ---
@@ -166,7 +140,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   creationTimestamp: null
-  name: {{ .Values.wes.appName }}-autocert
+  name: {{ .Values.wes.appName }}-autoadmin
 
 ---
 apiVersion: v1
@@ -175,18 +149,5 @@ metadata:
   name: {{ .Values.tlsSecret }}
 type: Opaque
 
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: admin-{{ .Values.wes.appName }}-autocert
-roleRef:
-  name: admin
-  kind: ClusterRole
-  apiGroup: rbac.authorization.k8s.io
-subjects:
-- kind: ServiceAccount
-  name: {{ .Values.wes.appName }}-autocert
-
 {{ end }} # Create autocert system
 
diff --git a/deployment/templates/wes/wes-volume.yaml b/deployment/templates/wes/wes-volume.yaml
index c41cba5..b30fbef 100644
--- a/deployment/templates/wes/wes-volume.yaml
+++ b/deployment/templates/wes/wes-volume.yaml
@@ -3,6 +3,10 @@ kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
   name: {{ .Values.wes.appName }}-volume
+  {{ if eq .Values.clusterType "kubernetes" }}
+  annotations:
+    volume.beta.kubernetes.io/storage-class: {{ .Values.wes.storageClass }}
+  {{ end }}
 spec:
   accessModes:
     - ReadWriteMany
diff --git a/deployment/values.yaml b/deployment/values.yaml
index 95e9266..e8f3aec 100644
--- a/deployment/values.yaml
+++ b/deployment/values.yaml
@@ -2,12 +2,13 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-applicationDomain: c03.k8s-popup.csc.fi
+applicationDomain: c03.k8s-popup.csc.fi  # change to wes server
 clusterType: openshift  # either 'kubernetes' or 'openshift'
-tlsSecret: mytls-secret
+tlsSecret: mytls-secret  # put name of tlsSecret
+storageAccessMode: ReadWriteOnce  # mongodb-pvc.yaml/rabbitmq-pvc.yaml, change to ReadWriteMany if storageClass can do RWX
 
 autocert:
-  createJob: "true"  # actually create autocert cronjob
+  createJob: "true"  # actually create autocert cronjob, for K8S with autocert installed set to "false"
   schedule: "0 0 1 */2 *"  # cronjob schedule
   testCert: "true"  # use test certificates so as not to exceed quota
   image: elixircloud/autocert:devel
@@ -22,9 +23,10 @@ flower:
 wes:
   appName: cwlwes
   image: elixircloud/cwl-wes:latest
-  netrcMachine: defaultmachine
-  netrcLogin: defaultnetrclogin
-  netrcPassword: defaultnetrcpassword
+  netrcMachine: defaultmachine  # change this
+  netrcLogin: defaultnetrclogin  # change this
+  netrcPassword: defaultnetrcpassword  # change this
+  storageClass: nfs-client  # <- wes-volume.yaml, only for K8S, a storageClass with readWriteMany capability is required
   volumeSize: 1Gi
 
 celeryWorker:
@@ -39,9 +41,26 @@ mongodb:
   databaseUser: cwlwes-user
   volumeSize: 1Gi
   image: centos/mongodb-36-centos7
+  mountPath: /var/lib/mongodb/data
+  pullPolicy: Always
+  securityContext:  # only for K8S
+    enabled: true
+    fsGroup: 1001
+    runAsUser: 1001
+    runAsNonRoot: true
 
 rabbitmq:
   appName: rabbitmq
   volumeSize: 1Gi
   image: rabbitmq:3-management
 
+ingress: # <- wes-ingress-kubernetes.yaml, only for system ingress for K8S, ensure wes namespace has rights to access it
+  letsencryptSystem: "true"  # if true, ensure wes namespace has rights to access letsencrypt pods - rbac
+  nginx_image: nginx/nginx-ingress:1.3.0
+  tls_letsencrypt:
+    annotations:
+      ingressclass: "nginx"
+      tlsacme: "\"true\""
+      clusterissuer: "letsencrypt-prod"
+
+

From 243ac7aae9fc9463daacb9152391076de2f1bbd8 Mon Sep 17 00:00:00 2001
From: Viktoria Spisakova <spisakova@ics.muni.cz>
Date: Fri, 5 Jun 2020 13:39:04 +0200
Subject: [PATCH 2/9] ci: Added ingress for K8S and separate file for
 ServiceAcoount and RoleBinding

---
 .../templates/wes/wes-ingress-kubernetes.yaml | 52 +++++++++++++++++++
 .../wes/wes-serviceaccount-autoadmin.yaml     | 18 +++++++
 2 files changed, 70 insertions(+)
 create mode 100644 deployment/templates/wes/wes-ingress-kubernetes.yaml
 create mode 100644 deployment/templates/wes/wes-serviceaccount-autoadmin.yaml

diff --git a/deployment/templates/wes/wes-ingress-kubernetes.yaml b/deployment/templates/wes/wes-ingress-kubernetes.yaml
new file mode 100644
index 0000000..6eda18a
--- /dev/null
+++ b/deployment/templates/wes/wes-ingress-kubernetes.yaml
@@ -0,0 +1,52 @@
+{{ if eq .Values.clusterType "kubernetes" }}
+{{ if .Values.ingress.letsencryptSystem }}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: wes-ingress-kubernetes {{ if .Values.ingress.tls_letsencrypt }}
+  annotations:
+    kubernetes.io/ingress.class: {{ .Values.ingress.tls_letsencrypt.annotations.ingressclass }}
+    kubernetes.io/tls-acme: {{ .Values.ingress.tls_letsencrypt.annotations.tlsacme }}
+    cert-manager.io/cluster-issuer: {{ .Values.ingress.tls_letsencrypt.annotations.clusterissuer }}
+  {{ end }}
+spec: {{ if .Values.tlsSecret }}
+  tls:
+    - hosts:
+        - {{ .Values.applicationDomain }}
+      secretName: {{ .Values.tlsSecret }} {{ end }}
+  rules:
+    - host: {{ .Values.applicationDomain }}
+      http:
+        paths:
+          - path: /
+            backend:
+              serviceName: {{ .Values.wes.appName }}
+              servicePort: 8080
+{{ else }}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: {{ .Values.wes.appName }}
+  labels:
+    app: {{ .Values.wes.appName }}
+spec:
+  rules:
+  - host: {{ .Values.wes.appName }}.{{ .Values.applicationDomain }}
+    http:
+      paths:
+      - path: "/.well-known"
+        backend:
+          serviceName: {{ .Values.wes.appName }}-certbot
+          servicePort: 8080
+      - path: "/ga4gh/wes/v1"
+        backend:
+          serviceName: {{ .Values.wes.appName }}
+          servicePort: 8080
+  {{ if ne .Values.tlsSecret "" }}
+  tls:
+  - secretName: {{ .Values.tlsSecret }}
+    hosts:
+    - {{ .Values.wes.appName }}.{{ .Values.applicationDomain }}
+  {{ end }}
+{{ end }}
+{{ end }}
diff --git a/deployment/templates/wes/wes-serviceaccount-autoadmin.yaml b/deployment/templates/wes/wes-serviceaccount-autoadmin.yaml
new file mode 100644
index 0000000..77533de
--- /dev/null
+++ b/deployment/templates/wes/wes-serviceaccount-autoadmin.yaml
@@ -0,0 +1,18 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  name: {{ .Values.wes.appName }}-autoadmin
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: admin-{{ .Values.wes.appName }}-autoadmin
+roleRef:
+  name: admin
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: {{ .Values.wes.appName }}-autoadmin
\ No newline at end of file

From b4e921b5c8e466208435623e6bcdb71aa342a974 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vikt=C3=B3ria=20Spi=C5=A1akov=C3=A1?=
 <viktoria.spisakovaa@gmail.com>
Date: Wed, 10 Mar 2021 10:56:08 +0100
Subject: [PATCH 3/9] ingre--kubernetes: add app-root label

---
 deployment/templates/wes/wes-ingress-kubernetes.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deployment/templates/wes/wes-ingress-kubernetes.yaml b/deployment/templates/wes/wes-ingress-kubernetes.yaml
index 6eda18a..5b9f34b 100644
--- a/deployment/templates/wes/wes-ingress-kubernetes.yaml
+++ b/deployment/templates/wes/wes-ingress-kubernetes.yaml
@@ -8,6 +8,7 @@ metadata:
     kubernetes.io/ingress.class: {{ .Values.ingress.tls_letsencrypt.annotations.ingressclass }}
     kubernetes.io/tls-acme: {{ .Values.ingress.tls_letsencrypt.annotations.tlsacme }}
     cert-manager.io/cluster-issuer: {{ .Values.ingress.tls_letsencrypt.annotations.clusterissuer }}
+    nginx.ingress.kubernetes.io/app-root: "/ga4gh/wes/v1/ui/"
   {{ end }}
 spec: {{ if .Values.tlsSecret }}
   tls:

From 4102a5407051240d664df21663f60087c6e40c15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vikt=C3=B3ria=20Spi=C5=A1akov=C3=A1?=
 <viktoria.spisakovaa@gmail.com>
Date: Wed, 10 Mar 2021 11:10:19 +0100
Subject: [PATCH 4/9] ingress-kubernetes: add app-root label

---
 .../templates/wes/wes-ingress-kubernetes.yaml      |  3 ---
 deployment/templates/wes/wes-ingress.yaml          | 10 ----------
 deployment/values.yaml                             | 14 --------------
 3 files changed, 27 deletions(-)

diff --git a/deployment/templates/wes/wes-ingress-kubernetes.yaml b/deployment/templates/wes/wes-ingress-kubernetes.yaml
index 1a2bc39..5b9f34b 100644
--- a/deployment/templates/wes/wes-ingress-kubernetes.yaml
+++ b/deployment/templates/wes/wes-ingress-kubernetes.yaml
@@ -8,10 +8,7 @@ metadata:
     kubernetes.io/ingress.class: {{ .Values.ingress.tls_letsencrypt.annotations.ingressclass }}
     kubernetes.io/tls-acme: {{ .Values.ingress.tls_letsencrypt.annotations.tlsacme }}
     cert-manager.io/cluster-issuer: {{ .Values.ingress.tls_letsencrypt.annotations.clusterissuer }}
-<<<<<<< HEAD
     nginx.ingress.kubernetes.io/app-root: "/ga4gh/wes/v1/ui/"
-=======
->>>>>>> refs/remotes/origin/dev
   {{ end }}
 spec: {{ if .Values.tlsSecret }}
   tls:
diff --git a/deployment/templates/wes/wes-ingress.yaml b/deployment/templates/wes/wes-ingress.yaml
index b4778ba..a27e9a8 100644
--- a/deployment/templates/wes/wes-ingress.yaml
+++ b/deployment/templates/wes/wes-ingress.yaml
@@ -137,16 +137,6 @@ status: {}
 
 ---
 apiVersion: v1
-<<<<<<< HEAD
-=======
-kind: ServiceAccount
-metadata:
-  creationTimestamp: null
-  name: {{ .Values.wes.appName }}-autoadmin
-
----
-apiVersion: v1
->>>>>>> refs/remotes/origin/dev
 kind: Secret
 metadata:
   name: {{ .Values.tlsSecret }}
diff --git a/deployment/values.yaml b/deployment/values.yaml
index 946f128..da783c3 100644
--- a/deployment/values.yaml
+++ b/deployment/values.yaml
@@ -6,13 +6,10 @@ applicationDomain: c03.k8s-popup.csc.fi  # change to wes server
 clusterType: openshift  # either 'kubernetes' or 'openshift'
 tlsSecret: mytls-secret  # put name of tlsSecret
 storageAccessMode: ReadWriteOnce  # mongodb-pvc.yaml/rabbitmq-pvc.yaml, change to ReadWriteMany if storageClass can do RWX
-<<<<<<< HEAD
 
 extra_config:
   folder: /etc/app_config
   file: app_config.yaml
-=======
->>>>>>> refs/remotes/origin/dev
 
 autocert:
   createJob: "true"  # actually create autocert cronjob, for K8S with autocert installed set to "false"
@@ -30,16 +27,10 @@ flower:
 wes:
   appName: cwlwes
   image: elixircloud/cwl-wes:latest
-<<<<<<< HEAD
   #netrc: # change this, add one entry per FTP/HTTP server
   #- Machine: defaultmachine
   #  Login: defaultnetrclogin
   #  Password: defaultnetrcpassword
-=======
-  netrcMachine: defaultmachine  # change this
-  netrcLogin: defaultnetrclogin  # change this
-  netrcPassword: defaultnetrcpassword  # change this
->>>>>>> refs/remotes/origin/dev
   storageClass: nfs-client  # <- wes-volume.yaml, only for K8S, a storageClass with readWriteMany capability is required
   volumeSize: 1Gi
 
@@ -76,8 +67,3 @@ ingress: # <- wes-ingress-kubernetes.yaml, only for system ingress for K8S, ensu
       ingressclass: "nginx"
       tlsacme: "\"true\""
       clusterissuer: "letsencrypt-prod"
-<<<<<<< HEAD
-=======
-
-
->>>>>>> refs/remotes/origin/dev

From 0127f4942998906e3023c12c875af814383611ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vikt=C3=B3ria=20Spi=C5=A1akov=C3=A1?=
 <viktoria.spisakovaa@gmail.com>
Date: Thu, 17 Mar 2022 17:09:51 +0100
Subject: [PATCH 5/9] add securityContext, chmod cookie, fix mongo initc if

---
 deployment/templates/flower/flower-deployment.yaml     |  5 ++++-
 deployment/templates/mongodb/mongodb-deployment.yaml   |  2 +-
 deployment/templates/rabbitmq/rabbitmq-deployment.yaml | 10 ++++++++++
 deployment/templates/wes/celery-deployment.yaml        |  5 ++++-
 deployment/templates/wes/wes-configmap.yaml            |  4 ++++
 deployment/templates/wes/wes-deployment.yaml           |  5 ++++-
 deployment/templates/wes/wes-ingress-kubernetes.yaml   |  1 -
 deployment/values.yaml                                 |  3 +++
 8 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/deployment/templates/flower/flower-deployment.yaml b/deployment/templates/flower/flower-deployment.yaml
index f9dc8ba..88c2f62 100644
--- a/deployment/templates/flower/flower-deployment.yaml
+++ b/deployment/templates/flower/flower-deployment.yaml
@@ -19,4 +19,7 @@ spec:
         command: ['flower']
         args: ['--broker=amqp://guest:guest@rabbitmq:5672//', '--port=5555', '--basic_auth={{ .Values.flower.basicAuth }}']
         name: flower
-
+{{- if .Values.securityContext }}
+      securityContext:
+{{- toYaml .Values.securityContext | nindent 8 -}} 
+{{- end }}
diff --git a/deployment/templates/mongodb/mongodb-deployment.yaml b/deployment/templates/mongodb/mongodb-deployment.yaml
index 75c01fc..c8cc96a 100644
--- a/deployment/templates/mongodb/mongodb-deployment.yaml
+++ b/deployment/templates/mongodb/mongodb-deployment.yaml
@@ -14,7 +14,7 @@ spec:
       labels:
         app: {{ .Values.mongodb.appName }}
     spec:
-      {{ if eq .Values.clusterType "kubernetes" }}
+      {{ if and (eq .Values.clusterType "kubernetes") (eq ( .Values.mongodb.securityContext.runAsUser | int) 0) }}
       initContainers:
         - name: volume-permissions
           image: busybox
diff --git a/deployment/templates/rabbitmq/rabbitmq-deployment.yaml b/deployment/templates/rabbitmq/rabbitmq-deployment.yaml
index b31bdeb..7e4fd19 100644
--- a/deployment/templates/rabbitmq/rabbitmq-deployment.yaml
+++ b/deployment/templates/rabbitmq/rabbitmq-deployment.yaml
@@ -17,6 +17,12 @@ spec:
       containers:
       - name: rabbitmq
         image: {{ .Values.rabbitmq.image }}
+        command:
+          - /bin/sh
+          - -c
+          - |
+            chmod g-rw /var/lib/rabbitmq/.erlang.cookie;
+            /opt/rabbitmq/sbin/rabbitmq-server
         volumeMounts:
         - mountPath: /var/lib/rabbitmq
           name: rabbitmq-volume
@@ -24,4 +30,8 @@ spec:
       - name: rabbitmq-volume
         persistentVolumeClaim:
           claimName: {{ .Values.rabbitmq.appName }}-volume
+{{- if .Values.securityContext }}
+      securityContext:
+{{- toYaml .Values.securityContext | nindent 8 -}}
+{{- end }}
 
diff --git a/deployment/templates/wes/celery-deployment.yaml b/deployment/templates/wes/celery-deployment.yaml
index 50ba977..fc84540 100644
--- a/deployment/templates/wes/celery-deployment.yaml
+++ b/deployment/templates/wes/celery-deployment.yaml
@@ -82,4 +82,7 @@ spec:
           items:
           - key: netrc
             path: .netrc
-
+{{- if .Values.securityContext }}
+      securityContext:
+{{- toYaml .Values.securityContext | nindent 8 -}} 
+{{- end }}
diff --git a/deployment/templates/wes/wes-configmap.yaml b/deployment/templates/wes/wes-configmap.yaml
index a38212e..90dc2d0 100644
--- a/deployment/templates/wes/wes-configmap.yaml
+++ b/deployment/templates/wes/wes-configmap.yaml
@@ -22,5 +22,9 @@ spec:
           value: {{ .Values.wes.appName }}
       restartPolicy: Never
       serviceAccountName: {{ .Values.wes.appName }}-autoadmin
+{{- if .Values.securityContext }}
+      securityContext:
+{{- toYaml .Values.securityContext | nindent 8 -}} 
+{{- end }}
 status: {}
 
diff --git a/deployment/templates/wes/wes-deployment.yaml b/deployment/templates/wes/wes-deployment.yaml
index 93ca230..182917a 100644
--- a/deployment/templates/wes/wes-deployment.yaml
+++ b/deployment/templates/wes/wes-deployment.yaml
@@ -97,4 +97,7 @@ spec:
           defaultMode: 420
           name: app-config
         name: app-config
-
+{{- if .Values.securityContext }}
+      securityContext:
+{{- toYaml .Values.securityContext | nindent 8 -}}
+{{- end }}
diff --git a/deployment/templates/wes/wes-ingress-kubernetes.yaml b/deployment/templates/wes/wes-ingress-kubernetes.yaml
index 30d2605..f04e526 100644
--- a/deployment/templates/wes/wes-ingress-kubernetes.yaml
+++ b/deployment/templates/wes/wes-ingress-kubernetes.yaml
@@ -8,7 +8,6 @@ metadata:
     kubernetes.io/ingress.class: {{ .Values.ingress.tls_letsencrypt.annotations.ingressclass }}
     kubernetes.io/tls-acme: {{ .Values.ingress.tls_letsencrypt.annotations.tlsacme }}
     cert-manager.io/cluster-issuer: {{ .Values.ingress.tls_letsencrypt.annotations.clusterissuer }}
-    nginx.ingress.kubernetes.io/app-root: "/ga4gh/wes/v1/ui/"
   {{ end }}
 spec: {{ if .Values.tlsSecret }}
   tls:
diff --git a/deployment/values.yaml b/deployment/values.yaml
index d125585..5067cb5 100644
--- a/deployment/values.yaml
+++ b/deployment/values.yaml
@@ -7,6 +7,9 @@ clusterType: openshift  # either 'kubernetes' or 'openshift'
 tlsSecret: mytls-secret  # put name of tlsSecret
 storageAccessMode: ReadWriteOnce  # mongodb-pvc.yaml/rabbitmq-pvc.yaml, change to ReadWriteMany if storageClass can do RWX
 
+securityContext:
+  runAsUser: 1000
+
 extra_config:
   folder: /etc/app_config
   file: app_config.yaml

From 3918cba6c1ac7576d16ab8cc8a4a58332b5fda54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vikt=C3=B3ria=20Spi=C5=A1akov=C3=A1?=
 <viktoria.spisakovaa@gmail.com>
Date: Mon, 21 Mar 2022 14:59:26 +0100
Subject: [PATCH 6/9] modify README, comment securityContext

---
 deployment/README.md                           | 18 ++++++++++++++++++
 .../templates/mongodb/mongodb-deployment.yaml  |  8 ++++----
 deployment/templates/wes/wes-deployment.yaml   |  1 -
 deployment/values.yaml                         |  6 ++++--
 4 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/deployment/README.md b/deployment/README.md
index acdfecd..bfe1e5d 100644
--- a/deployment/README.md
+++ b/deployment/README.md
@@ -80,6 +80,23 @@ curl -X POST \
 The helm chart utilizes scheduled TLS certificate fetching from [Let's
 Encrypt](https://letsencrypt.org/).
 
+## Security context
+Many clusters feature security policy that forbids various actions in cluster. Usually, security policy includes requirement that resources must be deployed under non-root user. The requirement is satisfied by setting `securityContext` section in resources. 
+
+`Values.yaml` offer setting security context only for Kubernetes clusters. It is set on two places:
+- `mongodb.securityContext` for settings related to mongoDB
+- `securityContext` for all other resources supporting security context
+
+If you wish to run all your deployments under root, leave `securityContext` commented and set `mongodb.securityContext.runAsUser` to `0` and `mongodb.securityContext.runAsNonRoot` to `false`. If you can't or don't want to run under root, you should set `securityContext` and `mongodb.securityContext` sections to your needs. `securityContext` is map of key value pairs that are directly translated to Kubernetes security context so you can set all key-value pairs allowed in the section, e.g.:
+```
+securityContext:                                                                
+  runAsUser: 1000
+  runAsNonRoot: true
+  fsGroup: 1001
+```
+
+[MongoDB deployment](https://github.com/elixir-cloud-aai/cwl-WES/blob/dev/deployment/templates/mongodb/mongodb-deployment.yaml#L17) includes init container that runs only as root. Setting `mongodb.securityContext.runAsUser` to uid different from `0`  will disable the init container.
+
 ## To do
 
 - Test autocert with vanilla Kubernetes
@@ -121,6 +138,7 @@ See [`values.yaml`](values.yaml) for default values.
 | rabbitmq.appName | string | name of RabbitMQ app on Kubernetes cluster |
 | rabbitmq.image | string | container image to be used to run RabbitMQ |
 | rabbitmq.volumeSize | string | size of volume reserved for RabbitMQ broker |
+| securityContext | map | for K8s, if uncommented the whole section is translated into Kubernetes `securityContext`, see section `Security Context` |
 | storageAccessMode | string | access mode for MongoDB and RabbitMQ PVC |
 | tlsSecret | string | secret for TLS encryption |
 | wes.appName | string | name of the main application on Kubernetes cluster |
diff --git a/deployment/templates/mongodb/mongodb-deployment.yaml b/deployment/templates/mongodb/mongodb-deployment.yaml
index c8cc96a..a227b65 100644
--- a/deployment/templates/mongodb/mongodb-deployment.yaml
+++ b/deployment/templates/mongodb/mongodb-deployment.yaml
@@ -14,7 +14,7 @@ spec:
       labels:
         app: {{ .Values.mongodb.appName }}
     spec:
-      {{ if and (eq .Values.clusterType "kubernetes") (eq ( .Values.mongodb.securityContext.runAsUser | int) 0) }}
+      {{- if and (eq .Values.clusterType "kubernetes") (eq (.Values.mongodb.securityContext.runAsUser | int) 0) }}
       initContainers:
         - name: volume-permissions
           image: busybox
@@ -25,7 +25,7 @@ spec:
           volumeMounts:
             - name: mongodb-data
               mountPath: {{ .Values.mongodb.mountPath }}
-      {{ end }}
+      {{- end }}
       containers:
         - env:
           - name: MONGODB_USER
@@ -79,11 +79,11 @@ spec:
           resources:
             limits:
               memory: 512Mi
-          {{ if eq .Values.clusterType "kubernetes" }}
+          {{- if eq .Values.clusterType "kubernetes" }}
           securityContext:
             runAsNonRoot: {{ .Values.mongodb.securityContext.runAsNonRoot }}
             runAsUser: {{ .Values.mongodb.securityContext.runAsUser }}
-          {{ end }}
+          {{- end }}
           volumeMounts:
             - mountPath: /var/lib/mongodb/data
               name: mongodb-data
diff --git a/deployment/templates/wes/wes-deployment.yaml b/deployment/templates/wes/wes-deployment.yaml
index 182917a..9a89c92 100644
--- a/deployment/templates/wes/wes-deployment.yaml
+++ b/deployment/templates/wes/wes-deployment.yaml
@@ -86,7 +86,6 @@ spec:
       - name: wes-volume
         persistentVolumeClaim:
           claimName: {{ .Values.wes.appName }}-volume
-
       - name: wes-netrc-secret
         secret:
           secretName: netrc
diff --git a/deployment/values.yaml b/deployment/values.yaml
index 5067cb5..f79e0b9 100644
--- a/deployment/values.yaml
+++ b/deployment/values.yaml
@@ -7,8 +7,10 @@ clusterType: openshift  # either 'kubernetes' or 'openshift'
 tlsSecret: mytls-secret  # put name of tlsSecret
 storageAccessMode: ReadWriteOnce  # mongodb-pvc.yaml/rabbitmq-pvc.yaml, change to ReadWriteMany if storageClass can do RWX
 
-securityContext:
-  runAsUser: 1000
+# If cluster has security policy enabled, this security context will be propagated
+# Uncomment whole section to take effect, see README for more details
+#securityContext:
+#  runAsUser: 1000
 
 extra_config:
   folder: /etc/app_config

From d707b3db9f5b21427337b30ba568751de74e136a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vikt=C3=B3ria=20Spi=C5=A1akov=C3=A1?=
 <viktoria.spisakovaa@gmail.com>
Date: Mon, 21 Mar 2022 15:22:05 +0100
Subject: [PATCH 7/9] better initContainer sec context handling

---
 deployment/README.md                                 | 8 ++++++--
 deployment/templates/mongodb/mongodb-deployment.yaml | 2 +-
 deployment/values.yaml                               | 2 ++
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/deployment/README.md b/deployment/README.md
index bfe1e5d..b146fd6 100644
--- a/deployment/README.md
+++ b/deployment/README.md
@@ -84,10 +84,13 @@ Encrypt](https://letsencrypt.org/).
 Many clusters feature security policy that forbids various actions in cluster. Usually, security policy includes requirement that resources must be deployed under non-root user. The requirement is satisfied by setting `securityContext` section in resources. 
 
 `Values.yaml` offer setting security context only for Kubernetes clusters. It is set on two places:
+- `mongodb.initContainer.runAsRoot` for settings related to mongoDB init container
 - `mongodb.securityContext` for settings related to mongoDB
 - `securityContext` for all other resources supporting security context
 
-If you wish to run all your deployments under root, leave `securityContext` commented and set `mongodb.securityContext.runAsUser` to `0` and `mongodb.securityContext.runAsNonRoot` to `false`. If you can't or don't want to run under root, you should set `securityContext` and `mongodb.securityContext` sections to your needs. `securityContext` is map of key value pairs that are directly translated to Kubernetes security context so you can set all key-value pairs allowed in the section, e.g.:
+If you wish to run all your deployments under root, leave `securityContext`, set `mongodb.securityContext.runAsUser` to `0`, `mongodb.securityContext.runAsNonRoot` to `false` and `mongodb.initContainer.runAsRoot` to `true`. 
+
+[MongoDB deployment](https://github.com/elixir-cloud-aai/cwl-WES/blob/dev/deployment/templates/mongodb/mongodb-deployment.yaml#L17) includes init container that runs only as root. If you can't run deployments under root, you should set `securityContext` and `mongodb.securityContext` sections to your needs and `mongodb.initContainer.runAsRoot` to `false` (leads to disabling root initContainer). `securityContext` is map of key value pairs that are directly translated to Kubernetes security context so you can set all key-value pairs allowed in the section, e.g.:
 ```
 securityContext:                                                                
   runAsUser: 1000
@@ -95,7 +98,7 @@ securityContext:
   fsGroup: 1001
 ```
 
-[MongoDB deployment](https://github.com/elixir-cloud-aai/cwl-WES/blob/dev/deployment/templates/mongodb/mongodb-deployment.yaml#L17) includes init container that runs only as root. Setting `mongodb.securityContext.runAsUser` to uid different from `0`  will disable the init container.
+If you don't want to run under root but there you are not forced to run non-root, you can set security contexts as you wish where e.g. the `securityContext` and  `mongodb.securityContext` will be set to non-root and `mongodb.initContainer.runAsRoot` to `true` to keep the init container (chown can be done only under root user). 
 
 ## To do
 
@@ -128,6 +131,7 @@ See [`values.yaml`](values.yaml) for default values.
 | mongodb.databasePassword | string | user password for MongoDB |
 | mongodb.databaseUser | string | username for MongoDB |
 | mongodb.image | string | container image to be used to run MongoDB |
+| mongodb.initContainer.runAsRoot | bool | whether run init container under root user, see section `Security Context` for more information |
 | mongodb.mountPath| string | for K8S, where to mount the PVC |
 | mongodb.pullPolicy | string | pull Policy for container image |
 | mongodb.securityContext.enabled | string | for K8S, whether security is enabled (to solve issues with newly created PVC) |
diff --git a/deployment/templates/mongodb/mongodb-deployment.yaml b/deployment/templates/mongodb/mongodb-deployment.yaml
index a227b65..b06ced0 100644
--- a/deployment/templates/mongodb/mongodb-deployment.yaml
+++ b/deployment/templates/mongodb/mongodb-deployment.yaml
@@ -14,7 +14,7 @@ spec:
       labels:
         app: {{ .Values.mongodb.appName }}
     spec:
-      {{- if and (eq .Values.clusterType "kubernetes") (eq (.Values.mongodb.securityContext.runAsUser | int) 0) }}
+      {{- if and (eq .Values.clusterType "kubernetes") .Values.mongodb.initContainer.runAsRoot }}
       initContainers:
         - name: volume-permissions
           image: busybox
diff --git a/deployment/values.yaml b/deployment/values.yaml
index f79e0b9..82aee08 100644
--- a/deployment/values.yaml
+++ b/deployment/values.yaml
@@ -55,6 +55,8 @@ mongodb:
   databaseUser: cwlwes-user
   volumeSize: 1Gi
   image: centos/mongodb-36-centos7
+  initContainer:
+    runAsRoot: false
   mountPath: /var/lib/mongodb/data
   pullPolicy: Always
   securityContext:  # only for K8S

From 0610dd81305cdf18dd78ae389a18acbbf61826c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vikt=C3=B3ria=20Spi=C5=A1akov=C3=A1?=
 <viktoria.spisakovaa@gmail.com>
Date: Mon, 21 Mar 2022 15:27:56 +0100
Subject: [PATCH 8/9] fix typo

---
 deployment/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/deployment/README.md b/deployment/README.md
index b146fd6..e0ac514 100644
--- a/deployment/README.md
+++ b/deployment/README.md
@@ -83,7 +83,7 @@ Encrypt](https://letsencrypt.org/).
 ## Security context
 Many clusters feature security policy that forbids various actions in cluster. Usually, security policy includes requirement that resources must be deployed under non-root user. The requirement is satisfied by setting `securityContext` section in resources. 
 
-`Values.yaml` offer setting security context only for Kubernetes clusters. It is set on two places:
+`Values.yaml` offer setting security context only for Kubernetes clusters. It is set on three places:
 - `mongodb.initContainer.runAsRoot` for settings related to mongoDB init container
 - `mongodb.securityContext` for settings related to mongoDB
 - `securityContext` for all other resources supporting security context
@@ -98,7 +98,7 @@ securityContext:
   fsGroup: 1001
 ```
 
-If you don't want to run under root but there you are not forced to run non-root, you can set security contexts as you wish where e.g. the `securityContext` and  `mongodb.securityContext` will be set to non-root and `mongodb.initContainer.runAsRoot` to `true` to keep the init container (chown can be done only under root user). 
+If you don't want to run under root but you are not forced to run non-root, you can set security contexts as you wish where e.g. the `securityContext` and  `mongodb.securityContext` will be set to non-root and `mongodb.initContainer.runAsRoot` to `true` to keep the init container (chown can be done only under root user). 
 
 ## To do
 
@@ -142,7 +142,7 @@ See [`values.yaml`](values.yaml) for default values.
 | rabbitmq.appName | string | name of RabbitMQ app on Kubernetes cluster |
 | rabbitmq.image | string | container image to be used to run RabbitMQ |
 | rabbitmq.volumeSize | string | size of volume reserved for RabbitMQ broker |
-| securityContext | map | for K8s, if uncommented the whole section is translated into Kubernetes `securityContext`, see section `Security Context` |
+| securityContext | map | for K8s, if uncommented the section is used as Kubernetes `securityContext`, see section `Security Context` |
 | storageAccessMode | string | access mode for MongoDB and RabbitMQ PVC |
 | tlsSecret | string | secret for TLS encryption |
 | wes.appName | string | name of the main application on Kubernetes cluster |

From 87bfa8eccf5c21c4cc03d48cb91cbbfc0f1ad4ca Mon Sep 17 00:00:00 2001
From: viktoriaas <viktoria.spisakovaa@gmail.com>
Date: Thu, 31 Mar 2022 12:19:13 +0200
Subject: [PATCH 9/9] comment chmoding cookie

---
 deployment/templates/rabbitmq/rabbitmq-deployment.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deployment/templates/rabbitmq/rabbitmq-deployment.yaml b/deployment/templates/rabbitmq/rabbitmq-deployment.yaml
index 7e4fd19..d92a026 100644
--- a/deployment/templates/rabbitmq/rabbitmq-deployment.yaml
+++ b/deployment/templates/rabbitmq/rabbitmq-deployment.yaml
@@ -21,8 +21,8 @@ spec:
           - /bin/sh
           - -c
           - |
-            chmod g-rw /var/lib/rabbitmq/.erlang.cookie;
-            /opt/rabbitmq/sbin/rabbitmq-server
+            chmod g-rw /var/lib/rabbitmq/.erlang.cookie;    # If rabbitMQ deployment is restarted (e.g. cluster failure) cookie has incorrect permissions
+            /opt/rabbitmq/sbin/rabbitmq-server              # Solved by chmod before calling rabbitmq (https://github.com/elixir-cloud-aai/cwl-WES/issues/232)
         volumeMounts:
         - mountPath: /var/lib/rabbitmq
           name: rabbitmq-volume