[patch] added support for MariaDB and Minio - both dependencies are mandatory for AIBrooker FVT integration

ibm/mas_devops/roles/aibroker/tasks/aibroker/main.yml

@@ -17,6 +17,7 @@
     kind: Namespace
     name: "{{ aibroker_namespace }}"
 
+
 # Install the operator & create entitlement secret
 # -----------------------------------------------------------------------------
 - name: "Create IBM Entitlement Key"

ibm/mas_devops/roles/arcgis/tasks/main.yml

@@ -43,6 +43,7 @@
     state: present
     template: templates/namespace.yml.j2
 
+
 # 5. Install the operator & create entitlement secret
 # -----------------------------------------------------------------------------
 - name: "Create IBM Entitlement Key"

ibm/mas_devops/roles/cp4d/tasks/install-cp4d.yml

@@ -160,15 +160,6 @@
       retries: 20 # Approximately 20 minutes before we give up
       delay: 60 # 1 minute
 
-# 3-pre. Wait for Zen Metastore Cluster  to be ready
-# -----------------------------------------------------------------------------
-# There have been issues with Zen Metastore Cluster not starting due to Persistent Storage,
-# This task restarts any failing pods
-- name: "install-cp4d : Wait for zen-metadata to be ready again (60s delay)"
-  include_tasks: "tasks/wait/wait-zenmetastore-edb.yml"
-  when:
-    - cpd_48_or_higher
-
 # 3. Wait for zenStatus
 # -----------------------------------------------------------------------------
 # oc get ZenService lite-cr -o jsonpath="{.status.zenStatus}{'\n'}"

ibm/mas_devops/roles/db2/tasks/install/main.yml

@@ -157,6 +157,7 @@
 - name: "Delete old db2 subscription, operand request and csv from {{ ibm_common_services_namespace }}"
   include_tasks: "tasks/delete_db2_operand_request.yml"
 
+
 - name: "Create Db2 Universal Operator Subscription in {{ db2_namespace }} namespace"
   ibm.mas_devops.apply_subscription:
     namespace: "{{ db2_namespace }}"

ibm/mas_devops/roles/kafka/tasks/determine-storage-classes.yml

@@ -26,6 +26,7 @@
     that: zookeeper_storage_class is defined and zookeeper_storage_class != ""
     fail_msg: "zookeeper_storage_class must be defined"
 
+
 # 3. Debug storage class configuration
 # -----------------------------------------------------------------------------
 - name: "Debug Kafka storage classes configuration"

ibm/mas_devops/roles/mariadb/README.md

@@ -0,0 +1,68 @@
+# Mariadb
+=====
+
+This role provides support to install and configure MariaDB 
+
+* Install MariaDB
+
+Role Variables
+--------------
+
+### MAS core instance
+Action to be performed by mariadb role. Valid values are `string`.
+
+* **Required**
+* Environment Variable: `MAS_INSTANCE_ID`
+* Default Value: ""
+
+### mariadb_namespace
+Action to be performed by mariadb role. Valid values are `string`.
+
+* Optional
+* Environment Variable: `MARIADB_NAMESPACE`
+* Default Value: `mariadb`
+
+### mariadb_instance_name
+Action to be performed by mariadb role. Valid values are `string`.
+
+* Optional
+* Environment Variable: `MARIADB_INSTANCE_NAME`
+* Default Value: `mariadb-instance`
+
+### mariadb_user
+Action to be performed by mariadb role. Valid values are `string`.
+
+* Optional
+* Environment Variable: `MARIADB_USER`
+* Default Value: `mariadb`
+
+### mariadb_password
+Action to be performed by mariadb role. Valid values are `string`.
+
+* **Required**
+* Environment Variable: `MARIADB_PASSWORD`
+* Default Value: ``
+
+### mariadb_database
+Action to be performed by mariadb role. Valid values are `string`.
+
+* Optional  
+* Environment Variable: `MARIADB_DATABASE`
+* Default Value: `kmpipeline`
+
+
+Example Playbook
+----------------
+
+```yaml
+---
+- hosts: localhost
+  any_errors_fatal: true
+  roles:
+    - ibm.mas_devops.mariadb
+```
+
+License
+-------
+
+EPL-2.0

ibm/mas_devops/roles/mariadb/defaults/main.yml

@@ -0,0 +1,27 @@
+---
+# When these are defined we will generate a MAS MongoCfg template
+mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}"
+mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}"
+
+# Source container registry
+# -----------------------------------------------------------------------------
+mas_icr_cp: "{{ lookup('env', 'MAS_ICR_CP') | default('cp.icr.io/cp', true) }}"
+mas_icr_cpopen: "{{ lookup('env', 'MAS_ICR_CPOPEN') | default('icr.io/cpopen', true) }}"
+
+# MAS Entitlement
+# -----------------------------------------------------------------------------
+mas_entitlement_username: "{{ lookup('env', 'MAS_ENTITLEMENT_USERNAME') | default('cp', true) }}"
+ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}"
+mas_entitlement_key: "{{ lookup('env', 'MAS_ENTITLEMENT_KEY') | default(ibm_entitlement_key, true) }}"
+
+# MAS Annotation block
+# -----------------------------------------------------------------------------
+mas_annotations: "{{ lookup('env', 'MAS_ANNOTATIONS') | default(None, true) }}"
+
+# MariaDB
+mariadb_namespace: "{{ lookup('env', 'MARIADB_NAMESPACE') | default('mariadb', True) }}"
+mariadb_instance_name: "{{ lookup('env', 'MARIADB_INSTANCE_NAME') | default('mariadb-instance', True) }}"
+mariadb_user: "{{ lookup('env', 'MARIADB_USER') | default('mariadb', True) }}"
+mariadb_password: "{{ lookup('env', 'MARIADB_PASSWORD') | default('', True) }}"
+mariadb_database: "{{ lookup('env', 'MARIADB_DATABASE') | default('kmpipeline', True) }}"
+mariadb_version: "registry.redhat.io/rhel8/mariadb-103:1-188"

ibm/mas_devops/roles/mariadb/meta/main.yml

@@ -0,0 +1,21 @@
+galaxy_info:
+  author: Karol Czarnecki (@karol-czarnecki)
+  description: Deploy Minio storage
+  company: IBM
+
+  license: EPL-2.0
+
+  min_ansible_version: 2.10
+
+  platforms:
+    - name: GenericLinux
+      versions:
+        - all
+
+  galaxy_tags:
+    - ibm
+    - mas
+    - devops
+
+dependencies:
+  - role: ibm.mas_devops.ansible_version_check

ibm/mas_devops/roles/mariadb/tasks/determine-storage-classes.yml

@@ -0,0 +1,20 @@
+# Lookup storage class availabiity
+# -----------------------------------------------------------------------------
+- name: "determine-storage-classes : Load default storage class information"
+  include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml"
+
+# Set Primary Storage (Required)
+# -----------------------------------------------------------------------------
+- name: "determine-storage-classes : Default Primary Storage if not set by user"
+  when: primary_storage_class is not defined or primary_storage_class == ""
+  set_fact:
+    primary_storage_class: "{{ defaultStorageClasses.rwx }}"
+
+- name: "determine-storage-classes : Assert that primary storage class has been defined"
+  assert:
+    that: primary_storage_class is defined and primary_storage_class != ""
+    fail_msg: "primary_storage_class must be defined"
+
+- name: Set primary StorageClass
+  debug:
+    msg: "Primary storage class ...... {{ primary_storage_class }}"

ibm/mas_devops/roles/mariadb/tasks/main.yml

@@ -0,0 +1,49 @@
+---
+- name: Load variables (main)
+  include_vars: "vars/main.yml"
+
+# Load default storage class (if not provided by the user)
+# -----------------------------------------------------------------------------
+- include_tasks: tasks/determine-storage-classes.yml
+
+# create mariadb namespace
+- name: 'Create mariadb namespace'
+  kubernetes.core.k8s:
+    apply: yes
+    definition: "{{ lookup('template', 'templates/mariadb/mariadb-ns.yml.j2') }}"
+
+# create mariadb PVC
+- name: 'Create mariadb pvc'
+  kubernetes.core.k8s:
+    apply: yes
+    definition: "{{ lookup('template', 'templates/mariadb/mariadb-pvc.yml.j2') }}"
+
+# create mariadb sa
+- name: 'Create mariadb sa'
+  kubernetes.core.k8s:
+    apply: yes
+    definition: "{{ lookup('template', 'templates/mariadb/mariadb-sa.yml.j2') }}"
+
+# create mariadb np
+- name: 'Create mariadb np'
+  kubernetes.core.k8s:
+    apply: yes
+    definition: "{{ lookup('template', 'templates/mariadb/mariadb-np.yml.j2') }}"
+
+# create mariadb secret
+- name: 'Create mariadb secret'
+  kubernetes.core.k8s:
+    apply: yes
+    definition: "{{ lookup('template', 'templates/mariadb/mariadb-secret.yml.j2') }}"
+
+# create mariadb deployment
+- name: 'Create mariadb deployment'
+  kubernetes.core.k8s:
+    apply: yes
+    definition: "{{ lookup('template', 'templates/mariadb/mariadb-deployment.yml.j2') }}"
+
+# create mariadb service
+- name: 'Create mariadb service'
+  kubernetes.core.k8s:
+    apply: yes
+    definition: "{{ lookup('template', 'templates/mariadb/mariadb-service.yml.j2') }}"

ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-deployment.yml.j2

@@ -0,0 +1,79 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ mariadb_instance_name }}"
+  namespace: "{{ mariadb_namespace }}"
+  labels:
+    app: "{{ mariadb_instance_name }}"
+    component: data-science-pipelines
+    dspa: instance
+spec:
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: "{{ mariadb_instance_name }}"
+      component: data-science-pipelines
+      dspa: instance
+  template:
+    metadata:
+      labels:
+        app: "{{ mariadb_instance_name }}"
+        component: data-science-pipelines
+        dspa: instance
+    spec:
+      serviceAccountName: ds-pipelines-mariadb-sa-instance
+      containers:
+        - name: mariadb
+          image: "{{ mariadb_version }}"
+          ports:
+            - containerPort: 3306
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - "-i"
+                - "-c"
+                - >-
+                  MYSQL_PWD=$MYSQL_PASSWORD mysql -h 127.0.0.1 -u $MYSQL_USER -D
+                  $MYSQL_DATABASE -e 'SELECT 1'
+            failureThreshold: 3
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            tcpSocket:
+              port: 3306
+            timeoutSeconds: 1
+          env:
+            - name: MYSQL_USER
+              value: "{{ mariadb_user }}"
+            - name: MYSQL_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: password
+                  name: "mariadb-instance"
+            - name: MYSQL_DATABASE
+              value: "{{ mariadb_database }}"
+            - name: MYSQL_ALLOW_EMPTY_PASSWORD
+              value: "true"
+          resources:
+            requests:
+              cpu: 300m
+              memory: 800Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
+          volumeMounts:
+            - name: mariadb-persistent-storage
+              mountPath: /var/lib/mysql
+      volumes:
+        - name: mariadb-persistent-storage
+          persistentVolumeClaim:
+            claimName: mariadb-pvc

ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-np.yml.j2

@@ -0,0 +1,38 @@
+---
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: "{{ mariadb_instance_name }}"
+  namespace: "{{ mariadb_namespace }}"
+spec:
+  podSelector:
+    matchLabels:
+      app: "{{ mariadb_instance_name }}"
+      component: data-science-pipelines
+  ingress:
+    - ports:
+        - protocol: TCP
+          port: 3306
+      from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: data-science-pipelines-operator
+          namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: opendatahub
+        - podSelector:
+            matchLabels:
+              app: ds-pipeline-instance
+              component: data-science-pipelines
+          namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: "mas-{{ mas_instance_id }}-aibroker"
+        - podSelector:
+            matchLabels:
+              app: ds-pipeline-metadata-grpc-instance
+              component: data-science-pipelines
+          namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: "mas-{{ mas_instance_id }}-aibroker"
+  policyTypes:
+    - Ingress

ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-ns.yml.j2

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: "{{ mariadb_namespace }}"

ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-pvc.yml.j2

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: mariadb-pvc
+  namespace: "{{ mariadb_namespace }}"
+  labels:
+    app: "{{ mariadb_instance_name }}"
+    component: data-science-pipelines
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: "{{ primary_storage_class }}"
+  resources:
+    requests:
+      storage: 20Gi

ibm/mas_devops/roles/mariadb/templates/mariadb/mariadb-sa.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ds-pipelines-mariadb-sa-instance
+  namespace: "{{ mariadb_namespace }}"
+  labels:
+    app: "{{ mariadb_instance_name }}"
+    component: data-science-pipelines