Merge branch 'main' into EVEREST-1350-ui-display-ip-address-with-netm…

…ask-same-as-backend

.github/minio.conf.yaml

@@ -50,11 +50,15 @@ spec:
       initContainers:
         - name: init-minio
           image: busybox
+          env:
+            - name: BUCKET_DIRS
+              value: "bucket-1,bucket-2,bucket-3,bucket-4,bucket-5"
           command:
             - /bin/sh
             - -c
           args:
-            - mkdir -p /data/bucket-1 /data/bucket-2 /data/bucket-3 /data/bucket-4 /data/bucket-5
+            - |
+              IFS=',' ; for i in $BUCKET_DIRS; do mkdir -p /data/$i || true ; done
           volumeMounts:
             - mountPath: /data
               name: minio-data-vol

dev/.env.example

@@ -0,0 +1,31 @@
+# Copy this file to .env in dev/ directory and set the values as needed.
+
+# Path to Everest server operator source code directory.
+# Hint: <path to github.com/percona/everest-operator repository directory>
+# Mandatory.
+EVEREST_OPERATOR_DIR=
+
+# Path to Everest helm charts source code directory.
+# Hint:  <path to github.com/percona/percona-helm-charts repository directory>/charts/everest
+# Mandatory.
+EVEREST_CHART_DIR=
+
+# PXC operator version for deploying into Everest DB namespaces.
+# Optional, default: 1.16.1
+#PXC_OPERATOR_VERSION=1.16.1
+
+# PSMDB operator version for deploying into Everest DB namespaces.
+# Optional, default: 1.19.1
+#PSMDB_OPERATOR_VERSION=1.19.1
+
+# PG operator version for deploying into Everest DB namespaces.
+# Optional, default: 2.5.0
+#PG_OPERATOR_VERSION=2.5.0
+
+# Enable building debug image for Everest server.
+# Optional, default: false
+#EVEREST_DEBUG=true
+
+# Enable building debug image for Everest operator.
+# Optional, default: false
+#EVEREST_OPERATOR_DEBUG=true

dev/.gitignore

@@ -0,0 +1,2 @@
+.env
+config.yaml

dev/README.md

@@ -50,20 +50,33 @@ gcloud auth configure-docker <REGISTRY_REGION>-docker.pkg.dev
 
 
 ### 2. Run tilt
-1. Set the following environment variables:
+1. Set environment variables:
+
+Copy file dev/.env.example to dev/.env and set the following environment variables:
 ```sh
-export EVEREST_OPERATOR_DIR=</path/to/everest-operator>
-export EVEREST_CHART_DIR=</path/to/percona-helm-charts>/charts/everest
+EVEREST_OPERATOR_DIR=<path to github.com/percona/everest-operator repository directory>
+EVEREST_CHART_DIR=<path to github.com/percona/percona-helm-charts>/charts/everest
 ```
 
-2. (Optional) If you want to test a specific version of a given DB operator you can set the following environment variables
+or set environment variables manually in the terminal:
+
+```sh
+export EVEREST_OPERATOR_DIR=<path to github.com/percona/everest-operator repository directory>
+export EVEREST_CHART_DIR=<path to github.com/percona/percona-helm-charts>/charts/everest
+```
+
+2. Set namespaces for the Everest components:
+
+Copy file dev/config.yaml.example to dev/config.yaml and set the needed DB namespaces that will be created automatically.
+
+3. (Optional) If you want to test a specific version of a given DB operator you can set the following environment variables in .env file or in the terminal:
 ```sh
-export PXC_OPERATOR_VERSION=1.13.0
-export PSMDB_OPERATOR_VERSION=1.15.0
-export PG_OPERATOR_VERSION=2.3.1
+export PXC_OPERATOR_VERSION=1.16.1
+export PSMDB_OPERATOR_VERSION=1.19.1
+export PG_OPERATOR_VERSION=2.5.0
 ```
 
-3. (Optional) If you want to debug Everest Server and/or Everest operator remotely, you can set the following environment variables 
+4. (Optional) If you want to debug Everest Server and/or Everest operator remotely, you can set the following environment variables in .env file or in the terminal: 
 ```sh
 export EVEREST_DEBUG=true
 export EVEREST_OPERATOR_DEBUG=true
@@ -78,7 +91,7 @@ Refer to instructions in your IDE on how to setup remote debugging.
 
 For GoLand, you can refer to [this](https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html#step-2-create-the-go-remote-run-debug-configuration) link.
 
-4. Run tilt
+5. Run tilt
 ```sh
 tilt up
 ```

dev/Tiltfile

@@ -1,75 +1,81 @@
-# Config
+# Some utilities
+def str2bool(v):
+    return v.lower() in ("yes", "true", "t", "1")
+
+# ===========================
+# CONSTANTS
+EVEREST_NAMESPACE = 'everest-system'
+EVEREST_MONITORING_NAMESPACE = 'everest-monitoring'
+UI_ADMIN_PASSWORD = 'admin'
+
+# Load Config
 config_yaml=read_yaml('config.yaml')
 
-# Vars
-everest_namespace = 'everest-system'
-everest_monitoring_namespace = 'everest-monitoring'
+# Load local Envfile with env vars
+load('ext://dotenv', 'dotenv')
+
+local_env_file='{0}/.env'.format(config.main_dir)
+if os.path.exists(local_env_file):
+    print ('Loading local .env file={0}'.format(local_env_file))
+    dotenv(fn=local_env_file)
+
+# Variables
 pxc_operator_version = os.getenv('PXC_OPERATOR_VERSION', '1.16.1')
-print('Using PXC operator version: %s' % pxc_operator_version)
-psmdb_operator_version = os.getenv('PSMDB_OPERATOR_VERSION', '1.19.0')
-print('Using PSMDB operator version: %s' % psmdb_operator_version)
+print('Using PXC operator version: {0}'.format(pxc_operator_version))
+psmdb_operator_version = os.getenv('PSMDB_OPERATOR_VERSION', '1.19.1')
+print('Using PSMDB operator version: {0}'.format(psmdb_operator_version))
 pg_operator_version = os.getenv('PG_OPERATOR_VERSION', '2.5.0')
-print('Using PG operator version: %s' % pg_operator_version)
+print('Using PG operator version: {0}'.format(pg_operator_version))
 
 # External resources set up
 # uncomment the settings below and insert your k8s context & registry names
 # to get your current context name run `kubectl config view`
 #allow_k8s_contexts("gke_percona-everest_europe-west1-c_everest-dev")
 #default_registry("us-central1-docker.pkg.dev/percona-everest/quickstart-docker-repo")
 
-
 # Check for required env vars
-backend_dir = os.getenv('EVEREST_BACKEND_DIR')
-if not backend_dir:
-  backend_dir = '..'
-backend_dir = os.path.abspath(backend_dir)
+backend_dir = os.path.abspath('{0}/../'.format(config.main_dir))
 if not os.path.exists(backend_dir):
-  fail('Backend dir does not exist: %s' % backend_dir)
-print('Using backend dir: %s' % backend_dir)
+  fail('Backend dir does not exist: {0}'.format(backend_dir))
+print('Using backend dir: {0}'.format(backend_dir))
 
-frontend_dir = os.getenv('EVEREST_FRONTEND_DIR')
-if not frontend_dir:
-  frontend_dir = '../ui'
-frontend_dir = os.path.abspath(frontend_dir)
+frontend_dir = os.path.abspath('{0}/../ui/'.format(config.main_dir))
 if not os.path.exists(frontend_dir):
-  fail('Frontend dir does not exist: %s' % frontend_dir)
-print('Using frontend dir: %s' % frontend_dir)
-
-cli_dir = os.getenv('EVEREST_CLI_DIR')
-if not cli_dir:
-  cli_dir = '..'
-cli_dir = os.path.abspath(cli_dir)
-if not os.path.exists(cli_dir):
-  fail('cli dir does not exist: %s' % cli_dir)
-print('Using cli dir: %s' % cli_dir)
+  fail('Frontend dir does not exist: {0}'.format(frontend_dir))
+print('Using frontend dir: {0}'.format(frontend_dir))
 
 operator_dir = os.getenv('EVEREST_OPERATOR_DIR')
 if not operator_dir:
   fail('EVEREST_OPERATOR_DIR must be set')
 operator_dir = os.path.abspath(operator_dir)
 if not os.path.exists(operator_dir):
-  fail('Operator dir does not exist: %s' % operator_dir)
-print('Using operator dir: %s' % operator_dir)
+  fail('Operator dir does not exist: {0}'.format(operator_dir))
+print('Using operator dir: {0}'.format(operator_dir))
 
 everest_chart_dir = os.getenv('EVEREST_CHART_DIR')
 if not everest_chart_dir:
   fail('EVEREST_CHART_DIR must be set')
 everest_chart_dir = os.path.abspath(everest_chart_dir)
 if not os.path.exists(everest_chart_dir):
-  fail('Chart dir does not exist: %s' % everest_chart_dir)
-print('Using chart dir: %s' % everest_chart_dir)
+  fail('Chart dir does not exist: {0}'.format(everest_chart_dir))
+print('Using chart dir: {0}'.format(everest_chart_dir))
 
 # Check for Everest debug
-everest_debug = bool(os.getenv('EVEREST_DEBUG'))
-if not everest_debug:
-    everest_debug = False
-print('Using Everest debug: %s' % everest_debug)
+everest_debug = str2bool(os.getenv('EVEREST_DEBUG', 'False'))
+print('Using Everest debug: {0}'.format(everest_debug))
 
 # Check for Everest Operator debug
-everest_operator_debug = bool(os.getenv('EVEREST_OPERATOR_DEBUG'))
-if not everest_operator_debug:
-    everest_operator_debug = False
-print('Using Everest Operator debug: %s' % everest_operator_debug)
+everest_operator_debug = str2bool(os.getenv('EVEREST_OPERATOR_DEBUG', 'False'))
+print('Using Everest Operator debug: {0}'.format(everest_operator_debug))
+
+# Check whether we need to create backup storage
+need_backup_storage=False
+for namespace in config_yaml['namespaces']:
+    if 'backupStorages' in namespace and len(namespace['backupStorages']) > 0:
+        need_backup_storage=True
+        break
+
+print('Need backup storage: {0}'.format(need_backup_storage))
 
 watch_settings(
     ignore=[
@@ -92,10 +98,10 @@ local('HELM=%s/bin/helm make -C %s deps' % (backend_dir, everest_chart_dir), qui
 
 # Create namespaces
 load('ext://namespace', 'namespace_create', 'namespace_inject')
-namespace_create(everest_namespace)
+namespace_create(EVEREST_NAMESPACE)
 k8s_resource(
   objects=[
-    '%s:namespace' % everest_namespace
+    '{0}:namespace'.format(EVEREST_NAMESPACE)
   ],
   new_name='everest-namespace',
 )
@@ -259,7 +265,7 @@ k8s_yaml(everest_crds)
 everest_yaml = helm(
     everest_chart_dir,
     name='everest',
-    namespace=everest_namespace,
+    namespace=EVEREST_NAMESPACE,
     skip_crds=True,
     set=[
         'dbNamespace.enabled=false',
@@ -268,8 +274,8 @@ everest_yaml = helm(
         'createMonitoringResources=false',
         'monitoring.crds.enabled=true',
         'monitoring.crds.plain=false',
-        'monitoring.namespaceOverride=%s' % everest_monitoring_namespace,
-        'server.initialAdminPassword=admin',
+        'monitoring.namespaceOverride={0}'.format(EVEREST_MONITORING_NAMESPACE),
+        'server.initialAdminPassword={0}'.format(UI_ADMIN_PASSWORD),
     ],
 )
 # we don't need the catalog source
@@ -325,14 +331,14 @@ k8s_yaml(everest_yaml)
 
 k8s_resource(
   objects=[
-    '%s:namespace' % everest_monitoring_namespace
+    '{0}:namespace'.format(EVEREST_MONITORING_NAMESPACE)
   ],
   new_name='everest-monitoring-namespace',
 )
 k8s_resource(
     workload='kube-state-metrics',
     objects=[
-        'kube-state-metrics:serviceaccount:%s' % everest_monitoring_namespace,
+        'kube-state-metrics:serviceaccount:{0}'.format(EVEREST_MONITORING_NAMESPACE),
         'kube-state-metrics:clusterrole',
         'kube-state-metrics:clusterrolebinding',
         'customresource-config-ksm:configmap'
@@ -455,9 +461,9 @@ k8s_resource(
 k8s_resource(
     workload='everest-operator',
     objects=[
-    'everest-operator-controller-manager:serviceaccount:%s' % everest_namespace,
-    'everest-operator-leader-election-role:role:%s' % everest_namespace,
-    'everest-operator-local:rolebinding:%s' % everest_namespace,
+    'everest-operator-controller-manager:serviceaccount:{0}'.format(EVEREST_NAMESPACE),
+    'everest-operator-leader-election-role:role:{0}'.format(EVEREST_NAMESPACE),
+    'everest-operator-local:rolebinding:{0}'.format(EVEREST_NAMESPACE),
     'everest-operator-metrics-reader:clusterrole',
     'everest-operator-metrics-auth-role:clusterrole',
     'everest-operator-metrics-auth-rolebinding:clusterrolebinding',
@@ -552,14 +558,14 @@ docker_build_with_restart('perconalab/everest',
 k8s_resource(
     workload='everest-server',
     objects=[
-    'everest-accounts:secret:%s' % everest_namespace,
-    'everest-jwt:secret:%s' % everest_namespace,
-    'everest-rbac:configmap:%s' % everest_namespace,
-    'everest-settings:configmap:%s' % everest_namespace,
-    'everest-admin-role:role:%s' % everest_namespace,
+    'everest-accounts:secret:{0}'.format(EVEREST_NAMESPACE),
+    'everest-jwt:secret:{0}'.format(EVEREST_NAMESPACE),
+    'everest-rbac:configmap:{0}'.format(EVEREST_NAMESPACE),
+    'everest-settings:configmap:{0}'.format(EVEREST_NAMESPACE),
+    'everest-admin-role:role:{0}'.format(EVEREST_NAMESPACE),
     'everest-admin-cluster-role:clusterrole',
     'everest-admin-cluster-role-binding:clusterrolebinding',
-    'everest-admin:serviceaccount:%s' % everest_namespace,
+    'everest-admin:serviceaccount:{0}'.format(EVEREST_NAMESPACE),
     'everest-admin-role-binding:rolebinding'
     ],
     resource_deps = [
@@ -569,3 +575,74 @@ k8s_resource(
     new_name='everest-server-deploy',
     labels=["everest"]
 )
+
+#################################
+############ Backup Storages ##########
+#################################
+def getBackupStorageNames():
+    to_return = []
+    for ns in config_yaml['namespaces']:
+        to_return.extend([storage_name for storage_name in ns.get('backupStorages', [])])
+    return to_return
+
+if need_backup_storage:
+    # Deploy minIO as backup storage
+    print('Deploying minIO as backup storage')
+    minio_path = os.path.abspath('{0}/.github/minio.conf.yaml'.format(backend_dir))
+    minio_objs=read_yaml_stream(minio_path)
+    bsListStr=','.join(getBackupStorageNames())
+    for minio_obj in minio_objs:
+        if minio_obj['kind'] == 'Deployment' and minio_obj['metadata']['name'] == 'minio':
+            # Redefine the env variable with the list of bucket names
+            minio_obj['spec']['template']['spec']['initContainers'][0]['env'][0]['value']=bsListStr
+            break
+    k8s_yaml(encode_yaml_stream(minio_objs))
+    k8s_resource(
+        workload='minio',
+        objects=[
+            'minio:Namespace',
+            'minio-data:PersistentVolumeClaim:minio',
+            'minio-cert:Secret:minio'
+        ],
+        new_name='minIO-deploy',
+        port_forwards=[
+            port_forward(9090, 9090, 'MinIO Console'),
+        ],
+        labels=["backup-storage"]
+    )
+
+    # Create BackupStorage CRs
+    bs_crd_path = os.path.abspath('{0}/resources/backupstorage.yaml'.format(config.main_dir))
+    for namespace in config_yaml['namespaces']:
+        ns_name = namespace['name']
+        for storage_name in namespace.get('backupStorages', []):
+            bs_objs=read_yaml_stream(bs_crd_path)
+            for obj in bs_objs:
+                if obj['kind'] == 'Secret':
+                    obj.update({
+                        'metadata': {
+                            'namespace': ns_name,
+                            'name': storage_name
+                        }
+                    })
+                elif obj['kind'] == 'BackupStorage':
+                    obj.update({
+                        'metadata': {
+                            'namespace': ns_name,
+                            'name': storage_name
+                        }
+                    })
+                    obj['spec'].update({
+                            'bucket': '{0}'.format(storage_name),
+                            'description': storage_name,
+                            'credentialsSecretName': storage_name,
+                    })
+            k8s_yaml(encode_yaml_stream(bs_objs))
+            k8s_resource(
+                objects=[
+                    '{0}:Secret:{1}'.format(storage_name, ns_name),
+                    '{0}:BackupStorage:{1}'.format(storage_name, ns_name),
+                ],
+                new_name='bs-{0}-{1}'.format(ns_name, storage_name),
+                labels=["backup-storage"]
+            )