Add plugins.query.datasources.encryption.masterkey (#198)

Signed-off-by: Vamsi Manohar <reddyvam@amazon.com>

Author: vamsimanohar

.github/workflows/deployment-template.yml

@@ -38,6 +38,8 @@ on:
         required: false
       kibanaserver:
         required: true
+      query_datasources_master_key:
+        required: false 
 jobs:
 
   OS-OSD-Deployment:
@@ -61,7 +63,17 @@ jobs:
           GA_TRACKING_ID: ${{ secrets.ga-tracking-id }}
           KIBANASERVER: ${{ secrets.kibanaserver }}
 
-      - name: Step 2 - Replace Tokens for dashboards
+      - name: Step 2 - Replace Token in opensearch.yml
+        if: ${{ (inputs.deploy-env == 'observability') || (inputs.deploy-env == 'dev') }}
+        uses: cschleiden/replace-tokens@v1
+        with:
+          files: '["${{ github.workspace }}/config/playground/helm/${{inputs.deploy-env}}/helm-opensearch.yaml"]'
+          tokenPrefix: '${'
+          tokenSuffix: '}'
+        env:
+          QUERY_DATASOURCES_MASTER_KEY: ${{ secrets.query_datasources_master_key }}
+
+      - name: Step 3 - Replace Tokens for dashboards
         uses: cschleiden/replace-tokens@v1
         with:
           files: '["${{ github.workspace }}/config/playground/helm/${{inputs.deploy-env}}/helm-opensearch-dashboards.yaml"]'
@@ -71,7 +83,7 @@ jobs:
           KIBANASERVER: ${{ secrets.kibanaserver }}
           GA_TRACKING_ID: ${{ secrets.ga-tracking-id }}
 
-      - name: Step 3 - Replace Tokens for logstash configuration
+      - name: Step 4 - Replace Tokens for logstash configuration
         if: ${{ (inputs.deploy-env == 'preview') || (inputs.deploy-env == 'searchapps') || (inputs.deploy-env == 'observability') }}
         uses: cschleiden/replace-tokens@v1
         with:
@@ -82,7 +94,7 @@ jobs:
           OSD_USER: ${{ secrets.osd_user }}
           OSD_USER_PASSWORD: ${{ secrets.osd_user_password }}
 
-      - name: Step 4 - Replace Tokens for tracing configuration
+      - name: Step 5 - Replace Tokens for tracing configuration
         if: ${{ (inputs.deploy-env == 'preview') || (inputs.deploy-env == 'searchapps') || (inputs.deploy-env == 'observability') }}
         uses: cschleiden/replace-tokens@v1
         with:
@@ -95,14 +107,14 @@ jobs:
           OTEL_ROOT_CA: ${{ secrets.otel_root_ca }}
           OTEL_DATA_PREPPER_CERT: ${{ secrets.otel_data_prepper_cert }}
 
-      - name: Step 5 - Configure AWS Credentials
+      - name: Step 6 - Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v2
         with:
           aws-access-key-id: ${{ secrets.access-key-id }}
           aws-secret-access-key: ${{ secrets.secret-access-key }}
           aws-region: ${{ secrets.region }}
 
-      - name: Step 6 - Delete Logstash
+      - name: Step 7 - Delete Logstash
         if: ${{ (inputs.deploy-env == 'preview') || (inputs.deploy-env == 'searchapps') || (inputs.deploy-env == 'observability') }}
         uses: elastic-analytics/dashboards-action@main
         env:
@@ -112,7 +124,7 @@ jobs:
           command: |
             kubectl delete ns logstash
 
-      - name: Step 7 - Delete Tracing
+      - name: Step 8 - Delete Tracing
         if: ${{ (inputs.deploy-env == 'preview') || (inputs.deploy-env == 'searchapps') || (inputs.deploy-env == 'observability') }}
         uses: elastic-analytics/dashboards-action@main
         env:
@@ -125,7 +137,7 @@ jobs:
             kubectl delete -f config/playground/metrics/tracing/preview/otel-collector.yaml
             kubectl delete -f config/playground/metrics/tracing/preview/data-prepper.yaml
 
-      - name: Step 8 - Deploy OpenSearch and OpenSearch Dashboards By Helm Chart
+      - name: Step 9 - Deploy OpenSearch and OpenSearch Dashboards By Helm Chart
         if: ${{ (inputs.deploy-env == 'searchapps') || (inputs.deploy-env == 'preview') || (inputs.deploy-env == 'observability')}}
         uses: elastic-analytics/dashboards-action@main
         env:
@@ -146,7 +158,7 @@ jobs:
             helm install opensearch opensearch/opensearch -f config/playground/helm/${{inputs.deploy-env}}/helm-opensearch.yaml
             helm install dashboards opensearch/opensearch-dashboards -f config/playground/helm/${{inputs.deploy-env}}/helm-opensearch-dashboards.yaml
 
-      - name: Step 9 - Deploy OpenSearch/OpenSearch Dashboards/ML By Helm Chart
+      - name: Step 10 - Deploy OpenSearch/OpenSearch Dashboards/ML By Helm Chart
         if: ${{ (inputs.deploy-env == 'dev') || (inputs.deploy-env == 'prod') || (inputs.deploy-env == 'ml') }}
         uses: elastic-analytics/dashboards-action@main
         env:
@@ -170,7 +182,7 @@ jobs:
             helm install dashboards opensearch/opensearch-dashboards -f config/playground/helm/${{inputs.deploy-env}}/helm-opensearch-dashboards.yaml
             helm install machinelearning opensearch/opensearch -f config/playground/helm/${{inputs.deploy-env}}/helm-machinelearning.yaml
 
-      - name: Step 10 - Install Logstash
+      - name: Step 11 - Install Logstash
         if: ${{ (inputs.deploy-env == 'preview') || (inputs.deploy-env == 'searchapps') || (inputs.deploy-env == 'observability') }}
         uses: elastic-analytics/dashboards-action@main
         env:
@@ -182,7 +194,7 @@ jobs:
             kubectl create -f config/playground/metrics/logstash/preview/logstash-configmap.yaml
             kubectl apply -f config/playground/metrics/logstash/preview/logstash.yaml
 
-      - name: Step 11 - Install Tracing
+      - name: Step 12 - Install Tracing
         if: ${{ (inputs.deploy-env == 'preview') || (inputs.deploy-env == 'searchapps') || (inputs.deploy-env == 'observability') }}
         uses: elastic-analytics/dashboards-action@main
         env:

.github/workflows/os-osd-deployment-scheduled.yml

@@ -74,6 +74,7 @@ jobs:
       otel_root_ca: ${{ secrets.OTEL_ROOT_CA }}
       otel_data_prepper_cert: ${{ secrets.OTEL_DATA_PREPPER_CERT }}
       kibanaserver: ${{ secrets.KIBANASERVER }}
+      query_datasources_master_key: ${{ secrets.QUERY_DATASOURCES_MASTER_KEY }}
 
   OSD-Functional-Test-Preview:
     needs: OS-OSD-Preview-Scheduled-Deployment

.github/workflows/os-osd-deployment.yml

@@ -46,6 +46,7 @@ jobs:
       kube-config: ${{ secrets.KUBE_CONFIG_DATA_DEV }}
       ga-tracking-id: ${{ secrets.GA_TRACKING_ID }}
       kibanaserver: ${{ secrets.KIBANASERVER }}
+      query_datasources_master_key: ${{ secrets.QUERY_DATASOURCES_MASTER_KEY }}
 
   OS-OSD-Prod-Deployment:
     needs: Pre-Deployment

config/playground/helm/dev/helm-opensearch.yaml

@@ -30,6 +30,7 @@ opensearchHome: /usr/share/opensearch
 config:
   opensearch.yml: |
     cluster.name: opensearch-cluster
+    plugins.query.datasources.encryption.masterkey: ${QUERY_DATASOURCES_MASTER_KEY}
     # Bind to all interfaces because we don't know what IP address Docker will assign to us.
     network.host: 0.0.0.0
     # # minimum_master_nodes need to be explicitly set when bound on a public IP

config/playground/helm/observability/helm-opensearch.yaml

@@ -30,6 +30,7 @@ opensearchHome: /usr/share/opensearch
 config:
   opensearch.yml: |
     cluster.name: opensearch-cluster
+    plugins.query.datasources.encryption.masterkey: ${QUERY_DATASOURCES_MASTER_KEY}
     # Bind to all interfaces because we don't know what IP address Docker will assign to us.
     network.host: 0.0.0.0
     # # minimum_master_nodes need to be explicitly set when bound on a public IP