Merge remote-tracking branch 'upstream/main' into smoke-test-workflow

Author: peterzhuamazon

.github/workflows/get-ci-image-tag.yml

@@ -25,25 +25,33 @@ on:
       ci-image-version-linux:
         description: The ci image version for linux build
         value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-version-linux }}
+      ci-image-start-options:
+        description: The ci image start options to set when starting the container
+        value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-start-options }}
+      ci-image-start-command:
+        description: The ci image start commands to run after initialization
+        value: ${{ jobs.Get-CI-Image-Tag.outputs.output-ci-image-start-command }}
 
 jobs:
   Get-CI-Image-Tag:
     runs-on: ubuntu-latest
     outputs:
-      output-ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }}
+      output-ci-image-version-linux: ${{ steps.step-ci-image-setups.outputs.ci-image-version-linux }}
+      output-ci-image-start-command: ${{ steps.step-ci-image-setups.outputs.ci-image-start-command }}
+      output-ci-image-start-options: ${{ steps.step-ci-image-setups.outputs.ci-image-start-options }}
     steps:
       - name: Install crane
         uses: iarekylew00t/crane-installer@v1
         with:
           crane-release: v0.15.2
       - name: Checkout opensearch-build repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: 'opensearch-project/opensearch-build'
           ref: ${{ inputs.build_ref }}
           path: 'opensearch-build'
       - name: Get ci image version from opensearch-build repository scripts
-        id: step-ci-image-version-linux
+        id: step-ci-image-setups
         run: |
           PRODUCT=${{ inputs.product }}
           PLATFORM=${{ inputs.platform }}
@@ -55,6 +63,18 @@ jobs:
                   PLATFORM="almalinux8"
               fi
           fi
+
+          if [[ "$PLATFORM" = "al2" ]]; then
+              CI_IMAGE_CMD="cp -a /node_al2/* /node && /node/bin/node -v"
+              echo "ci-image-start-command=$CI_IMAGE_CMD" >> $GITHUB_OUTPUT
+              CI_IMAGE_OPTIONS="--user root -v /node:/node:rw,rshared -v /node:/__e/node20:ro,rshared"
+              echo "ci-image-start-options=$CI_IMAGE_OPTIONS" >> $GITHUB_OUTPUT
+          else
+              CI_IMAGE_CMD="echo pass"
+              echo "ci-image-start-command=$CI_IMAGE_CMD" >> $GITHUB_OUTPUT
+              CI_IMAGE_OPTIONS="--user root"
+              echo "ci-image-start-options=$CI_IMAGE_OPTIONS" >> $GITHUB_OUTPUT
+          fi
           crane version
           echo $PRODUCT $PLATFORM
           CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p $PLATFORM -u $PRODUCT -t $TYPE | head -1`

config/opensearch_dashboards-2.x.yml

@@ -201,7 +201,7 @@
 # Set the value of this setting to true to enable the ml commons dashboards
 # ml_commons_dashboards.enabled: false
 
-# 2.12 New experimental Assistant Dashboards Feature
+# 2.12 New Experimental Assistant Dashboards Feature
 # Set the value of this setting to true to enable the assistant dashboards
 # assistant.chat.enabled: false
 
@@ -213,6 +213,13 @@
 # Set the value of this setting to true to enable UI Metric collections
 # usageCollection.uiMetric.enabled: false
 
+# 2.18 New Experimental Settings
+# Set the value to true to enable
+# assistant.alertInsight.enabled: false
+# assistant.smartAnomalyDetector.enabled: false
+# assistant.text2viz.enabled: false
+# queryEnhancements.queryAssist.summary.enabled: false
+
 opensearch.hosts: [https://localhost:9200]
 opensearch.ssl.verificationMode: none
 opensearch.username: kibanaserver

config/opensearch_dashboards-default.x.yml

@@ -201,7 +201,7 @@
 # Set the value of this setting to true to enable the ml commons dashboards
 # ml_commons_dashboards.enabled: false
 
-# 2.12 New experimental Assistant Dashboards Feature
+# 2.12 New Experimental Assistant Dashboards Feature
 # Set the value of this setting to true to enable the assistant dashboards
 # assistant.chat.enabled: false
 
@@ -213,6 +213,13 @@
 # Set the value of this setting to true to enable UI Metric collections
 # usageCollection.uiMetric.enabled: false
 
+# 2.18 New Experimental Settings
+# Set the value to true to enable
+# assistant.alertInsight.enabled: false
+# assistant.smartAnomalyDetector.enabled: false
+# assistant.text2viz.enabled: false
+# queryEnhancements.queryAssist.summary.enabled: false
+
 opensearch.hosts: [https://localhost:9200]
 opensearch.ssl.verificationMode: none
 opensearch.username: kibanaserver

docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile

@@ -76,11 +76,21 @@ ENV GEM_HOME=$CONTAINER_USER_HOME/.gem
 ENV GEM_PATH=$GEM_HOME
 ENV PATH=$RUBY_HOME:$RVM_HOME:$PATH
 
+# Installing openssl1.1.1
+# Support requests >= 2.28.1 version
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib
+RUN yum install -y curl libcurl-devel libfaketime perl-core pcre-devel && yum remove -y openssl-devel && yum clean all && \
+    mkdir -p /tmp/openssl && cd /tmp/openssl && \
+    curl -sSL -o- https://www.openssl.org/source/openssl-1.1.1g.tar.gz | tar -xz --strip-components 1 && \
+    ./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib shared zlib-dynamic && make && make install && \
+    echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib" > /etc/profile.d/openssl.sh && openssl version
+
 # Install Python binary
 RUN curl https://www.python.org/ftp/python/3.9.7/Python-3.9.7.tgz | tar xzvf - && \
     cd Python-3.9.7 && \
-    ./configure --enable-optimizations && \
-    make altinstall
+    env LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib ./configure --enable-optimizations --with-openssl=/usr --prefix=/usr/local && \
+    make altinstall && cd ../ && rm -rf Python-3.9.7.tgz Python-3.9.7 && \
+    cp -v /etc/ssl/certs/ca-bundle.crt /etc/ssl/cert.pem
 
 # Setup Python links
 RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \
@@ -114,6 +124,21 @@ RUN git clone -b v0.3.27 --single-branch https://github.com/xianyi/OpenBLAS.git
 ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"
 RUN pip3 install cmake==3.26.4
 
+# NodeJS Unofficial Builds
+# https://github.com/opensearch-project/opensearch-build/issues/5178
+# https://github.com/actions/runner/issues/2906
+# https://github.com/actions/runner/issues/3475
+# GitHub enforce nodejs 20 official build in runner 2.317.0 of their actions and CentOS7/AL2 would fail due to having older glibc versions
+# Until https://github.com/actions/runner/pull/3128 is merged or AL2 is deprecated (2025/06) this is a quick fix with unofficial builds support glibc 2.17
+# With changes done similar to this PR (https://github.com/opensearch-project/job-scheduler/pull/702) alongside the image here
+# Only linux x64 is supported in unofficial build until https://github.com/nodejs/unofficial-builds/pull/91 is merged
+RUN if [ `uname -m` = "x86_64" ]; then \
+        curl -SL https://unofficial-builds.nodejs.org/download/release/v20.10.0/node-v20.10.0-linux-x64-glibc-217.tar.xz -o /node20.tar.xz; \
+        mkdir /node_al2; \
+        tar -xf /node20.tar.xz --strip-components 1 -C /node_al2; \
+        rm -v /node20.tar.xz; \
+    fi
+
 # Change User
 USER $CONTAINER_USER
 WORKDIR $CONTAINER_USER_HOME

docker/release/config/opensearch-dashboards/opensearch-dashboards-docker-entrypoint-2.x.sh

@@ -167,10 +167,14 @@ opensearch_dashboards_vars=(
     data_source.audit.appender.layout.highlight
     data_source.audit.appender.layout.pattern
     ml_commons_dashboards.enabled
-    assistant.chat.enabled
     observability.query_assist.enabled
     usageCollection.uiMetric.enabled
     workspace.enabled
+    assistant.chat.enabled
+    assistant.alertInsight.enabled
+    assistant.smartAnomalyDetector.enabled
+    assistant.text2viz.enabled
+    queryEnhancements.queryAssist.summary.enabled
 )
 
 function setupSecurityDashboardsPlugin {

docker/release/config/opensearch-dashboards/opensearch-dashboards-docker-entrypoint-default.x.sh

@@ -167,10 +167,14 @@ opensearch_dashboards_vars=(
     data_source.audit.appender.layout.highlight
     data_source.audit.appender.layout.pattern
     ml_commons_dashboards.enabled
-    assistant.chat.enabled
     observability.query_assist.enabled
     usageCollection.uiMetric.enabled
     workspace.enabled
+    assistant.chat.enabled
+    assistant.alertInsight.enabled
+    assistant.smartAnomalyDetector.enabled
+    assistant.text2viz.enabled
+    queryEnhancements.queryAssist.summary.enabled
 )
 
 function setupSecurityDashboardsPlugin {

jenkins/check-for-build.jenkinsfile

@@ -23,6 +23,8 @@ pipeline {
     }
     triggers {
         parameterizedCron '''
+            H 1 * * * %INPUT_MANIFEST=2.18.1/opensearch-2.18.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.18.1/opensearch-2.18.1-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar
+            H 1 * * * %INPUT_MANIFEST=1.3.20/opensearch-dashboards-1.3.20.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=1.3.20/opensearch-dashboards-1.3.20-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar
             H 1 * * * %INPUT_MANIFEST=2.19.0/opensearch-2.19.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.19.0/opensearch-2.19.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar
             H 4 * * * %INPUT_MANIFEST=1.3.20/opensearch-1.3.20.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=1.3.20/opensearch-1.3.20-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar
             H 4 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip

jenkins/docker/docker-scan.jenkinsfile

@@ -17,6 +17,10 @@ pipeline {
         timeout(time: 30)
     }
     agent none  
+    environment {
+        TRIVY_DB_REPOSITORY = 'public.ecr.aws/aquasecurity/trivy-db'
+        TRIVY_JAVA_DB_REPOSITORY = 'public.ecr.aws/aquasecurity/trivy-java-db'
+    }
     parameters {
         string(
             name: 'IMAGE_FULL_NAME', 

jenkins/gradle/gradle-check.jenkinsfile

@@ -7,7 +7,7 @@
  * compatible open source license.
  */
 
-lib = library(identifier: 'jenkins@7.2.0', retriever: modernSCM([
+lib = library(identifier: 'jenkins@7.3.2', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))

jenkins/integ-test-notification.jenkinsfile

@@ -28,6 +28,8 @@ pipeline {
     }
     triggers {
         parameterizedCron('''
+            H */6 * * * %INPUT_MANIFEST=2.18.1/opensearch-2.18.1.yml
+            H */6 * * * %INPUT_MANIFEST=1.3.20/opensearch-dashboards-1.3.20.yml
             H */6 * * * %INPUT_MANIFEST=2.19.0/opensearch-2.19.0.yml
             H */6 * * * %INPUT_MANIFEST=2.18.0/opensearch-2.18.0.yml
             H */6 * * * %INPUT_MANIFEST=2.18.0/opensearch-dashboards-2.18.0.yml