Support new OS core systemd and integTests in 3.0.0 and manifest updates (#5345)

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

Author: peterzhuamazon

docker/ci/dockerfiles/current/test.almalinux8.systemd-base.x64.arm64.ppc64le.dockerfile

@@ -112,7 +112,7 @@ RUN dnf install -y sudo && \
     usermod -a -G opensearch-dashboards $CONTAINER_USER && \
     usermod -a -G adm $CONTAINER_USER && \
     id && \
-    echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which usermod`, `which dnf`, `which yum`, `which rpm`, `which chmod`, `which kill`, `which curl`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER
+    echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which su`, `which usermod`, `which dnf`, `which yum`, `which rpm`, `which chmod`, `which kill`, `which curl`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER
 
 # Copy from Stage0
 COPY --from=linux_stage_0 --chown=$CONTAINER_USER:$CONTAINER_USER $CONTAINER_USER_HOME $CONTAINER_USER_HOME

docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.ppc64le.dockerfile

@@ -130,7 +130,7 @@ RUN apt-get install -y sudo && \
     usermod -a -G opensearch-dashboards $CONTAINER_USER && \
     usermod -a -G adm $CONTAINER_USER && \
     id && \
-    echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which usermod`, `which apt`, `which apt-get`, `which apt-key`, `which dpkg`, `which chmod`, `which kill`, `which curl`, `which tee`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER
+    echo "$CONTAINER_USER ALL=(root) NOPASSWD:`which systemctl`, `which env`, `which su`, `which usermod`, `which apt`, `which apt-get`, `which apt-key`, `which dpkg`, `which chmod`, `which kill`, `which curl`, `which tee`, `which rm`, /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin" >> /etc/sudoers.d/$CONTAINER_USER
 
 # Copy from Stage0
 COPY --from=linux_stage_0 --chown=$CONTAINER_USER:$CONTAINER_USER $CONTAINER_USER_HOME $CONTAINER_USER_HOME

jenkins/check-for-build.jenkinsfile

@@ -27,8 +27,8 @@ pipeline {
             H 1 * * * %INPUT_MANIFEST=2.20.0/opensearch-2.20.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 3 * * * %INPUT_MANIFEST=3.0.0-alpha1/opensearch-3.0.0-alpha1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 3 * * * %INPUT_MANIFEST=3.0.0-alpha1/opensearch-dashboards-3.0.0-alpha1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
-            H 6 * * * %INPUT_MANIFEST=3.0.0-beta1/opensearch-3.0.0-beta1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
-            H 6 * * * %INPUT_MANIFEST=3.0.0-beta1/opensearch-dashboards-3.0.0-beta1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
+            H 6 * * * %INPUT_MANIFEST=3.0.0-beta1/opensearch-3.0.0-beta1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;UPDATE_GITHUB_ISSUE=false
+            H 6 * * * %INPUT_MANIFEST=3.0.0-beta1/opensearch-dashboards-3.0.0-beta1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;UPDATE_GITHUB_ISSUE=false
         '''
     }
     parameters {

manifests/2.19.1/opensearch-2.19.1-test.yml

@@ -6,7 +6,7 @@ ci:
     name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1
     args: -e JAVA_HOME=/opt/java/openjdk-21
 components:
-  - name: opensearch
+  - name: OpenSearch
     smoke-test:
       test-spec: opensearch.yml
   - name: alerting

manifests/2.20.0/opensearch-2.20.0-test.yml

@@ -6,7 +6,7 @@ ci:
     name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1
     args: -e JAVA_HOME=/opt/java/openjdk-21
 components:
-  - name: opensearch
+  - name: OpenSearch
     smoke-test:
       test-spec: opensearch.yml
   - name: alerting

manifests/3.0.0-alpha1/opensearch-3.0.0-alpha1-test.yml

@@ -6,7 +6,11 @@ ci:
     name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1
     args: -e JAVA_HOME=/opt/java/openjdk-23
 components:
-  - name: opensearch
+  - name: OpenSearch
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
     smoke-test:
       test-spec: opensearch.yml
   - name: alerting

manifests/3.0.0-beta1/opensearch-3.0.0-beta1-test.yml

@@ -6,7 +6,11 @@ ci:
     name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1
     args: -e JAVA_HOME=/opt/java/openjdk-23
 components:
-  - name: opensearch
+  - name: OpenSearch
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
     smoke-test:
       test-spec: opensearch.yml
   - name: alerting

manifests/3.0.0-beta1/opensearch-dashboards-3.0.0-beta1.yml

@@ -10,7 +10,7 @@ ci:
 components:
   - name: OpenSearch-Dashboards
     repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
-    ref: 'node-2x'
+    ref: 'feature/node-2x'
   - name: functionalTestDashboards
     repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git
     ref: main

scripts/components/OpenSearch/install.sh

@@ -68,6 +68,11 @@ if [ -z "$VERSION" ]; then
     exit 1
 fi
 
+if ! command -v yq > /dev/null; then
+    echo "Error: yq not found, please install v4 version of yq"
+    exit 1
+fi
+
 [ -z "$SNAPSHOT" ] && SNAPSHOT="false"
 [ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}')
 [ -z "$ARCHITECTURE" ] && ARCHITECTURE=`uname -m`
@@ -82,12 +87,23 @@ cd $DIR
 MAJOR_VERSION=`echo $VERSION | cut -d. -f1`
 if [ "$DISTRIBUTION" = "tar" ]; then
     cp -v ../../../scripts/startup/tar/linux/opensearch-tar-install.sh "$OUTPUT/"
-elif [ "$DISTRIBUTION" = "deb" -o "$DISTRIBUTION" = "rpm" ]; then
+elif [ "$DISTRIBUTION" = "deb" ] || [ "$DISTRIBUTION" = "rpm" ]; then
     cp -va ../../../scripts/pkg/service_templates/opensearch/* "$OUTPUT/../"
     if [ "$MAJOR_VERSION" = "1" ]; then
         cp -va ../../../scripts/pkg/build_templates/legacy/opensearch/$DISTRIBUTION/* "$OUTPUT/../"
+    elif [ "$MAJOR_VERSION" = "2" ]; then
+        cp -va ../../../scripts/pkg/build_templates/current/opensearch/$DISTRIBUTION/* "$OUTPUT/../"
     else
         cp -va ../../../scripts/pkg/build_templates/current/opensearch/$DISTRIBUTION/* "$OUTPUT/../"
+        OS_REF=`yq -e '.components[] | select(.name == "OpenSearch") | .ref' ../../../manifests/$VERSION/opensearch-$VERSION.yml`
+        curl -SfL "https://raw.githubusercontent.com/opensearch-project/OpenSearch/$OS_REF/distribution/packages/src/common/env/opensearch" -o "$OUTPUT/../etc/sysconfig/opensearch" || { echo "Failed to download env file"; exit 1; }
+        curl -SfL "https://raw.githubusercontent.com/opensearch-project/OpenSearch/$OS_REF/distribution/packages/src/common/systemd/opensearch.service" -o "$OUTPUT/../usr/lib/systemd/system/opensearch.service" || { echo "Failed to download env file"; exit 1; }
+        # k-NN lib setups
+        echo -e "\n\n################################" >> "$OUTPUT/../etc/sysconfig/opensearch"
+        echo -e "# Plugin properties" >> "$OUTPUT/../etc/sysconfig/opensearch"
+        echo -e "################################" >> "$OUTPUT/../etc/sysconfig/opensearch"
+        echo -e "\n# k-NN Lib Path" >> "$OUTPUT/../etc/sysconfig/opensearch"
+        echo "LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/share/opensearch/plugins/opensearch-knn/lib" >> "$OUTPUT/../etc/sysconfig/opensearch"
     fi
 elif [ "$DISTRIBUTION" = "zip" ] && [ "$PLATFORM" = "windows" ]; then
     cp -v ../../../scripts/startup/zip/windows/opensearch-windows-install.bat "$OUTPUT/"

scripts/components/OpenSearch/integtest.sh

@@ -7,4 +7,5 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
-# TODO: This is used by ScriptFinder tests, needs a working integtest.sh for OpenSearch.
+echo "Run systemd integTest for OpenSearch core engine"
+./gradlew qa:systemd-test:integTest --tests org.opensearch.systemdinteg.SystemdIT --console=plain