Merge branch 'main' into update-beta1-manifest

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

Author: peterzhuamazon

jenkins/check-for-build.jenkinsfile

@@ -23,6 +23,7 @@ pipeline {
     }
     triggers {
         parameterizedCron '''
+            H 1 * * * %INPUT_MANIFEST=2.19.2/opensearch-2.19.2.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 6 * * * %INPUT_MANIFEST=3.0.0-beta1/opensearch-3.0.0-beta1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;UPDATE_GITHUB_ISSUE=false
             H 6 * * * %INPUT_MANIFEST=3.0.0-beta1/opensearch-dashboards-3.0.0-beta1.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;UPDATE_GITHUB_ISSUE=false
         '''

jenkins/opensearch/benchmark-test.jenkinsfile

@@ -70,12 +70,12 @@ pipeline {
         H 3 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/bundle/opensearch/2.19.1/opensearch-2.19.1-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.19.1;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;DATA_NODE_STORAGE=200;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"}
         H 3 * * * %DISTRIBUTION_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0-alpha1/latest/linux/x64/tar/dist/opensearch/opensearch-3.0.0-alpha1-linux-x64.tar.gz;DISTRIBUTION_VERSION=3.0.0-alpha1;TEST_WORKLOAD=noaa;SINGLE_NODE_CLUSTER=true;DATA_INSTANCE_TYPE=c5.2xlarge;DATA_NODE_STORAGE=200;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:3x,cluster-config:x64-c5.2xlarge-1-shard-0-replica-snapshot;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"}
 
-        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:c6g.2xlarge,major-version:2x,cluster-config:arm64-c6g.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-218","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-218","snapshot_name":"http_logs_3_shards"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c6g.2xlarge
-        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:c6g.2xlarge,major-version:2x,cluster-config:arm64-c6g.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-218","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-218","snapshot_name":"nyc_taxis_3_shards"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c6g.2xlarge
-        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-218","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-218","snapshot_name":"http_logs_3_shards"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c5.2xlarge
-        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-218","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-218","snapshot_name":"nyc_taxis_3_shards"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c5.2xlarge
-        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:c6g.2xlarge,major-version:2x,cluster-config:arm64-c6g.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-218","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-218","snapshot_name":"big5_3_shards_single_client"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c6g.2xlarge
-        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-218","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-218","snapshot_name":"big5_3_shards_single_client"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c5.2xlarge
+        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:c6g.2xlarge,major-version:2x,cluster-config:arm64-c6g.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-219","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-219","snapshot_name":"http_logs_3_shards"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c6g.2xlarge
+        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:c6g.2xlarge,major-version:2x,cluster-config:arm64-c6g.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-219","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-219","snapshot_name":"nyc_taxis_3_shards"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c6g.2xlarge
+        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-219","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-219","snapshot_name":"http_logs_3_shards"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c5.2xlarge
+        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-219","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-219","snapshot_name":"nyc_taxis_3_shards"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c5.2xlarge
+        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:c6g.2xlarge,major-version:2x,cluster-config:arm64-c6g.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-219","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-219","snapshot_name":"big5_3_shards_single_client"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c6g.2xlarge
+        H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=big5;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:c5.2xlarge,major-version:2x,cluster-config:x64-c5.2xlarge-3-data-3-shards-0-replica-snapshot;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo-219","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots-219","snapshot_name":"big5_3_shards_single_client"};TEST_PROCEDURE=restore-from-snapshot;DATA_INSTANCE_TYPE=c5.2xlarge
 
 
         H 5 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.19.1/latest/linux/arm64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=vectorsearch;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;DATA_INSTANCE_TYPE=r6g.2xlarge;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.2xlarge,major-version:2x,cluster-config:arm64-r6g.2xlarge-3-data-3-shards-1-replica-nmslib-cohere-1m;ADDITIONAL_CONFIG=knn.algo_param.index_thread_qty:2;WORKLOAD_PARAMS={"target_index_name":"target_index","target_field_name":"target_field","target_index_body":"indices/nmslib-index.json","target_index_primary_shards":3,"target_index_dimension":768,"target_index_space_type":"innerproduct","target_index_bulk_size":100,"target_index_bulk_index_data_set_format":"hdf5","target_index_bulk_index_data_set_corpus":"cohere-1m","target_index_bulk_indexing_clients":10,"target_index_max_num_segments":1,"hnsw_ef_search":256,"hnsw_ef_construction":256,"query_k":100,"query_body":{"docvalue_fields":["_id"],"stored_fields":"_none_"},"query_data_set_format":"hdf5","query_data_set_corpus":"cohere-1m","neighbors_data_set_corpus":"cohere-1m","neighbors_data_set_format":"hdf5","query_count":10000}

jenkins/release-workflows/promote-repos.jenkinsfile

@@ -7,7 +7,7 @@
  * compatible open source license.
  */
 
-lib = library(identifier: 'jenkins@6.4.7', retriever: modernSCM([
+lib = library(identifier: 'jenkins@8.3.4', retriever: modernSCM([
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
 ]))

manifests/2.19.2/opensearch-2.19.2-test.yml

@@ -0,0 +1,172 @@
+---
+schema-version: '1.0'
+name: OpenSearch
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1
+    args: -e JAVA_HOME=/opt/java/openjdk-21
+components:
+  - name: OpenSearch
+    smoke-test:
+      test-spec: opensearch.yml
+  - name: alerting
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        plugins.destination.host.deny_list:
+          - 10.0.0.0/8
+          - 127.0.0.1
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: anomaly-detection
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: flow-framework
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: asynchronous-search
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: cross-cluster-replication
+    integ-test:
+      topology:
+        - cluster_name: leader
+          data_nodes: 2
+        - cluster_name: follower
+          data_nodes: 2
+      test-configs:
+        - with-security
+        - without-security
+  - name: geospatial
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: index-management
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        path.repo:
+          - /tmp
+    bwc-test:
+      test-configs:
+        - with-security
+    smoke-test:
+      test-spec: index-management.yml
+  - name: k-NN
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        node.attr.knn_cb_tier: integ
+        path.repo:
+          - /tmp
+          - C:\
+    smoke-test:
+      test-spec: k-NN.yml
+  - name: ml-commons
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: opensearch-learning-to-rank-base
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: neural-search
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: notifications
+    working-directory: notifications
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: opensearch-observability
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: opensearch-reports
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: security
+    integ-test:
+      test-configs:
+        - with-security
+    smoke-test:
+      test-spec: security.yml
+  - name: security-analytics
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: sql
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        script.context.field.max_compilations_rate: 1000/1m
+        plugins.query.datasources.encryption.masterkey: 4fc8fee6a3fd7d6ca01772e5
+    bwc-test:
+      test-configs:
+        - with-security
+    smoke-test:
+      test-spec: sql.yml
+  - name: custom-codecs
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: skills
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: query-insights
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: opensearch-system-templates
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        opensearch.experimental.feature.application_templates.enabled: true
+        cluster.application_templates.enabled: true