Merge branch 'opensearch-project:main' into remoteinteg-main

.github/pull_request_template.md

@@ -1,15 +1,16 @@
 ### Description
 [Describe what this change achieves]
-
-### Issues Resolved
-[List any issues this PR will resolve]
-
+
+### Related Issues
+Resolves #[Issue number to be closed when this PR is merged]
+<!-- List any other related issues here -->
+
 ### Check List
 - [ ] New functionality includes testing.
-  - [ ] All tests pass
 - [ ] New functionality has been documented.
-  - [ ] New functionality has javadoc added
-- [ ] Commits are signed per the DCO using --signoff 
+- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md).
+- [ ] Commits are signed per the DCO using `--signoff`.
+- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose).
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
-For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).
+For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/cross-cluster-replication/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).

.github/workflows/auto-release.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: ncipollo/release-action@v1
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}

.github/workflows/build-and-test.yml

@@ -32,7 +32,7 @@ jobs:
 
     steps:
       - name: Checkout CCR
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
@@ -44,7 +44,7 @@ jobs:
           chown -R 1000:1000 `pwd`
           su `id -un 1000` -c 'whoami && java -version && ./gradlew --refresh-dependencies clean release -D"build.snapshot=true"'
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: logs

.github/workflows/build.yml

@@ -31,7 +31,7 @@ jobs:
           java-version: ${{ matrix.java }}
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Build and run Replication tests
         run: |
           ./gradlew --refresh-dependencies clean release -D"build.snapshot=true" -x test -x IntegTest

.github/workflows/bwc.yml

@@ -21,14 +21,14 @@ jobs:
           java-version: 11
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Build and run Replication tests
         run: |
           echo "Running backwards compatibility tests ..."
           ./gradlew --refresh-dependencies clean release -Dbuild.snapshot=true -x test -x IntegTest
           ./gradlew fullRestartClusterTask --stacktrace
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: logs

.github/workflows/security-knn-tests.yml

@@ -24,7 +24,7 @@ jobs:
     steps:
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - id: plugin-availability-check
         name: "plugin check"
         run: |
@@ -71,14 +71,14 @@ jobs:
           java-version: 17
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Build and run Replication tests
         run: |
           chown -R 1000:1000 `pwd`
           ls -al src/test/resources/security/plugin
           su `id -un 1000` -c "whoami && java -version && ./gradlew --refresh-dependencies clean release -Dbuild.snapshot=true -Psecurity=true"
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: logs
@@ -116,13 +116,13 @@ jobs:
           java-version: 17
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Build and run Replication tests
         run: |
           chown -R 1000:1000 `pwd`
           su `id -un 1000` -c 'whoami && java -version && ./gradlew --refresh-dependencies clean release -Dbuild.snapshot=true -PnumNodes=1  -Dtests.class=org.opensearch.replication.BasicReplicationIT   -Dtests.method="test knn index replication"  -Pknn=true'
       - name: Upload failed logs
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: logs

SECURITY.md

@@ -1,3 +1,3 @@
 ## Reporting a Vulnerability
 
-If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue.
+If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do **not** create a public GitHub issue.

build.gradle

@@ -76,7 +76,7 @@ buildscript {
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}"
         classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}"
         classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.0.0-RC15"
-        classpath "org.jacoco:org.jacoco.agent:0.8.5"
+        classpath "org.jacoco:org.jacoco.agent:0.8.12"
     }
 }
 
@@ -380,8 +380,6 @@ testClusters {
             testDistribution = "ARCHIVE"
         }
         int debugPort = 5005
-        //adding it to test migration
-        systemProperty('opensearch.experimental.feature.remote_store.migration.enabled','true')
 
         if (_numNodes > 1) numberOfNodes = _numNodes
         //numberOfNodes = 3
@@ -940,8 +938,7 @@ task integTestRemote (type: RestIntegTestTask) {
 
     }
     filter {
-            setExcludePatterns("org.opensearch.replication.bwc.BackwardsCompatibilityIT","org.opensearch.replication.singleCluster.SingleClusterSanityIT",
-                    "org.opensearch.replication.integ.rest.StartReplicationIT.test operations are fetched from lucene when leader is in mixed mode")
+            setExcludePatterns("org.opensearch.replication.bwc.BackwardsCompatibilityIT","org.opensearch.replication.singleCluster.SingleClusterSanityIT")
     }
 }
 

perf_workflow/requirements.txt

@@ -3,7 +3,7 @@ validators
 yamlfix
 cerberus
 pipenv
-requests~=2.31.0
+requests~=2.32.0
 retry
 ndg-httpsclient
 psutil

release-notes/opensearch-cross-cluster-replication.release-notes-2.15.0.0.md

@@ -0,0 +1,6 @@
+## Version 2.15.0.0 Release Notes
+
+Compatible with OpenSearch 2.15.0
+
+### Maintenance
+* Disabling docrep to remote store migration test for integTestRemote suite ([#1379](https://github.com/opensearch-project/cross-cluster-replication/pull/1379))

release-notes/opensearch-cross-cluster-replication.release-notes-2.16.0.0.md

@@ -0,0 +1,13 @@
+## Version 2.16.0 Release Notes
+
+Compatible with OpenSearch 2.16.0
+
+## Security Fixes
+* Upgraded the version of the requests to 2.32.0 from 2.31.0. ([#1417](https://github.com/opensearch-project/cross-cluster-replication/pull/1417/))
+
+
+
+
+
+
+

release-notes/opensearch-cross-cluster-replication.release-notes-2.17.0.0.md

@@ -0,0 +1,6 @@
+## Version 2.17.0.0 Release Notes
+
+Compatible with OpenSearch 2.17.0
+
+### Bug Fixes
+* Updating remote-migration IT with correct setting name ([#1412](https://github.com/opensearch-project/cross-cluster-replication/pull/1412))

src/main/kotlin/org/opensearch/replication/action/autofollow/TransportAutoFollowClusterManagerNodeAction.kt

@@ -111,6 +111,7 @@ class TransportAutoFollowClusterManagerNodeAction @Inject constructor(transportS
         } catch(e: ResourceAlreadyExistsException) {
             // Log and bail as task is already running
             log.warn("Task already started for '$clusterAlias:$patternName'", e)
+            throw OpenSearchException("Exisiting autofollow replication rule cannot be recreated/updated", e)
         } catch (e: Exception) {
             log.error("Failed to start auto follow task for cluster '$clusterAlias:$patternName'", e)
             throw OpenSearchException(AUTOFOLLOW_EXCEPTION_GENERIC_STRING)

src/main/kotlin/org/opensearch/replication/action/autofollow/UpdateAutoFollowPatternRequest.kt

@@ -14,6 +14,7 @@ package org.opensearch.replication.action.autofollow
 import org.opensearch.replication.action.index.ReplicateIndexRequest
 import org.opensearch.replication.metadata.store.KEY_SETTINGS
 import org.opensearch.replication.util.ValidationUtil.validateName
+import org.opensearch.replication.util.ValidationUtil.validatePattern
 import org.opensearch.action.ActionRequestValidationException
 import org.opensearch.action.support.master.AcknowledgedRequest
 import org.opensearch.core.ParseField
@@ -113,8 +114,11 @@ class UpdateAutoFollowPatternRequest: AcknowledgedRequest<UpdateAutoFollowPatter
             if(pattern != null) {
                 validationException.addValidationError("Unexpected pattern")
             }
-        } else if(pattern == null) {
-            validationException.addValidationError("Missing pattern")
+        } else {
+            if(pattern == null)
+               validationException.addValidationError("Missing pattern")
+            else
+               validatePattern(pattern, validationException)
         }
 
         return if(validationException.validationErrors().isEmpty()) return null else validationException

src/main/kotlin/org/opensearch/replication/repository/RemoteClusterRepository.kt

@@ -74,6 +74,7 @@ import java.util.UUID
 import java.util.function.Consumer
 import java.util.function.Function
 import kotlin.collections.ArrayList
+import org.opensearch.common.Priority
 
 const val REMOTE_REPOSITORY_PREFIX = "replication-remote-repo-"
 const val REMOTE_REPOSITORY_TYPE = "replication-remote-repository"
@@ -122,6 +123,13 @@ class RemoteClusterRepository(private val repositoryMetadata: RepositoryMetadata
         throw UnsupportedOperationException("Operation not permitted")
     }
 
+    override fun finalizeSnapshot(shardGenerations: ShardGenerations?, repositoryStateId: Long, clusterMetadata: Metadata?,
+                                  snapshotInfo: SnapshotInfo?, repositoryMetaVersion: Version?,
+                                  stateTransformer: Function<ClusterState, ClusterState>?, repositoryUpdatePriority: Priority,
+                                  listener: ActionListener<RepositoryData>?) {
+        throw UnsupportedOperationException("Operation not permitted")
+    }
+
     override fun deleteSnapshots(snapshotIds: MutableCollection<SnapshotId>?, repositoryStateId: Long,
                                  repositoryMetaVersion: Version?, listener: ActionListener<RepositoryData>?) {
         throw UnsupportedOperationException("Operation not permitted")

src/main/kotlin/org/opensearch/replication/util/ValidationUtil.kt

@@ -105,6 +105,26 @@ object ValidationUtil {
             validationException.addValidationError("Value $name must not start with '.'")
     }
 
+    /**
+     * Validate the pattern against the rules that we have for indexPattern name.
+     */
+
+    fun validatePattern(pattern: String?, validationException: ValidationException) {
+
+        if (!Strings.validFileNameExcludingAstrix(pattern))
+            validationException.addValidationError("Autofollow pattern: $pattern must not contain the following characters ${Strings.INVALID_FILENAME_CHARS}")
+
+        if (pattern.isNullOrEmpty() == true)
+            validationException.addValidationError("Autofollow pattern: $pattern must not be empty")
+
+        if ((pattern?.contains("#") ?: false)|| (pattern?.contains(":") ?: false))
+            validationException.addValidationError("Autofollow pattern: $pattern must not contain '#' or ':'")
+
+        if ((pattern?.startsWith('_') ?: false) || (pattern?.startsWith('-') ?: false))
+            validationException.addValidationError("Autofollow pattern: $pattern must not start with '_' or '-'")
+
+    }
+
     /**
      * validate leader index version for compatibility
      * If on higher version - Replication will not be allowed

src/test/kotlin/org/opensearch/replication/ReplicationHelpers.kt

@@ -20,6 +20,7 @@ import org.opensearch.action.support.master.AcknowledgedResponse
 import org.opensearch.client.Request
 import org.opensearch.client.RequestOptions
 import org.opensearch.client.Response
+import org.opensearch.client.ResponseException
 import org.opensearch.client.RestHighLevelClient
 import org.opensearch.common.settings.Settings
 import org.opensearch.common.unit.TimeValue
@@ -325,7 +326,8 @@ fun RestHighLevelClient.waitForReplicationStop(index: String, waitFor : TimeValu
 fun RestHighLevelClient.updateAutoFollowPattern(connection: String, patternName: String, pattern: String,
                                                 settings: Settings = Settings.EMPTY,
                                                 useRoles: UseRoles = UseRoles(),
-                                                requestOptions: RequestOptions = RequestOptions.DEFAULT) {
+                                                requestOptions: RequestOptions = RequestOptions.DEFAULT,
+                                                ignoreIfExists: Boolean = false) {
     val lowLevelRequest = Request("POST", REST_AUTO_FOLLOW_PATTERN)
     if (settings == Settings.EMPTY) {
         lowLevelRequest.setJsonEntity("""{
@@ -350,9 +352,14 @@ fun RestHighLevelClient.updateAutoFollowPattern(connection: String, patternName:
                                      }""")
     }
     lowLevelRequest.setOptions(requestOptions)
-    val lowLevelResponse = lowLevelClient.performRequest(lowLevelRequest)
-    val response = getAckResponse(lowLevelResponse)
-    assertThat(response.isAcknowledged).isTrue()
+    try {
+        val lowLevelResponse = lowLevelClient.performRequest(lowLevelRequest)
+        val response = getAckResponse(lowLevelResponse)
+        assertThat(response.isAcknowledged).isTrue()
+    } catch (e: ResponseException) {
+        // Skip if ignoreIfExists is true and exception contains resource_already_exists_exception
+        if  ((ignoreIfExists == true && e.message?.contains("resource_already_exists_exception")!!) == false) throw e
+    }
 }
 
 fun RestHighLevelClient.AutoFollowStats() : Map<String, Any>  {

src/test/kotlin/org/opensearch/replication/integ/rest/StartReplicationIT.kt

@@ -1278,8 +1278,8 @@ class StartReplicationIT: MultiClusterRestTestCase() {
         val entityAsString = """
                         {
                           "persistent": {
-                             "remote_store.compatibility_mode": "mixed",
-                             "migration.direction" : "remote_store"
+                             "cluster.remote_store.compatibility_mode": "mixed",
+                             "cluster.migration.direction" : "remote_store"
                           }
                         }""".trimMargin()