increase the wait timeout to fetch the master key (opensearch-project#3151) (opensearch-project#3177)

* increase the wait timeout to fetch the master key

Signed-off-by: Jing Zhang <jngz@amazon.com>

* fix UTs

Signed-off-by: Jing Zhang <jngz@amazon.com>

---------

Signed-off-by: Jing Zhang <jngz@amazon.com>
(cherry picked from commit c7e564c)

Co-authored-by: Jing Zhang <jngz@amazon.com>

Author: opensearch-trigger-bot[bot]

ml-algorithms/src/main/java/org/opensearch/ml/engine/encryptor/EncryptorImpl.java

@@ -181,7 +181,10 @@ private void initMasterKey() {
         }));
 
         try {
-            latch.await(1, SECONDS);
+            boolean completed = latch.await(3, SECONDS);
+            if (!completed) {
+                throw new MLException("Fetching master key timed out.");
+            }
         } catch (InterruptedException e) {
             throw new IllegalStateException(e);
         }

ml-algorithms/src/test/java/org/opensearch/ml/engine/encryptor/EncryptorImplTest.java

@@ -462,8 +462,8 @@ public void decrypt() {
 
     @Test
     public void encrypt_NullMasterKey_NullMasterKey_MasterKeyNotExistInIndex() {
-        exceptionRule.expect(ResourceNotFoundException.class);
-        exceptionRule.expectMessage(MASTER_KEY_NOT_READY_ERROR);
+        exceptionRule.expect(MLException.class);
+        exceptionRule.expectMessage("Fetching master key timed out.");
 
         doAnswer(invocation -> {
             ActionListener<GetResponse> listener = invocation.getArgument(1);
@@ -517,8 +517,8 @@ public void decrypt_NoResponseToInitConfigIndex() {
 
     @Test
     public void decrypt_MLConfigIndexNotFound() {
-        exceptionRule.expect(ResourceNotFoundException.class);
-        exceptionRule.expectMessage(MASTER_KEY_NOT_READY_ERROR);
+        exceptionRule.expect(MLException.class);
+        exceptionRule.expectMessage("Fetching master key timed out.");
 
         Metadata metadata = new Metadata.Builder().indices(ImmutableMap.of()).build();
         when(clusterState.metadata()).thenReturn(metadata);