Fix Flaky Test SpecificClusterManagerNodesIT.testElectOnlyBetweenClusterManagerNodes

Signed-off-by: kkewwei <kewei.11@bytedance.com>
Signed-off-by: kkewwei <kkewwei@163.com>

Author: kkewwei

server/src/internalClusterTest/java/org/opensearch/cluster/SpecificClusterManagerNodesIT.java

@@ -44,6 +44,7 @@
 import org.opensearch.test.OpenSearchIntegTestCase.Scope;
 
 import java.io.IOException;
+import java.util.function.Supplier;
 
 import static org.opensearch.test.NodeRoles.clusterManagerNode;
 import static org.opensearch.test.NodeRoles.dataOnlyNode;
@@ -254,9 +255,9 @@ public void testElectOnlyBetweenClusterManagerNodes() throws Exception {
         logger.info("--> closing cluster-manager node (1)");
         client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(clusterManagerNodeName)).get();
         // removing the cluster-manager from the voting configuration immediately triggers the cluster-manager to step down
-        assertBusy(() -> {
-            assertThat(
-                internalCluster().nonClusterManagerClient()
+        Supplier<String> getClusterManagerIfElected = () -> {
+            try {
+                return internalCluster().nonClusterManagerClient()
                     .admin()
                     .cluster()
                     .prepareState()
@@ -265,9 +266,14 @@ public void testElectOnlyBetweenClusterManagerNodes() throws Exception {
                     .getState()
                     .nodes()
                     .getClusterManagerNode()
-                    .getName(),
-                equalTo(nextClusterManagerEligableNodeName)
-            );
+                    .getName();
+            } catch (ClusterManagerNotDiscoveredException e) {
+                logger.debug("failed to get cluster-manager name", e);
+                return null;
+            }
+        };
+        assertBusy(() -> {
+            assertThat(getClusterManagerIfElected.get(), equalTo(nextClusterManagerEligableNodeName));
             assertThat(
                 internalCluster().clusterManagerClient()
                     .admin()

test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java

@@ -89,6 +89,7 @@
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.indices.breaker.CircuitBreakerService;
 import org.opensearch.core.util.FileSystemUtils;
+import org.opensearch.discovery.ClusterManagerNotDiscoveredException;
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
 import org.opensearch.env.ShardLockObtainFailedException;
@@ -2171,7 +2172,7 @@ public String getClusterManagerName(@Nullable String viaNode) {
             return client.admin().cluster().prepareState().get().getState().nodes().getClusterManagerNode().getName();
         } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
-            throw new RuntimeException("Can't get cluster-manager node " + e.getMessage(), e);
+            throw new ClusterManagerNotDiscoveredException(e);
         }
     }