Fix Flaky Test SpecificClusterManagerNodesIT.testElectOnlyBetweenClusterManagerNodes

Signed-off-by: kkewwei <kkewwei@163.com>

Author: kkewwei

server/src/internalClusterTest/java/org/opensearch/cluster/SpecificClusterManagerNodesIT.java

@@ -225,7 +225,7 @@ public void testElectOnlyBetweenClusterManagerNodes() throws Exception {
             equalTo(clusterManagerNodeName)
         );
         assertThat(
-            internalCluster().nonClusterManagerClient()
+            internalCluster().clusterManagerClient()
                 .admin()
                 .cluster()
                 .prepareState()
@@ -237,6 +237,31 @@ public void testElectOnlyBetweenClusterManagerNodes() throws Exception {
                 .getName(),
             equalTo(clusterManagerNodeName)
         );
+
+        logger.info("--> closing cluster-manager node (1)");
+        client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(clusterManagerNodeName)).get();
+        // removing the cluster-manager from the voting configuration immediately triggers the cluster-manager to step down
+
+        // make sure `nextClusterManagerEligableNodeName` has been elected as the new cluster-manager
+        waitUntil(() -> {
+            try {
+                return nextClusterManagerEligableNodeName.equals(
+                    internalCluster().nonClusterManagerClient()
+                        .admin()
+                        .cluster()
+                        .prepareState()
+                        .execute()
+                        .actionGet()
+                        .getState()
+                        .nodes()
+                        .getClusterManagerNode()
+                        .getName()
+                );
+            } catch (Exception e) {
+                return false;
+            }
+        });
+
         assertThat(
             internalCluster().clusterManagerClient()
                 .admin()
@@ -248,40 +273,9 @@ public void testElectOnlyBetweenClusterManagerNodes() throws Exception {
                 .nodes()
                 .getClusterManagerNode()
                 .getName(),
-            equalTo(clusterManagerNodeName)
+            equalTo(nextClusterManagerEligableNodeName)
         );
 
-        logger.info("--> closing cluster-manager node (1)");
-        client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(clusterManagerNodeName)).get();
-        // removing the cluster-manager from the voting configuration immediately triggers the cluster-manager to step down
-        assertBusy(() -> {
-            assertThat(
-                internalCluster().nonClusterManagerClient()
-                    .admin()
-                    .cluster()
-                    .prepareState()
-                    .execute()
-                    .actionGet()
-                    .getState()
-                    .nodes()
-                    .getClusterManagerNode()
-                    .getName(),
-                equalTo(nextClusterManagerEligableNodeName)
-            );
-            assertThat(
-                internalCluster().clusterManagerClient()
-                    .admin()
-                    .cluster()
-                    .prepareState()
-                    .execute()
-                    .actionGet()
-                    .getState()
-                    .nodes()
-                    .getClusterManagerNode()
-                    .getName(),
-                equalTo(nextClusterManagerEligableNodeName)
-            );
-        });
         internalCluster().stopRandomNode(InternalTestCluster.nameFilter(clusterManagerNodeName));
         assertThat(
             internalCluster().nonClusterManagerClient()