Skip to content

Commit c6cebc7

Browse files
authored
Fix flaky RemoteIndexRecoveryIT testRerouteRecovery test #9580 (#11918)
Signed-off-by: Ashish Singh <ssashish@amazon.com>
1 parent 2f8d267 commit c6cebc7

File tree

3 files changed

+35
-8
lines changed

3 files changed

+35
-8
lines changed

server/src/internalClusterTest/java/org/opensearch/indices/recovery/IndexRecoveryIT.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -523,12 +523,12 @@ public void testRerouteRecovery() throws Exception {
523523

524524
logger.info("--> waiting for recovery to start both on source and target");
525525
final Index index = resolveIndex(INDEX_NAME);
526-
assertBusy(() -> {
526+
assertBusyWithFixedSleepTime(() -> {
527527
IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeA);
528528
assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(), equalTo(1));
529529
indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
530530
assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(), equalTo(1));
531-
});
531+
}, TimeValue.timeValueSeconds(10), TimeValue.timeValueMillis(500));
532532

533533
logger.info("--> request recoveries");
534534
RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();

server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java

-6
Original file line numberDiff line numberDiff line change
@@ -157,10 +157,4 @@ public void testDisconnectsDuringRecovery() {
157157
public void testReplicaRecovery() {
158158

159159
}
160-
161-
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9580")
162-
public void testRerouteRecovery() {
163-
164-
}
165-
166160
}

test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java

+33
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
import org.opensearch.common.settings.Settings;
8484
import org.opensearch.common.time.DateUtils;
8585
import org.opensearch.common.time.FormatNames;
86+
import org.opensearch.common.unit.TimeValue;
8687
import org.opensearch.common.util.MockBigArrays;
8788
import org.opensearch.common.util.MockPageCacheRecycler;
8889
import org.opensearch.common.util.concurrent.ThreadContext;
@@ -1095,6 +1096,38 @@ public static void assertBusy(CheckedRunnable<Exception> codeBlock, long maxWait
10951096
}
10961097
}
10971098

1099+
/**
1100+
* Runs the code block for the provided max wait time and sleeping for fixed sleep time, waiting for no assertions to trip.
1101+
*/
1102+
public static void assertBusyWithFixedSleepTime(CheckedRunnable<Exception> codeBlock, TimeValue maxWaitTime, TimeValue sleepTime)
1103+
throws Exception {
1104+
long maxTimeInMillis = maxWaitTime.millis();
1105+
long sleepTimeInMillis = sleepTime.millis();
1106+
if (sleepTimeInMillis > maxTimeInMillis) {
1107+
throw new IllegalArgumentException("sleepTime is more than the maxWaitTime");
1108+
}
1109+
long sum = 0;
1110+
List<AssertionError> failures = new ArrayList<>();
1111+
while (sum <= maxTimeInMillis) {
1112+
try {
1113+
codeBlock.run();
1114+
return;
1115+
} catch (AssertionError e) {
1116+
failures.add(e);
1117+
}
1118+
sum += sleepTimeInMillis;
1119+
Thread.sleep(sleepTimeInMillis);
1120+
}
1121+
try {
1122+
codeBlock.run();
1123+
} catch (AssertionError e) {
1124+
for (AssertionError failure : failures) {
1125+
e.addSuppressed(failure);
1126+
}
1127+
throw e;
1128+
}
1129+
}
1130+
10981131
/**
10991132
* Periodically execute the supplied function until it returns true, or a timeout
11001133
* is reached. This version uses a timeout of 10 seconds. If at all possible,

0 commit comments

Comments
 (0)