|
40 | 40 | import org.opensearch.cluster.metadata.IndexMetadata;
|
41 | 41 | import org.opensearch.cluster.metadata.Metadata;
|
42 | 42 | import org.opensearch.cluster.node.DiscoveryNode;
|
| 43 | +import org.opensearch.cluster.node.DiscoveryNodeRole; |
43 | 44 | import org.opensearch.cluster.node.DiscoveryNodes;
|
44 | 45 | import org.opensearch.cluster.routing.RoutingNodes;
|
45 | 46 | import org.opensearch.cluster.routing.RoutingTable;
|
46 | 47 | import org.opensearch.cluster.routing.ShardRouting;
|
47 | 48 | import org.opensearch.cluster.routing.allocation.command.AllocationCommands;
|
48 | 49 | import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
|
49 | 50 | import org.opensearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
|
| 51 | +import org.opensearch.common.UUIDs; |
50 | 52 | import org.opensearch.common.settings.Settings;
|
| 53 | +import org.opensearch.common.util.FeatureFlags; |
51 | 54 | import org.opensearch.core.index.shard.ShardId;
|
52 | 55 | import org.opensearch.indices.replication.common.ReplicationType;
|
| 56 | +import org.opensearch.node.remotestore.RemoteStoreNodeService; |
53 | 57 | import org.opensearch.test.VersionUtils;
|
54 | 58 |
|
55 | 59 | import java.util.ArrayList;
|
56 | 60 | import java.util.HashSet;
|
| 61 | +import java.util.List; |
| 62 | +import java.util.Map; |
57 | 63 | import java.util.Set;
|
58 | 64 |
|
59 | 65 | import static org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING;
|
60 | 66 | import static org.opensearch.cluster.routing.ShardRoutingState.INITIALIZING;
|
61 | 67 | import static org.opensearch.cluster.routing.ShardRoutingState.RELOCATING;
|
62 | 68 | import static org.opensearch.cluster.routing.ShardRoutingState.STARTED;
|
63 | 69 | import static org.opensearch.cluster.routing.ShardRoutingState.UNASSIGNED;
|
| 70 | +import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL; |
| 71 | +import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; |
| 72 | +import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING; |
| 73 | +import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; |
64 | 74 | import static org.hamcrest.Matchers.anyOf;
|
65 | 75 | import static org.hamcrest.Matchers.equalTo;
|
66 | 76 | import static org.hamcrest.Matchers.lessThan;
|
@@ -812,4 +822,134 @@ private void testReplicaIsPromoted(boolean isSegmentReplicationEnabled) {
|
812 | 822 | }
|
813 | 823 | }
|
814 | 824 | }
|
| 825 | + |
| 826 | + public void testPreferReplicaOnRemoteNodeForPrimaryPromotion() { |
| 827 | + FeatureFlags.initializeFeatureFlags(Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build()); |
| 828 | + AllocationService allocation = createAllocationService(Settings.builder().build()); |
| 829 | + |
| 830 | + // segment replication enabled |
| 831 | + Settings.Builder settingsBuilder = settings(Version.CURRENT).put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); |
| 832 | + |
| 833 | + // remote store migration metadata settings |
| 834 | + Metadata metadata = Metadata.builder() |
| 835 | + .put(IndexMetadata.builder("test").settings(settingsBuilder).numberOfShards(1).numberOfReplicas(4)) |
| 836 | + .persistentSettings( |
| 837 | + Settings.builder() |
| 838 | + .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.MIXED.mode) |
| 839 | + .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE.direction) |
| 840 | + .build() |
| 841 | + ) |
| 842 | + .build(); |
| 843 | + |
| 844 | + RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build(); |
| 845 | + |
| 846 | + ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) |
| 847 | + .metadata(metadata) |
| 848 | + .routingTable(initialRoutingTable) |
| 849 | + .build(); |
| 850 | + |
| 851 | + ShardId shardId = new ShardId(metadata.index("test").getIndex(), 0); |
| 852 | + |
| 853 | + // add a remote node and start primary shard |
| 854 | + Map<String, String> remoteStoreNodeAttributes = Map.of( |
| 855 | + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, |
| 856 | + "REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_VALUE" |
| 857 | + ); |
| 858 | + DiscoveryNode remoteNode1 = new DiscoveryNode( |
| 859 | + UUIDs.base64UUID(), |
| 860 | + buildNewFakeTransportAddress(), |
| 861 | + remoteStoreNodeAttributes, |
| 862 | + DiscoveryNodeRole.BUILT_IN_ROLES, |
| 863 | + Version.CURRENT |
| 864 | + ); |
| 865 | + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(remoteNode1)).build(); |
| 866 | + clusterState = ClusterState.builder(clusterState).routingTable(allocation.reroute(clusterState, "reroute").routingTable()).build(); |
| 867 | + assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); |
| 868 | + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(4)); |
| 869 | + |
| 870 | + clusterState = startInitializingShardsAndReroute(allocation, clusterState); |
| 871 | + assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1)); |
| 872 | + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(4)); |
| 873 | + |
| 874 | + // add remote and non-remote nodes and start replica shards |
| 875 | + DiscoveryNode remoteNode2 = new DiscoveryNode( |
| 876 | + UUIDs.base64UUID(), |
| 877 | + buildNewFakeTransportAddress(), |
| 878 | + remoteStoreNodeAttributes, |
| 879 | + DiscoveryNodeRole.BUILT_IN_ROLES, |
| 880 | + Version.CURRENT |
| 881 | + ); |
| 882 | + DiscoveryNode remoteNode3 = new DiscoveryNode( |
| 883 | + UUIDs.base64UUID(), |
| 884 | + buildNewFakeTransportAddress(), |
| 885 | + remoteStoreNodeAttributes, |
| 886 | + DiscoveryNodeRole.BUILT_IN_ROLES, |
| 887 | + Version.CURRENT |
| 888 | + ); |
| 889 | + DiscoveryNode nonRemoteNode1 = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); |
| 890 | + DiscoveryNode nonRemoteNode2 = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT); |
| 891 | + List<DiscoveryNode> replicaShardNodes = List.of(remoteNode2, remoteNode3, nonRemoteNode1, nonRemoteNode2); |
| 892 | + |
| 893 | + for (int i = 0; i < 4; i++) { |
| 894 | + clusterState = ClusterState.builder(clusterState) |
| 895 | + .nodes(DiscoveryNodes.builder(clusterState.nodes()).add(replicaShardNodes.get(i))) |
| 896 | + .build(); |
| 897 | + |
| 898 | + clusterState = allocation.reroute(clusterState, "reroute"); |
| 899 | + assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1 + i)); |
| 900 | + assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); |
| 901 | + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(4 - (i + 1))); |
| 902 | + |
| 903 | + clusterState = startInitializingShardsAndReroute(allocation, clusterState); |
| 904 | + assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1 + (i + 1))); |
| 905 | + assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(4 - (i + 1))); |
| 906 | + } |
| 907 | + |
| 908 | + // fail primary shard |
| 909 | + ShardRouting primaryShard0 = clusterState.routingTable().index("test").shard(0).primaryShard(); |
| 910 | + ClusterState newState = allocation.applyFailedShard(clusterState, primaryShard0, randomBoolean()); |
| 911 | + assertNotEquals(clusterState, newState); |
| 912 | + clusterState = newState; |
| 913 | + |
| 914 | + // verify that promoted replica exists on a remote node |
| 915 | + assertEquals(4, clusterState.getRoutingNodes().shardsWithState(STARTED).size()); |
| 916 | + ShardRouting primaryShardRouting1 = clusterState.routingTable().index("test").shard(0).primaryShard(); |
| 917 | + assertNotEquals(primaryShard0, primaryShardRouting1); |
| 918 | + assertTrue( |
| 919 | + primaryShardRouting1.currentNodeId().equals(remoteNode2.getId()) |
| 920 | + || primaryShardRouting1.currentNodeId().equals(remoteNode3.getId()) |
| 921 | + ); |
| 922 | + |
| 923 | + // fail primary shard again |
| 924 | + newState = allocation.applyFailedShard(clusterState, primaryShardRouting1, randomBoolean()); |
| 925 | + assertNotEquals(clusterState, newState); |
| 926 | + clusterState = newState; |
| 927 | + |
| 928 | + // verify that promoted replica again exists on a remote node |
| 929 | + assertEquals(3, clusterState.getRoutingNodes().shardsWithState(STARTED).size()); |
| 930 | + ShardRouting primaryShardRouting2 = clusterState.routingTable().index("test").shard(0).primaryShard(); |
| 931 | + assertNotEquals(primaryShardRouting1, primaryShardRouting2); |
| 932 | + assertTrue( |
| 933 | + primaryShardRouting2.currentNodeId().equals(remoteNode2.getId()) |
| 934 | + || primaryShardRouting2.currentNodeId().equals(remoteNode3.getId()) |
| 935 | + ); |
| 936 | + assertNotEquals(primaryShardRouting1.currentNodeId(), primaryShardRouting2.currentNodeId()); |
| 937 | + |
| 938 | + ShardRouting expectedCandidateForSegRep = clusterState.getRoutingNodes().activeReplicaWithOldestVersion(shardId); |
| 939 | + |
| 940 | + // fail primary shard again |
| 941 | + newState = allocation.applyFailedShard(clusterState, primaryShardRouting2, randomBoolean()); |
| 942 | + assertNotEquals(clusterState, newState); |
| 943 | + clusterState = newState; |
| 944 | + |
| 945 | + // verify that promoted replica exists on a non-remote node |
| 946 | + assertEquals(2, clusterState.getRoutingNodes().shardsWithState(STARTED).size()); |
| 947 | + ShardRouting primaryShardRouting3 = clusterState.routingTable().index("test").shard(0).primaryShard(); |
| 948 | + assertNotEquals(primaryShardRouting2, primaryShardRouting3); |
| 949 | + assertTrue( |
| 950 | + primaryShardRouting3.currentNodeId().equals(nonRemoteNode1.getId()) |
| 951 | + || primaryShardRouting3.currentNodeId().equals(nonRemoteNode2.getId()) |
| 952 | + ); |
| 953 | + assertEquals(expectedCandidateForSegRep.allocationId(), primaryShardRouting3.allocationId()); |
| 954 | + } |
815 | 955 | }
|
0 commit comments