Skip to content

Commit 25f3367

Browse files
committed
Process seeding of split fate operations in batches
Updates the Seeder in the Manager that handles seeding split fate ops to use a single thread and to submit multiple outstanding operations to be seeded together instead of individually in order to improve performance. The user fate store will now track outstanding fate operations and return a future for each pending operation that will be completed when the batch is submitted. This closes apache#5160
1 parent 1c35e23 commit 25f3367

File tree

2 files changed

+6
-4
lines changed
  • core/src/main/java/org/apache/accumulo/core/conf
  • server/manager/src/main/java/org/apache/accumulo/manager/split

2 files changed

+6
-4
lines changed

core/src/main/java/org/apache/accumulo/core/conf/Property.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -461,7 +461,7 @@ public enum Property {
461461
+ "indefinitely. Default is 0 to block indefinitely. Only valid when tserver available "
462462
+ "threshold is set greater than 0.",
463463
"1.10.0"),
464-
// TODO: Mark this as deprecated or delete?
464+
@Deprecated(since = "4.0.0")
465465
MANAGER_SPLIT_WORKER_THREADS("manager.split.seed.threadpool.size", "8", PropertyType.COUNT,
466466
"The number of threads used to seed fate split task, the actual split work is done by fate"
467467
+ " threads.",

server/manager/src/main/java/org/apache/accumulo/manager/split/Splitter.java

+5-3
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ public void run() {
7474
final Map<Text,KeyExtent> userSplits = new HashMap<>();
7575
final Map<Text,KeyExtent> metaSplits = new HashMap<>();
7676

77+
// Go through all the queued up splits and partition
78+
// into the different store types to be submitted.
7779
queuedTablets.forEach((metaRow, extent) -> {
7880
switch (FateInstanceType.fromTableId((extent.tableId()))) {
7981
case USER:
@@ -87,6 +89,9 @@ public void run() {
8789
}
8890
});
8991

92+
// see the user and then meta splits
93+
// The meta plits (zk) will be processed one at a time but there will not be
94+
// many of those splits. The user splits are processed as a batch.
9095
seedSplits(FateInstanceType.USER, userSplits);
9196
seedSplits(FateInstanceType.META, metaSplits);
9297
}
@@ -212,9 +217,6 @@ public Splitter(Manager manager) {
212217
this.manager = manager;
213218
ServerContext context = manager.getContext();
214219

215-
// TODO: Mark this as deprecated or delete?
216-
// int numThreads = context.getConfiguration().getCount(Property.MANAGER_SPLIT_WORKER_THREADS);
217-
218220
this.splitExecutor = context.threadPools().getPoolBuilder("split_seeder").numCoreThreads(1)
219221
.numMaxThreads(1).withTimeOut(0L, TimeUnit.MILLISECONDS).enableThreadPoolMetrics().build();
220222

0 commit comments

Comments
 (0)