Skip to content

Commit cdd86f1

Browse files
authored
Better error when compaction executors are not set (apache#4212)
* Better error when compaction executors are not set Replaces NPE with proper exception type and message for when the Default Compaction Planner's "executors" property is set and empty or not set at all.
1 parent 8bf0970 commit cdd86f1

File tree

2 files changed

+84
-44
lines changed

2 files changed

+84
-44
lines changed

core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java

+50-44
Original file line numberDiff line numberDiff line change
@@ -167,62 +167,68 @@ public String toString() {
167167
justification = "Field is written by Gson")
168168
@Override
169169
public void init(InitParameters params) {
170-
ExecutorConfig[] execConfigs =
171-
new Gson().fromJson(params.getOptions().get("executors"), ExecutorConfig[].class);
172170

173-
List<Executor> tmpExec = new ArrayList<>();
171+
if (params.getOptions().containsKey("executors")
172+
&& !params.getOptions().get("executors").isBlank()) {
174173

175-
for (ExecutorConfig executorConfig : execConfigs) {
176-
Long maxSize = executorConfig.maxSize == null ? null
177-
: ConfigurationTypeHelper.getFixedMemoryAsBytes(executorConfig.maxSize);
174+
ExecutorConfig[] execConfigs =
175+
new Gson().fromJson(params.getOptions().get("executors"), ExecutorConfig[].class);
178176

179-
CompactionExecutorId ceid;
177+
List<Executor> tmpExec = new ArrayList<>();
180178

181-
// If not supplied, GSON will leave type null. Default to internal
182-
if (executorConfig.type == null) {
183-
executorConfig.type = "internal";
184-
}
179+
for (ExecutorConfig executorConfig : execConfigs) {
180+
Long maxSize = executorConfig.maxSize == null ? null
181+
: ConfigurationTypeHelper.getFixedMemoryAsBytes(executorConfig.maxSize);
185182

186-
switch (executorConfig.type) {
187-
case "internal":
188-
Preconditions.checkArgument(null == executorConfig.queue,
189-
"'queue' should not be specified for internal compactions");
190-
int numThreads = Objects.requireNonNull(executorConfig.numThreads,
191-
"'numThreads' must be specified for internal type");
192-
ceid = params.getExecutorManager().createExecutor(executorConfig.name, numThreads);
193-
break;
194-
case "external":
195-
Preconditions.checkArgument(null == executorConfig.numThreads,
196-
"'numThreads' should not be specified for external compactions");
197-
String queue = Objects.requireNonNull(executorConfig.queue,
198-
"'queue' must be specified for external type");
199-
ceid = params.getExecutorManager().getExternalExecutor(queue);
200-
break;
201-
default:
202-
throw new IllegalArgumentException("type must be 'internal' or 'external'");
203-
}
204-
tmpExec.add(new Executor(ceid, maxSize));
205-
}
183+
CompactionExecutorId ceid;
206184

207-
Collections.sort(tmpExec, Comparator.comparing(Executor::getMaxSize,
208-
Comparator.nullsLast(Comparator.naturalOrder())));
185+
// If not supplied, GSON will leave type null. Default to internal
186+
if (executorConfig.type == null) {
187+
executorConfig.type = "internal";
188+
}
209189

210-
executors = List.copyOf(tmpExec);
190+
switch (executorConfig.type) {
191+
case "internal":
192+
Preconditions.checkArgument(null == executorConfig.queue,
193+
"'queue' should not be specified for internal compactions");
194+
int numThreads = Objects.requireNonNull(executorConfig.numThreads,
195+
"'numThreads' must be specified for internal type");
196+
ceid = params.getExecutorManager().createExecutor(executorConfig.name, numThreads);
197+
break;
198+
case "external":
199+
Preconditions.checkArgument(null == executorConfig.numThreads,
200+
"'numThreads' should not be specified for external compactions");
201+
String queue = Objects.requireNonNull(executorConfig.queue,
202+
"'queue' must be specified for external type");
203+
ceid = params.getExecutorManager().getExternalExecutor(queue);
204+
break;
205+
default:
206+
throw new IllegalArgumentException("type must be 'internal' or 'external'");
207+
}
208+
tmpExec.add(new Executor(ceid, maxSize));
209+
}
211210

212-
if (executors.stream().filter(e -> e.getMaxSize() == null).count() > 1) {
213-
throw new IllegalArgumentException(
214-
"Can only have one executor w/o a maxSize. " + params.getOptions().get("executors"));
215-
}
211+
Collections.sort(tmpExec, Comparator.comparing(Executor::getMaxSize,
212+
Comparator.nullsLast(Comparator.naturalOrder())));
213+
214+
executors = List.copyOf(tmpExec);
216215

217-
// use the add method on the Set interface to check for duplicate maxSizes
218-
Set<Long> maxSizes = new HashSet<>();
219-
executors.forEach(e -> {
220-
if (!maxSizes.add(e.getMaxSize())) {
216+
if (executors.stream().filter(e -> e.getMaxSize() == null).count() > 1) {
221217
throw new IllegalArgumentException(
222-
"Duplicate maxSize set in executors. " + params.getOptions().get("executors"));
218+
"Can only have one executor w/o a maxSize. " + params.getOptions().get("executors"));
223219
}
224-
});
225220

221+
// use the add method on the Set interface to check for duplicate maxSizes
222+
Set<Long> maxSizes = new HashSet<>();
223+
executors.forEach(e -> {
224+
if (!maxSizes.add(e.getMaxSize())) {
225+
throw new IllegalArgumentException(
226+
"Duplicate maxSize set in executors. " + params.getOptions().get("executors"));
227+
}
228+
});
229+
} else {
230+
throw new IllegalStateException("No defined executors for this planner");
231+
}
226232
determineMaxFilesToCompact(params);
227233
}
228234

core/src/test/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlannerTest.java

+34
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,40 @@ public void testErrorDuplicateMaxSize() {
470470
assertTrue(e.getMessage().contains("maxSize"), "Error message didn't contain maxSize");
471471
}
472472

473+
/**
474+
* Tests when "executors" is defined but empty.
475+
*/
476+
@Test
477+
public void testErrorEmptyExecutors() {
478+
DefaultCompactionPlanner planner = new DefaultCompactionPlanner();
479+
String executors = "";
480+
481+
var e = assertThrows(IllegalStateException.class,
482+
() -> planner.init(getInitParams(defaultConf, executors)), "Failed to throw error");
483+
assertTrue(e.getMessage().contains("No defined executors"),
484+
"Error message didn't contain 'No defined executors'");
485+
}
486+
487+
/**
488+
* Tests when "executors" doesn't exist
489+
*/
490+
@Test
491+
public void testErrorNoExecutors() {
492+
493+
ServiceEnvironment senv = EasyMock.createMock(ServiceEnvironment.class);
494+
EasyMock.expect(senv.getConfiguration()).andReturn(defaultConf).anyTimes();
495+
EasyMock.replay(senv);
496+
497+
var initParams = new CompactionPlannerInitParams(csid, new HashMap<>(), senv);
498+
499+
DefaultCompactionPlanner dcPlanner = new DefaultCompactionPlanner();
500+
501+
var e = assertThrows(IllegalStateException.class, () -> dcPlanner.init(initParams),
502+
"Failed to throw error");
503+
assertTrue(e.getMessage().contains("No defined executors"),
504+
"Error message didn't contain 'No defined executors'");
505+
}
506+
473507
// Test cases where a tablet has more than table.file.max files, but no files were found using the
474508
// compaction ratio. The planner should try to find the highest ratio that will result in a
475509
// compaction.

0 commit comments

Comments
 (0)