Skip to content

Commit 7cf46f9

Browse files
DomGarguiloDomGarguilo
committed
Merge remote-tracking branch 'upstream/elasticity' into replaceCompImpl
2 parents c3434ed + cca3683 commit 7cf46f9

File tree

71 files changed

+1288
-381
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+1288
-381
lines changed

‎assemble/bin/accumulo-cluster

+1-5
Original file line numberDiff line numberDiff line change
@@ -526,11 +526,7 @@ tserver:
526526
- localhost
527527
528528
compactor:
529-
accumulo_meta:
530-
- localhost
531-
user_small:
532-
- localhost
533-
user_large:
529+
default:
534530
- localhost
535531
536532
sserver:

‎core/src/main/java/org/apache/accumulo/core/conf/Property.java

+3-4
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ public enum Property {
6565
COMPACTION_SERVICE_ROOT_MAX_OPEN(COMPACTION_SERVICE_PREFIX + "root.planner.opts.maxOpen", "30",
6666
PropertyType.COUNT, "The maximum number of files a compaction will open.", "4.0.0"),
6767
COMPACTION_SERVICE_ROOT_GROUPS(COMPACTION_SERVICE_PREFIX + "root.planner.opts.groups",
68-
"[{'name':'accumulo_meta'}]".replaceAll("'", "\""), PropertyType.STRING,
68+
"[{'group':'default'}]".replaceAll("'", "\""), PropertyType.JSON,
6969
"See {% jlink -f org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
7070
"4.0.0"),
7171
COMPACTION_SERVICE_META_PLANNER(COMPACTION_SERVICE_PREFIX + "meta.planner",
@@ -74,7 +74,7 @@ public enum Property {
7474
COMPACTION_SERVICE_META_MAX_OPEN(COMPACTION_SERVICE_PREFIX + "meta.planner.opts.maxOpen", "30",
7575
PropertyType.COUNT, "The maximum number of files a compaction will open.", "4.0.0"),
7676
COMPACTION_SERVICE_META_GROUPS(COMPACTION_SERVICE_PREFIX + "meta.planner.opts.groups",
77-
"[{'name':'accumulo_meta'}]".replaceAll("'", "\""), PropertyType.JSON,
77+
"[{'group':'default'}]".replaceAll("'", "\""), PropertyType.JSON,
7878
"See {% jlink -f org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
7979
"4.0.0"),
8080
COMPACTION_SERVICE_DEFAULT_PLANNER(COMPACTION_SERVICE_PREFIX + "default.planner",
@@ -83,8 +83,7 @@ public enum Property {
8383
COMPACTION_SERVICE_DEFAULT_MAX_OPEN(COMPACTION_SERVICE_PREFIX + "default.planner.opts.maxOpen",
8484
"10", PropertyType.COUNT, "The maximum number of files a compaction will open.", "4.0.0"),
8585
COMPACTION_SERVICE_DEFAULT_GROUPS(COMPACTION_SERVICE_PREFIX + "default.planner.opts.groups",
86-
("[{'name':'user_small','maxSize':'128M'}, {'name':'user_large'}]").replaceAll("'", "\""),
87-
PropertyType.STRING,
86+
("[{'group':'default'}]").replaceAll("'", "\""), PropertyType.JSON,
8887
"See {% jlink -f org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
8988
"4.0.0"),
9089
COMPACTION_WARN_TIME(COMPACTION_PREFIX + "warn.time", "10m", PropertyType.TIMEDURATION,

‎core/src/main/java/org/apache/accumulo/core/fate/accumulo/AccumuloStore.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -361,8 +361,8 @@ public void pop() {
361361
verifyReserved(true);
362362

363363
Optional<Integer> top = findTop();
364-
top.ifPresent(
365-
t -> newMutator(fateId).requireStatus(TStatus.FAILED_IN_PROGRESS).deleteRepo(t).mutate());
364+
top.ifPresent(t -> newMutator(fateId)
365+
.requireStatus(TStatus.FAILED_IN_PROGRESS, TStatus.SUCCESSFUL).deleteRepo(t).mutate());
366366
}
367367

368368
@Override

‎core/src/main/java/org/apache/accumulo/core/fate/zookeeper/DistributedReadWriteLock.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
*/
4141
public class DistributedReadWriteLock implements java.util.concurrent.locks.ReadWriteLock {
4242

43-
public static enum LockType {
43+
public enum LockType {
4444
READ, WRITE,
4545
}
4646

‎core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java

+1-4
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,7 @@
4242
*/
4343
public class TabletManagement {
4444

45-
public static final EnumSet<ColumnType> CONFIGURED_COLUMNS =
46-
EnumSet.of(ColumnType.PREV_ROW, ColumnType.LOCATION, ColumnType.SUSPEND, ColumnType.LOGS,
47-
ColumnType.AVAILABILITY, ColumnType.HOSTING_REQUESTED, ColumnType.FILES, ColumnType.LAST,
48-
ColumnType.OPID, ColumnType.ECOMP, ColumnType.DIR, ColumnType.SELECTED);
45+
public static final EnumSet<ColumnType> CONFIGURED_COLUMNS = EnumSet.allOf(ColumnType.class);
4946

5047
private static final Text ERROR_COLUMN_NAME = new Text("ERROR");
5148
private static final Text REASONS_COLUMN_NAME = new Text("REASONS");

‎core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java

+4
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,10 @@ interface TabletUpdates<T> {
415415
T setMerged();
416416

417417
T deleteMerged();
418+
419+
T putUserCompactionRequested(FateId fateId);
420+
421+
T deleteUserCompactionRequested(FateId fateId);
418422
}
419423

420424
interface TabletMutator extends TabletUpdates<TabletMutator> {

‎core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java

+9
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,15 @@ public static class CompactedColumnFamily {
421421
public static final Text NAME = new Text(STR_NAME);
422422
}
423423

424+
/**
425+
* Column family for indicating that a user has requested to compact a tablet. The column
426+
* qualifier is expected to contain the fate transaction id that is executing the request.
427+
*/
428+
public static class UserCompactionRequestedColumnFamily {
429+
public static final String STR_NAME = "userRequestToCompact";
430+
public static final Text NAME = new Text(STR_NAME);
431+
}
432+
424433
// TODO when removing the Upgrader12to13 class in the upgrade package, also remove this class.
425434
public static class Upgrade12to13 {
426435

‎core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java

+18-5
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@
7272
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
7373
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
7474
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
75+
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.UserCompactionRequestedColumnFamily;
7576
import org.apache.accumulo.core.tabletserver.log.LogEntry;
7677
import org.apache.commons.lang3.builder.ToStringBuilder;
7778
import org.apache.commons.lang3.builder.ToStringStyle;
@@ -118,6 +119,7 @@ public class TabletMetadata {
118119
private TabletOperationId operationId;
119120
private boolean futureAndCurrentLocationSet = false;
120121
private Set<FateId> compacted;
122+
private Set<FateId> userCompactionsRequested;
121123

122124
public static TabletMetadataBuilder builder(KeyExtent extent) {
123125
return new TabletMetadataBuilder(extent);
@@ -147,7 +149,8 @@ public enum ColumnType {
147149
HOSTING_REQUESTED,
148150
OPID,
149151
SELECTED,
150-
COMPACTED
152+
COMPACTED,
153+
USER_COMPACTION_REQUESTED
151154
}
152155

153156
public static class Location {
@@ -358,6 +361,11 @@ public boolean hasMerged() {
358361
return merged;
359362
}
360363

364+
public Set<FateId> getUserCompactionsRequested() {
365+
ensureFetched(ColumnType.USER_COMPACTION_REQUESTED);
366+
return userCompactionsRequested;
367+
}
368+
361369
public TabletAvailability getTabletAvailability() {
362370
if (AccumuloTable.ROOT.tableId().equals(getTableId())
363371
|| AccumuloTable.METADATA.tableId().equals(getTableId())) {
@@ -385,7 +393,8 @@ public String toString() {
385393
.append("extCompactions", extCompactions).append("availability", availability)
386394
.append("onDemandHostingRequested", onDemandHostingRequested)
387395
.append("operationId", operationId).append("selectedFiles", selectedFiles)
388-
.append("futureAndCurrentLocationSet", futureAndCurrentLocationSet).toString();
396+
.append("futureAndCurrentLocationSet", futureAndCurrentLocationSet)
397+
.append("userCompactionsRequested", userCompactionsRequested).toString();
389398
}
390399

391400
public SortedMap<Key,Value> getKeyValues() {
@@ -431,6 +440,7 @@ public static <E extends Entry<Key,Value>> TabletMetadata convertRow(Iterator<E>
431440
final var extCompBuilder = ImmutableMap.<ExternalCompactionId,CompactionMetadata>builder();
432441
final var loadedFilesBuilder = ImmutableMap.<StoredTabletFile,FateId>builder();
433442
final var compactedBuilder = ImmutableSet.<FateId>builder();
443+
final var userCompactionsRequestedBuilder = ImmutableSet.<FateId>builder();
434444
ByteSequence row = null;
435445

436446
while (rowIter.hasNext()) {
@@ -488,7 +498,7 @@ public static <E extends Entry<Key,Value>> TabletMetadata convertRow(Iterator<E>
488498
te.flushNonce = OptionalLong.of(Long.parseUnsignedLong(val, 16));
489499
break;
490500
case OPID_QUAL:
491-
te.setOperationIdOnce(val, suppressLocationError);
501+
te.setOperationIdOnce(val);
492502
break;
493503
case SELECTED_QUAL:
494504
te.selectedFiles = SelectedFiles.from(val);
@@ -532,6 +542,9 @@ public static <E extends Entry<Key,Value>> TabletMetadata convertRow(Iterator<E>
532542
case CompactedColumnFamily.STR_NAME:
533543
compactedBuilder.add(FateId.from(qual));
534544
break;
545+
case UserCompactionRequestedColumnFamily.STR_NAME:
546+
userCompactionsRequestedBuilder.add(FateId.from(qual));
547+
break;
535548
default:
536549
throw new IllegalStateException("Unexpected family " + fam);
537550

@@ -551,6 +564,7 @@ public static <E extends Entry<Key,Value>> TabletMetadata convertRow(Iterator<E>
551564
te.logs = logsBuilder.build();
552565
te.extCompactions = extCompBuilder.build();
553566
te.compacted = compactedBuilder.build();
567+
te.userCompactionsRequested = userCompactionsRequestedBuilder.build();
554568
if (buildKeyValueMap) {
555569
te.keyValues = kvBuilder.build();
556570
}
@@ -581,10 +595,9 @@ private void setLocationOnce(String val, String qual, LocationType lt, boolean s
581595
* Sets an operation ID only once.
582596
*
583597
* @param val operation id to set
584-
* @param suppressError set to true to suppress an exception being thrown, else false
585598
* @throws IllegalStateException if an operation id or location is already set
586599
*/
587-
private void setOperationIdOnce(String val, boolean suppressError) {
600+
private void setOperationIdOnce(String val) {
588601
Preconditions.checkState(operationId == null);
589602
operationId = TabletOperationId.from(val);
590603
}

‎core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java

+13
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED;
3737
import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND;
3838
import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.TIME;
39+
import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.USER_COMPACTION_REQUESTED;
3940

4041
import java.util.Arrays;
4142
import java.util.EnumSet;
@@ -282,6 +283,18 @@ public TabletMetadataBuilder deleteMerged() {
282283
throw new UnsupportedOperationException();
283284
}
284285

286+
@Override
287+
public TabletMetadataBuilder putUserCompactionRequested(FateId fateId) {
288+
fetched.add(USER_COMPACTION_REQUESTED);
289+
internalBuilder.putUserCompactionRequested(fateId);
290+
return this;
291+
}
292+
293+
@Override
294+
public TabletMetadataBuilder deleteUserCompactionRequested(FateId fateId) {
295+
throw new UnsupportedOperationException();
296+
}
297+
285298
/**
286299
* @param extraFetched Anything that was put on the builder will automatically be added to the
287300
* fetched set. However, for the case where something was not put and it needs to be

‎core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java

+13
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
4747
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
4848
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
49+
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.UserCompactionRequestedColumnFamily;
4950
import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
5051
import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
5152
import org.apache.accumulo.core.tabletserver.log.LogEntry;
@@ -334,6 +335,18 @@ public T deleteMerged() {
334335
return getThis();
335336
}
336337

338+
@Override
339+
public T putUserCompactionRequested(FateId fateId) {
340+
mutation.put(UserCompactionRequestedColumnFamily.STR_NAME, fateId.canonical(), "");
341+
return getThis();
342+
}
343+
344+
@Override
345+
public T deleteUserCompactionRequested(FateId fateId) {
346+
mutation.putDelete(UserCompactionRequestedColumnFamily.STR_NAME, fateId.canonical());
347+
return getThis();
348+
}
349+
337350
public void setCloseAfterMutate(AutoCloseable closeable) {
338351
this.closeAfterMutate = closeable;
339352
}

‎core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java

+4
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@
8282
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily;
8383
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
8484
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
85+
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.UserCompactionRequestedColumnFamily;
8586
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
8687
import org.apache.accumulo.core.security.Authorizations;
8788
import org.apache.accumulo.core.util.ColumnFQ;
@@ -392,6 +393,9 @@ public Options fetch(ColumnType... colsToFetch) {
392393
case COMPACTED:
393394
families.add(CompactedColumnFamily.NAME);
394395
break;
396+
case USER_COMPACTION_REQUESTED:
397+
families.add(UserCompactionRequestedColumnFamily.NAME);
398+
break;
395399
default:
396400
throw new IllegalArgumentException("Unknown col type " + colToFetch);
397401
}

‎core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java

+7-7
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,8 @@
7272
* <th>Description</th>
7373
* </tr>
7474
* <tr>
75-
* <td>name</td>
76-
* <td>name of the compactor group (required)</td>
75+
* <td>group</td>
76+
* <td>name of the compactor resource group (required)</td>
7777
* </tr>
7878
* <tr>
7979
* <td>maxSize</td>
@@ -97,9 +97,9 @@
9797
* <pre>
9898
* {@code
9999
* [
100-
* {"name":"small", "maxSize":"100M"},
101-
* {"name":"medium", "maxSize":"500M"},
102-
* {"name": "large"}
100+
* {"group":"small", "maxSize":"100M"},
101+
* {"group":"medium", "maxSize":"500M"},
102+
* {"group": "large"}
103103
* ]}
104104
* </pre>
105105
* </ul>
@@ -127,7 +127,7 @@ public class DefaultCompactionPlanner implements CompactionPlanner {
127127
private final static Logger log = LoggerFactory.getLogger(DefaultCompactionPlanner.class);
128128

129129
private static class GroupConfig {
130-
String name;
130+
String group;
131131
String maxSize;
132132
}
133133

@@ -192,7 +192,7 @@ public void init(InitParameters params) {
192192
: ConfigurationTypeHelper.getFixedMemoryAsBytes(groupConfig.maxSize);
193193

194194
CompactorGroupId cgid;
195-
String group = Objects.requireNonNull(groupConfig.name, "'name' must be specified");
195+
String group = Objects.requireNonNull(groupConfig.group, "'group' must be specified");
196196
cgid = params.getGroupManager().getGroup(group);
197197
tmpGroups.add(new CompactionGroup(cgid, maxSize));
198198
}

0 commit comments

Comments
 (0)