Skip to content

Commit 8dd9435

Browse files
DomGarguiloDomGarguilo
committed
Merge remote-tracking branch 'upstream/elasticity' into importWithHostingGoal
# Conflicts: # server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportedTableInfo.java # server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/PopulateMetadataTable.java # test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
2 parents 1430a7e + 0e82f76 commit 8dd9435

File tree

23 files changed

+343
-53
lines changed

23 files changed

+343
-53
lines changed

‎core/pom.xml

-2
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,6 @@
198198
<exclude>src/main/java/org/apache/accumulo/core/bloomfilter/*.java</exclude>
199199
<exclude>src/main/java/org/apache/accumulo/core/util/HostAndPort.java</exclude>
200200
<exclude>src/test/resources/*.jceks</exclude>
201-
<exclude>src/test/resources/org/apache/accumulo/core/file/rfile/*.rf</exclude>
202201
</excludes>
203202
</licenseSet>
204203
</licenseSets>
@@ -211,7 +210,6 @@
211210
<excludes>
212211
<exclude>src/main/java/org/apache/accumulo/core/bloomfilter/*.java</exclude>
213212
<exclude>src/test/resources/*.jceks</exclude>
214-
<exclude>src/test/resources/org/apache/accumulo/core/file/rfile/*.rf</exclude>
215213
</excludes>
216214
</configuration>
217215
</plugin>

‎core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java

+15
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@
3333
import org.apache.hadoop.fs.Path;
3434
import org.apache.hadoop.io.DataInputBuffer;
3535
import org.apache.hadoop.io.Text;
36+
import org.checkerframework.checker.nullness.qual.NonNull;
3637

38+
import com.google.common.annotations.VisibleForTesting;
3739
import com.google.gson.Gson;
3840

3941
/**
@@ -263,6 +265,19 @@ private static Text decodeRow(byte[] serialized) {
263265
}
264266
}
265267

268+
/**
269+
* Quick validation to see if value has been converted by checking if the candidate looks like
270+
* json by checking the candidate starts with "{" and ends with "}".
271+
*
272+
* @param candidate a possible file: reference.
273+
* @return false if a likely a json object, true if not a likely json object
274+
*/
275+
@VisibleForTesting
276+
public static boolean fileNeedsConversion(@NonNull final String candidate) {
277+
String trimmed = candidate.trim();
278+
return !trimmed.startsWith("{") || !trimmed.endsWith("}");
279+
}
280+
266281
private static class TabletFileCq {
267282
public final Path path;
268283
public final Range range;

‎core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java

+2
Original file line numberDiff line numberDiff line change
@@ -459,6 +459,8 @@ public static <E extends Entry<Key,Value>> TabletMetadata convertRow(Iterator<E>
459459
case REQUESTED_QUAL:
460460
te.onDemandHostingRequested = true;
461461
break;
462+
default:
463+
throw new IllegalStateException("Unexpected TabletColumnFamily qualifier: " + qual);
462464
}
463465
break;
464466
case ServerColumnFamily.STR_NAME:

‎core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* https://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.accumulo.core.metadata;
20+
21+
import static org.junit.jupiter.api.Assertions.assertFalse;
22+
import static org.junit.jupiter.api.Assertions.assertTrue;
23+
24+
import org.junit.jupiter.api.Test;
25+
26+
public class StoredTabletFileTest {
27+
28+
@Test
29+
public void fileConversionTest() {
30+
String s21 = "hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf";
31+
String s31 =
32+
"{\"path\":\"hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf\",\"startRow\":\"\",\"endRow\":\"\"}";
33+
String s31_untrimmed =
34+
" { \"path\":\"hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf\",\"startRow\":\"\",\"endRow\":\"\" } ";
35+
36+
assertTrue(StoredTabletFile.fileNeedsConversion(s21));
37+
assertFalse(StoredTabletFile.fileNeedsConversion(s31));
38+
assertFalse(StoredTabletFile.fileNeedsConversion(s31_untrimmed));
39+
}
40+
}

‎pom.xml

+2
Original file line numberDiff line numberDiff line change
@@ -655,6 +655,7 @@
655655
<exclude>**/LICENSE</exclude>
656656
<exclude>**/NOTICE</exclude>
657657
<exclude>**/target/**</exclude>
658+
<exclude>**/*.rf</exclude>
658659
</excludes>
659660
</licenseSet>
660661
</licenseSets>
@@ -860,6 +861,7 @@
860861
<exclude>.vscode/**</exclude>
861862
<exclude>.factorypath</exclude>
862863
<exclude>.github/**</exclude>
864+
<exclude>**/*.rf</exclude>
863865
</excludes>
864866
</configuration>
865867
</plugin>

‎server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableExport/ExportTable.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,15 @@ public void undo(long tid, Manager env) throws Exception {
5454
Utils.unreserveHdfsDirectory(env, new Path(tableInfo.exportDir).toString(), tid);
5555
}
5656

57-
public static final int VERSION = 1;
57+
/**
58+
* Defines export / version.
59+
* <ul>
60+
* <li>version 1 exported by Accumulo &lt; 3.1</li>
61+
* <li>version 2 exported by Accumulo =&gt; 3.1 - uses file references with ranges.</li>
62+
* </ul>
63+
*/
64+
public static final int VERSION_2 = 2;
65+
public static final int CURR_VERSION = VERSION_2;
5866

5967
public static final String DATA_VERSION_PROP = "srcDataVersion";
6068
public static final String EXPORT_VERSION_PROP = "exportVersion";

‎server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableExport/WriteExportFiles.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ public static void exportTable(VolumeManager fs, ServerContext context, String t
168168
try (OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8)) {
169169

170170
zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_INFO_FILE));
171-
osw.append(ExportTable.EXPORT_VERSION_PROP + ":" + ExportTable.VERSION + "\n");
171+
osw.append(ExportTable.EXPORT_VERSION_PROP + ":" + ExportTable.CURR_VERSION + "\n");
172172
osw.append("srcInstanceName:" + context.getInstanceName() + "\n");
173173
osw.append("srcInstanceID:" + context.getInstanceID() + "\n");
174174
osw.append("srcZookeepers:" + context.getZooKeepers() + "\n");

‎server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportTable.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ public void checkVersions(Manager env) throws AcceptableThriftTableOperationExce
115115

116116
log.debug("Searching for export file in {}", exportDirs);
117117

118-
Integer exportVersion = null;
118+
tableInfo.exportedVersion = null;
119119
Integer dataVersion = null;
120120

121121
try {
@@ -132,7 +132,7 @@ public void checkVersions(Manager env) throws AcceptableThriftTableOperationExce
132132
while ((line = in.readLine()) != null) {
133133
String[] sa = line.split(":", 2);
134134
if (sa[0].equals(ExportTable.EXPORT_VERSION_PROP)) {
135-
exportVersion = Integer.parseInt(sa[1]);
135+
tableInfo.exportedVersion = Integer.parseInt(sa[1]);
136136
} else if (sa[0].equals(ExportTable.DATA_VERSION_PROP)) {
137137
dataVersion = Integer.parseInt(sa[1]);
138138
}
@@ -147,10 +147,10 @@ public void checkVersions(Manager env) throws AcceptableThriftTableOperationExce
147147
"Failed to read export metadata " + e.getMessage());
148148
}
149149

150-
if (exportVersion == null || exportVersion > ExportTable.VERSION) {
150+
if (tableInfo.exportedVersion == null || tableInfo.exportedVersion > ExportTable.CURR_VERSION) {
151151
throw new AcceptableThriftTableOperationException(null, tableInfo.tableName,
152152
TableOperation.IMPORT, TableOperationExceptionType.OTHER,
153-
"Incompatible export version " + exportVersion);
153+
"Incompatible export version " + tableInfo.exportedVersion);
154154
}
155155

156156
if (dataVersion == null || dataVersion > AccumuloDataVersion.get()) {

‎server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportedTableInfo.java

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ class ImportedTableInfo implements Serializable {
3737
public String exportFile;
3838
public boolean keepMappings;
3939
public boolean keepOffline;
40+
public Integer exportedVersion = null;
4041
public TabletAvailability initialAvailability = TabletAvailability.ONDEMAND;
4142

4243
static class DirectoryMapping implements Serializable {

‎server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/PopulateMetadataTable.java

+13-3
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import static java.nio.charset.StandardCharsets.UTF_8;
2222
import static org.apache.accumulo.core.Constants.IMPORT_MAPPINGS_FILE;
2323
import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.AVAILABILITY_COLUMN;
24+
import static org.apache.accumulo.manager.tableOps.tableExport.ExportTable.VERSION_2;
2425

2526
import java.io.BufferedInputStream;
2627
import java.io.BufferedReader;
@@ -146,8 +147,17 @@ public Repo<Manager> call(long tid, Manager manager) throws Exception {
146147
Text cq;
147148

148149
if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
149-
final StoredTabletFile oldTabletFile = StoredTabletFile.of(key.getColumnQualifier());
150-
String oldName = oldTabletFile.getFileName();
150+
StoredTabletFile exportedRef;
151+
var dataFileCQ = key.getColumnQualifier().toString();
152+
if (tableInfo.exportedVersion == null || tableInfo.exportedVersion < VERSION_2) {
153+
// written without fenced range information (accumulo < 3.1), use default
154+
// (null,null)
155+
exportedRef = StoredTabletFile.of(new Path(dataFileCQ));
156+
} else {
157+
exportedRef = StoredTabletFile.of(key.getColumnQualifier());
158+
}
159+
160+
String oldName = exportedRef.getFileName();
151161
String newName = fileNameMappings.get(oldName);
152162

153163
if (newName == null) {
@@ -157,7 +167,7 @@ public Repo<Manager> call(long tid, Manager manager) throws Exception {
157167
}
158168

159169
// Copy over the range for the new file
160-
cq = StoredTabletFile.of(URI.create(newName), oldTabletFile.getRange())
170+
cq = StoredTabletFile.of(URI.create(newName), exportedRef.getRange())
161171
.getMetadataText();
162172
} else {
163173
cq = key.getColumnQualifier();

‎server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader11to12.java

+4-16
Original file line numberDiff line numberDiff line change
@@ -197,28 +197,16 @@ void processReferences(MutationWriter batchWriter, Iterable<Map.Entry<Key,Value>
197197
}
198198

199199
@VisibleForTesting
200-
void upgradeDataFileCF(final Key key, final Value value, final Mutation m) {
200+
static void upgradeDataFileCF(final Key key, final Value value, final Mutation m) {
201201
String file = key.getColumnQualifier().toString();
202202
// filter out references if they are in the correct format already.
203-
if (fileNeedsConversion(file)) {
203+
boolean needsConversion = StoredTabletFile.fileNeedsConversion(file);
204+
log.trace("file: {} needs conversion: {}", file, needsConversion);
205+
if (needsConversion) {
204206
var fileJson = StoredTabletFile.of(new Path(file)).getMetadataText();
205207
m.at().family(DataFileColumnFamily.STR_NAME).qualifier(fileJson).put(value);
206208
m.at().family(DataFileColumnFamily.STR_NAME).qualifier(file).delete();
207209
}
208210
}
209211

210-
/**
211-
* Quick validation to see if value has been converted by checking if the candidate looks like
212-
* json by checking the candidate starts with "{" and ends with "}".
213-
*
214-
* @param candidate a possible file: reference.
215-
* @return false if a likely a json object, true if not a likely json object
216-
*/
217-
@VisibleForTesting
218-
boolean fileNeedsConversion(@NonNull final String candidate) {
219-
String trimmed = candidate.trim();
220-
boolean needsConversion = !trimmed.startsWith("{") || !trimmed.endsWith("}");
221-
log.trace("file: {} needs conversion: {}", candidate, needsConversion);
222-
return needsConversion;
223-
}
224212
}

‎server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java

+4-21
Original file line numberDiff line numberDiff line change
@@ -71,15 +71,13 @@ public class Upgrader11to12Test {
7171

7272
@Test
7373
void upgradeDataFileCF2Test() {
74-
Upgrader11to12 upgrader = new Upgrader11to12();
75-
7674
String fileName = "hdfs://localhost:8020/accumulo/tables/12/default_tablet/A000000v.rf";
7775
Key k = Key.builder().row(new Text("12;")).family(DataFileColumnFamily.NAME)
7876
.qualifier(new Text(fileName)).build();
7977
Value v = new Value("1234,5678");
8078

8179
Mutation upgrade = new Mutation(k.getRow());
82-
upgrader.upgradeDataFileCF(k, v, upgrade);
80+
Upgrader11to12.upgradeDataFileCF(k, v, upgrade);
8381

8482
var pending = upgrade.getUpdates();
8583
assertEquals(2, pending.size());
@@ -105,7 +103,7 @@ void upgradeDataFileCF2Test() {
105103
}
106104

107105
@Test
108-
public void processReferencesTest() throws Exception {
106+
public void processReferencesTest() {
109107

110108
// create sample data "served" by the mocked scanner
111109
TreeMap<Key,Value> scanData = new TreeMap<>();
@@ -161,7 +159,7 @@ public void processReferencesTest() throws Exception {
161159
}
162160

163161
@Test
164-
public void skipConvertedFileTest() throws Exception {
162+
public void skipConvertedFileTest() {
165163
// create sample data "served" by the mocked scanner
166164
TreeMap<Key,Value> scanData = new TreeMap<>();
167165
Text row1 = new Text("123");
@@ -288,7 +286,7 @@ void unexpectedColFailsTest() throws Exception {
288286
* called for those rows
289287
*/
290288
@Test
291-
public void verifyEmptyMutation() throws Exception {
289+
public void verifyEmptyMutation() {
292290
// create sample data "served" by the mocked scanner
293291
TreeMap<Key,Value> scanData = new TreeMap<>();
294292

@@ -377,21 +375,6 @@ public void upgradeZooKeeperTest() throws Exception {
377375
verify(context, zrw);
378376
}
379377

380-
@Test
381-
public void fileConversionTest() {
382-
String s21 = "hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf";
383-
String s31 =
384-
"{\"path\":\"hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf\",\"startRow\":\"\",\"endRow\":\"\"}";
385-
String s31_untrimmed =
386-
" { \"path\":\"hdfs://localhost:8020/accumulo/tables/1/t-0000000/A000003v.rf\",\"startRow\":\"\",\"endRow\":\"\" } ";
387-
388-
Upgrader11to12 upgrader = new Upgrader11to12();
389-
390-
assertTrue(upgrader.fileNeedsConversion(s21));
391-
assertFalse(upgrader.fileNeedsConversion(s31));
392-
assertFalse(upgrader.fileNeedsConversion(s31_untrimmed));
393-
}
394-
395378
@Test
396379
public void convertRoot1File() {
397380
String root21ZkData =

‎test/pom.xml

+17
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,23 @@
232232
<mapping>
233233
<shellit>SCRIPT_STYLE</shellit>
234234
</mapping>
235+
<licenseSets>
236+
<licenseSet>
237+
<header>${rootlocation}/src/build/license-header.txt</header>
238+
<excludes>
239+
<exclude>src/main/resources/v2_import_test/data/distcp.txt</exclude>
240+
</excludes>
241+
</licenseSet>
242+
</licenseSets>
243+
</configuration>
244+
</plugin>
245+
<plugin>
246+
<groupId>org.apache.rat</groupId>
247+
<artifactId>apache-rat-plugin</artifactId>
248+
<configuration>
249+
<excludes>
250+
<exclude>src/main/resources/v2_import_test/data/distcp.txt</exclude>
251+
</excludes>
235252
</configuration>
236253
</plugin>
237254
<plugin>

0 commit comments

Comments
 (0)