Skip to content

Commit bed997a

Browse files
DomGarguiloDomGarguilo
committed
Merge remote-tracking branch 'upstream/main' into avoidZooCacheClearException
2 parents aad0082 + 55f05c6 commit bed997a

File tree

6 files changed

+145
-120
lines changed

6 files changed

+145
-120
lines changed

‎.github/workflows/maven.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ jobs:
4141
- name: Set up JDK 17
4242
uses: actions/setup-java@v4
4343
with:
44-
distribution: adopt
44+
distribution: temurin
4545
java-version: 17
4646
cache: 'maven'
4747
- name: Show the first log message
@@ -72,7 +72,7 @@ jobs:
7272
- name: Set up JDK ${{ matrix.profile.javaver }}
7373
uses: actions/setup-java@v4
7474
with:
75-
distribution: adopt
75+
distribution: temurin
7676
java-version: ${{ matrix.profile.javaver }}
7777
cache: 'maven'
7878
- name: Override DNS to fix IP address for hostname

‎core/src/main/java/org/apache/accumulo/core/cli/ConfigOpts.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,10 @@ public List<String> split(String value) {
5656

5757
@Parameter(names = "-o", splitter = NullSplitter.class,
5858
description = "Overrides configuration set in accumulo.properties (but NOT system-wide config"
59-
+ " set in Zookeeper). Expected format: -o <key>=<value>")
59+
+ " set in Zookeeper). This is useful when you have process specific configuration items"
60+
+ " that are one-offs from a shared common configuration. Setting the bind address,"
61+
+ " for example, can be done with the arguments \"-o general.process.bind.addr=127.0.0.1\"."
62+
+ " Expected format: -o <key>=<value> [-o <key>=<value>]")
6063
private List<String> overrides = new ArrayList<>();
6164

6265
private SiteConfiguration siteConfig = null;

‎core/src/test/java/org/apache/accumulo/core/cli/ConfigOptsTest.java

+9
Original file line numberDiff line numberDiff line change
@@ -56,4 +56,13 @@ public void testOverrideConfig() {
5656
assertEquals("test:123", opts.getSiteConfiguration().get(Property.INSTANCE_ZK_HOST));
5757
}
5858

59+
@Test
60+
public void testOverrideMultiple() {
61+
opts.parseArgs(ConfigOptsTest.class.getName(),
62+
new String[] {"-o", Property.GENERAL_PROCESS_BIND_ADDRESS.getKey() + "=1.2.3.4", "-o",
63+
Property.COMPACTOR_GROUP_NAME.getKey() + "=test"});
64+
assertEquals("1.2.3.4", opts.getSiteConfiguration().get(Property.GENERAL_PROCESS_BIND_ADDRESS));
65+
assertEquals("test", opts.getSiteConfiguration().get(Property.COMPACTOR_GROUP_NAME));
66+
}
67+
5968
}

‎server/manager/src/main/java/org/apache/accumulo/manager/tableOps/create/PopulateZookeeper.java

-3
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,10 @@
3030
import org.apache.accumulo.manager.tableOps.Utils;
3131
import org.apache.accumulo.server.conf.store.TablePropKey;
3232
import org.apache.accumulo.server.util.PropUtil;
33-
import org.slf4j.Logger;
34-
import org.slf4j.LoggerFactory;
3533

3634
class PopulateZookeeper extends ManagerRepo {
3735

3836
private static final long serialVersionUID = 1L;
39-
private static final Logger log = LoggerFactory.getLogger(PopulateZookeeper.class);
4037

4138
private final TableInfo tableInfo;
4239

‎test/src/main/java/org/apache/accumulo/test/ImportExportIT.java

+130-113
Original file line numberDiff line numberDiff line change
@@ -103,139 +103,156 @@ protected Duration defaultTimeout() {
103103
return Duration.ofMinutes(1);
104104
}
105105

106-
@ParameterizedTest
107-
@ValueSource(booleans = {true, false})
108-
public void testExportImportThenScan(boolean fenced) throws Exception {
109-
try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
106+
private void doExportImportThenScan(boolean fenced, AccumuloClient client, String srcTable,
107+
String destTable) throws Exception {
110108

111-
String[] tableNames = getUniqueNames(2);
112-
String srcTable = tableNames[0], destTable = tableNames[1];
113-
client.tableOperations().create(srcTable);
109+
client.tableOperations().create(srcTable);
114110

115-
try (BatchWriter bw = client.createBatchWriter(srcTable)) {
116-
for (int row = 0; row < 1000; row++) {
117-
Mutation m = new Mutation("row_" + String.format("%010d", row));
118-
for (int col = 0; col < 100; col++) {
119-
m.put(Integer.toString(col), "", Integer.toString(col * 2));
120-
}
121-
bw.addMutation(m);
111+
try (BatchWriter bw = client.createBatchWriter(srcTable)) {
112+
for (int row = 0; row < 1000; row++) {
113+
Mutation m = new Mutation("row_" + String.format("%010d", row));
114+
for (int col = 0; col < 100; col++) {
115+
m.put(Integer.toString(col), "", Integer.toString(col * 2));
122116
}
117+
bw.addMutation(m);
123118
}
119+
}
124120

125-
client.tableOperations().compact(srcTable, null, null, true, true);
126-
127-
int expected = 100000;
128-
// Test that files with ranges and are fenced work with export/import
129-
if (fenced) {
130-
// Split file into 3 ranges of 10000, 20000, and 5000 for a total of 35000
131-
FileMetadataUtil.splitFilesIntoRanges(getServerContext(), srcTable, createRanges());
132-
expected = 35000;
133-
}
134-
135-
// Make a directory we can use to throw the export and import directories
136-
// Must exist on the filesystem the cluster is running.
137-
FileSystem fs = cluster.getFileSystem();
138-
log.info("Using FileSystem: " + fs);
139-
Path baseDir = new Path(cluster.getTemporaryPath(), getClass().getName());
140-
fs.deleteOnExit(baseDir);
141-
if (fs.exists(baseDir)) {
142-
log.info("{} exists on filesystem, deleting", baseDir);
143-
assertTrue(fs.delete(baseDir, true), "Failed to deleted " + baseDir);
144-
}
145-
log.info("Creating {}", baseDir);
146-
assertTrue(fs.mkdirs(baseDir), "Failed to create " + baseDir);
147-
Path exportDir = new Path(baseDir, "export");
148-
fs.deleteOnExit(exportDir);
149-
Path importDirA = new Path(baseDir, "import-a");
150-
Path importDirB = new Path(baseDir, "import-b");
151-
fs.deleteOnExit(importDirA);
152-
fs.deleteOnExit(importDirB);
153-
for (Path p : new Path[] {exportDir, importDirA, importDirB}) {
154-
assertTrue(fs.mkdirs(p), "Failed to create " + p);
155-
}
156-
157-
Set<String> importDirs = Set.of(importDirA.toString(), importDirB.toString());
158-
159-
Path[] importDirAry = new Path[] {importDirA, importDirB};
121+
client.tableOperations().compact(srcTable, null, null, true, true);
160122

161-
log.info("Exporting table to {}", exportDir);
162-
log.info("Importing table from {}", importDirs);
123+
int expected = 100000;
124+
// Test that files with ranges and are fenced work with export/import
125+
if (fenced) {
126+
// Split file into 3 ranges of 10000, 20000, and 5000 for a total of 35000
127+
FileMetadataUtil.splitFilesIntoRanges(getServerContext(), srcTable, createRanges());
128+
expected = 35000;
129+
}
163130

164-
// test fast fail offline check
165-
assertThrows(IllegalStateException.class,
166-
() -> client.tableOperations().exportTable(srcTable, exportDir.toString()));
131+
// Make a directory we can use to throw the export and import directories
132+
// Must exist on the filesystem the cluster is running.
133+
FileSystem fs = cluster.getFileSystem();
134+
log.info("Using FileSystem: " + fs);
135+
Path baseDir = new Path(cluster.getTemporaryPath(), getClass().getName());
136+
fs.deleteOnExit(baseDir);
137+
if (fs.exists(baseDir)) {
138+
log.info("{} exists on filesystem, deleting", baseDir);
139+
assertTrue(fs.delete(baseDir, true), "Failed to deleted " + baseDir);
140+
}
141+
log.info("Creating {}", baseDir);
142+
assertTrue(fs.mkdirs(baseDir), "Failed to create " + baseDir);
143+
Path exportDir = new Path(baseDir, "export");
144+
fs.deleteOnExit(exportDir);
145+
Path importDirA = new Path(baseDir, "import-a");
146+
Path importDirB = new Path(baseDir, "import-b");
147+
fs.deleteOnExit(importDirA);
148+
fs.deleteOnExit(importDirB);
149+
for (Path p : new Path[] {exportDir, importDirA, importDirB}) {
150+
assertTrue(fs.mkdirs(p), "Failed to create " + p);
151+
}
167152

168-
// Offline the table
169-
client.tableOperations().offline(srcTable, true);
170-
// Then export it
171-
client.tableOperations().exportTable(srcTable, exportDir.toString());
153+
Set<String> importDirs = Set.of(importDirA.toString(), importDirB.toString());
172154

173-
// Make sure the distcp.txt file that exporttable creates is available
174-
Path distcp = new Path(exportDir, "distcp.txt");
175-
fs.deleteOnExit(distcp);
176-
assertTrue(fs.exists(distcp), "Distcp file doesn't exist");
177-
FSDataInputStream is = fs.open(distcp);
178-
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
155+
Path[] importDirAry = new Path[] {importDirA, importDirB};
179156

180-
// Copy each file that was exported to one of the imports directory
181-
String line;
157+
log.info("Exporting table to {}", exportDir);
158+
log.info("Importing table from {}", importDirs);
182159

183-
while ((line = reader.readLine()) != null) {
184-
Path p = new Path(line.substring(5));
185-
assertTrue(fs.exists(p), "File doesn't exist: " + p);
186-
Path importDir = importDirAry[RANDOM.get().nextInt(importDirAry.length)];
187-
Path dest = new Path(importDir, p.getName());
188-
assertFalse(fs.exists(dest), "Did not expect " + dest + " to exist");
189-
FileUtil.copy(fs, p, fs, dest, false, fs.getConf());
190-
}
160+
// test fast fail offline check
161+
assertThrows(IllegalStateException.class,
162+
() -> client.tableOperations().exportTable(srcTable, exportDir.toString()));
191163

192-
reader.close();
164+
// Offline the table
165+
client.tableOperations().offline(srcTable, true);
166+
// Then export it
167+
client.tableOperations().exportTable(srcTable, exportDir.toString());
193168

194-
log.info("Import dir A: {}", Arrays.toString(fs.listStatus(importDirA)));
195-
log.info("Import dir B: {}", Arrays.toString(fs.listStatus(importDirB)));
169+
// Make sure the distcp.txt file that exporttable creates is available
170+
Path distcp = new Path(exportDir, "distcp.txt");
171+
fs.deleteOnExit(distcp);
172+
assertTrue(fs.exists(distcp), "Distcp file doesn't exist");
173+
FSDataInputStream is = fs.open(distcp);
174+
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
196175

197-
// Import the exported data into a new table
198-
client.tableOperations().importTable(destTable, importDirs, ImportConfiguration.empty());
176+
// Copy each file that was exported to one of the imports directory
177+
String line;
199178

200-
// Get the table ID for the table that the importtable command created
201-
final String tableId = client.tableOperations().tableIdMap().get(destTable);
202-
assertNotNull(tableId);
179+
while ((line = reader.readLine()) != null) {
180+
Path p = new Path(line.substring(5));
181+
assertTrue(fs.exists(p), "File doesn't exist: " + p);
182+
Path importDir = importDirAry[RANDOM.get().nextInt(importDirAry.length)];
183+
Path dest = new Path(importDir, p.getName());
184+
assertFalse(fs.exists(dest), "Did not expect " + dest + " to exist");
185+
FileUtil.copy(fs, p, fs, dest, false, fs.getConf());
186+
}
203187

204-
// Get all `file` colfams from the metadata table for the new table
205-
log.info("Imported into table with ID: {}", tableId);
188+
reader.close();
189+
190+
log.info("Import dir A: {}", Arrays.toString(fs.listStatus(importDirA)));
191+
log.info("Import dir B: {}", Arrays.toString(fs.listStatus(importDirB)));
192+
193+
// Import the exported data into a new table
194+
client.tableOperations().importTable(destTable, importDirs, ImportConfiguration.empty());
195+
196+
// Get the table ID for the table that the importtable command created
197+
final String tableId = client.tableOperations().tableIdMap().get(destTable);
198+
assertNotNull(tableId);
199+
200+
// Get all `file` colfams from the metadata table for the new table
201+
log.info("Imported into table with ID: {}", tableId);
202+
203+
try (Scanner s =
204+
client.createScanner(AccumuloTable.METADATA.tableName(), Authorizations.EMPTY)) {
205+
s.setRange(TabletsSection.getRange(TableId.of(tableId)));
206+
s.fetchColumnFamily(DataFileColumnFamily.NAME);
207+
ServerColumnFamily.DIRECTORY_COLUMN.fetch(s);
208+
209+
// Should find a single entry
210+
for (Entry<Key,Value> fileEntry : s) {
211+
Key k = fileEntry.getKey();
212+
String value = fileEntry.getValue().toString();
213+
if (k.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
214+
// The file should be an absolute URI (file:///...), not a relative path
215+
// (/b-000.../I000001.rf)
216+
var tabFile = StoredTabletFile.of(k.getColumnQualifier());
217+
// Verify that the range is set correctly on the StoredTabletFile
218+
assertEquals(fenced,
219+
!tabFile.getRange().isInfiniteStartKey() || !tabFile.getRange().isInfiniteStopKey());
220+
assertFalse(looksLikeRelativePath(tabFile.getMetadataPath()),
221+
"Imported files should have absolute URIs, not relative: " + tabFile);
222+
} else if (k.getColumnFamily().equals(ServerColumnFamily.NAME)) {
223+
assertFalse(looksLikeRelativePath(value),
224+
"Server directory should have absolute URI, not relative: " + value);
225+
} else {
226+
fail("Got expected pair: " + k + "=" + fileEntry.getValue());
227+
}
228+
}
206229

207-
try (Scanner s =
208-
client.createScanner(AccumuloTable.METADATA.tableName(), Authorizations.EMPTY)) {
209-
s.setRange(TabletsSection.getRange(TableId.of(tableId)));
210-
s.fetchColumnFamily(DataFileColumnFamily.NAME);
211-
ServerColumnFamily.DIRECTORY_COLUMN.fetch(s);
230+
}
231+
// Online the original table before we verify equivalence
232+
client.tableOperations().online(srcTable, true);
212233

213-
// Should find a single entry
214-
for (Entry<Key,Value> fileEntry : s) {
215-
Key k = fileEntry.getKey();
216-
String value = fileEntry.getValue().toString();
217-
if (k.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
218-
// The file should be an absolute URI (file:///...), not a relative path
219-
// (/b-000.../I000001.rf)
220-
var tabFile = StoredTabletFile.of(k.getColumnQualifier());
221-
// Verify that the range is set correctly on the StoredTabletFile
222-
assertEquals(fenced, !tabFile.getRange().isInfiniteStartKey()
223-
|| !tabFile.getRange().isInfiniteStopKey());
224-
assertFalse(looksLikeRelativePath(tabFile.getMetadataPath()),
225-
"Imported files should have absolute URIs, not relative: " + tabFile);
226-
} else if (k.getColumnFamily().equals(ServerColumnFamily.NAME)) {
227-
assertFalse(looksLikeRelativePath(value),
228-
"Server directory should have absolute URI, not relative: " + value);
229-
} else {
230-
fail("Got expected pair: " + k + "=" + fileEntry.getValue());
231-
}
232-
}
234+
verifyTableEquality(client, srcTable, destTable, expected);
235+
}
233236

234-
}
235-
// Online the original table before we verify equivalence
236-
client.tableOperations().online(srcTable, true);
237+
@ParameterizedTest
238+
@ValueSource(booleans = {true, false})
239+
public void testExportImportThenScan(boolean fenced) throws Exception {
240+
try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
241+
String[] tableNames = getUniqueNames(2);
242+
doExportImportThenScan(fenced, client, tableNames[0], tableNames[1]);
243+
}
244+
}
237245

238-
verifyTableEquality(client, srcTable, destTable, expected);
246+
@ParameterizedTest
247+
@ValueSource(booleans = {true, false})
248+
public void testExportImportSameTableNameThenScan(boolean fenced) throws Exception {
249+
try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
250+
String ns1 = "namespace1";
251+
client.namespaceOperations().create(ns1);
252+
String ns2 = "namespace2";
253+
client.namespaceOperations().create(ns2);
254+
String tableName = getUniqueNames(1)[0];
255+
doExportImportThenScan(fenced, client, ns1 + "." + tableName, ns2 + "." + tableName);
239256
}
240257
}
241258

‎test/src/main/java/org/apache/accumulo/test/TableOperationsIT.java

-1
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,6 @@ public void createTableWithBadProperties()
228228
@Test
229229
public void createTablesWithSameNameInDifferentNamespace() throws Exception {
230230
TableOperations tableOps = accumuloClient.tableOperations();
231-
String[] names = getUniqueNames(2);
232231

233232
accumuloClient.namespaceOperations().create("test1");
234233
accumuloClient.namespaceOperations().create("test2");

0 commit comments

Comments
 (0)