Merge branch 'main' into accumulo-5160

Author: cshannon

core/src/main/java/org/apache/accumulo/core/file/blockfile/impl/CachableBlockFile.java

@@ -46,6 +46,8 @@
 
 import com.github.benmanes.caffeine.cache.Cache;
 
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
 /**
  * This is a wrapper class for BCFile that includes a cache for independent caches for datablocks
  * and metadatablocks
@@ -331,6 +333,8 @@ public Map<String,Loader> getDependencies() {
         return Collections.emptyMap();
       }
 
+      @SuppressFBWarnings(value = {"NP_LOAD_OF_KNOWN_NULL_VALUE"},
+          justification = "Spotbugs false positive, see spotbugs issue 2836.")
       @Override
       public byte[] load(int maxSize, Map<String,byte[]> dependencies) {
 
@@ -345,23 +349,18 @@ public byte[] load(int maxSize, Map<String,byte[]> dependencies) {
             }
           }
 
-          BlockReader _currBlock = getBlockReader(maxSize, reader);
-          if (_currBlock == null) {
-            return null;
-          }
+          try (BlockReader _currBlock = getBlockReader(maxSize, reader)) {
+            if (_currBlock == null) {
+              return null;
+            }
 
-          byte[] b = null;
-          try {
-            b = new byte[(int) _currBlock.getRawSize()];
+            byte[] b = new byte[(int) _currBlock.getRawSize()];
             _currBlock.readFully(b);
+            return b;
           } catch (IOException e) {
             log.debug("Error full blockRead for file " + cacheId + " for block " + getBlockId(), e);
             throw new UncheckedIOException(e);
-          } finally {
-            _currBlock.close();
           }
-
-          return b;
         } catch (IOException e) {
           throw new UncheckedIOException(e);
         }

hadoop-mapreduce/src/test/java/org/apache/accumulo/hadoop/its/mapreduce/AccumuloInputFormatIT.java

@@ -53,8 +53,7 @@
 import org.apache.accumulo.hadoop.mapreduce.InputFormatBuilder.InputFormatOptions;
 import org.apache.accumulo.hadoopImpl.mapreduce.BatchInputSplit;
 import org.apache.accumulo.hadoopImpl.mapreduce.RangeInputSplit;
-import org.apache.accumulo.harness.AccumuloClusterHarness;
-import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.io.Text;
@@ -66,7 +65,9 @@
 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -78,7 +79,7 @@
  *
  * @since 2.0
  */
-public class AccumuloInputFormatIT extends AccumuloClusterHarness {
+public class AccumuloInputFormatIT extends SharedMiniClusterBase {
 
   AccumuloInputFormat inputFormat;
   AccumuloClient client;
@@ -88,9 +89,15 @@ protected Duration defaultTimeout() {
     return Duration.ofMinutes(4);
   }
 
-  @Override
-  public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
-    cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1);
+  @BeforeAll
+  public static void setup() throws Exception {
+    SharedMiniClusterBase.startMiniClusterWithConfig(
+        (cfg, coreSite) -> cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1));
+  }
+
+  @AfterAll
+  public static void teardown() {
+    SharedMiniClusterBase.stopMiniCluster();
   }
 
   @BeforeEach

server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java

@@ -371,4 +371,11 @@ public void startServiceLockVerificationThread() {
   @Override
   public void close() {}
 
+  protected void waitForUpgrade() throws InterruptedException {
+    while (AccumuloDataVersion.getCurrentVersion(getContext()) < AccumuloDataVersion.get()) {
+      log.info("Waiting for upgrade to complete.");
+      Thread.sleep(1000);
+    }
+  }
+
 }

server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java

@@ -154,6 +154,14 @@ boolean inSafeMode() {
   @Override
   @SuppressFBWarnings(value = "DM_EXIT", justification = "main class can call System.exit")
   public void run() {
+
+    try {
+      waitForUpgrade();
+    } catch (InterruptedException e) {
+      log.error("Interrupted while waiting for upgrade to complete, exiting...");
+      System.exit(1);
+    }
+
     final VolumeManager fs = getContext().getVolumeManager();
 
     // Sleep for an initial period, giving the manager time to start up and

server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java

@@ -44,6 +44,7 @@
 import org.apache.accumulo.core.client.admin.TabletInformation;
 import org.apache.accumulo.core.client.admin.servers.ServerId;
 import org.apache.accumulo.core.compaction.thrift.TExternalCompaction;
+import org.apache.accumulo.core.data.TableId;
 import org.apache.accumulo.core.metrics.flatbuffers.FMetric;
 import org.apache.accumulo.core.process.thrift.MetricResponse;
 import org.apache.accumulo.monitor.Monitor;
@@ -313,31 +314,33 @@ public List<TExternalCompaction> getCompactions(@PathParam("group") String resou
   @GET
   @Path("tables")
   @Produces(MediaType.APPLICATION_JSON)
-  @Description("Returns a map of table name to table details")
-  public Map<String,TableSummary> getTables() {
+  @Description("Returns a map of TableId to table details")
+  public Map<TableId,TableSummary> getTables() {
     return monitor.getInformationFetcher().getSummary().getTables();
   }
 
   @GET
-  @Path("tables/{name}")
+  @Path("tables/{tableId}")
   @Produces(MediaType.APPLICATION_JSON)
-  @Description("Returns table details for the supplied table name")
-  public TableSummary getTable(@PathParam("name") String tableName) {
-    TableSummary ts = monitor.getInformationFetcher().getSummary().getTables().get(tableName);
+  @Description("Returns table details for the supplied TableId")
+  public TableSummary getTable(@PathParam("tableId") String tableId) {
+    TableSummary ts =
+        monitor.getInformationFetcher().getSummary().getTables().get(TableId.of(tableId));
     if (ts == null) {
-      throw new NotFoundException(tableName + " not found");
+      throw new NotFoundException(tableId + " not found");
     }
     return ts;
   }
 
   @GET
-  @Path("tables/{name}/tablets")
+  @Path("tables/{tableId}/tablets")
   @Produces(MediaType.APPLICATION_JSON)
   @Description("Returns tablet details for the supplied table name")
-  public List<TabletInformation> getTablets(@PathParam("name") String tableName) {
-    List<TabletInformation> ti = monitor.getInformationFetcher().getSummary().getTablets(tableName);
+  public List<TabletInformation> getTablets(@PathParam("tableId") String tableId) {
+    List<TabletInformation> ti =
+        monitor.getInformationFetcher().getSummary().getTablets(TableId.of(tableId));
     if (ti == null) {
-      throw new NotFoundException(tableName + " not found");
+      throw new NotFoundException(tableId + " not found");
     }
     return ti;
   }

server/monitor/src/main/java/org/apache/accumulo/monitor/next/InformationFetcher.java

@@ -43,6 +43,7 @@
 import org.apache.accumulo.core.compaction.thrift.TExternalCompactionList;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TableId;
 import org.apache.accumulo.core.process.thrift.MetricResponse;
 import org.apache.accumulo.core.process.thrift.ServerProcessService.Client;
 import org.apache.accumulo.core.rpc.ThriftUtil;
@@ -139,27 +140,28 @@ public void run() {
 
   private class TableInformationFetcher implements Runnable {
     private final ServerContext ctx;
-    private final String table;
+    private final TableId tableId;
     private final SystemInformation summary;
 
-    private TableInformationFetcher(ServerContext ctx, String tableName,
-        SystemInformation summary) {
+    private TableInformationFetcher(ServerContext ctx, TableId tableId, SystemInformation summary) {
       this.ctx = ctx;
-      this.table = tableName;
+      this.tableId = tableId;
       this.summary = summary;
     }
 
     @Override
     public void run() {
-      try (Stream<TabletInformation> tablets =
-          this.ctx.tableOperations().getTabletInformation(table, new Range())) {
-        tablets.forEach(t -> summary.processTabletInformation(table, t));
+      try {
+        final String tableName = ctx.getTableName(tableId);
+        try (Stream<TabletInformation> tablets =
+            this.ctx.tableOperations().getTabletInformation(tableName, new Range())) {
+          tablets.forEach(t -> summary.processTabletInformation(tableId, tableName, t));
+        }
       } catch (TableNotFoundException e) {
-        LOG.warn(
-            "TableNotFoundException thrown while trying to gather information for table: " + table,
-            e);
+        LOG.warn("TableNotFoundException thrown while trying to gather information for TableId: {}",
+            tableId, e);
       } catch (Exception e) {
-        LOG.warn("Interrupted while trying to gather information for table: {}", table, e);
+        LOG.warn("Interrupted while trying to gather information for TableId: {}", tableId, e);
       }
     }
   }
@@ -292,8 +294,8 @@ public void run() {
       futures.add(this.pool.submit(new CompactionListFetcher(summary)));
 
       // Fetch Tablet / Tablet information from the metadata table
-      for (String tName : this.ctx.tableOperations().list()) {
-        futures.add(this.pool.submit(new TableInformationFetcher(this.ctx, tName, summary)));
+      for (TableId tableId : this.ctx.getTableNameToIdMap().values()) {
+        futures.add(this.pool.submit(new TableInformationFetcher(this.ctx, tableId, summary)));
       }
 
       long monitorFetchTimeout =

server/monitor/src/main/java/org/apache/accumulo/monitor/next/SystemInformation.java

@@ -40,6 +40,7 @@
 import org.apache.accumulo.core.client.admin.servers.ServerId;
 import org.apache.accumulo.core.compaction.thrift.TExternalCompaction;
 import org.apache.accumulo.core.compaction.thrift.TExternalCompactionList;
+import org.apache.accumulo.core.data.TableId;
 import org.apache.accumulo.core.data.TabletId;
 import org.apache.accumulo.core.dataImpl.KeyExtent;
 import org.apache.accumulo.core.dataImpl.TabletIdImpl;
@@ -154,6 +155,11 @@ public static class TableSummary {
     private final AtomicLong totalHostedTablets = new AtomicLong();
     private final AtomicLong totalSuspendedTablets = new AtomicLong();
     private final AtomicLong totalUnassignedTablets = new AtomicLong();
+    private String tableName;
+
+    public TableSummary(String tableName) {
+      this.tableName = tableName;
+    }
 
     public long getTotalEntries() {
       return totalEntries.get();
@@ -207,6 +213,10 @@ public long getTotalUnassignedTablets() {
       return totalUnassignedTablets.get();
     }
 
+    public String getTableName() {
+      return tableName;
+    }
+
     public void addTablet(TabletInformation info) {
       totalEntries.addAndGet(info.getEstimatedEntries());
       totalSizeOnDisk.addAndGet(info.getEstimatedSize());
@@ -327,8 +337,8 @@ public Set<String> getNotRespondedHosts() {
       new AtomicReference<>();
 
   // Table Information
-  private final Map<String,TableSummary> tables = new ConcurrentHashMap<>();
-  private final Map<String,List<TabletInformation>> tablets = new ConcurrentHashMap<>();
+  private final Map<TableId,TableSummary> tables = new ConcurrentHashMap<>();
+  private final Map<TableId,List<TabletInformation>> tablets = new ConcurrentHashMap<>();
 
   // Deployment Overview
   private final Map<String,Map<String,ProcessSummary>> deployment = new ConcurrentHashMap<>();
@@ -460,11 +470,11 @@ public void processExternalCompactionList(Map<String,TExternalCompactionList> ru
     oldestCompactions.set(running);
   }
 
-  public void processTabletInformation(String tableName, TabletInformation info) {
+  public void processTabletInformation(TableId tableId, String tableName, TabletInformation info) {
     final SanitizedTabletInformation sti = new SanitizedTabletInformation(info);
-    tablets.computeIfAbsent(tableName, (t) -> Collections.synchronizedList(new ArrayList<>()))
+    tablets.computeIfAbsent(tableId, (t) -> Collections.synchronizedList(new ArrayList<>()))
         .add(sti);
-    tables.computeIfAbsent(tableName, (t) -> new TableSummary()).addTablet(sti);
+    tables.computeIfAbsent(tableId, (t) -> new TableSummary(tableName)).addTablet(sti);
     if (sti.getEstimatedEntries() == 0) {
       suggestions.add("Tablet " + sti.getTabletId().toString() + " (tid: "
           + sti.getTabletId().getTable() + ") may have zero entries and could be merged.");
@@ -582,12 +592,12 @@ public List<TExternalCompaction> getCompactions(String group) {
     return list.getCompactions();
   }
 
-  public Map<String,TableSummary> getTables() {
+  public Map<TableId,TableSummary> getTables() {
     return this.tables;
   }
 
-  public List<TabletInformation> getTablets(String table) {
-    return this.tablets.get(table);
+  public List<TabletInformation> getTablets(TableId tableId) {
+    return this.tablets.get(tableId);
   }
 
   public Map<String,Map<String,ProcessSummary>> getDeploymentOverview() {

server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java

@@ -279,24 +279,23 @@ public Map<String,Object> getTables() {
   /**
    * Returns participating tservers template
    *
-   * @param tableID Table ID for participating tservers
+   * @param tableId Table ID for participating tservers
    * @return Participating tservers model
    */
   @GET
-  @Path("tables/{tableID}")
+  @Path("tables/{tableId}")
   @Template(name = "/default.ftl")
   public Map<String,Object> getTables(
-      @PathParam("tableID") @NotNull @Pattern(regexp = ALPHA_NUM_REGEX_TABLE_ID) String tableID)
+      @PathParam("tableId") @NotNull @Pattern(regexp = ALPHA_NUM_REGEX_TABLE_ID) String tableId)
       throws TableNotFoundException {
-
-    String tableName = monitor.getContext().getTableName(TableId.of(tableID));
+    String tableName = monitor.getContext().getTableName(TableId.of(tableId));
 
     Map<String,Object> model = getModel();
     model.put("title", "Table Status");
 
     model.put("template", "table.ftl");
     model.put("js", "table.js");
-    model.put("tableID", tableID);
+    model.put("tableId", tableId);
     model.put("table", tableName);
 
     return model;

server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/functions.js

@@ -25,7 +25,7 @@
 // Suffixes for quantity
 var QUANTITY_SUFFIX = ['', 'K', 'M', 'B', 'T', 'e15', 'e18', 'e21'];
 // Suffixes for size
-var SIZE_SUFFIX = ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z'];
+var SIZE_SUFFIX = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB'];
 
 /**
  * Initializes Auto Refresh to false if it is not set,
@@ -103,14 +103,14 @@ function bigNumber(big, suffixes, base) {
 }
 
 /**
- * Converts a number to a size with suffix
+ * Converts a size in bytes to a human-readable string with appropriate units.
  *
- * @param {number} size Number to convert
- * @return {string} Number with suffix added
+ * @param {number} size - The size in bytes to be converted.
+ * @returns {string} The human-readable string representation of the size.
  */
 function bigNumberForSize(size) {
-  if (size === null) {
-    size = 0;
+  if (size === 0) {
+    return '0B';
   }
   return bigNumber(size, SIZE_SUFFIX, 1024);
 }