tacks number of accesses to zoocache entry for logging (apache#5156)

Adds an access count to the trace logging when a zoocache entry is
removed because it was not used recently.  This will help find data that
is being stored in zoocache and accessed infrequently which may cause
uneedd watches.

Author: keith-turner

core/src/main/java/org/apache/accumulo/core/fate/zookeeper/ZcNode.java

@@ -20,6 +20,7 @@
 
 import java.util.List;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
 
 import com.google.common.base.Preconditions;
 
@@ -52,6 +53,8 @@ class ZcNode {
 
   static final ZcNode NON_EXISTENT = new ZcNode();
 
+  private final AtomicLong accessCount = new AtomicLong(0);
+
   private ZcNode() {
     this.data = null;
     this.stat = null;
@@ -96,6 +99,7 @@ private ZcNode() {
    */
   byte[] getData() {
     Preconditions.checkState(cachedData());
+    accessCount.incrementAndGet();
     return data;
   }
 
@@ -116,6 +120,7 @@ ZooCache.ZcStat getStat() {
    */
   List<String> getChildren() {
     Preconditions.checkState(cachedChildren());
+    accessCount.incrementAndGet();
     return children;
   }
 
@@ -139,4 +144,8 @@ boolean cachedData() {
   boolean notExists() {
     return stat == null && data == null && children == null;
   }
+
+  public long getAccessCount() {
+    return accessCount.get();
+  }
 }

core/src/main/java/org/apache/accumulo/core/fate/zookeeper/ZooCache.java

@@ -181,7 +181,8 @@ public ZooCache(ZooReader reader, Watcher watcher, Duration timeout) {
     this.externalWatcher = watcher;
     RemovalListener<String,ZcNode> removalListerner = (path, zcNode, reason) -> {
       try {
-        log.trace("{} removing watches for {} because {}", cacheId, path, reason);
+        log.trace("{} removing watches for {} because {} accesses {}", cacheId, path, reason,
+            zcNode == null ? -1 : zcNode.getAccessCount());
         reader.getZooKeeper().removeWatches(path, ZooCache.this.watcher, Watcher.WatcherType.Any,
             false);
       } catch (InterruptedException | KeeperException | RuntimeException e) {