first steps of forecast (#330)

* forecast error ranges

* changes, fixing the dependence on centrality + tests

* random seed

* changes

Author: sudiptoguha

Java/core/src/main/java/com/amazon/randomcutforest/RandomCutForest.java

@@ -60,6 +60,7 @@
 import com.amazon.randomcutforest.returntypes.Neighbor;
 import com.amazon.randomcutforest.returntypes.OneSidedConvergingDiVectorAccumulator;
 import com.amazon.randomcutforest.returntypes.OneSidedConvergingDoubleAccumulator;
+import com.amazon.randomcutforest.returntypes.RangeVector;
 import com.amazon.randomcutforest.returntypes.SampleSummary;
 import com.amazon.randomcutforest.sampler.CompactSampler;
 import com.amazon.randomcutforest.sampler.IStreamSampler;
@@ -568,10 +569,11 @@ public float[] lastShingledPoint() {
 
     /**
      *
-     * @return the sequence index of the last known shingled point
+     * @return the sequence index of the last known shingled point. If internal
+     *         shingling is not enabled, then this would correspond to the number of
+     *         updates
      */
     public long nextSequenceIndex() {
-        checkArgument(internalShinglingEnabled, "incorrect use");
         return stateCoordinator.getStore().getNextSequenceIndex();
     }
 
@@ -1001,7 +1003,8 @@ public List<ConditionalTreeSample> getConditionalField(float[] point, int number
 
         int[] liftedIndices = transformIndices(missingIndexes, point.length);
         IMultiVisitorFactory<ConditionalTreeSample> visitorFactory = (tree, y) -> new ImputeVisitor(y,
-                tree.projectToTree(y), liftedIndices, tree.projectMissingIndices(liftedIndices), centrality);
+                tree.projectToTree(y), liftedIndices, tree.projectMissingIndices(liftedIndices), centrality,
+                tree.getRandomSeed());
         return traverseForestMulti(transformToShingledPoint(point), visitorFactory, ConditionalTreeSample.collector);
     }
 
@@ -1072,25 +1075,41 @@ public double[] extrapolateBasic(double[] point, int horizon, int blockSize, boo
     }
 
     public float[] extrapolateBasic(float[] point, int horizon, int blockSize, boolean cyclic, int shingleIndex) {
+        return extrapolateWithRanges(point, horizon, blockSize, cyclic, shingleIndex, 1.0).values;
+    }
+
+    // the following is provided for maximum flexibilty from the calling entity;
+    // but likely use is extrapolateFromShingle(), which abstracts away rotation
+    // etc.
+    public RangeVector extrapolateWithRanges(float[] point, int horizon, int blockSize, boolean cyclic,
+            int shingleIndex, double centrality) {
         checkArgument(0 < blockSize && blockSize < dimensions,
                 "blockSize must be between 0 and dimensions (exclusive)");
         checkArgument(dimensions % blockSize == 0, "dimensions must be evenly divisible by blockSize");
         checkArgument(0 <= shingleIndex && shingleIndex < dimensions / blockSize,
                 "shingleIndex must be between 0 (inclusive) and dimensions / blockSize");
 
-        float[] result = new float[blockSize * horizon];
+        RangeVector result = new RangeVector(blockSize * horizon);
         int[] missingIndexes = new int[blockSize];
         float[] queryPoint = Arrays.copyOf(point, dimensions);
 
         if (cyclic) {
-            extrapolateBasicCyclic(result, horizon, blockSize, shingleIndex, queryPoint, missingIndexes);
+            extrapolateBasicCyclic(result, horizon, blockSize, shingleIndex, queryPoint, missingIndexes, centrality);
         } else {
-            extrapolateBasicSliding(result, horizon, blockSize, queryPoint, missingIndexes);
+            extrapolateBasicSliding(result, horizon, blockSize, queryPoint, missingIndexes, centrality);
         }
 
         return result;
     }
 
+    // external management of shingle; can function for both internal and external
+    // shingling
+    // however blocksize has to be externally managed
+    public RangeVector extrapolateFromShingle(float[] shingle, int horizon, int blockSize, double centrality) {
+        return extrapolateWithRanges(shingle, horizon, blockSize, isRotationEnabled(),
+                ((int) nextSequenceIndex()) % shingleSize, centrality);
+    }
+
     /**
      * Given an initial shingled point, extrapolate the stream into the future to
      * produce a forecast. This method is intended to be called when the input data
@@ -1130,7 +1149,8 @@ public double[] extrapolateBasic(ShingleBuilder builder, int horizon) {
                 builder.getShingleIndex());
     }
 
-    void extrapolateBasicSliding(float[] result, int horizon, int blockSize, float[] queryPoint, int[] missingIndexes) {
+    void extrapolateBasicSliding(RangeVector result, int horizon, int blockSize, float[] queryPoint,
+            int[] missingIndexes, double centrality) {
         int resultIndex = 0;
 
         Arrays.fill(missingIndexes, 0);
@@ -1142,16 +1162,20 @@ void extrapolateBasicSliding(float[] result, int horizon, int blockSize, float[]
             // shift all entries in the query point left by 1 block
             System.arraycopy(queryPoint, blockSize, queryPoint, 0, dimensions - blockSize);
 
-            float[] imputedPoint = imputeMissingValues(queryPoint, blockSize, missingIndexes);
+            SampleSummary imputedSummary = getConditionalFieldSummary(queryPoint, blockSize, missingIndexes,
+                    centrality);
             for (int y = 0; y < blockSize; y++) {
-                result[resultIndex++] = queryPoint[dimensions - blockSize + y] = imputedPoint[dimensions - blockSize
-                        + y];
+                result.values[resultIndex] = queryPoint[dimensions - blockSize + y] = imputedSummary.median[dimensions
+                        - blockSize + y];
+                result.lower[resultIndex] = imputedSummary.lower[dimensions - blockSize + y];
+                result.upper[resultIndex] = imputedSummary.upper[dimensions - blockSize + y];
+                resultIndex++;
             }
         }
     }
 
-    void extrapolateBasicCyclic(float[] result, int horizon, int blockSize, int shingleIndex, float[] queryPoint,
-            int[] missingIndexes) {
+    void extrapolateBasicCyclic(RangeVector result, int horizon, int blockSize, int shingleIndex, float[] queryPoint,
+            int[] missingIndexes, double centrality) {
 
         int resultIndex = 0;
         int currentPosition = shingleIndex;
@@ -1162,11 +1186,15 @@ void extrapolateBasicCyclic(float[] result, int horizon, int blockSize, int shin
                 missingIndexes[y] = (currentPosition + y) % dimensions;
             }
 
-            float[] imputedPoint = imputeMissingValues(queryPoint, blockSize, missingIndexes);
+            SampleSummary imputedSummary = getConditionalFieldSummary(queryPoint, blockSize, missingIndexes,
+                    centrality);
 
             for (int y = 0; y < blockSize; y++) {
-                result[resultIndex++] = queryPoint[(currentPosition + y)
-                        % dimensions] = imputedPoint[(currentPosition + y) % dimensions];
+                result.values[resultIndex] = queryPoint[(currentPosition + y)
+                        % dimensions] = imputedSummary.median[(currentPosition + y) % dimensions];
+                result.lower[resultIndex] = imputedSummary.lower[(currentPosition + y) % dimensions];
+                result.upper[resultIndex] = imputedSummary.upper[(currentPosition + y) % dimensions];
+                resultIndex++;
             }
 
             currentPosition = (currentPosition + blockSize) % dimensions;

Java/core/src/main/java/com/amazon/randomcutforest/imputation/ImputeVisitor.java

@@ -18,6 +18,7 @@
 import static com.amazon.randomcutforest.CommonUtils.checkArgument;
 
 import java.util.Arrays;
+import java.util.Random;
 
 import com.amazon.randomcutforest.CommonUtils;
 import com.amazon.randomcutforest.MultiVisitor;
@@ -66,6 +67,10 @@ public class ImputeVisitor implements MultiVisitor<ConditionalTreeSample> {
      */
     protected double centrality;
 
+    protected long randomSeed;
+
+    protected double randomRank;
+
     protected boolean converged;
 
     protected int pointIndex;
@@ -84,10 +89,11 @@ public class ImputeVisitor implements MultiVisitor<ConditionalTreeSample> {
      *                             space
      */
     public ImputeVisitor(float[] liftedPoint, float[] queryPoint, int[] liftedMissingIndexes, int[] missingIndexes,
-            double centrality) {
+            double centrality, long randomSeed) {
         this.queryPoint = Arrays.copyOf(queryPoint, queryPoint.length);
         this.missing = new boolean[queryPoint.length];
         this.centrality = centrality;
+        this.randomSeed = randomSeed;
         this.dimensionsUsed = new int[queryPoint.length];
 
         if (missingIndexes == null) {
@@ -108,7 +114,7 @@ public ImputeVisitor(float[] liftedPoint, float[] queryPoint, int[] liftedMissin
     public ImputeVisitor(float[] queryPoint, int numberOfMissingIndices, int[] missingIndexes) {
         this(queryPoint, Arrays.copyOf(queryPoint, queryPoint.length),
                 Arrays.copyOf(missingIndexes, Math.min(numberOfMissingIndices, missingIndexes.length)),
-                Arrays.copyOf(missingIndexes, Math.min(numberOfMissingIndices, missingIndexes.length)), 1.0);
+                Arrays.copyOf(missingIndexes, Math.min(numberOfMissingIndices, missingIndexes.length)), 1.0, 0L);
     }
 
     /**
@@ -121,6 +127,8 @@ public ImputeVisitor(float[] queryPoint, int numberOfMissingIndices, int[] missi
         this.queryPoint = Arrays.copyOf(original.queryPoint, length);
         this.missing = Arrays.copyOf(original.missing, length);
         this.dimensionsUsed = new int[original.dimensionsUsed.length];
+        this.randomSeed = new Random(original.randomSeed).nextLong();
+        this.centrality = original.centrality;
         anomalyRank = DEFAULT_INIT_VALUE;
         distance = DEFAULT_INIT_VALUE;
     }
@@ -174,6 +182,12 @@ public void acceptLeaf(final INodeView leafNode, final int depthOfNode) {
             }
         }
 
+        if (centrality < 1.0) {
+            Random rng = new Random(randomSeed);
+            randomSeed = rng.nextLong();
+            randomRank = rng.nextDouble();
+        }
+
         this.distance = distance;
         if (distance <= 0) {
             converged = true;
@@ -226,8 +240,12 @@ public MultiVisitor<ConditionalTreeSample> newCopy() {
         return new ImputeVisitor(this);
     }
 
+    double adjustedRank() {
+        return (1 - centrality) * randomRank + centrality * anomalyRank;
+    }
+
     protected boolean updateCombine(ImputeVisitor other) {
-        return other.anomalyRank < anomalyRank;
+        return other.adjustedRank() < adjustedRank();
     }
 
     /**

Java/core/src/main/java/com/amazon/randomcutforest/returntypes/RangeVector.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.amazon.randomcutforest.returntypes;
+
+import static com.amazon.randomcutforest.CommonUtils.checkArgument;
+
+import java.util.Arrays;
+
+/**
+ * A RangeVector is used when we want to track a quantity and its upper and
+ * lower bounds
+ */
+public class RangeVector {
+
+    public final float[] values;
+
+    /**
+     * An array of values corresponding to the upper ranges in each dimension.
+     */
+    public final float[] upper;
+    /**
+     * An array of values corresponding to the lower ranges in each dimension
+     */
+    public final float[] lower;
+
+    public RangeVector(int dimensions) {
+        checkArgument(dimensions > 0, "dimensions must be greater than 0");
+        values = new float[dimensions];
+        upper = new float[dimensions];
+        lower = new float[dimensions];
+    }
+
+    /**
+     * Construct a new RangeVector with the given number of spatial dimensions.
+     * 
+     * @param values the values being estimated in a range
+     * @param upper  the higher values of the ranges
+     * @param lower  the lower values in the ranges
+     */
+    public RangeVector(float[] values, float[] upper, float[] lower) {
+        checkArgument(values.length > 0, " dimensions must be > 0");
+        checkArgument(values.length == upper.length && upper.length == lower.length, "dimensions must be equal");
+        this.values = Arrays.copyOf(values, values.length);
+        this.upper = Arrays.copyOf(upper, upper.length);
+        this.lower = Arrays.copyOf(lower, lower.length);
+    }
+
+    public RangeVector(float[] values) {
+        checkArgument(values.length > 0, "dimensions must be > 0 ");
+        this.values = Arrays.copyOf(values, values.length);
+        this.upper = Arrays.copyOf(values, values.length);
+        this.lower = Arrays.copyOf(values, values.length);
+    }
+
+    /**
+     * Create a deep copy of the base RangeVector.
+     *
+     * @param base The RangeVector to copy.
+     */
+    public RangeVector(RangeVector base) {
+        int dimensions = base.values.length;
+        this.values = Arrays.copyOf(base.values, dimensions);
+        this.upper = Arrays.copyOf(base.upper, dimensions);
+        this.lower = Arrays.copyOf(base.lower, dimensions);
+    }
+}