Version 1.6.2 of the Amazon Kinesis Client Library

META-INF/MANIFEST.MF

@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: Amazon Kinesis Client Library for Java
 Bundle-SymbolicName: com.amazonaws.kinesisclientlibrary;singleton:=true
-Bundle-Version: 1.6.1
+Bundle-Version: 1.6.2
 Bundle-Vendor: Amazon Technologies, Inc
 Bundle-RequiredExecutionEnvironment: JavaSE-1.7
 Require-Bundle: org.apache.commons.codec;bundle-version="1.6",
@@ -12,7 +12,7 @@ Require-Bundle: org.apache.commons.codec;bundle-version="1.6",
  com.fasterxml.jackson.core.jackson-annotations;bundle-version="2.5.0",
  org.apache.httpcomponents.httpcore;bundle-version="4.3.3",
  org.apache.httpcomponents.httpclient;bundle-version="4.3.6"
- com.amazonaws.sdk;bundle-version="1.10.20",
+ com.amazonaws.sdk;bundle-version="1.10.61",
 Export-Package: com.amazonaws.services.kinesis,
  com.amazonaws.services.kinesis.clientlibrary,
  com.amazonaws.services.kinesis.clientlibrary.config,

README.md

@@ -29,6 +29,14 @@ For producer-side developers using the **[Kinesis Producer Library (KPL)][kinesi
 To make it easier for developers to write record processors in other languages, we have implemented a Java based daemon, called MultiLangDaemon that does all the heavy lifting. Our approach has the daemon spawn a sub-process, which in turn runs the record processor, which can be written in any language. The MultiLangDaemon process and the record processor sub-process communicate with each other over [STDIN and STDOUT using a defined protocol][multi-lang-protocol]. There will be a one to one correspondence amongst record processors, child processes, and shards. For Python developers specifically, we have abstracted these implementation details away and [expose an interface][kclpy] that enables you to focus on writing record processing logic in Python. This approach enables KCL to be language agnostic, while providing identical features and similar parallel processing model across all languages.
 
 ## Release Notes
+### Release 1.6.2 (March 23, 2016)
+* Support for specifying max leases per worker and max leases to steal at a time.
+* Support for specifying initial DynamoDB table read and write capacity.
+* Support for parallel lease renewal.
+* Support for graceful worker shutdown.
+* Change DefaultCWMetricsPublisher log level to debug. [PR # 49](https://github.com/awslabs/amazon-kinesis-client/pull/49)
+* Avoid NPE in MLD record processor shutdown if record processor was not initialized. [Issue # 29](https://github.com/awslabs/amazon-kinesis-client/issues/29)
+
 ### Release 1.6.1 (September 23, 2015)
 * Expose [approximateArrivalTimestamp](http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetRecords.html) for Records in processRecords API call.
 

pom.xml

@@ -6,7 +6,7 @@
     <artifactId>amazon-kinesis-client</artifactId>
     <packaging>jar</packaging>
     <name>Amazon Kinesis Client Library for Java</name>
-    <version>1.6.1</version>
+    <version>1.6.2</version>
     <description>The Amazon Kinesis Client Library for Java enables Java developers to easily consume and process data from Amazon Kinesis.</description>
     <url>https://aws.amazon.com/kinesis</url>
 
@@ -23,7 +23,7 @@
     </licenses>
 
     <properties>
-        <aws-java-sdk.version>1.10.20</aws-java-sdk.version>
+        <aws-java-sdk.version>1.10.61</aws-java-sdk.version>
     </properties>
 
     <dependencies>

src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
  * Licensed under the Amazon Software License (the "License").
  * You may not use this file except in compliance with the License.
@@ -119,14 +119,41 @@ public class KinesisClientLibConfiguration {
     /**
      * User agent set when Amazon Kinesis Client Library makes AWS requests.
      */
-    public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.6.1";
+    public static final String KINESIS_CLIENT_LIB_USER_AGENT = "amazon-kinesis-client-library-java-1.6.2";
 
     /**
      * KCL will validate client provided sequence numbers with a call to Amazon Kinesis before checkpointing for calls
      * to {@link RecordProcessorCheckpointer#checkpoint(String)} by default.
      */
     public static final boolean DEFAULT_VALIDATE_SEQUENCE_NUMBER_BEFORE_CHECKPOINTING = true;
 
+    /**
+     * The max number of leases (shards) this worker should process.
+     * This can be useful to avoid overloading (and thrashing) a worker when a host has resource constraints
+     * or during deployment.
+     * NOTE: Setting this to a low value can cause data loss if workers are not able to pick up all shards in the
+     * stream due to the max limit.
+     */
+    public static final int DEFAULT_MAX_LEASES_FOR_WORKER = Integer.MAX_VALUE;
+
+    /**
+     * Max leases to steal from another worker at one time (for load balancing).
+     * Setting this to a higher number can allow for faster load convergence (e.g. during deployments, cold starts),
+     * but can cause higher churn in the system.
+     */
+    public static final int DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME = 1;
+
+    /**
+     * The Amazon DynamoDB table used for tracking leases will be provisioned with this read capacity.
+     */
+    public static final int DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY = 10;
+
+    /**
+     * The Amazon DynamoDB table used for tracking leases will be provisioned with this write capacity.
+     */
+    public static final int DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY = 10;
+
+
     private String applicationName;
     private String streamName;
     private String kinesisEndpoint;
@@ -153,6 +180,10 @@ public class KinesisClientLibConfiguration {
     private Set<String> metricsEnabledDimensions;
     private boolean validateSequenceNumberBeforeCheckpointing;
     private String regionName;
+    private int maxLeasesForWorker;
+    private int maxLeasesToStealAtOneTime;
+    private int initialLeaseTableReadCapacity;
+    private int initialLeaseTableWriteCapacity;
 
     /**
      * Constructor.
@@ -293,6 +324,10 @@ public KinesisClientLibConfiguration(String applicationName,
         this.metricsEnabledDimensions = DEFAULT_METRICS_ENABLED_DIMENSIONS;
         this.validateSequenceNumberBeforeCheckpointing = validateSequenceNumberBeforeCheckpointing;
         this.regionName = regionName;
+        this.maxLeasesForWorker = DEFAULT_MAX_LEASES_FOR_WORKER;
+        this.maxLeasesToStealAtOneTime = DEFAULT_MAX_LEASES_TO_STEAL_AT_ONE_TIME;
+        this.initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY;
+        this.initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY;
     }
 
     // Check if value is positive, otherwise throw an exception
@@ -508,6 +543,34 @@ public String getRegionName() {
         return regionName;
     }
 
+    /**
+     * @return Max leases this Worker can handle at a time
+     */
+    public int getMaxLeasesForWorker() {
+        return maxLeasesForWorker;
+    }
+
+    /**
+     * @return Max leases to steal at one time (for load balancing)
+     */
+    public int getMaxLeasesToStealAtOneTime() {
+        return maxLeasesToStealAtOneTime;
+    }
+
+    /**
+     * @return Read capacity to provision when creating the lease table.
+     */
+    public int getInitialLeaseTableReadCapacity() {
+        return initialLeaseTableReadCapacity;
+    }
+
+    /**
+     * @return Write capacity to provision when creating the lease table.
+     */
+    public int getInitialLeaseTableWriteCapacity() {
+        return initialLeaseTableWriteCapacity;
+    }
+
     // CHECKSTYLE:IGNORE HiddenFieldCheck FOR NEXT 190 LINES
     /**
      * @param kinesisEndpoint Kinesis endpoint
@@ -748,4 +811,57 @@ public KinesisClientLibConfiguration withRegionName(String regionName) {
         this.regionName = regionName;
         return this;
     }
+
+    /**
+     * Worker will not acquire more than the specified max number of leases even if there are more
+     * shards that need to be processed. This can be used in scenarios where a worker is resource constrained or
+     * to prevent lease thrashing when small number of workers pick up all leases for small amount of time during
+     * deployment.
+     * Note that setting a low value may cause data loss (e.g. if there aren't enough Workers to make progress on all
+     * shards). When setting the value for this property, one must ensure enough workers are present to process
+     * shards and should consider future resharding, child shards that may be blocked on parent shards, some workers
+     * becoming unhealthy, etc.
+     *
+     * @param maxLeasesForWorker Max leases this Worker can handle at a time
+     * @return KinesisClientLibConfiguration
+     */
+    public KinesisClientLibConfiguration withMaxLeasesForWorker(int maxLeasesForWorker) {
+        checkIsValuePositive("maxLeasesForWorker", maxLeasesForWorker);
+        this.maxLeasesForWorker = maxLeasesForWorker;
+        return this;
+    }
+
+    /**
+     * Max leases to steal from a more loaded Worker at one time (for load balancing).
+     * Setting this to a higher number can allow for faster load convergence (e.g. during deployments, cold starts),
+     * but can cause higher churn in the system.
+     *
+     * @param maxLeasesToStealAtOneTime Steal up to this many leases at one time (for load balancing)
+     * @return KinesisClientLibConfiguration
+     */
+    public KinesisClientLibConfiguration withMaxLeasesToStealAtOneTime(int maxLeasesToStealAtOneTime) {
+        checkIsValuePositive("maxLeasesToStealAtOneTime", maxLeasesToStealAtOneTime);
+        this.maxLeasesToStealAtOneTime = maxLeasesToStealAtOneTime;
+        return this;
+    }
+
+    /**
+     * @param initialLeaseTableReadCapacity Read capacity to provision when creating the lease table.
+     * @return KinesisClientLibConfiguration
+     */
+    public KinesisClientLibConfiguration withInitialLeaseTableReadCapacity(int initialLeaseTableReadCapacity) {
+        checkIsValuePositive("initialLeaseTableReadCapacity", initialLeaseTableReadCapacity);
+        this.initialLeaseTableReadCapacity = initialLeaseTableReadCapacity;
+        return this;
+    }
+
+    /**
+     * @param initialLeaseTableWriteCapacity Write capacity to provision when creating the lease table.
+     * @return KinesisClientLibConfiguration
+     */
+    public KinesisClientLibConfiguration withInitialLeaseTableWriteCapacity(int initialLeaseTableWriteCapacity) {
+        checkIsValuePositive("initialLeaseTableWriteCapacity", initialLeaseTableWriteCapacity);
+        this.initialLeaseTableWriteCapacity = initialLeaseTableWriteCapacity;
+        return this;
+    }
 }

src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibLeaseCoordinator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
  * Licensed under the Amazon Software License (the "License").
  * You may not use this file except in compliance with the License.
@@ -44,9 +44,14 @@
 class KinesisClientLibLeaseCoordinator extends LeaseCoordinator<KinesisClientLease> implements ICheckpoint {
 
     private static final Log LOG = LogFactory.getLog(KinesisClientLibLeaseCoordinator.class);
+
+    private static final long DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY = 10L;
+    private static final long DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY = 10L;
+
     private final ILeaseManager<KinesisClientLease> leaseManager;
-    private final long initialLeaseTableReadCapacity = 10L;
-    private final long initialLeaseTableWriteCapacity = 10L;
+
+    private long initialLeaseTableReadCapacity = DEFAULT_INITIAL_LEASE_TABLE_READ_CAPACITY;
+    private long initialLeaseTableWriteCapacity = DEFAULT_INITIAL_LEASE_TABLE_WRITE_CAPACITY;
 
     /**
      * @param leaseManager Lease manager which provides CRUD lease operations.
@@ -78,6 +83,53 @@ public KinesisClientLibLeaseCoordinator(ILeaseManager<KinesisClientLease> leaseM
         this.leaseManager = leaseManager;
     }
 
+    /**
+     * @param leaseManager Lease manager which provides CRUD lease operations.
+     * @param workerIdentifier Used to identify this worker process
+     * @param leaseDurationMillis Duration of a lease in milliseconds
+     * @param epsilonMillis Delta for timing operations (e.g. checking lease expiry)
+     * @param maxLeasesForWorker Max leases this worker can handle at a time
+     * @param maxLeasesToStealAtOneTime Steal up to this many leases at a time (for load balancing)
+     * @param metricsFactory Metrics factory used to emit metrics
+     */
+    public KinesisClientLibLeaseCoordinator(ILeaseManager<KinesisClientLease> leaseManager,
+            String workerIdentifier,
+            long leaseDurationMillis,
+            long epsilonMillis,
+            int maxLeasesForWorker,
+            int maxLeasesToStealAtOneTime,
+            IMetricsFactory metricsFactory) {
+        super(leaseManager, workerIdentifier, leaseDurationMillis, epsilonMillis, maxLeasesForWorker,
+                maxLeasesToStealAtOneTime, metricsFactory);
+        this.leaseManager = leaseManager;
+    }
+
+    /**
+     * @param readCapacity The DynamoDB table used for tracking leases will be provisioned with the specified initial
+     *        read capacity
+     * @return KinesisClientLibLeaseCoordinator
+     */
+    public KinesisClientLibLeaseCoordinator withInitialLeaseTableReadCapacity(long readCapacity) {
+        if (readCapacity <= 0) {
+            throw new IllegalArgumentException("readCapacity should be >= 1");
+        }
+        this.initialLeaseTableReadCapacity = readCapacity;
+        return this;
+    }
+
+    /**
+     * @param writeCapacity The DynamoDB table used for tracking leases will be provisioned with the specified initial
+     *        write capacity
+     * @return KinesisClientLibLeaseCoordinator
+     */
+    public KinesisClientLibLeaseCoordinator withInitialLeaseTableWriteCapacity(long writeCapacity) {
+        if (writeCapacity <= 0) {
+            throw new IllegalArgumentException("writeCapacity should be >= 1");
+        }
+        this.initialLeaseTableWriteCapacity = writeCapacity;
+        return this;
+    }
+
     /**
      * Sets the checkpoint for a shard and updates ownerSwitchesSinceCheckpoint.
      * 
@@ -173,7 +225,9 @@ void initialize() throws ProvisionedThroughputException, DependencyException, Il
         final boolean newTableCreated =
                 leaseManager.createLeaseTableIfNotExists(initialLeaseTableReadCapacity, initialLeaseTableWriteCapacity);
         if (newTableCreated) {
-            LOG.info("Created new lease table for coordinator");
+            LOG.info(String.format(
+                "Created new lease table for coordinator with initial read capacity of %d and write capacity of %d.",
+                initialLeaseTableReadCapacity, initialLeaseTableWriteCapacity));
         }
         // Need to wait for table in active state.
         final long secondsBetweenPolls = 10L;

src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/ShardConsumer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
  * Licensed under the Amazon Software License (the "License").
  * You may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -71,8 +72,8 @@ enum ShardConsumerState {
      * Used to track if we lost the primary responsibility. Once set to true, we will start shutting down.
      * If we regain primary responsibility before shutdown is complete, Worker should create a new ShardConsumer object.
      */
-    private boolean beginShutdown;
-    private ShutdownReason shutdownReason;
+    private volatile boolean beginShutdown;
+    private volatile ShutdownReason shutdownReason;
 
     /**
      * @param shardInfo Shard information
@@ -154,16 +155,26 @@ private synchronized boolean checkAndSubmitNextTask() {
                     }
                 } catch (Exception e) {
                     throw new RuntimeException(e);
+                } finally {
+                    // Setting future to null so we don't misinterpret task completion status in case of exceptions
+                    future = null;
                 }
             }
             updateState(taskCompletedSuccessfully);
             ITask nextTask = getNextTask();
             if (nextTask != null) {
                 currentTask = nextTask;
-                future = executorService.submit(currentTask);
-                currentTaskSubmitTime = System.currentTimeMillis();
-                submittedNewTask = true;
-                LOG.debug("Submitted new " + currentTask.getTaskType() + " task for shard " + shardInfo.getShardId());
+                try {
+                    future = executorService.submit(currentTask);
+                    currentTaskSubmitTime = System.currentTimeMillis();
+                    submittedNewTask = true;
+                    LOG.debug("Submitted new " + currentTask.getTaskType()
+                            + " task for shard " + shardInfo.getShardId());
+                } catch (RejectedExecutionException e) {
+                    LOG.info(currentTask.getTaskType() + " task was not accepted for execution.", e);
+                } catch (RuntimeException e) {
+                    LOG.info(currentTask.getTaskType() + " task encountered exception ", e);
+                }
             } else {
                 if (LOG.isDebugEnabled()) {
                     LOG.debug(String.format("No new task to submit for shard %s, currentState %s",