Merge pull request #43 from bartelink/init-reorg

Clarify ctor/Creation/Verification lifecycle

README.md

@@ -17,26 +17,27 @@ Table items can be represented using F# records:
 open FSharp.AWS.DynamoDB
 
 type WorkItemInfo =
-	{
-		[<HashKey>]
-		ProcessId : int64
-		[<RangeKey>]
-		WorkItemId : int64
-
-		Name : string
-		UUID : Guid
-		Dependencies : Set<string>
-		Started : DateTimeOffset option
-	}
+    {
+        [<HashKey>]
+        ProcessId : int64
+        [<RangeKey>]
+        WorkItemId : int64
+
+        Name : string
+        UUID : Guid
+        Dependencies : Set<string>
+        Started : DateTimeOffset option
+    }
 ```
 
 We can now perform table operations on DynamoDB like so:
 
 ```fsharp
 open Amazon.DynamoDBv2
+open FSharp.AWS.DynamoDB.Scripting // Expose non-Async methods, e.g. PutItem/GetItem
 
 let client : IAmazonDynamoDB = ``your DynamoDB client instance``
-let table = TableContext.Create<WorkItemInfo>(client, tableName = "workItems", createIfNotExists = true)
+let table = TableContext.Initialize<WorkItemInfo>(client, tableName = "workItems", Throughput.OnDemand)
 
 let workItem = { ProcessId = 0L ; WorkItemId = 1L ; Name = "Test" ; UUID = guid() ; Dependencies = set ["mscorlib"] ; Started = None }
 
@@ -48,7 +49,7 @@ Queries and scans can be performed using quoted predicates:
 
 ```fsharp
 let qResults = table.Query(keyCondition = <@ fun r -> r.ProcessId = 0 @>, 
-                            filterCondition = <@ fun r -> r.Name = "test" @>)
+                           filterCondition = <@ fun r -> r.Name = "test" @>)
                             
 let sResults = table.Scan <@ fun r -> r.Started.Value >= DateTimeOffset.Now - TimeSpan.FromMinutes 1.  @>
 ```
@@ -57,10 +58,10 @@ Values can be updated using quoted update expressions:
 
 ```fsharp
 let updated = table.UpdateItem(<@ fun r -> { r with Started = Some DateTimeOffset.Now } @>, 
-                                preCondition = <@ fun r -> r.DateTimeOffset = None @>)
+                               preCondition = <@ fun r -> r.DateTimeOffset = None @>)
 ```
 
-Or they can be updated using the `UpdateOp` DSL,
+Or they can be updated using [the `SET`, `ADD`, `REMOVE` and `DELETE` operations of the UpdateOp` DSL](./src/FSharp.AWS.DynamoDB/Types.fs#263),
 which is closer to the underlying DynamoDB API:
 
 ```fsharp
@@ -99,24 +100,35 @@ Update expressions support the following F# value constructors:
 * `Option.Value` and `Option.get`.
 * `fst` and `snd` for tuple records.
 
-## Example: Creating an atomic counter
+## Example: Representing an atomic counter as an Item in a DynamoDB Table 
 
 ```fsharp
 type private CounterEntry = { [<HashKey>] Id : Guid ; Value : int64 }
 
 type Counter private (table : TableContext<CounterEntry>, key : TableKey) =
-    member _.Value = table.GetItem(key).Value
-    member _.Incr() = 
-        let updated = table.UpdateItem(key, <@ fun e -> { e with Value = e.Value + 1L } @>)
-        updated.Value
-
-    static member Create(client : IAmazonDynamoDB, table : string) =
-        let table = TableContext.Create<CounterEntry>(client, table, createIfNotExists = true)
-        let entry = { Id = Guid.NewGuid() ; Value = 0L }
-        let key = table.PutItem entry
-        new Counter(table, key)
+    
+    member _.Value = async {
+        let! current = table.GetItemAsync(key)
+        return current.Value
+    }
+    
+    member _.Incr() = async { 
+        let! updated = table.UpdateItemAsync(key, <@ fun e -> { e with Value = e.Value + 1L } @>)
+        return updated.Value
+    }
+
+    static member Create(client : IAmazonDynamoDB, tableName : string) = async {
+        let table = TableContext<CounterEntry>(client, tableName)
+        let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L)        
+        do! table.VerifyOrCreateTableAsync(Throughput.Provisioned throughput)
+        let initialEntry = { Id = Guid.NewGuid() ; Value = 0L }
+        let! key = table.PutItemAsync(initialEntry)
+        return Counter(table, key)
+    }
 ```
 
+_NOTE: It's advised to split single time initialization/verification of table creation from the application logic, see [`Script.fsx`](src/FSharp.AWS.DynamoDB/Script.fsx#99) for further details_.
+
 ## Projection Expressions
 
 Projection expressions can be used to fetch a subset of table attributes, which can be useful when performing large queries:
@@ -125,7 +137,7 @@ Projection expressions can be used to fetch a subset of table attributes, which
 table.QueryProjected(<@ fun r -> r.HashKey = "Foo" @>, <@ fun r -> r.HashKey, r.Values.Nested.[0] @>)
 ```
 
-which returns a tuple of the specified attributes. Tuples can be of any arity and must contain non-conflicting document paths.
+the resulting value is a tuple of the specified attributes. Tuples can be of any arity but must contain non-conflicting document paths.
 
 ## Secondary Indices
 
@@ -141,7 +153,8 @@ type Record =
 ```
 
 Queries can now be performed on the `GSIH` and `GSIR` fields as if they were regular hashkey and rangekey attributes.
-Global secondary indices are created using the same provisioned throughput as the primary keys.
+
+_NOTE: Global secondary indices are created using the same provisioned throughput as for the primary keys_.
 
 [Local Secondary Indices](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/LSI.html) can be defined using the `LocalSecondaryIndex` attribute:
 ```fsharp
@@ -212,13 +225,13 @@ It is possible to precompute a DynamoDB expression as follows:
 let precomputedConditional = table.Template.PrecomputeConditionalExpr <@ fun w -> w.Name <> "test" && w.Dependencies.Contains "mscorlib" @>
 ```
 
-This precomputed conditional can now be used in place of the original expression in the FSharp.AWS.DynamoDB API:
+This precomputed conditional can now be used in place of the original expression in the `FSharp.AWS.DynamoDB` API:
 
 ```fsharp
 let results = table.Scan precomputedConditional
 ```
 
-FSharp.AWS.DynamoDB also supports precomputation of parametric expressions:
+`FSharp.AWS.DynamoDB` also supports precomputation of parametric expressions:
 
 ```fsharp
 let startedBefore = table.Template.PrecomputeConditionalExpr <@ fun time w -> w.StartTime.Value <= time @>
@@ -232,16 +245,15 @@ table.Scan(startedBefore (DateTimeOffset.Now - TimeSpan.FromDays 1.))
 A hook is provided so metrics can be published via your preferred Observability provider. For example, using [Prometheus.NET](https://github.com/prometheus-net/prometheus-net):
 
 ```fsharp
-let dbCounter = Metrics.CreateCounter ("aws_dynamodb_requests_total", "Count of all DynamoDB requests", "table", "operation")
+let dbCounter = Prometheus.Metrics.CreateCounter("aws_dynamodb_requests_total", "Count of all DynamoDB requests", "table", "operation")
 let processMetrics (m : RequestMetrics) =
-    dbCounter.WithLabels(m.TableName, string m.Operation).Inc () |> ignore
-let table = TableContext.Create<WorkItemInfo>(client, tableName = "workItems", metricsCollector = processMetrics)
+    dbCounter.WithLabels(m.TableName, string m.Operation).Inc()
+let table = TableContext<WorkItemInfo>(client, tableName = "workItems", metricsCollector = processMetrics)
 ```
 
-If `metricsCollector` is supplied, the requests will include `ReturnConsumedCapacity = ReturnConsumedCapacity.INDEX` 
+If `metricsCollector` is supplied, the requests will set `ReturnConsumedCapacity` to `ReturnConsumedCapacity.INDEX` 
 and the `RequestMetrics` parameter will contain a list of `ConsumedCapacity` objects returned from the DynamoDB operations.
 
-
 ### Building & Running Tests
 
 To build using the dotnet SDK:

RELEASE_NOTES.md

@@ -1,3 +1,16 @@
+### 0.10.0-beta
+* Added `TableContext` constructor (replaces `TableContext.Create(verifyTable = false)`)
+* Added `TableContext.VerifyOrCreateTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`)
+* Added `TableContext.UpdateTableIfRequiredAsync` (conditional `UpdateTableAsync` to establish specified `throughput` or `streaming` only if required. Replaces `UpdateProvisionedThroughputAsync`)
+* Added `TableContext.Scripting.Initialize` (two overloads, replacing `TableContext.Create()` and `TableContext.Create(createIfNotExists = true)`)
+* Added `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST`, to go with the existing support for configuring `PROVISIONED` and a `ProvisionedThroughput`)
+* Added ability to configure DynamoDB streaming (via a `Streaming` DU) to `VerifyOrCreateTableAsync` and `UpdateTableIfRequiredAsync` 
+* Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize`, `TableContext.VerifyOrCreateTableAsync`, `TableContext.VerifyTableAsync`)
+* Obsoleted `TableContext.UpdateProvisionedThroughputAsync` (replace with `TableContext.UpdateTableIfRequiredAsync`)
+* (breaking) Obsoleted `TableContext.VerifyTableAsync` optional argument to create a Table (replace with `VerifyOrCreateTableAsync`)
+* (breaking) Changed `TableKeySchemata.CreateCreateTableRequest` to `ApplyToCreateTableRequest` (with minor signature change)
+* (breaking) Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `VerifyOrCreateTableAsync`)
+
 ### 0.9.4-beta
 * Moved Sync-over-Async versions of `TableContext` operations into `namespace FSharp.AWS.DynamoDB.Scripting`
 * Added `WithMetricsCollector()` copy method to allow separating metrics by context (eg by request)

src/FSharp.AWS.DynamoDB/FSharp.AWS.DynamoDB.fsproj

@@ -9,6 +9,7 @@
     <Company />
     <Copyright>Copyright 2016</Copyright>
     <Product />
+    <PackageLicense>MIT</PackageLicense>
     <PackageLicenseUrl>https://github.com/fsprojects/FSharp.AWS.DynamoDB/blob/master/License.md</PackageLicenseUrl>
     <PackageProjectUrl>https://github.com/fsprojects/FSharp.AWS.DynamoDB</PackageProjectUrl>
     <PackageIconUrl>https://avatars0.githubusercontent.com/u/6001315</PackageIconUrl>
@@ -44,4 +45,4 @@
     <None Include="paket.template" />
   </ItemGroup>
   <Import Project="..\..\.paket\Paket.Restore.targets" />
-</Project>
+</Project>

src/FSharp.AWS.DynamoDB/RecordKeySchema.fs

@@ -368,13 +368,10 @@ type TableKeySchemata with
                 yield! td.GlobalSecondaryIndexes |> Seq.map mkGlobalSecondaryIndex
                 yield! td.LocalSecondaryIndexes |> Seq.map mkLocalSecondaryIndex |])
 
-    /// Create a CreateTableRequest using supplied key schema
-    member schema.CreateCreateTableRequest (tableName : string, provisionedThroughput : ProvisionedThroughput) =
-        let ctr = CreateTableRequest(TableName = tableName)
+    /// Applies the settings implied by the schema to the supplied CreateTableRequest
+    member schema.ApplyToCreateTableRequest(ctr : CreateTableRequest) =
         let inline mkKSE n t = KeySchemaElement(n, t)
 
-        ctr.ProvisionedThroughput <- provisionedThroughput
-
         let keyAttrs = new Dictionary<string, KeyAttributeSchema>()
         for tks in schema.Schemata do
             keyAttrs.[tks.HashKey.AttributeName] <- tks.HashKey
@@ -391,7 +388,6 @@ type TableKeySchemata with
                 gsi.KeySchema.Add <| mkKSE tks.HashKey.AttributeName KeyType.HASH
                 tks.RangeKey |> Option.iter (fun rk -> gsi.KeySchema.Add <| mkKSE rk.AttributeName KeyType.RANGE)
                 gsi.Projection <- Projection(ProjectionType = ProjectionType.ALL)
-                gsi.ProvisionedThroughput <- provisionedThroughput
                 ctr.GlobalSecondaryIndexes.Add gsi
 
             | LocalSecondaryIndex name ->
@@ -405,5 +401,3 @@ type TableKeySchemata with
         for attr in keyAttrs.Values do
             let ad = AttributeDefinition(attr.AttributeName, attr.KeyType)
             ctr.AttributeDefinitions.Add ad
-
-        ctr

src/FSharp.AWS.DynamoDB/Script.fsx

@@ -1,7 +1,7 @@
 #if USE_PUBLISHED_NUGET // If you don't want to do a local build first
 #r "nuget: FSharp.AWS.DynamoDB, *-*" // *-* to white-list the fact that all releases to date have been `-beta` sufficed
 #else
-#I "../../bin/net6.0/"
+#I "../../tests/FSharp.AWS.DynamoDB.Tests/bin/Debug/net6.0/"
 #r "AWSSDK.Core.dll"
 #r "AWSSDK.DynamoDBv2.dll"
 #r "FSharp.AWS.DynamoDB.dll"
@@ -15,19 +15,18 @@ open FSharp.AWS.DynamoDB
 open FSharp.AWS.DynamoDB.Scripting // non-Async overloads
 
 #if USE_CLOUD
-open Amazon
-open Amazon.Util
-let account = AWSCredentialsProfile.LoadFrom("default").Credentials
-let ddb = new AmazonDynamoDBClient(account, RegionEndpoint.EUCentral1) :> IAmazonDynamoDB
+open Amazon.DynamoDBv2
+let ok, creds = Amazon.Runtime.CredentialManagement.CredentialProfileStoreChain().TryGetAWSCredentials("default")
+let ddb = if ok then new AmazonDynamoDBClient(creds) :> IAmazonDynamoDB else failwith "Unable to load default credentials"
 #else // Use Docker-hosted dynamodb-local instance
 // See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html#docker for details of how to deploy a simulator instance
-let clientConfig = AmazonDynamoDBConfig(ServiceURL = "http://localhost:8000")
 #if USE_CREDS_FROM_ENV_VARS // 'AWS_ACCESS_KEY_ID' and 'AWS_SECRET_ACCESS_KEY' must be set for this to work
 let credentials = AWSCredentials.FromEnvironmentVariables()
 #else
 // Credentials are not validated if connecting to local instance so anything will do (this avoids it looking for profiles to be configured)
 let credentials = Amazon.Runtime.BasicAWSCredentials("A", "A")
 #endif
+let clientConfig = AmazonDynamoDBConfig(ServiceURL = "http://localhost:8000")
 let ddb = new AmazonDynamoDBClient(credentials, clientConfig) :> IAmazonDynamoDB
 #endif
 
@@ -54,8 +53,8 @@ type Test =
         Bytes : byte[]
     }
 
-
-let table = TableContext.Create<Test>(ddb, "test", createIfNotExists = true)
+let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L)
+let table = TableContext.Initialize<Test>(ddb, "test", Throughput.Provisioned throughput)
 
 let value = { HashKey = Guid.NewGuid() ; List = [] ; RangeKey = "2" ; Value = 3.1415926 ; Date = DateTimeOffset.Now + TimeSpan.FromDays 2. ; Value2 = None ; Values = [|{ A = "foo" ; B = System.Reflection.BindingFlags.Instance }|] ; Map = Map.ofList [("A1",1)] ; Set = [set [1L];set [2L]] ; Bytes = [|1uy..10uy|]; String = ref "1a" ; Unions = [A 42; B("42",3)]}
 
@@ -94,3 +93,104 @@ let uexpr2 = table.Template.PrecomputeUpdateExpr <@ fun v r -> { r with Value2 =
 for i = 1 to 1000 do
     let _ = table.UpdateItem(key, uexpr2 (Some 42))
     ()
+
+(* Expanded version of README sample that illustrates how one can better split Table initialization from application logic *)
+
+type internal CounterEntry = { [<HashKey>] Id : Guid ; Value : int64 }
+
+/// Represents a single Item in a Counters Table
+type Counter internal (table : TableContext<CounterEntry>, key : TableKey) =
+
+    static member internal Start(table : TableContext<CounterEntry>) = async {
+        let initialEntry = { Id = Guid.NewGuid() ; Value = 0L }
+        let! key = table.PutItemAsync(initialEntry)
+        return Counter(table, key)
+    }
+
+    member _.Value = async {
+        let! current = table.GetItemAsync(key)
+        return current.Value
+    }
+
+    member _.Incr() = async {
+        let! updated = table.UpdateItemAsync(key, <@ fun (e : CounterEntry) -> { e with Value = e.Value + 1L } @>)
+        return updated.Value
+    }
+
+/// Wrapper that creates/verifies the table only once per call to Create()
+/// This does assume that your application will be sufficiently privileged to create tables on the fly
+type EasyCounters private (table : TableContext<CounterEntry>) =
+
+    // We only want to do the initialization bit once per instance of our application
+    static member Create(client : IAmazonDynamoDB, tableName : string) : Async<EasyCounters> = async {
+        let table = TableContext<CounterEntry>(client, tableName)
+        // Create the table if necessary. Verifies schema is correct if it has already been created
+        // NOTE the hard coded initial throughput provisioning - arguably this belongs outside of your application logic
+        let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L)
+        do! table.VerifyOrCreateTableAsync(Throughput.Provisioned throughput)
+        return EasyCounters(table)
+    }
+
+    member _.StartCounter() : Async<Counter> =
+        Counter.Start table
+
+/// Variant of EasyCounters that splits the provisioning step from the (optional) validation that the table is present
+type SimpleCounters private (table : TableContext<CounterEntry>) =
+
+    static member Provision(client : IAmazonDynamoDB, tableName : string, readCapacityUnits, writeCapacityUnits) : Async<unit> = async {
+        let table = TableContext<CounterEntry>(client, tableName)
+        let provisionedThroughput = ProvisionedThroughput(readCapacityUnits, writeCapacityUnits)
+        let throughput = Throughput.Provisioned provisionedThroughput
+        // normally, RCU/WCU provisioning only happens first time the Table is created and is then considered an external concern
+        // here we use `UpdateTableIfRequiredAsync` to reset it each time we deploy the app
+        do! table.VerifyOrCreateTableAsync(throughput)
+        do! table.UpdateTableIfRequiredAsync(throughput) }
+
+    static member ProvisionOnDemand(client : IAmazonDynamoDB, tableName : string) : Async<unit> = async {
+        let table = TableContext<CounterEntry>(client, tableName)
+        let throughput = Throughput.OnDemand
+        do! table.VerifyOrCreateTableAsync(throughput)
+        // as per the Provision, above, we reset to OnDemand, if it got reconfigured since it was originally created
+        do! table.UpdateTableIfRequiredAsync(throughput) }
+
+    /// We only want to do the initialization bit once per instance of our application
+    /// Similar to EasyCounters.Create in that it ensures the table is provisioned correctly
+    /// However it will never actually create the table
+    static member CreateWithVerify(client : IAmazonDynamoDB, tableName : string) : Async<SimpleCounters> = async {
+        let table = TableContext<CounterEntry>(client, tableName)
+        // This validates the Table has been created correctly
+        // (in general this is a good idea, but it is an optional step so it can be skipped, i.e. see Create() below)
+        do! table.VerifyTableAsync()
+        return SimpleCounters(table)
+    }
+
+    /// Assumes the table has been provisioned externally via Provision()
+    static member Create(client : IAmazonDynamoDB, tableName : string) : SimpleCounters =
+        // NOTE we are skipping
+        SimpleCounters(TableContext<CounterEntry>(client, tableName))
+
+    member _.StartCounter() : Async<Counter> =
+        Counter.Start table
+
+let e = EasyCounters.Create(ddb, "testing") |> Async.RunSynchronously
+let e1 = e.StartCounter() |> Async.RunSynchronously
+let e2 = e.StartCounter() |> Async.RunSynchronously
+e1.Incr() |> Async.RunSynchronously
+e2.Incr() |> Async.RunSynchronously
+
+// First, we create it in On-Demand mode
+SimpleCounters.ProvisionOnDemand(ddb, "testing-pre-provisioned") |> Async.RunSynchronously
+// Then we flip it to Provisioned mode
+SimpleCounters.Provision(ddb, "testing-pre-provisioned", readCapacityUnits = 10L, writeCapacityUnits = 10L) |> Async.RunSynchronously
+// The consuming code can assume the provisioning has been carried out as part of the deploy
+// that allows the creation to be synchronous (and not impede application startup)
+let s = SimpleCounters.Create(ddb, "testing-pre-provisioned")
+let s1 = s.StartCounter() |> Async.RunSynchronously // Throws if Provision step has not been executed
+s1.Incr() |> Async.RunSynchronously
+
+// Alternately, we can have the app do an extra call (and have some asynchronous initialization work) to check the table is ready
+let v = SimpleCounters.CreateWithVerify(ddb, "testing-not-present") |> Async.RunSynchronously // Throws, as table not present
+let v2 = v.StartCounter() |> Async.RunSynchronously
+v2.Incr() |> Async.RunSynchronously
+
+// (TOCONSIDER: Illustrate how to use AsyncCacheCell from https://github.com/jet/equinox/blob/master/src/Equinox.Core/AsyncCacheCell.fs to make Verify call lazy)