From f03fe7894c9b87c3b63270750b1b65145598c046 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 23 Mar 2022 15:06:36 +0000 Subject: [PATCH 01/22] Add TableContext.CreateUnverified --- RELEASE_NOTES.md | 1 + src/FSharp.AWS.DynamoDB/TableContext.fs | 25 +++++++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c130ea4..b66388d 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,6 +4,7 @@ * Ensured metrics are reported even for failed requests * Added `TryGetItemAsync` (same as `GetItemAsync`, but returns `None`, instead of throwing, if an item is not present) * Switched test framework to Xunit, assertions to Unquote, runner to `dotnet test` +* Added `TableContext.CreateUnverified` (`TableContext.CreateAsync` without the optional store round-trips) ### 0.9.3-beta * Added `RequestMetrics` record type diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index d946a5b..1b5804f 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -930,24 +930,33 @@ type TableContext<'TRecord> internal type TableContext = /// - /// Creates a DynamoDB client instance for given F# record and table name. + /// Creates a DynamoDB client instance for given F# record and table name.
+ /// See CreateAsync for the ability to create and/or verify the Table. + /// + /// DynamoDB client instance. + /// Table name to target. + /// Function to receive request metrics. + static member CreateUnverified<'TRecord>(client : IAmazonDynamoDB, tableName : string, ?metricsCollector : RequestMetrics -> unit) : TableContext<'TRecord> = + if not <| isValidTableName tableName then invalidArg "tableName" "unsupported DynamoDB table name." + TableContext<'TRecord>(client, tableName, RecordTemplate.Define<'TRecord>(), metricsCollector) + + /// + /// Creates a DynamoDB client instance for given F# record and table name.
+ /// See CreateUnverified if your deployment phase handles the creation of the Table and verification of the schema. /// /// DynamoDB client instance. /// Table name to target. /// Verify that the table exists and is compatible with supplied record schema. Defaults to true. - /// Create the table now instance if it does not exist. Defaults to false. - /// Provisioned throughput for the table if newly created. Defaults to (10,10). + /// Create the table immediately if it does not exist. Defaults to false. + /// Provisioned throughput for the table if creation required. Defaults to (10,10). /// Function to receive request metrics. static member CreateAsync<'TRecord>(client : IAmazonDynamoDB, tableName : string, ?verifyTable : bool, ?createIfNotExists : bool, - ?provisionedThroughput : ProvisionedThroughput, ?metricsCollector : RequestMetrics -> unit) : Async> = async { - - if not <| isValidTableName tableName then invalidArg "tableName" "unsupported DynamoDB table name." + ?provisionedThroughput : ProvisionedThroughput, ?metricsCollector : RequestMetrics -> unit) : Async> = async { + let context = TableContext.CreateUnverified(client, tableName, ?metricsCollector = metricsCollector) let verifyTable = defaultArg verifyTable true let createIfNotExists = defaultArg createIfNotExists false - let context = new TableContext<'TRecord>(client, tableName, RecordTemplate.Define<'TRecord>(), metricsCollector) if verifyTable || createIfNotExists then do! context.VerifyTableAsync(createIfNotExists = createIfNotExists, ?provisionedThroughput = provisionedThroughput) - return context } From 1f42d77bb8e85a30b4400bdb7f4bd48c44970d99 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 23 Mar 2022 20:10:54 +0000 Subject: [PATCH 02/22] README tweak --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3380319..ebbf77f 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ let updated = table.UpdateItem(<@ fun r -> { r with Started = Some DateTimeOffse preCondition = <@ fun r -> r.DateTimeOffset = None @>) ``` -Or they can be updated using the `UpdateOp` DSL, +Or they can be updated using [the `SET`, `ADD`, `REMOVE` and `DELETE` operations of the UpdateOp` DSL](./src/FSharp.AWS.DynamoDB/Types.fs#263), which is closer to the underlying DynamoDB API: ```fsharp @@ -125,7 +125,7 @@ Projection expressions can be used to fetch a subset of table attributes, which table.QueryProjected(<@ fun r -> r.HashKey = "Foo" @>, <@ fun r -> r.HashKey, r.Values.Nested.[0] @>) ``` -which returns a tuple of the specified attributes. Tuples can be of any arity and must contain non-conflicting document paths. +the resulting value is a tuple of the specified attributes. Tuples can be of any arity but must contain non-conflicting document paths. ## Secondary Indices @@ -141,7 +141,8 @@ type Record = ``` Queries can now be performed on the `GSIH` and `GSIR` fields as if they were regular hashkey and rangekey attributes. -Global secondary indices are created using the same provisioned throughput as the primary keys. + +_NOTE: Global secondary indices are created using the same provisioned throughput as for the primary keys_. [Local Secondary Indices](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/LSI.html) can be defined using the `LocalSecondaryIndex` attribute: ```fsharp @@ -212,13 +213,13 @@ It is possible to precompute a DynamoDB expression as follows: let precomputedConditional = table.Template.PrecomputeConditionalExpr <@ fun w -> w.Name <> "test" && w.Dependencies.Contains "mscorlib" @> ``` -This precomputed conditional can now be used in place of the original expression in the FSharp.AWS.DynamoDB API: +This precomputed conditional can now be used in place of the original expression in the `FSharp.AWS.DynamoDB` API: ```fsharp let results = table.Scan precomputedConditional ``` -FSharp.AWS.DynamoDB also supports precomputation of parametric expressions: +`FSharp.AWS.DynamoDB` also supports precomputation of parametric expressions: ```fsharp let startedBefore = table.Template.PrecomputeConditionalExpr <@ fun time w -> w.StartTime.Value <= time @> From 266b79689e4c1bf7322e7bc556efe20efba3ef97 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 28 Mar 2022 13:00:33 +0100 Subject: [PATCH 03/22] Clarified ctor and Creation/Verification APIs --- RELEASE_NOTES.md | 8 +- src/FSharp.AWS.DynamoDB/TableContext.fs | 192 +++++++++++------- .../ConditionalExpressionTests.fs | 2 +- .../MetricsCollectorTests.fs | 10 +- .../PaginationTests.fs | 2 +- .../ProjectionExpressionTests.fs | 2 +- .../SimpleTableOperationTests.fs | 2 +- .../SparseGSITests.fs | 2 +- .../UpdateExpressionTests.fs | 2 +- tests/FSharp.AWS.DynamoDB.Tests/Utils.fs | 7 +- 10 files changed, 138 insertions(+), 91 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index b66388d..80aeac9 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,7 +4,13 @@ * Ensured metrics are reported even for failed requests * Added `TryGetItemAsync` (same as `GetItemAsync`, but returns `None`, instead of throwing, if an item is not present) * Switched test framework to Xunit, assertions to Unquote, runner to `dotnet test` -* Added `TableContext.CreateUnverified` (`TableContext.CreateAsync` without the optional store round-trips) +* Clarified Creation/Verification APIs: + * Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize` and/or `TableContext.InitializeTableAsync`) + * Added `TableContext` constructor (replaces `TableContext.Create(verifyTable = false)`) + * Added `TableContext.Scripting.Initialize` (replaces `TableContext.Create()`) + * Added `TableContext.VerifyTableAsync` overload that only performs verification but never creates a Table + * Added `TableContext.InitializeTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) + * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `TableContext.InitializeTableAsync`) ### 0.9.3-beta * Added `RequestMetrics` record type diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 1b5804f..91be3ab 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -8,7 +8,6 @@ open Microsoft.FSharp.Quotations open Amazon.DynamoDBv2 open Amazon.DynamoDBv2.Model -open FSharp.AWS.DynamoDB.KeySchema open FSharp.AWS.DynamoDB.ExprCommon /// Exception raised by DynamoDB in case where write preconditions are not satisfied @@ -47,6 +46,16 @@ type private LimitType = All | Default | Count of int static member AllOrCount (l : int option) = l |> Option.map Count |> Option.defaultValue All static member DefaultOrCount (l : int option) = l |> Option.map Count |> Option.defaultValue Default +/// Defines Verification and/or Provisioning steps to be applied when initializing and/or validating a TableContext +[] +type InitializationMode = + /// Validate the Table exists and has a compatible schema + | VerifyOnly + /// Perform a Validation step as per VerifyOnly, but Create the Table with the specified Throughput if it was not found + | CreateIfNotExists of provisionedThroughput : ProvisionedThroughput + /// Validate and/or Create as per CreateIfNotExists, but also re-apply the specified Throughput in case it has diverged + | CreateOrUpdateThroughput of provisionedThroughput : ProvisionedThroughput + /// DynamoDB client object for performing table operations in the context of given F# record representations [] type TableContext<'TRecord> internal @@ -260,6 +269,19 @@ type TableContext<'TRecord> internal /// Record-induced table template member __.Template = template + + /// + /// Creates a DynamoDB client instance for given F# record and table name.
+ /// For creating, provisioning or verification, see InitializeTableAsync and VerifyTableAsync. + /// + /// DynamoDB client instance. + /// Table name to target. + /// Function to receive request metrics. + new (client : IAmazonDynamoDB, tableName : string, ?metricsCollector : RequestMetrics -> unit) = + if not <| isValidTableName tableName then invalidArg "tableName" "unsupported DynamoDB table name." + TableContext<'TRecord>(client, tableName, RecordTemplate.Define<'TRecord>(), metricsCollector) + + /// Creates a new table context instance that uses /// a new F# record type. The new F# record type /// must define a compatible key schema. @@ -857,24 +879,14 @@ type TableContext<'TRecord> internal } - /// - /// Asynchronously verify that the table exists and is compatible with record key schema. - /// - /// Create the table instance now instance if it does not exist. Defaults to false. - /// Provisioned throughput for the table if newly created. Defaults to (10,10). - member __.VerifyTableAsync(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) : Async = async { - let createIfNotExists = defaultArg createIfNotExists false + member __.InternalInitializeAsync(mode : InitializationMode) : Async = async { let (|Conflict|_|) (e : exn) = match e with | :? AmazonDynamoDBException as e when e.StatusCode = HttpStatusCode.Conflict -> Some() | :? ResourceInUseException -> Some () | _ -> None - let rec verify lastExn retries = async { - match lastExn with - | Some e when retries = 0 -> do! Async.Raise e - | _ -> () - + let rec verify retries = async { let! ct = Async.CancellationToken let! response = client.DescribeTableAsync(tableName, ct) @@ -886,7 +898,7 @@ type TableContext<'TRecord> internal if td.Table.TableStatus <> TableStatus.ACTIVE then do! Async.Sleep 2000 // wait indefinitely if table is in transition state - return! verify None retries + return! verify retries else let existingSchema = TableKeySchemata.OfTableDescription td.Table @@ -894,71 +906,105 @@ type TableContext<'TRecord> internal sprintf "table '%s' exists with key schema %A, which is incompatible with record '%O'." tableName existingSchema typeof<'TRecord> |> invalidOp - - | Choice2Of2 (:? ResourceNotFoundException) when createIfNotExists -> - let provisionedThroughput = - match provisionedThroughput with - | None -> ProvisionedThroughput(10L,10L) - | Some pt -> pt - - let ctr = template.Info.Schemata.CreateCreateTableRequest (tableName, provisionedThroughput) - let! ct = Async.CancellationToken + match mode with + | InitializationMode.VerifyOnly | InitializationMode.CreateIfNotExists _ -> () + | InitializationMode.CreateOrUpdateThroughput t -> do! __.UpdateProvisionedThroughputAsync(t) + + | Choice2Of2 (:? ResourceNotFoundException) when mode <> InitializationMode.VerifyOnly -> + let throughput = + match mode with + | InitializationMode.VerifyOnly -> failwith "Unexpected" // the when guard should preclude this + | InitializationMode.CreateIfNotExists t | InitializationMode.CreateOrUpdateThroughput t -> t + let ctr = template.Info.Schemata.CreateCreateTableRequest(tableName, throughput) let! response = client.CreateTableAsync(ctr, ct) |> Async.AwaitTaskCorrect |> Async.Catch match response with - | Choice1Of2 _ -> return! verify None retries - | Choice2Of2 (Conflict as e) -> + | Choice1Of2 _ -> return! verify retries + | Choice2Of2 Conflict when retries > 0 -> do! Async.Sleep 2000 - return! verify (Some e) (retries - 1) + return! verify (retries - 1) | Choice2Of2 e -> do! Async.Raise e - | Choice2Of2 (Conflict as e) -> + | Choice2Of2 Conflict when retries > 0 -> do! Async.Sleep 2000 - return! verify (Some e) (retries - 1) + return! verify (retries - 1) | Choice2Of2 e -> do! Async.Raise e } - do! verify None 10 + do! verify 9 // up to 9 retries, i.e. 10 attempts before we let exception propagate } -/// Table context factory methods -type TableContext = + /// + /// Asynchronously verify that the table exists and is compatible with record key schema. + /// + /// Create the table instance now instance if it does not exist. Defaults to false. + /// Provisioned throughput for the table if newly created. Defaults to (10,10). + [] + member __.VerifyTableAsync(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) : Async = + let mode = + if createIfNotExists = Some true then + let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) + InitializationMode.CreateIfNotExists throughput + else InitializationMode.VerifyOnly + __.InternalInitializeAsync(mode) /// - /// Creates a DynamoDB client instance for given F# record and table name.
- /// See CreateAsync for the ability to create and/or verify the Table. + /// Asynchronously verify that the table exists and is compatible with record key schema, or throw.
+ /// See also InitializeTableAsync, which performs the same check, but can create or re-provision the Table if required. /// - /// DynamoDB client instance. - /// Table name to target. - /// Function to receive request metrics. - static member CreateUnverified<'TRecord>(client : IAmazonDynamoDB, tableName : string, ?metricsCollector : RequestMetrics -> unit) : TableContext<'TRecord> = - if not <| isValidTableName tableName then invalidArg "tableName" "unsupported DynamoDB table name." - TableContext<'TRecord>(client, tableName, RecordTemplate.Define<'TRecord>(), metricsCollector) + member __.VerifyTableAsync() : Async = + __.InternalInitializeAsync(InitializationMode.VerifyOnly) /// - /// Creates a DynamoDB client instance for given F# record and table name.
- /// See CreateUnverified if your deployment phase handles the creation of the Table and verification of the schema. + /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
+ /// If the table is not present, it is provisioned, with the specified throughput.
+ /// Optionally can reapply the cited throughput level if the Table has already been created.
+ /// See also VerifyTableAsync, which only verifies the Table is present and correct. + /// + /// Provisioned throughput to use for the table. + /// Trigger re-provisioning of the Table's throughput if it was already present. Default: false + member __.InitializeTableAsync(throughput : ProvisionedThroughput, ?updateThroughputIfExists) : Async = + let mode = + if updateThroughputIfExists = Some true then InitializationMode.CreateOrUpdateThroughput + else InitializationMode.CreateIfNotExists + __.InternalInitializeAsync(mode throughput) + +// Deprecated factory method, to be removed. Replaced with +// 1. TableContext<'T> ctor (synchronous) +// 2. InitializeTableAsync OR VerifyTableAsync (explicitly async to signify that verification/creation is a costly and/or privileged operation) +type TableContext internal () = + + /// + /// Creates a DynamoDB client instance for given F# record and table name. /// /// DynamoDB client instance. /// Table name to target. /// Verify that the table exists and is compatible with supplied record schema. Defaults to true. - /// Create the table immediately if it does not exist. Defaults to false. - /// Provisioned throughput for the table if creation required. Defaults to (10,10). + /// Create the table now if it does not exist. Defaults to false. + /// Provisioned throughput for the table if newly created. /// Function to receive request metrics. - static member CreateAsync<'TRecord>(client : IAmazonDynamoDB, tableName : string, ?verifyTable : bool, ?createIfNotExists : bool, - ?provisionedThroughput : ProvisionedThroughput, ?metricsCollector : RequestMetrics -> unit) : Async> = async { - let context = TableContext.CreateUnverified(client, tableName, ?metricsCollector = metricsCollector) - let verifyTable = defaultArg verifyTable true - let createIfNotExists = defaultArg createIfNotExists false + [] + static member Create<'TRecord> + ( client : IAmazonDynamoDB, tableName : string, ?verifyTable : bool, + ?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput, + ?metricsCollector : RequestMetrics -> unit) = + let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) + let verifyTable, createIfNotExists = verifyTable <> Some false, createIfNotExists = Some true if verifyTable || createIfNotExists then - do! context.VerifyTableAsync(createIfNotExists = createIfNotExists, ?provisionedThroughput = provisionedThroughput) - return context - } + let mode = + if createIfNotExists then + let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) + InitializationMode.CreateIfNotExists throughput + else InitializationMode.VerifyOnly + context.InternalInitializeAsync(mode) |> Async.RunSynchronously + context /// /// Sync-over-Async helpers that can be opted-into when working in scripting scenarios. @@ -967,6 +1013,23 @@ type TableContext = /// module Scripting = + /// Factory method that allows one to include auto-initialization easily for scripting scenarios + type TableContext internal () = + + /// Creates a DynamoDB client instance for the specified F# record type, client and table name. + /// DynamoDB client instance. + /// Table name to target. + /// Allows one to define auto-creation or re-provisioning rules via . + /// Function to receive request metrics. + static member Initialize<'TRecord> + ( client : IAmazonDynamoDB, tableName : string, + ?mode : InitializationMode, + ?metricsCollector : RequestMetrics -> unit) : TableContext<'TRecord> = + let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) + let mode = defaultArg mode InitializationMode.VerifyOnly + context.InternalInitializeAsync(mode) |> Async.RunSynchronously + context + type TableContext<'TRecord> with /// @@ -1384,30 +1447,3 @@ module Scripting = /// Provisioned throughput to use on table. member __.UpdateProvisionedThroughput(provisionedThroughput : ProvisionedThroughput) = __.UpdateProvisionedThroughputAsync(provisionedThroughput) |> Async.RunSynchronously - - - /// - /// Asynchronously verify that the table exists and is compatible with record key schema. - /// - /// Create the table instance now if it does not exist. Defaults to false. - /// Provisioned throughput for the table if newly created. - member __.VerifyTable(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) = - __.VerifyTableAsync(?createIfNotExists = createIfNotExists, ?provisionedThroughput = provisionedThroughput) - |> Async.RunSynchronously - - type TableContext with - - /// - /// Creates a DynamoDB client instance for given F# record and table name. - /// - /// DynamoDB client instance. - /// Table name to target. - /// Verify that the table exists and is compatible with supplied record schema. Defaults to true. - /// Create the table now instance if it does not exist. Defaults to false. - /// Provisioned throughput for the table if newly created. - /// Function to receive request metrics. - static member Create<'TRecord>(client : IAmazonDynamoDB, tableName : string, ?verifyTable : bool, ?createIfNotExists : bool, - ?provisionedThroughput : ProvisionedThroughput, ?metricsCollector : RequestMetrics -> unit) = - TableContext.CreateAsync<'TRecord>(client, tableName, ?verifyTable = verifyTable, ?createIfNotExists = createIfNotExists, - ?provisionedThroughput = provisionedThroughput, ?metricsCollector = metricsCollector) - |> Async.RunSynchronously diff --git a/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs index 894569f..621a0f3 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs @@ -86,7 +86,7 @@ type ``Conditional Expression Tests`` (fixture : TableFixture) = Serialized = rand(), guid() } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() let [] ``Item exists precondition`` () = let item = mkItem() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs index d720074..100aba7 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs @@ -52,7 +52,7 @@ let (|TotalCu|) : ConsumedCapacity list -> float = Seq.sumBy (fun c -> c.Capacit /// Tests without common setup type Tests(fixture : TableFixture) = - let rawTable = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let rawTable = fixture.CreateContextAndTableIfNotExists() let (|ExpectedTableName|_|) name = if name = fixture.TableName then Some () else None @@ -84,7 +84,7 @@ type Tests(fixture : TableFixture) = /// Tests that look up a specific item. Each test run gets a fresh individual item type ItemTests(fixture : TableFixture) = - let rawTable = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let rawTable = fixture.CreateContextAndTableIfNotExists() let (|ExpectedTableName|_|) name = if name = fixture.TableName then Some () else None let item = mkItem (guid()) (guid()) 0 @@ -128,10 +128,10 @@ type ItemTests(fixture : TableFixture) = interface IClassFixture -/// Heavy tests reliant on establishing (and mutating) multiple items. Separate Test Class so Xunit will run thme in parallel with others +/// Heavy tests reliant on establishing (and mutating) multiple items. Separate Test Class so Xunit will run them in parallel with others type BulkMutationTests(fixture : TableFixture) = - let rawTable = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let rawTable = fixture.CreateContextAndTableIfNotExists() let (|ExpectedTableName|_|) name = if name = fixture.TableName then Some () else None // NOTE we mutate the items so they need to be established each time @@ -169,7 +169,7 @@ type ManyReadOnlyItemsFixture() = inherit TableFixture() // TOCONSIDER shift this into IAsyncLifetime.InitializeAsync - let table = TableContext.Create(base.Client, base.TableName, createIfNotExists = true) + let table = base.CreateContextAndTableIfNotExists() let hk = guid () do let gsk = guid () diff --git a/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs index a470d07..e4cfa49 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs @@ -43,7 +43,7 @@ type ``Pagination Tests`` (fixture : TableFixture) = LocalAttribute = int (rand () % 2L) } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() let [] ``Paginated Query on Primary Key`` () = let hk = guid() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs index 115bc78..db5f1e5 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs @@ -97,7 +97,7 @@ type ``Projection Expression Tests`` (fixture : TableFixture) = Serialized = rand(), guid() ; Serialized2 = { NV = guid() ; NE = enum (int (rand()) % 3) } ; } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() let [] ``Should fail on invalid projections`` () = let testProj (p : Expr 'T>) = diff --git a/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs index 5ed0710..752d307 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs @@ -56,7 +56,7 @@ type ``Simple Table Operation Tests`` (fixture : TableFixture) = Unions = [Choice1Of3 (guid()) ; Choice2Of3(rand()) ; Choice3Of3(Guid.NewGuid().ToByteArray())] } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() let [] ``Convert to compatible table`` () = let table' = table.WithRecordType () diff --git a/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs b/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs index 9a68be7..f553b30 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs @@ -31,7 +31,7 @@ type ``Sparse GSI Tests`` (fixture : TableFixture) = SecondaryHashKey = if rand() % 2L = 0L then Some (guid()) else None ; } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() let [] ``GSI Put Operation`` () = let value = mkItem() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs index ff4ff7d..ea95d2d 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs @@ -94,7 +94,7 @@ type ``Update Expression Tests``(fixture : TableFixture) = Serialized = rand(), guid() ; Serialized2 = { NV = guid() ; NE = enum (int (rand()) % 3) } ; } - let table = TableContext.Create(fixture.Client, fixture.TableName, createIfNotExists = true) + let table = fixture.CreateContextAndTableIfNotExists() let [] ``Attempt to update HashKey`` () = let item = mkItem() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs index fc59b79..264893e 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs @@ -6,6 +6,8 @@ open System.IO open FsCheck open Swensen.Unquote +open FSharp.AWS.DynamoDB + open Amazon.DynamoDBv2 open Amazon.Runtime open Xunit @@ -42,9 +44,12 @@ module Utils = member _.Client = client member _.TableName = tableName + member __.CreateContextAndTableIfNotExists<'TRecord>() = + let autoCreate = InitializationMode.CreateIfNotExists (ProvisionedThroughput(10, 10)) + Scripting.TableContext.Initialize<'TRecord>(__.Client, __.TableName, mode = autoCreate) + interface IAsyncLifetime with member _.InitializeAsync() = System.Threading.Tasks.Task.CompletedTask member _.DisposeAsync() = client.DeleteTableAsync(tableName) - From d0b8e74ff630e1440ea34124c51b2818172406b7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 28 Mar 2022 14:47:13 +0100 Subject: [PATCH 04/22] Readme updates and extended examples --- README.md | 42 +++++++----- src/FSharp.AWS.DynamoDB/Script.fsx | 85 +++++++++++++++++++++++- src/FSharp.AWS.DynamoDB/TableContext.fs | 6 +- tests/FSharp.AWS.DynamoDB.Tests/Utils.fs | 2 +- 4 files changed, 115 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ebbf77f..b2b0f10 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,11 @@ We can now perform table operations on DynamoDB like so: ```fsharp open Amazon.DynamoDBv2 +open FSharp.AWS.DynamoDB.Scripting // Expose non-Async methods, e.g. PutItem/GetItem let client : IAmazonDynamoDB = ``your DynamoDB client instance`` -let table = TableContext.Create(client, tableName = "workItems", createIfNotExists = true) +let autoCreate = InitializationMode.CreateIfNotExists (ProvisionedThroughput(10, 10)) +let table = TableContext.Initialize(client, tableName = "workItems", mode = autoCreate) let workItem = { ProcessId = 0L ; WorkItemId = 1L ; Name = "Test" ; UUID = guid() ; Dependencies = set ["mscorlib"] ; Started = None } @@ -99,24 +101,34 @@ Update expressions support the following F# value constructors: * `Option.Value` and `Option.get`. * `fst` and `snd` for tuple records. -## Example: Creating an atomic counter +## Example: Representing an atomic counter as an Item in a DynamoDB Table ```fsharp type private CounterEntry = { [] Id : Guid ; Value : int64 } type Counter private (table : TableContext, key : TableKey) = - member _.Value = table.GetItem(key).Value - member _.Incr() = - let updated = table.UpdateItem(key, <@ fun e -> { e with Value = e.Value + 1L } @>) - updated.Value - - static member Create(client : IAmazonDynamoDB, table : string) = - let table = TableContext.Create(client, table, createIfNotExists = true) - let entry = { Id = Guid.NewGuid() ; Value = 0L } - let key = table.PutItem entry - new Counter(table, key) + + member _.Value = async { + let! current = table.GetItemAsync(key) + return current.Value + } + + member _.Incr() = async { + let! updated = table.UpdateItemAsync(key, <@ fun e -> { e with Value = e.Value + 1L } @>) + return updated.Value + } + + static member Create(client : IAmazonDynamoDB, tableName : string) = async { + let table = TableContext(client, tableName) + do! table.InitializeTableAsync( ProvisionedThroughput(100L, 100L)) + let initialEntry = { Id = Guid.NewGuid() ; Value = 0L } + let! key = table.PutItemAsync(initialEntry) + return Counter(table, key) + } ``` +_NOTE: It's advised to split single time initialization/verification of table creation from the application logic, see [`Script.fsx`](src/FSharp.AWS.DynamoDB/Script.fsx#99) for further details_. + ## Projection Expressions Projection expressions can be used to fetch a subset of table attributes, which can be useful when performing large queries: @@ -233,10 +245,10 @@ table.Scan(startedBefore (DateTimeOffset.Now - TimeSpan.FromDays 1.)) A hook is provided so metrics can be published via your preferred Observability provider. For example, using [Prometheus.NET](https://github.com/prometheus-net/prometheus-net): ```fsharp -let dbCounter = Metrics.CreateCounter ("aws_dynamodb_requests_total", "Count of all DynamoDB requests", "table", "operation") +let dbCounter = Prometheus.Metrics.CreateCounter("aws_dynamodb_requests_total", "Count of all DynamoDB requests", "table", "operation") let processMetrics (m : RequestMetrics) = - dbCounter.WithLabels(m.TableName, string m.Operation).Inc () |> ignore -let table = TableContext.Create(client, tableName = "workItems", metricsCollector = processMetrics) + dbCounter.WithLabels(m.TableName, string m.Operation).Inc() +let table = TableContext(client, tableName = "workItems", metricsCollector = processMetrics) ``` If `metricsCollector` is supplied, the requests will include `ReturnConsumedCapacity = ReturnConsumedCapacity.INDEX` diff --git a/src/FSharp.AWS.DynamoDB/Script.fsx b/src/FSharp.AWS.DynamoDB/Script.fsx index 10c835b..b35627f 100644 --- a/src/FSharp.AWS.DynamoDB/Script.fsx +++ b/src/FSharp.AWS.DynamoDB/Script.fsx @@ -54,8 +54,8 @@ type Test = Bytes : byte[] } - -let table = TableContext.Create(ddb, "test", createIfNotExists = true) +let autoCreate = InitializationMode.CreateIfNotExists (ProvisionedThroughput (100L, 100L)) +let table = TableContext.Initialize(ddb, "test", mode = autoCreate) let value = { HashKey = Guid.NewGuid() ; List = [] ; RangeKey = "2" ; Value = 3.1415926 ; Date = DateTimeOffset.Now + TimeSpan.FromDays 2. ; Value2 = None ; Values = [|{ A = "foo" ; B = System.Reflection.BindingFlags.Instance }|] ; Map = Map.ofList [("A1",1)] ; Set = [set [1L];set [2L]] ; Bytes = [|1uy..10uy|]; String = ref "1a" ; Unions = [A 42; B("42",3)]} @@ -94,3 +94,84 @@ let uexpr2 = table.Template.PrecomputeUpdateExpr <@ fun v r -> { r with Value2 = for i = 1 to 1000 do let _ = table.UpdateItem(key, uexpr2 (Some 42)) () + +(* Expanded version of README sample that illustrates how one can better split Table initialization from application logic *) + +type internal CounterEntry = { [] Id : Guid ; Value : int64 } + +/// Represents a single Item in a Counters Table +type Counter internal (table : TableContext, key : TableKey) = + + static member internal Start(table : TableContext) = async { + let initialEntry = { Id = Guid.NewGuid() ; Value = 0L } + let! key = table.PutItemAsync(initialEntry) + return Counter(table, key) + } + + member _.Value = async { + let! current = table.GetItemAsync(key) + return current.Value + } + + member _.Incr() = async { + let! updated = table.UpdateItemAsync(key, <@ fun (e : CounterEntry) -> { e with Value = e.Value + 1L } @>) + return updated.Value + } + +/// Wrapper that creates/verifies the table only once per call to Create() +/// This does assume that your application will be sufficiently privileged to create tables on the fly +type EasyCounters private (table : TableContext) = + + // We only want to do the initialization bit once per instance of our application + static member Create(client : IAmazonDynamoDB, tableName : string) : Async = async { + let table = TableContext(client, tableName) + // Create the table if necessary. Verifies schema is correct if it has already been created + do! table.InitializeTableAsync( ProvisionedThroughput(100L, 100L)) + return EasyCounters(table) + } + + member _.StartCounter() : Async = + Counter.Start table + +/// Variant of EasyCounters that splits the provisioning step from the (optional) validation that the table is present +type SimpleCounters private (table : TableContext) = + + static member Provision(client : IAmazonDynamoDB, tableName : string, readCapacityUnits, writeCapacityUnits) : Async = + let table = TableContext(client, tableName) + // normally, RCU/WCU provisioning only happens first time the Table is created and is then considered an external concern + // here we use `updateThroughputIfExists = true` to reset it each time we start the app + table.InitializeTableAsync(ProvisionedThroughput (readCapacityUnits, writeCapacityUnits), updateThroughputIfExists = true) + + /// We only want to do the initialization bit once per instance of our application + /// Similar to EasyCounters.Create in that it ensures the table is provisioned correctly + /// However it will never actually create the table + static member CreateWithVerify(client : IAmazonDynamoDB, tableName : string) : Async = async { + let table = TableContext(client, tableName) + // This validates the Table has been created correctly + // (in general this is a good idea, but it is an optional step so it can be skipped, i.e. see Create() below) + do! table.VerifyTableAsync() + return SimpleCounters(table) + } + + /// Assumes the table has been provisioned externally via Provision() + static member Create(client : IAmazonDynamoDB, tableName : string) : SimpleCounters = + // NOTE we are skipping + SimpleCounters(TableContext(client, tableName)) + + member _.StartCounter() : Async = + Counter.Start table + +let e = EasyCounters.Create(ddb, "testing") |> Async.RunSynchronously +let e1 = e.StartCounter() |> Async.RunSynchronously +let e2 = e.StartCounter() |> Async.RunSynchronously +e1.Incr() |> Async.RunSynchronously +e2.Incr() |> Async.RunSynchronously + +SimpleCounters.Provision(ddb, "testing-pre-provisioned", 100L, 100L) |> Async.RunSynchronously +let s = SimpleCounters.Create(ddb, "testing-pre-provisioned") +let s1 = s.StartCounter() |> Async.RunSynchronously // Would throw if Provision has not been carried out +s1.Incr() |> Async.RunSynchronously + +let v = SimpleCounters.CreateWithVerify(ddb, "testing-not-present") |> Async.RunSynchronously // Throws, as table not present +let v2 = v.StartCounter() |> Async.RunSynchronously +v2.Incr() |> Async.RunSynchronously diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 91be3ab..a02df20 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -908,7 +908,9 @@ type TableContext<'TRecord> internal |> invalidOp match mode with | InitializationMode.VerifyOnly | InitializationMode.CreateIfNotExists _ -> () - | InitializationMode.CreateOrUpdateThroughput t -> do! __.UpdateProvisionedThroughputAsync(t) + | InitializationMode.CreateOrUpdateThroughput t -> + // TODO make this not throw when its a null update + do! __.UpdateProvisionedThroughputAsync(t) | Choice2Of2 (:? ResourceNotFoundException) when mode <> InitializationMode.VerifyOnly -> let throughput = @@ -986,7 +988,7 @@ type TableContext internal () = /// Table name to target. /// Verify that the table exists and is compatible with supplied record schema. Defaults to true. /// Create the table now if it does not exist. Defaults to false. - /// Provisioned throughput for the table if newly created. + /// Provisioned throughput for the table if newly created. Default: 10 RCU, 10 WCU /// Function to receive request metrics. [() = - let autoCreate = InitializationMode.CreateIfNotExists (ProvisionedThroughput(10, 10)) + let autoCreate = InitializationMode.CreateIfNotExists (ProvisionedThroughput(10L, 10L)) Scripting.TableContext.Initialize<'TRecord>(__.Client, __.TableName, mode = autoCreate) interface IAsyncLifetime with From 0eb827e212ccb8d0ff88467fe6ba3e10f5156af3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 28 Mar 2022 15:18:33 +0100 Subject: [PATCH 05/22] Add guard --- src/FSharp.AWS.DynamoDB/TableContext.fs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index a02df20..73ed3af 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -909,8 +909,10 @@ type TableContext<'TRecord> internal match mode with | InitializationMode.VerifyOnly | InitializationMode.CreateIfNotExists _ -> () | InitializationMode.CreateOrUpdateThroughput t -> - // TODO make this not throw when its a null update - do! __.UpdateProvisionedThroughputAsync(t) + let provisioned = td.Table.ProvisionedThroughput + if t.ReadCapacityUnits <> provisioned.ReadCapacityUnits + || t.WriteCapacityUnits <> provisioned.WriteCapacityUnits then + do! __.UpdateProvisionedThroughputAsync(t) | Choice2Of2 (:? ResourceNotFoundException) when mode <> InitializationMode.VerifyOnly -> let throughput = From 306a5d541a0f797911ede0bfcf72c99ae622ddb6 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 28 Mar 2022 16:54:09 +0100 Subject: [PATCH 06/22] Refactor InitializationMode to ProvisionTableAsync --- README.md | 7 +- RELEASE_NOTES.md | 3 +- src/FSharp.AWS.DynamoDB/Script.fsx | 12 +- src/FSharp.AWS.DynamoDB/TableContext.fs | 137 ++++++++++------------- tests/FSharp.AWS.DynamoDB.Tests/Utils.fs | 6 +- 5 files changed, 80 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index b2b0f10..72464e2 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ open Amazon.DynamoDBv2 open FSharp.AWS.DynamoDB.Scripting // Expose non-Async methods, e.g. PutItem/GetItem let client : IAmazonDynamoDB = ``your DynamoDB client instance`` -let autoCreate = InitializationMode.CreateIfNotExists (ProvisionedThroughput(10, 10)) -let table = TableContext.Initialize(client, tableName = "workItems", mode = autoCreate) +let throughput = ProvisionedThroughput (readCapacityUnits = 1L, writeCapacityUnits = 10L) +let table = TableContext.Initialize(client, tableName = "workItems", throughput) let workItem = { ProcessId = 0L ; WorkItemId = 1L ; Name = "Test" ; UUID = guid() ; Dependencies = set ["mscorlib"] ; Started = None } @@ -120,7 +120,8 @@ type Counter private (table : TableContext, key : TableKey) = static member Create(client : IAmazonDynamoDB, tableName : string) = async { let table = TableContext(client, tableName) - do! table.InitializeTableAsync( ProvisionedThroughput(100L, 100L)) + let throughput = ProvisionedThroughput(readCapacityUnits = 100L, writeCapacityUnits = 100L) + do! table.InitializeTableAsync throughput let initialEntry = { Id = Guid.NewGuid() ; Value = 0L } let! key = table.PutItemAsync(initialEntry) return Counter(table, key) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 80aeac9..855eaf7 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -10,7 +10,8 @@ * Added `TableContext.Scripting.Initialize` (replaces `TableContext.Create()`) * Added `TableContext.VerifyTableAsync` overload that only performs verification but never creates a Table * Added `TableContext.InitializeTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) - * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `TableContext.InitializeTableAsync`) + * Added `TableContext.ProvisionTableAsync` (as per `InitializeTableAsync` but does an `UpdateProvisionedThroughputAsync` if throughput has changed) + * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `InitializeTableAsync`) ### 0.9.3-beta * Added `RequestMetrics` record type diff --git a/src/FSharp.AWS.DynamoDB/Script.fsx b/src/FSharp.AWS.DynamoDB/Script.fsx index b35627f..b915661 100644 --- a/src/FSharp.AWS.DynamoDB/Script.fsx +++ b/src/FSharp.AWS.DynamoDB/Script.fsx @@ -126,7 +126,8 @@ type EasyCounters private (table : TableContext) = static member Create(client : IAmazonDynamoDB, tableName : string) : Async = async { let table = TableContext(client, tableName) // Create the table if necessary. Verifies schema is correct if it has already been created - do! table.InitializeTableAsync( ProvisionedThroughput(100L, 100L)) + let throughput = ProvisionedThroughput(readCapacityUnits = 100L, writeCapacityUnits = 100L) + do! table.InitializeTableAsync throughput return EasyCounters(table) } @@ -139,8 +140,8 @@ type SimpleCounters private (table : TableContext) = static member Provision(client : IAmazonDynamoDB, tableName : string, readCapacityUnits, writeCapacityUnits) : Async = let table = TableContext(client, tableName) // normally, RCU/WCU provisioning only happens first time the Table is created and is then considered an external concern - // here we use `updateThroughputIfExists = true` to reset it each time we start the app - table.InitializeTableAsync(ProvisionedThroughput (readCapacityUnits, writeCapacityUnits), updateThroughputIfExists = true) + // here we use `ProvisionTableAsync` instead of `InitializeAsync` to reset it each time we start the app + table.ProvisionTableAsync(ProvisionedThroughput (readCapacityUnits, writeCapacityUnits)) /// We only want to do the initialization bit once per instance of our application /// Similar to EasyCounters.Create in that it ensures the table is provisioned correctly @@ -168,10 +169,15 @@ e1.Incr() |> Async.RunSynchronously e2.Incr() |> Async.RunSynchronously SimpleCounters.Provision(ddb, "testing-pre-provisioned", 100L, 100L) |> Async.RunSynchronously +// The consuming code can assume the provisioning has been carried out as part of the deploy +// that allows the creation to be synchronous (and not impede application startup) let s = SimpleCounters.Create(ddb, "testing-pre-provisioned") let s1 = s.StartCounter() |> Async.RunSynchronously // Would throw if Provision has not been carried out s1.Incr() |> Async.RunSynchronously +// Alternately, we can have the app do an extra call (and have some asynchronous initialization work) to check the table is ready let v = SimpleCounters.CreateWithVerify(ddb, "testing-not-present") |> Async.RunSynchronously // Throws, as table not present let v2 = v.StartCounter() |> Async.RunSynchronously v2.Incr() |> Async.RunSynchronously + +// (TOCONSIDER: Illustrate how to use AsyncCacheCell from https://github.com/jet/equinox/blob/master/src/Equinox.Core/AsyncCacheCell.fs to make Verify call lazy) diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 73ed3af..76c2878 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -46,16 +46,6 @@ type private LimitType = All | Default | Count of int static member AllOrCount (l : int option) = l |> Option.map Count |> Option.defaultValue All static member DefaultOrCount (l : int option) = l |> Option.map Count |> Option.defaultValue Default -/// Defines Verification and/or Provisioning steps to be applied when initializing and/or validating a TableContext -[] -type InitializationMode = - /// Validate the Table exists and has a compatible schema - | VerifyOnly - /// Perform a Validation step as per VerifyOnly, but Create the Table with the specified Throughput if it was not found - | CreateIfNotExists of provisionedThroughput : ProvisionedThroughput - /// Validate and/or Create as per CreateIfNotExists, but also re-apply the specified Throughput in case it has diverged - | CreateOrUpdateThroughput of provisionedThroughput : ProvisionedThroughput - /// DynamoDB client object for performing table operations in the context of given F# record representations [] type TableContext<'TRecord> internal @@ -879,69 +869,61 @@ type TableContext<'TRecord> internal } - member __.InternalInitializeAsync(mode : InitializationMode) : Async = async { + member internal _.InternalDescribe() : Async = + let rec wait () = async { + let! ct = Async.CancellationToken + let! td = client.DescribeTableAsync(tableName, ct) |> Async.AwaitTaskCorrect + if td.Table.TableStatus = TableStatus.ACTIVE then + do! Async.Sleep 2000 + // wait indefinitely if table is in transition state + return! wait () + else + + return td.Table + } + wait () + + member internal __.CreateOrValidateTableAsync(createThroughput) : Async = let (|Conflict|_|) (e : exn) = match e with | :? AmazonDynamoDBException as e when e.StatusCode = HttpStatusCode.Conflict -> Some() | :? ResourceInUseException -> Some () | _ -> None - let rec verify retries = async { - let! ct = Async.CancellationToken - let! response = - client.DescribeTableAsync(tableName, ct) - |> Async.AwaitTaskCorrect - |> Async.Catch - - match response with - | Choice1Of2 td -> - if td.Table.TableStatus <> TableStatus.ACTIVE then - do! Async.Sleep 2000 - // wait indefinitely if table is in transition state - return! verify retries - else - - let existingSchema = TableKeySchemata.OfTableDescription td.Table + let rec checkOrCreate retries = async { + match! __.InternalDescribe() |> Async.Catch with + | Choice1Of2 desc -> + let existingSchema = TableKeySchemata.OfTableDescription desc if existingSchema <> template.Info.Schemata then sprintf "table '%s' exists with key schema %A, which is incompatible with record '%O'." tableName existingSchema typeof<'TRecord> |> invalidOp - match mode with - | InitializationMode.VerifyOnly | InitializationMode.CreateIfNotExists _ -> () - | InitializationMode.CreateOrUpdateThroughput t -> - let provisioned = td.Table.ProvisionedThroughput - if t.ReadCapacityUnits <> provisioned.ReadCapacityUnits - || t.WriteCapacityUnits <> provisioned.WriteCapacityUnits then - do! __.UpdateProvisionedThroughputAsync(t) - - | Choice2Of2 (:? ResourceNotFoundException) when mode <> InitializationMode.VerifyOnly -> - let throughput = - match mode with - | InitializationMode.VerifyOnly -> failwith "Unexpected" // the when guard should preclude this - | InitializationMode.CreateIfNotExists t | InitializationMode.CreateOrUpdateThroughput t -> t - let ctr = template.Info.Schemata.CreateCreateTableRequest(tableName, throughput) + return desc + + | Choice2Of2 (:? ResourceNotFoundException) when Option.isSome createThroughput -> + let! ct = Async.CancellationToken + let ctr = template.Info.Schemata.CreateCreateTableRequest(tableName, Option.get createThroughput) let! response = client.CreateTableAsync(ctr, ct) |> Async.AwaitTaskCorrect |> Async.Catch match response with - | Choice1Of2 _ -> return! verify retries + | Choice1Of2 _ -> return! checkOrCreate retries | Choice2Of2 Conflict when retries > 0 -> do! Async.Sleep 2000 - return! verify (retries - 1) + return! checkOrCreate (retries - 1) - | Choice2Of2 e -> do! Async.Raise e + | Choice2Of2 e -> return! Async.Raise e | Choice2Of2 Conflict when retries > 0 -> do! Async.Sleep 2000 - return! verify (retries - 1) + return! checkOrCreate (retries - 1) - | Choice2Of2 e -> do! Async.Raise e + | Choice2Of2 e -> return! Async.Raise e } - do! verify 9 // up to 9 retries, i.e. 10 attempts before we let exception propagate - } + checkOrCreate 9 // up to 9 retries, i.e. 10 attempts before we let exception propagate /// /// Asynchronously verify that the table exists and is compatible with record key schema. @@ -950,33 +932,41 @@ type TableContext<'TRecord> internal /// Provisioned throughput for the table if newly created. Defaults to (10,10). [] member __.VerifyTableAsync(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) : Async = - let mode = + let throughputIfCreate = if createIfNotExists = Some true then let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) - InitializationMode.CreateIfNotExists throughput - else InitializationMode.VerifyOnly - __.InternalInitializeAsync(mode) + Some throughput + else None + __.CreateOrValidateTableAsync(throughputIfCreate) |> Async.Ignore /// /// Asynchronously verify that the table exists and is compatible with record key schema, or throw.
/// See also InitializeTableAsync, which performs the same check, but can create or re-provision the Table if required. /// member __.VerifyTableAsync() : Async = - __.InternalInitializeAsync(InitializationMode.VerifyOnly) + __.CreateOrValidateTableAsync(None) |> Async.Ignore /// /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
/// If the table is not present, it is provisioned, with the specified throughput.
- /// Optionally can reapply the cited throughput level if the Table has already been created.
/// See also VerifyTableAsync, which only verifies the Table is present and correct. /// /// Provisioned throughput to use for the table. - /// Trigger re-provisioning of the Table's throughput if it was already present. Default: false - member __.InitializeTableAsync(throughput : ProvisionedThroughput, ?updateThroughputIfExists) : Async = - let mode = - if updateThroughputIfExists = Some true then InitializationMode.CreateOrUpdateThroughput - else InitializationMode.CreateIfNotExists - __.InternalInitializeAsync(mode throughput) + member __.InitializeTableAsync(throughput : ProvisionedThroughput) : Async = + __.CreateOrValidateTableAsync(Some throughput) |> Async.Ignore + + /// + /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
+ /// If the table is not present, it is provisioned, with the specified throughput.
+ /// If it is present, and the throughput is not as specified, uses UpdateProvisionedThroughputAsync to update it.
+ /// + /// Provisioned throughput to use for the table. + member __.ProvisionTableAsync(throughput : ProvisionedThroughput) : Async = async { + let! desc = __.CreateOrValidateTableAsync(Some throughput) + let provisioned = desc.ProvisionedThroughput + if throughput.ReadCapacityUnits <> provisioned.ReadCapacityUnits + || throughput.WriteCapacityUnits <> provisioned.WriteCapacityUnits then + do! __.UpdateProvisionedThroughputAsync(throughput) } // Deprecated factory method, to be removed. Replaced with // 1. TableContext<'T> ctor (synchronous) @@ -998,17 +988,14 @@ type TableContext internal () = static member Create<'TRecord> ( client : IAmazonDynamoDB, tableName : string, ?verifyTable : bool, ?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput, - ?metricsCollector : RequestMetrics -> unit) = + ?metricsCollector : RequestMetrics -> unit) = async { let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) - let verifyTable, createIfNotExists = verifyTable <> Some false, createIfNotExists = Some true - if verifyTable || createIfNotExists then - let mode = - if createIfNotExists then - let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) - InitializationMode.CreateIfNotExists throughput - else InitializationMode.VerifyOnly - context.InternalInitializeAsync(mode) |> Async.RunSynchronously - context + if createIfNotExists = Some true then + let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) + do! context.InitializeTableAsync throughput + elif verifyTable <> Some false then + do! context.VerifyTableAsync() + return context } /// /// Sync-over-Async helpers that can be opted-into when working in scripting scenarios. @@ -1023,15 +1010,15 @@ module Scripting = /// Creates a DynamoDB client instance for the specified F# record type, client and table name. /// DynamoDB client instance. /// Table name to target. - /// Allows one to define auto-creation or re-provisioning rules via . + /// Optional throughput to configure if the Table does not yet exist. /// Function to receive request metrics. static member Initialize<'TRecord> - ( client : IAmazonDynamoDB, tableName : string, - ?mode : InitializationMode, + ( client : IAmazonDynamoDB, tableName : string, ?throughput, ?metricsCollector : RequestMetrics -> unit) : TableContext<'TRecord> = let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) - let mode = defaultArg mode InitializationMode.VerifyOnly - context.InternalInitializeAsync(mode) |> Async.RunSynchronously + match throughput with + | None -> context.VerifyTableAsync() |> Async.RunSynchronously + | Some t -> context.InitializeTableAsync(t) |> Async.RunSynchronously context type TableContext<'TRecord> with diff --git a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs index 065ce9e..b1b0c40 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs @@ -44,9 +44,9 @@ module Utils = member _.Client = client member _.TableName = tableName - member __.CreateContextAndTableIfNotExists<'TRecord>() = - let autoCreate = InitializationMode.CreateIfNotExists (ProvisionedThroughput(10L, 10L)) - Scripting.TableContext.Initialize<'TRecord>(__.Client, __.TableName, mode = autoCreate) + member _.CreateContextAndTableIfNotExists<'TRecord>() = + let createThroughput = ProvisionedThroughput(10L, 10L) + Scripting.TableContext.Initialize<'TRecord>(client, tableName, createThroughput) interface IAsyncLifetime with member _.InitializeAsync() = From 88194b9ee45ef172bbce2e83b74476d44226bd11 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 28 Mar 2022 17:25:35 +0100 Subject: [PATCH 07/22] Extract InternalCreateOrValidateTableAsync --- src/FSharp.AWS.DynamoDB/TableContext.fs | 55 +++++++++++++------------ 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 76c2878..bb344f3 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -868,7 +868,6 @@ type TableContext<'TRecord> internal return () } - member internal _.InternalDescribe() : Async = let rec wait () = async { let! ct = Async.CancellationToken @@ -883,7 +882,7 @@ type TableContext<'TRecord> internal } wait () - member internal __.CreateOrValidateTableAsync(createThroughput) : Async = + member internal __.InternalProvision(?makeCreateTableRequest) : Async = let (|Conflict|_|) (e : exn) = match e with | :? AmazonDynamoDBException as e when e.StatusCode = HttpStatusCode.Conflict -> Some() @@ -900,11 +899,10 @@ type TableContext<'TRecord> internal |> invalidOp return desc - | Choice2Of2 (:? ResourceNotFoundException) when Option.isSome createThroughput -> + | Choice2Of2 (:? ResourceNotFoundException) when Option.isSome makeCreateTableRequest -> let! ct = Async.CancellationToken - let ctr = template.Info.Schemata.CreateCreateTableRequest(tableName, Option.get createThroughput) let! response = - client.CreateTableAsync(ctr, ct) + client.CreateTableAsync(makeCreateTableRequest.Value (), ct) |> Async.AwaitTaskCorrect |> Async.Catch @@ -925,26 +923,17 @@ type TableContext<'TRecord> internal checkOrCreate 9 // up to 9 retries, i.e. 10 attempts before we let exception propagate - /// - /// Asynchronously verify that the table exists and is compatible with record key schema. - /// - /// Create the table instance now instance if it does not exist. Defaults to false. - /// Provisioned throughput for the table if newly created. Defaults to (10,10). - [] - member __.VerifyTableAsync(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) : Async = - let throughputIfCreate = - if createIfNotExists = Some true then - let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) - Some throughput - else None - __.CreateOrValidateTableAsync(throughputIfCreate) |> Async.Ignore + member internal _.InternalCreateTableRequest(throughput) = + template.Info.Schemata.CreateCreateTableRequest(tableName, throughput) + member internal t.InternalCreateOrValidateTableAsync(createThroughput) = + t.InternalProvision(fun () -> t.InternalCreateTableRequest(createThroughput)) /// /// Asynchronously verify that the table exists and is compatible with record key schema, or throw.
/// See also InitializeTableAsync, which performs the same check, but can create or re-provision the Table if required. /// member __.VerifyTableAsync() : Async = - __.CreateOrValidateTableAsync(None) |> Async.Ignore + __.InternalProvision() |> Async.Ignore /// /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
@@ -952,8 +941,8 @@ type TableContext<'TRecord> internal /// See also VerifyTableAsync, which only verifies the Table is present and correct. /// /// Provisioned throughput to use for the table. - member __.InitializeTableAsync(throughput : ProvisionedThroughput) : Async = - __.CreateOrValidateTableAsync(Some throughput) |> Async.Ignore + member t.InitializeTableAsync(throughput : ProvisionedThroughput) : Async = + t.InternalCreateOrValidateTableAsync(throughput) |> Async.Ignore /// /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
@@ -961,12 +950,24 @@ type TableContext<'TRecord> internal /// If it is present, and the throughput is not as specified, uses UpdateProvisionedThroughputAsync to update it.
/// /// Provisioned throughput to use for the table. - member __.ProvisionTableAsync(throughput : ProvisionedThroughput) : Async = async { - let! desc = __.CreateOrValidateTableAsync(Some throughput) - let provisioned = desc.ProvisionedThroughput - if throughput.ReadCapacityUnits <> provisioned.ReadCapacityUnits - || throughput.WriteCapacityUnits <> provisioned.WriteCapacityUnits then - do! __.UpdateProvisionedThroughputAsync(throughput) } + member t.ProvisionTableAsync(throughput : ProvisionedThroughput) : Async = async { + let! tableDescription = t.InternalCreateOrValidateTableAsync(throughput) + let provisioned = tableDescription.ProvisionedThroughput + if throughput.ReadCapacityUnits <> provisioned.ReadCapacityUnits || throughput.WriteCapacityUnits <> provisioned.WriteCapacityUnits then + do! t.UpdateProvisionedThroughputAsync(throughput) } + + /// + /// Asynchronously verify that the table exists and is compatible with record key schema. + /// + /// Create the table instance now instance if it does not exist. Defaults to false. + /// Provisioned throughput for the table if newly created. Defaults to (10,10). + [] + member t.VerifyTableAsync(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) : Async = + if createIfNotExists = Some true then + let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) + t.InitializeTableAsync(throughput) + else + t.VerifyTableAsync() // Deprecated factory method, to be removed. Replaced with // 1. TableContext<'T> ctor (synchronous) From dcf37566aaafd6e24638641070f9daf1742aea17 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 29 Mar 2022 17:31:39 +0100 Subject: [PATCH 08/22] Add OnDemand mode --- README.md | 8 ++-- RELEASE_NOTES.md | 1 + src/FSharp.AWS.DynamoDB/RecordKeySchema.fs | 5 +-- src/FSharp.AWS.DynamoDB/Script.fsx | 16 ++++---- src/FSharp.AWS.DynamoDB/TableContext.fs | 45 +++++++++++++++------- tests/FSharp.AWS.DynamoDB.Tests/Utils.fs | 4 +- 6 files changed, 49 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 72464e2..009471b 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,8 @@ open Amazon.DynamoDBv2 open FSharp.AWS.DynamoDB.Scripting // Expose non-Async methods, e.g. PutItem/GetItem let client : IAmazonDynamoDB = ``your DynamoDB client instance`` -let throughput = ProvisionedThroughput (readCapacityUnits = 1L, writeCapacityUnits = 10L) -let table = TableContext.Initialize(client, tableName = "workItems", throughput) +let throughput = ProvisionedThroughput(readCapacityUnits = 1L, writeCapacityUnits = 10L) +let table = TableContext.Initialize(client, tableName = "workItems", Provisioned throughput) let workItem = { ProcessId = 0L ; WorkItemId = 1L ; Name = "Test" ; UUID = guid() ; Dependencies = set ["mscorlib"] ; Started = None } @@ -120,8 +120,8 @@ type Counter private (table : TableContext, key : TableKey) = static member Create(client : IAmazonDynamoDB, tableName : string) = async { let table = TableContext(client, tableName) - let throughput = ProvisionedThroughput(readCapacityUnits = 100L, writeCapacityUnits = 100L) - do! table.InitializeTableAsync throughput + let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) + do! table.InitializeTableAsync(Provisioned throughput) let initialEntry = { Id = Guid.NewGuid() ; Value = 0L } let! key = table.PutItemAsync(initialEntry) return Counter(table, key) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 855eaf7..7958f6c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -11,6 +11,7 @@ * Added `TableContext.VerifyTableAsync` overload that only performs verification but never creates a Table * Added `TableContext.InitializeTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) * Added `TableContext.ProvisionTableAsync` (as per `InitializeTableAsync` but does an `UpdateProvisionedThroughputAsync` if throughput has changed) + * Added Support for `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST` rather than attempting to configure a `ProvisionedThroughput`) * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `InitializeTableAsync`) ### 0.9.3-beta diff --git a/src/FSharp.AWS.DynamoDB/RecordKeySchema.fs b/src/FSharp.AWS.DynamoDB/RecordKeySchema.fs index 434166f..3780d86 100644 --- a/src/FSharp.AWS.DynamoDB/RecordKeySchema.fs +++ b/src/FSharp.AWS.DynamoDB/RecordKeySchema.fs @@ -369,12 +369,10 @@ type TableKeySchemata with yield! td.LocalSecondaryIndexes |> Seq.map mkLocalSecondaryIndex |]) /// Create a CreateTableRequest using supplied key schema - member schema.CreateCreateTableRequest (tableName : string, provisionedThroughput : ProvisionedThroughput) = + member schema.CreateCreateTableRequest(tableName : string) = let ctr = CreateTableRequest(TableName = tableName) let inline mkKSE n t = KeySchemaElement(n, t) - ctr.ProvisionedThroughput <- provisionedThroughput - let keyAttrs = new Dictionary() for tks in schema.Schemata do keyAttrs.[tks.HashKey.AttributeName] <- tks.HashKey @@ -391,7 +389,6 @@ type TableKeySchemata with gsi.KeySchema.Add <| mkKSE tks.HashKey.AttributeName KeyType.HASH tks.RangeKey |> Option.iter (fun rk -> gsi.KeySchema.Add <| mkKSE rk.AttributeName KeyType.RANGE) gsi.Projection <- Projection(ProjectionType = ProjectionType.ALL) - gsi.ProvisionedThroughput <- provisionedThroughput ctr.GlobalSecondaryIndexes.Add gsi | LocalSecondaryIndex name -> diff --git a/src/FSharp.AWS.DynamoDB/Script.fsx b/src/FSharp.AWS.DynamoDB/Script.fsx index b915661..596a89f 100644 --- a/src/FSharp.AWS.DynamoDB/Script.fsx +++ b/src/FSharp.AWS.DynamoDB/Script.fsx @@ -54,8 +54,8 @@ type Test = Bytes : byte[] } -let autoCreate = InitializationMode.CreateIfNotExists (ProvisionedThroughput (100L, 100L)) -let table = TableContext.Initialize(ddb, "test", mode = autoCreate) +let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) +let table = TableContext.Initialize(ddb, "test", Provisioned throughput) let value = { HashKey = Guid.NewGuid() ; List = [] ; RangeKey = "2" ; Value = 3.1415926 ; Date = DateTimeOffset.Now + TimeSpan.FromDays 2. ; Value2 = None ; Values = [|{ A = "foo" ; B = System.Reflection.BindingFlags.Instance }|] ; Map = Map.ofList [("A1",1)] ; Set = [set [1L];set [2L]] ; Bytes = [|1uy..10uy|]; String = ref "1a" ; Unions = [A 42; B("42",3)]} @@ -126,8 +126,9 @@ type EasyCounters private (table : TableContext) = static member Create(client : IAmazonDynamoDB, tableName : string) : Async = async { let table = TableContext(client, tableName) // Create the table if necessary. Verifies schema is correct if it has already been created - let throughput = ProvisionedThroughput(readCapacityUnits = 100L, writeCapacityUnits = 100L) - do! table.InitializeTableAsync throughput + // NOTE the hard coded initial throughput provisioning - arguably this belongs outside of your application logic + let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) + do! table.InitializeTableAsync(Provisioned throughput) return EasyCounters(table) } @@ -141,7 +142,8 @@ type SimpleCounters private (table : TableContext) = let table = TableContext(client, tableName) // normally, RCU/WCU provisioning only happens first time the Table is created and is then considered an external concern // here we use `ProvisionTableAsync` instead of `InitializeAsync` to reset it each time we start the app - table.ProvisionTableAsync(ProvisionedThroughput (readCapacityUnits, writeCapacityUnits)) + let provisionedThroughput = ProvisionedThroughput(readCapacityUnits, writeCapacityUnits) + table.ProvisionTableAsync(Provisioned provisionedThroughput) /// We only want to do the initialization bit once per instance of our application /// Similar to EasyCounters.Create in that it ensures the table is provisioned correctly @@ -168,11 +170,11 @@ let e2 = e.StartCounter() |> Async.RunSynchronously e1.Incr() |> Async.RunSynchronously e2.Incr() |> Async.RunSynchronously -SimpleCounters.Provision(ddb, "testing-pre-provisioned", 100L, 100L) |> Async.RunSynchronously +SimpleCounters.Provision(ddb, "testing-pre-provisioned", readCapacityUnits = 10L, writeCapacityUnits = 10L) |> Async.RunSynchronously // The consuming code can assume the provisioning has been carried out as part of the deploy // that allows the creation to be synchronous (and not impede application startup) let s = SimpleCounters.Create(ddb, "testing-pre-provisioned") -let s1 = s.StartCounter() |> Async.RunSynchronously // Would throw if Provision has not been carried out +let s1 = s.StartCounter() |> Async.RunSynchronously // Throws if Provision step has not been executed s1.Incr() |> Async.RunSynchronously // Alternately, we can have the app do an extra call (and have some asynchronous initialization work) to check the table is ready diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index bb344f3..020a2b8 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -19,6 +19,19 @@ type ResourceNotFoundException = Amazon.DynamoDBv2.Model.ResourceNotFoundExcepti /// Represents the provisioned throughput for given table or index type ProvisionedThroughput = Amazon.DynamoDBv2.Model.ProvisionedThroughput +/// Represents the throughput configuration for a Table +type Throughput = + | Provisioned of ProvisionedThroughput + | OnDemand +module internal Throughput = + let applyToCreateRequest (req : CreateTableRequest) = function + | Provisioned t -> + req.ProvisionedThroughput <- t + for gsi in req.GlobalSecondaryIndexes do + gsi.ProvisionedThroughput <- t + | OnDemand -> + req.BillingMode <- BillingMode.PAY_PER_REQUEST + /// Represents the operation performed on the table, for metrics collection purposes type Operation = GetItem | PutItem | UpdateItem | DeleteItem | BatchGetItems | BatchWriteItems | Scan | Query @@ -923,10 +936,13 @@ type TableContext<'TRecord> internal checkOrCreate 9 // up to 9 retries, i.e. 10 attempts before we let exception propagate - member internal _.InternalCreateTableRequest(throughput) = - template.Info.Schemata.CreateCreateTableRequest(tableName, throughput) - member internal t.InternalCreateOrValidateTableAsync(createThroughput) = - t.InternalProvision(fun () -> t.InternalCreateTableRequest(createThroughput)) + member internal _.InternalCreateCreateTableRequest(throughput) = + let req = template.Info.Schemata.CreateCreateTableRequest(tableName) + Throughput.applyToCreateRequest req throughput + req + + member internal t.InternalCreateOrValidateTableAsync(throughput) = + t.InternalProvision(fun () -> t.InternalCreateCreateTableRequest throughput) /// /// Asynchronously verify that the table exists and is compatible with record key schema, or throw.
@@ -940,8 +956,8 @@ type TableContext<'TRecord> internal /// If the table is not present, it is provisioned, with the specified throughput.
/// See also VerifyTableAsync, which only verifies the Table is present and correct. /// - /// Provisioned throughput to use for the table. - member t.InitializeTableAsync(throughput : ProvisionedThroughput) : Async = + /// Throughput configuration to use for the table. + member t.InitializeTableAsync(throughput : Throughput) : Async = t.InternalCreateOrValidateTableAsync(throughput) |> Async.Ignore /// @@ -949,12 +965,15 @@ type TableContext<'TRecord> internal /// If the table is not present, it is provisioned, with the specified throughput.
/// If it is present, and the throughput is not as specified, uses UpdateProvisionedThroughputAsync to update it.
/// - /// Provisioned throughput to use for the table. - member t.ProvisionTableAsync(throughput : ProvisionedThroughput) : Async = async { + /// Throughput configuration to use for the table. + member t.ProvisionTableAsync(throughput : Throughput) : Async = async { let! tableDescription = t.InternalCreateOrValidateTableAsync(throughput) - let provisioned = tableDescription.ProvisionedThroughput - if throughput.ReadCapacityUnits <> provisioned.ReadCapacityUnits || throughput.WriteCapacityUnits <> provisioned.WriteCapacityUnits then - do! t.UpdateProvisionedThroughputAsync(throughput) } + match throughput with + | Provisioned p -> + let current = tableDescription.ProvisionedThroughput + if p.ReadCapacityUnits <> current.ReadCapacityUnits || p.WriteCapacityUnits <> current.WriteCapacityUnits then + do! t.UpdateProvisionedThroughputAsync p + | OnDemand -> () } /// /// Asynchronously verify that the table exists and is compatible with record key schema. @@ -965,7 +984,7 @@ type TableContext<'TRecord> internal member t.VerifyTableAsync(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) : Async = if createIfNotExists = Some true then let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) - t.InitializeTableAsync(throughput) + t.InitializeTableAsync(Provisioned throughput) else t.VerifyTableAsync() @@ -993,7 +1012,7 @@ type TableContext internal () = let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) if createIfNotExists = Some true then let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) - do! context.InitializeTableAsync throughput + do! context.InitializeTableAsync(Throughput.Provisioned throughput) elif verifyTable <> Some false then do! context.VerifyTableAsync() return context } diff --git a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs index b1b0c40..6a5007e 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs @@ -45,8 +45,8 @@ module Utils = member _.TableName = tableName member _.CreateContextAndTableIfNotExists<'TRecord>() = - let createThroughput = ProvisionedThroughput(10L, 10L) - Scripting.TableContext.Initialize<'TRecord>(client, tableName, createThroughput) + let throughput = Model.ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) + Scripting.TableContext.Initialize<'TRecord>(client, tableName, Provisioned throughput) interface IAsyncLifetime with member _.InitializeAsync() = From a23ddc4a3936c4a690619fe5979958e84548093b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 4 Apr 2022 16:37:54 +0100 Subject: [PATCH 09/22] Added Streaming impl --- README.md | 4 +- RELEASE_NOTES.md | 3 +- src/FSharp.AWS.DynamoDB/Script.fsx | 6 +- src/FSharp.AWS.DynamoDB/TableContext.fs | 101 +++++++++++++++++------ tests/FSharp.AWS.DynamoDB.Tests/Utils.fs | 2 +- 5 files changed, 82 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 009471b..8a1ab93 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ open FSharp.AWS.DynamoDB.Scripting // Expose non-Async methods, e.g. PutItem/Get let client : IAmazonDynamoDB = ``your DynamoDB client instance`` let throughput = ProvisionedThroughput(readCapacityUnits = 1L, writeCapacityUnits = 10L) -let table = TableContext.Initialize(client, tableName = "workItems", Provisioned throughput) +let table = TableContext.Initialize(client, tableName = "workItems", Throughput.Provisioned throughput) let workItem = { ProcessId = 0L ; WorkItemId = 1L ; Name = "Test" ; UUID = guid() ; Dependencies = set ["mscorlib"] ; Started = None } @@ -121,7 +121,7 @@ type Counter private (table : TableContext, key : TableKey) = static member Create(client : IAmazonDynamoDB, tableName : string) = async { let table = TableContext(client, tableName) let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) - do! table.InitializeTableAsync(Provisioned throughput) + do! table.InitializeTableAsync(Throughput.Provisioned throughput) let initialEntry = { Id = Guid.NewGuid() ; Value = 0L } let! key = table.PutItemAsync(initialEntry) return Counter(table, key) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 7958f6c..c3382ea 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -10,8 +10,9 @@ * Added `TableContext.Scripting.Initialize` (replaces `TableContext.Create()`) * Added `TableContext.VerifyTableAsync` overload that only performs verification but never creates a Table * Added `TableContext.InitializeTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) - * Added `TableContext.ProvisionTableAsync` (as per `InitializeTableAsync` but does an `UpdateProvisionedThroughputAsync` if throughput has changed) + * Added `TableContext.ProvisionTableAsync` (as per `InitializeTableAsync` but does an `UpdateTableAsync` if `throughput` or `streaming` has changed) * Added Support for `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST` rather than attempting to configure a `ProvisionedThroughput`) + * Added ability to configure DynamoDB streaming (via `Streaming` DU) to `InitializeTableAsync` and `ProvisionTableAsync` * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `InitializeTableAsync`) ### 0.9.3-beta diff --git a/src/FSharp.AWS.DynamoDB/Script.fsx b/src/FSharp.AWS.DynamoDB/Script.fsx index 596a89f..5294302 100644 --- a/src/FSharp.AWS.DynamoDB/Script.fsx +++ b/src/FSharp.AWS.DynamoDB/Script.fsx @@ -55,7 +55,7 @@ type Test = } let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) -let table = TableContext.Initialize(ddb, "test", Provisioned throughput) +let table = TableContext.Initialize(ddb, "test", Throughput.Provisioned throughput) let value = { HashKey = Guid.NewGuid() ; List = [] ; RangeKey = "2" ; Value = 3.1415926 ; Date = DateTimeOffset.Now + TimeSpan.FromDays 2. ; Value2 = None ; Values = [|{ A = "foo" ; B = System.Reflection.BindingFlags.Instance }|] ; Map = Map.ofList [("A1",1)] ; Set = [set [1L];set [2L]] ; Bytes = [|1uy..10uy|]; String = ref "1a" ; Unions = [A 42; B("42",3)]} @@ -128,7 +128,7 @@ type EasyCounters private (table : TableContext) = // Create the table if necessary. Verifies schema is correct if it has already been created // NOTE the hard coded initial throughput provisioning - arguably this belongs outside of your application logic let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) - do! table.InitializeTableAsync(Provisioned throughput) + do! table.InitializeTableAsync(Throughput.Provisioned throughput) return EasyCounters(table) } @@ -143,7 +143,7 @@ type SimpleCounters private (table : TableContext) = // normally, RCU/WCU provisioning only happens first time the Table is created and is then considered an external concern // here we use `ProvisionTableAsync` instead of `InitializeAsync` to reset it each time we start the app let provisionedThroughput = ProvisionedThroughput(readCapacityUnits, writeCapacityUnits) - table.ProvisionTableAsync(Provisioned provisionedThroughput) + table.ProvisionTableAsync(Throughput.Provisioned provisionedThroughput) /// We only want to do the initialization bit once per instance of our application /// Similar to EasyCounters.Create in that it ensures the table is provisioned correctly diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 020a2b8..3abb820 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -20,17 +20,50 @@ type ResourceNotFoundException = Amazon.DynamoDBv2.Model.ResourceNotFoundExcepti type ProvisionedThroughput = Amazon.DynamoDBv2.Model.ProvisionedThroughput /// Represents the throughput configuration for a Table +[] type Throughput = | Provisioned of ProvisionedThroughput | OnDemand module internal Throughput = let applyToCreateRequest (req : CreateTableRequest) = function - | Provisioned t -> + | Throughput.Provisioned t -> req.ProvisionedThroughput <- t for gsi in req.GlobalSecondaryIndexes do gsi.ProvisionedThroughput <- t - | OnDemand -> + | Throughput.OnDemand -> req.BillingMode <- BillingMode.PAY_PER_REQUEST + let hasChanged (desc : TableDescription) = function + | Throughput.Provisioned t -> + let current = desc.ProvisionedThroughput + desc.BillingModeSummary.BillingMode <> BillingMode.PROVISIONED + || t.ReadCapacityUnits <> current.ReadCapacityUnits + || t.WriteCapacityUnits <> current.WriteCapacityUnits + | Throughput.OnDemand -> + desc.BillingModeSummary.BillingMode <> BillingMode.PAY_PER_REQUEST + let applyToUpdateRequest (req : UpdateTableRequest) = function + | Throughput.Provisioned t -> + req.BillingMode <- BillingMode.PROVISIONED + req.ProvisionedThroughput <- t + | Throughput.OnDemand -> + req.BillingMode <- BillingMode.PAY_PER_REQUEST + +/// Represents the streaming configuration for a Table +type Streaming = + | Enabled of StreamViewType + | Disabled +module internal Streaming = + let private (|Spec|) = function + | Streaming.Enabled svt -> StreamSpecification(StreamEnabled = true, StreamViewType = svt) + | Streaming.Disabled -> StreamSpecification(StreamEnabled = false) + let applyToCreateRequest (req : CreateTableRequest) (Spec spec) = + req.StreamSpecification <- spec + let hasChanged (desc : TableDescription) = function + | Streaming.Disabled -> desc.StreamSpecification.StreamEnabled + | Streaming.Enabled svt -> + not desc.StreamSpecification.StreamEnabled + || desc.StreamSpecification.StreamViewType <> svt + let applyToUpdateRequest (req : UpdateTableRequest) (Spec spec) = + req.StreamSpecification <- spec /// Represents the operation performed on the table, for metrics collection purposes type Operation = GetItem | PutItem | UpdateItem | DeleteItem | BatchGetItems | BatchWriteItems | Scan | Query @@ -871,16 +904,25 @@ type TableContext<'TRecord> internal /// - /// Asynchronously updates the underlying table with supplied provisioned throughput. + /// Asynchronously updates the underlying table with supplied configuration.
+ /// Will throw if customize does not apply any alterations. /// - /// Provisioned throughput to use on table. - member __.UpdateProvisionedThroughputAsync(provisionedThroughput : ProvisionedThroughput) : Async = async { - let request = UpdateTableRequest(tableName, provisionedThroughput) + /// Callback to apply any options desired. + member _.UpdateTableAsync(customize) : Async = async { + let request = UpdateTableRequest(TableName = tableName) + customize request let! ct = Async.CancellationToken - let! _response = client.UpdateTableAsync(request, ct) |> Async.AwaitTaskCorrect - return () + let! _response = client.UpdateTableAsync(request, ct) |> Async.AwaitTaskCorrect in () } + /// + /// Asynchronously updates the underlying table with supplied provisioned throughput. + /// + /// Provisioned throughput to use on table. + [] + member t.UpdateProvisionedThroughputAsync(provisionedThroughput : ProvisionedThroughput) : Async = + t.UpdateTableAsync(fun req -> Throughput.applyToUpdateRequest req (Throughput.Provisioned provisionedThroughput)) + member internal _.InternalDescribe() : Async = let rec wait () = async { let! ct = Async.CancellationToken @@ -936,13 +978,14 @@ type TableContext<'TRecord> internal checkOrCreate 9 // up to 9 retries, i.e. 10 attempts before we let exception propagate - member internal _.InternalCreateCreateTableRequest(throughput) = + member internal _.InternalCreateCreateTableRequest(?throughput, ?streaming) = let req = template.Info.Schemata.CreateCreateTableRequest(tableName) - Throughput.applyToCreateRequest req throughput + throughput |> Option.iter (Throughput.applyToCreateRequest req) + streaming |> Option.iter (Streaming.applyToCreateRequest req) req - member internal t.InternalCreateOrValidateTableAsync(throughput) = - t.InternalProvision(fun () -> t.InternalCreateCreateTableRequest throughput) + member internal t.InternalCreateOrValidateTableAsync(?throughput, ?streaming) = + t.InternalProvision(fun () -> t.InternalCreateCreateTableRequest(?throughput = throughput, ?streaming = streaming)) /// /// Asynchronously verify that the table exists and is compatible with record key schema, or throw.
@@ -957,23 +1000,26 @@ type TableContext<'TRecord> internal /// See also VerifyTableAsync, which only verifies the Table is present and correct. /// /// Throughput configuration to use for the table. - member t.InitializeTableAsync(throughput : Throughput) : Async = - t.InternalCreateOrValidateTableAsync(throughput) |> Async.Ignore + /// Optional Streaming configuration to use for the table. Default: Disabled. + member t.InitializeTableAsync(throughput : Throughput, ?streaming) : Async = + t.InternalCreateOrValidateTableAsync(throughput, defaultArg streaming Streaming.Disabled) |> Async.Ignore /// /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
- /// If the table is not present, it is provisioned, with the specified throughput.
- /// If it is present, and the throughput is not as specified, uses UpdateProvisionedThroughputAsync to update it.
+ /// If the table is not present, it is provisioned, with the specified throughput and optionally streaming.
+ /// If it is present, and the throughput or streaming are not as specified, uses UpdateTableAsync to adjust.
/// /// Throughput configuration to use for the table. - member t.ProvisionTableAsync(throughput : Throughput) : Async = async { - let! tableDescription = t.InternalCreateOrValidateTableAsync(throughput) - match throughput with - | Provisioned p -> - let current = tableDescription.ProvisionedThroughput - if p.ReadCapacityUnits <> current.ReadCapacityUnits || p.WriteCapacityUnits <> current.WriteCapacityUnits then - do! t.UpdateProvisionedThroughputAsync p - | OnDemand -> () } + /// Optional streaming configuration to apply for the table. Default (if creating): Disabled. Default: (if existing) do not change. + member t.ProvisionTableAsync(throughput : Throughput, ?streaming) : Async = async { + let! tableDescription = t.InternalCreateOrValidateTableAsync(throughput, defaultArg streaming Streaming.Disabled) + let tc = throughput |> Throughput.hasChanged tableDescription + let sc = streaming |> Option.exists (Streaming.hasChanged tableDescription) + if tc || sc then + let apply req = + if tc then Throughput.applyToUpdateRequest req throughput + if sc then streaming |> Option.iter (Streaming.applyToUpdateRequest req) + do! t.UpdateTableAsync(apply) } /// /// Asynchronously verify that the table exists and is compatible with record key schema. @@ -984,7 +1030,7 @@ type TableContext<'TRecord> internal member t.VerifyTableAsync(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) : Async = if createIfNotExists = Some true then let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) - t.InitializeTableAsync(Provisioned throughput) + t.InitializeTableAsync(Throughput.Provisioned throughput) else t.VerifyTableAsync() @@ -1456,5 +1502,6 @@ module Scripting = /// Updates the underlying table with supplied provisioned throughput. /// /// Provisioned throughput to use on table. - member __.UpdateProvisionedThroughput(provisionedThroughput : ProvisionedThroughput) = - __.UpdateProvisionedThroughputAsync(provisionedThroughput) |> Async.RunSynchronously + member t.UpdateProvisionedThroughput(provisionedThroughput : ProvisionedThroughput) = + let spec = Throughput.Provisioned provisionedThroughput + t.UpdateTableAsync(fun req -> Throughput.applyToUpdateRequest req spec) |> Async.RunSynchronously diff --git a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs index 6a5007e..1014c2a 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs @@ -46,7 +46,7 @@ module Utils = member _.CreateContextAndTableIfNotExists<'TRecord>() = let throughput = Model.ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) - Scripting.TableContext.Initialize<'TRecord>(client, tableName, Provisioned throughput) + Scripting.TableContext.Initialize<'TRecord>(client, tableName, Throughput.Provisioned throughput) interface IAsyncLifetime with member _.InitializeAsync() = From bf4852badbcfa8ed45979a40eb466432cce51bc8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 5 Apr 2022 10:11:34 +0100 Subject: [PATCH 10/22] MetricsCollector merge updates --- tests/FSharp.AWS.DynamoDB.Tests/Utils.fs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs index 1014c2a..bcf7400 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs @@ -44,6 +44,8 @@ module Utils = member _.Client = client member _.TableName = tableName + member _.TableName = tableName + member _.CreateContextAndTableIfNotExists<'TRecord>() = let throughput = Model.ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) Scripting.TableContext.Initialize<'TRecord>(client, tableName, Throughput.Provisioned throughput) From 6479f049288dd06203fbc1501c7a057a459538ba Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 5 Apr 2022 10:12:56 +0100 Subject: [PATCH 11/22] extract CreateTableRequest and UpdateTableRequest --- README.md | 3 +- src/FSharp.AWS.DynamoDB/TableContext.fs | 257 ++++++++++++++---------- 2 files changed, 151 insertions(+), 109 deletions(-) diff --git a/README.md b/README.md index 8a1ab93..3416945 100644 --- a/README.md +++ b/README.md @@ -37,8 +37,7 @@ open Amazon.DynamoDBv2 open FSharp.AWS.DynamoDB.Scripting // Expose non-Async methods, e.g. PutItem/GetItem let client : IAmazonDynamoDB = ``your DynamoDB client instance`` -let throughput = ProvisionedThroughput(readCapacityUnits = 1L, writeCapacityUnits = 10L) -let table = TableContext.Initialize(client, tableName = "workItems", Throughput.Provisioned throughput) +let table = TableContext.Initialize(client, tableName = "workItems", Throughput.OnDemand) let workItem = { ProcessId = 0L ; WorkItemId = 1L ; Name = "Test" ; UUID = guid() ; Dependencies = set ["mscorlib"] ; Started = None } diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 3abb820..565bcc0 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -32,7 +32,7 @@ module internal Throughput = gsi.ProvisionedThroughput <- t | Throughput.OnDemand -> req.BillingMode <- BillingMode.PAY_PER_REQUEST - let hasChanged (desc : TableDescription) = function + let requiresUpdate (desc : TableDescription) = function | Throughput.Provisioned t -> let current = desc.ProvisionedThroughput desc.BillingModeSummary.BillingMode <> BillingMode.PROVISIONED @@ -57,14 +57,120 @@ module internal Streaming = | Streaming.Disabled -> StreamSpecification(StreamEnabled = false) let applyToCreateRequest (req : CreateTableRequest) (Spec spec) = req.StreamSpecification <- spec - let hasChanged (desc : TableDescription) = function + let requiresUpdate (desc : TableDescription) = function | Streaming.Disabled -> desc.StreamSpecification.StreamEnabled - | Streaming.Enabled svt -> - not desc.StreamSpecification.StreamEnabled - || desc.StreamSpecification.StreamViewType <> svt + | Streaming.Enabled svt -> not desc.StreamSpecification.StreamEnabled || desc.StreamSpecification.StreamViewType <> svt let applyToUpdateRequest (req : UpdateTableRequest) (Spec spec) = req.StreamSpecification <- spec +module internal CreateTableRequest = + + let create (tableName, template : RecordTemplate<'TRecord>) throughput streaming customize = + let req = template.Info.Schemata.CreateCreateTableRequest(tableName) + throughput |> Option.iter (Throughput.applyToCreateRequest req) + streaming |> Option.iter (Streaming.applyToCreateRequest req) + customize |> Option.iter (fun c -> c req) + req + +module internal UpdateTableRequest = + + let create tableName = + UpdateTableRequest(TableName = tableName) + + let apply throughput streaming request = + throughput |> Option.iter (Throughput.applyToUpdateRequest request) + streaming |> Option.iter (Streaming.applyToUpdateRequest request) + + // Yields a request only if throughput, streaming or customize determine te update is warranted + let createIfRequired (tableName, template : RecordTemplate<'TRecord>) tableDescription throughput streaming customize : UpdateTableRequest option= + + let request = create tableName + let tc = throughput |> Option.filter (Throughput.requiresUpdate tableDescription) + let sc = streaming |> Option.filter (Streaming.requiresUpdate tableDescription) + match tc, sc, customize with + | None, None, None -> None + | Some _ as tc, sc, None + | tc, (Some _ as sc), None -> + apply tc sc request + Some request + | tc, sc, Some customize -> + apply tc sc request + customize request + + let execute (client : IAmazonDynamoDB) request = async { + let! ct = Async.CancellationToken + let! _response = client.UpdateTableAsync(request, ct) |> Async.AwaitTaskCorrect in () + } + +module Provisioning = + + let private describe (client : IAmazonDynamoDB, tableName : string) : Async = + let rec wait () = async { + let! ct = Async.CancellationToken + let! td = client.DescribeTableAsync(tableName, ct) |> Async.AwaitTaskCorrect + if td.Table.TableStatus = TableStatus.ACTIVE then + do! Async.Sleep 2000 + // wait indefinitely if table is in transition state + return! wait () + else + + return td.Table + } + wait () + + let (|Conflict|_|) (e : exn) = + match e with + | :? AmazonDynamoDBException as e when e.StatusCode = HttpStatusCode.Conflict -> Some() + | :? ResourceInUseException -> Some () + | _ -> None + + let private checkOrCreate (client, tableName) validateDescription maybeMakeCreateTableRequest : Async = + let rec aux retries = async { + match! describe (client, tableName) |> Async.Catch with + | Choice1Of2 desc -> + validateDescription desc + return desc + + | Choice2Of2 (:? ResourceNotFoundException) when Option.isSome maybeMakeCreateTableRequest -> + let! ct = Async.CancellationToken + let! response = + client.CreateTableAsync(maybeMakeCreateTableRequest.Value (), ct) + |> Async.AwaitTaskCorrect + |> Async.Catch + + match response with + | Choice1Of2 _ -> return! aux retries + | Choice2Of2 Conflict when retries > 0 -> + do! Async.Sleep 2000 + return! aux (retries - 1) + + | Choice2Of2 e -> return! Async.Raise e + + | Choice2Of2 Conflict when retries > 0 -> + do! Async.Sleep 2000 + return! aux (retries - 1) + + | Choice2Of2 e -> return! Async.Raise e + } + aux 9 // up to 9 retries, i.e. 10 attempts before we let exception propagate + + let private validateDescription (tableName, template : RecordTemplate<'TRecord>) desc = + let existingSchema = TableKeySchemata.OfTableDescription desc + if existingSchema <> template.Info.Schemata then + sprintf "table '%s' exists with key schema %A, which is incompatible with record '%O'." + tableName existingSchema typeof<'TRecord> + |> invalidOp + + let private run (client, tableName, template) maybeMakeCreateRequest : Async = + let validate = validateDescription (tableName, template) + checkOrCreate (client, tableName) validate maybeMakeCreateRequest + + let validateOnly (client, tableName, template) = run (client, tableName, template) None |> Async.Ignore + + let createOrValidate (client, tableName, template) throughput streaming customize : Async = + let generateCreateRequest () = CreateTableRequest.create (tableName, template) throughput streaming customize + run (client, tableName, template) (Some generateCreateRequest) + /// Represents the operation performed on the table, for metrics collection purposes type Operation = GetItem | PutItem | UpdateItem | DeleteItem | BatchGetItems | BatchWriteItems | Scan | Query @@ -903,127 +1009,64 @@ type TableContext<'TRecord> internal } - /// - /// Asynchronously updates the underlying table with supplied configuration.
- /// Will throw if customize does not apply any alterations. - /// - /// Callback to apply any options desired. - member _.UpdateTableAsync(customize) : Async = async { - let request = UpdateTableRequest(TableName = tableName) - customize request - let! ct = Async.CancellationToken - let! _response = client.UpdateTableAsync(request, ct) |> Async.AwaitTaskCorrect in () - } - - /// - /// Asynchronously updates the underlying table with supplied provisioned throughput. - /// - /// Provisioned throughput to use on table. - [] - member t.UpdateProvisionedThroughputAsync(provisionedThroughput : ProvisionedThroughput) : Async = - t.UpdateTableAsync(fun req -> Throughput.applyToUpdateRequest req (Throughput.Provisioned provisionedThroughput)) - - member internal _.InternalDescribe() : Async = - let rec wait () = async { - let! ct = Async.CancellationToken - let! td = client.DescribeTableAsync(tableName, ct) |> Async.AwaitTaskCorrect - if td.Table.TableStatus = TableStatus.ACTIVE then - do! Async.Sleep 2000 - // wait indefinitely if table is in transition state - return! wait () - else - - return td.Table - } - wait () - - member internal __.InternalProvision(?makeCreateTableRequest) : Async = - let (|Conflict|_|) (e : exn) = - match e with - | :? AmazonDynamoDBException as e when e.StatusCode = HttpStatusCode.Conflict -> Some() - | :? ResourceInUseException -> Some () - | _ -> None - - let rec checkOrCreate retries = async { - match! __.InternalDescribe() |> Async.Catch with - | Choice1Of2 desc -> - let existingSchema = TableKeySchemata.OfTableDescription desc - if existingSchema <> template.Info.Schemata then - sprintf "table '%s' exists with key schema %A, which is incompatible with record '%O'." - tableName existingSchema typeof<'TRecord> - |> invalidOp - return desc - - | Choice2Of2 (:? ResourceNotFoundException) when Option.isSome makeCreateTableRequest -> - let! ct = Async.CancellationToken - let! response = - client.CreateTableAsync(makeCreateTableRequest.Value (), ct) - |> Async.AwaitTaskCorrect - |> Async.Catch - - match response with - | Choice1Of2 _ -> return! checkOrCreate retries - | Choice2Of2 Conflict when retries > 0 -> - do! Async.Sleep 2000 - return! checkOrCreate (retries - 1) - - | Choice2Of2 e -> return! Async.Raise e - - | Choice2Of2 Conflict when retries > 0 -> - do! Async.Sleep 2000 - return! checkOrCreate (retries - 1) - - | Choice2Of2 e -> return! Async.Raise e - } - - checkOrCreate 9 // up to 9 retries, i.e. 10 attempts before we let exception propagate - - member internal _.InternalCreateCreateTableRequest(?throughput, ?streaming) = - let req = template.Info.Schemata.CreateCreateTableRequest(tableName) - throughput |> Option.iter (Throughput.applyToCreateRequest req) - streaming |> Option.iter (Streaming.applyToCreateRequest req) - req - - member internal t.InternalCreateOrValidateTableAsync(?throughput, ?streaming) = - t.InternalProvision(fun () -> t.InternalCreateCreateTableRequest(?throughput = throughput, ?streaming = streaming)) - /// /// Asynchronously verify that the table exists and is compatible with record key schema, or throw.
/// See also InitializeTableAsync, which performs the same check, but can create or re-provision the Table if required. /// - member __.VerifyTableAsync() : Async = - __.InternalProvision() |> Async.Ignore + member _.VerifyTableAsync() : Async = + Provisioning.validateOnly (client, tableName, template) + + member internal _.InternalCreateOrValidate(?throughput, ?streaming, ?customize) : Async = + Provisioning.createOrValidate (client, tableName, template) throughput streaming customize /// /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
/// If the table is not present, it is provisioned, with the specified throughput.
- /// See also VerifyTableAsync, which only verifies the Table is present and correct. + /// See also VerifyTableAsync, which only verifies the Table is present and correct.
+ /// See also ProvisionTableAsync, which will adjust throughput and streaming if they are not as specified. /// /// Throughput configuration to use for the table. - /// Optional Streaming configuration to use for the table. Default: Disabled. - member t.InitializeTableAsync(throughput : Throughput, ?streaming) : Async = - t.InternalCreateOrValidateTableAsync(throughput, defaultArg streaming Streaming.Disabled) |> Async.Ignore + /// Optional streaming configuration to apply for the table. Default: Disabled.. + /// Callback to post-process the CreateTableRequest. + member t.InitializeTableAsync(throughput : Throughput, ?streaming, ?customize) : Async = + t.InternalCreateOrValidate(throughput, ?streaming = streaming, ?customize = customize) |> Async.Ignore /// /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
/// If the table is not present, it is provisioned, with the specified throughput and optionally streaming.
/// If it is present, and the throughput or streaming are not as specified, uses UpdateTableAsync to adjust.
/// - /// Throughput configuration to use for the table. + /// Throughput configuration to use for the table. Always applied via either CreateTable or UpdateTable. /// Optional streaming configuration to apply for the table. Default (if creating): Disabled. Default: (if existing) do not change. - member t.ProvisionTableAsync(throughput : Throughput, ?streaming) : Async = async { - let! tableDescription = t.InternalCreateOrValidateTableAsync(throughput, defaultArg streaming Streaming.Disabled) - let tc = throughput |> Throughput.hasChanged tableDescription - let sc = streaming |> Option.exists (Streaming.hasChanged tableDescription) - if tc || sc then - let apply req = - if tc then Throughput.applyToUpdateRequest req throughput - if sc then streaming |> Option.iter (Streaming.applyToUpdateRequest req) - do! t.UpdateTableAsync(apply) } + /// Callback to post-process the CreateTableRequest if desired. + /// Callback to post-process the UpdateTableRequest if desired. When supplied, UpdateTable is inhibited if it returns None. + member t.ProvisionTableAsync(throughput : Throughput, ?streaming, ?customizeCreate, ?customizeUpdate) : Async = async { + let! tableDescription = t.InternalCreateOrValidate(throughput, ?streaming = streaming, ?customize = customizeCreate) + let maybeRequest = UpdateTableRequest.createIfRequired (tableName, template) tableDescription (Some throughput) streaming customizeUpdate + match maybeRequest with + | None -> () + | Some request -> do! UpdateTableRequest.execute client request } /// - /// Asynchronously verify that the table exists and is compatible with record key schema. + /// Asynchronously updates the underlying table with supplied configuration.
+ /// Underlying processing will throw if none of the options represent a change.
/// + /// Optional Throughput configuration to apply. + /// Optional Streaming configuration to apply. + /// Callback to apply any further options desired. + member _.UpdateTableAsync(?throughput, ?streaming, ?customize : UpdateTableRequest -> unit) : Async = + let request = UpdateTableRequest.create tableName + UpdateTableRequest.apply throughput streaming request + customize |> Option.iter (fun c -> c request) + UpdateTableRequest.execute client request + + /// Asynchronously updates the underlying table with supplied provisioned throughput. + /// Provisioned throughput to use on table. + [] + member t.UpdateProvisionedThroughputAsync(provisionedThroughput : ProvisionedThroughput) : Async = + t.UpdateTableAsync(Throughput.Provisioned provisionedThroughput) + + /// Asynchronously verify that the table exists and is compatible with record key schema. /// Create the table instance now instance if it does not exist. Defaults to false. /// Provisioned throughput for the table if newly created. Defaults to (10,10). [] @@ -1504,4 +1547,4 @@ module Scripting = /// Provisioned throughput to use on table. member t.UpdateProvisionedThroughput(provisionedThroughput : ProvisionedThroughput) = let spec = Throughput.Provisioned provisionedThroughput - t.UpdateTableAsync(fun req -> Throughput.applyToUpdateRequest req spec) |> Async.RunSynchronously + t.UpdateTableAsync(spec) |> Async.RunSynchronously From 8aed1892f5048a1c979fd8e382f960887d9b6bd3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 5 Apr 2022 10:40:11 +0100 Subject: [PATCH 12/22] Cleanup --- RELEASE_NOTES.md | 1 + src/FSharp.AWS.DynamoDB/RecordKeySchema.fs | 7 ++--- src/FSharp.AWS.DynamoDB/TableContext.fs | 32 +++++++++++----------- tests/FSharp.AWS.DynamoDB.Tests/Utils.fs | 14 +++++----- 4 files changed, 26 insertions(+), 28 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c3382ea..b787e5c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -14,6 +14,7 @@ * Added Support for `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST` rather than attempting to configure a `ProvisionedThroughput`) * Added ability to configure DynamoDB streaming (via `Streaming` DU) to `InitializeTableAsync` and `ProvisionTableAsync` * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `InitializeTableAsync`) + * Replaced `TableKeySchemata.CreateCreateTableRequest` with `ApplyToCreateTableRequest` ### 0.9.3-beta * Added `RequestMetrics` record type diff --git a/src/FSharp.AWS.DynamoDB/RecordKeySchema.fs b/src/FSharp.AWS.DynamoDB/RecordKeySchema.fs index 3780d86..81f089d 100644 --- a/src/FSharp.AWS.DynamoDB/RecordKeySchema.fs +++ b/src/FSharp.AWS.DynamoDB/RecordKeySchema.fs @@ -368,9 +368,8 @@ type TableKeySchemata with yield! td.GlobalSecondaryIndexes |> Seq.map mkGlobalSecondaryIndex yield! td.LocalSecondaryIndexes |> Seq.map mkLocalSecondaryIndex |]) - /// Create a CreateTableRequest using supplied key schema - member schema.CreateCreateTableRequest(tableName : string) = - let ctr = CreateTableRequest(TableName = tableName) + /// Applies the settings implied by the schema to the supplied CreateTableRequest + member schema.ApplyToCreateTableRequest(ctr : CreateTableRequest) = let inline mkKSE n t = KeySchemaElement(n, t) let keyAttrs = new Dictionary() @@ -402,5 +401,3 @@ type TableKeySchemata with for attr in keyAttrs.Values do let ad = AttributeDefinition(attr.AttributeName, attr.KeyType) ctr.AttributeDefinitions.Add ad - - ctr diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 565bcc0..0cb4f37 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -66,12 +66,18 @@ module internal Streaming = module internal CreateTableRequest = let create (tableName, template : RecordTemplate<'TRecord>) throughput streaming customize = - let req = template.Info.Schemata.CreateCreateTableRequest(tableName) + let req = CreateTableRequest(TableName = tableName) + template.Info.Schemata.ApplyToCreateTableRequest req throughput |> Option.iter (Throughput.applyToCreateRequest req) streaming |> Option.iter (Streaming.applyToCreateRequest req) customize |> Option.iter (fun c -> c req) req + let execute (client : IAmazonDynamoDB) request : Async = async { + let! ct = Async.CancellationToken + return! client.CreateTableAsync(request, ct) |> Async.AwaitTaskCorrect + } + module internal UpdateTableRequest = let create tableName = @@ -81,9 +87,8 @@ module internal UpdateTableRequest = throughput |> Option.iter (Throughput.applyToUpdateRequest request) streaming |> Option.iter (Streaming.applyToUpdateRequest request) - // Yields a request only if throughput, streaming or customize determine te update is warranted - let createIfRequired (tableName, template : RecordTemplate<'TRecord>) tableDescription throughput streaming customize : UpdateTableRequest option= - + // Yields a request only if throughput, streaming or customize determine the update is warranted + let createIfRequired tableName tableDescription throughput streaming customize : UpdateTableRequest option= let request = create tableName let tc = throughput |> Option.filter (Throughput.requiresUpdate tableDescription) let sc = streaming |> Option.filter (Streaming.requiresUpdate tableDescription) @@ -97,7 +102,7 @@ module internal UpdateTableRequest = apply tc sc request customize request - let execute (client : IAmazonDynamoDB) request = async { + let execute (client : IAmazonDynamoDB) request : Async = async { let! ct = Async.CancellationToken let! _response = client.UpdateTableAsync(request, ct) |> Async.AwaitTaskCorrect in () } @@ -132,13 +137,8 @@ module Provisioning = return desc | Choice2Of2 (:? ResourceNotFoundException) when Option.isSome maybeMakeCreateTableRequest -> - let! ct = Async.CancellationToken - let! response = - client.CreateTableAsync(maybeMakeCreateTableRequest.Value (), ct) - |> Async.AwaitTaskCorrect - |> Async.Catch - - match response with + let req = maybeMakeCreateTableRequest.Value () + match! CreateTableRequest.execute client req |> Async.Catch with | Choice1Of2 _ -> return! aux retries | Choice2Of2 Conflict when retries > 0 -> do! Async.Sleep 2000 @@ -165,12 +165,13 @@ module Provisioning = let validate = validateDescription (tableName, template) checkOrCreate (client, tableName) validate maybeMakeCreateRequest - let validateOnly (client, tableName, template) = run (client, tableName, template) None |> Async.Ignore - let createOrValidate (client, tableName, template) throughput streaming customize : Async = let generateCreateRequest () = CreateTableRequest.create (tableName, template) throughput streaming customize run (client, tableName, template) (Some generateCreateRequest) + let validateOnly (client, tableName, template) = + run (client, tableName, template) None |> Async.Ignore + /// Represents the operation performed on the table, for metrics collection purposes type Operation = GetItem | PutItem | UpdateItem | DeleteItem | BatchGetItems | BatchWriteItems | Scan | Query @@ -1042,8 +1043,7 @@ type TableContext<'TRecord> internal /// Callback to post-process the UpdateTableRequest if desired. When supplied, UpdateTable is inhibited if it returns None. member t.ProvisionTableAsync(throughput : Throughput, ?streaming, ?customizeCreate, ?customizeUpdate) : Async = async { let! tableDescription = t.InternalCreateOrValidate(throughput, ?streaming = streaming, ?customize = customizeCreate) - let maybeRequest = UpdateTableRequest.createIfRequired (tableName, template) tableDescription (Some throughput) streaming customizeUpdate - match maybeRequest with + match UpdateTableRequest.createIfRequired tableName tableDescription (Some throughput) streaming customizeUpdate with | None -> () | Some request -> do! UpdateTableRequest.execute client request } diff --git a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs index bcf7400..6e4e946 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs @@ -15,10 +15,10 @@ open Xunit [] module Utils = - let getRandomTableName() = - sprintf "fsdynamodb-%s" <| Guid.NewGuid().ToString("N") + let guid () = Guid.NewGuid().ToString("N") - let guid() = Guid.NewGuid().ToString("N") + let getRandomTableName() = + sprintf "fsdynamodb-%s" <| guid () let shouldFailwith<'T, 'Exn when 'Exn :> exn>(f : unit -> 'T) = <@ f () |> ignore @> @@ -26,8 +26,8 @@ module Utils = let getDynamoDBAccount () = let credentials = BasicAWSCredentials("Fake", "Fake") - let config = AmazonDynamoDBConfig() - config.ServiceURL <- "http://localhost:8000" + let config = AmazonDynamoDBConfig(ServiceURL = "http://localhost:8000") + new AmazonDynamoDBClient(credentials, config) :> IAmazonDynamoDB @@ -39,11 +39,11 @@ module Utils = type TableFixture() = + let client = getDynamoDBAccount() let tableName = getRandomTableName() - member _.Client = client - member _.TableName = tableName + member _.Client = client member _.TableName = tableName member _.CreateContextAndTableIfNotExists<'TRecord>() = From a959dda36ece1f75027b31e2ea4e84dd542a5f0c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 6 Apr 2022 10:27:07 +0100 Subject: [PATCH 13/22] Fixes from testing --- src/FSharp.AWS.DynamoDB/FSharp.AWS.DynamoDB.fsproj | 3 ++- src/FSharp.AWS.DynamoDB/Script.fsx | 11 +++++++++-- src/FSharp.AWS.DynamoDB/TableContext.fs | 13 ++++++++----- tests/FSharp.AWS.DynamoDB.Tests/Utils.fs | 4 ++-- tests/FSharp.AWS.DynamoDB.Tests/packages.config | 4 ---- 5 files changed, 21 insertions(+), 14 deletions(-) delete mode 100644 tests/FSharp.AWS.DynamoDB.Tests/packages.config diff --git a/src/FSharp.AWS.DynamoDB/FSharp.AWS.DynamoDB.fsproj b/src/FSharp.AWS.DynamoDB/FSharp.AWS.DynamoDB.fsproj index 0aa6e6c..f391456 100644 --- a/src/FSharp.AWS.DynamoDB/FSharp.AWS.DynamoDB.fsproj +++ b/src/FSharp.AWS.DynamoDB/FSharp.AWS.DynamoDB.fsproj @@ -9,6 +9,7 @@ Copyright 2016 + MIT https://github.com/fsprojects/FSharp.AWS.DynamoDB/blob/master/License.md https://github.com/fsprojects/FSharp.AWS.DynamoDB https://avatars0.githubusercontent.com/u/6001315 @@ -44,4 +45,4 @@ - \ No newline at end of file + diff --git a/src/FSharp.AWS.DynamoDB/Script.fsx b/src/FSharp.AWS.DynamoDB/Script.fsx index 5294302..1e99b14 100644 --- a/src/FSharp.AWS.DynamoDB/Script.fsx +++ b/src/FSharp.AWS.DynamoDB/Script.fsx @@ -1,7 +1,7 @@ #if USE_PUBLISHED_NUGET // If you don't want to do a local build first #r "nuget: FSharp.AWS.DynamoDB, *-*" // *-* to white-list the fact that all releases to date have been `-beta` sufficed #else -#I "../../bin/net6.0/" +#I "../../tests/FSharp.AWS.DynamoDB.Tests/bin/Debug/net6.0/" #r "AWSSDK.Core.dll" #r "AWSSDK.DynamoDBv2.dll" #r "FSharp.AWS.DynamoDB.dll" @@ -21,13 +21,13 @@ let account = AWSCredentialsProfile.LoadFrom("default").Credentials let ddb = new AmazonDynamoDBClient(account, RegionEndpoint.EUCentral1) :> IAmazonDynamoDB #else // Use Docker-hosted dynamodb-local instance // See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html#docker for details of how to deploy a simulator instance -let clientConfig = AmazonDynamoDBConfig(ServiceURL = "http://localhost:8000") #if USE_CREDS_FROM_ENV_VARS // 'AWS_ACCESS_KEY_ID' and 'AWS_SECRET_ACCESS_KEY' must be set for this to work let credentials = AWSCredentials.FromEnvironmentVariables() #else // Credentials are not validated if connecting to local instance so anything will do (this avoids it looking for profiles to be configured) let credentials = Amazon.Runtime.BasicAWSCredentials("A", "A") #endif +let clientConfig = AmazonDynamoDBConfig(ServiceURL = "http://localhost:8000") let ddb = new AmazonDynamoDBClient(credentials, clientConfig) :> IAmazonDynamoDB #endif @@ -145,6 +145,10 @@ type SimpleCounters private (table : TableContext) = let provisionedThroughput = ProvisionedThroughput(readCapacityUnits, writeCapacityUnits) table.ProvisionTableAsync(Throughput.Provisioned provisionedThroughput) + static member ProvisionOnDemand(client : IAmazonDynamoDB, tableName : string) : Async = + let table = TableContext(client, tableName) + table.ProvisionTableAsync(Throughput.OnDemand) + /// We only want to do the initialization bit once per instance of our application /// Similar to EasyCounters.Create in that it ensures the table is provisioned correctly /// However it will never actually create the table @@ -170,6 +174,9 @@ let e2 = e.StartCounter() |> Async.RunSynchronously e1.Incr() |> Async.RunSynchronously e2.Incr() |> Async.RunSynchronously +// First, we create it in On-Demand mode +SimpleCounters.ProvisionOnDemand(ddb, "testing-pre-provisioned") |> Async.RunSynchronously +// Then we flip it to Provisioned mode SimpleCounters.Provision(ddb, "testing-pre-provisioned", readCapacityUnits = 10L, writeCapacityUnits = 10L) |> Async.RunSynchronously // The consuming code can assume the provisioning has been carried out as part of the deploy // that allows the creation to be synchronous (and not impede application startup) diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 0cb4f37..003b3d6 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -35,11 +35,15 @@ module internal Throughput = let requiresUpdate (desc : TableDescription) = function | Throughput.Provisioned t -> let current = desc.ProvisionedThroughput - desc.BillingModeSummary.BillingMode <> BillingMode.PROVISIONED + match desc.BillingModeSummary with + | null -> false + | bms -> bms.BillingMode <> BillingMode.PROVISIONED || t.ReadCapacityUnits <> current.ReadCapacityUnits || t.WriteCapacityUnits <> current.WriteCapacityUnits | Throughput.OnDemand -> - desc.BillingModeSummary.BillingMode <> BillingMode.PAY_PER_REQUEST + match desc.BillingModeSummary with + | null -> false + | bms -> bms.BillingMode <> BillingMode.PAY_PER_REQUEST let applyToUpdateRequest (req : UpdateTableRequest) = function | Throughput.Provisioned t -> req.BillingMode <- BillingMode.PROVISIONED @@ -113,13 +117,12 @@ module Provisioning = let rec wait () = async { let! ct = Async.CancellationToken let! td = client.DescribeTableAsync(tableName, ct) |> Async.AwaitTaskCorrect - if td.Table.TableStatus = TableStatus.ACTIVE then + if td.Table.TableStatus <> TableStatus.ACTIVE then do! Async.Sleep 2000 // wait indefinitely if table is in transition state return! wait () else - - return td.Table + return td.Table } wait () diff --git a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs index 6e4e946..4d93c40 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs @@ -5,12 +5,12 @@ open System.IO open FsCheck open Swensen.Unquote +open Xunit open FSharp.AWS.DynamoDB open Amazon.DynamoDBv2 open Amazon.Runtime -open Xunit [] module Utils = @@ -47,7 +47,7 @@ module Utils = member _.TableName = tableName member _.CreateContextAndTableIfNotExists<'TRecord>() = - let throughput = Model.ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) + let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) Scripting.TableContext.Initialize<'TRecord>(client, tableName, Throughput.Provisioned throughput) interface IAsyncLifetime with diff --git a/tests/FSharp.AWS.DynamoDB.Tests/packages.config b/tests/FSharp.AWS.DynamoDB.Tests/packages.config deleted file mode 100644 index c899fcc..0000000 --- a/tests/FSharp.AWS.DynamoDB.Tests/packages.config +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file From 56f0be6bfd3df27fab38419c8b8bc4353f7997a1 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 6 Apr 2022 11:08:26 +0100 Subject: [PATCH 14/22] Add comments re BillingModeSummary --- src/FSharp.AWS.DynamoDB/TableContext.fs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 003b3d6..2837847 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -27,7 +27,7 @@ type Throughput = module internal Throughput = let applyToCreateRequest (req : CreateTableRequest) = function | Throughput.Provisioned t -> - req.ProvisionedThroughput <- t + req.ProvisionedThroughput <- t // TOCONSIDER also do req.BillingMode <- BillingMode.PROVISIONED or DescribeTableResponse does not include a BillingModeSummary on dynamodb-local for gsi in req.GlobalSecondaryIndexes do gsi.ProvisionedThroughput <- t | Throughput.OnDemand -> @@ -36,13 +36,13 @@ module internal Throughput = | Throughput.Provisioned t -> let current = desc.ProvisionedThroughput match desc.BillingModeSummary with - | null -> false + | null -> false // can happen if initial create did not explicitly specify a BillingMode when creating | bms -> bms.BillingMode <> BillingMode.PROVISIONED || t.ReadCapacityUnits <> current.ReadCapacityUnits || t.WriteCapacityUnits <> current.WriteCapacityUnits | Throughput.OnDemand -> match desc.BillingModeSummary with - | null -> false + | null -> false // // can happen if initial create did not explicitly specify a BillingMode when creating, i.e. Table was initially created in Provisioned mode | bms -> bms.BillingMode <> BillingMode.PAY_PER_REQUEST let applyToUpdateRequest (req : UpdateTableRequest) = function | Throughput.Provisioned t -> @@ -71,8 +71,8 @@ module internal CreateTableRequest = let create (tableName, template : RecordTemplate<'TRecord>) throughput streaming customize = let req = CreateTableRequest(TableName = tableName) - template.Info.Schemata.ApplyToCreateTableRequest req - throughput |> Option.iter (Throughput.applyToCreateRequest req) + template.Info.Schemata.ApplyToCreateTableRequest req // NOTE needs to precede the throughput application as that walks the GSIs list + throughput |> Option.iter (Throughput.applyToCreateRequest req) // NOTE needs to succeed Schemata.ApplyToCreateTableRequest streaming |> Option.iter (Streaming.applyToCreateRequest req) customize |> Option.iter (fun c -> c req) req From bbd92c4c0e6e06821515b720b5b64c270d399673 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 7 Apr 2022 10:28:55 +0100 Subject: [PATCH 15/22] Create / Update semantics tweaks --- src/FSharp.AWS.DynamoDB/Script.fsx | 2 +- src/FSharp.AWS.DynamoDB/TableContext.fs | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/FSharp.AWS.DynamoDB/Script.fsx b/src/FSharp.AWS.DynamoDB/Script.fsx index 1e99b14..7841757 100644 --- a/src/FSharp.AWS.DynamoDB/Script.fsx +++ b/src/FSharp.AWS.DynamoDB/Script.fsx @@ -141,7 +141,7 @@ type SimpleCounters private (table : TableContext) = static member Provision(client : IAmazonDynamoDB, tableName : string, readCapacityUnits, writeCapacityUnits) : Async = let table = TableContext(client, tableName) // normally, RCU/WCU provisioning only happens first time the Table is created and is then considered an external concern - // here we use `ProvisionTableAsync` instead of `InitializeAsync` to reset it each time we start the app + // here we use `ProvisionTableAsync` instead of `InitializeTableAsync` to reset it each time we deploy the app let provisionedThroughput = ProvisionedThroughput(readCapacityUnits, writeCapacityUnits) table.ProvisionTableAsync(Throughput.Provisioned provisionedThroughput) diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 2837847..ee75398 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -16,7 +16,7 @@ type ConditionalCheckFailedException = Amazon.DynamoDBv2.Model.ConditionalCheckF /// Exception raised by DynamoDB in case where resources are not found type ResourceNotFoundException = Amazon.DynamoDBv2.Model.ResourceNotFoundException -/// Represents the provisioned throughput for given table or index +/// Represents the provisioned throughput for a Table or Global Secondary Index type ProvisionedThroughput = Amazon.DynamoDBv2.Model.ProvisionedThroughput /// Represents the throughput configuration for a Table @@ -27,7 +27,8 @@ type Throughput = module internal Throughput = let applyToCreateRequest (req : CreateTableRequest) = function | Throughput.Provisioned t -> - req.ProvisionedThroughput <- t // TOCONSIDER also do req.BillingMode <- BillingMode.PROVISIONED or DescribeTableResponse does not include a BillingModeSummary on dynamodb-local + req.BillingMode <- BillingMode.PROVISIONED + req.ProvisionedThroughput <- t for gsi in req.GlobalSecondaryIndexes do gsi.ProvisionedThroughput <- t | Throughput.OnDemand -> @@ -42,7 +43,7 @@ module internal Throughput = || t.WriteCapacityUnits <> current.WriteCapacityUnits | Throughput.OnDemand -> match desc.BillingModeSummary with - | null -> false // // can happen if initial create did not explicitly specify a BillingMode when creating, i.e. Table was initially created in Provisioned mode + | null -> true // CreateTable without setting BillingMode is equivalent to it being BullingMode.PROVISIONED | bms -> bms.BillingMode <> BillingMode.PAY_PER_REQUEST let applyToUpdateRequest (req : UpdateTableRequest) = function | Throughput.Provisioned t -> @@ -52,6 +53,7 @@ module internal Throughput = req.BillingMode <- BillingMode.PAY_PER_REQUEST /// Represents the streaming configuration for a Table +[] type Streaming = | Enabled of StreamViewType | Disabled @@ -111,7 +113,7 @@ module internal UpdateTableRequest = let! _response = client.UpdateTableAsync(request, ct) |> Async.AwaitTaskCorrect in () } -module Provisioning = +module internal Provisioning = let private describe (client : IAmazonDynamoDB, tableName : string) : Async = let rec wait () = async { From 0858c5bace63b5311c1c97cdbe9dff880e2c29c6 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 7 Apr 2022 10:36:25 +0100 Subject: [PATCH 16/22] InitializeAsync -> CreateTableIfNotExistsAsync --- README.md | 2 +- RELEASE_NOTES.md | 10 +++++----- src/FSharp.AWS.DynamoDB/Script.fsx | 4 ++-- src/FSharp.AWS.DynamoDB/TableContext.fs | 20 ++++++++++---------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3416945..5552b0a 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ type Counter private (table : TableContext, key : TableKey) = static member Create(client : IAmazonDynamoDB, tableName : string) = async { let table = TableContext(client, tableName) let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) - do! table.InitializeTableAsync(Throughput.Provisioned throughput) + do! table.CreateTableIfNotExistsAsync(Throughput.Provisioned throughput) let initialEntry = { Id = Guid.NewGuid() ; Value = 0L } let! key = table.PutItemAsync(initialEntry) return Counter(table, key) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index b787e5c..9237554 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -5,15 +5,15 @@ * Added `TryGetItemAsync` (same as `GetItemAsync`, but returns `None`, instead of throwing, if an item is not present) * Switched test framework to Xunit, assertions to Unquote, runner to `dotnet test` * Clarified Creation/Verification APIs: - * Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize` and/or `TableContext.InitializeTableAsync`) + * Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize`, `TableContext.CreateTableIfNotExistsAsync`, `TableContext.VerifyTableAsync`) * Added `TableContext` constructor (replaces `TableContext.Create(verifyTable = false)`) * Added `TableContext.Scripting.Initialize` (replaces `TableContext.Create()`) * Added `TableContext.VerifyTableAsync` overload that only performs verification but never creates a Table - * Added `TableContext.InitializeTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) - * Added `TableContext.ProvisionTableAsync` (as per `InitializeTableAsync` but does an `UpdateTableAsync` if `throughput` or `streaming` has changed) + * Added `TableContext.CreateTableIfNotExistsAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) + * Added `TableContext.ProvisionTableAsync` (as per `CreateTableIfNotExistsAsync` but does an `UpdateTableAsync` if `throughput` or `streaming` has changed) * Added Support for `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST` rather than attempting to configure a `ProvisionedThroughput`) - * Added ability to configure DynamoDB streaming (via `Streaming` DU) to `InitializeTableAsync` and `ProvisionTableAsync` - * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `InitializeTableAsync`) + * Added ability to configure DynamoDB streaming (via `Streaming` DU) to `CreateTableIfNotExistsAsync` and `ProvisionTableAsync` + * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `CreateTableIfNotExistsAsync`) * Replaced `TableKeySchemata.CreateCreateTableRequest` with `ApplyToCreateTableRequest` ### 0.9.3-beta diff --git a/src/FSharp.AWS.DynamoDB/Script.fsx b/src/FSharp.AWS.DynamoDB/Script.fsx index 7841757..5ac9325 100644 --- a/src/FSharp.AWS.DynamoDB/Script.fsx +++ b/src/FSharp.AWS.DynamoDB/Script.fsx @@ -128,7 +128,7 @@ type EasyCounters private (table : TableContext) = // Create the table if necessary. Verifies schema is correct if it has already been created // NOTE the hard coded initial throughput provisioning - arguably this belongs outside of your application logic let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) - do! table.InitializeTableAsync(Throughput.Provisioned throughput) + do! table.CreateTableIfNotExistsAsync(Throughput.Provisioned throughput) return EasyCounters(table) } @@ -141,7 +141,7 @@ type SimpleCounters private (table : TableContext) = static member Provision(client : IAmazonDynamoDB, tableName : string, readCapacityUnits, writeCapacityUnits) : Async = let table = TableContext(client, tableName) // normally, RCU/WCU provisioning only happens first time the Table is created and is then considered an external concern - // here we use `ProvisionTableAsync` instead of `InitializeTableAsync` to reset it each time we deploy the app + // here we use `ProvisionTableAsync` instead of `CreateTableIfNotExistsAsync` to reset it each time we deploy the app let provisionedThroughput = ProvisionedThroughput(readCapacityUnits, writeCapacityUnits) table.ProvisionTableAsync(Throughput.Provisioned provisionedThroughput) diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index ee75398..1a03eff 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -420,7 +420,7 @@ type TableContext<'TRecord> internal /// /// Creates a DynamoDB client instance for given F# record and table name.
- /// For creating, provisioning or verification, see InitializeTableAsync and VerifyTableAsync. + /// For creating, provisioning or verification, see CreateTableIfNotExistsAsync and VerifyTableAsync. /// /// DynamoDB client instance. /// Table name to target. @@ -1017,7 +1017,7 @@ type TableContext<'TRecord> internal /// /// Asynchronously verify that the table exists and is compatible with record key schema, or throw.
- /// See also InitializeTableAsync, which performs the same check, but can create or re-provision the Table if required. + /// See also CreateTableIfNotExistsAsync, which performs the same check, but can create or re-provision the Table if required. /// member _.VerifyTableAsync() : Async = Provisioning.validateOnly (client, tableName, template) @@ -1027,14 +1027,14 @@ type TableContext<'TRecord> internal /// /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
- /// If the table is not present, it is provisioned, with the specified throughput.
+ /// If the table is not present, it is created, with the specified throughput.
/// See also VerifyTableAsync, which only verifies the Table is present and correct.
/// See also ProvisionTableAsync, which will adjust throughput and streaming if they are not as specified. /// /// Throughput configuration to use for the table. /// Optional streaming configuration to apply for the table. Default: Disabled.. /// Callback to post-process the CreateTableRequest. - member t.InitializeTableAsync(throughput : Throughput, ?streaming, ?customize) : Async = + member t.CreateTableIfNotExistsAsync(throughput : Throughput, ?streaming, ?customize) : Async = t.InternalCreateOrValidate(throughput, ?streaming = streaming, ?customize = customize) |> Async.Ignore /// @@ -1074,17 +1074,17 @@ type TableContext<'TRecord> internal /// Asynchronously verify that the table exists and is compatible with record key schema. /// Create the table instance now instance if it does not exist. Defaults to false. /// Provisioned throughput for the table if newly created. Defaults to (10,10). - [] + [] member t.VerifyTableAsync(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) : Async = if createIfNotExists = Some true then let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) - t.InitializeTableAsync(Throughput.Provisioned throughput) + t.CreateTableIfNotExistsAsync(Throughput.Provisioned throughput) else t.VerifyTableAsync() // Deprecated factory method, to be removed. Replaced with // 1. TableContext<'T> ctor (synchronous) -// 2. InitializeTableAsync OR VerifyTableAsync (explicitly async to signify that verification/creation is a costly and/or privileged operation) +// 2. CreateTableIfNotExistsAsync OR VerifyTableAsync (explicitly async to signify that verification/creation is a costly and/or privileged operation) type TableContext internal () = /// @@ -1097,7 +1097,7 @@ type TableContext internal () = /// Provisioned throughput for the table if newly created. Default: 10 RCU, 10 WCU /// Function to receive request metrics. [] static member Create<'TRecord> ( client : IAmazonDynamoDB, tableName : string, ?verifyTable : bool, @@ -1106,7 +1106,7 @@ type TableContext internal () = let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) if createIfNotExists = Some true then let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) - do! context.InitializeTableAsync(Throughput.Provisioned throughput) + do! context.CreateTableIfNotExistsAsync(Throughput.Provisioned throughput) elif verifyTable <> Some false then do! context.VerifyTableAsync() return context } @@ -1132,7 +1132,7 @@ module Scripting = let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) match throughput with | None -> context.VerifyTableAsync() |> Async.RunSynchronously - | Some t -> context.InitializeTableAsync(t) |> Async.RunSynchronously + | Some t -> context.CreateTableIfNotExistsAsync(t) |> Async.RunSynchronously context type TableContext<'TRecord> with From 4ca36df9cd5a3c9eb6c1939cdcc9949ef5a15537 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 7 Apr 2022 12:29:06 +0100 Subject: [PATCH 17/22] Provision -> UpdateTableIfRequired refactor --- README.md | 2 +- RELEASE_NOTES.md | 12 +-- src/FSharp.AWS.DynamoDB/Script.fsx | 19 +++-- src/FSharp.AWS.DynamoDB/TableContext.fs | 98 +++++++++++++------------ 4 files changed, 72 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 5552b0a..2b53d41 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ type Counter private (table : TableContext, key : TableKey) = static member Create(client : IAmazonDynamoDB, tableName : string) = async { let table = TableContext(client, tableName) let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) - do! table.CreateTableIfNotExistsAsync(Throughput.Provisioned throughput) + do! table.VerifyOrCreateTableAsync(Throughput.Provisioned throughput) let initialEntry = { Id = Guid.NewGuid() ; Value = 0L } let! key = table.PutItemAsync(initialEntry) return Counter(table, key) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 9237554..8f91bda 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -5,15 +5,15 @@ * Added `TryGetItemAsync` (same as `GetItemAsync`, but returns `None`, instead of throwing, if an item is not present) * Switched test framework to Xunit, assertions to Unquote, runner to `dotnet test` * Clarified Creation/Verification APIs: - * Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize`, `TableContext.CreateTableIfNotExistsAsync`, `TableContext.VerifyTableAsync`) + * Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize`, `TableContext.VerifyOrCreateTableAsync`, `TableContext.VerifyTableAsync`) * Added `TableContext` constructor (replaces `TableContext.Create(verifyTable = false)`) * Added `TableContext.Scripting.Initialize` (replaces `TableContext.Create()`) * Added `TableContext.VerifyTableAsync` overload that only performs verification but never creates a Table - * Added `TableContext.CreateTableIfNotExistsAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) - * Added `TableContext.ProvisionTableAsync` (as per `CreateTableIfNotExistsAsync` but does an `UpdateTableAsync` if `throughput` or `streaming` has changed) - * Added Support for `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST` rather than attempting to configure a `ProvisionedThroughput`) - * Added ability to configure DynamoDB streaming (via `Streaming` DU) to `CreateTableIfNotExistsAsync` and `ProvisionTableAsync` - * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `CreateTableIfNotExistsAsync`) + * Added `TableContext.VerifyOrCreateTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) + * Added `TableContext.UpdateTableIfRequiredAsync` (conditional `UpdateTableAsync` to establish specified `throughput` or `streaming` only if required) + * Added `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST` rather than attempting to configure a `ProvisionedThroughput`) + * Added ability to configure DynamoDB streaming (via `Streaming` DU) to `VerifyOrCreateTableAsync` and `UpdateTableIfRequiredAsync` + * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `VerifyOrCreateTableAsync`) * Replaced `TableKeySchemata.CreateCreateTableRequest` with `ApplyToCreateTableRequest` ### 0.9.3-beta diff --git a/src/FSharp.AWS.DynamoDB/Script.fsx b/src/FSharp.AWS.DynamoDB/Script.fsx index 5ac9325..b08e6a6 100644 --- a/src/FSharp.AWS.DynamoDB/Script.fsx +++ b/src/FSharp.AWS.DynamoDB/Script.fsx @@ -128,7 +128,7 @@ type EasyCounters private (table : TableContext) = // Create the table if necessary. Verifies schema is correct if it has already been created // NOTE the hard coded initial throughput provisioning - arguably this belongs outside of your application logic let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) - do! table.CreateTableIfNotExistsAsync(Throughput.Provisioned throughput) + do! table.VerifyOrCreateTableAsync(Throughput.Provisioned throughput) return EasyCounters(table) } @@ -138,16 +138,21 @@ type EasyCounters private (table : TableContext) = /// Variant of EasyCounters that splits the provisioning step from the (optional) validation that the table is present type SimpleCounters private (table : TableContext) = - static member Provision(client : IAmazonDynamoDB, tableName : string, readCapacityUnits, writeCapacityUnits) : Async = + static member Provision(client : IAmazonDynamoDB, tableName : string, readCapacityUnits, writeCapacityUnits) : Async = async { let table = TableContext(client, tableName) - // normally, RCU/WCU provisioning only happens first time the Table is created and is then considered an external concern - // here we use `ProvisionTableAsync` instead of `CreateTableIfNotExistsAsync` to reset it each time we deploy the app let provisionedThroughput = ProvisionedThroughput(readCapacityUnits, writeCapacityUnits) - table.ProvisionTableAsync(Throughput.Provisioned provisionedThroughput) + let throughput = Throughput.Provisioned provisionedThroughput + // normally, RCU/WCU provisioning only happens first time the Table is created and is then considered an external concern + // here we use `UpdateTableIfRequiredAsync` to reset it each time we deploy the app + do! table.VerifyOrCreateTableAsync(throughput) + do! table.UpdateTableIfRequiredAsync(throughput) } - static member ProvisionOnDemand(client : IAmazonDynamoDB, tableName : string) : Async = + static member ProvisionOnDemand(client : IAmazonDynamoDB, tableName : string) : Async = async { let table = TableContext(client, tableName) - table.ProvisionTableAsync(Throughput.OnDemand) + let throughput = Throughput.OnDemand + do! table.VerifyOrCreateTableAsync(throughput) + // as per the Provision, above, we reset to OnDemand, if it got reconfigured since it was originally created + do! table.UpdateTableIfRequiredAsync(throughput) } /// We only want to do the initialization bit once per instance of our application /// Similar to EasyCounters.Create in that it ensures the table is provisioned correctly diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 1a03eff..975604e 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -106,7 +106,8 @@ module internal UpdateTableRequest = Some request | tc, sc, Some customize -> apply tc sc request - customize request + if customize request then Some request + else None let execute (client : IAmazonDynamoDB) request : Async = async { let! ct = Async.CancellationToken @@ -115,16 +116,20 @@ module internal UpdateTableRequest = module internal Provisioning = - let private describe (client : IAmazonDynamoDB, tableName : string) : Async = + let tryDescribe (client : IAmazonDynamoDB, tableName : string) : Async = async { + let! ct = Async.CancellationToken + let! td = client.DescribeTableAsync(tableName, ct) |> Async.AwaitTaskCorrect + return match td.Table with t when t.TableStatus = TableStatus.ACTIVE -> Some t | _ -> None + } + + let private waitForActive (client : IAmazonDynamoDB, tableName : string) : Async = let rec wait () = async { - let! ct = Async.CancellationToken - let! td = client.DescribeTableAsync(tableName, ct) |> Async.AwaitTaskCorrect - if td.Table.TableStatus <> TableStatus.ACTIVE then - do! Async.Sleep 2000 + match! tryDescribe (client, tableName) with + | Some t -> return t + | None -> + do! Async.Sleep 1000 // wait indefinitely if table is in transition state return! wait () - else - return td.Table } wait () @@ -136,7 +141,7 @@ module internal Provisioning = let private checkOrCreate (client, tableName) validateDescription maybeMakeCreateTableRequest : Async = let rec aux retries = async { - match! describe (client, tableName) |> Async.Catch with + match! waitForActive (client, tableName) |> Async.Catch with | Choice1Of2 desc -> validateDescription desc return desc @@ -166,16 +171,16 @@ module internal Provisioning = tableName existingSchema typeof<'TRecord> |> invalidOp - let private run (client, tableName, template) maybeMakeCreateRequest : Async = + let private run (client, tableName, template) maybeMakeCreateRequest : Async = let validate = validateDescription (tableName, template) - checkOrCreate (client, tableName) validate maybeMakeCreateRequest + checkOrCreate (client, tableName) validate maybeMakeCreateRequest |> Async.Ignore - let createOrValidate (client, tableName, template) throughput streaming customize : Async = + let verifyOrCreate (client, tableName, template) throughput streaming customize : Async = let generateCreateRequest () = CreateTableRequest.create (tableName, template) throughput streaming customize run (client, tableName, template) (Some generateCreateRequest) - let validateOnly (client, tableName, template) = - run (client, tableName, template) None |> Async.Ignore + let validateOnly (client, tableName, template) : Async = + run (client, tableName, template) None /// Represents the operation performed on the table, for metrics collection purposes type Operation = GetItem | PutItem | UpdateItem | DeleteItem | BatchGetItems | BatchWriteItems | Scan | Query @@ -420,7 +425,7 @@ type TableContext<'TRecord> internal /// /// Creates a DynamoDB client instance for given F# record and table name.
- /// For creating, provisioning or verification, see CreateTableIfNotExistsAsync and VerifyTableAsync. + /// For creating, provisioning or verification, see VerifyOrCreateTableAsync and VerifyTableAsync. /// /// DynamoDB client instance. /// Table name to target. @@ -1015,46 +1020,49 @@ type TableContext<'TRecord> internal } - /// - /// Asynchronously verify that the table exists and is compatible with record key schema, or throw.
- /// See also CreateTableIfNotExistsAsync, which performs the same check, but can create or re-provision the Table if required. - /// - member _.VerifyTableAsync() : Async = - Provisioning.validateOnly (client, tableName, template) - - member internal _.InternalCreateOrValidate(?throughput, ?streaming, ?customize) : Async = - Provisioning.createOrValidate (client, tableName, template) throughput streaming customize - /// /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
- /// If the table is not present, it is created, with the specified throughput.
+ /// If the table is not present, it is created, with the specified throughput (and optionally streaming) configuration.
/// See also VerifyTableAsync, which only verifies the Table is present and correct.
- /// See also ProvisionTableAsync, which will adjust throughput and streaming if they are not as specified. + /// See also UpdateTableIfRequiredAsync, which will adjust throughput and streaming if they are not as specified. /// /// Throughput configuration to use for the table. /// Optional streaming configuration to apply for the table. Default: Disabled.. /// Callback to post-process the CreateTableRequest. - member t.CreateTableIfNotExistsAsync(throughput : Throughput, ?streaming, ?customize) : Async = - t.InternalCreateOrValidate(throughput, ?streaming = streaming, ?customize = customize) |> Async.Ignore + member t.VerifyOrCreateTableAsync(throughput : Throughput, ?streaming, ?customize) : Async = + Provisioning.verifyOrCreate (client, tableName, template) (Some throughput) streaming customize /// - /// Asynchronously verifies that the table exists and is compatible with record key schema, throwing if it is incompatible.
- /// If the table is not present, it is provisioned, with the specified throughput and optionally streaming.
- /// If it is present, and the throughput or streaming are not as specified, uses UpdateTableAsync to adjust.
+ /// Asynchronously verify that the table exists and is compatible with record key schema, or throw.
+ /// See also VerifyOrCreateTableAsync, which performs the same check, but can create or re-provision the Table if required. + /// + member _.VerifyTableAsync() : Async = + Provisioning.validateOnly (client, tableName, template) + + /// + /// Adjusts the Table's configuration via UpdateTable if the throughput or streaming are not as specified.
+ /// NOTE: The underlying API can throw if a change is currently in progress; see the DynamoDB UpdateTable API documentation.
+ /// NOTE: Throws InvalidOperationException if the table is not yet Active. It is recommended to ensure the Table is prepared via VerifyTableAsync or VerifyOrCreateTableAsync to guard against the potential for this state. /// /// Throughput configuration to use for the table. Always applied via either CreateTable or UpdateTable. /// Optional streaming configuration to apply for the table. Default (if creating): Disabled. Default: (if existing) do not change. - /// Callback to post-process the CreateTableRequest if desired. - /// Callback to post-process the UpdateTableRequest if desired. When supplied, UpdateTable is inhibited if it returns None. - member t.ProvisionTableAsync(throughput : Throughput, ?streaming, ?customizeCreate, ?customizeUpdate) : Async = async { - let! tableDescription = t.InternalCreateOrValidate(throughput, ?streaming = streaming, ?customize = customizeCreate) - match UpdateTableRequest.createIfRequired tableName tableDescription (Some throughput) streaming customizeUpdate with + /// Callback to post-process the UpdateTableRequest. UpdateTable is inhibited if it returns false and no other configuration requires a change. + /// Current table configuration, if known. Retrieved via DescribeTable if not supplied. + member t.UpdateTableIfRequiredAsync(?throughput : Throughput, ?streaming, ?custom, ?currentTableDescription : TableDescription) : Async = async { + let! tableDescription = async { + match currentTableDescription with + | Some d -> return d + | None -> + match! Provisioning.tryDescribe (client, tableName) with + | Some d -> return d + | None -> return invalidOp "Table is not currently Active. Please use VerifyTableAsync or VerifyOrCreateTableAsync to guard against this state." } + match UpdateTableRequest.createIfRequired tableName tableDescription throughput streaming custom with | None -> () | Some request -> do! UpdateTableRequest.execute client request } /// /// Asynchronously updates the underlying table with supplied configuration.
- /// Underlying processing will throw if none of the options represent a change.
+ /// NOTE: The underlying API can throw if none the options represent a change or a change is in currently progress; see the DynamoDB UpdateTable API documentation. /// /// Optional Throughput configuration to apply. /// Optional Streaming configuration to apply. @@ -1067,24 +1075,24 @@ type TableContext<'TRecord> internal /// Asynchronously updates the underlying table with supplied provisioned throughput. /// Provisioned throughput to use on table. - [] + [] member t.UpdateProvisionedThroughputAsync(provisionedThroughput : ProvisionedThroughput) : Async = t.UpdateTableAsync(Throughput.Provisioned provisionedThroughput) /// Asynchronously verify that the table exists and is compatible with record key schema. /// Create the table instance now instance if it does not exist. Defaults to false. /// Provisioned throughput for the table if newly created. Defaults to (10,10). - [] + [] member t.VerifyTableAsync(?createIfNotExists : bool, ?provisionedThroughput : ProvisionedThroughput) : Async = if createIfNotExists = Some true then let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) - t.CreateTableIfNotExistsAsync(Throughput.Provisioned throughput) + t.VerifyOrCreateTableAsync(Throughput.Provisioned throughput) else t.VerifyTableAsync() // Deprecated factory method, to be removed. Replaced with // 1. TableContext<'T> ctor (synchronous) -// 2. CreateTableIfNotExistsAsync OR VerifyTableAsync (explicitly async to signify that verification/creation is a costly and/or privileged operation) +// 2. VerifyOrCreateTableAsync OR VerifyTableAsync (explicitly async to signify that verification/creation is a costly and/or privileged operation) type TableContext internal () = /// @@ -1097,7 +1105,7 @@ type TableContext internal () = /// Provisioned throughput for the table if newly created. Default: 10 RCU, 10 WCU /// Function to receive request metrics. [] static member Create<'TRecord> ( client : IAmazonDynamoDB, tableName : string, ?verifyTable : bool, @@ -1106,7 +1114,7 @@ type TableContext internal () = let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) if createIfNotExists = Some true then let throughput = match provisionedThroughput with Some p -> p | None -> ProvisionedThroughput(10L, 10L) - do! context.CreateTableIfNotExistsAsync(Throughput.Provisioned throughput) + do! context.VerifyOrCreateTableAsync(Throughput.Provisioned throughput) elif verifyTable <> Some false then do! context.VerifyTableAsync() return context } @@ -1132,7 +1140,7 @@ module Scripting = let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) match throughput with | None -> context.VerifyTableAsync() |> Async.RunSynchronously - | Some t -> context.CreateTableIfNotExistsAsync(t) |> Async.RunSynchronously + | Some t -> context.VerifyOrCreateTableAsync(t) |> Async.RunSynchronously context type TableContext<'TRecord> with From d68a400a98897a3a17ac0b4c451e970df66f48ef Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 8 Apr 2022 08:16:15 +0100 Subject: [PATCH 18/22] Updated creds retrieval logic from @samritchie --- src/FSharp.AWS.DynamoDB/Script.fsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/FSharp.AWS.DynamoDB/Script.fsx b/src/FSharp.AWS.DynamoDB/Script.fsx index b08e6a6..16047b6 100644 --- a/src/FSharp.AWS.DynamoDB/Script.fsx +++ b/src/FSharp.AWS.DynamoDB/Script.fsx @@ -15,10 +15,9 @@ open FSharp.AWS.DynamoDB open FSharp.AWS.DynamoDB.Scripting // non-Async overloads #if USE_CLOUD -open Amazon -open Amazon.Util -let account = AWSCredentialsProfile.LoadFrom("default").Credentials -let ddb = new AmazonDynamoDBClient(account, RegionEndpoint.EUCentral1) :> IAmazonDynamoDB +open Amazon.DynamoDBv2 +let ok, creds = CredentialProfileStoreChain().TryGetAWSCredentials("default") +let ddb = if ok then new AmazonDynamoDBClient(creds) :> IAmazonDynamoDB else failwith "Unable to load default credentials" #else // Use Docker-hosted dynamodb-local instance // See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html#docker for details of how to deploy a simulator instance #if USE_CREDS_FROM_ENV_VARS // 'AWS_ACCESS_KEY_ID' and 'AWS_SECRET_ACCESS_KEY' must be set for this to work From e50078ae557ae30d688f4907ab34652a71057a59 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 8 Apr 2022 08:32:44 +0100 Subject: [PATCH 19/22] Flatten release notes --- RELEASE_NOTES.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 8f91bda..15572ea 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,20 +1,21 @@ +### 0.10.0-beta +* Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize`, `TableContext.VerifyOrCreateTableAsync`, `TableContext.VerifyTableAsync`) +* Added `TableContext` constructor (replaces `TableContext.Create(verifyTable = false)`) +* Added `TableContext.Scripting.Initialize` (replaces `TableContext.Create()`) +* Added `TableContext.VerifyTableAsync` overload that only performs verification but never creates a Table +* Added `TableContext.VerifyOrCreateTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) +* Added `TableContext.UpdateTableIfRequiredAsync` (conditional `UpdateTableAsync` to establish specified `throughput` or `streaming` only if required) +* Added `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST` rather than attempting to configure a `ProvisionedThroughput`) +* Added ability to configure DynamoDB streaming (via `Streaming` DU) to `VerifyOrCreateTableAsync` and `UpdateTableIfRequiredAsync` +* Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `VerifyOrCreateTableAsync`) +* Replaced `TableKeySchemata.CreateCreateTableRequest` with `ApplyToCreateTableRequest` + ### 0.9.4-beta * Moved Sync-over-Async versions of `TableContext` operations into `namespace FSharp.AWS.DynamoDB.Scripting` * Added `WithMetricsCollector()` copy method to allow separating metrics by context (eg by request) * Ensured metrics are reported even for failed requests * Added `TryGetItemAsync` (same as `GetItemAsync`, but returns `None`, instead of throwing, if an item is not present) * Switched test framework to Xunit, assertions to Unquote, runner to `dotnet test` -* Clarified Creation/Verification APIs: - * Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize`, `TableContext.VerifyOrCreateTableAsync`, `TableContext.VerifyTableAsync`) - * Added `TableContext` constructor (replaces `TableContext.Create(verifyTable = false)`) - * Added `TableContext.Scripting.Initialize` (replaces `TableContext.Create()`) - * Added `TableContext.VerifyTableAsync` overload that only performs verification but never creates a Table - * Added `TableContext.VerifyOrCreateTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) - * Added `TableContext.UpdateTableIfRequiredAsync` (conditional `UpdateTableAsync` to establish specified `throughput` or `streaming` only if required) - * Added `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST` rather than attempting to configure a `ProvisionedThroughput`) - * Added ability to configure DynamoDB streaming (via `Streaming` DU) to `VerifyOrCreateTableAsync` and `UpdateTableIfRequiredAsync` - * Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `VerifyOrCreateTableAsync`) - * Replaced `TableKeySchemata.CreateCreateTableRequest` with `ApplyToCreateTableRequest` ### 0.9.3-beta * Added `RequestMetrics` record type From f0feecf95594157fd7fd922bb32fc5a7e81ad470 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 8 Apr 2022 08:56:59 +0100 Subject: [PATCH 20/22] Review changes --- README.md | 29 +++++++++---------- src/FSharp.AWS.DynamoDB/TableContext.fs | 7 ++--- .../ConditionalExpressionTests.fs | 2 +- .../MetricsCollectorTests.fs | 8 ++--- .../PaginationTests.fs | 2 +- .../ProjectionExpressionTests.fs | 2 +- .../SimpleTableOperationTests.fs | 2 +- .../SparseGSITests.fs | 2 +- .../UpdateExpressionTests.fs | 2 +- tests/FSharp.AWS.DynamoDB.Tests/Utils.fs | 2 +- 10 files changed, 27 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 2b53d41..8cf5064 100644 --- a/README.md +++ b/README.md @@ -17,17 +17,17 @@ Table items can be represented using F# records: open FSharp.AWS.DynamoDB type WorkItemInfo = - { - [] - ProcessId : int64 - [] - WorkItemId : int64 - - Name : string - UUID : Guid - Dependencies : Set - Started : DateTimeOffset option - } + { + [] + ProcessId : int64 + [] + WorkItemId : int64 + + Name : string + UUID : Guid + Dependencies : Set + Started : DateTimeOffset option + } ``` We can now perform table operations on DynamoDB like so: @@ -49,7 +49,7 @@ Queries and scans can be performed using quoted predicates: ```fsharp let qResults = table.Query(keyCondition = <@ fun r -> r.ProcessId = 0 @>, - filterCondition = <@ fun r -> r.Name = "test" @>) + filterCondition = <@ fun r -> r.Name = "test" @>) let sResults = table.Scan <@ fun r -> r.Started.Value >= DateTimeOffset.Now - TimeSpan.FromMinutes 1. @> ``` @@ -58,7 +58,7 @@ Values can be updated using quoted update expressions: ```fsharp let updated = table.UpdateItem(<@ fun r -> { r with Started = Some DateTimeOffset.Now } @>, - preCondition = <@ fun r -> r.DateTimeOffset = None @>) + preCondition = <@ fun r -> r.DateTimeOffset = None @>) ``` Or they can be updated using [the `SET`, `ADD`, `REMOVE` and `DELETE` operations of the UpdateOp` DSL](./src/FSharp.AWS.DynamoDB/Types.fs#263), @@ -251,10 +251,9 @@ let processMetrics (m : RequestMetrics) = let table = TableContext(client, tableName = "workItems", metricsCollector = processMetrics) ``` -If `metricsCollector` is supplied, the requests will include `ReturnConsumedCapacity = ReturnConsumedCapacity.INDEX` +If `metricsCollector` is supplied, the requests will set `ReturnConsumedCapacity` to `ReturnConsumedCapacity.INDEX` and the `RequestMetrics` parameter will contain a list of `ConsumedCapacity` objects returned from the DynamoDB operations. - ### Building & Running Tests To build using the dotnet SDK: diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 975604e..1e1c265 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -1133,11 +1133,8 @@ module Scripting = /// DynamoDB client instance. /// Table name to target. /// Optional throughput to configure if the Table does not yet exist. - /// Function to receive request metrics. - static member Initialize<'TRecord> - ( client : IAmazonDynamoDB, tableName : string, ?throughput, - ?metricsCollector : RequestMetrics -> unit) : TableContext<'TRecord> = - let context = TableContext<'TRecord>(client, tableName, ?metricsCollector = metricsCollector) + static member Initialize<'TRecord>(client : IAmazonDynamoDB, tableName : string, ?throughput) : TableContext<'TRecord> = + let context = TableContext<'TRecord>(client, tableName) match throughput with | None -> context.VerifyTableAsync() |> Async.RunSynchronously | Some t -> context.VerifyOrCreateTableAsync(t) |> Async.RunSynchronously diff --git a/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs index 621a0f3..13ae79a 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/ConditionalExpressionTests.fs @@ -86,7 +86,7 @@ type ``Conditional Expression Tests`` (fixture : TableFixture) = Serialized = rand(), guid() } - let table = fixture.CreateContextAndTableIfNotExists() + let table = fixture.CreateEmpty() let [] ``Item exists precondition`` () = let item = mkItem() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs index 100aba7..5bf6ef7 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs @@ -52,7 +52,7 @@ let (|TotalCu|) : ConsumedCapacity list -> float = Seq.sumBy (fun c -> c.Capacit /// Tests without common setup type Tests(fixture : TableFixture) = - let rawTable = fixture.CreateContextAndTableIfNotExists() + let rawTable = fixture.CreateEmpty() let (|ExpectedTableName|_|) name = if name = fixture.TableName then Some () else None @@ -84,7 +84,7 @@ type Tests(fixture : TableFixture) = /// Tests that look up a specific item. Each test run gets a fresh individual item type ItemTests(fixture : TableFixture) = - let rawTable = fixture.CreateContextAndTableIfNotExists() + let rawTable = fixture.CreateEmpty() let (|ExpectedTableName|_|) name = if name = fixture.TableName then Some () else None let item = mkItem (guid()) (guid()) 0 @@ -131,7 +131,7 @@ type ItemTests(fixture : TableFixture) = /// Heavy tests reliant on establishing (and mutating) multiple items. Separate Test Class so Xunit will run them in parallel with others type BulkMutationTests(fixture : TableFixture) = - let rawTable = fixture.CreateContextAndTableIfNotExists() + let rawTable = fixture.CreateEmpty() let (|ExpectedTableName|_|) name = if name = fixture.TableName then Some () else None // NOTE we mutate the items so they need to be established each time @@ -169,7 +169,7 @@ type ManyReadOnlyItemsFixture() = inherit TableFixture() // TOCONSIDER shift this into IAsyncLifetime.InitializeAsync - let table = base.CreateContextAndTableIfNotExists() + let table = base.CreateEmpty() let hk = guid () do let gsk = guid () diff --git a/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs index e4cfa49..93d6833 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/PaginationTests.fs @@ -43,7 +43,7 @@ type ``Pagination Tests`` (fixture : TableFixture) = LocalAttribute = int (rand () % 2L) } - let table = fixture.CreateContextAndTableIfNotExists() + let table = fixture.CreateEmpty() let [] ``Paginated Query on Primary Key`` () = let hk = guid() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs index db5f1e5..5b039cd 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/ProjectionExpressionTests.fs @@ -97,7 +97,7 @@ type ``Projection Expression Tests`` (fixture : TableFixture) = Serialized = rand(), guid() ; Serialized2 = { NV = guid() ; NE = enum (int (rand()) % 3) } ; } - let table = fixture.CreateContextAndTableIfNotExists() + let table = fixture.CreateEmpty() let [] ``Should fail on invalid projections`` () = let testProj (p : Expr 'T>) = diff --git a/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs index 752d307..ec671a9 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs @@ -56,7 +56,7 @@ type ``Simple Table Operation Tests`` (fixture : TableFixture) = Unions = [Choice1Of3 (guid()) ; Choice2Of3(rand()) ; Choice3Of3(Guid.NewGuid().ToByteArray())] } - let table = fixture.CreateContextAndTableIfNotExists() + let table = fixture.CreateEmpty() let [] ``Convert to compatible table`` () = let table' = table.WithRecordType () diff --git a/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs b/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs index f553b30..70e9bda 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/SparseGSITests.fs @@ -31,7 +31,7 @@ type ``Sparse GSI Tests`` (fixture : TableFixture) = SecondaryHashKey = if rand() % 2L = 0L then Some (guid()) else None ; } - let table = fixture.CreateContextAndTableIfNotExists() + let table = fixture.CreateEmpty() let [] ``GSI Put Operation`` () = let value = mkItem() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs index ea95d2d..f9d1343 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/UpdateExpressionTests.fs @@ -94,7 +94,7 @@ type ``Update Expression Tests``(fixture : TableFixture) = Serialized = rand(), guid() ; Serialized2 = { NV = guid() ; NE = enum (int (rand()) % 3) } ; } - let table = fixture.CreateContextAndTableIfNotExists() + let table = fixture.CreateEmpty() let [] ``Attempt to update HashKey`` () = let item = mkItem() diff --git a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs index 4d93c40..606ed0c 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/Utils.fs @@ -46,7 +46,7 @@ module Utils = member _.Client = client member _.TableName = tableName - member _.CreateContextAndTableIfNotExists<'TRecord>() = + member _.CreateEmpty<'TRecord>() = let throughput = ProvisionedThroughput(readCapacityUnits = 10L, writeCapacityUnits = 10L) Scripting.TableContext.Initialize<'TRecord>(client, tableName, Throughput.Provisioned throughput) From 3bce1eff62ae6569f2e44f1a672848b2b99063a5 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 8 Apr 2022 13:47:41 +0100 Subject: [PATCH 21/22] Remove UpdateTableAsync --- RELEASE_NOTES.md | 11 ++++++----- src/FSharp.AWS.DynamoDB/TableContext.fs | 23 ++++------------------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 15572ea..5d0a91c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,14 +1,15 @@ ### 0.10.0-beta -* Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize`, `TableContext.VerifyOrCreateTableAsync`, `TableContext.VerifyTableAsync`) * Added `TableContext` constructor (replaces `TableContext.Create(verifyTable = false)`) * Added `TableContext.Scripting.Initialize` (replaces `TableContext.Create()`) -* Added `TableContext.VerifyTableAsync` overload that only performs verification but never creates a Table * Added `TableContext.VerifyOrCreateTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) -* Added `TableContext.UpdateTableIfRequiredAsync` (conditional `UpdateTableAsync` to establish specified `throughput` or `streaming` only if required) +* Added `TableContext.UpdateTableIfRequiredAsync` (conditional `UpdateTableAsync` to establish specified `throughput` or `streaming` only if required. Replaces `UpdateProvisionedThroughputAsync`) * Added `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST` rather than attempting to configure a `ProvisionedThroughput`) * Added ability to configure DynamoDB streaming (via `Streaming` DU) to `VerifyOrCreateTableAsync` and `UpdateTableIfRequiredAsync` -* Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `VerifyOrCreateTableAsync`) -* Replaced `TableKeySchemata.CreateCreateTableRequest` with `ApplyToCreateTableRequest` +* Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize`, `TableContext.VerifyOrCreateTableAsync`, `TableContext.VerifyTableAsync`) +* Obsoleted `TableContext.UpdateProvisionedThroughputAsync` (replace with `TableContext.UpdateTableIfRequiredAsync`) +* (breaking) Obsoleted `TableContext.VerifyTableAsync` optional argument to create a Table (replace with `VerifyOrCreateTableAsync`) +* (breaking) Changed `TableKeySchemata.CreateCreateTableRequest` to `ApplyToCreateTableRequest` (with minor signature change) +* (breaking) Removed `TableContext.CreateAsync` (replace with `TableContext.VerifyTableAsync` or `VerifyOrCreateTableAsync`) ### 0.9.4-beta * Moved Sync-over-Async versions of `TableContext` operations into `namespace FSharp.AWS.DynamoDB.Scripting` diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 1e1c265..0a807c4 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -1060,24 +1060,11 @@ type TableContext<'TRecord> internal | None -> () | Some request -> do! UpdateTableRequest.execute client request } - /// - /// Asynchronously updates the underlying table with supplied configuration.
- /// NOTE: The underlying API can throw if none the options represent a change or a change is in currently progress; see the DynamoDB UpdateTable API documentation. - /// - /// Optional Throughput configuration to apply. - /// Optional Streaming configuration to apply. - /// Callback to apply any further options desired. - member _.UpdateTableAsync(?throughput, ?streaming, ?customize : UpdateTableRequest -> unit) : Async = - let request = UpdateTableRequest.create tableName - UpdateTableRequest.apply throughput streaming request - customize |> Option.iter (fun c -> c request) - UpdateTableRequest.execute client request - /// Asynchronously updates the underlying table with supplied provisioned throughput. /// Provisioned throughput to use on table. - [] + [] member t.UpdateProvisionedThroughputAsync(provisionedThroughput : ProvisionedThroughput) : Async = - t.UpdateTableAsync(Throughput.Provisioned provisionedThroughput) + t.UpdateTableIfRequiredAsync(Throughput.Provisioned provisionedThroughput) /// Asynchronously verify that the table exists and is compatible with record key schema. /// Create the table instance now instance if it does not exist. Defaults to false. @@ -1551,10 +1538,8 @@ module Scripting = |> Async.RunSynchronously - /// - /// Updates the underlying table with supplied provisioned throughput. - /// + /// Updates the underlying table with supplied provisioned throughput. /// Provisioned throughput to use on table. member t.UpdateProvisionedThroughput(provisionedThroughput : ProvisionedThroughput) = let spec = Throughput.Provisioned provisionedThroughput - t.UpdateTableAsync(spec) |> Async.RunSynchronously + t.UpdateTableIfRequiredAsync(spec) |> Async.RunSynchronously From fe3570b4abff88d1f9dba25ca23016077a861e09 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 8 Apr 2022 14:05:38 +0100 Subject: [PATCH 22/22] Split Initialize into two overloads with separate xmldoc --- RELEASE_NOTES.md | 6 +++--- src/FSharp.AWS.DynamoDB/Script.fsx | 2 +- src/FSharp.AWS.DynamoDB/TableContext.fs | 26 ++++++++++++++++++------- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5d0a91c..53a0807 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,10 +1,10 @@ ### 0.10.0-beta * Added `TableContext` constructor (replaces `TableContext.Create(verifyTable = false)`) -* Added `TableContext.Scripting.Initialize` (replaces `TableContext.Create()`) * Added `TableContext.VerifyOrCreateTableAsync` (replaces `TableContext.VerifyTableAsync(createIfNotExists = true)`) * Added `TableContext.UpdateTableIfRequiredAsync` (conditional `UpdateTableAsync` to establish specified `throughput` or `streaming` only if required. Replaces `UpdateProvisionedThroughputAsync`) -* Added `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST` rather than attempting to configure a `ProvisionedThroughput`) -* Added ability to configure DynamoDB streaming (via `Streaming` DU) to `VerifyOrCreateTableAsync` and `UpdateTableIfRequiredAsync` +* Added `TableContext.Scripting.Initialize` (two overloads, replacing `TableContext.Create()` and `TableContext.Create(createIfNotExists = true)`) +* Added `Throughput.OnDemand` mode (sets `BillingMode` to `PAY_PER_REQUEST`, to go with the existing support for configuring `PROVISIONED` and a `ProvisionedThroughput`) +* Added ability to configure DynamoDB streaming (via a `Streaming` DU) to `VerifyOrCreateTableAsync` and `UpdateTableIfRequiredAsync` * Obsoleted `TableContext.Create` (replace with `TableContext.Scripting.Initialize`, `TableContext.VerifyOrCreateTableAsync`, `TableContext.VerifyTableAsync`) * Obsoleted `TableContext.UpdateProvisionedThroughputAsync` (replace with `TableContext.UpdateTableIfRequiredAsync`) * (breaking) Obsoleted `TableContext.VerifyTableAsync` optional argument to create a Table (replace with `VerifyOrCreateTableAsync`) diff --git a/src/FSharp.AWS.DynamoDB/Script.fsx b/src/FSharp.AWS.DynamoDB/Script.fsx index 16047b6..38b9417 100644 --- a/src/FSharp.AWS.DynamoDB/Script.fsx +++ b/src/FSharp.AWS.DynamoDB/Script.fsx @@ -16,7 +16,7 @@ open FSharp.AWS.DynamoDB.Scripting // non-Async overloads #if USE_CLOUD open Amazon.DynamoDBv2 -let ok, creds = CredentialProfileStoreChain().TryGetAWSCredentials("default") +let ok, creds = Amazon.Runtime.CredentialManagement.CredentialProfileStoreChain().TryGetAWSCredentials("default") let ddb = if ok then new AmazonDynamoDBClient(creds) :> IAmazonDynamoDB else failwith "Unable to load default credentials" #else // Use Docker-hosted dynamodb-local instance // See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html#docker for details of how to deploy a simulator instance diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 0a807c4..aa521f2 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -1113,18 +1113,30 @@ type TableContext internal () = /// module Scripting = - /// Factory method that allows one to include auto-initialization easily for scripting scenarios + /// Factory methods for scripting scenarios type TableContext internal () = - /// Creates a DynamoDB client instance for the specified F# record type, client and table name. + /// + /// Creates a DynamoDB client instance for the specified F# record type, client and table name.
+ /// Validates the table exists, and has the correct schema as per VerifyTableAsync.
+ /// See other overload for VerifyOrCreateTableAsync semantics. + /// + /// DynamoDB client instance. + /// Table name to target. + static member Initialize<'TRecord>(client : IAmazonDynamoDB, tableName : string) : TableContext<'TRecord> = + let context = TableContext<'TRecord>(client, tableName) + context.VerifyTableAsync() |> Async.RunSynchronously + context + + /// Creates a DynamoDB client instance for the specified F# record type, client and table name.
+ /// Either validates the table exists and has the correct schema, or creates a fresh one, as per VerifyOrCreateTableAsync.
+ /// See other overload for VerifyTableAsync semantics. /// DynamoDB client instance. /// Table name to target. - /// Optional throughput to configure if the Table does not yet exist. - static member Initialize<'TRecord>(client : IAmazonDynamoDB, tableName : string, ?throughput) : TableContext<'TRecord> = + /// Throughput to configure if the Table does not yet exist. + static member Initialize<'TRecord>(client : IAmazonDynamoDB, tableName : string, throughput) : TableContext<'TRecord> = let context = TableContext<'TRecord>(client, tableName) - match throughput with - | None -> context.VerifyTableAsync() |> Async.RunSynchronously - | Some t -> context.VerifyOrCreateTableAsync(t) |> Async.RunSynchronously + context.VerifyOrCreateTableAsync(throughput) |> Async.RunSynchronously context type TableContext<'TRecord> with