From be71263c5a7f2ccda742b5277b5ca0d9bab01e1d Mon Sep 17 00:00:00 2001 From: Sam Ritchie Date: Fri, 8 Nov 2024 16:44:11 +0800 Subject: [PATCH 1/3] Implemented transaction API feedback --- README.md | 9 ++-- src/FSharp.AWS.DynamoDB/TableContext.fs | 50 +++++++---------- .../MetricsCollectorTests.fs | 15 +++--- .../SimpleTableOperationTests.fs | 53 +++++++++---------- 4 files changed, 54 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 71a8adc..9cdced3 100644 --- a/README.md +++ b/README.md @@ -269,25 +269,26 @@ let doesntExistCondition = compile <@ fun t -> NOT_EXISTS t.Value @> let existsCondition = compile <@ fun t -> EXISTS t.Value @> let key = TableKey.Combined(hashKey, rangeKey) -let transaction = Transaction() +let transaction = table.CreateTransaction() transaction.Check(table, key, doesntExistCondition) transaction.Put(table, item2, None) transaction.Put(table, item3, Some existsCondition) transaction.Delete (table ,table.Template.ExtractKey item5, None) + do! transaction.TransactWriteItems() ``` Failed preconditions (or `TransactWrite.Check`s) are signalled as per the underlying API: via a `TransactionCanceledException`. -Use `TransactWriteItemsRequest.TransactionCanceledConditionalCheckFailed` to trap such conditions: +Use `Transaction.TransactionCanceledConditionalCheckFailed` to trap such conditions: ```fsharp try do! transaction.TransactWriteItems() return Some result -with TransactWriteItemsRequest.TransactionCanceledConditionalCheckFailed -> return None +with Transaction.TransactionCanceledConditionalCheckFailed -> return None ``` -See [`TransactWriteItems tests`](./tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs#130) for more details and examples. +See [`TransactWriteItems tests`](./tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs#156) for more details and examples. It generally costs [double or more the Write Capacity Units charges compared to using precondition expressions](https://zaccharles.medium.com/calculating-a-dynamodb-items-size-and-consumed-capacity-d1728942eb7c) on individual operations. diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index c35a315..0e73450 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -1318,22 +1318,19 @@ type TableContext<'TRecord> else t.VerifyTableAsync() - member t.Transaction() = + /// Creates a new `Transaction`, using the DynamoDB client and metricsCollector configured for this `TableContext` + member _.CreateTransaction() = match metricsCollector with - | Some metricsCollector -> Transaction(metricsCollector = metricsCollector) - | None -> Transaction() + | Some metricsCollector -> Transaction(client, metricsCollector = metricsCollector) + | None -> Transaction(client) /// /// Represents a transactional set of operations to be applied atomically to a arbitrary number of DynamoDB tables. /// +/// DynamoDB client instance /// Function to receive request metrics. -and Transaction(?metricsCollector: (RequestMetrics -> unit)) = +and Transaction(client: IAmazonDynamoDB, ?metricsCollector: (RequestMetrics -> unit)) = let transactionItems = ResizeArray() - let mutable (dynamoDbClient: IAmazonDynamoDB) = null - - let setClient client = - if dynamoDbClient = null then - dynamoDbClient <- client let reportMetrics collector (tableName: string) (operation: Operation) (consumedCapacity: ConsumedCapacity list) (itemCount: int) = collector @@ -1353,20 +1350,18 @@ and Transaction(?metricsCollector: (RequestMetrics -> unit)) = /// Table context to operate on. /// Item to be put. /// Optional precondition expression. - member this.Put<'TRecord> + member _.Put<'TRecord> ( tableContext: TableContext<'TRecord>, item: 'TRecord, ?precondition: ConditionExpression<'TRecord> - ) : Transaction = - setClient tableContext.Client + ) = let req = Put(TableName = tableContext.TableName, Item = tableContext.Template.ToAttributeValues item) precondition |> Option.iter (fun cond -> let writer = AttributeWriter(req.ExpressionAttributeNames, req.ExpressionAttributeValues) req.ConditionExpression <- cond.Conditional.Write writer) transactionItems.Add(TransactWriteItem(Put = req)) - this /// /// Adds a ConditionCheck operation to the transaction. @@ -1374,14 +1369,11 @@ and Transaction(?metricsCollector: (RequestMetrics -> unit)) = /// Table context to operate on. /// Key of item to check. /// Condition to check. - member this.Check(tableContext: TableContext<'TRecord>, key: TableKey, condition: ConditionExpression<'TRecord>) : Transaction = - setClient tableContext.Client - + member _.Check(tableContext: TableContext<'TRecord>, key: TableKey, condition: ConditionExpression<'TRecord>) = let req = ConditionCheck(TableName = tableContext.TableName, Key = tableContext.Template.ToAttributeValues key) let writer = AttributeWriter(req.ExpressionAttributeNames, req.ExpressionAttributeValues) req.ConditionExpression <- condition.Conditional.Write writer transactionItems.Add(TransactWriteItem(ConditionCheck = req)) - this /// /// Adds an Update operation to the transaction. @@ -1390,22 +1382,19 @@ and Transaction(?metricsCollector: (RequestMetrics -> unit)) = /// Key of item to update. /// Update expression. /// Optional precondition expression. - member this.Update + member _.Update ( tableContext: TableContext<'TRecord>, key: TableKey, updater: UpdateExpression<'TRecord>, ?precondition: ConditionExpression<'TRecord> - ) : Transaction = - setClient tableContext.Client - + ) = let req = Update(TableName = tableContext.TableName, Key = tableContext.Template.ToAttributeValues key) let writer = AttributeWriter(req.ExpressionAttributeNames, req.ExpressionAttributeValues) req.UpdateExpression <- updater.UpdateOps.Write(writer) precondition |> Option.iter (fun cond -> req.ConditionExpression <- cond.Conditional.Write writer) transactionItems.Add(TransactWriteItem(Update = req)) - this /// /// Adds a Delete operation to the transaction. @@ -1413,21 +1402,18 @@ and Transaction(?metricsCollector: (RequestMetrics -> unit)) = /// Table context to operate on. /// Key of item to delete. /// Optional precondition expression. - member this.Delete + member _.Delete ( tableContext: TableContext<'TRecord>, key: TableKey, - precondition: option> - ) : Transaction = - setClient tableContext.Client - + ?precondition: ConditionExpression<'TRecord> + ) = let req = Delete(TableName = tableContext.TableName, Key = tableContext.Template.ToAttributeValues key) precondition |> Option.iter (fun cond -> let writer = AttributeWriter(req.ExpressionAttributeNames, req.ExpressionAttributeValues) req.ConditionExpression <- cond.Conditional.Write writer) transactionItems.Add(TransactWriteItem(Delete = req)) - this /// /// Atomically applies a set of 1-100 operations to the table.
@@ -1436,13 +1422,13 @@ and Transaction(?metricsCollector: (RequestMetrics -> unit)) = /// /// The ClientRequestToken to supply as an idempotency key (10 minute window). member _.TransactWriteItems(?clientRequestToken) : Async = async { - if (Seq.length transactionItems) = 0 || (Seq.length transactionItems) > 100 then + if transactionItems.Count = 0 || transactionItems.Count > 100 then raise <| System.ArgumentOutOfRangeException(nameof transactionItems, "must be between 1 and 100 items.") - let req = TransactWriteItemsRequest(ReturnConsumedCapacity = returnConsumedCapacity, TransactItems = (ResizeArray transactionItems)) + let req = TransactWriteItemsRequest(ReturnConsumedCapacity = returnConsumedCapacity, TransactItems = transactionItems) clientRequestToken |> Option.iter (fun x -> req.ClientRequestToken <- x) let! ct = Async.CancellationToken - let! response = dynamoDbClient.TransactWriteItemsAsync(req, ct) |> Async.AwaitTaskCorrect + let! response = client.TransactWriteItemsAsync(req, ct) |> Async.AwaitTaskCorrect maybeReport |> Option.iter (fun r -> response.ConsumedCapacity @@ -2151,7 +2137,7 @@ module Scripting = t.UpdateTableIfRequiredAsync(spec) |> Async.Ignore |> Async.RunSynchronously /// Helpers for working with TransactWriteItemsRequest -module TransactWriteItemsRequest = +module Transaction = /// Exception filter to identify whether a TransactWriteItems call has failed due to /// one or more of the supplied precondition checks failing. let (|TransactionCanceledConditionalCheckFailed|_|): exn -> unit option = diff --git a/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs index dddcb83..a204cac 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/MetricsCollectorTests.fs @@ -106,10 +106,9 @@ type Tests(fixture: TableFixture) = collector.Clear() let item = mkItem (guid ()) (guid ()) 0 - do! - Transaction(collector.Collect) - .Put(sut, item, compile <@ fun t -> NOT_EXISTS t.RangeKey @>) - .TransactWriteItems() + let transaction = Transaction(sut.Client, collector.Collect) + transaction.Put(sut, item, compile <@ fun t -> NOT_EXISTS t.RangeKey @>) + do! transaction.TransactWriteItems() test <@ @@ -131,14 +130,14 @@ type Tests(fixture: TableFixture) = let sut = rawTable.WithMetricsCollector(collector.Collect) let item = mkItem (guid ()) (guid ()) 0 + let transaction = rawTable.CreateTransaction() + transaction.Put(sut, item, compile <@ fun t -> EXISTS t.RangeKey @>) let mutable failed = false try do! // The check will fail, which triggers a throw from the underlying AWS SDK; there's no way to extract the consumption info in that case - Transaction() - .Put(sut, item, compile <@ fun t -> EXISTS t.RangeKey @>) - .TransactWriteItems() - with TransactWriteItemsRequest.TransactionCanceledConditionalCheckFailed -> + transaction.TransactWriteItems() + with Transaction.TransactionCanceledConditionalCheckFailed -> failed <- true true =! failed [] =! collector.Metrics diff --git a/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs b/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs index a1db252..561acb4 100644 --- a/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs +++ b/tests/FSharp.AWS.DynamoDB.Tests/SimpleTableOperationTests.fs @@ -182,10 +182,9 @@ type ``TransactWriteItems tests``(table1: TableFixture, table2: TableFixture) = [] let ``Minimal happy path`` () = async { let item = mkItem () - do! - Transaction() - .Put(table1, item, doesntExistConditionTable1) - .TransactWriteItems() + let transaction = table1.CreateTransaction() + transaction.Put(table1, item, doesntExistConditionTable1) + do! transaction.TransactWriteItems() let! itemFound = table1.ContainsKeyAsync(table1.Template.ExtractKey item) true =! itemFound @@ -196,11 +195,10 @@ type ``TransactWriteItems tests``(table1: TableFixture, table2: TableFixture) = let item = mkItem () let compatibleItem = mkCompatibleItem () - do! - Transaction() - .Put(table1, item, doesntExistConditionTable1) - .Put(table2, compatibleItem, doesntExistConditionTable2) - .TransactWriteItems() + let transaction = table1.CreateTransaction() + transaction.Put(table1, item, doesntExistConditionTable1) + transaction.Put(table2, compatibleItem, doesntExistConditionTable2) + do! transaction.TransactWriteItems() let! itemFound = table1.ContainsKeyAsync(table1.Template.ExtractKey item) true =! itemFound @@ -213,13 +211,12 @@ type ``TransactWriteItems tests``(table1: TableFixture, table2: TableFixture) = let ``Minimal Canceled path`` () = async { let item = mkItem () + let transaction = table1.CreateTransaction() + transaction.Put(table1, item, existsConditionTable1) let mutable failed = false try - do! - Transaction() - .Put(table1, item, existsConditionTable1) - .TransactWriteItems() - with TransactWriteItemsRequest.TransactionCanceledConditionalCheckFailed -> + do! transaction.TransactWriteItems() + with Transaction.TransactionCanceledConditionalCheckFailed -> failed <- true true =! failed @@ -233,18 +230,17 @@ type ``TransactWriteItems tests``(table1: TableFixture, table2: TableFixture) = let item, item2 = mkItem (), mkItem () let! key = table1.PutItemAsync item - let transaction = - if shouldFail then - Transaction().Check(table1, key, doesntExistConditionTable1) - else - Transaction() - .Check(table1, key, existsConditionTable1) - .Put(table1, item2) + let transaction = table1.CreateTransaction() + if shouldFail then + transaction.Check(table1, key, doesntExistConditionTable1) + else + transaction.Check(table1, key, existsConditionTable1) + transaction.Put(table1, item2) let mutable failed = false try do! transaction.TransactWriteItems() - with TransactWriteItemsRequest.TransactionCanceledConditionalCheckFailed -> + with Transaction.TransactionCanceledConditionalCheckFailed -> failed <- true failed =! shouldFail @@ -257,14 +253,14 @@ type ``TransactWriteItems tests``(table1: TableFixture, table2: TableFixture) = let ``All paths`` shouldFail = async { let item, item2, item3, item4, item5, item6, item7 = mkItem (), mkItem (), mkItem (), mkItem (), mkItem (), mkItem (), mkItem () let! key = table1.PutItemAsync item - let transaction = Transaction() + let transaction = table1.CreateTransaction() let requests = [ transaction.Update(table1, key, compileUpdateTable1 <@ fun t -> { t with Value = 42 } @>, existsConditionTable1) transaction.Put(table1, item2) transaction.Put(table1, item3, doesntExistConditionTable1) - transaction.Delete(table1, table1.Template.ExtractKey item4, Some doesntExistConditionTable1) - transaction.Delete(table1, table1.Template.ExtractKey item5, None) + transaction.Delete(table1, table1.Template.ExtractKey item4, doesntExistConditionTable1) + transaction.Delete(table1, table1.Template.ExtractKey item5) transaction.Check( table1, table1.Template.ExtractKey item6, @@ -281,7 +277,7 @@ type ``TransactWriteItems tests``(table1: TableFixture, table2: TableFixture) = let mutable failed = false try do! transaction.TransactWriteItems() - with TransactWriteItemsRequest.TransactionCanceledConditionalCheckFailed -> + with Transaction.TransactionCanceledConditionalCheckFailed -> failed <- true failed =! shouldFail @@ -310,13 +306,12 @@ type ``TransactWriteItems tests``(table1: TableFixture, table2: TableFixture) = [] let ``Empty request list is rejected with AORE`` () = - shouldBeRejectedWithArgumentOutOfRangeException (Transaction()) + shouldBeRejectedWithArgumentOutOfRangeException (Transaction(table1.Client)) |> Async.RunSynchronously - |> ignore [] let ``Over 100 writes are rejected with AORE`` () = - let Transaction = Transaction() + let Transaction = Transaction(table1.Client) for _x in 1..101 do Transaction.Put(table1, mkItem ()) |> ignore From 3804cd740dcbb7de0a52c84de7057af2f340d0fc Mon Sep 17 00:00:00 2001 From: Sam Ritchie Date: Thu, 14 Nov 2024 14:26:53 +0800 Subject: [PATCH 2/3] Incorporate feedback from @bartelink --- README.md | 4 ++-- src/FSharp.AWS.DynamoDB/TableContext.fs | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9cdced3..6a114e7 100644 --- a/README.md +++ b/README.md @@ -274,12 +274,12 @@ let transaction = table.CreateTransaction() transaction.Check(table, key, doesntExistCondition) transaction.Put(table, item2, None) transaction.Put(table, item3, Some existsCondition) -transaction.Delete (table ,table.Template.ExtractKey item5, None) +transaction.Delete(table, table.Template.ExtractKey item5, None) do! transaction.TransactWriteItems() ``` -Failed preconditions (or `TransactWrite.Check`s) are signalled as per the underlying API: via a `TransactionCanceledException`. +Failed preconditions (or `Check`s) are signalled as per the underlying API: via a `TransactionCanceledException`. Use `Transaction.TransactionCanceledConditionalCheckFailed` to trap such conditions: ```fsharp diff --git a/src/FSharp.AWS.DynamoDB/TableContext.fs b/src/FSharp.AWS.DynamoDB/TableContext.fs index 0e73450..c442607 100644 --- a/src/FSharp.AWS.DynamoDB/TableContext.fs +++ b/src/FSharp.AWS.DynamoDB/TableContext.fs @@ -1320,9 +1320,7 @@ type TableContext<'TRecord> /// Creates a new `Transaction`, using the DynamoDB client and metricsCollector configured for this `TableContext` member _.CreateTransaction() = - match metricsCollector with - | Some metricsCollector -> Transaction(client, metricsCollector = metricsCollector) - | None -> Transaction(client) + Transaction(client, ?metricsCollector = metricsCollector) /// /// Represents a transactional set of operations to be applied atomically to a arbitrary number of DynamoDB tables. @@ -2136,7 +2134,7 @@ module Scripting = let spec = Throughput.Provisioned provisionedThroughput t.UpdateTableIfRequiredAsync(spec) |> Async.Ignore |> Async.RunSynchronously -/// Helpers for working with TransactWriteItemsRequest +/// Helpers for working with Transaction module Transaction = /// Exception filter to identify whether a TransactWriteItems call has failed due to /// one or more of the supplied precondition checks failing. From 3b9a8830309d658b91dc93fe8ecea1002728f754 Mon Sep 17 00:00:00 2001 From: Sam Ritchie Date: Thu, 14 Nov 2024 14:39:14 +0800 Subject: [PATCH 3/3] Updated release notes --- RELEASE_NOTES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 829a09a..38d26c7 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,6 @@ +### 0.12.2-beta +* (breaking) Revised multi-table transaction API (thanks @bartelink) + ### 0.12.1-beta * Added support for `defaultArg` in update expressions on the same attribute, allowing SET if_not_exists semantics (eg { record with OptionalValue = Some (defaultArg record.OptionalValue "Default") }) * Allow empty strings in non-key attributes (thanks @purkhusid)