Merge pull request #1 from farbodahm/feature/init-project

Initialize Project

.gitignore

@@ -19,3 +19,7 @@
 
 # Go workspace file
 go.work
+
+# Terraform state files
+*.terraform*
+*.tfstate

README.md

@@ -1,2 +1,95 @@
-# dynamodb-optimistic-locking
-A simple project to implement optimistic locking (using versioning) on DynamoDB.
+# Dynamodb Optimistic Locking
+
+This is a simple project written in a weekend to 
+implement optimistic locking (versioning model) on DynamoDB.
+
+## Problem statement
+
+Assume we have a table called `products` on DynamoDB. One of the attributes is `quantity` showing the number of available products in the wharehouse.
+
+Now assume different users order the same product concurrently,
+resulting in race condition. How would you fix this problem?
+
+- Assume that we are not serializing the requests in one
+Goroutine (or in one thread, in generic);
+each request is handled by one Goroutine concurrently.
+- Special edge case of this problem, will be scenario
+where we have only 1 product available in the wharehouse;
+but 2 users order that exactly at the same time.
+
+![dynamodb-race-condition](./docs/update-no-condition.png)
+
+## Solution
+
+By default, DynamoDB is not ACID complient. But with a
+correct combination of [Transactional Write](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/transaction-apis.html)
+and
+[Conditional Write](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/WorkingWithItems.html#WorkingWithItems.ConditionalUpdate)
+we can achieve a similar behavour.
+
+### Architecture
+
+One of the easiest ways to solve this problem is
+to use [Optimistic Locking](https://en.wikipedia.org/wiki/Optimistic_concurrency_control)
+implemented with Conditional Write.
+
+1. To achieve this, we can add a `version` attribute
+(with a default value of 1) to the table.
+2. All of the requests, get the same item from DynamoDB.
+3. Next to the change each request want to do with the item,
+they should also bump the `version` by +1. Ex: if the previous
+version was 1, it will become 2 locally.
+4. All of the threads can now send the update request; with
+the following condition expression: `version = oldVersion`.
+    - Only request for one of the threads will be accepted
+and every other request will be rejected as the version is
+updated and is not equal to the old value.
+
+A picture is worth a thousand words:
+![dynamodb-optimistic-lock](./docs/update-with-condition.png)
+
+## How to run the project
+
+### Creating the table
+
+Make sure you have [Terraform](https://www.terraform.io/) installed.
+Then easily create the table using:
+
+```hcl
+cd infra/
+terraform init
+terraform apply
+```
+
+**Note:** Make sure your IAM role has the correct permissions
+to create DynamoDB table and has read/write access on that table.
+
+### Running the application
+
+First thing first, build the application:
+
+```go
+go mod tidy
+go build
+```
+
+Then to populate the table, you can easily provide the
+`--populate-table` so the script will populate the table
+with sample data for you:
+
+```sh
+./dynamodb-optimistic-locking --populate-table
+```
+
+**Note:** You can skip this part if you don't want to populate
+the table with sample data.
+
+and to run the application:
+
+```sh
+./dynamodb-optimistic-locking --number-of-requests 100
+```
+
+This will simulate submitting 100 orders simultaneously, which
+if everything goes well, 99 of them should fail:
+![result](./docs/result.png)

go.mod

@@ -0,0 +1,30 @@
+module github.com/farbodahm/dynamodb-optimistic-locking
+
+go 1.20
+
+require (
+	github.com/aws/aws-sdk-go-v2 v1.24.1
+	github.com/aws/aws-sdk-go-v2/config v1.26.5
+	github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.16
+	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.0
+)
+
+require (
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.16 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
+	github.com/aws/smithy-go v1.19.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+)

go.sum

@@ -0,0 +1,56 @@
+github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU=
+github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
+github.com/aws/aws-sdk-go-v2/config v1.26.5 h1:lodGSevz7d+kkFJodfauThRxK9mdJbyutUxGq1NNhvw=
+github.com/aws/aws-sdk-go-v2/config v1.26.5/go.mod h1:DxHrz6diQJOc9EwDslVRh84VjjrE17g+pVZXUeSxaDU=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0=
+github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.16 h1:KZvXflfyoL43jhDe2tDHPeK9C+edHJl2Rb07N7Dq3qY=
+github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.12.16/go.mod h1:SdkjT6MneWbTztIxA5cZ8QTvD4ASCeM7IhUkIIhvVa0=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.0 h1:e/HPLjLas04wKnmCUSSXD44cYdVjT/Dcd9CkmlYNyNU=
+github.com/aws/aws-sdk-go-v2/service/dynamodb v1.27.0/go.mod h1:N5tqZcYMM0N1PN7UQYJNWuGyO886OfnMhf/3MAbqMcI=
+github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.7 h1:srShyROqxzC7p18Ws8mqM2sqxJO/8L3Kpiqf+NboJLg=
+github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.18.7/go.mod h1:9efZgg4nJCGRp91MuHhkwd2kvyp7PWLRYYk5WjEQ5ts=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
+github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11 h1:e9AVb17H4x5FTE5KWIP5M1Du+9M86pS+Hw0lBUdN8EY=
+github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.8.11/go.mod h1:B90ZQJa36xo0ph9HsoteI1+r8owgQH/U1QNfqZQkj1Q=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U=
+github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
+github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

helpers.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"github.com/farbodahm/dynamodb-optimistic-locking/pkg/ddb"
+	"github.com/farbodahm/dynamodb-optimistic-locking/pkg/tables"
+)
+
+// populateProductsTable populates the Product table with sample data.
+func populateProductsTable(dynamo ddb.DynamoDB) error {
+
+	sampleProduct := []tables.Product{
+		{Id: "p#0", Name: "Product 0", Quantity: 10, Version: 1},
+		{Id: "p#1", Name: "Product 1", Quantity: 4, Version: 1},
+		{Id: "p#2", Name: "Product 2", Quantity: 2, Version: 1},
+		{Id: "p#3", Name: "Product 3", Quantity: 6, Version: 1},
+		{Id: "p#4", Name: "Product 4", Quantity: 1, Version: 1},
+		{Id: "p#5", Name: "Product 5", Quantity: 0, Version: 1},
+	}
+
+	Product := make([]ddb.DynamoDBWritable, len(sampleProduct))
+	for i, product := range sampleProduct {
+		Product[i] = product
+	}
+
+	_, err := dynamo.AddBatch("products", Product, 100)
+	return err
+}

infra/main.tf

@@ -0,0 +1,14 @@
+provider "aws" {
+  region = "eu-west-1"
+}
+
+resource "aws_dynamodb_table" "products" {
+  name         = "products"
+  hash_key     = "id"
+  billing_mode = "PAY_PER_REQUEST"
+
+  attribute {
+    name = "id"
+    type = "S"
+  }
+}

main.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"errors"
+	"log"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
+	"github.com/farbodahm/dynamodb-optimistic-locking/pkg/ddb"
+	"github.com/farbodahm/dynamodb-optimistic-locking/pkg/tables"
+	"github.com/spf13/cobra"
+)
+
+// Make sure all of the purchase simulations are finished
+var wg sync.WaitGroup
+
+// numberOfFailedRequests counts number of failed requests because of ConditionalCheckFailedException error
+var numberOfFailedRequests atomic.Int64
+
+// getCommandLineParser creates the command line parser using Cobra
+func getCommandLineParser() *cobra.Command {
+	return &cobra.Command{
+		Use:   "dynamodb-optimistic-locking",
+		Short: "Simple application to create a race condition on DynamoDB and solve it using optimistic locking (versioning)",
+	}
+}
+
+// simulateNewPurchase simulates a new purchase on DynamoDB using optimistic lock.
+// Intentionally it waits for a second to make sure all of the requests get the same
+// version from DDB to simulate a race condition on the table.
+func simulateNewPurchase(d ddb.DynamoDB, tableName string, productId string, requestId int) {
+	defer wg.Done()
+	product, err := tables.GetProduct(d, tableName, productId)
+	if err != nil {
+		log.Fatalf("Failed to get product %v error: %v\n", productId, err)
+	}
+
+	// this sleep is used to intentionally create a race condition between different
+	// goroutines trying to update the same product to test the optimistic lock mechanism.
+	time.Sleep(2 * time.Second)
+
+	product.Quantity -= 1
+	if err = tables.SafeUpdateProductQuantity(d, tableName, product); err != nil {
+		var e *types.ConditionalCheckFailedException
+		if x := errors.As(err, &e); x {
+			numberOfFailedRequests.Add(1)
+		} else {
+			log.Printf("WARN: Request Id %v failed with error %v", requestId, err)
+		}
+	}
+}
+
+func main() {
+	dynamo := ddb.NewDynamoDBClient()
+	cmd := getCommandLineParser()
+	tableName := "products"
+
+	var populateTable bool
+	var numberOfRequests int
+	cmd.Flags().BoolVar(&populateTable, "populate-table", false, "Populate the table with some sample data")
+	cmd.Flags().IntVar(&numberOfRequests, "number-of-requests", 5, "Number of requests to simulate a concurrent access on DynamoDB")
+
+	if err := cmd.Execute(); err != nil {
+		log.Fatalln("Failed to parse arguments:", err)
+	}
+
+	if populateTable {
+		log.Println("Populating `products` table with sample data...")
+		if err := populateProductsTable(*dynamo); err != nil {
+			log.Fatalln("Failed to populate products table:", err)
+		}
+	}
+
+	wg.Add(numberOfRequests)
+	for i := 0; i < numberOfRequests; i++ {
+		go simulateNewPurchase(*dynamo, tableName, "p#1", i)
+	}
+	wg.Wait()
+
+	log.Printf("Number of failed requests: %v\n", numberOfFailedRequests.Load())
+}

pkg/ddb/config.go

@@ -0,0 +1,23 @@
+package ddb
+
+import (
+	"context"
+
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
+)
+
+const Region = "eu-west-1"
+
+// GetDDBClient creates and returns DynamoDB client using default AWS config
+func CreateDDBClient() *dynamodb.Client {
+	cfg, err := config.LoadDefaultConfig(context.TODO(), func(opts *config.LoadOptions) error {
+		opts.Region = Region
+		return nil
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return dynamodb.NewFromConfig(cfg)
+}