diff --git a/pbm/storage/s3/retryer.go b/pbm/storage/s3/retryer.go new file mode 100644 index 000000000..034f6f1b6 --- /dev/null +++ b/pbm/storage/s3/retryer.go @@ -0,0 +1,58 @@ +package s3 + +import ( + "context" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/retry" +) + +type CustomRetryer struct { + // base is the AWS standard retryer. + base aws.Retryer + + // MinBackoff is the minimum delay that will be returned from RetryDelay. + MinBackoff time.Duration +} + +func (r CustomRetryer) IsErrorRetryable(err error) bool { + return r.base.IsErrorRetryable(err) +} + +func (r CustomRetryer) MaxAttempts() int { + return r.base.MaxAttempts() +} + +func (r CustomRetryer) RetryDelay(attempt int, opErr error) (time.Duration, error) { + delay, err := r.base.RetryDelay(attempt, opErr) + if err != nil { + return delay, err + } + + if delay < r.MinBackoff { + delay = r.MinBackoff + } + + return delay, nil +} + +func (r CustomRetryer) GetRetryToken(ctx context.Context, opErr error) (releaseToken func(error) error, err error) { + return r.base.GetRetryToken(ctx, opErr) +} + +func (r CustomRetryer) GetInitialToken() (releaseToken func(error) error) { + return r.base.GetInitialToken() +} + +func NewCustomRetryer(numMaxRetries int, minBackoff time.Duration, maxBackoff time.Duration) aws.Retryer { + baseRetryer := retry.NewStandard(func(o *retry.StandardOptions) { + o.MaxAttempts = numMaxRetries + o.MaxBackoff = maxBackoff + }) + + return &CustomRetryer{ + base: baseRetryer, + MinBackoff: minBackoff, + } +} diff --git a/pbm/storage/s3/s3.go b/pbm/storage/s3/s3.go index b28b2571b..3d73b8ea1 100644 --- a/pbm/storage/s3/s3.go +++ b/pbm/storage/s3/s3.go @@ -17,7 +17,6 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/credentials/stscreds" @@ -578,14 +577,13 @@ func (s *S3) buildLoadOptions() []func(*config.LoadOptions) error { } if s.opts.Retryer != nil { - customRetryer := func() aws.Retryer { - return retry.NewStandard(func(o *retry.StandardOptions) { - // v2 MaxAttempts includes the first try - o.MaxAttempts = s.opts.Retryer.NumMaxRetries + 1 - // TODO: determine Backoff based on MinRetryDelay and MaxRetryDelay - }) - } - cfgOpts = append(cfgOpts, config.WithRetryer(customRetryer)) + cfgOpts = append(cfgOpts, config.WithRetryer(func() aws.Retryer { + return NewCustomRetryer( + s.opts.Retryer.NumMaxRetries, + s.opts.Retryer.MinRetryDelay, + s.opts.Retryer.MaxRetryDelay, + ) + })) } if s.opts.Credentials.AccessKeyID != "" && s.opts.Credentials.SecretAccessKey != "" {