diff --git a/.changelog/bbab7da0e2504bebb9d999dacb2de133.json b/.changelog/bbab7da0e2504bebb9d999dacb2de133.json new file mode 100644 index 00000000000..a32b6ba38be --- /dev/null +++ b/.changelog/bbab7da0e2504bebb9d999dacb2de133.json @@ -0,0 +1,9 @@ +{ + "id": "bbab7da0-e250-4beb-b9d9-99dacb2de133", + "type": "feature", + "description": "port v1 sdk 100-continue http header customization for s3 PutObject/UploadPart request and enable user config", + "modules": [ + "service/internal/s3shared", + "service/s3" + ] +} \ No newline at end of file diff --git a/aws/signer/internal/v4/headers.go b/aws/signer/internal/v4/headers.go index 85a1d8f032f..64c4c4845ee 100644 --- a/aws/signer/internal/v4/headers.go +++ b/aws/signer/internal/v4/headers.go @@ -7,6 +7,7 @@ var IgnoredHeaders = Rules{ "Authorization": struct{}{}, "User-Agent": struct{}{}, "X-Amzn-Trace-Id": struct{}{}, + "Expect": struct{}{}, }, }, } diff --git a/aws/signer/internal/v4/headers_test.go b/aws/signer/internal/v4/headers_test.go index debf07788ae..6405ea97fc7 100644 --- a/aws/signer/internal/v4/headers_test.go +++ b/aws/signer/internal/v4/headers_test.go @@ -33,3 +33,31 @@ func TestAllowedQueryHoisting(t *testing.T) { }) } } + +func TestIgnoredHeaders(t *testing.T) { + cases := map[string]struct { + Header string + ExpectIgnored bool + }{ + "expect": { + Header: "Expect", + ExpectIgnored: true, + }, + "authorization": { + Header: "Authorization", + ExpectIgnored: true, + }, + "X-AMZ header": { + Header: "X-Amz-Content-Sha256", + ExpectIgnored: false, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + if e, a := c.ExpectIgnored, IgnoredHeaders.IsValid(c.Header); e == a { + t.Errorf("expect ignored %v, was %v", e, a) + } + }) + } +} diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3100Continue.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3100Continue.java new file mode 100644 index 00000000000..dffca079d60 --- /dev/null +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/S3100Continue.java @@ -0,0 +1,107 @@ +package software.amazon.smithy.aws.go.codegen.customization; + +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.go.codegen.GoDelegator; +import software.amazon.smithy.go.codegen.GoSettings; +import software.amazon.smithy.go.codegen.GoWriter; +import software.amazon.smithy.go.codegen.SymbolUtils; +import software.amazon.smithy.go.codegen.integration.ConfigField; +import software.amazon.smithy.go.codegen.integration.GoIntegration; +import software.amazon.smithy.go.codegen.integration.MiddlewareRegistrar; +import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.traits.HttpTrait; +import software.amazon.smithy.utils.ListUtils; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Add middleware, which adds {Expect: 100-continue} header for s3 client HTTP PUT request larger than 2MB + * or with unknown size streaming bodies, during operation builder step + */ +public class S3100Continue implements GoIntegration { + private static final String ADD_100Continue_Header = "add100Continue"; + private static final String ADD_100Continue_Header_INTERNAL = "Add100Continue"; + private static final String Continue_Client_Option = "ContinueHeaderThresholdBytes"; + private static final Set Put_Op_ShapeId_Set = new HashSet<>(Arrays.asList("com.amazonaws.s3#PutObject", "com.amazonaws.s3#UploadPart")); + + /** + * Return true if service is Amazon S3. + * + * @param model is the generation model. + * @param service is the service shape being audited. + */ + private static boolean isS3Service(Model model, ServiceShape service) { + return S3ModelUtils.isServiceS3(model, service); + } + + /** + * Gets the sort order of the customization from -128 to 127, with lowest + * executed first. + * + * @return Returns the sort order, defaults to -40. + */ + @Override + public byte getOrder() { + return 126; + } + + @Override + public void writeAdditionalFiles( + GoSettings settings, + Model model, + SymbolProvider symbolProvider, + GoDelegator goDelegator + ) { + ServiceShape service = settings.getService(model); + if (!isS3Service(model, service)) { + return; + } + + goDelegator.useShapeWriter(service, this::writeMiddlewareHelper); + } + + private void writeMiddlewareHelper(GoWriter writer) { + writer.openBlock("func $L(stack *middleware.Stack, options Options) error {", "}", ADD_100Continue_Header, () -> { + writer.write("return $T(stack, options.ContinueHeaderThresholdBytes)", + SymbolUtils.createValueSymbolBuilder(ADD_100Continue_Header_INTERNAL, + AwsCustomGoDependency.S3_SHARED_CUSTOMIZATION).build() + ); + }); + writer.insertTrailingNewline(); + } + + @Override + public List getClientPlugins() { + return ListUtils.of( + RuntimeClientPlugin.builder() + .operationPredicate((model, service, operation) -> + isS3Service(model, service) && Put_Op_ShapeId_Set.contains(operation.getId().toString()) + ) + .registerMiddleware(MiddlewareRegistrar.builder() + .resolvedFunction(SymbolUtils.createValueSymbolBuilder(ADD_100Continue_Header).build()) + .useClientOptions() + .build() + ) + .build(), + RuntimeClientPlugin.builder() + .servicePredicate(S3100Continue::isS3Service) + .configFields(ListUtils.of( + ConfigField.builder() + .name(Continue_Client_Option) + .type(SymbolUtils.createValueSymbolBuilder("int64") + .putProperty(SymbolUtils.GO_UNIVERSE_TYPE, true) + .build()) + .documentation("The threshold ContentLength in bytes for HTTP PUT request to receive {Expect: 100-continue} header. " + + "Setting to -1 will disable adding the Expect header to requests; setting to 0 will set the threshold " + + "to default 2MB") + .build() + )) + .build() + ); + } +} diff --git a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration index 1b0378be1ec..89074020cdc 100644 --- a/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration +++ b/codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration @@ -47,3 +47,4 @@ software.amazon.smithy.aws.go.codegen.customization.BackfillEc2UnboxedToBoxedSha software.amazon.smithy.aws.go.codegen.customization.AdjustAwsRestJsonContentType software.amazon.smithy.aws.go.codegen.customization.SQSValidateMessageChecksum software.amazon.smithy.aws.go.codegen.EndpointDiscoveryGenerator +software.amazon.smithy.aws.go.codegen.customization.S3100Continue \ No newline at end of file diff --git a/service/internal/s3shared/s3100continue.go b/service/internal/s3shared/s3100continue.go new file mode 100644 index 00000000000..0f43ec0d4fe --- /dev/null +++ b/service/internal/s3shared/s3100continue.go @@ -0,0 +1,54 @@ +package s3shared + +import ( + "context" + "fmt" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +const s3100ContinueID = "S3100Continue" +const default100ContinueThresholdBytes int64 = 1024 * 1024 * 2 + +// Add100Continue add middleware, which adds {Expect: 100-continue} header for s3 client HTTP PUT request larger than 2MB +// or with unknown size streaming bodies, during operation builder step +func Add100Continue(stack *middleware.Stack, continueHeaderThresholdBytes int64) error { + return stack.Build.Add(&s3100Continue{ + continueHeaderThresholdBytes: continueHeaderThresholdBytes, + }, middleware.After) +} + +type s3100Continue struct { + continueHeaderThresholdBytes int64 +} + +// ID returns the middleware identifier +func (m *s3100Continue) ID() string { + return s3100ContinueID +} + +func (m *s3100Continue) HandleBuild( + ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler, +) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error, +) { + sizeLimit := default100ContinueThresholdBytes + switch { + case m.continueHeaderThresholdBytes == -1: + return next.HandleBuild(ctx, in) + case m.continueHeaderThresholdBytes > 0: + sizeLimit = m.continueHeaderThresholdBytes + default: + } + + req, ok := in.Request.(*smithyhttp.Request) + if !ok { + return out, metadata, fmt.Errorf("unknown request type %T", req) + } + + if req.ContentLength == -1 || (req.ContentLength == 0 && req.Body != nil) || req.ContentLength >= sizeLimit { + req.Header.Set("Expect", "100-continue") + } + + return next.HandleBuild(ctx, in) +} diff --git a/service/internal/s3shared/s3100continue_test.go b/service/internal/s3shared/s3100continue_test.go new file mode 100644 index 00000000000..db815c30bdc --- /dev/null +++ b/service/internal/s3shared/s3100continue_test.go @@ -0,0 +1,96 @@ +package s3shared + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/internal/awstesting" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" + "testing" +) + +// unit test for service/internal/s3shared/s3100continue.go +func TestAdd100ContinueHttpHeader(t *testing.T) { + const HeaderKey = "Expect" + HeaderValue := "100-continue" + + cases := map[string]struct { + ContentLength int64 + Body *awstesting.ReadCloser + ExpectValueFound string + ContinueHeaderThresholdBytes int64 + }{ + "http request smaller than default 2MB": { + ContentLength: 1, + Body: &awstesting.ReadCloser{Size: 1}, + ExpectValueFound: "", + }, + "http request smaller than configured threshold": { + ContentLength: 1024 * 1024 * 2, + Body: &awstesting.ReadCloser{Size: 1024 * 1024 * 2}, + ExpectValueFound: "", + ContinueHeaderThresholdBytes: 1024 * 1024 * 3, + }, + "http request larger than default 2MB": { + ContentLength: 1024 * 1024 * 3, + Body: &awstesting.ReadCloser{Size: 1024 * 1024 * 3}, + ExpectValueFound: HeaderValue, + }, + "http request larger than configured threshold": { + ContentLength: 1024 * 1024 * 4, + Body: &awstesting.ReadCloser{Size: 1024 * 1024 * 4}, + ExpectValueFound: HeaderValue, + ContinueHeaderThresholdBytes: 1024 * 1024 * 3, + }, + "http put request with unknown -1 ContentLength": { + ContentLength: -1, + Body: &awstesting.ReadCloser{Size: 1024 * 1024 * 10}, + ExpectValueFound: HeaderValue, + }, + "http put request with 0 ContentLength but unknown non-nil body": { + ContentLength: 0, + Body: &awstesting.ReadCloser{Size: 1024 * 1024 * 3}, + ExpectValueFound: HeaderValue, + }, + "http put request with unknown -1 ContentLength and configured threshold": { + ContentLength: -1, + Body: &awstesting.ReadCloser{Size: 1024 * 1024 * 3}, + ExpectValueFound: HeaderValue, + ContinueHeaderThresholdBytes: 1024 * 1024 * 10, + }, + "http put request with continue header disabled": { + ContentLength: 1024 * 1024 * 3, + Body: &awstesting.ReadCloser{Size: 1024 * 1024 * 3}, + ExpectValueFound: "", + ContinueHeaderThresholdBytes: -1, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + var err error + req := smithyhttp.NewStackRequest().(*smithyhttp.Request) + + req.ContentLength = c.ContentLength + req.Body = c.Body + var updatedRequest *smithyhttp.Request + m := s3100Continue{ + continueHeaderThresholdBytes: c.ContinueHeaderThresholdBytes, + } + _, _, err = m.HandleBuild(context.Background(), + middleware.BuildInput{Request: req}, + middleware.BuildHandlerFunc(func(ctx context.Context, input middleware.BuildInput) ( + out middleware.BuildOutput, metadata middleware.Metadata, err error) { + updatedRequest = input.Request.(*smithyhttp.Request) + return out, metadata, nil + }), + ) + if err != nil { + t.Fatalf("expect no error, got %v", err) + } + + if e, a := c.ExpectValueFound, updatedRequest.Header.Get(HeaderKey); e != a { + t.Errorf("expect header value %v found, got %v", e, a) + } + }) + } +} diff --git a/service/s3/api_client.go b/service/s3/api_client.go index f228819fc28..e462917a707 100644 --- a/service/s3/api_client.go +++ b/service/s3/api_client.go @@ -80,6 +80,11 @@ type Options struct { // Configures the events that will be sent to the configured logger. ClientLogMode aws.ClientLogMode + // The threshold ContentLength in bytes for HTTP PUT request to receive {Expect: + // 100-continue} header. Setting to -1 will disable adding the Expect header to + // requests; setting to 0 will set the threshold to default 2MB + ContinueHeaderThresholdBytes int64 + // The credentials object to use when signing requests. Credentials aws.CredentialsProvider @@ -530,6 +535,10 @@ func addMetadataRetrieverMiddleware(stack *middleware.Stack) error { return s3shared.AddMetadataRetrieverMiddleware(stack) } +func add100Continue(stack *middleware.Stack, options Options) error { + return s3shared.Add100Continue(stack, options.ContinueHeaderThresholdBytes) +} + // ComputedInputChecksumsMetadata provides information about the algorithms used to // compute the checksum(s) of the input payload. type ComputedInputChecksumsMetadata struct { diff --git a/service/s3/api_op_PutObject.go b/service/s3/api_op_PutObject.go index 6433640616c..aa13f0e775d 100644 --- a/service/s3/api_op_PutObject.go +++ b/service/s3/api_op_PutObject.go @@ -500,6 +500,9 @@ func (c *Client) addOperationPutObjectMiddlewares(stack *middleware.Stack, optio if err = addMetadataRetrieverMiddleware(stack); err != nil { return err } + if err = add100Continue(stack, options); err != nil { + return err + } if err = addPutObjectInputChecksumMiddlewares(stack, options); err != nil { return err } diff --git a/service/s3/api_op_UploadPart.go b/service/s3/api_op_UploadPart.go index 2617bed6036..18d30a10f08 100644 --- a/service/s3/api_op_UploadPart.go +++ b/service/s3/api_op_UploadPart.go @@ -383,6 +383,9 @@ func (c *Client) addOperationUploadPartMiddlewares(stack *middleware.Stack, opti if err = addMetadataRetrieverMiddleware(stack); err != nil { return err } + if err = add100Continue(stack, options); err != nil { + return err + } if err = addUploadPartInputChecksumMiddlewares(stack, options); err != nil { return err }