Skip to content

Commit

Permalink
Merge pull request #998 from bashofmann/add-aws-s3-v1-signer
Browse files Browse the repository at this point in the history
Allow to use AWS Signature v1 for creating signed AWS urls
  • Loading branch information
skriss committed Dec 4, 2018
2 parents 555f73c + e13806e commit 8955199
Show file tree
Hide file tree
Showing 6 changed files with 212 additions and 6 deletions.
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
* Initialize empty schedule metrics on server init (#1054, @cbeneke)
* Update CHANGELOGs (#1063, @wwitzel3)
* Remove default token from all service accounts (#1048, @ncdc)

* Allow to use AWS Signature v1 for creating signed AWS urls (#811, @bashofmann)

## Current release:
* [CHANGELOG-0.10.md][8]

Expand Down
1 change: 1 addition & 0 deletions docs/api-types/backupstoragelocation.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ The configurable parameters are as follows:
| `s3Url` | string | Required field for non-AWS-hosted storage| *Example*: http://minio:9000<br><br>You can specify the AWS S3 URL here for explicitness, but Ark can already generate it from `region`, and `bucket`. This field is primarily for local storage services like Minio.|
| `publicUrl` | string | Empty | *Example*: https://minio.mycluster.com<br><br>If specified, use this instead of `s3Url` when generating download URLs (e.g., for logs). This field is primarily for local storage services like Minio.|
| `kmsKeyId` | string | Empty | *Example*: "502b409c-4da1-419f-a16e-eif453b3i49f" or "alias/`<KMS-Key-Alias-Name>`"<br><br>Specify an [AWS KMS key][10] id or alias to enable encryption of the backups stored in S3. Only works with AWS S3 and may require explicitly granting key usage rights.|
| `signatureVersion` | string | `"4"` | Version of the signature algorithm used to create signed URLs that are used by ark cli to download backups or fetch logs. Possible versions are "1" and "4". Usually the default version 4 is correct, but some S3-compatible providers like Quobyte only support version 1.|

#### Azure

Expand Down
4 changes: 4 additions & 0 deletions docs/support-matrix.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ _Note that these providers are not regularly tested by the Ark team._
* [Minio][9]
* Ceph RADOS v12.2.7
* [DigitalOcean][7]
* Quobyte

_Some storage providers, like Quobyte, may need a different [signature algorithm version][15]._

## Volume Snapshot Providers

Expand Down Expand Up @@ -52,3 +55,4 @@ After you publish your plugin, open a PR that adds your plugin to the appropriat
[12]: https://github.com/aws/aws-sdk-go/aws
[13]: https://portworx.slack.com/messages/px-k8s
[14]: https://github.com/portworx/ark-plugin/issues
[15]: api-types/backupstoragelocation.md#aws
34 changes: 29 additions & 5 deletions pkg/cloudprovider/aws/object_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/pkg/errors"
Expand All @@ -38,27 +39,38 @@ const (
kmsKeyIDKey = "kmsKeyId"
s3ForcePathStyleKey = "s3ForcePathStyle"
bucketKey = "bucket"
signatureVersionKey = "signatureVersion"
)

type objectStore struct {
log logrus.FieldLogger
s3 *s3.S3
preSignS3 *s3.S3
s3Uploader *s3manager.Uploader
kmsKeyID string
log logrus.FieldLogger
s3 *s3.S3
preSignS3 *s3.S3
s3Uploader *s3manager.Uploader
kmsKeyID string
signatureVersion string
}

func NewObjectStore(logger logrus.FieldLogger) cloudprovider.ObjectStore {
return &objectStore{log: logger}
}

func isValidSignatureVersion(signatureVersion string) bool {
switch signatureVersion {
case "1", "4":
return true
}
return false
}

func (o *objectStore) Init(config map[string]string) error {
var (
region = config[regionKey]
s3URL = config[s3URLKey]
publicURL = config[publicURLKey]
kmsKeyID = config[kmsKeyIDKey]
s3ForcePathStyleVal = config[s3ForcePathStyleKey]
signatureVersion = config[signatureVersionKey]

// note that bucket is automatically added to the config map
// by the server from the ObjectStorageProviderConfig so
Expand Down Expand Up @@ -100,6 +112,13 @@ func (o *objectStore) Init(config map[string]string) error {
o.s3Uploader = s3manager.NewUploader(serverSession)
o.kmsKeyID = kmsKeyID

if signatureVersion != "" {
if !isValidSignatureVersion(signatureVersion) {
return errors.Errorf("invalid signature version: %s", signatureVersion)
}
o.signatureVersion = signatureVersion
}

if publicURL != "" {
publicConfig, err := newAWSConfig(publicURL, region, s3ForcePathStyle)
if err != nil {
Expand Down Expand Up @@ -239,5 +258,10 @@ func (o *objectStore) CreateSignedURL(bucket, key string, ttl time.Duration) (st
Key: aws.String(key),
})

if o.signatureVersion == "1" {
req.Handlers.Sign.Remove(v4.SignRequestHandler)
req.Handlers.Sign.PushBackNamed(v1SignRequestHandler)
}

return req.Presign(ttl)
}
29 changes: 29 additions & 0 deletions pkg/cloudprovider/aws/object_store_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
Copyright 2018 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestIsValidSignatureVersion(t *testing.T) {
assert.True(t, isValidSignatureVersion("1"))
assert.True(t, isValidSignatureVersion("4"))
assert.False(t, isValidSignatureVersion("3"))
}
147 changes: 147 additions & 0 deletions pkg/cloudprovider/aws/v1_sign_request_handler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
/*
Copyright 2018 the Heptio Ark contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"fmt"
"net/url"
"strconv"
"strings"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/pkg/errors"
)

var (
errInvalidMethod = errors.New("v1 signer only handles HTTP GET")
)

type signer struct {
// Values that must be populated from the request
request *request.Request
time time.Time
credentials *credentials.Credentials
debug aws.LogLevelType
logger aws.Logger

query url.Values
stringToSign string
signature string
}

// SignRequestHandler is a named request handler the SDK will use to sign
// service client request with using the V4 signature.
var v1SignRequestHandler = request.NamedHandler{
Name: "v1.SignRequestHandler", Fn: signSDKRequest,
}

func signSDKRequest(req *request.Request) {
// If the request does not need to be signed ignore the signing of the
// request if the AnonymousCredentials object is used.
if req.Config.Credentials == credentials.AnonymousCredentials {
return
}

if req.HTTPRequest.Method != "GET" {
// The V1 signer only supports GET
req.Error = errInvalidMethod
return
}

v1 := signer{
request: req,
time: req.Time,
credentials: req.Config.Credentials,
debug: req.Config.LogLevel.Value(),
logger: req.Config.Logger,
}

req.Error = v1.sign()

if req.Error != nil {
return
}

req.HTTPRequest.URL.RawQuery = v1.query.Encode()
}

func (v1 *signer) sign() error {
credentialsValue, err := v1.credentials.Get()
if err != nil {
return errors.Wrap(err, "error getting credentials")
}

httpRequest := v1.request.HTTPRequest

v1.query = httpRequest.URL.Query()

// Set new query parameters
v1.query.Set("AWSAccessKeyId", credentialsValue.AccessKeyID)
if credentialsValue.SessionToken != "" {
v1.query.Set("SecurityToken", credentialsValue.SessionToken)
}

// in case this is a retry, ensure no signature present
v1.query.Del("Signature")

method := httpRequest.Method
path := httpRequest.URL.Path
if path == "" {
path = "/"
}

duration := int64(v1.request.ExpireTime / time.Second)
expires := strconv.FormatInt(duration, 10)
// build the canonical string for the v1 signature
v1.stringToSign = strings.Join([]string{
method,
"",
"",
expires,
path,
}, "\n")

hash := hmac.New(sha1.New, []byte(credentialsValue.SecretAccessKey))
hash.Write([]byte(v1.stringToSign))
v1.signature = base64.StdEncoding.EncodeToString(hash.Sum(nil))
v1.query.Set("Signature", v1.signature)
v1.query.Set("Expires", expires)

if v1.debug.Matches(aws.LogDebugWithSigning) {
v1.logSigningInfo()
}

return nil
}

const logSignInfoMsg = `DEBUG: Request Signature:
---[ STRING TO SIGN ]--------------------------------
%s
---[ SIGNATURE ]-------------------------------------
%s
-----------------------------------------------------`

func (v1 *signer) logSigningInfo() {
msg := fmt.Sprintf(logSignInfoMsg, v1.stringToSign, v1.query.Get("Signature"))
v1.logger.Log(msg)
}

0 comments on commit 8955199

Please sign in to comment.