diff --git a/cmd/dotmesh-server/s3.go b/cmd/dotmesh-server/s3.go index f3924b996..4e0eafe35 100644 --- a/cmd/dotmesh-server/s3.go +++ b/cmd/dotmesh-server/s3.go @@ -115,17 +115,20 @@ func getS3Client(transferRequest S3TransferRequest) (*s3.S3, error) { } func downloadS3Bucket(svc *s3.S3, bucketName, destPath, transferRequestId string, prefixes []string, pollResult *TransferPollResult, currentKeyVersions map[string]string) (bool, map[string]string, error) { + log.Printf("Prefixes: %#v, len: %d", prefixes, len(prefixes)) if len(prefixes) == 0 { return downloadPartialS3Bucket(svc, bucketName, destPath, transferRequestId, "", pollResult, currentKeyVersions) } var changed bool var err error for _, prefix := range prefixes { + log.Printf("[downloadS3Bucket] Pulling down objects prefixed %s", prefix) changed, currentKeyVersions, err = downloadPartialS3Bucket(svc, bucketName, destPath, transferRequestId, prefix, pollResult, currentKeyVersions) if err != nil { return false, nil, err } } + fmt.Printf("[downloadS3Bucket] currentVersions: %#v", currentKeyVersions) return changed, currentKeyVersions, nil } @@ -135,14 +138,13 @@ func downloadPartialS3Bucket(svc *s3.S3, bucketName, destPath, transferRequestId // 2. Download new versions of things that have changed // 3. Return a map of object key -> s3 version id, plus an indicator of whether anything actually changed during this process so we know whether to make a snapshot. var bucketChanged bool - // TODO refactor this a lil so we can get the folder structure easily - fmt.Printf("[downloadS3Bucket] currentVersions: %#v", currentKeyVersions) params := &s3.ListObjectVersionsInput{ Bucket: aws.String(bucketName), } if prefix != "" { - params.Prefix = prefix + params.SetPrefix(prefix) } + log.Printf("Params: %#v", *params) downloader := s3manager.NewDownloaderWithClient(svc) var innerError error err := svc.ListObjectVersionsPages(params, @@ -203,7 +205,7 @@ func downloadPartialS3Bucket(svc *s3.S3, bucketName, destPath, transferRequestId } else if innerError != nil { return bucketChanged, nil, innerError } - fmt.Printf("New key versions: %#v", currentKeyVersions) + fmt.Printf("[downloadPartialS3Bucket] New key versions: %#v", currentKeyVersions) return bucketChanged, currentKeyVersions, nil } diff --git a/cmd/dotmesh-server/transfers.go b/cmd/dotmesh-server/transfers.go index f8aaaaa44..148454e7c 100644 --- a/cmd/dotmesh-server/transfers.go +++ b/cmd/dotmesh-server/transfers.go @@ -15,11 +15,16 @@ func s3TransferRequestify(in interface{}) (S3TransferRequest, error) { "Unable to cast %s to map[string]interface{}", in, ) } + prefixInter, ok := typed["Prefixes"].([]interface{}) + var prefixes []string + for _, pref := range prefixInter { + prefixes = append(prefixes, pref.(string)) + } return S3TransferRequest{ KeyID: typed["KeyID"].(string), SecretKey: typed["SecretKey"].(string), Endpoint: typed["Endpoint"].(string), - Prefixes: typed["Prefixes"].([]string), + Prefixes: prefixes, Direction: typed["Direction"].(string), LocalNamespace: typed["LocalNamespace"].(string), LocalName: typed["LocalName"].(string),