Merge pull request #206 from scality/feature/ZENKO-367/backbeat-end-t…

…o-end-tests-gcp

feature: ZENKO-367 CRR to GCP end-to-end tests

tests/.env

@@ -33,6 +33,7 @@ export GCP_CRR_MPU_BUCKET_NAME=ci-zenko-gcp-crr-mpu-bucket
 export GCP_CRR_SRC_BUCKET_NAME=ci-zenko-gcp-crr-src-bucket
 export GCP_MPU_BUCKET_NAME=ci-zenko-gcp-mpu-bucket
 export GCP_MPU_BUCKET_NAME_2=ci-zenko-gcp-mpu-bucket-2
+export GCP_BACKEND_DESTINATION_LOCATION=ci-zenko-gcp-destination-location
 export MULTI_CRR_SRC_BUCKET=ci-zenko-multi-crr-src-bucket
 export TRANSIENT_SRC_BUCKET=ci-transient-src-bucket
 

tests/.gitignore

@@ -2,7 +2,7 @@
 *~
 
 .env
-
+gcp_key.json
 .eggs/
 .pytest_cache/
 .tox/

tests/.secrets.env.example

@@ -18,3 +18,5 @@ export AZURE_BACKEND_ACCOUNT_NAME=
 export AZURE_BACKEND_ACCOUNT_NAME_2=
 export AZURE_BACKEND_ENDPOINT=
 export AZURE_BACKEND_ENDPOINT_2=
+export GCP_BACKEND_PRIVATE_KEY=
+export GCP_BACKEND_CLIENT_EMAIL=

tests/zenko_e2e/backbeat/ReplicationUtility.js

@@ -8,6 +8,7 @@ const { scalityS3Client, awsS3Client } = require('./s3SDK');
 const srcLocation = process.env.AWS_S3_BACKEND_SOURCE_LOCATION;
 const destAWSLocation = process.env.AWS_S3_BACKEND_DESTINATION_LOCATION;
 const destAzureLocation = process.env.AZURE_BACKEND_DESTINATION_LOCATION;
+const destGCPLocation = process.env.GCP_BACKEND_DESTINATION_LOCATION;
 const REPLICATION_TIMEOUT = 10000;
 
 class ReplicationUtility {
@@ -85,6 +86,11 @@ class ReplicationUtility {
             });
     }
 
+    deleteAllFiles(bucketName, filePrefix, cb) {
+        const bucket = this.gcpStorage.bucket(bucketName);
+        bucket.deleteFiles({ prefix: filePrefix }, cb);
+    }
+
     putObject(bucketName, objectName, content, cb) {
         this.s3.putObject({
             Bucket: bucketName,
@@ -264,6 +270,16 @@ class ReplicationUtility {
             true, false, cb);
     }
 
+    completeMPUGCP(bucketName, objectName, howManyParts, cb) {
+        this.genericCompleteMPU(bucketName, objectName, howManyParts, true,
+            false, false, cb);
+    }
+
+    completeMPUGCPWithProperties(bucketName, objectName, howManyParts, cb) {
+        this.genericCompleteMPU(bucketName, objectName, howManyParts, true,
+            true, false, cb);
+    }
+
     completeMPUWithPartCopy(bucketName, objectName, copySource, byteRange,
         howManyParts, cb) {
         let uploadId;
@@ -353,6 +369,18 @@ class ReplicationUtility {
         request.on('error', err => cb(err));
     }
 
+    getMetadata(bucketName, fileName, cb) {
+        const bucket = this.gcpStorage.bucket(bucketName);
+        const file = bucket.file(fileName);
+        file.getMetadata(cb);
+    }
+
+    download(bucketName, fileName, cb) {
+        const bucket = this.gcpStorage.bucket(bucketName);
+        const file = bucket.file(fileName);
+        file.download(cb);
+    }
+
     createBucket(bucketName, cb) {
         this.s3.createBucket({ Bucket: bucketName }, cb);
     }
@@ -596,6 +624,38 @@ class ReplicationUtility {
         });
     }
 
+    compareObjectsGCP(srcBucket, destBucket, key, cb) {
+        return async.series({
+            wait: next =>
+                this.waitUntilReplicated(srcBucket, key, undefined, next),
+            srcData: next => this.getObject(srcBucket, key, next),
+            destMetadata: next => this.getMetadata(destBucket,
+                `${srcBucket}/${key}`, next),
+            destData: next => this.download(destBucket,
+                `${srcBucket}/${key}`, next),
+        }, (err, data) => {
+            if (err) {
+                return cb(err);
+            }
+            const { srcData, destMetadata, destData } = data;
+            assert.strictEqual(srcData.ReplicationStatus, 'COMPLETED');
+            assert.strictEqual(srcData.ContentLength, destMetadata[0].size);
+            const srcUserMD = srcData.Metadata;
+            const destUserMD = destMetadata[0].metadata;
+            assert.strictEqual(
+                srcUserMD[`${destGCPLocation}-replication-status`],
+                'COMPLETED');
+            assert.strictEqual(srcUserMD[`${destGCPLocation}-version-id`],
+                destMetadata[0].generation);
+            assert.strictEqual(destUserMD['scal-replication-status'],
+                'REPLICA');
+            assert.strictEqual(destUserMD['scal-version-id'],
+                srcData.VersionId);
+            this._compareObjectBody(srcData.Body, destData);
+            return cb();
+        });
+    }
+
     compareAzureObjectProperties(srcBucket, containerName, key, cb) {
         return async.series([
             next => this.waitUntilReplicated(srcBucket, key, undefined, next),
@@ -633,6 +693,36 @@ class ReplicationUtility {
         });
     };
 
+    compareGCPObjectProperties(srcBucket, destBucket, file, cb) {
+        return async.series({
+            wait: next =>
+                this.waitUntilReplicated(srcBucket, file, undefined, next),
+            srcData: next => this.getHeadObject(srcBucket, file, next),
+            destData: next => this.getMetadata(destBucket,
+                `${srcBucket}/${file}`, next),
+        }, (err, data) => {
+            if (err) {
+                return cb(err);
+            }
+            const { srcData, destData } = data;
+            const destProperties = destData[0];
+            const destMetadata = destProperties.metadata;
+            let expectedVal = srcData.Metadata.customkey;
+            assert.strictEqual(expectedVal, destMetadata['customkey']);
+            expectedVal = srcData.ContentType;
+            assert.strictEqual(expectedVal, destProperties.contentType);
+            expectedVal = srcData.CacheControl;
+            assert.strictEqual(expectedVal, destProperties.cacheControl);
+            expectedVal = srcData.ContentEncoding;
+            assert.strictEqual(expectedVal, destProperties.contentEncoding);
+            expectedVal = srcData.ContentDisposition;
+            assert.strictEqual(expectedVal, destProperties.contentDisposition);
+            expectedVal = srcData.ContentLanguage;
+            assert.strictEqual(expectedVal, destProperties.contentLanguage);
+            return cb();
+        });
+    };
+
     compareACLsAWS(srcBucket, destBucket, key, cb) {
         return async.series([
             next => this.waitUntilReplicated(srcBucket, key, undefined, next),
@@ -701,6 +791,38 @@ class ReplicationUtility {
         });
     }
 
+    compareObjectTagsGCP(srcBucket, destContainer, file, scalityVersionId, cb) {
+        return async.series({
+            wait: next =>
+                this.waitUntilReplicated(srcBucket, file, scalityVersionId,
+                    next),
+            srcData: next => this.getObjectTagging(srcBucket, file,
+                scalityVersionId, next),
+            destData: next => this.getMetadata(destContainer,
+                `${srcBucket}/${file}`, next),
+        }, (err, data) => {
+            if (err) {
+                return cb(err);
+            }
+            const { srcData, destData } = data;
+            const destTags = destData[0].metadata;
+            const destTagSet = [];
+            if (destTags) {
+                Object.keys(destTags).forEach(key => {
+                    const tag = key.split('aws-tag-')[1];
+                    if (tag) {
+                        destTagSet.push({
+                            Key: tag,
+                            Value: destTags[key],
+                        })
+                    }
+                });
+            }
+            assert.deepStrictEqual(srcData.TagSet, destTagSet);
+            return cb();
+        });
+    }
+
     assertNoObject(bucketName, key, cb) {
         this.getObject(bucketName, key, err => {
             assert.strictEqual(err.code, 'NoSuchKey');

tests/zenko_e2e/backbeat/Using.md

@@ -31,8 +31,13 @@ export ZENKO_STORAGE_ACCOUNT_SECRET_KEY=<zenko-secret-key>
 3. In Orbit, create an AWS location `<source-aws-location-name>`.
 4. Create a container on Azure `<destination-azure-container-name>`.
 5. In Orbit, create an Azure storage location
-   `<destination-azure-location-name>` with an Azure container `<destination-azure-container-name>`.
-6. Export the keys, bucket name, container name, and storage location names
+   `<destination-azure-location-name>` with an Azure container
+   `<destination-azure-container-name>`.
+6. Create a bucket on GCP `<destination-gcp-bucket-name>`.
+7. In Orbit, create a GCP storage location
+  `<destination-gcp-location-name>` with an GCP bucket
+  `<destination-gcp-bucket-name>`.
+8. Export the keys, bucket name, container name, and storage location names
    (for example, in `.env` and `.secrets.env`):
 
 ```
@@ -46,15 +51,30 @@ export AZURE_BACKEND_ACCESS_KEY=<azure-access-key>
 export AZURE_BACKEND_ENDPOINT=<azure-endpoint>
 export AZURE_BACKBEAT_CONTAINER_NAME=<destination-azure-container-name>
 export AZURE_BACKEND_DESTINATION_LOCATION=<destination-azure-location-name>
+export GCP_CRR_BUCKET_NAME=<destination-gcp-bucket-name>
+export GCP_BACKEND_DESTINATION_LOCATION=<destination-gcp-location-name>
+export GCP_BACKEND_PRIVATE_KEY=<gcp-private-key>
+export GCP_BACKEND_CLIENT_EMAIL=<gcp-client-email>
 ```
 
-7. If using `*.env` files, source the files:
+9. If using `*.env` files, source the files:
 
 ```
 source .env && source .secrets.env
 ```
 
-8. Run the test suite: `npm run test_crr`.
+10. Create the GCP credential file in `Zenko/tests/zenko_e2e/backbeat`:
+
+```
+cat >gcp_key.json <<EOF
+{
+  "private_key": "${GCP_BACKEND_PRIVATE_KEY}",
+  "client_email": "${GCP_BACKEND_CLIENT_EMAIL}"
+}
+EOF
+```
+
+9. Run the test suite: `npm run test_crr`.
 
 ### Tests for Backbeat API:
 

tests/zenko_e2e/backbeat/gcpStorage.js

@@ -0,0 +1,7 @@
+const Storage = require('@google-cloud/storage');
+
+const storage = new Storage({
+    keyFilename: `${process.cwd()}/gcp_key.json`,
+});
+
+module.exports = storage;