Marketplace vm solution v0.5.8 (#348)

* fixed a bug in the VM startup script when applying the Istio AuthN/Z policies, removed cloudcommerceprocurement API from VM startup script.

* fixes #317

* updated release notes with changes and added new fields to test_config.yaml

* fixing image size

* fixing image size

* fixing images sizes

* fixes #344

* fixes #339

api/deploy_ds_api.py

@@ -13,6 +13,13 @@
 # limitations under the License.
 """Creates service account, custom role and builds DS API image and deploys to Cloud Run."""
 
+def domain_has_protocol(domain):
+    if domain.find("https://") >= 0:
+        return True
+    elif domain.find("http://") >= 0:
+        raise Exception('Invalid protocol provided in uiDomainName (http:// should be https:// or not included)')
+    else:
+        return False
 
 def GenerateConfig(context):
     """Generate YAML resource configuration."""
@@ -28,6 +35,10 @@ def GenerateConfig(context):
     service_acct_name = context.properties['serviceAccountName']
     service_acct_descr = context.properties['serviceAccountDesc']
     custom_role_name = context.properties['customRoleName']
+    ui_domain_name = ""
+    # if hasattr(context.properties, 'uiDomainName'):
+    if context.properties['uiDomainName'] != None:
+        ui_domain_name = context.properties['uiDomainName']
     delete_timeout = '120s'
     general_timeout = context.properties['timeout']
     # admin_sa = context.properties['adminServiceAccount']
@@ -100,6 +111,14 @@ def GenerateConfig(context):
             '--platform=gke',
             '--service-account=' + service_acct_name
             ]
+
+    # if a user includes the UI domain name then include it as an environment variable
+    domain_protocol = 'https://'
+    if ui_domain_name is not "":
+        if domain_has_protocol(ui_domain_name):
+            steps[5]['args'].append('--set-env-vars=UI_BASE_URL=' + ui_domain_name)
+        else:
+            steps[5]['args'].append('--set-env-vars=UI_BASE_URL=' + domain_protocol + ui_domain_name)
 
     git_release = {  # Checkout the correct release
                 'name': 'gcr.io/cloud-builders/git',

frontend/deploy_ui_cloud_run.py

@@ -15,6 +15,13 @@
 
 import json
 
+def domain_has_protocol(domain):
+    if domain.find("https://") >= 0:
+        return True
+    elif domain.find("http://") >= 0:
+        raise Exception('Invalid protocol provided in API Domain Name (http:// should be https:// or not included)')
+    else:
+        return False
 
 def GenerateConfig(context):
   """Generate YAML resource configuration."""
@@ -23,15 +30,21 @@ def GenerateConfig(context):
   datashare_ui_name_with_tag = "ds-frontend-ui:dev"
   container_tag = context.properties['containerTag']
   cloud_run_deploy_name = context.properties['cloudRunDeployName']
-  datashare_ui_name = "ds-frontend-ui"
+  #datashare_ui_name = "ds-frontend-ui"
   gcp_region = context.properties['region']
+  client_id = context.properties['clientId']
   delete_timeout = '120s'
   general_timeout = context.properties['timeout']
   cmd = "https://github.com/GoogleCloudPlatform/datashare-toolkit.git"
   git_release_version = "master"
   if context.properties['datashareGitReleaseTag'] != None:
       git_release_version = context.properties['datashareGitReleaseTag']
 
+  api_domain_name = ""
+  # if hasattr(context.properties, 'uiDomainName'):
+  if context.properties['apiDomainName'] != None:
+      api_domain_name = context.properties['apiDomainName']
+
   steps = [
               { # Clone the Datashare repository only if the ds-frontend-ui:dev is not present
                 'name': 'gcr.io/cloud-builders/git',
@@ -65,7 +78,20 @@ def GenerateConfig(context):
                           ]
               }
           ]
-
+  # if a user includes the UI domain name then include it as an environment variable
+  domain_protocol = 'https://'
+  base_path = '/v1alpha'
+  environment_variables = ""
+  if api_domain_name is not "":
+      if domain_has_protocol(api_domain_name):
+          environment_variables = 'VUE_APP_API_BASE_URL=' + api_domain_name + base_path
+      else:
+          environment_variables = 'VUE_APP_API_BASE_URL=' + domain_protocol + api_domain_name + base_path
+  # set the Project ID and Client ID environment variables
+  environment_variables += ',VUE_APP_PROJECT_ID=$PROJECT_ID'
+  environment_variables += ',VUE_APP_GOOGLE_APP_CLIENT_ID=' + client_id
+  steps[2]['args'].append('--set-env-vars=' + environment_variables)
+
   if git_release_version != "master":
       git_release = { # Checkout the correct release
                 'name': 'gcr.io/cloud-builders/git',

frontend/deploy_ui_cloud_run.py.schema

@@ -21,7 +21,8 @@ required:
 - containerTag
 - region
 - timeout
-
+- clientId
+- apiDomainName
 
 properties:
   cloudRunDeployName:
@@ -52,6 +53,14 @@ properties:
     type: boolean
     description: Deploy all the components (UI, API, Ingestion)
     default: False
+
+  clientId:
+    type: string
+    description: The web app client ID.
+
+  apiDomainName:
+    type: string
+    description: The web app client ID.
 
 outputs:
   properties:

frontend/deploy_ui_cloud_run.yaml

@@ -8,4 +8,6 @@ resources:
     containerTag: dev
     region: us-central1
     timeout: 600s
-    datashareGitReleaseTag: master
+    datashareGitReleaseTag: master
+    clientId: XYZ
+    apiDomainName: api.datashare.yourdomain.com

marketplace/PREREQUISITES.md

@@ -19,9 +19,10 @@ Enable the Kubernetes API as well, since the Datashare API is deployed to Cloud
 1. [Enable the Kubernetes API from Cloud Console](https://console.cloud.google.com/apis/library/container.googleapis.com)
 2. [Enable the Runtime Config API from Cloud Console](https://console.cloud.google.com/apis/library/runtimeconfig.googleapis.com)
 3. [Enable the Cloud Build API from Cloud Console](https://console.cloud.google.com/apis/library/cloudbuild.googleapis.com)
+4. [Enable the Commerce API from Cloud Console](https://console.cloud.google.com/apis/library/cloudcommerceprocurement.googleapis.com)
 
 ### From the command line
-1. `gcloud services enable container.googleapis.com runtimeconfig.googleapis.com cloudbuild.googleapis.com`
+1. `gcloud services enable container.googleapis.com runtimeconfig.googleapis.com cloudbuild.googleapis.com cloudcommerceprocurement.googleapis.com`
 
 ## Update service account from Google Cloud Console
 1. Login to Google Cloud Console and select `IAM` from the menu.
@@ -30,26 +31,26 @@ Enable the Kubernetes API as well, since the Datashare API is deployed to Cloud
 
 2. Select `Service Accounts` on the left side of the screen
 
-![Service Accounts](images/iam-select-service-account.png)
+<img src="images/iam-select-service-account.png" width="400" title="Service Accounts"> 
 
 3. Click `Create Service Account`.
 
-![Create SA](images/iam-create-sa.png)
+<img src="images/iam-create-sa.png" width="400" title="Create SA"> 
 
 4. Enter the following and then click the `Create` button. 
 * `Service account name` as `datashare-deployment-manager`
 * `Service account description` as `Datashare deployment manager`
 
 5. Select the `Editor`, `Security Admin` and `Kubernetes Admin` roles.
 
-![Assign roles](images/iam-assign-roles-to-sa.png)
+<img src="images/iam-assign-roles-to-sa.png" width="500"> 
 
 6. Next add two `Service account users roles` to this service account.  These two members need to be able to execute commands on behalf of this service account. Then click the `Done` button. 
 * `PROJECT_NUMBER-compute@developer.gserviceaccount.com`
 * `PROJECT_NUMBER@cloudservices.gserviceaccount.com`
 * `PROJECT_NUMBER@cloudbuild.gserviceaccount.com`
 
-![Assign members](images/iam-assign-members-to-sa.png)
+<img src="images/iam-assign-members-to-sa.png" width="500" title="Assign Members">
 
 Now you can click the `Launch` button on the Marketplace and deploy the Datashare solution within your GCP project. 
 

marketplace/releases/RELEASE_NOTES_VM_SOLUTION.md

@@ -1,5 +1,27 @@
 # Google Cloud Platform Release Notes for Marketplace VM Solution
 
+## 0.5.8 Release Notes
+* Build - 10/14/2020
+
+### Datashare
+* Datashare release version is 0.4.4
+* Kubernetes cluster version is 1.16 to support incremental changes
+
+### Updates  
+* Added new fields to the launch page, to help complete the UI and API deployment successfully.
+  * UI domain name - optional
+  * API domain name - mandatory
+* Changed the cloud ingestion bucket name from `PROJECT-cds-bucket` to `PROJECT-datashare-ingestion`
+
+### Bug Fixes
+* Fixed the VM startup script that applied the Istio AuthN/AuthZ policies
+* Added enabling the cloudcommerceprocurement API as a prerequisite step 
+  * removed this from the VM startup script as it caused a permission error
+
+### Known Bugs
+* None
+
+
 ## 0.5.7 Release Notes
 * Build - 10/1/2020
 

marketplace/vm-solution/cloud_function.py

@@ -20,7 +20,7 @@ def GenerateConfig(context):
   source_archive_url = 'gs://%s/%s' % (context.properties['codeBucket'],
                                        'datashare-toolkit-cloud-function.zip')
   print(source_archive_url)
-
+  ingestion_bucket_name = context.properties['ingestionBucketName']
   useWaiter = context.properties['useRuntimeConfigWaiter']    
   #cmd = "echo '%s' | base64 -d > /function/function.zip;" % (content.decode('ascii'))
 
@@ -44,7 +44,7 @@ def GenerateConfig(context):
               context.properties['entryPoint'],
           'eventTrigger': {
               'eventType': 'providers/cloud.storage/eventTypes/object.change',
-              'resource': 'projects/' + context.env['project'] + '/buckets/' + context.env['project'] + '-cds-bucket'
+              'resource': 'projects/' + context.env['project'] + '/buckets/' + ingestion_bucket_name # + context.env['project'] + '-cds-bucket'
           },
           'timeout':
               context.properties['timeout'],

marketplace/vm-solution/cloud_function.py.schema

@@ -22,6 +22,7 @@ required:
 - codeBucket
 - codeBucketObject
 - runtime
+- ingestionBucketName
 
 properties:
   codeLocation:
@@ -56,6 +57,10 @@ properties:
     type: string
     description: Node.js runtime
 
+  ingestionBucketName:
+    type: string
+    description: Storage bucket where customer uploads their source data files to be ingested by Datashare
+
 outputs:
   properties:
     functionName:

marketplace/vm-solution/cluster.py

@@ -19,10 +19,10 @@
 def GenerateConfig(context):
   """Generate YAML resource configuration."""
 
-  name_prefix = context.env['deployment'] + '-' + context.env['name']
+  # name_prefix = context.env['deployment'] + '-' + context.env['name']
   cluster_name = 'datashare-cluster-resource'
   acutal_cluster_name = 'datashare'
-  type_name = name_prefix + '-type'
+  # type_name = name_prefix + '-type'
   cluster_version = '1.16'
   workload_pool = context.env['project'] + '.svc.id.goog'
   machine_type = 'e2-standard-2'

marketplace/vm-solution/common/vm_instance.py

@@ -1,4 +1,4 @@
-# Copyright 2015 Google Inc. All rights reserved.
+# Copyright 2020 Google Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Creates an Instance VM with common defaults."""
-# pylint: disable=g-import-not-at-top
+
 import copy
 import re
 import common
@@ -111,7 +111,8 @@ def GenerateComputeVM(context, create_disks_separately=True):
   project = context.env[default.PROJECT]
   deployApiToGke = context.properties['deployApiToGke']
   datashare_install_bucket_name = project + '-install-bucket'
-  datashare_ingestion_bucket_name = project + '-cds-bucket'
+  ingestion_bucket_name = context.properties['ingestionBucketName']
+
   k8s_cluster_name = 'datashare-cluster-resource'
   # gce_service_account = context.properties['gceServiceAccount']
 
@@ -162,7 +163,7 @@ def GenerateComputeVM(context, create_disks_separately=True):
           'type': default.INSTANCE,
           'metadata': {
               'dependsOn': [ 
-                datashare_ingestion_bucket_name
+                ingestion_bucket_name
               ]
           },
           'properties': {