Switch Google Cloud SDK from DockerHub to GCR BA-4640

CHANGELOG.md

@@ -13,6 +13,13 @@ Cromwell 47 now supports GCS parallel composite uploads which can greatly improv
 This feature is turned off by default, it can be turned on by either a backend-level configuration setting or
 on a per-workflow basis with workflow options. More details [here](https://cromwell.readthedocs.io/en/stable/backends/Google/).
 
+### Papi V2 Localization Using GCR [(#5200)](https://github.com/broadinstitute/cromwell/pull/5200)
+
+The Docker image for the Google Cloud SDK was previously only [published on Docker
+Hub](https://hub.docker.com/r/google/cloud-sdk). Now that the image is [publicly hosted in
+GCR](http://gcr.io/google.com/cloudsdktool/cloud-sdk), Papi V2 jobs will localize inputs and delocalize outputs using
+the GCR image.
+
 ## 46 Release Notes
 
 ### Nvidia GPU Driver Update

centaur/src/main/resources/engineUpgradeTestCases/call_cache_cha_cha/call_cache_cha_cha_papi.inputs.json

@@ -1,5 +1,5 @@
 {
   "call_cache_cha_cha.modify_file_command_prefix": "echo \"override\" | gsutil cp -",
   "call_cache_cha_cha.modify_file_using_cloud": true,
-  "call_cache_cha_cha.modify_file_docker": "google/cloud-sdk"
+  "call_cache_cha_cha.modify_file_docker": "gcr.io/google.com/cloudsdktool/cloud-sdk"
 }

centaur/src/main/resources/standardTestCases/call_cache_capoeira/call_cache_capoeira_jes.wdl

@@ -83,7 +83,7 @@ task modify_file_gcs {
         echo "override" | gsutil cp - ${file_path_raw}
     }
     runtime {
-        docker: "google/cloud-sdk"
+        docker: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     }
     output {
         Boolean done = true

centaur/src/main/resources/standardTestCases/cwl_resources/cwl_disk_resources.cwl

@@ -20,7 +20,7 @@ $graph:
   hints:
   - class: ShellCommandRequirement
   - class: DockerRequirement
-    dockerPull: google/cloud-sdk:slim
+    dockerPull: gcr.io/google.com/cloudsdktool/cloud-sdk:slim
   - class: dx:InputResourceRequirement
     indirMin: 10240
   inputs: []

centaur/src/main/resources/standardTestCases/cwl_resources/cwl_inputdir_zero_doesnt_localize.cwl

@@ -7,7 +7,7 @@ $graph:
   hints:
   - class: ShellCommandRequirement
   - class: DockerRequirement
-    dockerPull: google/cloud-sdk:slim
+    dockerPull: gcr.io/google.com/cloudsdktool/cloud-sdk:slim
   - class: dx:InputResourceRequirement
     indirMin: 0
   inputs: []

centaur/src/main/resources/standardTestCases/dedup_localizations_papi_v2/dedup_localizations_papi_v2.wdl

@@ -55,5 +55,5 @@ task check_log {
       File out = stdout()
       Int num_input_localizations = read_int(stdout())
   }
-  runtime { docker: "google/cloud-sdk" }
+  runtime { docker: "gcr.io/google.com/cloudsdktool/cloud-sdk" }
 }

centaur/src/main/resources/standardTestCases/error_10_preemptible/error_10_preemptible.wdl

@@ -20,7 +20,7 @@ task delete_self_if_preemptible {
 
   runtime {
     preemptible: 1
-    docker: "google/cloud-sdk:slim"
+    docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:slim"
   }
 }
 

centaur/src/main/resources/standardTestCases/google_labels/google_labels.wdl

@@ -49,7 +49,7 @@ task check_labels {
     ~{sep="\n" label_checker_lines}
   }
   runtime {
-    docker: "google/cloud-sdk:slim"
+    docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:slim"
     # Specify this zone because it's used in the curl commands above. We probably *could* work this out ad-hoc but it's easier to hard-code it here:
     zones: ["us-central1-c"]
     failOnStderr: true

centaur/src/main/resources/standardTestCases/gpu_on_papi/gpu_on_papi.wdl

@@ -35,7 +35,7 @@ task task_with_gpu {
   runtime {
     gpuCount: 1
     gpuType: gpuTypeInput
-    docker: "google/cloud-sdk:slim"
+    docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:slim"
     zones: ["us-central1-c"]
   }
 }

centaur/src/main/resources/standardTestCases/papi_cpu_platform/papi_cpu_platform.wdl

@@ -14,7 +14,7 @@ task cpu_platform {
     }
 
     runtime {
-        docker: "google/cloud-sdk:slim"
+        docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:slim"
         cpuPlatform: cpu_platform
     }
 

centaur/src/main/resources/standardTestCases/papi_v2_gcsa/papi_v2_gcsa.wdl

@@ -26,7 +26,7 @@ task get_token_info {
     >>>
 
     runtime {
-        docker: "google/cloud-sdk:slim"
+        docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:slim"
         backend: "papi-v2-gcsa"
     }
 

centaur/src/main/resources/standardTestCases/papi_v2_log/papi_v2_log.wdl

@@ -60,5 +60,5 @@ task check_log {
         File out = stdout()
         Array[Int] out_counts = read_lines(stdout())
     }
-    runtime { docker: "google/cloud-sdk" }
+    runtime { docker: "gcr.io/google.com/cloudsdktool/cloud-sdk" }
 }

centaur/src/main/resources/standardTestCases/virtual_private_cloud/check_network_in_vpc.wdl

@@ -16,7 +16,7 @@ task get_network {
   }
 
   runtime {
-    docker: "google/cloud-sdk:slim"
+    docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:slim"
     backend: "Papiv2-Virtual-Private-Cloud"
   }
 

centaur/src/main/resources/standardTestCases/wdl_draft3/call_cache_capoeira/draft3_call_cache_capoeira_jes.wdl

@@ -91,7 +91,7 @@ task modify_file_gcs {
     echo "override" | gsutil cp - ~{file_path_raw}
   }
   runtime {
-    docker: "google/cloud-sdk"
+    docker: "gcr.io/google.com/cloudsdktool/cloud-sdk"
   }
   output {
     Boolean done = true

centaur/src/main/resources/standardTestCases/wdl_draft3/draft3_nio_file/draft3_nio_file_papi1.wdl

@@ -131,7 +131,7 @@ task nio_file {
     grep -q "draft3_nio_file.nio_file.y" <<< $PAPI_METADATA && echo "y was incorrectly localized" >> errors.txt
   }
   runtime {
-    docker: "google/cloud-sdk:slim"
+    docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:slim"
     zones: ["us-central1-c"]
 
     # Depending on the final 'grep', the return code is probably going to be '1'... which is fine!

centaur/src/main/resources/standardTestCases/wdl_draft3/draft3_nio_file/draft3_nio_file_papi2.wdl

@@ -124,7 +124,7 @@ task nio_file {
     find . -name y >> errors.txt
   }
   runtime {
-    docker: "google/cloud-sdk:slim"
+    docker: "gcr.io/google.com/cloudsdktool/cloud-sdk:slim"
     zones: ["us-central1-c"]
   }
   output {

scripts/perf/helper.inc.sh

@@ -154,7 +154,8 @@ read_service_account_from_vault() {
 gcloud_run_as_service_account() {
   local name="$1"
   local command="$2"
-  docker run --name $name -v "$(pwd)"/mnt:${DOCKER_ETC_PATH} --rm google/cloud-sdk:slim /bin/bash -c "\
+  docker run --name $name -v "$(pwd)"/mnt:${DOCKER_ETC_PATH} --rm gcr.io/google.com/cloudsdktool/cloud-sdk:slim \
+    /bin/bash -c "\
     gcloud auth activate-service-account --key-file ${DOCKER_ETC_PATH}/sa.json 2> /dev/null &&\
     ${command}"
 }

supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala

@@ -178,10 +178,6 @@ case class GenomicsFactory(applicationName: String, authMode: GoogleAuthMode, en
        * Adjust using docker images used by Cromwell as well as the tool's docker image size if available
        */
       val adjustedBootDiskSize = {
-        /*
-         * At the moment, google/cloud-sdk:slim (664MB on 12/3/18) and possibly stedolan/jq (182MB) decompressed ~= 1 GB
-         */
-        val cromwellImagesSize = 1
         val fromRuntimeAttributes = createPipelineParameters.runtimeAttributes.bootDiskSize
         // Compute the decompressed size based on the information available
         val actualSizeInBytes = createPipelineParameters.jobDescriptor.dockerSize.map(_.toFullSize(DockerConfiguration.instance.sizeCompressionFactor)).getOrElse(0L)
@@ -193,7 +189,7 @@ case class GenomicsFactory(applicationName: String, authMode: GoogleAuthMode, en
           jobLogger.info(s"Adjusting boot disk size to $actualSizeRoundedUpInGB GB to account for docker image size")
         }
 
-        Math.max(fromRuntimeAttributes, actualSizeRoundedUpInGB) + cromwellImagesSize
+        Math.max(fromRuntimeAttributes, actualSizeRoundedUpInGB) + GenomicsFactory.CromwellImagesSizeRoundedUpInGB
       }
 
       val virtualMachine = new VirtualMachine()
@@ -251,6 +247,22 @@ object GenomicsFactory {
     * For some reason we couldn't find this scope within Google libraries
     */
   val MonitoringWrite = "https://www.googleapis.com/auth/monitoring.write"
+
+  /**
+    * An image with the Google Cloud SDK installed.
+    * http://gcr.io/google.com/cloudsdktool/cloud-sdk
+    *
+    * FYI additional older versions are available on DockerHub at:
+    * https://hub.docker.com/r/google/cloud-sdk
+    *
+    * When updating this value, also consider updating the CromwellImagesSizeRoundedUpInGB below.
+    */
+  val CloudSdkImage = "gcr.io/google.com/cloudsdktool/cloud-sdk:slim"
+
+  /*
+   * At the moment, the cloud-sdk:slim (727MB on 2019-09-26) and possibly stedolan/jq (182MB) decompressed ~= 1 GB
+   */
+  val CromwellImagesSizeRoundedUpInGB = 1
 }
 
 case class ProjectLabels(labels: Map[String, String])

supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/PipelinesParameterConversions.scala

@@ -70,7 +70,7 @@ trait PipelinesParameterConversions {
             .withFlags(List(ActionFlag.RunInBackground, ActionFlag.EnableFuse))
           List(ActionBuilder.describeParameter(fileInput, labels), localizationAction)
         case _: HttpPath =>
-          val dockerImage = "google/cloud-sdk:slim"
+          val dockerImage = GenomicsFactory.CloudSdkImage
           val command = s"curl --silent --create-dirs --output ${fileInput.containerPath} ${fileInput.cloudPath}"
           val localizationAction = ActionBuilder
             .withImage(dockerImage)

supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/api/ActionBuilder.scala

@@ -3,6 +3,7 @@ package cromwell.backend.google.pipelines.v2alpha1.api
 import com.google.api.services.genomics.v2alpha1.model.{Action, Mount, Secret}
 import cromwell.backend.google.pipelines.common.api.PipelinesApiRequestFactory.CreatePipelineDockerKeyAndToken
 import cromwell.backend.google.pipelines.common.{PipelinesApiInput, PipelinesApiOutput, PipelinesParameter}
+import cromwell.backend.google.pipelines.v2alpha1.GenomicsFactory
 import cromwell.backend.google.pipelines.v2alpha1.api.ActionBuilder.Labels._
 import cromwell.backend.google.pipelines.v2alpha1.api.ActionFlag.ActionFlag
 import cromwell.docker.DockerImageIdentifier
@@ -59,10 +60,7 @@ object ActionBuilder {
     }
   }
 
-  // TODO revert this to google/cloud-sdk:slim once latest is unbroken
-  // TODO https://github.com/GoogleCloudPlatform/gsutil/issues/806
-  val cloudSdkImage = "google/cloud-sdk:251.0.0-slim"
-  def cloudSdkAction: Action = new Action().setImageUri(cloudSdkImage)
+  def cloudSdkAction: Action = new Action().setImageUri(GenomicsFactory.CloudSdkImage)
 
   def withImage(image: String) = new Action()
     .setImageUri(image)

supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/api/Delocalization.scala

@@ -9,7 +9,7 @@ import cromwell.backend.google.pipelines.common.PipelinesApiConfigurationAttribu
 import cromwell.backend.google.pipelines.common.PipelinesApiJobPaths.GcsDelocalizationScriptName
 import cromwell.backend.google.pipelines.common.api.PipelinesApiRequestFactory.CreatePipelineParameters
 import cromwell.backend.google.pipelines.v2alpha1.PipelinesConversions._
-import cromwell.backend.google.pipelines.v2alpha1.RuntimeOutputMapping
+import cromwell.backend.google.pipelines.v2alpha1.{GenomicsFactory, RuntimeOutputMapping}
 import cromwell.backend.google.pipelines.v2alpha1.ToParameter.ops._
 import cromwell.backend.google.pipelines.v2alpha1.api.ActionBuilder.Labels.{Key, Value}
 import cromwell.backend.google.pipelines.v2alpha1.api.ActionBuilder._
@@ -63,7 +63,7 @@ trait Delocalization {
     )
 
     ActionBuilder
-      .withImage(womOutputRuntimeExtractor.dockerImage.getOrElse(cloudSdkImage))
+      .withImage(womOutputRuntimeExtractor.dockerImage.getOrElse(GenomicsFactory.CloudSdkImage))
       .setCommands(commands.asJava)
       .withMounts(mounts)
       .setEntrypoint("/bin/bash")

supportedBackends/google/pipelines/v2alpha1/src/test/scala/cromwell/backend/google/pipelines/v2alpha1/PipelinesConversionsSpec.scala

@@ -9,7 +9,6 @@ import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.backend.google.pipelines.common.PipelinesApiConfigurationAttributes.GcsTransferConfiguration
 import cromwell.backend.google.pipelines.common.PipelinesApiFileInput
 import cromwell.backend.google.pipelines.common.io.{DiskType, PipelinesApiWorkingDisk}
-import cromwell.backend.google.pipelines.v2alpha1.api.ActionBuilder
 import cromwell.core.path.DefaultPathBuilder
 import cromwell.filesystems.drs.DrsPathBuilder
 import eu.timepit.refined.refineMV
@@ -70,7 +69,7 @@ class PipelinesConversionsSpec extends FlatSpec with Matchers {
     logging.get("mounts") should be(a[java.util.List[_]])
     logging.get("mounts").asInstanceOf[java.util.List[_]] should be (empty)
 
-    logging.get("imageUri") should be(ActionBuilder.cloudSdkImage)
+    logging.get("imageUri") should be(GenomicsFactory.CloudSdkImage)
 
     val loggingLabels = logging.get("labels").asInstanceOf[java.util.Map[_, _]]
     loggingLabels.keySet.asScala should contain theSameElementsAs List("logging", "inputName")

supportedBackends/google/pipelines/v2alpha1/src/test/scala/cromwell/backend/google/pipelines/v2alpha1/api/ActionBuilderSpec.scala

@@ -3,6 +3,7 @@ package cromwell.backend.google.pipelines.v2alpha1.api
 import java.util
 
 import com.google.api.services.genomics.v2alpha1.model.{Action, Mount}
+import cromwell.backend.google.pipelines.v2alpha1.GenomicsFactory
 import cromwell.backend.google.pipelines.v2alpha1.api.ActionBuilder.Labels.{Key, Value}
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.{FlatSpec, Matchers}
@@ -15,18 +16,18 @@ class ActionBuilderSpec extends FlatSpec with Matchers with TableDrivenPropertyC
 
   val dockerRunActions = Table(
     ("description", "action", "command"),
-    ("a cloud sdk action", ActionBuilder.cloudSdkAction, s"docker run ${ActionBuilder.cloudSdkImage}"),
+    ("a cloud sdk action", ActionBuilder.cloudSdkAction, s"docker run ${GenomicsFactory.CloudSdkImage}"),
     ("a cloud sdk action with args",
       ActionBuilder.cloudSdkAction.setCommands(List("bash", "-c", "echo hello").asJava),
-      s"docker run ${ActionBuilder.cloudSdkImage} bash -c echo\\ hello"
+      s"docker run ${GenomicsFactory.CloudSdkImage} bash -c echo\\ hello"
     ),
     ("a cloud sdk action with quotes in the args",
       ActionBuilder.cloudSdkAction.setCommands(List("bash", "-c", "echo hello m'lord").asJava),
-      s"docker run ${ActionBuilder.cloudSdkImage} bash -c echo\\ hello\\ m\\'lord"
+      s"docker run ${GenomicsFactory.CloudSdkImage} bash -c echo\\ hello\\ m\\'lord"
     ),
     ("a cloud sdk action with a newline in the args",
       ActionBuilder.cloudSdkAction.setCommands(List("bash", "-c", "echo hello\\\nworld").asJava),
-      s"docker run ${ActionBuilder.cloudSdkImage} bash -c echo\\ hello\\\\world"
+      s"docker run ${GenomicsFactory.CloudSdkImage} bash -c echo\\ hello\\\\world"
     ),
     ("an action with multiple args",
       new Action()