Make GitHub-hosted runners available right from the matrix generation

.github/workflows/build_wheels_linux.yml

@@ -11,10 +11,6 @@ on:
         description: 'Reference to checkout, defaults to "nightly"'
         default: "nightly"
         type: string
-      runner:
-        description: "Instance to run the workflow on"
-        default: ' '
-        type: string
       test-infra-repository:
         description: "Test infra repository to use"
         default: "pytorch/test-infra"
@@ -88,7 +84,7 @@ jobs:
       UPLOAD_TO_BASE_BUCKET: ${{ matrix.upload_to_base_bucket }}
       ARCH: ${{ inputs.architecture }}
     name: ${{ matrix.build_name }}
-    runs-on: ${{ inputs.runner == ' ' && matrix.validation_runner || inputs.runner }}
+    runs-on: ${{ matrix.validation_runner }}
     container:
       image: ${{ matrix.container_image }}
       options: ${{ matrix.gpu_arch_type == 'cuda' && '--gpus all' || ' ' }}

.github/workflows/generate_binary_build_matrix.yml

@@ -39,6 +39,10 @@ on:
         description: "Use only download.pytorch.org when generating wheel install command?"
         default: "false"
         type: string
+      use-github-hosted-runners:
+        description: "Use GitHub-Hosted Runners for your build jobs?"
+        default: "disable"
+        type: string
     outputs:
       matrix:
         description: "Generated build matrix"
@@ -68,6 +72,7 @@ jobs:
           WITH_CUDA: ${{ inputs.with-cuda }}
           WITH_ROCM: ${{ inputs.with-rocm }}
           WITH_CPU: ${{ inputs.with-cpu }}
+          USE_GITHUB_HOSTED_RUNNERS: ${{ inputs.use-github-hosted-runners }}
           # limit pull request builds to one version of python unless ciflow/binaries/all is applied to the workflow
           # should not affect builds that are from events that are not the pull_request event
           LIMIT_PR_BUILDS: ${{ github.event_name == 'pull_request' && !contains( github.event.pull_request.labels.*.name, 'ciflow/binaries/all') }}

.github/workflows/test_build_wheels_linux_without_cuda.yml

@@ -16,7 +16,7 @@ permissions:
   contents: read
 
 jobs:
-  generate-matrix:
+  generate-matrix-default:
     uses: ./.github/workflows/generate_binary_build_matrix.yml
     with:
       package-type: wheel
@@ -25,7 +25,7 @@ jobs:
       test-infra-ref: ${{ github.ref }}
       with-cuda: disable
   test_default_runner:
-    needs: generate-matrix
+    needs: generate-matrix-default
     strategy:
       fail-fast: false
       matrix:
@@ -48,14 +48,22 @@ jobs:
       smoke-test-script: ${{ matrix.smoke-test-script }}
       package-name: ${{ matrix.package-name }}
       trigger-event: "${{ github.event_name }}"
+  generate-matrix-github-hosted:
+    uses: ./.github/workflows/generate_binary_build_matrix.yml
+    with:
+      package-type: wheel
+      os: linux
+      test-infra-repository: ${{ github.repository }}
+      test-infra-ref: ${{ github.ref }}
+      with-cuda: disable
+      use-github-hosted-runners: enable
   test_github_runner:
-    needs: generate-matrix
+    needs: generate-matrix-github-hosted
     uses: ./.github/workflows/build_wheels_linux.yml
     name: pytorch/text
     with:
       repository: pytorch/text
       ref: nightly
-      runner: ubuntu-latest
       test-infra-repository: ${{ github.repository }}
       test-infra-ref: ${{ github.ref }}
       build-matrix: ${{ needs.generate-matrix.outputs.matrix }}

tools/scripts/generate_binary_build_matrix.py

@@ -67,6 +67,7 @@
 mod.ROCM_ARCHES = ROCM_ARCHES_DICT[NIGHTLY]
 mod.PYTHON_ARCHES = PYTHON_ARCHES_DICT[NIGHTLY]
 
+# Default Runners (A mix of self-hosted runners on AWS and GH-hosted)
 LINUX_GPU_RUNNER = "linux.g5.4xlarge.nvidia.gpu"
 LINUX_CPU_RUNNER = "linux.2xlarge"
 LINUX_AARCH64_RUNNER = "linux.arm64.2xlarge"
@@ -75,6 +76,13 @@
 MACOS_M1_RUNNER = "macos-m1-stable"
 MACOS_RUNNER = "macos-12"
 
+# GitHub-Hosted Alternatives to some of the above
+# Note that a Linux aarch64 runner is not available through GitHub
+LINUX_GPU_GITHUB_RUNNER = "4-core-ubuntu-gpu-t4"
+LINUX_CPU_GITHUB_RUNNER = "ubuntu-latest"
+WIN_GPU_GITHUB_RUNNER = "4-core-windows-gpu-t4"
+WIN_CPU_GITHUB_RUNNER = "windows-latest"
+
 PACKAGES_TO_INSTALL_WHL = "torch torchvision torchaudio"
 
 PACKAGES_TO_INSTALL_CONDA = "pytorch torchvision torchaudio"
@@ -97,19 +105,31 @@ def arch_type(arch_version: str) -> str:
         return CPU
 
 
-def validation_runner(arch_type: str, os: str) -> str:
+def validation_runner(arch_type: str, os: str, use_github_hosted_runners: str) -> str:
     if os == LINUX:
         if arch_type == CUDA:
-            return LINUX_GPU_RUNNER
+            if use_github_hosted_runners:
+                return LINUX_GPU_GITHUB_RUNNER
+            else:
+                return LINUX_GPU_RUNNER
         else:
-            return LINUX_CPU_RUNNER
+            if use_github_hosted_runners:
+                return LINUX_CPU_GITHUB_RUNNER
+            else:
+                return LINUX_CPU_RUNNER
     elif os == LINUX_AARCH64:
         return LINUX_AARCH64_RUNNER
     elif os == WINDOWS:
         if arch_type == CUDA:
-            return WIN_GPU_RUNNER
+            if use_github_hosted_runners:
+                return WIN_GPU_GITHUB_RUNNER
+            else:
+                return WIN_GPU_RUNNER
         else:
-            return WIN_CPU_RUNNER
+            if use_github_hosted_runners:
+                return WIN_CPU_GITHUB_RUNNER
+            else:
+                return WIN_CPU_RUNNER
     elif os == MACOS_ARM64:
         return MACOS_M1_RUNNER
     elif os == MACOS:
@@ -297,6 +317,7 @@ def generate_conda_matrix(
     with_cuda: str,
     with_rocm: str,
     with_cpu: str,
+    use_github_hosted_runners: str,
     limit_pr_builds: bool,
     use_only_dl_pytorch_org: bool,
 ) -> List[Dict[str, str]]:
@@ -334,7 +355,7 @@ def generate_conda_matrix(
                     "build_name": f"conda-py{python_version}-{gpu_arch_type}{gpu_arch_version}".replace(
                         ".", "_"
                     ),
-                    "validation_runner": validation_runner(gpu_arch_type, os),
+                    "validation_runner": validation_runner(gpu_arch_type, os, use_github_hosted_runners),
                     "channel": channel,
                     "stable_version": mod.CURRENT_VERSION,
                     "installation": get_conda_install_command(
@@ -352,6 +373,7 @@ def generate_libtorch_matrix(
     with_cuda: str,
     with_rocm: str,
     with_cpu: str,
+    use_github_hosted_runners: str,
     limit_pr_builds: bool,
     use_only_dl_pytorch_org: bool,
     abi_versions: Optional[List[str]] = None,
@@ -417,7 +439,7 @@ def generate_libtorch_matrix(
                             ".", "_"
                         ),
                         # Please noe since libtorch validations are minimal, we use CPU runners
-                        "validation_runner": validation_runner(CPU, os),
+                        "validation_runner": validation_runner(CPU, os, use_github_hosted_runners),
                         "installation": get_libtorch_install_command(
                             os,
                             channel,
@@ -440,6 +462,7 @@ def generate_wheels_matrix(
     with_cuda: str,
     with_rocm: str,
     with_cpu: str,
+    use_github_hosted_runners: str,
     limit_pr_builds: bool,
     use_only_dl_pytorch_org: bool,
     arches: Optional[List[str]] = None,
@@ -502,7 +525,7 @@ def generate_wheels_matrix(
                     "build_name": f"{package_type}-py{python_version}-{gpu_arch_type}{gpu_arch_version}".replace(
                         ".", "_"
                     ),
-                    "validation_runner": validation_runner(gpu_arch_type, os),
+                    "validation_runner": validation_runner(gpu_arch_type, os, use_github_hosted_runners),
                     "installation": get_wheel_install_command(
                         os,
                         channel,
@@ -536,6 +559,7 @@ def generate_build_matrix(
     with_cpu: str,
     limit_pr_builds: str,
     use_only_dl_pytorch_org: str,
+    use_github_hosted_runners: str,
 ) -> Dict[str, List[Dict[str, str]]]:
     includes = []
 
@@ -555,6 +579,7 @@ def generate_build_matrix(
                     with_cuda,
                     with_rocm,
                     with_cpu,
+                    use_github_hosted_runners,
                     limit_pr_builds == "true",
                     use_only_dl_pytorch_org == "true",
                 )
@@ -624,6 +649,13 @@ def main(args) -> None:
         choices=["true", "false"],
         default=os.getenv("USE_ONLY_DL_PYTORCH_ORG", "false"),
     )
+    parser.add_argument(
+        "--use-github-runners",
+        help="Use GitHub Hosted Runners?",
+        type=str,
+        choices=[ENABLE, DISABLE],
+        default=os.getenv("USE_GITHUB_HOSTED_RUNNERS", DISABLE),
+    )
 
     options = parser.parse_args(args)
 
@@ -640,6 +672,7 @@ def main(args) -> None:
         options.with_cpu,
         options.limit_pr_builds,
         options.use_only_dl_pytorch_org,
+        options.use_github_hosted_runners,
     )
 
     print(json.dumps(build_matrix))