Skip to content

Commit

Permalink
[Split Build] limit split build matrix generation to cuda nightly jobs
Browse files Browse the repository at this point in the history
Validation this works

```
sahanp@sahanp-mbp ~/test-infra [2]> python tools/scripts/generate_binary_build_matrix.py --use-split-build true                (base)

{
  "include": [
    {
      "python_version": "3.8",
      "gpu_arch_type": "cpu",
      "gpu_arch_version": "",
      "desired_cuda": "cpu",
      "container_image": "pytorch/manylinux-builder:cpu",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cpu",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "11.8",
      "desired_cuda": "cu118",
      "container_image": "pytorch/manylinux-builder:cuda11.8",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda11_8",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "11.8",
      "desired_cuda": "cu118",
      "container_image": "pytorch/manylinux-builder:cuda11.8",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda11_8-split",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": true
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "12.1",
      "desired_cuda": "cu121",
      "container_image": "pytorch/manylinux-builder:cuda12.1",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda12_1",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "12.1",
      "desired_cuda": "cu121",
      "container_image": "pytorch/manylinux-builder:cuda12.1",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda12_1-split",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": true
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "12.4",
      "desired_cuda": "cu124",
      "container_image": "pytorch/manylinux-builder:cuda12.4",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda12_4",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "12.4",
      "desired_cuda": "cu124",
      "container_image": "pytorch/manylinux-builder:cuda12.4",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda12_4-split",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": true
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "rocm",
      "gpu_arch_version": "6.0",
      "desired_cuda": "rocm6.0",
      "container_image": "pytorch/manylinux-builder:rocm6.0",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-rocm6_0",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.0",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "rocm",
      "gpu_arch_version": "6.1",
      "desired_cuda": "rocm6.1",
      "container_image": "pytorch/manylinux-builder:rocm6.1",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-rocm6_1",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.1",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    }
  ]
}
```
  • Loading branch information
PaliC committed Jun 25, 2024
1 parent b62e5f1 commit 796d69f
Showing 1 changed file with 1 addition and 1 deletion.
2 changes: 1 addition & 1 deletion tools/scripts/generate_binary_build_matrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -545,7 +545,7 @@ def generate_wheels_matrix(
"use_split_build": False,
}
ret.append(entry)
if use_split_build:
if use_split_build and (gpu_arch_version in CUDA_ARCHES) and (os == LINUX) and (channel == NIGHTLY):
entry = entry.copy()
entry["build_name"] = f"{package_type}-py{python_version}-{gpu_arch_type}{gpu_arch_version}-split".replace(
".", "_"
Expand Down

0 comments on commit 796d69f

Please sign in to comment.