Skip to content

Commit

Permalink
[Split Build] Add split build to binary build generation
Browse files Browse the repository at this point in the history 
Adds split build to binary generation script. The outputs of the json is as follows. We will need to also add it to testing proper.

```
sahanp@sahanp-mbp ~/test-infra> python tools/scripts/generate_binary_build_matrix.py                                                                                                                                                                                    (pytorch)
{
  "include": [
    {
      "python_version": "3.8",
      "gpu_arch_type": "cpu",
      "gpu_arch_version": "",
      "desired_cuda": "cpu",
      "container_image": "pytorch/manylinux-builder:cpu",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cpu",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "11.8",
      "desired_cuda": "cu118",
      "container_image": "pytorch/manylinux-builder:cuda11.8",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda11_8",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "12.1",
      "desired_cuda": "cu121",
      "container_image": "pytorch/manylinux-builder:cuda12.1",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda12_1",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "12.4",
      "desired_cuda": "cu124",
      "container_image": "pytorch/manylinux-builder:cuda12.4",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda12_4",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "rocm",
      "gpu_arch_version": "6.0",
      "desired_cuda": "rocm6.0",
      "container_image": "pytorch/manylinux-builder:rocm6.0",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-rocm6_0",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.0",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "rocm",
      "gpu_arch_version": "6.1",
      "desired_cuda": "rocm6.1",
      "container_image": "pytorch/manylinux-builder:rocm6.1",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-rocm6_1",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.1",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    }
  ]
}
sahanp@sahanp-mbp ~/test-infra> python tools/scripts/generate_binary_build_matrix.py --use-split-build true                                                                                                                                                             (pytorch)
{
  "include": [
    {
      "python_version": "3.8",
      "gpu_arch_type": "cpu",
      "gpu_arch_version": "",
      "desired_cuda": "cpu",
      "container_image": "pytorch/manylinux-builder:cpu",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cpu",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cpu",
      "gpu_arch_version": "",
      "desired_cuda": "cpu",
      "container_image": "pytorch/manylinux-builder:cpu",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cpu-split",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": true
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "11.8",
      "desired_cuda": "cu118",
      "container_image": "pytorch/manylinux-builder:cuda11.8",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda11_8",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "11.8",
      "desired_cuda": "cu118",
      "container_image": "pytorch/manylinux-builder:cuda11.8",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda11_8-split",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": true
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "12.1",
      "desired_cuda": "cu121",
      "container_image": "pytorch/manylinux-builder:cuda12.1",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda12_1",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "12.1",
      "desired_cuda": "cu121",
      "container_image": "pytorch/manylinux-builder:cuda12.1",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda12_1-split",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu121",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": true
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "12.4",
      "desired_cuda": "cu124",
      "container_image": "pytorch/manylinux-builder:cuda12.4",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda12_4",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "cuda",
      "gpu_arch_version": "12.4",
      "desired_cuda": "cu124",
      "container_image": "pytorch/manylinux-builder:cuda12.4",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-cuda12_4-split",
      "validation_runner": "linux.g5.4xlarge.nvidia.gpu",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu124",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": true
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "rocm",
      "gpu_arch_version": "6.0",
      "desired_cuda": "rocm6.0",
      "container_image": "pytorch/manylinux-builder:rocm6.0",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-rocm6_0",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.0",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "rocm",
      "gpu_arch_version": "6.0",
      "desired_cuda": "rocm6.0",
      "container_image": "pytorch/manylinux-builder:rocm6.0",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-rocm6_0-split",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.0",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": true
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "rocm",
      "gpu_arch_version": "6.1",
      "desired_cuda": "rocm6.1",
      "container_image": "pytorch/manylinux-builder:rocm6.1",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-rocm6_1",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.1",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": false
    },
    {
      "python_version": "3.8",
      "gpu_arch_type": "rocm",
      "gpu_arch_version": "6.1",
      "desired_cuda": "rocm6.1",
      "container_image": "pytorch/manylinux-builder:rocm6.1",
      "package_type": "manywheel",
      "build_name": "manywheel-py3_8-rocm6_1-split",
      "validation_runner": "linux.2xlarge",
      "installation": "pip3 install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/rocm6.1",
      "channel": "nightly",
      "upload_to_base_bucket": "no",
      "stable_version": "2.3.1",
      "use_split_build": true
    }
  ]
}
```
  • Loading branch information
PaliC committed Jun 25, 2024
1 parent 31528be commit 0da87a1
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 4 deletions.
9 changes: 9 additions & 0 deletions .github/workflows/generate_binary_build_matrix.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ on:
description: "Generate binary build matrix for a python only package (i.e. only one python version)"
default: "disable"
type: string
use_split_build:
description: |
[Experimental] Build a libtorch only wheel and build pytorch such that
are built from the libtorch wheel.
required: false
type: boolean
default: false

outputs:
matrix:
description: "Generated build matrix"
Expand Down Expand Up @@ -79,6 +87,7 @@ jobs:
# In cases when pipy binaries are not published yet.
USE_ONLY_DL_PYTORCH_ORG: ${{ inputs.use-only-dl-pytorch-org }}
BUILD_PYTHON_ONLY: ${{ inputs.build-python-only }}
USE_SPLIT_BUILD: ${{ inputs.use_split_build }}
run: |
set -eou pipefail
MATRIX_BLOB="$(python3 tools/scripts/generate_binary_build_matrix.py)"
Expand Down
35 changes: 31 additions & 4 deletions tools/scripts/generate_binary_build_matrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,8 +287,13 @@ def get_wheel_install_command(
desired_cuda: str,
python_version: str,
use_only_dl_pytorch_org: bool,
use_split_build: bool = False,
) -> str:

if use_split_build:
if (gpu_arch_version in CUDA_ARCHES) and (os == LINUX) and (channel == NIGHTLY):
return f"{WHL_INSTALL_BASE} {PACKAGES_TO_INSTALL_WHL} --index-url {get_base_download_url_for_repo('whl', channel, gpu_arch_type, desired_cuda)}_pypi_pkg"
else:
raise ValueError("Split build is not supported for this configuration. It is only supported for CUDA 11.8, 12.1, 12.4 on Linux nightly builds.")
if channel == RELEASE and (not use_only_dl_pytorch_org) and (
(gpu_arch_version == "12.1" and os == LINUX)
or (
Expand All @@ -315,6 +320,7 @@ def generate_conda_matrix(
with_cpu: str,
limit_pr_builds: bool,
use_only_dl_pytorch_org: bool,
use_split_build: bool = False,
) -> List[Dict[str, str]]:
ret: List[Dict[str, str]] = []
python_versions = list(PYTHON_ARCHES)
Expand Down Expand Up @@ -370,6 +376,7 @@ def generate_libtorch_matrix(
with_cpu: str,
limit_pr_builds: bool,
use_only_dl_pytorch_org: bool,
use_split_build: bool = False,
abi_versions: Optional[List[str]] = None,
arches: Optional[List[str]] = None,
libtorch_variants: Optional[List[str]] = None,
Expand Down Expand Up @@ -458,6 +465,7 @@ def generate_wheels_matrix(
with_cpu: str,
limit_pr_builds: bool,
use_only_dl_pytorch_org: bool,
use_split_build: bool = False,
arches: Optional[List[str]] = None,
python_versions: Optional[List[str]] = None,
) -> List[Dict[str, str]]:
Expand Down Expand Up @@ -511,8 +519,7 @@ def generate_wheels_matrix(
)

desired_cuda = translate_desired_cuda(gpu_arch_type, gpu_arch_version)
ret.append(
{
entry = {
"python_version": python_version,
"gpu_arch_type": gpu_arch_type,
"gpu_arch_version": gpu_arch_version,
Expand All @@ -535,8 +542,17 @@ def generate_wheels_matrix(
"channel": channel,
"upload_to_base_bucket": upload_to_base_bucket,
"stable_version": CURRENT_VERSION,
"use_split_build": False,
}
)
ret.append(entry)
if use_split_build:
entry = entry.copy()
entry["build_name"] = f"{package_type}-py{python_version}-{gpu_arch_type}{gpu_arch_version}-split".replace(
".", "_"
)
entry["use_split_build"] = True
ret.append(entry)

return ret


Expand All @@ -557,6 +573,7 @@ def generate_build_matrix(
limit_pr_builds: str,
use_only_dl_pytorch_org: str,
build_python_only: str,
use_split_build: str,
) -> Dict[str, List[Dict[str, str]]]:
includes = []

Expand All @@ -578,6 +595,7 @@ def generate_build_matrix(
with_cpu,
limit_pr_builds == "true",
use_only_dl_pytorch_org == "true",
use_split_build == "true",
)
)

Expand Down Expand Up @@ -657,6 +675,14 @@ def main(args: List[str]) -> None:
default=os.getenv("BUILD_PYTHON_ONLY", ENABLE),
)

parser.add_argument(
"--use-split-build",
help="Use split build for wheel",
type=str,
choices=["true", "false"],
default=os.getenv("USE_SPLIT_BUILD", DISABLE),
)

options = parser.parse_args(args)

assert (
Expand All @@ -673,6 +699,7 @@ def main(args: List[str]) -> None:
options.limit_pr_builds,
options.use_only_dl_pytorch_org,
options.build_python_only,
options.use_split_build,
)

print(json.dumps(build_matrix))
Expand Down

0 comments on commit 0da87a1

Please sign in to comment.