From 29fd875bc125582f29abbdf5559d3941899680be Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 12 Mar 2025 18:12:34 +0000 Subject: [PATCH] Automate stable CUDA update and linter using min Python verison (#148912) 1. Fixes: https://github.com/pytorch/pytorch/issues/145571 . Cuda Stable is the same cuda version that is published to pypi, also used to set Metadata section in the rest of whl scripts and tag the docker releases with latest tag. 2. Updates min python version used in linter Pull Request resolved: https://github.com/pytorch/pytorch/pull/148912 Approved by: https://github.com/Skylion007, https://github.com/malfet --- .../scripts/generate_binary_build_matrix.py | 5 ++-- .github/scripts/get_ci_variable.py | 30 +++++++++++++++++++ .github/workflows/docker-release.yml | 7 +++-- .github/workflows/lint.yml | 15 +++++++--- 4 files changed, 49 insertions(+), 8 deletions(-) create mode 100755 .github/scripts/get_ci_variable.py diff --git a/.github/scripts/generate_binary_build_matrix.py b/.github/scripts/generate_binary_build_matrix.py index 4e5ac3ac0fc..c7e5c13ba06 100644 --- a/.github/scripts/generate_binary_build_matrix.py +++ b/.github/scripts/generate_binary_build_matrix.py @@ -17,6 +17,7 @@ from typing import Optional # NOTE: Also update the CUDA sources in tools/nightly.py when changing this list CUDA_ARCHES = ["11.8", "12.6", "12.8"] +CUDA_STABLE = "12.6" CUDA_ARCHES_FULL_VERSION = { "11.8": "11.8.0", "12.6": "12.6.3", @@ -373,7 +374,7 @@ def generate_wheels_matrix( } ) # Special build building to use on Colab. Python 3.11 for 12.6 CUDA - if python_version == "3.11" and arch_version == "12.6": + if python_version == "3.11" and arch_version == CUDA_STABLE: ret.append( { "python_version": python_version, @@ -416,7 +417,7 @@ def generate_wheels_matrix( "pytorch_extra_install_requirements": ( PYTORCH_EXTRA_INSTALL_REQUIREMENTS["xpu"] if gpu_arch_type == "xpu" - else PYTORCH_EXTRA_INSTALL_REQUIREMENTS["12.6"] + else PYTORCH_EXTRA_INSTALL_REQUIREMENTS[CUDA_STABLE] if os != "linux" else "" ), diff --git a/.github/scripts/get_ci_variable.py b/.github/scripts/get_ci_variable.py new file mode 100755 index 00000000000..b2d5755bce6 --- /dev/null +++ b/.github/scripts/get_ci_variable.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Helper script - Return CI variables such as stable cuda, min python version, etc.""" + +import argparse +import sys + + +def main(args: list[str]) -> None: + import generate_binary_build_matrix + + parser = argparse.ArgumentParser() + parser.add_argument( + "--cuda-stable-version", + action="store_true", + help="get cuda stable version", + ) + parser.add_argument( + "--min-python-version", + action="store_true", + help="get min supported python version", + ) + options = parser.parse_args(args) + if options.cuda_stable_version: + return print(generate_binary_build_matrix.CUDA_STABLE) + if options.min_python_version: + return print(generate_binary_build_matrix.FULL_PYTHON_VERSIONS[0]) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index d92bb72535b..4334a94e8f8 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -117,7 +117,10 @@ jobs: # To get QEMU binaries in our PATH echo "${RUNNER_TEMP}/bin" >> "${GITHUB_PATH}" # Generate PyTorch version to use - echo "PYTORCH_VERSION=$(python3 .github/scripts/generate_pytorch_version.py --no-build-suffix)" >> "${GITHUB_ENV}" + { + echo "PYTORCH_VERSION=$(python3 .github/scripts/generate_pytorch_version.py --no-build-suffix)"; + echo "STABLE_CUDA_VERSION=$(python3 .github/scripts/get_ci_variable.py --stable-cuda-version)" + } >> "${GITHUB_ENV}" - name: Setup test specific variables if: ${{ startsWith(github.event.ref, 'refs/tags/v') }} run: | @@ -154,7 +157,7 @@ jobs: docker push ghcr.io/pytorch/pytorch-nightly:"${PYTORCH_NIGHTLY_COMMIT}${CUDA_SUFFIX}" # Please note, here we ned to pin specific verison of CUDA as with latest label - if [[ ${CUDA_VERSION_SHORT} == "12.4" ]]; then + if [[ ${CUDA_VERSION_SHORT} == "${STABLE_CUDA_VERSION}" ]]; then docker tag ghcr.io/pytorch/pytorch-nightly:"${PYTORCH_NIGHTLY_COMMIT}${CUDA_SUFFIX}" \ ghcr.io/pytorch/pytorch-nightly:latest docker push ghcr.io/pytorch/pytorch-nightly:latest diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 84372121096..f15056afb92 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -253,21 +253,28 @@ jobs: with: submodules: false fetch-depth: 1 - - name: Setup Python 3.6 + - name: Get min python version + id: get-min-python-version + if: matrix.test_type == 'older_python_version' + run: | + set -eou pipefail + # Generate PyTorch version to use + echo "MIN_PYTHON_VERSION=$(python3 .github/scripts/get_ci_variable.py --min-python-version)" >> "${GITHUB_OUTPUT}" + - name: Setup Old Python version if: matrix.test_type == 'older_python_version' uses: actions/setup-python@v4 with: - python-version: '3.6' + python-version: 3.6 architecture: x64 check-latest: false cache: pip cache-dependency-path: | **/requirements.txt - - name: Setup Python 3.9 + - name: Setup Min Python version if: matrix.test_type != 'older_python_version' uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: ${{ steps.get-min-python-version.outputs.MIN_PYTHON_VERSION }} architecture: x64 check-latest: false cache: pip