From 6974ba84f6efa10f58fb86e766feee7492f9137a Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 19 Mar 2025 21:56:44 +0000 Subject: [PATCH] [ci][anaconda] Remove conda from linter docker images (#147789) Remove conda usage from the linter docker images Handles part of https://github.com/pytorch/pytorch/issues/148110 Pull Request resolved: https://github.com/pytorch/pytorch/pull/147789 Approved by: https://github.com/atalman --- .ci/docker/build.sh | 10 ++++++---- .ci/docker/common/install_linter.sh | 6 ++---- .ci/docker/common/install_python.sh | 18 ++++++++++++++++++ .ci/docker/linter-cuda/Dockerfile | 22 +++++++++++----------- .ci/docker/linter/Dockerfile | 18 ++++++++---------- .github/scripts/lintrunner.sh | 5 ----- .github/workflows/lint.yml | 16 ---------------- 7 files changed, 45 insertions(+), 50 deletions(-) create mode 100644 .ci/docker/common/install_python.sh diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index e2b4e672618..a7d052b2dc1 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -382,13 +382,13 @@ case "$image" in # TODO: Use 3.9 here because of this issue https://github.com/python/mypy/issues/13627. # We will need to update mypy version eventually, but that's for another day. The task # would be to upgrade mypy to 1.0.0 with Python 3.11 - ANACONDA_PYTHON_VERSION=3.9 - CONDA_CMAKE=yes + PYTHON_VERSION=3.9 + PIP_CMAKE=yes ;; pytorch-linux-jammy-cuda11.8-cudnn9-py3.9-linter) - ANACONDA_PYTHON_VERSION=3.9 + PYTHON_VERSION=3.9 CUDA_VERSION=11.8 - CONDA_CMAKE=yes + PIP_CMAKE=yes ;; pytorch-linux-jammy-aarch64-py3.10-gcc11) ANACONDA_PYTHON_VERSION=3.10 @@ -478,6 +478,7 @@ docker build \ --build-arg "GLIBC_VERSION=${GLIBC_VERSION}" \ --build-arg "CLANG_VERSION=${CLANG_VERSION}" \ --build-arg "ANACONDA_PYTHON_VERSION=${ANACONDA_PYTHON_VERSION}" \ + --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \ --build-arg "GCC_VERSION=${GCC_VERSION}" \ --build-arg "CUDA_VERSION=${CUDA_VERSION}" \ --build-arg "CUDNN_VERSION=${CUDNN_VERSION}" \ @@ -494,6 +495,7 @@ docker build \ --build-arg "UCX_COMMIT=${UCX_COMMIT}" \ --build-arg "UCC_COMMIT=${UCC_COMMIT}" \ --build-arg "CONDA_CMAKE=${CONDA_CMAKE}" \ + --build-arg "PIP_CMAKE=${PIP_CMAKE}" \ --build-arg "TRITON=${TRITON}" \ --build-arg "TRITON_CPU=${TRITON_CPU}" \ --build-arg "ONNX=${ONNX}" \ diff --git a/.ci/docker/common/install_linter.sh b/.ci/docker/common/install_linter.sh index a7f008fb735..d080bb40379 100644 --- a/.ci/docker/common/install_linter.sh +++ b/.ci/docker/common/install_linter.sh @@ -2,8 +2,6 @@ set -ex -source "$(dirname "${BASH_SOURCE[0]}")/common_utils.sh" - if [ -n "${UBUNTU_VERSION}" ]; then apt update apt-get install -y clang doxygen git graphviz nodejs npm libtinfo5 @@ -15,8 +13,8 @@ chown -R jenkins pytorch pushd pytorch # Install all linter dependencies -pip_install -r requirements.txt -conda_run lintrunner init +pip install -r requirements.txt +lintrunner init # Cache .lintbin directory as part of the Docker image cp -r .lintbin /tmp diff --git a/.ci/docker/common/install_python.sh b/.ci/docker/common/install_python.sh new file mode 100644 index 00000000000..7f92deae9b0 --- /dev/null +++ b/.ci/docker/common/install_python.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -ex + +apt-get update +# Use deadsnakes in case we need an older python version +sudo add-apt-repository ppa:deadsnakes/ppa +apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python3-pip python${PYTHON_VERSION}-venv + +# Use a venv because uv and some other package managers don't support --user install +ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python +python -m venv /var/lib/jenkins/ci_env +source /var/lib/jenkins/ci_env/bin/activate + +python -mpip install --upgrade pip +python -mpip install -r /opt/requirements-ci.txt +if [ -n "${PIP_CMAKE}" ]; then + python -mpip install cmake==3.31.6 +fi diff --git a/.ci/docker/linter-cuda/Dockerfile b/.ci/docker/linter-cuda/Dockerfile index 8084bf62712..d93f69a149f 100644 --- a/.ci/docker/linter-cuda/Dockerfile +++ b/.ci/docker/linter-cuda/Dockerfile @@ -18,15 +18,14 @@ COPY ./common/install_user.sh install_user.sh RUN bash ./install_user.sh && rm install_user.sh # Install conda and other packages (e.g., numpy, pytest) -ARG ANACONDA_PYTHON_VERSION -ARG CONDA_CMAKE -ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION -ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH -COPY requirements-ci.txt /opt/conda/requirements-ci.txt -COPY ./common/install_conda.sh install_conda.sh -COPY ./common/common_utils.sh common_utils.sh -COPY ./common/install_magma_conda.sh install_magma_conda.sh -RUN bash ./install_conda.sh && rm install_conda.sh install_magma_conda.sh common_utils.sh /opt/conda/requirements-ci.txt +ARG PYTHON_VERSION +ARG PIP_CMAKE +# Put venv into the env vars so users don't need to activate it +ENV PATH /var/lib/jenkins/ci_env/bin:$PATH +ENV VIRTUAL_ENV /var/lib/jenkins/ci_env +COPY requirements-ci.txt /opt/requirements-ci.txt +COPY ./common/install_python.sh install_python.sh +RUN bash ./install_python.sh && rm install_python.sh /opt/requirements-ci.txt # Install cuda and cudnn ARG CUDA_VERSION @@ -37,9 +36,10 @@ ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:$PATH # Note that Docker build forbids copying file outside the build context COPY ./common/install_linter.sh install_linter.sh -COPY ./common/common_utils.sh common_utils.sh RUN bash ./install_linter.sh -RUN rm install_linter.sh common_utils.sh +RUN rm install_linter.sh + +RUN chown -R jenkins:jenkins /var/lib/jenkins/ci_env USER jenkins CMD ["bash"] diff --git a/.ci/docker/linter/Dockerfile b/.ci/docker/linter/Dockerfile index 968918a3617..d5bac900361 100644 --- a/.ci/docker/linter/Dockerfile +++ b/.ci/docker/linter/Dockerfile @@ -15,20 +15,18 @@ COPY ./common/install_user.sh install_user.sh RUN bash ./install_user.sh && rm install_user.sh # Install conda and other packages (e.g., numpy, pytest) -ARG ANACONDA_PYTHON_VERSION -ARG CONDA_CMAKE -ENV ANACONDA_PYTHON_VERSION=$ANACONDA_PYTHON_VERSION -ENV PATH /opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:/opt/conda/bin:$PATH -COPY requirements-ci.txt /opt/conda/requirements-ci.txt -COPY ./common/install_conda.sh install_conda.sh -COPY ./common/common_utils.sh common_utils.sh -RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt +ARG PYTHON_VERSION +ARG PIP_CMAKE +ENV PATH /var/lib/jenkins/ci_env/bin:$PATH +ENV VIRTUAL_ENV /var/lib/jenkins/ci_env +COPY requirements-ci.txt /opt/requirements-ci.txt +COPY ./common/install_python.sh install_python.sh +RUN bash ./install_python.sh && rm install_python.sh /opt/requirements-ci.txt # Note that Docker build forbids copying file outside the build context COPY ./common/install_linter.sh install_linter.sh -COPY ./common/common_utils.sh common_utils.sh RUN bash ./install_linter.sh -RUN rm install_linter.sh common_utils.sh +RUN rm install_linter.sh USER jenkins CMD ["bash"] diff --git a/.github/scripts/lintrunner.sh b/.github/scripts/lintrunner.sh index a3d78d116b3..1ae27f062d2 100755 --- a/.github/scripts/lintrunner.sh +++ b/.github/scripts/lintrunner.sh @@ -1,11 +1,6 @@ #!/usr/bin/env bash set -ex -# The generic Linux job chooses to use base env, not the one setup by the image -CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") -eval "$(command conda 'shell.bash' 'hook' 2> /dev/null)" -conda activate "${CONDA_ENV}" - # Use uv to speed up lintrunner init python3 -m pip install uv==0.1.45 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 65625058156..7989be53147 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -68,10 +68,6 @@ jobs: fetch-depth: 0 ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" - # Ensure no non-breaking spaces # NB: We use 'printf' below rather than '\u000a' since bash pre-4.2 # does not support the '\u000a' syntax (which is relevant for local linters) @@ -126,10 +122,6 @@ jobs: submodules: true ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" - # Regenerate workflows .github/scripts/generate_ci_workflows.py @@ -163,10 +155,6 @@ jobs: fetch-depth: 0 ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" - # Regenerate ToCs and check that they didn't change set -eu @@ -203,10 +191,6 @@ jobs: fetch-depth: 0 ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} script: | - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" - # Test tools PYTHONPATH=$(pwd) pytest tools/stats PYTHONPATH=$(pwd) pytest tools/test -o "python_files=test*.py"