OSSForks/pytorch
1
0
Fork 0
mirror of https://github.com/zebrajr/pytorch.git synced 2025-12-06 12:20:52 +01:00
Code Issues Actions 38 Packages Projects Releases Wiki Activity

Use dedicated docker-build environment for manywheel, libtorch and conda Docker builds (#133699)

Browse Source 
BE change. Apply logic simiar to: https://github.com/pytorch/pytorch/blob/main/.github/workflows/docker-builds.yml

Pull Request resolved: https://github.com/pytorch/pytorch/pull/133699
Approved by: https://github.com/seemethere
This commit is contained in:
atalman 2024-08-16 18:10:41 +00:00 committed by PyTorch MergeBot
parent b833990a8f
commit caaa339e0f
3 changed files with 53 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/build-conda-images.yml vendored
View File

@ -23,8 +23,6 @@ on:
env: env:
DOCKER_REGISTRY: "docker.io" DOCKER_REGISTRY: "docker.io"
DOCKER_BUILDKIT: 1 DOCKER_BUILDKIT: 1
DOCKER_ID: ${{ secrets.DOCKER_ID }}
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }} WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }}
concurrency: concurrency:
@ -33,6 +31,7 @@ concurrency:
jobs: jobs:
build-docker: build-docker:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.9xlarge.ephemeral runs-on: linux.9xlarge.ephemeral
strategy: strategy:
matrix: matrix:
@ -54,6 +53,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin

14
.github/workflows/build-libtorch-images.yml vendored
View File

@ -22,8 +22,6 @@ on:
env: env:
DOCKER_REGISTRY: "docker.io" DOCKER_REGISTRY: "docker.io"
DOCKER_BUILDKIT: 1 DOCKER_BUILDKIT: 1
DOCKER_ID: ${{ secrets.DOCKER_ID }}
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }} WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }}
concurrency: concurrency:
@ -32,6 +30,7 @@ concurrency:
jobs: jobs:
build-docker-cuda: build-docker-cuda:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.9xlarge.ephemeral runs-on: linux.9xlarge.ephemeral
strategy: strategy:
matrix: matrix:
@ -54,6 +53,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin
@ -63,6 +65,7 @@ jobs:
run: | run: |
.ci/docker/libtorch/build.sh libtorch-cxx11-builder:cuda${{matrix.cuda_version}} .ci/docker/libtorch/build.sh libtorch-cxx11-builder:cuda${{matrix.cuda_version}}
build-docker-rocm: build-docker-rocm:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.9xlarge.ephemeral runs-on: linux.9xlarge.ephemeral
strategy: strategy:
matrix: matrix:
@ -85,6 +88,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin
@ -94,6 +100,7 @@ jobs:
run: | run: |
.ci/docker/libtorch/build.sh libtorch-cxx11-builder:rocm${{matrix.rocm_version}} .ci/docker/libtorch/build.sh libtorch-cxx11-builder:rocm${{matrix.rocm_version}}
build-docker-cpu: build-docker-cpu:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.9xlarge.ephemeral runs-on: linux.9xlarge.ephemeral
steps: steps:
- name: Checkout PyTorch - name: Checkout PyTorch
@ -110,6 +117,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin

39
.github/workflows/build-manywheel-images.yml vendored
View File

@ -24,8 +24,6 @@ on:
env: env:
DOCKER_REGISTRY: "docker.io" DOCKER_REGISTRY: "docker.io"
DOCKER_BUILDKIT: 1 DOCKER_BUILDKIT: 1
DOCKER_ID: ${{ secrets.DOCKER_ID }}
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }} WITH_PUSH: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release')) }}
concurrency: concurrency:
@ -34,6 +32,7 @@ concurrency:
jobs: jobs:
build-docker-cuda: build-docker-cuda:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.9xlarge.ephemeral runs-on: linux.9xlarge.ephemeral
strategy: strategy:
matrix: matrix:
@ -58,6 +57,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin
@ -68,6 +70,7 @@ jobs:
.ci/docker/manywheel/build.sh manylinux-builder:cuda${{matrix.cuda_version}} .ci/docker/manywheel/build.sh manylinux-builder:cuda${{matrix.cuda_version}}
# NOTE: manylinux_2_28 are still experimental, see https://github.com/pytorch/pytorch/issues/123649 # NOTE: manylinux_2_28 are still experimental, see https://github.com/pytorch/pytorch/issues/123649
build-docker-cuda-manylinux_2_28: build-docker-cuda-manylinux_2_28:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.9xlarge.ephemeral runs-on: linux.9xlarge.ephemeral
strategy: strategy:
matrix: matrix:
@ -92,6 +95,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin
@ -101,6 +107,7 @@ jobs:
run: | run: |
.ci/docker/manywheel/build.sh manylinux2_28-builder:cuda${{matrix.cuda_version}} .ci/docker/manywheel/build.sh manylinux2_28-builder:cuda${{matrix.cuda_version}}
build-docker-cuda-aarch64: build-docker-cuda-aarch64:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.arm64.2xlarge runs-on: linux.arm64.2xlarge
strategy: strategy:
matrix: matrix:
@ -121,6 +128,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin
@ -130,6 +140,7 @@ jobs:
run: | run: |
.ci/docker/manywheel/build.sh manylinuxaarch64-builder:cuda${{matrix.cuda_version}} .ci/docker/manywheel/build.sh manylinuxaarch64-builder:cuda${{matrix.cuda_version}}
build-docker-rocm: build-docker-rocm:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.9xlarge.ephemeral runs-on: linux.9xlarge.ephemeral
strategy: strategy:
matrix: matrix:
@ -152,6 +163,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin
@ -161,6 +175,7 @@ jobs:
run: | run: |
.ci/docker/manywheel/build.sh manylinux-builder:rocm${{matrix.rocm_version}} .ci/docker/manywheel/build.sh manylinux-builder:rocm${{matrix.rocm_version}}
build-docker-cpu: build-docker-cpu:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.9xlarge.ephemeral runs-on: linux.9xlarge.ephemeral
steps: steps:
- name: Checkout PyTorch - name: Checkout PyTorch
@ -177,6 +192,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin
@ -186,6 +204,7 @@ jobs:
run: | run: |
.ci/docker/manywheel/build.sh manylinux-builder:cpu .ci/docker/manywheel/build.sh manylinux-builder:cpu
build-docker-cpu-manylinux_2_28: build-docker-cpu-manylinux_2_28:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.9xlarge.ephemeral runs-on: linux.9xlarge.ephemeral
env: env:
GPU_ARCH_TYPE: cpu-manylinux_2_28 GPU_ARCH_TYPE: cpu-manylinux_2_28
@ -213,6 +232,7 @@ jobs:
run: | run: |
.ci/docker/manywheel/build.sh manylinux2_28-builder:cpu .ci/docker/manywheel/build.sh manylinux2_28-builder:cpu
build-docker-cpu-aarch64: build-docker-cpu-aarch64:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.arm64.2xlarge runs-on: linux.arm64.2xlarge
env: env:
GPU_ARCH_TYPE: cpu-aarch64 GPU_ARCH_TYPE: cpu-aarch64
@ -231,6 +251,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin
@ -240,6 +263,7 @@ jobs:
run: | run: |
.ci/docker/manywheel/build.sh manylinuxaarch64-builder:cpu-aarch64 .ci/docker/manywheel/build.sh manylinuxaarch64-builder:cpu-aarch64
build-docker-cpu-aarch64-2_28: build-docker-cpu-aarch64-2_28:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.arm64.2xlarge runs-on: linux.arm64.2xlarge
env: env:
GPU_ARCH_TYPE: cpu-aarch64-2_28 GPU_ARCH_TYPE: cpu-aarch64-2_28
@ -264,9 +288,13 @@ jobs:
fi fi
- name: Build Docker Image - name: Build Docker Image
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
.ci/docker/manywheel/build.sh manylinux2_28_aarch64-builder:cpu-aarch64 .ci/docker/manywheel/build.sh manylinux2_28_aarch64-builder:cpu-aarch64
build-docker-cpu-cxx11-abi: build-docker-cpu-cxx11-abi:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.9xlarge.ephemeral runs-on: linux.9xlarge.ephemeral
env: env:
GPU_ARCH_TYPE: cpu-cxx11-abi GPU_ARCH_TYPE: cpu-cxx11-abi
@ -285,6 +313,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin
@ -294,6 +325,7 @@ jobs:
run: | run: |
.ci/docker/manywheel/build.sh manylinuxcxx11-abi-builder:cpu-cxx11-abi .ci/docker/manywheel/build.sh manylinuxcxx11-abi-builder:cpu-cxx11-abi
build-docker-xpu: build-docker-xpu:
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }}
runs-on: linux.9xlarge.ephemeral runs-on: linux.9xlarge.ephemeral
env: env:
GPU_ARCH_TYPE: xpu GPU_ARCH_TYPE: xpu
@ -312,6 +344,9 @@ jobs:
push: true push: true
- name: Authenticate if WITH_PUSH - name: Authenticate if WITH_PUSH
if: env.WITH_PUSH == 'true' if: env.WITH_PUSH == 'true'
env:
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_ID: ${{ secrets.DOCKER_ID }}
run: | run: |
if [[ "${WITH_PUSH}" == true ]]; then if [[ "${WITH_PUSH}" == true ]]; then
echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin echo "${DOCKER_TOKEN}" | docker login -u "${DOCKER_ID}" --password-stdin