diff --git a/.circleci/cimodel/lib/miniyaml.py b/.circleci/cimodel/lib/miniyaml.py index 713864f6362..a4bbe19ab3b 100644 --- a/.circleci/cimodel/lib/miniyaml.py +++ b/.circleci/cimodel/lib/miniyaml.py @@ -44,7 +44,7 @@ def render(fh, data, depth, is_list_member=False): render(fh, v, depth, True) else: - # use empty quotes to denote an empty sting value instead of blank space + # use empty quotes to denote an empty string value instead of blank space modified_data = miniutils.quote(data) if data == '' else data list_member_prefix = indentation + LIST_MARKER if is_list_member else "" diff --git a/.circleci/codegen_validation/compare_normalized_yaml.sh b/.circleci/codegen_validation/compare_normalized_yaml.sh index 94cf8e03d21..af2a8268d3c 100755 --- a/.circleci/codegen_validation/compare_normalized_yaml.sh +++ b/.circleci/codegen_validation/compare_normalized_yaml.sh @@ -1,7 +1,8 @@ #!/bin/bash -xe -YAML_FILENAME=verbatim-sources/windows-build-test.yml +YAML_FILENAME=verbatim-sources/workflows-pytorch-ge-config-tests.yml +DIFF_TOOL=meld # Allows this script to be invoked from any directory: @@ -10,7 +11,7 @@ cd $(dirname "$0") pushd .. -meld $YAML_FILENAME <(./normalize_yaml_fragment.py < $YAML_FILENAME) +$DIFF_TOOL $YAML_FILENAME <(./codegen_validation/normalize_yaml_fragment.py < $YAML_FILENAME) -popd \ No newline at end of file +popd diff --git a/.circleci/normalize_yaml_fragment.py b/.circleci/codegen_validation/normalize_yaml_fragment.py similarity index 76% rename from .circleci/normalize_yaml_fragment.py rename to .circleci/codegen_validation/normalize_yaml_fragment.py index 3d4d5ebacbb..0778404e8bb 100755 --- a/.circleci/normalize_yaml_fragment.py +++ b/.circleci/codegen_validation/normalize_yaml_fragment.py @@ -1,7 +1,12 @@ #!/usr/bin/env python3 +import os import sys import yaml + +# Need to import modules that lie on an upward-relative path +sys.path.append(os.path.join(sys.path[0], '..')) + import cimodel.lib.miniyaml as miniyaml diff --git a/.circleci/codegen_validation/overwrite_with_normalized.sh b/.circleci/codegen_validation/overwrite_with_normalized.sh new file mode 100755 index 00000000000..7665984cedb --- /dev/null +++ b/.circleci/codegen_validation/overwrite_with_normalized.sh @@ -0,0 +1,15 @@ +#!/bin/bash -xe + +YAML_FILENAME=$1 + +# Allows this script to be invoked from any directory: +cd $(dirname "$0") + +pushd .. + +TEMP_FILENAME=$(mktemp) + +cat $YAML_FILENAME | ./codegen_validation/normalize_yaml_fragment.py > $TEMP_FILENAME +mv $TEMP_FILENAME $YAML_FILENAME + +popd diff --git a/.circleci/config.yml b/.circleci/config.yml index c6c45cb4543..71571e26bec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2780,93 +2780,90 @@ workflows: - setup - pytorch_macos_10_13_py3_build - pytorch_android_gradle_build-x86_32: - name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-x86_32 filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ + name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-x86_32 requires: - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build - - pytorch_android_gradle_build: - name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ + name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build requires: - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build - # Pytorch iOS PR builds - pytorch_ios_build: + build_environment: pytorch-ios-11.2.1-x86_64_build + ios_arch: x86_64 + ios_platform: SIMULATOR name: pytorch_ios_11_2_1_x86_64_build - build_environment: "pytorch-ios-11.2.1-x86_64_build" - ios_arch: "x86_64" - ios_platform: "SIMULATOR" requires: - setup - pytorch_ios_build: + build_environment: pytorch-ios-11.2.1-arm64_build + context: org-member + ios_arch: arm64 + ios_platform: OS name: pytorch_ios_11_2_1_arm64_build - context: org-member - build_environment: "pytorch-ios-11.2.1-arm64_build" - ios_arch: "arm64" - ios_platform: "OS" requires: - setup - pytorch_ios_build: - name: pytorch_ios_11_2_1_arm64_custom_build + build_environment: pytorch-ios-11.2.1-arm64_custom_build context: org-member - build_environment: "pytorch-ios-11.2.1-arm64_custom_build" - ios_arch: "arm64" - ios_platform: "OS" - op_list: "mobilenetv2.yaml" + ios_arch: arm64 + ios_platform: OS + name: pytorch_ios_11_2_1_arm64_custom_build + op_list: mobilenetv2.yaml requires: - setup - # PyTorch Mobile PR builds (use linux host toolchain + mobile build options) - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-mobile-build + build_only: "1" + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 name: pytorch_linux_xenial_py3_clang5_mobile_build requires: - setup - build_environment: "pytorch-linux-xenial-py3-clang5-mobile-build" - build_only: "1" - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-mobile-custom-build-static + build_only: "1" + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 name: pytorch_linux_xenial_py3_clang5_mobile_custom_build_static requires: - setup - build_environment: "pytorch-linux-xenial-py3-clang5-mobile-custom-build-static" - build_only: "1" - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-mobile-custom-build-dynamic + build_only: "1" + # Use LLVM-DEV toolchain in android-ndk-r19c docker image + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 name: pytorch_linux_xenial_py3_clang5_mobile_custom_build_dynamic requires: - setup - build_environment: "pytorch-linux-xenial-py3-clang5-mobile-custom-build-dynamic" - build_only: "1" - # Use LLVM-DEV toolchain in android-ndk-r19c docker image - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" - pytorch_linux_build: - name: pytorch_linux_xenial_py3_clang5_mobile_code_analysis - requires: - - setup + # Use LLVM-DEV toolchain in android-ndk-r19c docker image # Most of this CI is already covered by "mobile-custom-build-dynamic" job + build_environment: pytorch-linux-xenial-py3-clang5-mobile-code-analysis + build_only: "1" + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - build_environment: "pytorch-linux-xenial-py3-clang5-mobile-code-analysis" - build_only: "1" - # Use LLVM-DEV toolchain in android-ndk-r19c docker image - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" + name: pytorch_linux_xenial_py3_clang5_mobile_code_analysis + requires: + - setup - pytorch_linux_test: name: pytorch_linux_xenial_py3_6_gcc5_4_ge_config_legacy_test requires: @@ -2910,19 +2907,19 @@ workflows: use_cuda_docker_runtime: "1" resource_class: gpu.medium - pytorch_linux_bazel_build: + build_environment: pytorch-linux-xenial-py3.6-gcc7-bazel-build + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026 name: pytorch_bazel_build requires: - setup - build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-build" - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026" resource_class: large - pytorch_linux_bazel_test: + build_environment: pytorch-linux-xenial-py3.6-gcc7-bazel-test + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026 name: pytorch_bazel_test requires: - setup - pytorch_bazel_build - build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-test" - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026" - caffe2_linux_build: name: caffe2_onnx_main_py3_6_clang7_ubuntu16_04_build requires: @@ -2961,116 +2958,117 @@ workflows: # the binary_and_smoke_tests_on_pr variable in # pytorch-ci-hud to adjust the list of whitelisted builds # at https://github.com/ezyang/pytorch-ci-hud/blob/master/src/BuildHistoryDisplay.js - - - binary_linux_build: - name: binary_linux_manywheel_3_7m_cu102_devtoolset7_build - build_environment: "manywheel 3.7m cu102 devtoolset7" - requires: - - setup - docker_image: "pytorch/manylinux-cuda102" - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ + # + # Note: # This binary build is currently broken, see https://github_com/pytorch/pytorch/issues/16710 # - binary_linux_conda_3_6_cu90_devtoolset7_build - - binary_linux_build: - name: binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build - build_environment: "libtorch 3.7m cpu devtoolset7" - requires: - - setup - libtorch_variant: "shared-with-deps" - docker_image: "pytorch/manylinux-cuda102" - - binary_linux_build: - name: binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build - build_environment: "libtorch 3.7m cpu gcc5.4_cxx11-abi" - requires: - - setup - libtorch_variant: "shared-with-deps" - docker_image: "pytorch/pytorch-binary-docker-image-ubuntu16.04:latest" - # TODO we should test a libtorch cuda build, but they take too long + # - binary_linux_conda_3_6_cu90_devtoolset7_test + # + # TODO + # we should test a libtorch cuda build, but they take too long # - binary_linux_libtorch_3_6m_cu90_devtoolset7_static-without-deps_build + - binary_linux_build: + build_environment: manywheel 3.7m cu102 devtoolset7 + docker_image: pytorch/manylinux-cuda102 + filters: + branches: + only: + - master + - /ci-all\/.*/ + - /release\/.*/ + name: binary_linux_manywheel_3_7m_cu102_devtoolset7_build + requires: + - setup + - binary_linux_build: + build_environment: libtorch 3.7m cpu devtoolset7 + docker_image: pytorch/manylinux-cuda102 + libtorch_variant: shared-with-deps + name: binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build + requires: + - setup + - binary_linux_build: + build_environment: libtorch 3.7m cpu gcc5.4_cxx11-abi + docker_image: pytorch/pytorch-binary-docker-image-ubuntu16.04:latest + libtorch_variant: shared-with-deps + name: binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build + requires: + - setup - binary_mac_build: + build_environment: wheel 3.7 cpu + filters: + branches: + only: + - master + - /ci-all\/.*/ + - /release\/.*/ name: binary_macos_wheel_3_7_cpu_build - build_environment: "wheel 3.7 cpu" requires: - setup - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - # This job has an average run time of 3 hours o.O - # Now only running this on master to reduce overhead - binary_mac_build: + # This job has an average run time of 3 hours o.O + # Now only running this on master to reduce overhead + build_environment: libtorch 3.7 cpu + filters: + branches: + only: + - master + - /ci-all\/.*/ + - /release\/.*/ name: binary_macos_libtorch_3_7_cpu_build - build_environment: "libtorch 3.7 cpu" requires: - setup - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - binary_windows_build: + build_environment: libtorch 3.7 cpu debug name: binary_windows_libtorch_3_7_cpu_debug_build - build_environment: "libtorch 3.7 cpu debug" requires: - setup - binary_windows_build: + build_environment: libtorch 3.7 cpu release name: binary_windows_libtorch_3_7_cpu_release_build - build_environment: "libtorch 3.7 cpu release" requires: - setup - binary_windows_build: - name: binary_windows_wheel_3_7_cu102_build - build_environment: "wheel 3.7 cu102" - requires: - - setup + build_environment: wheel 3.7 cu102 filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ + name: binary_windows_wheel_3_7_cu102_build + requires: + - setup - binary_linux_test: + build_environment: manywheel 3.7m cu102 devtoolset7 + docker_image: pytorch/manylinux-cuda102 + filters: + branches: + only: + - master + - /ci-all\/.*/ + - /release\/.*/ name: binary_linux_manywheel_3_7m_cu102_devtoolset7_test - build_environment: "manywheel 3.7m cu102 devtoolset7" requires: - setup - binary_linux_manywheel_3_7m_cu102_devtoolset7_build - docker_image: "pytorch/manylinux-cuda102" - use_cuda_docker_runtime: "1" resource_class: gpu.medium - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - # This binary build is currently broken, see https://github_com/pytorch/pytorch/issues/16710 - # - binary_linux_conda_3_6_cu90_devtoolset7_test: + use_cuda_docker_runtime: "1" - binary_linux_test: + build_environment: libtorch 3.7m cpu devtoolset7 + docker_image: pytorch/manylinux-cuda102 + libtorch_variant: shared-with-deps name: binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_test - build_environment: "libtorch 3.7m cpu devtoolset7" requires: - setup - binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build - libtorch_variant: "shared-with-deps" - docker_image: "pytorch/manylinux-cuda102" - binary_linux_test: + build_environment: libtorch 3.7m cpu gcc5.4_cxx11-abi + docker_image: pytorch/pytorch-binary-docker-image-ubuntu16.04:latest + libtorch_variant: shared-with-deps name: binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_test - build_environment: "libtorch 3.7m cpu gcc5.4_cxx11-abi" requires: - setup - binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build - libtorch_variant: "shared-with-deps" - docker_image: "pytorch/pytorch-binary-docker-image-ubuntu16.04:latest" - - smoke_linux_test: name: smoke_linux_manywheel_3_6m_cpu_devtoolset7_nightly build_environment: "manywheel 3.6m cpu devtoolset7" @@ -4911,95 +4909,92 @@ workflows: only: nightly tags: only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - # Pytorch iOS binary builds - binary_ios_build: + build_environment: libtorch-ios-11.2.1-nightly-x86_64-build + context: org-member + filters: + branches: + only: nightly + ios_arch: x86_64 + ios_platform: SIMULATOR name: pytorch_ios_11_2_1_nightly_x86_64_build - build_environment: "libtorch-ios-11.2.1-nightly-x86_64-build" - context: org-member - ios_platform: "SIMULATOR" - ios_arch: "x86_64" - requires: + requires: - setup - filters: - branches: - only: nightly - binary_ios_build: - name: pytorch_ios_11_2_1_nightly_arm64_build - build_environment: "libtorch-ios-11.2.1-nightly-arm64-build" + build_environment: libtorch-ios-11.2.1-nightly-arm64-build context: org-member - ios_arch: "arm64" - ios_platform: "OS" - requires: - - setup filters: branches: only: nightly + ios_arch: arm64 + ios_platform: OS + name: pytorch_ios_11_2_1_nightly_arm64_build + requires: + - setup - binary_ios_upload: - build_environment: "libtorch-ios-11.2.1-nightly-binary-build-upload" + build_environment: libtorch-ios-11.2.1-nightly-binary-build-upload context: org-member + filters: + branches: + only: nightly requires: - setup - pytorch_ios_11_2_1_nightly_x86_64_build - pytorch_ios_11_2_1_nightly_arm64_build + - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32 + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 filters: branches: only: nightly - - pytorch_linux_build: name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build - build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32" requires: - setup - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" + - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64 + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 filters: branches: only: nightly - - pytorch_linux_build: name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build - build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64" requires: - setup - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" + - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 filters: branches: only: nightly - - pytorch_linux_build: name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build - build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a" requires: - setup - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" - filters: - branches: - only: nightly - pytorch_linux_build: - name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build - build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a" - requires: - - setup - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" + build_environment: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 filters: branches: only: nightly - + name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build + requires: + - setup - pytorch_android_gradle_build: + filters: + branches: + only: nightly name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_android_gradle_build requires: - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build - filters: - branches: - only: nightly - - pytorch_android_publish_snapshot: - name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_android_publish_snapshot - requires: - - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_android_gradle_build context: org-member filters: branches: only: nightly + name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_android_publish_snapshot + requires: + - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_android_gradle_build ############################################################################## # Nightly tests ############################################################################## @@ -7086,42 +7081,42 @@ workflows: # CircleCI security context - promote_approval: context: org-member - type: approval filters: + branches: + ignore: /.*/ tags: only: /v[0-9]+(\.[0-9]+)*/ - branches: - ignore: /.*/ + type: approval - promote_s3: context: org-member + filters: + branches: + ignore: /.*/ + tags: + only: /v[0-9]+(\.[0-9]+)*/ name: promote_s3_libtorch package_name: libtorch requires: - promote_approval - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - branches: - ignore: /.*/ - promote_s3: context: org-member + filters: + branches: + ignore: /.*/ + tags: + only: /v[0-9]+(\.[0-9]+)*/ name: promote_s3_torch package_name: torch requires: - promote_approval - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - branches: - ignore: /.*/ - promote_conda: context: org-member + filters: + branches: + ignore: /.*/ + tags: + only: /v[0-9]+(\.[0-9]+)*/ name: promote_conda_pytorch package_name: pytorch requires: - promote_approval - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - branches: - ignore: /.*/ diff --git a/.circleci/verbatim-sources/workflows-binary-builds-smoke-subset.yml b/.circleci/verbatim-sources/workflows-binary-builds-smoke-subset.yml index 4ec23b304b8..24e9f3d4c97 100644 --- a/.circleci/verbatim-sources/workflows-binary-builds-smoke-subset.yml +++ b/.circleci/verbatim-sources/workflows-binary-builds-smoke-subset.yml @@ -6,113 +6,114 @@ # the binary_and_smoke_tests_on_pr variable in # pytorch-ci-hud to adjust the list of whitelisted builds # at https://github.com/ezyang/pytorch-ci-hud/blob/master/src/BuildHistoryDisplay.js - - - binary_linux_build: - name: binary_linux_manywheel_3_7m_cu102_devtoolset7_build - build_environment: "manywheel 3.7m cu102 devtoolset7" - requires: - - setup - docker_image: "pytorch/manylinux-cuda102" - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ + # + # Note: # This binary build is currently broken, see https://github_com/pytorch/pytorch/issues/16710 # - binary_linux_conda_3_6_cu90_devtoolset7_build - - binary_linux_build: - name: binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build - build_environment: "libtorch 3.7m cpu devtoolset7" - requires: - - setup - libtorch_variant: "shared-with-deps" - docker_image: "pytorch/manylinux-cuda102" - - binary_linux_build: - name: binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build - build_environment: "libtorch 3.7m cpu gcc5.4_cxx11-abi" - requires: - - setup - libtorch_variant: "shared-with-deps" - docker_image: "pytorch/pytorch-binary-docker-image-ubuntu16.04:latest" - # TODO we should test a libtorch cuda build, but they take too long + # - binary_linux_conda_3_6_cu90_devtoolset7_test + # + # TODO + # we should test a libtorch cuda build, but they take too long # - binary_linux_libtorch_3_6m_cu90_devtoolset7_static-without-deps_build + - binary_linux_build: + build_environment: manywheel 3.7m cu102 devtoolset7 + docker_image: pytorch/manylinux-cuda102 + filters: + branches: + only: + - master + - /ci-all\/.*/ + - /release\/.*/ + name: binary_linux_manywheel_3_7m_cu102_devtoolset7_build + requires: + - setup + - binary_linux_build: + build_environment: libtorch 3.7m cpu devtoolset7 + docker_image: pytorch/manylinux-cuda102 + libtorch_variant: shared-with-deps + name: binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build + requires: + - setup + - binary_linux_build: + build_environment: libtorch 3.7m cpu gcc5.4_cxx11-abi + docker_image: pytorch/pytorch-binary-docker-image-ubuntu16.04:latest + libtorch_variant: shared-with-deps + name: binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build + requires: + - setup - binary_mac_build: + build_environment: wheel 3.7 cpu + filters: + branches: + only: + - master + - /ci-all\/.*/ + - /release\/.*/ name: binary_macos_wheel_3_7_cpu_build - build_environment: "wheel 3.7 cpu" requires: - setup - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - # This job has an average run time of 3 hours o.O - # Now only running this on master to reduce overhead - binary_mac_build: + # This job has an average run time of 3 hours o.O + # Now only running this on master to reduce overhead + build_environment: libtorch 3.7 cpu + filters: + branches: + only: + - master + - /ci-all\/.*/ + - /release\/.*/ name: binary_macos_libtorch_3_7_cpu_build - build_environment: "libtorch 3.7 cpu" requires: - setup - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - binary_windows_build: + build_environment: libtorch 3.7 cpu debug name: binary_windows_libtorch_3_7_cpu_debug_build - build_environment: "libtorch 3.7 cpu debug" requires: - setup - binary_windows_build: + build_environment: libtorch 3.7 cpu release name: binary_windows_libtorch_3_7_cpu_release_build - build_environment: "libtorch 3.7 cpu release" requires: - setup - binary_windows_build: - name: binary_windows_wheel_3_7_cu102_build - build_environment: "wheel 3.7 cu102" - requires: - - setup + build_environment: wheel 3.7 cu102 filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ + name: binary_windows_wheel_3_7_cu102_build + requires: + - setup - binary_linux_test: + build_environment: manywheel 3.7m cu102 devtoolset7 + docker_image: pytorch/manylinux-cuda102 + filters: + branches: + only: + - master + - /ci-all\/.*/ + - /release\/.*/ name: binary_linux_manywheel_3_7m_cu102_devtoolset7_test - build_environment: "manywheel 3.7m cu102 devtoolset7" requires: - setup - binary_linux_manywheel_3_7m_cu102_devtoolset7_build - docker_image: "pytorch/manylinux-cuda102" - use_cuda_docker_runtime: "1" resource_class: gpu.medium - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - # This binary build is currently broken, see https://github_com/pytorch/pytorch/issues/16710 - # - binary_linux_conda_3_6_cu90_devtoolset7_test: + use_cuda_docker_runtime: "1" - binary_linux_test: + build_environment: libtorch 3.7m cpu devtoolset7 + docker_image: pytorch/manylinux-cuda102 + libtorch_variant: shared-with-deps name: binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_test - build_environment: "libtorch 3.7m cpu devtoolset7" requires: - setup - binary_linux_libtorch_3_7m_cpu_devtoolset7_shared-with-deps_build - libtorch_variant: "shared-with-deps" - docker_image: "pytorch/manylinux-cuda102" - binary_linux_test: + build_environment: libtorch 3.7m cpu gcc5.4_cxx11-abi + docker_image: pytorch/pytorch-binary-docker-image-ubuntu16.04:latest + libtorch_variant: shared-with-deps name: binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_test - build_environment: "libtorch 3.7m cpu gcc5.4_cxx11-abi" requires: - setup - binary_linux_libtorch_3_7m_cpu_gcc5_4_cxx11-abi_shared-with-deps_build - libtorch_variant: "shared-with-deps" - docker_image: "pytorch/pytorch-binary-docker-image-ubuntu16.04:latest" - diff --git a/.circleci/verbatim-sources/workflows-nightly-android-binary-builds.yml b/.circleci/verbatim-sources/workflows-nightly-android-binary-builds.yml index 3b92f53eed5..793386c6f75 100644 --- a/.circleci/verbatim-sources/workflows-nightly-android-binary-builds.yml +++ b/.circleci/verbatim-sources/workflows-nightly-android-binary-builds.yml @@ -1,56 +1,54 @@ - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32 + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 + filters: + branches: + only: nightly name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build - build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32" requires: - setup - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" + - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64 + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 filters: branches: only: nightly - - pytorch_linux_build: name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build - build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64" requires: - setup - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" + - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 filters: branches: only: nightly - - pytorch_linux_build: name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build - build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a" requires: - setup - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" - filters: - branches: - only: nightly - pytorch_linux_build: - name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build - build_environment: "pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a" - requires: - - setup - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" + build_environment: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 filters: branches: only: nightly - + name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build + requires: + - setup - pytorch_android_gradle_build: + filters: + branches: + only: nightly name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_android_gradle_build requires: - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v7a_build - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_arm_v8a_build - filters: - branches: - only: nightly - - pytorch_android_publish_snapshot: - name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_android_publish_snapshot - requires: - - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_android_gradle_build context: org-member filters: branches: only: nightly + name: nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_android_publish_snapshot + requires: + - nightly_pytorch_linux_xenial_py3_clang5_android_ndk_r19c_android_gradle_build diff --git a/.circleci/verbatim-sources/workflows-nightly-ios-binary-builds.yml b/.circleci/verbatim-sources/workflows-nightly-ios-binary-builds.yml index 62d8dc855f4..e92001a6c3c 100644 --- a/.circleci/verbatim-sources/workflows-nightly-ios-binary-builds.yml +++ b/.circleci/verbatim-sources/workflows-nightly-ios-binary-builds.yml @@ -1,33 +1,32 @@ - # Pytorch iOS binary builds - binary_ios_build: + build_environment: libtorch-ios-11.2.1-nightly-x86_64-build + context: org-member + filters: + branches: + only: nightly + ios_arch: x86_64 + ios_platform: SIMULATOR name: pytorch_ios_11_2_1_nightly_x86_64_build - build_environment: "libtorch-ios-11.2.1-nightly-x86_64-build" - context: org-member - ios_platform: "SIMULATOR" - ios_arch: "x86_64" - requires: + requires: - setup - filters: - branches: - only: nightly - binary_ios_build: - name: pytorch_ios_11_2_1_nightly_arm64_build - build_environment: "libtorch-ios-11.2.1-nightly-arm64-build" + build_environment: libtorch-ios-11.2.1-nightly-arm64-build context: org-member - ios_arch: "arm64" - ios_platform: "OS" - requires: - - setup filters: branches: only: nightly + ios_arch: arm64 + ios_platform: OS + name: pytorch_ios_11_2_1_nightly_arm64_build + requires: + - setup - binary_ios_upload: - build_environment: "libtorch-ios-11.2.1-nightly-binary-build-upload" + build_environment: libtorch-ios-11.2.1-nightly-binary-build-upload context: org-member + filters: + branches: + only: nightly requires: - setup - pytorch_ios_11_2_1_nightly_x86_64_build - pytorch_ios_11_2_1_nightly_arm64_build - filters: - branches: - only: nightly diff --git a/.circleci/verbatim-sources/workflows-promote.yml b/.circleci/verbatim-sources/workflows-promote.yml index bd94361ec2e..d3afc0862d6 100644 --- a/.circleci/verbatim-sources/workflows-promote.yml +++ b/.circleci/verbatim-sources/workflows-promote.yml @@ -5,42 +5,42 @@ # CircleCI security context - promote_approval: context: org-member - type: approval filters: + branches: + ignore: /.*/ tags: only: /v[0-9]+(\.[0-9]+)*/ - branches: - ignore: /.*/ + type: approval - promote_s3: context: org-member + filters: + branches: + ignore: /.*/ + tags: + only: /v[0-9]+(\.[0-9]+)*/ name: promote_s3_libtorch package_name: libtorch requires: - promote_approval - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - branches: - ignore: /.*/ - promote_s3: context: org-member + filters: + branches: + ignore: /.*/ + tags: + only: /v[0-9]+(\.[0-9]+)*/ name: promote_s3_torch package_name: torch requires: - promote_approval - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - branches: - ignore: /.*/ - promote_conda: context: org-member + filters: + branches: + ignore: /.*/ + tags: + only: /v[0-9]+(\.[0-9]+)*/ name: promote_conda_pytorch package_name: pytorch requires: - promote_approval - filters: - tags: - only: /v[0-9]+(\.[0-9]+)*/ - branches: - ignore: /.*/ diff --git a/.circleci/verbatim-sources/workflows-pytorch-android-gradle-build.yml b/.circleci/verbatim-sources/workflows-pytorch-android-gradle-build.yml index b3829054460..17cfda8711a 100644 --- a/.circleci/verbatim-sources/workflows-pytorch-android-gradle-build.yml +++ b/.circleci/verbatim-sources/workflows-pytorch-android-gradle-build.yml @@ -1,22 +1,21 @@ - pytorch_android_gradle_build-x86_32: - name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-x86_32 filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ + name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-x86_32 requires: - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build - - pytorch_android_gradle_build: - name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ + name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build requires: - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build - pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_64_build diff --git a/.circleci/verbatim-sources/workflows-pytorch-bazel-builds.yml b/.circleci/verbatim-sources/workflows-pytorch-bazel-builds.yml index 825ea500ad1..20181ca9f2b 100644 --- a/.circleci/verbatim-sources/workflows-pytorch-bazel-builds.yml +++ b/.circleci/verbatim-sources/workflows-pytorch-bazel-builds.yml @@ -1,14 +1,14 @@ - pytorch_linux_bazel_build: + build_environment: pytorch-linux-xenial-py3.6-gcc7-bazel-build + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026 name: pytorch_bazel_build requires: - setup - build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-build" - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026" resource_class: large - pytorch_linux_bazel_test: + build_environment: pytorch-linux-xenial-py3.6-gcc7-bazel-test + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026 name: pytorch_bazel_test requires: - setup - pytorch_bazel_build - build_environment: "pytorch-linux-xenial-py3.6-gcc7-bazel-test" - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc7:f990c76a-a798-42bb-852f-5be5006f8026" diff --git a/.circleci/verbatim-sources/workflows-pytorch-ios-builds.yml b/.circleci/verbatim-sources/workflows-pytorch-ios-builds.yml index 93aa3287444..56eedf7c276 100644 --- a/.circleci/verbatim-sources/workflows-pytorch-ios-builds.yml +++ b/.circleci/verbatim-sources/workflows-pytorch-ios-builds.yml @@ -1,26 +1,24 @@ - # Pytorch iOS PR builds - pytorch_ios_build: + build_environment: pytorch-ios-11.2.1-x86_64_build + ios_arch: x86_64 + ios_platform: SIMULATOR name: pytorch_ios_11_2_1_x86_64_build - build_environment: "pytorch-ios-11.2.1-x86_64_build" - ios_arch: "x86_64" - ios_platform: "SIMULATOR" requires: - setup - pytorch_ios_build: + build_environment: pytorch-ios-11.2.1-arm64_build + context: org-member + ios_arch: arm64 + ios_platform: OS name: pytorch_ios_11_2_1_arm64_build - context: org-member - build_environment: "pytorch-ios-11.2.1-arm64_build" - ios_arch: "arm64" - ios_platform: "OS" requires: - setup - pytorch_ios_build: - name: pytorch_ios_11_2_1_arm64_custom_build + build_environment: pytorch-ios-11.2.1-arm64_custom_build context: org-member - build_environment: "pytorch-ios-11.2.1-arm64_custom_build" - ios_arch: "arm64" - ios_platform: "OS" - op_list: "mobilenetv2.yaml" + ios_arch: arm64 + ios_platform: OS + name: pytorch_ios_11_2_1_arm64_custom_build + op_list: mobilenetv2.yaml requires: - setup - diff --git a/.circleci/verbatim-sources/workflows-pytorch-mobile-builds.yml b/.circleci/verbatim-sources/workflows-pytorch-mobile-builds.yml index 2a91b464526..0b958cf4057 100644 --- a/.circleci/verbatim-sources/workflows-pytorch-mobile-builds.yml +++ b/.circleci/verbatim-sources/workflows-pytorch-mobile-builds.yml @@ -1,38 +1,38 @@ # PyTorch Mobile PR builds (use linux host toolchain + mobile build options) - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-mobile-build + build_only: "1" + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 name: pytorch_linux_xenial_py3_clang5_mobile_build requires: - setup - build_environment: "pytorch-linux-xenial-py3-clang5-mobile-build" - build_only: "1" - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-mobile-custom-build-static + build_only: "1" + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 name: pytorch_linux_xenial_py3_clang5_mobile_custom_build_static requires: - setup - build_environment: "pytorch-linux-xenial-py3-clang5-mobile-custom-build-static" - build_only: "1" - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-asan:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" - pytorch_linux_build: + build_environment: pytorch-linux-xenial-py3-clang5-mobile-custom-build-dynamic + build_only: "1" + # Use LLVM-DEV toolchain in android-ndk-r19c docker image + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 name: pytorch_linux_xenial_py3_clang5_mobile_custom_build_dynamic requires: - setup - build_environment: "pytorch-linux-xenial-py3-clang5-mobile-custom-build-dynamic" - build_only: "1" - # Use LLVM-DEV toolchain in android-ndk-r19c docker image - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" - pytorch_linux_build: - name: pytorch_linux_xenial_py3_clang5_mobile_code_analysis - requires: - - setup + # Use LLVM-DEV toolchain in android-ndk-r19c docker image # Most of this CI is already covered by "mobile-custom-build-dynamic" job + build_environment: pytorch-linux-xenial-py3-clang5-mobile-code-analysis + build_only: "1" + docker_image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2 filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - build_environment: "pytorch-linux-xenial-py3-clang5-mobile-code-analysis" - build_only: "1" - # Use LLVM-DEV toolchain in android-ndk-r19c docker image - docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:9a3986fa-7ce7-4a36-a001-3c9bef9892e2" + name: pytorch_linux_xenial_py3_clang5_mobile_code_analysis + requires: + - setup