diff --git a/.circleci/cimodel/data/pytorch_build_data.py b/.circleci/cimodel/data/pytorch_build_data.py index 76d11d7a860..aeee86f82d7 100644 --- a/.circleci/cimodel/data/pytorch_build_data.py +++ b/.circleci/cimodel/data/pytorch_build_data.py @@ -140,6 +140,7 @@ class XlaConfigNode(TreeConfigNode): def child_constructor(self): return ImportantConfigNode + class ParallelTBBConfigNode(TreeConfigNode): def modify_label(self, label): return "PARALLELTBB=" + str(label) @@ -150,6 +151,7 @@ class ParallelTBBConfigNode(TreeConfigNode): def child_constructor(self): return ImportantConfigNode + class ParallelNativeConfigNode(TreeConfigNode): def modify_label(self, label): return "PARALLELNATIVE=" + str(label) @@ -160,6 +162,7 @@ class ParallelNativeConfigNode(TreeConfigNode): def child_constructor(self): return ImportantConfigNode + class LibTorchConfigNode(TreeConfigNode): def modify_label(self, label): return "BUILD_TEST_LIBTORCH=" + str(label) @@ -170,6 +173,7 @@ class LibTorchConfigNode(TreeConfigNode): def child_constructor(self): return ImportantConfigNode + class AndroidAbiConfigNode(TreeConfigNode): def init2(self, node_name): @@ -178,6 +182,7 @@ class AndroidAbiConfigNode(TreeConfigNode): def child_constructor(self): return ImportantConfigNode + class ImportantConfigNode(TreeConfigNode): def modify_label(self, label): return "IMPORTANT=" + str(label) @@ -202,6 +207,7 @@ class XenialCompilerConfigNode(TreeConfigNode): return XenialCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode + class BionicCompilerConfigNode(TreeConfigNode): def modify_label(self, label): @@ -215,6 +221,7 @@ class BionicCompilerConfigNode(TreeConfigNode): return BionicCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode + class XenialCompilerVersionConfigNode(TreeConfigNode): def init2(self, node_name): self.props["compiler_version"] = node_name @@ -223,6 +230,7 @@ class XenialCompilerVersionConfigNode(TreeConfigNode): def child_constructor(self): return PyVerConfigNode + class BionicCompilerVersionConfigNode(TreeConfigNode): def init2(self, node_name): self.props["compiler_version"] = node_name diff --git a/.circleci/cimodel/data/windows_build_definitions.py b/.circleci/cimodel/data/windows_build_definitions.py new file mode 100644 index 00000000000..8cbdc29132c --- /dev/null +++ b/.circleci/cimodel/data/windows_build_definitions.py @@ -0,0 +1,139 @@ +import cimodel.lib.miniutils as miniutils + + +NON_PR_BRANCH_LIST = [ + "master", + r"/ci-all\/.*/", + r"/release\/.*/", +] + + +class WindowJob: + def __init__(self, + test_index, + vscode_spec, + cuda_version, + force_on_cpu=False, + run_on_prs_pred=lambda job: job.vscode_spec.year != 2019): + + self.test_index = test_index + self.vscode_spec = vscode_spec + self.cuda_version = cuda_version + self.force_on_cpu = force_on_cpu + self.run_on_prs_pred = run_on_prs_pred + + def gen_tree(self): + + base_phase = "build" if self.test_index is None else "test" + numbered_phase = base_phase if self.test_index is None else base_phase + str(self.test_index) + + key_name = "_".join(["pytorch", "windows", base_phase]) + + cpu_forcing_name_parts = ["on", "cpu"] if self.force_on_cpu else [] + + target_arch = self.cuda_version if self.cuda_version else "cpu" + + base_name_parts = [ + "pytorch", + "windows", + self.vscode_spec.render(), + "py36", + target_arch, + ] + + prerequisite_jobs = [] + if base_phase == "test": + prerequisite_jobs.append("_".join(base_name_parts + ["build"])) + + arch_env_elements = ["cuda10", "cudnn7"] if self.cuda_version else ["cpu"] + + build_environment_string = "-".join([ + "pytorch", + "win", + ] + self.vscode_spec.get_elements() + arch_env_elements + [ + "py3", + ]) + + is_running_on_cuda = bool(self.cuda_version) and not self.force_on_cpu + + props_dict = { + "build_environment": build_environment_string, + "python_version": miniutils.quote("3.6"), + "vc_version": miniutils.quote(self.vscode_spec.dotted_version()), + "vc_year": miniutils.quote(str(self.vscode_spec.year)), + "vc_product": self.vscode_spec.get_product(), + "use_cuda": miniutils.quote(str(int(is_running_on_cuda))), + "requires": ["setup"] + prerequisite_jobs, + } + + if self.run_on_prs_pred(self): + props_dict["filters"] = { + "branches": { + "only": NON_PR_BRANCH_LIST, + }, + } + + name_parts = base_name_parts + cpu_forcing_name_parts + [ + numbered_phase, + ] + + if base_phase == "test": + test_name = "-".join(["pytorch", "windows", numbered_phase]) + props_dict["test_name"] = test_name + + if is_running_on_cuda: + props_dict["executor"] = "windows-with-nvidia-gpu" + + props_dict["cuda_version"] = miniutils.quote(str(10)) if self.cuda_version else "cpu" + props_dict["name"] = "_".join(name_parts) + + return [{key_name: props_dict}] + + +class VcSpec: + def __init__(self, year, version_elements=None): + self.year = year + self.version_elements = version_elements or [] + + def get_elements(self): + return [self.prefixed_year()] + self.version_elements + + def get_product(self): + return "Community" if self.year == 2019 else "BuildTools" + + def dotted_version(self): + return ".".join(self.version_elements) + + def prefixed_year(self): + return "vs" + str(self.year) + + def render(self): + return "_".join(filter(None, [self.prefixed_year(), self.dotted_version()])) + + +WINDOWS_WORKFLOW_DATA = [ + WindowJob(None, VcSpec(2017, ["14", "11"]), "cuda10.1"), + WindowJob(1, VcSpec(2017, ["14", "11"]), "cuda10.1"), + WindowJob(2, VcSpec(2017, ["14", "11"]), "cuda10.1"), + WindowJob(None, VcSpec(2017, ["14", "16"]), "cuda10.1"), + WindowJob(1, VcSpec(2017, ["14", "16"]), "cuda10.1"), + WindowJob(2, VcSpec(2017, ["14", "16"]), "cuda10.1"), + WindowJob(None, VcSpec(2019), "cuda10.1"), + WindowJob(1, VcSpec(2019), "cuda10.1"), + WindowJob(2, VcSpec(2019), "cuda10.1"), + WindowJob(None, VcSpec(2017, ["14", "11"]), None), + WindowJob(1, VcSpec(2017, ["14", "11"]), None), + WindowJob(2, VcSpec(2017, ["14", "11"]), None), + WindowJob(None, VcSpec(2017, ["14", "16"]), None), + WindowJob(1, VcSpec(2017, ["14", "16"]), None), + WindowJob(2, VcSpec(2017, ["14", "16"]), None), + WindowJob(None, VcSpec(2019), None), + WindowJob(1, VcSpec(2019), None), + WindowJob(2, VcSpec(2019), None), + WindowJob(1, VcSpec(2019), "cuda10.1", force_on_cpu=True), + WindowJob(2, VcSpec(2019), "cuda10.1", force_on_cpu=True), +] + + +def get_windows_workflows(): + return [item.gen_tree() for item in WINDOWS_WORKFLOW_DATA] diff --git a/.circleci/cimodel/lib/miniyaml.py b/.circleci/cimodel/lib/miniyaml.py index 94c3430b6f1..713864f6362 100644 --- a/.circleci/cimodel/lib/miniyaml.py +++ b/.circleci/cimodel/lib/miniyaml.py @@ -1,4 +1,5 @@ from collections import OrderedDict +import cimodel.lib.miniutils as miniutils LIST_MARKER = "- " @@ -43,5 +44,8 @@ def render(fh, data, depth, is_list_member=False): render(fh, v, depth, True) else: + # use empty quotes to denote an empty sting value instead of blank space + modified_data = miniutils.quote(data) if data == '' else data + list_member_prefix = indentation + LIST_MARKER if is_list_member else "" - fh.write(list_member_prefix + str(data) + "\n") + fh.write(list_member_prefix + str(modified_data) + "\n") diff --git a/.circleci/codegen_validation/compare_normalized_yaml.sh b/.circleci/codegen_validation/compare_normalized_yaml.sh new file mode 100755 index 00000000000..94cf8e03d21 --- /dev/null +++ b/.circleci/codegen_validation/compare_normalized_yaml.sh @@ -0,0 +1,16 @@ +#!/bin/bash -xe + + +YAML_FILENAME=verbatim-sources/windows-build-test.yml + + +# Allows this script to be invoked from any directory: +cd $(dirname "$0") + +pushd .. + + +meld $YAML_FILENAME <(./normalize_yaml_fragment.py < $YAML_FILENAME) + + +popd \ No newline at end of file diff --git a/.circleci/config.yml b/.circleci/config.yml index c0db5477b3b..37e9fd465c1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2066,334 +2066,332 @@ workflows: only: /.*/ branches: only: /.*/ - # Warning: indentation here matters! - - pytorch_windows_build: + build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3 + cuda_version: "10" + filters: + branches: + only: + - master + - /ci-all\/.*/ + - /release\/.*/ name: pytorch_windows_vs2017_14.11_py36_cuda10.1_build - build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3 - cuda_version: "10" python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" requires: - setup + use_cuda: "1" + vc_product: BuildTools + vc_version: "14.11" + vc_year: "2017" + - pytorch_windows_test: + build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3 + cuda_version: "10" + executor: windows-with-nvidia-gpu filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - - pytorch_windows_test: name: pytorch_windows_vs2017_14.11_py36_cuda10.1_test1 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3 - test_name: pytorch-windows-test1 - cuda_version: "10" python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" requires: - setup - pytorch_windows_vs2017_14.11_py36_cuda10.1_build + test_name: pytorch-windows-test1 + use_cuda: "1" + vc_product: BuildTools + vc_version: "14.11" + vc_year: "2017" + - pytorch_windows_test: + build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3 + cuda_version: "10" + executor: windows-with-nvidia-gpu filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - - pytorch_windows_test: name: pytorch_windows_vs2017_14.11_py36_cuda10.1_test2 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3 - test_name: pytorch-windows-test2 - cuda_version: "10" python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" requires: - setup - pytorch_windows_vs2017_14.11_py36_cuda10.1_build + test_name: pytorch-windows-test2 + use_cuda: "1" + vc_product: BuildTools + vc_version: "14.11" + vc_year: "2017" + - pytorch_windows_build: + build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3 + cuda_version: "10" filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - - pytorch_windows_build: name: pytorch_windows_vs2017_14.16_py36_cuda10.1_build - build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3 - cuda_version: "10" python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" requires: - setup + use_cuda: "1" + vc_product: BuildTools + vc_version: "14.16" + vc_year: "2017" + - pytorch_windows_test: + build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3 + cuda_version: "10" + executor: windows-with-nvidia-gpu filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - - pytorch_windows_test: name: pytorch_windows_vs2017_14.16_py36_cuda10.1_test1 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3 - test_name: pytorch-windows-test1 - cuda_version: "10" python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" requires: - setup - pytorch_windows_vs2017_14.16_py36_cuda10.1_build + test_name: pytorch-windows-test1 + use_cuda: "1" + vc_product: BuildTools + vc_version: "14.16" + vc_year: "2017" + - pytorch_windows_test: + build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3 + cuda_version: "10" + executor: windows-with-nvidia-gpu filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - - pytorch_windows_test: name: pytorch_windows_vs2017_14.16_py36_cuda10.1_test2 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3 - test_name: pytorch-windows-test2 - cuda_version: "10" python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" requires: - setup - pytorch_windows_vs2017_14.16_py36_cuda10.1_build - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ + test_name: pytorch-windows-test2 + use_cuda: "1" + vc_product: BuildTools + vc_version: "14.16" + vc_year: "2017" - pytorch_windows_build: + build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 + cuda_version: "10" name: pytorch_windows_vs2019_py36_cuda10.1_build - build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 - cuda_version: "10" python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "1" requires: - setup + use_cuda: "1" + vc_product: Community + vc_version: "" + vc_year: "2019" - pytorch_windows_test: + build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 + cuda_version: "10" + executor: windows-with-nvidia-gpu name: pytorch_windows_vs2019_py36_cuda10.1_test1 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 - test_name: pytorch-windows-test1 - cuda_version: "10" python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "1" requires: - setup - pytorch_windows_vs2019_py36_cuda10.1_build + test_name: pytorch-windows-test1 + use_cuda: "1" + vc_product: Community + vc_version: "" + vc_year: "2019" - pytorch_windows_test: + build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 + cuda_version: "10" + executor: windows-with-nvidia-gpu name: pytorch_windows_vs2019_py36_cuda10.1_test2 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 - test_name: pytorch-windows-test2 - cuda_version: "10" python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" + requires: + - setup + - pytorch_windows_vs2019_py36_cuda10.1_build + test_name: pytorch-windows-test2 use_cuda: "1" - requires: - - setup - - pytorch_windows_vs2019_py36_cuda10.1_build + vc_product: Community + vc_version: "" + vc_year: "2019" - pytorch_windows_build: + build_environment: pytorch-win-vs2017-14-11-cpu-py3 + cuda_version: cpu + filters: + branches: + only: + - master + - /ci-all\/.*/ + - /release\/.*/ name: pytorch_windows_vs2017_14.11_py36_cpu_build - build_environment: pytorch-win-vs2017-14-11-cpu-py3 - cuda_version: "cpu" python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" requires: - setup + use_cuda: "0" + vc_product: BuildTools + vc_version: "14.11" + vc_year: "2017" + - pytorch_windows_test: + build_environment: pytorch-win-vs2017-14-11-cpu-py3 + cuda_version: cpu filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - - pytorch_windows_test: name: pytorch_windows_vs2017_14.11_py36_cpu_test1 - build_environment: pytorch-win-vs2017-14-11-cpu-py3 - test_name: pytorch-windows-test1 - cuda_version: "cpu" python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" requires: - setup - pytorch_windows_vs2017_14.11_py36_cpu_build + test_name: pytorch-windows-test1 + use_cuda: "0" + vc_product: BuildTools + vc_version: "14.11" + vc_year: "2017" + - pytorch_windows_test: + build_environment: pytorch-win-vs2017-14-11-cpu-py3 + cuda_version: cpu filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - - pytorch_windows_test: name: pytorch_windows_vs2017_14.11_py36_cpu_test2 - build_environment: pytorch-win-vs2017-14-11-cpu-py3 - test_name: pytorch-windows-test2 - cuda_version: "cpu" python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" requires: - setup - pytorch_windows_vs2017_14.11_py36_cpu_build + test_name: pytorch-windows-test2 + use_cuda: "0" + vc_product: BuildTools + vc_version: "14.11" + vc_year: "2017" + - pytorch_windows_build: + build_environment: pytorch-win-vs2017-14-16-cpu-py3 + cuda_version: cpu filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - - pytorch_windows_build: name: pytorch_windows_vs2017_14.16_py36_cpu_build - build_environment: pytorch-win-vs2017-14-16-cpu-py3 - cuda_version: "cpu" python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" requires: - setup + use_cuda: "0" + vc_product: BuildTools + vc_version: "14.16" + vc_year: "2017" + - pytorch_windows_test: + build_environment: pytorch-win-vs2017-14-16-cpu-py3 + cuda_version: cpu filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - - pytorch_windows_test: name: pytorch_windows_vs2017_14.16_py36_cpu_test1 - build_environment: pytorch-win-vs2017-14-16-cpu-py3 - test_name: pytorch-windows-test1 - cuda_version: "cpu" python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" requires: - setup - pytorch_windows_vs2017_14.16_py36_cpu_build + test_name: pytorch-windows-test1 + use_cuda: "0" + vc_product: BuildTools + vc_version: "14.16" + vc_year: "2017" + - pytorch_windows_test: + build_environment: pytorch-win-vs2017-14-16-cpu-py3 + cuda_version: cpu filters: branches: only: - master - /ci-all\/.*/ - /release\/.*/ - - pytorch_windows_test: name: pytorch_windows_vs2017_14.16_py36_cpu_test2 - build_environment: pytorch-win-vs2017-14-16-cpu-py3 - test_name: pytorch-windows-test2 - cuda_version: "cpu" python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" requires: - setup - pytorch_windows_vs2017_14.16_py36_cpu_build - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ + test_name: pytorch-windows-test2 + use_cuda: "0" + vc_product: BuildTools + vc_version: "14.16" + vc_year: "2017" - pytorch_windows_build: + build_environment: pytorch-win-vs2019-cpu-py3 + cuda_version: cpu name: pytorch_windows_vs2019_py36_cpu_build - build_environment: pytorch-win-vs2019-cpu-py3 - cuda_version: "cpu" python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "0" requires: - setup + use_cuda: "0" + vc_product: Community + vc_version: "" + vc_year: "2019" - pytorch_windows_test: + build_environment: pytorch-win-vs2019-cpu-py3 + cuda_version: cpu name: pytorch_windows_vs2019_py36_cpu_test1 - build_environment: pytorch-win-vs2019-cpu-py3 - test_name: pytorch-windows-test1 - cuda_version: "cpu" python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "0" requires: - setup - pytorch_windows_vs2019_py36_cpu_build + test_name: pytorch-windows-test1 + use_cuda: "0" + vc_product: Community + vc_version: "" + vc_year: "2019" - pytorch_windows_test: + build_environment: pytorch-win-vs2019-cpu-py3 + cuda_version: cpu name: pytorch_windows_vs2019_py36_cpu_test2 - build_environment: pytorch-win-vs2019-cpu-py3 - test_name: pytorch-windows-test2 - cuda_version: "cpu" python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "0" requires: - setup - pytorch_windows_vs2019_py36_cpu_build - - pytorch_windows_test: - name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test1 - build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 - test_name: pytorch-windows-test1 - cuda_version: "10" - python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "0" - requires: - - setup - - pytorch_windows_vs2019_py36_cuda10.1_build - - pytorch_windows_test: - name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test2 - build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 test_name: pytorch-windows-test2 - cuda_version: "10" - python_version: "3.6" + use_cuda: "0" + vc_product: Community vc_version: "" vc_year: "2019" - vc_product: "Community" - use_cuda: "0" + - pytorch_windows_test: + build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 + cuda_version: "10" + name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test1 + python_version: "3.6" requires: - setup - pytorch_windows_vs2019_py36_cuda10.1_build + test_name: pytorch-windows-test1 + use_cuda: "0" + vc_product: Community + vc_version: "" + vc_year: "2019" + - pytorch_windows_test: + build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 + cuda_version: "10" + name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test2 + python_version: "3.6" + requires: + - setup + - pytorch_windows_vs2019_py36_cuda10.1_build + test_name: pytorch-windows-test2 + use_cuda: "0" + vc_product: Community + vc_version: "" + vc_year: "2019" - pytorch_linux_build: name: pytorch_linux_xenial_pynightly_build requires: diff --git a/.circleci/generate_config_yml.py b/.circleci/generate_config_yml.py index a6f2d4b0f80..691bd3b6cbb 100755 --- a/.circleci/generate_config_yml.py +++ b/.circleci/generate_config_yml.py @@ -11,6 +11,7 @@ import shutil from collections import namedtuple, OrderedDict import cimodel.data.pytorch_build_definitions as pytorch_build_definitions +import cimodel.data.windows_build_definitions as windows_build_definitions import cimodel.data.binary_build_definitions as binary_build_definitions import cimodel.data.caffe2_build_definitions as caffe2_build_definitions import cimodel.lib.miniutils as miniutils @@ -94,7 +95,7 @@ YAML_SOURCES = [ File("workflows.yml"), File("workflows-setup-job.yml"), - File("windows-build-test.yml"), + Listgen(windows_build_definitions.get_windows_workflows, 3), Listgen(pytorch_build_definitions.get_workflow_jobs, 3), File("workflows-pytorch-macos-builds.yml"), File("workflows-pytorch-android-gradle-build.yml"), diff --git a/.circleci/normalize_yaml_fragment.py b/.circleci/normalize_yaml_fragment.py new file mode 100755 index 00000000000..3d4d5ebacbb --- /dev/null +++ b/.circleci/normalize_yaml_fragment.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +import sys +import yaml +import cimodel.lib.miniyaml as miniyaml + + +def regurgitate(depth, use_pyyaml_formatter=False): + data = yaml.safe_load(sys.stdin) + + if use_pyyaml_formatter: + output = yaml.dump(data, sort_keys=True) + sys.stdout.write(output) + else: + miniyaml.render(sys.stdout, data, depth) + + +if __name__ == "__main__": + regurgitate(3) diff --git a/.circleci/verbatim-sources/windows-build-test.yml b/.circleci/verbatim-sources/windows-build-test.yml deleted file mode 100644 index 25a8f548ab0..00000000000 --- a/.circleci/verbatim-sources/windows-build-test.yml +++ /dev/null @@ -1,328 +0,0 @@ - # Warning: indentation here matters! - - - pytorch_windows_build: - name: pytorch_windows_vs2017_14.11_py36_cuda10.1_build - build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3 - cuda_version: "10" - python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" - requires: - - setup - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_test: - name: pytorch_windows_vs2017_14.11_py36_cuda10.1_test1 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3 - test_name: pytorch-windows-test1 - cuda_version: "10" - python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" - requires: - - setup - - pytorch_windows_vs2017_14.11_py36_cuda10.1_build - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_test: - name: pytorch_windows_vs2017_14.11_py36_cuda10.1_test2 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3 - test_name: pytorch-windows-test2 - cuda_version: "10" - python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" - requires: - - setup - - pytorch_windows_vs2017_14.11_py36_cuda10.1_build - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_build: - name: pytorch_windows_vs2017_14.16_py36_cuda10.1_build - build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3 - cuda_version: "10" - python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" - requires: - - setup - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_test: - name: pytorch_windows_vs2017_14.16_py36_cuda10.1_test1 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3 - test_name: pytorch-windows-test1 - cuda_version: "10" - python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" - requires: - - setup - - pytorch_windows_vs2017_14.16_py36_cuda10.1_build - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_test: - name: pytorch_windows_vs2017_14.16_py36_cuda10.1_test2 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3 - test_name: pytorch-windows-test2 - cuda_version: "10" - python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "1" - requires: - - setup - - pytorch_windows_vs2017_14.16_py36_cuda10.1_build - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_build: - name: pytorch_windows_vs2019_py36_cuda10.1_build - build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 - cuda_version: "10" - python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "1" - requires: - - setup - - pytorch_windows_test: - name: pytorch_windows_vs2019_py36_cuda10.1_test1 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 - test_name: pytorch-windows-test1 - cuda_version: "10" - python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "1" - requires: - - setup - - pytorch_windows_vs2019_py36_cuda10.1_build - - pytorch_windows_test: - name: pytorch_windows_vs2019_py36_cuda10.1_test2 - executor: windows-with-nvidia-gpu - build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 - test_name: pytorch-windows-test2 - cuda_version: "10" - python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "1" - requires: - - setup - - pytorch_windows_vs2019_py36_cuda10.1_build - - pytorch_windows_build: - name: pytorch_windows_vs2017_14.11_py36_cpu_build - build_environment: pytorch-win-vs2017-14-11-cpu-py3 - cuda_version: "cpu" - python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" - requires: - - setup - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_test: - name: pytorch_windows_vs2017_14.11_py36_cpu_test1 - build_environment: pytorch-win-vs2017-14-11-cpu-py3 - test_name: pytorch-windows-test1 - cuda_version: "cpu" - python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" - requires: - - setup - - pytorch_windows_vs2017_14.11_py36_cpu_build - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_test: - name: pytorch_windows_vs2017_14.11_py36_cpu_test2 - build_environment: pytorch-win-vs2017-14-11-cpu-py3 - test_name: pytorch-windows-test2 - cuda_version: "cpu" - python_version: "3.6" - vc_version: "14.11" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" - requires: - - setup - - pytorch_windows_vs2017_14.11_py36_cpu_build - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_build: - name: pytorch_windows_vs2017_14.16_py36_cpu_build - build_environment: pytorch-win-vs2017-14-16-cpu-py3 - cuda_version: "cpu" - python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" - requires: - - setup - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_test: - name: pytorch_windows_vs2017_14.16_py36_cpu_test1 - build_environment: pytorch-win-vs2017-14-16-cpu-py3 - test_name: pytorch-windows-test1 - cuda_version: "cpu" - python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" - requires: - - setup - - pytorch_windows_vs2017_14.16_py36_cpu_build - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_test: - name: pytorch_windows_vs2017_14.16_py36_cpu_test2 - build_environment: pytorch-win-vs2017-14-16-cpu-py3 - test_name: pytorch-windows-test2 - cuda_version: "cpu" - python_version: "3.6" - vc_version: "14.16" - vc_year: "2017" - vc_product: "BuildTools" - use_cuda: "0" - requires: - - setup - - pytorch_windows_vs2017_14.16_py36_cpu_build - filters: - branches: - only: - - master - - /ci-all\/.*/ - - /release\/.*/ - - pytorch_windows_build: - name: pytorch_windows_vs2019_py36_cpu_build - build_environment: pytorch-win-vs2019-cpu-py3 - cuda_version: "cpu" - python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "0" - requires: - - setup - - pytorch_windows_test: - name: pytorch_windows_vs2019_py36_cpu_test1 - build_environment: pytorch-win-vs2019-cpu-py3 - test_name: pytorch-windows-test1 - cuda_version: "cpu" - python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "0" - requires: - - setup - - pytorch_windows_vs2019_py36_cpu_build - - pytorch_windows_test: - name: pytorch_windows_vs2019_py36_cpu_test2 - build_environment: pytorch-win-vs2019-cpu-py3 - test_name: pytorch-windows-test2 - cuda_version: "cpu" - python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "0" - requires: - - setup - - pytorch_windows_vs2019_py36_cpu_build - - pytorch_windows_test: - name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test1 - build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 - test_name: pytorch-windows-test1 - cuda_version: "10" - python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "0" - requires: - - setup - - pytorch_windows_vs2019_py36_cuda10.1_build - - pytorch_windows_test: - name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test2 - build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3 - test_name: pytorch-windows-test2 - cuda_version: "10" - python_version: "3.6" - vc_version: "" - vc_year: "2019" - vc_product: "Community" - use_cuda: "0" - requires: - - setup - - pytorch_windows_vs2019_py36_cuda10.1_build