Codegen CircleCI Windows configs (#38292)

Summary:
This is a step toward re-automating most of the CircleCI `config.yml` generation so that it can be safely refactored into multiple `workflow`s.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/38292

Differential Revision: D21519337

Pulled By: kostmo

fbshipit-source-id: 09cc4f97ac52f37ef6d8a6fb8f49eeead052b446

.circleci/cimodel/data/pytorch_build_data.py

@@ -140,6 +140,7 @@ class XlaConfigNode(TreeConfigNode):
     def child_constructor(self):
         return ImportantConfigNode
 
+
 class ParallelTBBConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "PARALLELTBB=" + str(label)
@@ -150,6 +151,7 @@ class ParallelTBBConfigNode(TreeConfigNode):
     def child_constructor(self):
         return ImportantConfigNode
 
+
 class ParallelNativeConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "PARALLELNATIVE=" + str(label)
@@ -160,6 +162,7 @@ class ParallelNativeConfigNode(TreeConfigNode):
     def child_constructor(self):
         return ImportantConfigNode
 
+
 class LibTorchConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "BUILD_TEST_LIBTORCH=" + str(label)
@@ -170,6 +173,7 @@ class LibTorchConfigNode(TreeConfigNode):
     def child_constructor(self):
         return ImportantConfigNode
 
+
 class AndroidAbiConfigNode(TreeConfigNode):
 
     def init2(self, node_name):
@@ -178,6 +182,7 @@ class AndroidAbiConfigNode(TreeConfigNode):
     def child_constructor(self):
         return ImportantConfigNode
 
+
 class ImportantConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "IMPORTANT=" + str(label)
@@ -202,6 +207,7 @@ class XenialCompilerConfigNode(TreeConfigNode):
 
         return XenialCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode
 
+
 class BionicCompilerConfigNode(TreeConfigNode):
 
     def modify_label(self, label):
@@ -215,6 +221,7 @@ class BionicCompilerConfigNode(TreeConfigNode):
 
         return BionicCompilerVersionConfigNode if self.props["compiler_name"] else PyVerConfigNode
 
+
 class XenialCompilerVersionConfigNode(TreeConfigNode):
     def init2(self, node_name):
         self.props["compiler_version"] = node_name
@@ -223,6 +230,7 @@ class XenialCompilerVersionConfigNode(TreeConfigNode):
     def child_constructor(self):
         return PyVerConfigNode
 
+
 class BionicCompilerVersionConfigNode(TreeConfigNode):
     def init2(self, node_name):
         self.props["compiler_version"] = node_name

.circleci/cimodel/data/windows_build_definitions.py

@@ -0,0 +1,139 @@
+import cimodel.lib.miniutils as miniutils
+
+
+NON_PR_BRANCH_LIST = [
+    "master",
+    r"/ci-all\/.*/",
+    r"/release\/.*/",
+]
+
+
+class WindowJob:
+    def __init__(self,
+                 test_index,
+                 vscode_spec,
+                 cuda_version,
+                 force_on_cpu=False,
+                 run_on_prs_pred=lambda job: job.vscode_spec.year != 2019):
+
+        self.test_index = test_index
+        self.vscode_spec = vscode_spec
+        self.cuda_version = cuda_version
+        self.force_on_cpu = force_on_cpu
+        self.run_on_prs_pred = run_on_prs_pred
+
+    def gen_tree(self):
+
+        base_phase = "build" if self.test_index is None else "test"
+        numbered_phase = base_phase if self.test_index is None else base_phase + str(self.test_index)
+
+        key_name = "_".join(["pytorch", "windows", base_phase])
+
+        cpu_forcing_name_parts = ["on", "cpu"] if self.force_on_cpu else []
+
+        target_arch = self.cuda_version if self.cuda_version else "cpu"
+
+        base_name_parts = [
+            "pytorch",
+            "windows",
+            self.vscode_spec.render(),
+            "py36",
+            target_arch,
+        ]
+
+        prerequisite_jobs = []
+        if base_phase == "test":
+            prerequisite_jobs.append("_".join(base_name_parts + ["build"]))
+
+        arch_env_elements = ["cuda10", "cudnn7"] if self.cuda_version else ["cpu"]
+
+        build_environment_string = "-".join([
+            "pytorch",
+            "win",
+        ] + self.vscode_spec.get_elements() + arch_env_elements + [
+            "py3",
+        ])
+
+        is_running_on_cuda = bool(self.cuda_version) and not self.force_on_cpu
+
+        props_dict = {
+            "build_environment": build_environment_string,
+            "python_version": miniutils.quote("3.6"),
+            "vc_version": miniutils.quote(self.vscode_spec.dotted_version()),
+            "vc_year": miniutils.quote(str(self.vscode_spec.year)),
+            "vc_product": self.vscode_spec.get_product(),
+            "use_cuda": miniutils.quote(str(int(is_running_on_cuda))),
+            "requires": ["setup"] + prerequisite_jobs,
+        }
+
+        if self.run_on_prs_pred(self):
+            props_dict["filters"] = {
+                "branches": {
+                    "only": NON_PR_BRANCH_LIST,
+                },
+            }
+
+        name_parts = base_name_parts + cpu_forcing_name_parts + [
+            numbered_phase,
+        ]
+
+        if base_phase == "test":
+            test_name = "-".join(["pytorch", "windows", numbered_phase])
+            props_dict["test_name"] = test_name
+
+            if is_running_on_cuda:
+                props_dict["executor"] = "windows-with-nvidia-gpu"
+
+        props_dict["cuda_version"] = miniutils.quote(str(10)) if self.cuda_version else "cpu"
+        props_dict["name"] = "_".join(name_parts)
+
+        return [{key_name: props_dict}]
+
+
+class VcSpec:
+    def __init__(self, year, version_elements=None):
+        self.year = year
+        self.version_elements = version_elements or []
+
+    def get_elements(self):
+        return [self.prefixed_year()] + self.version_elements
+
+    def get_product(self):
+        return "Community" if self.year == 2019 else "BuildTools"
+
+    def dotted_version(self):
+        return ".".join(self.version_elements)
+
+    def prefixed_year(self):
+        return "vs" + str(self.year)
+
+    def render(self):
+        return "_".join(filter(None, [self.prefixed_year(), self.dotted_version()]))
+
+
+WINDOWS_WORKFLOW_DATA = [
+    WindowJob(None, VcSpec(2017, ["14", "11"]), "cuda10.1"),
+    WindowJob(1, VcSpec(2017, ["14", "11"]), "cuda10.1"),
+    WindowJob(2, VcSpec(2017, ["14", "11"]), "cuda10.1"),
+    WindowJob(None, VcSpec(2017, ["14", "16"]), "cuda10.1"),
+    WindowJob(1, VcSpec(2017, ["14", "16"]), "cuda10.1"),
+    WindowJob(2, VcSpec(2017, ["14", "16"]), "cuda10.1"),
+    WindowJob(None, VcSpec(2019), "cuda10.1"),
+    WindowJob(1, VcSpec(2019), "cuda10.1"),
+    WindowJob(2, VcSpec(2019), "cuda10.1"),
+    WindowJob(None, VcSpec(2017, ["14", "11"]), None),
+    WindowJob(1, VcSpec(2017, ["14", "11"]), None),
+    WindowJob(2, VcSpec(2017, ["14", "11"]), None),
+    WindowJob(None, VcSpec(2017, ["14", "16"]), None),
+    WindowJob(1, VcSpec(2017, ["14", "16"]), None),
+    WindowJob(2, VcSpec(2017, ["14", "16"]), None),
+    WindowJob(None, VcSpec(2019), None),
+    WindowJob(1, VcSpec(2019), None),
+    WindowJob(2, VcSpec(2019), None),
+    WindowJob(1, VcSpec(2019), "cuda10.1", force_on_cpu=True),
+    WindowJob(2, VcSpec(2019), "cuda10.1", force_on_cpu=True),
+]
+
+
+def get_windows_workflows():
+    return [item.gen_tree() for item in WINDOWS_WORKFLOW_DATA]

.circleci/cimodel/lib/miniyaml.py

@@ -1,4 +1,5 @@
 from collections import OrderedDict
+import cimodel.lib.miniutils as miniutils
 
 
 LIST_MARKER = "- "
@@ -43,5 +44,8 @@ def render(fh, data, depth, is_list_member=False):
             render(fh, v, depth, True)
 
     else:
+        # use empty quotes to denote an empty sting value instead of blank space
+        modified_data = miniutils.quote(data) if data == '' else data
+
         list_member_prefix = indentation + LIST_MARKER if is_list_member else ""
-        fh.write(list_member_prefix + str(data) + "\n")
+        fh.write(list_member_prefix + str(modified_data) + "\n")

.circleci/codegen_validation/compare_normalized_yaml.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -xe
+
+
+YAML_FILENAME=verbatim-sources/windows-build-test.yml
+
+
+# Allows this script to be invoked from any directory:
+cd $(dirname "$0")
+
+pushd ..
+
+
+meld $YAML_FILENAME <(./normalize_yaml_fragment.py < $YAML_FILENAME)
+
+
+popd

.circleci/config.yml

@@ -2066,334 +2066,332 @@ workflows:
               only: /.*/
             branches:
               only: /.*/
-      # Warning: indentation here matters!
-
       - pytorch_windows_build:
+          build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3
+          cuda_version: "10"
+          filters:
+            branches:
+              only:
+                - master
+                - /ci-all\/.*/
+                - /release\/.*/
           name: pytorch_windows_vs2017_14.11_py36_cuda10.1_build
-          build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3
-          cuda_version: "10"
           python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
           requires:
             - setup
+          use_cuda: "1"
+          vc_product: BuildTools
+          vc_version: "14.11"
+          vc_year: "2017"
+      - pytorch_windows_test:
+          build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3
+          cuda_version: "10"
+          executor: windows-with-nvidia-gpu
           filters:
             branches:
               only:
                 - master
                 - /ci-all\/.*/
                 - /release\/.*/
-      - pytorch_windows_test:
           name: pytorch_windows_vs2017_14.11_py36_cuda10.1_test1
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "10"
           python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
           requires:
             - setup
             - pytorch_windows_vs2017_14.11_py36_cuda10.1_build
+          test_name: pytorch-windows-test1
+          use_cuda: "1"
+          vc_product: BuildTools
+          vc_version: "14.11"
+          vc_year: "2017"
+      - pytorch_windows_test:
+          build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3
+          cuda_version: "10"
+          executor: windows-with-nvidia-gpu
           filters:
             branches:
               only:
                 - master
                 - /ci-all\/.*/
                 - /release\/.*/
-      - pytorch_windows_test:
           name: pytorch_windows_vs2017_14.11_py36_cuda10.1_test2
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "10"
           python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
           requires:
             - setup
             - pytorch_windows_vs2017_14.11_py36_cuda10.1_build
+          test_name: pytorch-windows-test2
+          use_cuda: "1"
+          vc_product: BuildTools
+          vc_version: "14.11"
+          vc_year: "2017"
+      - pytorch_windows_build:
+          build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3
+          cuda_version: "10"
           filters:
             branches:
               only:
                 - master
                 - /ci-all\/.*/
                 - /release\/.*/
-      - pytorch_windows_build:
           name: pytorch_windows_vs2017_14.16_py36_cuda10.1_build
-          build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3
-          cuda_version: "10"
           python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
           requires:
             - setup
+          use_cuda: "1"
+          vc_product: BuildTools
+          vc_version: "14.16"
+          vc_year: "2017"
+      - pytorch_windows_test:
+          build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3
+          cuda_version: "10"
+          executor: windows-with-nvidia-gpu
           filters:
             branches:
               only:
                 - master
                 - /ci-all\/.*/
                 - /release\/.*/
-      - pytorch_windows_test:
           name: pytorch_windows_vs2017_14.16_py36_cuda10.1_test1
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "10"
           python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
           requires:
             - setup
             - pytorch_windows_vs2017_14.16_py36_cuda10.1_build
+          test_name: pytorch-windows-test1
+          use_cuda: "1"
+          vc_product: BuildTools
+          vc_version: "14.16"
+          vc_year: "2017"
+      - pytorch_windows_test:
+          build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3
+          cuda_version: "10"
+          executor: windows-with-nvidia-gpu
           filters:
             branches:
               only:
                 - master
                 - /ci-all\/.*/
                 - /release\/.*/
-      - pytorch_windows_test:
           name: pytorch_windows_vs2017_14.16_py36_cuda10.1_test2
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "10"
           python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
           requires:
             - setup
             - pytorch_windows_vs2017_14.16_py36_cuda10.1_build
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
+          test_name: pytorch-windows-test2
+          use_cuda: "1"
+          vc_product: BuildTools
+          vc_version: "14.16"
+          vc_year: "2017"
       - pytorch_windows_build:
+          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
+          cuda_version: "10"
           name: pytorch_windows_vs2019_py36_cuda10.1_build
-          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
-          cuda_version: "10"
           python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "1"
           requires:
             - setup
+          use_cuda: "1"
+          vc_product: Community
+          vc_version: ""
+          vc_year: "2019"
       - pytorch_windows_test:
+          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
+          cuda_version: "10"
+          executor: windows-with-nvidia-gpu
           name: pytorch_windows_vs2019_py36_cuda10.1_test1
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "10"
           python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "1"
           requires:
             - setup
             - pytorch_windows_vs2019_py36_cuda10.1_build
+          test_name: pytorch-windows-test1
+          use_cuda: "1"
+          vc_product: Community
+          vc_version: ""
+          vc_year: "2019"
       - pytorch_windows_test:
+          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
+          cuda_version: "10"
+          executor: windows-with-nvidia-gpu
           name: pytorch_windows_vs2019_py36_cuda10.1_test2
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "10"
           python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
+          requires:
+            - setup
+            - pytorch_windows_vs2019_py36_cuda10.1_build
+          test_name: pytorch-windows-test2
           use_cuda: "1"
-          requires:
-            - setup
-            - pytorch_windows_vs2019_py36_cuda10.1_build
+          vc_product: Community
+          vc_version: ""
+          vc_year: "2019"
       - pytorch_windows_build:
+          build_environment: pytorch-win-vs2017-14-11-cpu-py3
+          cuda_version: cpu
+          filters:
+            branches:
+              only:
+                - master
+                - /ci-all\/.*/
+                - /release\/.*/
           name: pytorch_windows_vs2017_14.11_py36_cpu_build
-          build_environment: pytorch-win-vs2017-14-11-cpu-py3
-          cuda_version: "cpu"
           python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
           requires:
             - setup
+          use_cuda: "0"
+          vc_product: BuildTools
+          vc_version: "14.11"
+          vc_year: "2017"
+      - pytorch_windows_test:
+          build_environment: pytorch-win-vs2017-14-11-cpu-py3
+          cuda_version: cpu
           filters:
             branches:
               only:
                 - master
                 - /ci-all\/.*/
                 - /release\/.*/
-      - pytorch_windows_test:
           name: pytorch_windows_vs2017_14.11_py36_cpu_test1
-          build_environment: pytorch-win-vs2017-14-11-cpu-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "cpu"
           python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
           requires:
             - setup
             - pytorch_windows_vs2017_14.11_py36_cpu_build
+          test_name: pytorch-windows-test1
+          use_cuda: "0"
+          vc_product: BuildTools
+          vc_version: "14.11"
+          vc_year: "2017"
+      - pytorch_windows_test:
+          build_environment: pytorch-win-vs2017-14-11-cpu-py3
+          cuda_version: cpu
           filters:
             branches:
               only:
                 - master
                 - /ci-all\/.*/
                 - /release\/.*/
-      - pytorch_windows_test:
           name: pytorch_windows_vs2017_14.11_py36_cpu_test2
-          build_environment: pytorch-win-vs2017-14-11-cpu-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "cpu"
           python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
           requires:
             - setup
             - pytorch_windows_vs2017_14.11_py36_cpu_build
+          test_name: pytorch-windows-test2
+          use_cuda: "0"
+          vc_product: BuildTools
+          vc_version: "14.11"
+          vc_year: "2017"
+      - pytorch_windows_build:
+          build_environment: pytorch-win-vs2017-14-16-cpu-py3
+          cuda_version: cpu
           filters:
             branches:
               only:
                 - master
                 - /ci-all\/.*/
                 - /release\/.*/
-      - pytorch_windows_build:
           name: pytorch_windows_vs2017_14.16_py36_cpu_build
-          build_environment: pytorch-win-vs2017-14-16-cpu-py3
-          cuda_version: "cpu"
           python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
           requires:
             - setup
+          use_cuda: "0"
+          vc_product: BuildTools
+          vc_version: "14.16"
+          vc_year: "2017"
+      - pytorch_windows_test:
+          build_environment: pytorch-win-vs2017-14-16-cpu-py3
+          cuda_version: cpu
           filters:
             branches:
               only:
                 - master
                 - /ci-all\/.*/
                 - /release\/.*/
-      - pytorch_windows_test:
           name: pytorch_windows_vs2017_14.16_py36_cpu_test1
-          build_environment: pytorch-win-vs2017-14-16-cpu-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "cpu"
           python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
           requires:
             - setup
             - pytorch_windows_vs2017_14.16_py36_cpu_build
+          test_name: pytorch-windows-test1
+          use_cuda: "0"
+          vc_product: BuildTools
+          vc_version: "14.16"
+          vc_year: "2017"
+      - pytorch_windows_test:
+          build_environment: pytorch-win-vs2017-14-16-cpu-py3
+          cuda_version: cpu
           filters:
             branches:
               only:
                 - master
                 - /ci-all\/.*/
                 - /release\/.*/
-      - pytorch_windows_test:
           name: pytorch_windows_vs2017_14.16_py36_cpu_test2
-          build_environment: pytorch-win-vs2017-14-16-cpu-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "cpu"
           python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
           requires:
             - setup
             - pytorch_windows_vs2017_14.16_py36_cpu_build
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
+          test_name: pytorch-windows-test2
+          use_cuda: "0"
+          vc_product: BuildTools
+          vc_version: "14.16"
+          vc_year: "2017"
       - pytorch_windows_build:
+          build_environment: pytorch-win-vs2019-cpu-py3
+          cuda_version: cpu
           name: pytorch_windows_vs2019_py36_cpu_build
-          build_environment: pytorch-win-vs2019-cpu-py3
-          cuda_version: "cpu"
           python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "0"
           requires:
             - setup
+          use_cuda: "0"
+          vc_product: Community
+          vc_version: ""
+          vc_year: "2019"
       - pytorch_windows_test:
+          build_environment: pytorch-win-vs2019-cpu-py3
+          cuda_version: cpu
           name: pytorch_windows_vs2019_py36_cpu_test1
-          build_environment: pytorch-win-vs2019-cpu-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "cpu"
           python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "0"
           requires:
             - setup
             - pytorch_windows_vs2019_py36_cpu_build
+          test_name: pytorch-windows-test1
+          use_cuda: "0"
+          vc_product: Community
+          vc_version: ""
+          vc_year: "2019"
       - pytorch_windows_test:
+          build_environment: pytorch-win-vs2019-cpu-py3
+          cuda_version: cpu
           name: pytorch_windows_vs2019_py36_cpu_test2
-          build_environment: pytorch-win-vs2019-cpu-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "cpu"
           python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "0"
           requires:
             - setup
             - pytorch_windows_vs2019_py36_cpu_build
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test1
-          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "0"
-          requires:
-            - setup
-            - pytorch_windows_vs2019_py36_cuda10.1_build
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test2
-          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
           test_name: pytorch-windows-test2
-          cuda_version: "10"
-          python_version: "3.6"
+          use_cuda: "0"
+          vc_product: Community
           vc_version: ""
           vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "0"
+      - pytorch_windows_test:
+          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
+          cuda_version: "10"
+          name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test1
+          python_version: "3.6"
           requires:
             - setup
             - pytorch_windows_vs2019_py36_cuda10.1_build
+          test_name: pytorch-windows-test1
+          use_cuda: "0"
+          vc_product: Community
+          vc_version: ""
+          vc_year: "2019"
+      - pytorch_windows_test:
+          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
+          cuda_version: "10"
+          name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test2
+          python_version: "3.6"
+          requires:
+            - setup
+            - pytorch_windows_vs2019_py36_cuda10.1_build
+          test_name: pytorch-windows-test2
+          use_cuda: "0"
+          vc_product: Community
+          vc_version: ""
+          vc_year: "2019"
       - pytorch_linux_build:
           name: pytorch_linux_xenial_pynightly_build
           requires:

.circleci/generate_config_yml.py

@@ -11,6 +11,7 @@ import shutil
 from collections import namedtuple, OrderedDict
 
 import cimodel.data.pytorch_build_definitions as pytorch_build_definitions
+import cimodel.data.windows_build_definitions as windows_build_definitions
 import cimodel.data.binary_build_definitions as binary_build_definitions
 import cimodel.data.caffe2_build_definitions as caffe2_build_definitions
 import cimodel.lib.miniutils as miniutils
@@ -94,7 +95,7 @@ YAML_SOURCES = [
     File("workflows.yml"),
 
     File("workflows-setup-job.yml"),
-    File("windows-build-test.yml"),
+    Listgen(windows_build_definitions.get_windows_workflows, 3),
     Listgen(pytorch_build_definitions.get_workflow_jobs, 3),
     File("workflows-pytorch-macos-builds.yml"),
     File("workflows-pytorch-android-gradle-build.yml"),

.circleci/normalize_yaml_fragment.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+import sys
+import yaml
+import cimodel.lib.miniyaml as miniyaml
+
+
+def regurgitate(depth, use_pyyaml_formatter=False):
+    data = yaml.safe_load(sys.stdin)
+
+    if use_pyyaml_formatter:
+        output = yaml.dump(data, sort_keys=True)
+        sys.stdout.write(output)
+    else:
+        miniyaml.render(sys.stdout, data, depth)
+
+
+if __name__ == "__main__":
+    regurgitate(3)

.circleci/verbatim-sources/windows-build-test.yml

@@ -1,328 +0,0 @@
-      # Warning: indentation here matters!
-
-      - pytorch_windows_build:
-          name: pytorch_windows_vs2017_14.11_py36_cuda10.1_build
-          build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
-          requires:
-            - setup
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2017_14.11_py36_cuda10.1_test1
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
-          requires:
-            - setup
-            - pytorch_windows_vs2017_14.11_py36_cuda10.1_build
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2017_14.11_py36_cuda10.1_test2
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2017-14-11-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
-          requires:
-            - setup
-            - pytorch_windows_vs2017_14.11_py36_cuda10.1_build
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_build:
-          name: pytorch_windows_vs2017_14.16_py36_cuda10.1_build
-          build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
-          requires:
-            - setup
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2017_14.16_py36_cuda10.1_test1
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
-          requires:
-            - setup
-            - pytorch_windows_vs2017_14.16_py36_cuda10.1_build
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2017_14.16_py36_cuda10.1_test2
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2017-14-16-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "1"
-          requires:
-            - setup
-            - pytorch_windows_vs2017_14.16_py36_cuda10.1_build
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_build:
-          name: pytorch_windows_vs2019_py36_cuda10.1_build
-          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "1"
-          requires:
-            - setup
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2019_py36_cuda10.1_test1
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "1"
-          requires:
-            - setup
-            - pytorch_windows_vs2019_py36_cuda10.1_build
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2019_py36_cuda10.1_test2
-          executor: windows-with-nvidia-gpu
-          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "1"
-          requires:
-            - setup
-            - pytorch_windows_vs2019_py36_cuda10.1_build
-      - pytorch_windows_build:
-          name: pytorch_windows_vs2017_14.11_py36_cpu_build
-          build_environment: pytorch-win-vs2017-14-11-cpu-py3
-          cuda_version: "cpu"
-          python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
-          requires:
-            - setup
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2017_14.11_py36_cpu_test1
-          build_environment: pytorch-win-vs2017-14-11-cpu-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "cpu"
-          python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
-          requires:
-            - setup
-            - pytorch_windows_vs2017_14.11_py36_cpu_build
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2017_14.11_py36_cpu_test2
-          build_environment: pytorch-win-vs2017-14-11-cpu-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "cpu"
-          python_version: "3.6"
-          vc_version: "14.11"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
-          requires:
-            - setup
-            - pytorch_windows_vs2017_14.11_py36_cpu_build
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_build:
-          name: pytorch_windows_vs2017_14.16_py36_cpu_build
-          build_environment: pytorch-win-vs2017-14-16-cpu-py3
-          cuda_version: "cpu"
-          python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
-          requires:
-            - setup
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2017_14.16_py36_cpu_test1
-          build_environment: pytorch-win-vs2017-14-16-cpu-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "cpu"
-          python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
-          requires:
-            - setup
-            - pytorch_windows_vs2017_14.16_py36_cpu_build
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2017_14.16_py36_cpu_test2
-          build_environment: pytorch-win-vs2017-14-16-cpu-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "cpu"
-          python_version: "3.6"
-          vc_version: "14.16"
-          vc_year: "2017"
-          vc_product: "BuildTools"
-          use_cuda: "0"
-          requires:
-            - setup
-            - pytorch_windows_vs2017_14.16_py36_cpu_build
-          filters:
-            branches:
-              only:
-                - master
-                - /ci-all\/.*/
-                - /release\/.*/
-      - pytorch_windows_build:
-          name: pytorch_windows_vs2019_py36_cpu_build
-          build_environment: pytorch-win-vs2019-cpu-py3
-          cuda_version: "cpu"
-          python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "0"
-          requires:
-            - setup
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2019_py36_cpu_test1
-          build_environment: pytorch-win-vs2019-cpu-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "cpu"
-          python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "0"
-          requires:
-            - setup
-            - pytorch_windows_vs2019_py36_cpu_build
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2019_py36_cpu_test2
-          build_environment: pytorch-win-vs2019-cpu-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "cpu"
-          python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "0"
-          requires:
-            - setup
-            - pytorch_windows_vs2019_py36_cpu_build
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test1
-          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test1
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "0"
-          requires:
-            - setup
-            - pytorch_windows_vs2019_py36_cuda10.1_build
-      - pytorch_windows_test:
-          name: pytorch_windows_vs2019_py36_cuda10.1_on_cpu_test2
-          build_environment: pytorch-win-vs2019-cuda10-cudnn7-py3
-          test_name: pytorch-windows-test2
-          cuda_version: "10"
-          python_version: "3.6"
-          vc_version: ""
-          vc_year: "2019"
-          vc_product: "Community"
-          use_cuda: "0"
-          requires:
-            - setup
-            - pytorch_windows_vs2019_py36_cuda10.1_build