#!/usr/bin/env python3 from pathlib import Path from typing import Any, Dict import jinja2 DOCKER_REGISTRY = "308535385114.dkr.ecr.us-east-1.amazonaws.com" GITHUB_DIR = Path(__file__).parent.parent # it would be nice to statically specify that build_environment must be # present, but currently Python has no easy way to do that # https://github.com/python/mypy/issues/4617 PyTorchWorkflow = Dict[str, Any] WINDOWS_CPU_TEST_RUNNER = "windows.4xlarge" WINDOWS_CUDA_TEST_RUNNER = "windows.8xlarge.nvidia.gpu" def PyTorchWindowsWorkflow( *, build_environment: str, test_runner_type: str, on_pull_request: bool = False ) -> PyTorchWorkflow: return { "build_environment": build_environment, "test_runner_type": test_runner_type, "on_pull_request": on_pull_request, } LINUX_CPU_TEST_RUNNER = "linux.2xlarge" LINUX_CUDA_TEST_RUNNER = "linux.8xlarge.nvidia.gpu" def PyTorchLinuxWorkflow( *, build_environment: str, docker_image_base: str, test_runner_type: str, on_pull_request: bool = False, enable_doc_jobs: bool = False, ) -> PyTorchWorkflow: return { "build_environment": build_environment, "docker_image_base": docker_image_base, "test_runner_type": test_runner_type, "on_pull_request": on_pull_request, "enable_doc_jobs": enable_doc_jobs, } def generate_workflow_file( *, workflow: PyTorchWorkflow, workflow_template: jinja2.Template, ) -> Path: output_file_path = GITHUB_DIR / f"workflows/{workflow['build_environment']}.yml" with open(output_file_path, "w") as output_file: GENERATED = "generated" output_file.writelines([f"# @{GENERATED} DO NOT EDIT MANUALLY\n"]) output_file.write(workflow_template.render(**workflow)) output_file.write("\n") return output_file_path WINDOWS_WORKFLOWS = [ PyTorchWindowsWorkflow( build_environment="pytorch-win-vs2019-cpu-py3", test_runner_type=WINDOWS_CPU_TEST_RUNNER, ) ] LINUX_WORKFLOWS = [ PyTorchLinuxWorkflow( build_environment="pytorch-linux-xenial-py3.6-gcc5.4", docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4", test_runner_type=LINUX_CPU_TEST_RUNNER, on_pull_request=True, enable_doc_jobs=True, ), # PyTorchLinuxWorkflow( # build_environment="pytorch-paralleltbb-linux-xenial-py3.6-gcc5.4", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-parallelnative-linux-xenial-py3.6-gcc5.4", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-pure_torch-linux-xenial-py3.6-gcc5.4", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-py3.6-gcc7", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc7", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-py3.6-clang5-asan", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-asan", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-py3.6-clang7-onnx", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang7-onnx", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), PyTorchLinuxWorkflow( build_environment="pytorch-linux-xenial-cuda10.2-cudnn7-py3.6-gcc7", docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7", test_runner_type=LINUX_CUDA_TEST_RUNNER, ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7", # test_runner_type=LINUX_CUDA_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-libtorch-linux-xenial-cuda11.1-cudnn8-py3.6-gcc7", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7", # test_runner_type=LINUX_CUDA_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-bionic-py3.6-clang9-noarch", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-xla-linux-bionic-py3.6-clang9", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-vulkan-linux-bionic-py3.6-clang9", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-bionic-py3.8-gcc9-coverage", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.8-gcc9", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-bionic-rocm3.9-py3.6", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-rocm3.9-py3.6", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-py3.6-clang5-android-ndk-r19c-x86_32", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-py3.6-clang5-android-ndk-r19c-x86_64", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-py3.6-clang5-android-ndk-r19c-arm-v7a", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-py3.6-clang5-android-ndk-r19c-arm-v8a", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-py3.6-clang5-mobile", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-asan", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-py3.6-clang5-mobile-custom-dynamic", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-py3.6-clang5-mobile-custom-static", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), # PyTorchLinuxWorkflow( # build_environment="pytorch-linux-xenial-py3.6-clang5-mobile-code-analysis", # docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c", # test_runner_type=LINUX_CPU_TEST_RUNNER, # ), ] if __name__ == "__main__": jinja_env = jinja2.Environment( variable_start_string="!{{", loader=jinja2.FileSystemLoader(str(GITHUB_DIR.joinpath("templates"))), ) template_and_workflows = [ (jinja_env.get_template("linux_ci_workflow.yml.in"), LINUX_WORKFLOWS), (jinja_env.get_template("windows_ci_workflow.yml.in"), WINDOWS_WORKFLOWS) ] for template, workflows in template_and_workflows: for workflow in workflows: print(generate_workflow_file(workflow=workflow, workflow_template=template))