mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-06 12:20:52 +01:00
7d410bc3c8
Summary: This is a commandeer of https://github.com/pytorch/pytorch/issues/54091. TODO: - [x] understand why the build is [failing](https://github.com/pytorch/pytorch/pull/55176/checks?check_run_id=2254742265) here when it was [succeeding](https://github.com/pytorch/pytorch/pull/54091/checks?check_run_id=2177844748) on https://github.com/pytorch/pytorch/issues/54091 - [x] fix the build failure - [x] fix the test failure(s) - [x] add CI check to generate YAML workflows from templates, similar to https://github.com/pytorch/pytorch/issues/55171 - [ ] uncomment the rest of the matrix Pull Request resolved: https://github.com/pytorch/pytorch/pull/55176 Reviewed By: walterddr Differential Revision: D27803529 Pulled By: seemethere fbshipit-source-id: 52a65ec8f7a83b929fed47f0bbdca544210ec9c2
165 lines
7.4 KiB
Python
Executable File
165 lines
7.4 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
from pathlib import Path
|
|
|
|
import jinja2
|
|
|
|
DOCKER_REGISTRY = "308535385114.dkr.ecr.us-east-1.amazonaws.com"
|
|
|
|
GITHUB_DIR = Path(__file__).parent.parent
|
|
|
|
CPU_TEST_RUNNER = "linux.2xlarge"
|
|
CUDA_TEST_RUNNER = "linux.8xlarge.nvidia.gpu"
|
|
|
|
|
|
class PyTorchLinuxWorkflow:
|
|
def __init__(self, build_environment: str, docker_image_base: str):
|
|
self.build_environment = build_environment
|
|
self.docker_image_base = docker_image_base
|
|
self.test_runner_type = CPU_TEST_RUNNER
|
|
if "cuda" in build_environment:
|
|
self.test_runner_type = CUDA_TEST_RUNNER
|
|
|
|
def generate_workflow_file(
|
|
self, workflow_template: jinja2.Template, jinja_env: jinja2.Environment
|
|
) -> Path:
|
|
output_file_path = GITHUB_DIR.joinpath(
|
|
f"workflows/{self.build_environment}.yml"
|
|
)
|
|
with open(output_file_path, "w") as output_file:
|
|
output_file.write(
|
|
workflow_template.render(
|
|
build_environment=self.build_environment,
|
|
docker_image_base=self.docker_image_base,
|
|
test_runner_type=self.test_runner_type
|
|
)
|
|
)
|
|
output_file.write('\n')
|
|
return output_file_path
|
|
|
|
|
|
WORKFLOWS = [
|
|
PyTorchLinuxWorkflow(
|
|
build_environment="pytorch-linux-xenial-py3.6-gcc5.4",
|
|
docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
|
|
),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-paralleltbb-linux-xenial-py3.6-gcc5.4",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-parallelnative-linux-xenial-py3.6-gcc5.4",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-pure_torch-linux-xenial-py3.6-gcc5.4",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc5.4",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3.6-gcc7",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3.6-gcc7",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-asan",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-asan",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang7-onnx",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang7-onnx",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-libtorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-cuda11.1-cudnn8-py3-gcc7",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-bionic-py3.6-clang9-noarch",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-xla-linux-bionic-py3.6-clang9",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-vulkan-linux-bionic-py3.6-clang9",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.6-clang9",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-bionic-py3.8-gcc9-coverage",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-py3.8-gcc9",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-bionic-rocm3.9-py3.6",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-bionic-rocm3.9-py3.6",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-mobile",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-asan",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-mobile-custom-dynamic",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-mobile-custom-static",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-mobile-code-analysis",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_32",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-x86_64",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v7a",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
|
|
# ),
|
|
# PyTorchLinuxWorkflow(
|
|
# build_environment="pytorch-linux-xenial-py3-clang5-android-ndk-r19c-arm-v8a",
|
|
# docker_image_base=f"{DOCKER_REGISTRY}/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c",
|
|
# ),
|
|
]
|
|
|
|
if __name__ == "__main__":
|
|
jinja_env = jinja2.Environment(
|
|
variable_start_string="!{{",
|
|
loader=jinja2.FileSystemLoader(str(GITHUB_DIR.joinpath("templates"))),
|
|
)
|
|
workflow_template = jinja_env.get_template("linux_ci_workflow.yml.in")
|
|
for workflow in WORKFLOWS:
|
|
print(
|
|
workflow.generate_workflow_file(
|
|
workflow_template=workflow_template,
|
|
jinja_env=jinja_env
|
|
)
|
|
)
|