From 8ca3913f47ecc403de785635834ef8794fa9aa07 Mon Sep 17 00:00:00 2001 From: Rong Rong Date: Tue, 1 Sep 2020 10:16:46 -0700 Subject: [PATCH] Introduce BUILD_CAFFE2 flag (#43673) Summary: introduce BUILD_CAFFE2 flag. default to `ON`. Pull Request resolved: https://github.com/pytorch/pytorch/pull/43673 Reviewed By: malfet Differential Revision: D23381035 Pulled By: walterddr fbshipit-source-id: 1f4582987fa0c4a911f0b18d311c04fdbf8dd8f0 --- .circleci/cimodel/data/pytorch_build_data.py | 13 +++++++++++++ .../cimodel/data/pytorch_build_definitions.py | 10 +++++++++- .circleci/config.yml | 15 +++++++++++++++ .../job-specs/pytorch-job-specs.yml | 3 +++ CMakeLists.txt | 15 ++++++++++++--- cmake/Summary.cmake | 3 ++- 6 files changed, 54 insertions(+), 5 deletions(-) diff --git a/.circleci/cimodel/data/pytorch_build_data.py b/.circleci/cimodel/data/pytorch_build_data.py index 81479d98120..5234bf17f30 100644 --- a/.circleci/cimodel/data/pytorch_build_data.py +++ b/.circleci/cimodel/data/pytorch_build_data.py @@ -16,6 +16,7 @@ CONFIG_TREE_DATA = [ ("important", [X(True)]), ("parallel_tbb", [X(True)]), ("parallel_native", [X(True)]), + ("pure_torch", [X(True)]), ]), ]), # TODO: bring back libtorch test @@ -153,10 +154,22 @@ class ExperimentalFeatureConfigNode(TreeConfigNode): "build_only": BuildOnlyConfigNode, "cuda_gcc_override": CudaGccOverrideConfigNode, "coverage": CoverageConfigNode, + "pure_torch": PureTorchConfigNode, } return next_nodes[experimental_feature] +class PureTorchConfigNode(TreeConfigNode): + def modify_label(self, label): + return "PURE_TORCH=" + str(label) + + def init2(self, node_name): + self.props["is_pure_torch"] = node_name + + def child_constructor(self): + return ImportantConfigNode + + class XlaConfigNode(TreeConfigNode): def modify_label(self, label): return "XLA=" + str(label) diff --git a/.circleci/cimodel/data/pytorch_build_definitions.py b/.circleci/cimodel/data/pytorch_build_definitions.py index 0df2c8ed920..254379b6f45 100644 --- a/.circleci/cimodel/data/pytorch_build_definitions.py +++ b/.circleci/cimodel/data/pytorch_build_definitions.py @@ -23,6 +23,7 @@ class Conf: # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453608) is_xla: bool = False is_vulkan: bool = False + is_pure_torch: bool = False restrict_phases: Optional[List[str]] = None gpu_resource: Optional[str] = None dependent_tests: List = field(default_factory=list) @@ -50,6 +51,8 @@ class Conf: leading.append("vulkan") if self.is_libtorch and not for_docker: leading.append("libtorch") + if self.is_pure_torch and not for_docker: + leading.append("pure_torch") if self.parallel_backend is not None and not for_docker: leading.append(self.parallel_backend) @@ -260,6 +263,7 @@ def instantiate_configs(): compiler_version = fc.find_prop("compiler_version") is_xla = fc.find_prop("is_xla") or False is_asan = fc.find_prop("is_asan") or False + is_pure_torch = fc.find_prop("is_pure_torch") or False is_vulkan = fc.find_prop("is_vulkan") or False parms_list_ignored_for_docker_image = [] @@ -307,7 +311,8 @@ def instantiate_configs(): parallel_backend = fc.find_prop("parallel_backend") or None build_only = fc.find_prop("build_only") or False is_coverage = fc.find_prop("is_coverage") or False - if build_only: + # TODO: fix pure_torch python test packaging issue. + if build_only or is_pure_torch: restrict_phases = ["build"] if is_coverage and restrict_phases is None: restrict_phases = ["build", "coverage_test"] @@ -326,6 +331,7 @@ def instantiate_configs(): rocm_version, is_xla, is_vulkan, + is_pure_torch, restrict_phases, gpu_resource, is_libtorch=is_libtorch, @@ -341,6 +347,7 @@ def instantiate_configs(): and cuda_version is None and parallel_backend is None and not is_vulkan + and not is_pure_torch and compiler_name == "gcc" and fc.find_prop("compiler_version") == "5.4" ): @@ -354,6 +361,7 @@ def instantiate_configs(): and compiler_version == "5.4" and not is_libtorch and not is_vulkan + and not is_pure_torch and parallel_backend is None ): bc_breaking_check = Conf( diff --git a/.circleci/config.yml b/.circleci/config.yml index b9be825f5d0..c80803de59b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -480,6 +480,9 @@ jobs: if [[ "${DOCKER_IMAGE}" == *rocm* ]]; then export DOCKER_TAG="ab1632df-fa59-40e6-8c23-98e004f61148" fi + if [[ ${BUILD_ENVIRONMENT} == *"pure_torch"* ]]; then + echo 'BUILD_CAFFE2=OFF' >> "${BASH_ENV}" + fi if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then echo 'ATEN_THREADING=TBB' >> "${BASH_ENV}" echo 'USE_TBB=1' >> "${BASH_ENV}" @@ -6222,6 +6225,18 @@ workflows: build_environment: "pytorch-parallelnative-linux-xenial-py3.6-gcc5.4-test" docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4" resource_class: large + - pytorch_linux_build: + name: pytorch_pure_torch_linux_xenial_py3_6_gcc5_4_build + requires: + - "docker-pytorch-linux-xenial-py3.6-gcc5.4" + filters: + branches: + only: + - master + - /ci-all\/.*/ + - /release\/.*/ + build_environment: "pytorch-pure_torch-linux-xenial-py3.6-gcc5.4-build" + docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4" - pytorch_linux_build: name: pytorch_linux_xenial_py3_6_gcc7_build requires: diff --git a/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml b/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml index 96d33b5d08c..0d0ec262db3 100644 --- a/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml +++ b/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml @@ -19,6 +19,9 @@ jobs: if [[ "${DOCKER_IMAGE}" == *rocm* ]]; then export DOCKER_TAG="ab1632df-fa59-40e6-8c23-98e004f61148" fi + if [[ ${BUILD_ENVIRONMENT} == *"pure_torch"* ]]; then + echo 'BUILD_CAFFE2=OFF' >> "${BASH_ENV}" + fi if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then echo 'ATEN_THREADING=TBB' >> "${BASH_ENV}" echo 'USE_TBB=1' >> "${BASH_ENV}" diff --git a/CMakeLists.txt b/CMakeLists.txt index fb7b306547b..eadef572423 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,9 +121,14 @@ option(BUILD_BINARY "Build C++ binaries" OFF) option(BUILD_DOCS "Build Caffe2 documentation" OFF) option(BUILD_CUSTOM_PROTOBUF "Build and use Caffe2's own protobuf under third_party" ON) option(BUILD_PYTHON "Build Python binaries" ON) -option(BUILD_CAFFE2_OPS "Build Caffe2 operators" ON) +option(BUILD_CAFFE2 "Master flag to build Caffe2" ON) +cmake_dependent_option( + BUILD_CAFFE2_OPS "Build Caffe2 operators" ON + "BUILD_CAFFE2" OFF) +cmake_dependent_option( + BUILD_CAFFE2_MOBILE "Build libcaffe2 for mobile (deprecating)" OFF + "BUILD_CAFFE2" OFF) option(BUILD_SHARED_LIBS "Build libcaffe2.so" ON) -option(BUILD_CAFFE2_MOBILE "Build libcaffe2 for mobile (deprecating)" OFF) cmake_dependent_option( CAFFE2_LINK_LOCAL_PROTOBUF "If set, build protobuf inside libcaffe2.so." ON "BUILD_SHARED_LIBS AND BUILD_CUSTOM_PROTOBUF" OFF) @@ -774,7 +779,11 @@ else() endif() # ---[ Modules -add_subdirectory(modules) +# If master flag for buildling Caffe2 is disabled, we also disable the +# build for Caffe2 related operator modules. +if(BUILD_CAFFE2) + add_subdirectory(modules) +endif() # ---[ Binaries # Binaries will be built after the Caffe2 main libraries and the modules diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake index 6cd9429b43e..16fc3cd16b0 100644 --- a/cmake/Summary.cmake +++ b/cmake/Summary.cmake @@ -20,6 +20,8 @@ function(caffe2_print_configuration_summary) message(STATUS " TORCH_VERSION : ${TORCH_VERSION}") message(STATUS " CAFFE2_VERSION : ${CAFFE2_VERSION}") + message(STATUS " BUILD_CAFFE2 : ${BUILD_CAFFE2}") + message(STATUS " BUILD_CAFFE2_OPS : ${BUILD_CAFFE2_OPS}") message(STATUS " BUILD_CAFFE2_MOBILE : ${BUILD_CAFFE2_MOBILE}") message(STATUS " BUILD_BINARY : ${BUILD_BINARY}") message(STATUS " BUILD_CUSTOM_PROTOBUF : ${BUILD_CUSTOM_PROTOBUF}") @@ -40,7 +42,6 @@ function(caffe2_print_configuration_summary) message(STATUS " Python includes : ${PYTHON_INCLUDE_DIRS}") message(STATUS " Python site-packages: ${PYTHON_SITE_PACKAGES}") endif() - message(STATUS " BUILD_CAFFE2_OPS : ${BUILD_CAFFE2_OPS}") message(STATUS " BUILD_SHARED_LIBS : ${BUILD_SHARED_LIBS}") message(STATUS " BUILD_TEST : ${BUILD_TEST}") message(STATUS " BUILD_JNI : ${BUILD_JNI}")