Introduce BUILD_CAFFE2 flag (#43673)

Summary:
introduce BUILD_CAFFE2 flag. default to `ON`.

Pull Request resolved: https://github.com/pytorch/pytorch/pull/43673

Reviewed By: malfet

Differential Revision: D23381035

Pulled By: walterddr

fbshipit-source-id: 1f4582987fa0c4a911f0b18d311c04fdbf8dd8f0

.circleci/cimodel/data/pytorch_build_data.py

@@ -16,6 +16,7 @@ CONFIG_TREE_DATA = [
                     ("important", [X(True)]),
                     ("parallel_tbb", [X(True)]),
                     ("parallel_native", [X(True)]),
+                    ("pure_torch", [X(True)]),
                 ]),
             ]),
             # TODO: bring back libtorch test
@@ -153,10 +154,22 @@ class ExperimentalFeatureConfigNode(TreeConfigNode):
             "build_only": BuildOnlyConfigNode,
             "cuda_gcc_override": CudaGccOverrideConfigNode,
             "coverage": CoverageConfigNode,
+            "pure_torch": PureTorchConfigNode,
         }
         return next_nodes[experimental_feature]
 
 
+class PureTorchConfigNode(TreeConfigNode):
+    def modify_label(self, label):
+        return "PURE_TORCH=" + str(label)
+
+    def init2(self, node_name):
+        self.props["is_pure_torch"] = node_name
+
+    def child_constructor(self):
+        return ImportantConfigNode
+
+
 class XlaConfigNode(TreeConfigNode):
     def modify_label(self, label):
         return "XLA=" + str(label)

.circleci/cimodel/data/pytorch_build_definitions.py

@@ -23,6 +23,7 @@ class Conf:
     # (from https://github.com/pytorch/pytorch/pull/17323#discussion_r259453608)
     is_xla: bool = False
     is_vulkan: bool = False
+    is_pure_torch: bool = False
     restrict_phases: Optional[List[str]] = None
     gpu_resource: Optional[str] = None
     dependent_tests: List = field(default_factory=list)
@@ -50,6 +51,8 @@ class Conf:
             leading.append("vulkan")
         if self.is_libtorch and not for_docker:
             leading.append("libtorch")
+        if self.is_pure_torch and not for_docker:
+            leading.append("pure_torch")
         if self.parallel_backend is not None and not for_docker:
             leading.append(self.parallel_backend)
 
@@ -260,6 +263,7 @@ def instantiate_configs():
         compiler_version = fc.find_prop("compiler_version")
         is_xla = fc.find_prop("is_xla") or False
         is_asan = fc.find_prop("is_asan") or False
+        is_pure_torch = fc.find_prop("is_pure_torch") or False
         is_vulkan = fc.find_prop("is_vulkan") or False
         parms_list_ignored_for_docker_image = []
 
@@ -307,7 +311,8 @@ def instantiate_configs():
         parallel_backend = fc.find_prop("parallel_backend") or None
         build_only = fc.find_prop("build_only") or False
         is_coverage = fc.find_prop("is_coverage") or False
-        if build_only:
+        # TODO: fix pure_torch python test packaging issue.
+        if build_only or is_pure_torch:
             restrict_phases = ["build"]
         if is_coverage and restrict_phases is None:
             restrict_phases = ["build", "coverage_test"]
@@ -326,6 +331,7 @@ def instantiate_configs():
             rocm_version,
             is_xla,
             is_vulkan,
+            is_pure_torch,
             restrict_phases,
             gpu_resource,
             is_libtorch=is_libtorch,
@@ -341,6 +347,7 @@ def instantiate_configs():
             and cuda_version is None
             and parallel_backend is None
             and not is_vulkan
+            and not is_pure_torch
             and compiler_name == "gcc"
             and fc.find_prop("compiler_version") == "5.4"
         ):
@@ -354,6 +361,7 @@ def instantiate_configs():
             and compiler_version == "5.4"
             and not is_libtorch
             and not is_vulkan
+            and not is_pure_torch
             and parallel_backend is None
         ):
             bc_breaking_check = Conf(

.circleci/config.yml

@@ -480,6 +480,9 @@ jobs:
           if [[ "${DOCKER_IMAGE}" == *rocm* ]]; then
             export DOCKER_TAG="ab1632df-fa59-40e6-8c23-98e004f61148"
           fi
+          if [[ ${BUILD_ENVIRONMENT} == *"pure_torch"* ]]; then
+            echo 'BUILD_CAFFE2=OFF' >> "${BASH_ENV}"
+          fi
           if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
             echo 'ATEN_THREADING=TBB' >> "${BASH_ENV}"
             echo 'USE_TBB=1' >> "${BASH_ENV}"
@@ -6222,6 +6225,18 @@ workflows:
           build_environment: "pytorch-parallelnative-linux-xenial-py3.6-gcc5.4-test"
           docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
           resource_class: large
+      - pytorch_linux_build:
+          name: pytorch_pure_torch_linux_xenial_py3_6_gcc5_4_build
+          requires:
+            - "docker-pytorch-linux-xenial-py3.6-gcc5.4"
+          filters:
+            branches:
+              only:
+                - master
+                - /ci-all\/.*/
+                - /release\/.*/
+          build_environment: "pytorch-pure_torch-linux-xenial-py3.6-gcc5.4-build"
+          docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3.6-gcc5.4"
       - pytorch_linux_build:
           name: pytorch_linux_xenial_py3_6_gcc7_build
           requires:

.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml

@@ -19,6 +19,9 @@ jobs:
           if [[ "${DOCKER_IMAGE}" == *rocm* ]]; then
             export DOCKER_TAG="ab1632df-fa59-40e6-8c23-98e004f61148"
           fi
+          if [[ ${BUILD_ENVIRONMENT} == *"pure_torch"* ]]; then
+            echo 'BUILD_CAFFE2=OFF' >> "${BASH_ENV}"
+          fi
           if [[ ${BUILD_ENVIRONMENT} == *"paralleltbb"* ]]; then
             echo 'ATEN_THREADING=TBB' >> "${BASH_ENV}"
             echo 'USE_TBB=1' >> "${BASH_ENV}"

CMakeLists.txt

@@ -121,9 +121,14 @@ option(BUILD_BINARY "Build C++ binaries" OFF)
 option(BUILD_DOCS "Build Caffe2 documentation" OFF)
 option(BUILD_CUSTOM_PROTOBUF "Build and use Caffe2's own protobuf under third_party" ON)
 option(BUILD_PYTHON "Build Python binaries" ON)
-option(BUILD_CAFFE2_OPS "Build Caffe2 operators" ON)
+option(BUILD_CAFFE2 "Master flag to build Caffe2" ON)
+cmake_dependent_option(
+    BUILD_CAFFE2_OPS "Build Caffe2 operators" ON
+    "BUILD_CAFFE2" OFF)
+cmake_dependent_option(
+    BUILD_CAFFE2_MOBILE "Build libcaffe2 for mobile (deprecating)" OFF
+    "BUILD_CAFFE2" OFF)
 option(BUILD_SHARED_LIBS "Build libcaffe2.so" ON)
-option(BUILD_CAFFE2_MOBILE "Build libcaffe2 for mobile (deprecating)" OFF)
 cmake_dependent_option(
     CAFFE2_LINK_LOCAL_PROTOBUF "If set, build protobuf inside libcaffe2.so." ON
     "BUILD_SHARED_LIBS AND BUILD_CUSTOM_PROTOBUF" OFF)
@@ -774,7 +779,11 @@ else()
 endif()
 
 # ---[ Modules
-add_subdirectory(modules)
+# If master flag for buildling Caffe2 is disabled, we also disable the
+# build for Caffe2 related operator modules.
+if(BUILD_CAFFE2)
+  add_subdirectory(modules)
+endif()
 
 # ---[ Binaries
 # Binaries will be built after the Caffe2 main libraries and the modules

cmake/Summary.cmake

@@ -20,6 +20,8 @@ function(caffe2_print_configuration_summary)
 
   message(STATUS "  TORCH_VERSION         : ${TORCH_VERSION}")
   message(STATUS "  CAFFE2_VERSION        : ${CAFFE2_VERSION}")
+  message(STATUS "  BUILD_CAFFE2          : ${BUILD_CAFFE2}")
+  message(STATUS "  BUILD_CAFFE2_OPS      : ${BUILD_CAFFE2_OPS}")
   message(STATUS "  BUILD_CAFFE2_MOBILE   : ${BUILD_CAFFE2_MOBILE}")
   message(STATUS "  BUILD_BINARY          : ${BUILD_BINARY}")
   message(STATUS "  BUILD_CUSTOM_PROTOBUF : ${BUILD_CUSTOM_PROTOBUF}")
@@ -40,7 +42,6 @@ function(caffe2_print_configuration_summary)
     message(STATUS "    Python includes     : ${PYTHON_INCLUDE_DIRS}")
     message(STATUS "    Python site-packages: ${PYTHON_SITE_PACKAGES}")
   endif()
-  message(STATUS "  BUILD_CAFFE2_OPS      : ${BUILD_CAFFE2_OPS}")
   message(STATUS "  BUILD_SHARED_LIBS     : ${BUILD_SHARED_LIBS}")
   message(STATUS "  BUILD_TEST            : ${BUILD_TEST}")
   message(STATUS "  BUILD_JNI             : ${BUILD_JNI}")