Remove FULL_CAFFE2 flag (#11321)

Summary:
Continuing pjh5's work to remove FULL_CAFFE2 flag completely.

With these changes you'll be able to also do something like

```
NO_TEST=1 python setup.py build_deps
```
and this will skip building tests in caffe2, aten, and c10d. By default the tests are built.

cc mingzhe09088 Yangqing
Pull Request resolved: https://github.com/pytorch/pytorch/pull/11321

Reviewed By: mingzhe09088

Differential Revision: D9694950

Pulled By: orionr

fbshipit-source-id: ff5c4937a23d1a263378a196a5eda0cba98af0a8

.jenkins/caffe2/build.sh

@@ -226,7 +226,7 @@ else
     export MAX_JOBS=`expr $(nproc) - 1`
   fi
 
-  FULL_CAFFE2=1 python setup.py install --user
+  USE_OPENCV=1 BUILD_BINARY=1 python setup.py install --user
 
   # This is to save test binaries for testing
   cp -r torch/lib/tmp_install $INSTALL_PREFIX

CMakeLists.txt

@@ -56,7 +56,7 @@ include(CMakeDependentOption)
 option(BUILD_TORCH "Build Torch" OFF)
 option(ATEN_NO_TEST "Do not build ATen test binaries" OFF)
 option(BUILD_ATEN_MOBILE "Build ATen for Android and iOS" OFF)
-option(BUILD_BINARY "Build C++ binaries" ON)
+option(BUILD_BINARY "Build C++ binaries" OFF)
 option(BUILD_DOCS "Build Caffe2 documentation" OFF)
 option(BUILD_CUSTOM_PROTOBUF "Build and use Caffe2's own protobuf under third_party" ON)
 option(BUILD_PYTHON "Build Python binaries" ON)
@@ -115,7 +115,7 @@ option(USE_IDEEP "Use IDEEP interface in MKL BLAS" ON)
 option(USE_MKLML "Use MKLML interface in MKL BLAS" ON)
 option(USE_DISTRIBUTED "Use distributed" ON)
 cmake_dependent_option(
-    USE_MPI "Use MPI. Only available if USE_DISTRIBUTED is on." ON
+    USE_MPI "Use MPI for Caffe2. Only available if USE_DISTRIBUTED is on." OFF
     "USE_DISTRIBUTED" OFF)
 cmake_dependent_option(
     USE_GLOO "Use Gloo. Only available if USE_DISTRIBUTED is on." ON

caffe2/CMakeLists.txt

@@ -425,25 +425,12 @@ if (BUILD_TEST)
       endif()
     endforeach()
   endif()
-endif()
 
-set(__aten_test_dir "test/aten")
+  if (NOT USE_ROCM)
-if (NOT USE_ROCM)
+    set(__aten_test_dir "test/aten")
-  foreach(test_src ${ATen_CPU_TEST_SRCS})
+    foreach(test_src ${ATen_CPU_TEST_SRCS})
-    get_filename_component(test_name ${test_src} NAME_WE)
-    add_executable(${test_name} "${test_src}")
-    target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
-    target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
-    target_include_directories(${test_name} SYSTEM PRIVATE ${Caffe2_DEPENDENCY_INCLUDE})
-    target_link_libraries(${test_name} ${Caffe2_MAIN_LIBS})
-    add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
-    install(TARGETS ${test_name} DESTINATION ${__aten_test_dir})
-  endforeach()
-
-  if(USE_CUDA OR USE_ROCM)
-    foreach(test_src ${ATen_CUDA_TEST_SRCS})
       get_filename_component(test_name ${test_src} NAME_WE)
-      torch_cuda_based_add_executable(${test_name} "${test_src}")
+      add_executable(${test_name} "${test_src}")
       target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
       target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
       target_include_directories(${test_name} SYSTEM PRIVATE ${Caffe2_DEPENDENCY_INCLUDE})
@@ -451,6 +438,19 @@ if (NOT USE_ROCM)
       add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
       install(TARGETS ${test_name} DESTINATION ${__aten_test_dir})
     endforeach()
+
+    if(USE_CUDA OR USE_ROCM)
+      foreach(test_src ${ATen_CUDA_TEST_SRCS})
+        get_filename_component(test_name ${test_src} NAME_WE)
+        torch_cuda_based_add_executable(${test_name} "${test_src}")
+        target_include_directories(${test_name} PRIVATE $<INSTALL_INTERFACE:include>)
+        target_include_directories(${test_name} PRIVATE ${Caffe2_CPU_INCLUDE})
+        target_include_directories(${test_name} SYSTEM PRIVATE ${Caffe2_DEPENDENCY_INCLUDE})
+        target_link_libraries(${test_name} ${Caffe2_MAIN_LIBS})
+        add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
+        install(TARGETS ${test_name} DESTINATION ${__aten_test_dir})
+      endforeach()
+    endif()
   endif()
 endif()
 

caffe2/utils/Array.h

@@ -291,13 +291,13 @@ constexpr inline array<T, N-1> tail(const array<T, N>& arg) {
 
 namespace detail {
 template<class T, size_t N, size_t... I>
-constexpr inline array<T, N+1> prepend_(T head, const array<T, N>& tail, guts::index_sequence<I...>) {
+constexpr inline array<T, N+1> prepend_(T&& head, const array<T, N>& tail, guts::index_sequence<I...>) {
-  return {{std::move(head), get<I>(tail)...}};
+  return {{std::forward<T>(head), get<I>(tail)...}};
 }
 }
 template<class T, size_t N>
-constexpr inline array<T, N+1> prepend(T head, const array<T, N>& tail) {
+constexpr inline array<T, N+1> prepend(T&& head, const array<T, N>& tail) {
-  return detail::prepend_(std::move(head), tail, guts::make_index_sequence<N>());
+  return detail::prepend_(std::forward<T>(head), tail, guts::make_index_sequence<N>());
 }
 
 /**

caffe2/utils/Array_test.cpp

@@ -78,9 +78,11 @@ namespace test_tail {
     static_assert(array < int, 0 > {{}} == tail(array < int, 1 > {{3}}), "");
 }
 
-namespace test_prepend {
+TEST(ArrayTest, TestPrepend) {
-    static_assert(array < int, 3 > {{2, 3, 4}} == prepend(2, array < int, 2 > {{3, 4}}), "");
+  // Some compilers can't handle move results as constexpr, so use
-    static_assert(array < int, 1 > {{3}} == prepend(3, array < int, 0 > {{}}), "");
+  // gtest assert for this test
+  ASSERT_EQ((array<int, 3> {{2, 3, 4}}), (prepend(2, array<int, 2> {{3, 4}})));
+  ASSERT_EQ((array<int, 1> {{3}}), (prepend(3, array<int, 0> {{}})));
 }
 
 namespace test_to_std_array {

cmake/Dependencies.cmake

@@ -185,9 +185,13 @@ endif()
 
 # ---[ Googletest and benchmark
 if(BUILD_TEST)
+  # Preserve build options.
   set(TEMP_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+  set(TEMP_CMAKE_DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX})
+
   # We will build gtest as static libs and embed it directly into the binary.
-  set(BUILD_SHARED_LIBS OFF)
+  set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libs" FORCE)
+
   # For gtest, we will simply embed it into our test binaries, so we won't
   # need to install it.
   set(BUILD_GTEST ON)
@@ -208,8 +212,10 @@ if(BUILD_TEST)
   add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/benchmark)
   include_directories(${CMAKE_CURRENT_LIST_DIR}/../third_party/benchmark/include)
 
-  # Recover the build shared libs option.
+  # Recover build options. Unfortunately gtest modifies CMAKE_DEBUG_POSTFIX
-  set(BUILD_SHARED_LIBS ${TEMP_BUILD_SHARED_LIBS})
+  # in some versions as detailed at https://github.com/google/googletest/issues/1334
+  set(BUILD_SHARED_LIBS ${TEMP_BUILD_SHARED_LIBS} CACHE BOOL "Build shared libs" FORCE)
+  set(CMAKE_DEBUG_POSTFIX ${TEMP_CMAKE_DEBUG_POSTFIX} CACHE BOOL "Debug postfix" FORCE)
 endif()
 
 # ---[ LMDB

conda/integrated/build.sh

@@ -51,10 +51,10 @@ fi
 # Build Caffe2 and PyTorch
 ###########################################################
 if [[ "$OSTYPE" == "darwin"* ]]; then
-  FULL_CAFFE2=1 MACOSX_DEPLOYMENT_TARGET=10.9 python setup.py install
+  USE_OPENCV=1 BUILD_BINARY=1 MACOSX_DEPLOYMENT_TARGET=10.9 python setup.py install
   exit 0
 fi
-FULL_CAFFE2=1 python setup.py install
+USE_OPENCV=1 BUILD_BINARY=1 python setup.py install
 
 
 

scripts/onnx/install-develop.sh

@@ -15,4 +15,4 @@ pip install -e "$tp2_dir/onnx"
 # Install caffe2 and pytorch
 pip install -r "$top_dir/caffe2/requirements.txt"
 pip install -r "$top_dir/requirements.txt"
-FULL_CAFFE2=1 python setup.py build_deps develop 
+USE_OPENCV=1 BUILD_BINARY=1 python setup.py build_deps develop

scripts/onnx/install.sh

@@ -35,4 +35,4 @@ _pip_install -b "$BUILD_DIR/onnx" "file://$tp2_dir/onnx#egg=onnx"
 # Install caffe2 and pytorch
 pip install -r "$top_dir/caffe2/requirements.txt"
 pip install -r "$top_dir/requirements.txt"
-FULL_CAFFE2=1 python setup.py install
+USE_OPENCV=1 BUILD_BINARY=1 python setup.py install

setup.py

@@ -27,6 +27,9 @@
 #   NO_CUDNN
 #     disables the cuDNN build
 #
+#   NO_TEST
+#     disables the test build
+#
 #   NO_MIOPEN
 #     disables the MIOpen build
 #
@@ -46,6 +49,12 @@
 #   USE_GLOO_IBVERBS
 #     toggle features related to distributed support
 #
+#   USE_OPENCV
+#     enables use of OpenCV for additional operators
+#
+#   BUILD_BINARY
+#     enables the additional binaries/ build
+#
 #   PYTORCH_BUILD_VERSION
 #   PYTORCH_BUILD_NUMBER
 #     specify the version of PyTorch, rather than the hard-coded version
@@ -117,25 +126,29 @@ import importlib
 
 from tools.setup_helpers.env import check_env_flag, check_negative_env_flag
 
-# Before we run the setup_helpers, let's look for NO_* and WITH_*
-# variables and hotpatch the environment with the USE_* equivalent
-config_env_vars = ['CUDA', 'CUDNN', 'MIOPEN', 'MKLDNN', 'NNPACK', 'DISTRIBUTED',
-                   'SYSTEM_NCCL', 'GLOO_IBVERBS']
 
-
+def hotpatch_var(var, prefix='USE_'):
-def hotpatch_var(var):
     if check_env_flag('NO_' + var):
-        os.environ['USE_' + var] = '0'
+        os.environ[prefix + var] = '0'
     elif check_negative_env_flag('NO_' + var):
-        os.environ['USE_' + var] = '1'
+        os.environ[prefix + var] = '1'
     elif check_env_flag('WITH_' + var):
-        os.environ['USE_' + var] = '1'
+        os.environ[prefix + var] = '1'
     elif check_negative_env_flag('WITH_' + var):
-        os.environ['USE_' + var] = '0'
+        os.environ[prefix + var] = '0'
 
-list(map(hotpatch_var, config_env_vars))
+# Before we run the setup_helpers, let's look for NO_* and WITH_*
+# variables and hotpatch environment with the USE_* equivalent
+use_env_vars = ['CUDA', 'CUDNN', 'MIOPEN', 'MKLDNN', 'NNPACK', 'DISTRIBUTED',
+                'OPENCV', 'SYSTEM_NCCL', 'GLOO_IBVERBS']
+list(map(hotpatch_var, use_env_vars))
+
+# Also hotpatch a few with BUILD_* equivalent
+build_env_vars = ['BINARY', 'TEST']
+[hotpatch_var(v, 'BUILD_') for v in build_env_vars]
 
 from tools.setup_helpers.cuda import USE_CUDA, CUDA_HOME, CUDA_VERSION
+from tools.setup_helpers.build import BUILD_BINARY, BUILD_TEST, USE_OPENCV
 from tools.setup_helpers.rocm import USE_ROCM, ROCM_HOME, ROCM_VERSION
 from tools.setup_helpers.cudnn import (USE_CUDNN, CUDNN_LIBRARY,
                                        CUDNN_LIB_DIR, CUDNN_INCLUDE_DIR)
@@ -161,7 +174,6 @@ IS_WINDOWS = (platform.system() == 'Windows')
 IS_DARWIN = (platform.system() == 'Darwin')
 IS_LINUX = (platform.system() == 'Linux')
 
-FULL_CAFFE2 = check_env_flag('FULL_CAFFE2')
 BUILD_PYTORCH = check_env_flag('BUILD_PYTORCH')
 USE_CUDA_STATIC_LINK = check_env_flag('USE_CUDA_STATIC_LINK')
 
@@ -359,10 +371,13 @@ def build_libs(libs):
         build_libs_cmd += ['--use-mkldnn']
     if USE_GLOO_IBVERBS:
         build_libs_cmd += ['--use-gloo-ibverbs']
-    if FULL_CAFFE2:
-        build_libs_cmd += ['--full-caffe2']
 
     my_env["BUILD_TORCH"] = "ON"
+    my_env["BUILD_PYTHON"] = "ON"
+    my_env["BUILD_BINARY"] = "ON" if BUILD_BINARY else "OFF"
+    my_env["BUILD_TEST"] = "ON" if BUILD_TEST else "OFF"
+    my_env["INSTALL_TEST"] = "ON" if BUILD_TEST else "OFF"
+    my_env["USE_OPENCV"] = "ON" if USE_OPENCV else "OFF"
 
     try:
         os.mkdir('build')
@@ -1073,11 +1088,12 @@ extensions.append(
         name=str('caffe2.python.caffe2_pybind11_state'),
         sources=[]),
 )
-extensions.append(
+if USE_CUDA:
-    setuptools.Extension(
+    extensions.append(
-        name=str('caffe2.python.caffe2_pybind11_state_gpu'),
+        setuptools.Extension(
-        sources=[]),
+            name=str('caffe2.python.caffe2_pybind11_state_gpu'),
-)
+            sources=[]),
+    )
 
 cmdclass = {
     'create_version_file': create_version_file,

tools/build_pytorch_libs.bat

@@ -63,8 +63,9 @@ IF "%REL_WITH_DEB_INFO%"=="1" (
   set BUILD_TYPE=RelWithDebInfo
 )
 
+:: sccache will fail if all cores are used for compiling
 IF NOT DEFINED MAX_JOBS (
-  set MAX_JOBS=%NUMBER_OF_PROCESSORS%
+  set /a MAX_JOBS=%NUMBER_OF_PROCESSORS% - 1
 )
 
 IF NOT DEFINED BUILD_SHARED_LIBS (
@@ -151,6 +152,8 @@ goto:eof
                   -DTHNN_SO_VERSION=1 ^
                   -DTHCUNN_SO_VERSION=1 ^
                   -DUSE_CUDA=%USE_CUDA% ^
+                  -DBUILD_EXAMPLES=OFF ^
+                  -DBUILD_TEST=%BUILD_TEST% ^
                   -DNO_NNPACK=%NO_NNPACK% ^
                   -DCMAKE_BUILD_TYPE=%BUILD_TYPE%
 
@@ -174,11 +177,14 @@ goto:eof
                   -DNO_API=ON ^
                   -DBUILD_SHARED_LIBS="%BUILD_SHARED_LIBS%" ^
                   -DBUILD_PYTHON=OFF ^
-                  -DBUILD_BINARY=OFF ^
+                  -DBUILD_BINARY=%BUILD_BINARY% ^
+                  -DBUILD_TEST=OFF ^
+                  -DINSTALL_TEST=%INSTALL_TEST% ^
                   -DONNX_NAMESPACE=%ONNX_NAMESPACE% ^
                   -DUSE_CUDA=%USE_CUDA% ^
                   -DUSE_CUDNN=OFF ^
                   -DUSE_NNPACK=%USE_NNPACK% ^
+                  -DUSE_OPENCV=%USE_OPENCV% ^
                   -DUSE_GLOG=OFF ^
                   -DUSE_GFLAGS=OFF ^
                   -DUSE_SYSTEM_EIGEN_INSTALL=OFF ^

tools/build_pytorch_libs.sh

@@ -16,7 +16,6 @@ USE_ROCM=0
 USE_NNPACK=0
 USE_MKLDNN=0
 USE_GLOO_IBVERBS=0
-FULL_CAFFE2=0
 CAFFE2_STATIC_LINK_CUDA=0
 while [[ $# -gt 0 ]]; do
     case "$1" in
@@ -35,9 +34,6 @@ while [[ $# -gt 0 ]]; do
       --use-gloo-ibverbs)
           USE_GLOO_IBVERBS=1
           ;;
-      --full-caffe2)
-          FULL_CAFFE2=1
-          ;;
       --cuda-static-link)
           CAFFE2_STATIC_LINK_CUDA=1
           ;;
@@ -183,6 +179,8 @@ function build() {
               -DTHCUNN_SO_VERSION=1 \
               -DTHD_SO_VERSION=1 \
               -DUSE_CUDA=$USE_CUDA \
+              -DBUILD_EXAMPLES=OFF \
+              -DBUILD_TEST=$BUILD_TEST \
               -DNO_NNPACK=$((1-$USE_NNPACK)) \
               -DNCCL_EXTERNAL=1 \
               -DCMAKE_DEBUG_POSTFIX="" \
@@ -260,16 +258,17 @@ function build_caffe2() {
       -DBUILDING_WITH_TORCH_LIBS=ON \
       -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
       -DBUILD_TORCH=$BUILD_TORCH \
-      -DBUILD_PYTHON=ON \
+      -DBUILD_PYTHON=$BUILD_PYTHON \
       -DBUILD_SHARED_LIBS=$BUILD_SHARED_LIBS \
-      -DBUILD_BINARY=$FULL_CAFFE2 \
+      -DBUILD_BINARY=$BUILD_BINARY \
-      -DBUILD_TEST=$FULL_CAFFE2 \
+      -DBUILD_TEST=$BUILD_TEST \
-      -DINSTALL_TEST=$FULL_CAFFE2 \
+      -DINSTALL_TEST=$INSTALL_TEST \
       -DONNX_NAMESPACE=$ONNX_NAMESPACE \
       -DUSE_CUDA=$USE_CUDA \
       -DCAFFE2_STATIC_LINK_CUDA=$CAFFE2_STATIC_LINK_CUDA \
       -DUSE_ROCM=$USE_ROCM \
       -DUSE_NNPACK=$USE_NNPACK \
+      -DUSE_OPENCV=$USE_OPENCV \
       -DUSE_GLOG=OFF \
       -DUSE_GFLAGS=OFF \
       -DUSE_SYSTEM_EIGEN_INSTALL=OFF \

tools/setup_helpers/build.py

@@ -0,0 +1,5 @@
+from .env import check_env_flag, check_negative_env_flag
+
+BUILD_BINARY = check_env_flag('BUILD_BINARY')
+BUILD_TEST = not check_negative_env_flag('BUILD_TEST')
+USE_OPENCV = check_env_flag('USE_OPENCV')