Reorganize third-party libraries into top-level third_party directory (#6025)

- gloo, pybind11, nanopb and nccl now live in third_party.
- ATen builds in aten/build rather than torch/lib/build/aten
- A bit of faffing about in the scripts was necessary, because they used to assume that everything lived in the same directory. Now you are expected to cd into the correct directory before calling one of the build functions. The actual builder script lives in tools
- Lint now just unconditionally ignores third_party, rather than enumerating folders explicitly

.gitmodules

@@ -1,11 +1,11 @@
 [submodule "torch/lib/gloo"]
-	path = torch/lib/gloo
+	path = third_party/gloo
 	url = https://github.com/facebookincubator/gloo
 [submodule "torch/lib/pybind11"]
-	path = torch/lib/pybind11
+	path = third_party/pybind11
 	url = https://github.com/pybind/pybind11
 [submodule "torch/lib/nanopb"]
-	path = torch/lib/nanopb
+	path = third_party/nanopb
 	url = https://github.com/nanopb/nanopb.git
 [submodule "aten/src/ATen/cpu/cpuinfo"]
 	path = aten/src/ATen/cpu/cpuinfo

aten/src/README.md

@@ -1,5 +1,7 @@
-This directory contains the low-level tensor libraries for PyTorch.
-This code traces its lineage from the original Torch.  There are
+This directory contains the low-level tensor libraries for PyTorch,
+as well as the new ATen C++ bindings.
+
+The low-level libraries trace their lineage from the original Torch.  There are
 multiple variants of the library, summarized here:
 
 * TH = TorcH

aten/tools/subtree_dir_add.sh

@@ -1,17 +0,0 @@
-SRC_BRANCH=$1
-SRC_PATH=$2
-DST_PATH=$3
-
-CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-set -x
-git branch -q -D temporary-split-branch
-git checkout $SRC_BRANCH
-git subtree split -P $SRC_PATH -b temporary-split-branch
-git checkout $CURRENT_BRANCH
-git subtree add -P $DST_PATH temporary-split-branch ${@:5}
-git branch -D temporary-split-branch
-
-#./subtree_dir.sh add pytorch/master torch/lib/TH lib/TH
-#./subtree_dir.sh add pytorch/master torch/lib/THC lib/THC
-#./subtree_dir.sh add pytorch/master torch/lib/THNN lib/THNN
-#./subtree_dir.sh add pytorch/master torch/lib/THCUNN lib/THCUNN

aten/tools/subtree_dir_merge.sh

@@ -1,17 +0,0 @@
-SRC_BRANCH=$1
-SRC_PATH=$2
-DST_PATH=$3
-
-CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-set -x
-git branch -q -D temporary-split-branch
-git checkout $SRC_BRANCH
-git subtree split -P $SRC_PATH -b temporary-split-branch
-git checkout $CURRENT_BRANCH
-git subtree merge -P $DST_PATH temporary-split-branch -m "Merge commit '`git rev-parse temporary-split-branch`'"
-git branch -D temporary-split-branch
-
-#./subtree_dir.sh merge pytorch/master torch/lib/TH lib/TH
-#./subtree_dir.sh merge pytorch/master torch/lib/THC lib/THC
-#./subtree_dir.sh merge pytorch/master torch/lib/THNN lib/THNN
-#./subtree_dir.sh merge pytorch/master torch/lib/THCUNN lib/THCUNN

aten/tools/subtree_dir_merge_all.sh

@@ -1,10 +0,0 @@
-set -e
-set -x
-DIR=$(dirname $0)
-git fetch pytorch
-$DIR/subtree_dir_merge.sh pytorch/master torch/lib/TH lib/TH
-$DIR/subtree_dir_merge.sh pytorch/master torch/lib/THC lib/THC
-$DIR/subtree_dir_merge.sh pytorch/master torch/lib/THNN lib/THNN
-$DIR/subtree_dir_merge.sh pytorch/master torch/lib/THCUNN lib/THCUNN
-$DIR/subtree_dir_merge.sh pytorch/master torch/lib/THS lib/THS
-$DIR/subtree_dir_merge.sh pytorch/master torch/lib/THCS lib/THCS

setup.py

@@ -35,7 +35,7 @@
 #
 #   NO_SYSTEM_NCCL
 #     disables use of system-wide nccl (we will use our submoduled
-#     copy in torch/lib/nccl)
+#     copy in third_party/nccl)
 #
 #   WITH_GLOO_IBVERBS
 #     toggle features related to distributed support
@@ -190,9 +190,9 @@ def build_libs(libs):
     for lib in libs:
         assert lib in dep_libs, 'invalid lib: {}'.format(lib)
     if IS_WINDOWS:
-        build_libs_cmd = ['torch\\lib\\build_libs.bat']
+        build_libs_cmd = ['tools\\build_pytorch_libs.bat']
     else:
-        build_libs_cmd = ['bash', 'torch/lib/build_libs.sh']
+        build_libs_cmd = ['bash', 'tools/build_pytorch_libs.sh']
     my_env = os.environ.copy()
     my_env["PYTORCH_PYTHON"] = sys.executable
     my_env["NUM_JOBS"] = str(NUM_JOBS)
@@ -250,9 +250,9 @@ class build_deps(Command):
                 print("Could not find {}".format(f))
                 print("Did you run 'git submodule update --init'?")
                 sys.exit(1)
-        check_file(os.path.join(lib_path, "gloo", "CMakeLists.txt"))
-        check_file(os.path.join(lib_path, "nanopb", "CMakeLists.txt"))
-        check_file(os.path.join(lib_path, "pybind11", "CMakeLists.txt"))
+        check_file(os.path.join(third_party_path, "gloo", "CMakeLists.txt"))
+        check_file(os.path.join(third_party_path, "nanopb", "CMakeLists.txt"))
+        check_file(os.path.join(third_party_path, "pybind11", "CMakeLists.txt"))
         check_file(os.path.join('aten', 'src', 'ATen', 'cpu', 'cpuinfo', 'CMakeLists.txt'))
         check_file(os.path.join('aten', 'src', 'ATen', 'cpu', 'tbb', 'tbb_remote', 'Makefile'))
         check_file(os.path.join('aten', 'src', 'ATen', 'utils', 'catch', 'CMakeLists.txt'))
@@ -283,7 +283,7 @@ class build_deps(Command):
         # More information can be found in conversation thread of PR #5772
 
         self.copy_tree('torch/csrc', 'torch/lib/include/torch/csrc/')
-        self.copy_tree('torch/lib/pybind11/include/pybind11/',
+        self.copy_tree('third_party/pybind11/include/pybind11/',
                        'torch/lib/include/pybind11')
         self.copy_file('torch/torch.h', 'torch/lib/include/torch/torch.h')
 
@@ -487,13 +487,14 @@ else:
 
 cwd = os.path.dirname(os.path.abspath(__file__))
 lib_path = os.path.join(cwd, "torch", "lib")
+third_party_path = os.path.join(cwd, "third_party")
 
 
 tmp_install_path = lib_path + "/tmp_install"
 include_dirs += [
     cwd,
     os.path.join(cwd, "torch", "csrc"),
-    lib_path + "/pybind11/include",
+    third_party_path + "/pybind11/include",
     tmp_install_path + "/include",
     tmp_install_path + "/include/TH",
     tmp_install_path + "/include/THNN",

third_party/README.md

@@ -0,0 +1,2 @@
+This folder contains vendored copies of third-party libraries that we
+use.

tools/build_pytorch_libs.bat

@@ -1,11 +1,10 @@
-@echo off
-cd "%~dp0"
-cd "../.."
+:: @echo off
+cd "%~dp0/.."
 
 set BASE_DIR=%cd:\=/%
-cd torch/lib
-
-set INSTALL_DIR=%cd:\=/%/tmp_install
+set TORCH_LIB_DIR=%cd:\=/%/torch/lib
+set INSTALL_DIR=%cd:\=/%/torch/lib/tmp_install
+set THIRD_PARTY_DIR=%cd:\=/%/third_party
 set PATH=%INSTALL_DIR%/bin;%PATH%
 set BASIC_C_FLAGS= /DTH_INDEX_BASE=0 /I%INSTALL_DIR%/include /I%INSTALL_DIR%/include/TH /I%INSTALL_DIR%/include/THC /I%INSTALL_DIR%/include/THS /I%INSTALLDIR%/include/THCS /I%INSTALLDIR%/include/THPP /I%INSTALLDIR%/include/THNN /I%INSTALLDIR%/include/THCUNN
 set BASIC_CUDA_FLAGS= -DTH_INDEX_BASE=0 -I%INSTALL_DIR%/include -I%INSTALL_DIR%/include/TH -I%INSTALL_DIR%/include/THC -I%INSTALL_DIR%/include/THS -I%INSTALLDIR%/include/THCS -I%INSTALLDIR%/include/THPP -I%INSTALLDIR%/include/THNN -I%INSTALLDIR%/include/THCUNN
@@ -16,7 +15,7 @@ set CWRAP_FILES=%BASE_DIR%/torch/lib/ATen/Declarations.cwrap;%BASE_DIR%/torch/li
 set C_FLAGS=%BASIC_C_FLAGS% /D_WIN32 /Z7 /EHa /DNOMINMAX
 set LINK_FLAGS=/DEBUG:FULL
 
-mkdir tmp_install
+mkdir torch/lib/tmp_install
 
 IF "%~1"=="--with-cuda" (
   set /a NO_CUDA=0
@@ -60,15 +59,30 @@ IF "%CMAKE_GENERATOR%"=="" (
 :read_loop
 if "%1"=="" goto after_loop
 if "%1"=="ATen" (
+  cd aten
   call:build_aten %~1
+  cd ..
 ) ELSE (
-  call:build %~1
+  set "IS_OURS="
+  IF "%1"=="THD" set IS_OURS=1
+  IF "%1"=="libshm_windows" set IS_OURS=1
+  if defined IS_OURS (
+    cd torch\lib
+    call:build %~1
+    cd ..\..
+  ) ELSE (
+    cd third_party
+    call:build %~1
+    cd ..
+  )
 )
 shift
 goto read_loop
 
 :after_loop
 
+cd torch/lib
+
 copy /Y tmp_install\lib\* .
 IF EXIST ".\tmp_install\bin" (
   copy /Y tmp_install\bin\* .
@@ -77,6 +91,8 @@ xcopy /Y /E tmp_install\include\*.* include\*.*
 xcopy /Y ..\..\aten\src\THNN\generic\THNN.h  .
 xcopy /Y ..\..\aten\src\THCUNN\generic\THCUNN.h .
 
+cd ..\..
+
 goto:eof
 
 :build
@@ -121,9 +137,9 @@ goto:eof
 :build_aten
   @setlocal
   IF NOT "%PREBUILD_COMMAND%"=="" call "%PREBUILD_COMMAND%" %PREBUILD_COMMAND_ARGS%
-  mkdir build\%~1
-  cd build/%~1
-  cmake ../../../../%~1 %CMAKE_GENERATOR_COMMAND% ^
+  mkdir build
+  cd build
+  cmake .. %CMAKE_GENERATOR_COMMAND% ^
                   -DCMAKE_INSTALL_PREFIX="%INSTALL_DIR%" ^
                   -DNO_CUDA=%NO_CUDA% ^
                   -DNO_NNPACK=%NO_NNPACK% ^
@@ -134,7 +150,7 @@ goto:eof
 
   %MAKE_COMMAND%
   IF NOT %ERRORLEVEL%==0 exit 1
-  cd ../..
+  cd ..
   @endlocal
 
 goto:eof

tools/build_pytorch_libs.sh

@@ -1,10 +1,12 @@
 #!/usr/bin/env bash
 
-# Shell script used to build the torch/lib/* dependencies prior to
+# Shell script used to build the aten/* and third_party/* dependencies prior to
 # linking the libraries and passing the headers to the Python extension
 # compilation stage. This file is used from setup.py, but can also be
 # called standalone to compile the libraries outside of the overall PyTorch
 # build process.
+#
+# TODO: Replace this with a CMakeLists.txt
 
 set -ex
 
@@ -27,6 +29,8 @@ if [[ "$1" == "--with-gloo-ibverbs" ]]; then
   shift
 fi
 
+CMAKE_INSTALL=${CMAKE_INSTALL-make install}
+
 # Save user specified env vars, we will manually propagate them
 # to cmake.  We copy distutils semantics, referring to
 # cpython/Lib/distutils/sysconfig.py as the source of truth
@@ -45,11 +49,13 @@ if [[ -n "$CPPFLAGS" ]]; then
   USER_LDFLAGS="$USER_LDFLAGS $CPPFLAGS"
 fi
 
-cd "$(dirname "$0")/../.."
+cd "$(dirname "$0")/.."
 PWD=`printf "%q\n" "$(pwd)"`
 BASE_DIR="$PWD"
-cd torch/lib
-INSTALL_DIR="$PWD/tmp_install"
+TORCH_LIB_DIR="$BASE_DIR/torch/lib"
+INSTALL_DIR="$TORCH_LIB_DIR/tmp_install"
+THIRD_PARTY_DIR="$BASE_DIR/third_party"
+
 CMAKE_VERSION=${CMAKE_VERSION:="cmake"}
 C_FLAGS=" -DTH_INDEX_BASE=0 -I\"$INSTALL_DIR/include\" \
   -I\"$INSTALL_DIR/include/TH\" -I\"$INSTALL_DIR/include/THC\" \
@@ -94,12 +100,12 @@ if [ -z "$NUM_JOBS" ]; then
   NUM_JOBS="$(getconf _NPROCESSORS_ONLN)"
 fi
 
-# Used to build an individual library, e.g. build TH
+# Used to build an individual library
 function build() {
   # We create a build directory for the library, which will
   # contain the cmake output
   mkdir -p build/$1
-  cd build/$1
+  pushd build/$1
   BUILD_C_FLAGS=''
   case $1 in
       THCS | THCUNN ) BUILD_C_FLAGS=$C_FLAGS;;
@@ -141,7 +147,7 @@ function build() {
               ${@:2} \
               -DCMAKE_EXPORT_COMPILE_COMMANDS=1
   ${CMAKE_INSTALL} -j"$NUM_JOBS"
-  cd ../..
+  popd
 
   local lib_prefix=$INSTALL_DIR/lib/lib$1
   if [ -f "$lib_prefix$LD_POSTFIX" ]; then
@@ -149,32 +155,32 @@ function build() {
   fi
 
   if [[ $(uname) == 'Darwin' ]]; then
-    cd tmp_install/lib
+    pushd "$INSTALL_DIR/lib"
     for lib in *.dylib; do
       echo "Updating install_name for $lib"
       install_name_tool -id @rpath/$lib $lib
     done
-    cd ../..
+    popd
   fi
 }
 
 function build_nccl() {
-   mkdir -p build/nccl
-   cd build/nccl
-   ${CMAKE_VERSION} ../../nccl -DCMAKE_MODULE_PATH="$BASE_DIR/cmake/FindCUDA" \
-               ${CMAKE_GENERATOR} \
-               -DCMAKE_BUILD_TYPE=Release \
-               -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
-               -DCMAKE_C_FLAGS="$C_FLAGS $USER_CFLAGS" \
-               -DCMAKE_CXX_FLAGS="$C_FLAGS $CPP_FLAGS $USER_CFLAGS" \
-               -DCMAKE_SHARED_LINKER_FLAGS="$USER_LDFLAGS"
+  mkdir -p build/nccl
+  pushd build/nccl
+  ${CMAKE_VERSION} ../../nccl -DCMAKE_MODULE_PATH="$BASE_DIR/cmake/FindCUDA" \
+              ${CMAKE_GENERATOR} \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
+              -DCMAKE_C_FLAGS="$C_FLAGS $USER_CFLAGS" \
+              -DCMAKE_CXX_FLAGS="$C_FLAGS $CPP_FLAGS $USER_CFLAGS" \
+              -DCMAKE_SHARED_LINKER_FLAGS="$USER_LDFLAGS"
   ${CMAKE_INSTALL}
-   mkdir -p ${INSTALL_DIR}/lib
-   cp "lib/libnccl.so.1" "${INSTALL_DIR}/lib/libnccl.so.1"
-   if [ ! -f "${INSTALL_DIR}/lib/libnccl.so" ]; then
-     ln -s "${INSTALL_DIR}/lib/libnccl.so.1" "${INSTALL_DIR}/lib/libnccl.so"
-   fi
-   cd ../..
+  mkdir -p ${INSTALL_DIR}/lib
+  cp "lib/libnccl.so.1" "${INSTALL_DIR}/lib/libnccl.so.1"
+  if [ ! -f "${INSTALL_DIR}/lib/libnccl.so" ]; then
+    ln -s "${INSTALL_DIR}/lib/libnccl.so.1" "${INSTALL_DIR}/lib/libnccl.so"
+  fi
+  popd
 }
 
 # purpusefully not using build() because we need ATen to build the same
@@ -186,9 +192,9 @@ function build_nccl() {
 # detected them (to ensure that we have a consistent view between the
 # PyTorch and ATen builds.)
 function build_aten() {
-  mkdir -p build/aten
-  cd  build/aten
-  ${CMAKE_VERSION} ../../../../aten \
+  mkdir -p build
+  pushd build
+  ${CMAKE_VERSION} .. \
   ${CMAKE_GENERATOR} \
       -DCMAKE_BUILD_TYPE=$([ $DEBUG ] && echo Debug || echo Release) \
       -DNO_CUDA=$((1-$WITH_CUDA)) \
@@ -207,27 +213,43 @@ function build_aten() {
       # to aten/CMakeLists.txt, not here.  We need the vanilla
       # cmake build to work.
   ${CMAKE_INSTALL} -j"$NUM_JOBS"
-  cd ../..
+  popd
 }
 
 # In the torch/lib directory, create an installation directory
-mkdir -p tmp_install
+mkdir -p torch/lib/tmp_install
 
 # Build
 for arg in "$@"; do
     if [[ "$arg" == "nccl" ]]; then
+        pushd $THIRD_PARTY_DIR
         build_nccl
+        popd
     elif [[ "$arg" == "gloo" ]]; then
+        pushd "$THIRD_PARTY_DIR"
         build gloo $GLOO_FLAGS
+        popd
     elif [[ "$arg" == "ATen" ]]; then
+        pushd "$BASE_DIR/aten"
         build_aten
+        popd
     elif [[ "$arg" == "THD" ]]; then
+        pushd "$TORCH_LIB_DIR"
         build THD $THD_FLAGS
-    else
+        popd
+    elif [[ "$arg" == "libshm" ]] || [[ "$arg" == "libshm_windows" ]]; then
+        pushd "$TORCH_LIB_DIR"
         build $arg
+        popd
+    else
+        pushd "$THIRD_PARTY_DIR"
+        build $arg
+        popd
     fi
 done
 
+pushd torch/lib
+
 # If all the builds succeed we copy the libraries, headers,
 # binaries to torch/lib
 rm -rf "$INSTALL_DIR/lib/cmake"
@@ -250,3 +272,5 @@ then
     # copy over dependency libraries into the current dir
     cp "$PYTORCH_SO_DEPS" .
 fi
+
+popd

tools/cpp_build/libtorch/CMakeLists.txt

@@ -147,7 +147,7 @@ TARGET_INCLUDE_DIRECTORIES(torch
   "${ATEN_INCLUDE_DIR}/TH"
   "${CMAKE_CURRENT_SOURCE_DIR}"
   "${TORCH_SRC_DIR}/../"
-  "${TORCH_SRC_DIR}/lib/nanopb"
+  "${TORCH_SRC_DIR}/../third_party/nanopb"
   "${CUDA_INCLUDE_DIRS}"
   )
 

tox.ini

@@ -1,4 +1,4 @@
 [flake8]
 max-line-length = 120
 ignore = E305,E402,E721,E741,F401,F403,F405,F821,F841,F999
-exclude = docs/src,venv,torch/lib/gloo,torch/lib/pybind11,torch/lib/nanopb
+exclude = docs/src,venv,third_party