Statically linking CUDA for Anaconda builds (#6680)

* Statically linking CUDA for Anaconda builds

* typo

* Adding a summary line

* Comments

* Typo fix

* Fix faulty parameter passing

* Removing problem CUDA modules for now

* Fixing unused debugging function

* Turning off static cuda linking until script changes are in

* Disabling mkl

CMakeLists.txt

@@ -40,6 +40,7 @@ option(USE_ACL "Use ARM Compute Library" OFF)
 option(USE_ASAN "Use Address Sanitizer" OFF)
 option(USE_ATEN "Use ATen" OFF)
 option(USE_CUDA "Use Cuda" ON)
+option(CAFFE2_STATIC_LINK_CUDA "Statically link CUDA libraries" OFF)
 option(USE_FFMPEG "Use ffmpeg" OFF)
 option(USE_GFLAGS "Use GFLAGS" ON)
 option(USE_GLOG "Use GLOG" ON)

caffe2/CMakeLists.txt

@@ -141,16 +141,25 @@ if(USE_CUDA)
   # it. We will then manually add the cudart library as interface libs.
   set(__tmp ${CUDA_LIBRARIES})
   set(CUDA_LIBRARIES PRIVATE ${CUDA_LIBRARIES})
-  CUDA_ADD_LIBRARY(caffe2_gpu ${Caffe2_GPU_SRCS})
+  if(CAFFE2_STATIC_LINK_CUDA)
+    CUDA_ADD_LIBRARY(caffe2_gpu STATIC ${Caffe2_GPU_SRCS})
+  else()
+    CUDA_ADD_LIBRARY(caffe2_gpu ${Caffe2_GPU_SRCS})
+  endif()
   set(CUDA_LIBRARIES ${__tmp})
   target_link_libraries(caffe2_gpu INTERFACE caffe2::cudart)
 
   target_include_directories(
       caffe2_gpu INTERFACE $<INSTALL_INTERFACE:include>)
-  target_link_libraries(
-      caffe2_gpu PUBLIC caffe2 ${Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS})
   target_link_libraries(
       caffe2_gpu PRIVATE ${Caffe2_CUDA_DEPENDENCY_LIBS})
+
+  # These public dependencies must go after the previous dependencies, as the
+  # order of the libraries in the linker call matters here when statically
+  # linking; libculibos and cublas must be last.
+  target_link_libraries(
+      caffe2_gpu PUBLIC caffe2 ${Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS})
+
   caffe2_interface_library(caffe2_gpu caffe2_gpu_library)
   list(APPEND Caffe2_MAIN_LIBS caffe2_gpu_library)
   install(TARGETS caffe2_gpu EXPORT Caffe2Targets DESTINATION lib)

cmake/Dependencies.cmake

@@ -378,11 +378,18 @@ endif()
 if(USE_CUDA)
   include(cmake/public/cuda.cmake)
   if(CAFFE2_FOUND_CUDA)
-    # A helper variable recording the list of Caffe2 dependent librareis
+    # A helper variable recording the list of Caffe2 dependent libraries
     # caffe2::cudart is dealt with separately, due to CUDA_ADD_LIBRARY
     # design reason (it adds CUDA_LIBRARIES itself).
     set(Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS
-        caffe2::cuda caffe2::curand caffe2::cublas caffe2::cudnn caffe2::nvrtc)
+        caffe2::cuda caffe2::curand caffe2::cudnn caffe2::nvrtc)
+    if(CAFFE2_STATIC_LINK_CUDA)
+      # When statically linking, this must be the order of the libraries
+      LIST(APPEND Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS
+          "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libculibos.a" caffe2::cublas)
+    else()
+      LIST(APPEND Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS caffe2::cublas)
+    endif()
     if(USE_TENSORRT) 
       list(APPEND Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS caffe2::tensorrt) 
     endif()

cmake/Summary.cmake

@@ -38,12 +38,22 @@ function (caffe2_print_configuration_summary)
   message(STATUS "  USE_ASAN              : ${USE_ASAN}")
   message(STATUS "  USE_CUDA              : ${USE_CUDA}")
   if(${USE_CUDA})
+    message(STATUS "    CUDA static link    : ${CAFFE2_STATIC_LINK_CUDA}")
     message(STATUS "    CUDA version        : ${CUDA_VERSION}")
     message(STATUS "    CuDNN version       : ${CUDNN_VERSION}")
     message(STATUS "    CUDA root directory : ${CUDA_TOOLKIT_ROOT_DIR}")
-    message(STATUS "    CUDA library        : ${CUDA_CUDA_LIB}")
+    get_target_property(__tmp caffe2::cuda IMPORTED_LOCATION)
-    message(STATUS "    CUDA NVRTC library  : ${CUDA_NVRTC_LIB}")
+    message(STATUS "    CUDA library        : ${__tmp}")
-    message(STATUS "    CUDA runtime library: ${CUDA_CUDART_LIBRARY}")
+    get_target_property(__tmp caffe2::cudart INTERFACE_LINK_LIBRARIES)
+    message(STATUS "    cudart library      : ${__tmp}")
+    get_target_property(__tmp caffe2::cublas INTERFACE_LINK_LIBRARIES)
+    message(STATUS "    cublas library      : ${__tmp}")
+    get_target_property(__tmp caffe2::curand IMPORTED_LOCATION)
+    message(STATUS "    curand library      : ${__tmp}")
+    get_target_property(__tmp caffe2::cudnn IMPORTED_LOCATION)
+    message(STATUS "    CuDNN library       : ${__tmp}")
+    get_target_property(__tmp caffe2::nvrtc IMPORTED_LOCATION)
+    message(STATUS "    nvrtc               : ${__tmp}")
     message(STATUS "    CUDA include path   : ${CUDA_INCLUDE_DIRS}")
     message(STATUS "    NVCC executable     : ${CUDA_NVCC_EXECUTABLE}")
     message(STATUS "    CUDA host compiler  : ${CUDA_HOST_COMPILER}")

cmake/Utils.cmake

@@ -223,3 +223,31 @@ function(pycmd outvar cmd)
   string(STRIP "${_output}" _output)
   set(${outvar} "${_output}" PARENT_SCOPE)
 endfunction()
+
+###
+# Helper function to print out everything that cmake knows about a target
+#
+# Copied from https://stackoverflow.com/questions/32183975/how-to-print-all-the-properties-of-a-target-in-cmake
+# This isn't called anywhere, but it's very useful when debugging cmake
+# NOTE: This doesn't work for INTERFACE_LIBRARY or INTERFACE_LINK_LIBRARY targets
+
+function(print_target_properties tgt)
+  if(NOT TARGET ${tgt})
+    message("There is no target named '${tgt}'")
+    return()
+  endif()
+
+  # Get a list of all cmake properties TODO cache this lazily somehow
+  execute_process(COMMAND cmake --help-property-list OUTPUT_VARIABLE CMAKE_PROPERTY_LIST)
+  STRING(REGEX REPLACE ";" "\\\\;" CMAKE_PROPERTY_LIST "${CMAKE_PROPERTY_LIST}")
+  STRING(REGEX REPLACE "\n" ";" CMAKE_PROPERTY_LIST "${CMAKE_PROPERTY_LIST}")
+
+  foreach (prop ${CMAKE_PROPERTY_LIST})
+    string(REPLACE "<CONFIG>" "${CMAKE_BUILD_TYPE}" prop ${prop})
+    get_property(propval TARGET ${tgt} PROPERTY ${prop} SET)
+    if (propval)
+      get_target_property(propval ${tgt} ${prop})
+      message ("${tgt} ${prop} = ${propval}")
+    endif()
+  endforeach(prop)
+endfunction(print_target_properties)

cmake/public/cuda.cmake

@@ -13,12 +13,17 @@ if(NOT CUDA_FOUND)
 endif()
 
 # Find cudnn.
+if(CAFFE2_STATIC_LINK_CUDA)
+  SET(CUDNN_LIBNAME "libcudnn_static.a")
+else()
+  SET(CUDNN_LIBNAME "cudnn")
+endif()
 include(FindPackageHandleStandardArgs)
 set(CUDNN_ROOT_DIR "" CACHE PATH "Folder contains NVIDIA cuDNN")
 find_path(CUDNN_INCLUDE_DIR cudnn.h
     HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}
     PATH_SUFFIXES cuda/include include)
-find_library(CUDNN_LIBRARY cudnn
+find_library(CUDNN_LIBRARY ${CUDNN_LIBNAME}
     HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}
     PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64)
 find_package_handle_standard_args(
@@ -89,6 +94,10 @@ find_library(CUDA_NVRTC_LIB nvrtc
     PATH_SUFFIXES lib lib64 lib/x64)
 
 # Create new style imported libraries.
+# Several of these libraries have a hardcoded path if CAFFE2_STATIC_LINK_CUDA
+# is set. This path is where sane CUDA installations have their static
+# libraries installed. This flag should only be used for binary builds, so
+# end-users should never have this flag set.
 
 # cuda
 add_library(caffe2::cuda UNKNOWN IMPORTED)
@@ -102,14 +111,21 @@ set_property(
 # cudart. CUDA_LIBRARIES is actually a list, so we will make an interface
 # library.
 add_library(caffe2::cudart INTERFACE IMPORTED)
-set_property(
+if(CAFFE2_STATIC_LINK_CUDA)
-    TARGET caffe2::cudart PROPERTY INTERFACE_LINK_LIBRARIES
+    set_property(
-    ${CUDA_LIBRARIES})
+        TARGET caffe2::cudart PROPERTY INTERFACE_LINK_LIBRARIES
+        "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcudart_static.a")
+else()
+    set_property(
+        TARGET caffe2::cudart PROPERTY INTERFACE_LINK_LIBRARIES
+        ${CUDA_LIBRARIES})
+endif()
 set_property(
     TARGET caffe2::cudart PROPERTY INTERFACE_INCLUDE_DIRECTORIES
     ${CUDA_INCLUDE_DIRS})
 
 # cudnn
+# static linking is handled by USE_STATIC_CUDNN environment variable
 add_library(caffe2::cudnn UNKNOWN IMPORTED)
 set_property(
     TARGET caffe2::cudnn PROPERTY IMPORTED_LOCATION
@@ -120,9 +136,15 @@ set_property(
 
 # curand
 add_library(caffe2::curand UNKNOWN IMPORTED)
-set_property(
+if(CAFFE2_STATIC_LINK_CUDA)
-    TARGET caffe2::curand PROPERTY IMPORTED_LOCATION
+    set_property(
-    ${CUDA_curand_LIBRARY})
+        TARGET caffe2::curand PROPERTY IMPORTED_LOCATION
+        "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcurand_static.a")
+else()
+    set_property(
+        TARGET caffe2::curand PROPERTY IMPORTED_LOCATION
+        ${CUDA_curand_LIBRARY})
+endif()
 set_property(
     TARGET caffe2::curand PROPERTY INTERFACE_INCLUDE_DIRECTORIES
     ${CUDA_INCLUDE_DIRS})
@@ -141,9 +163,15 @@ endif()
 # cublas. CUDA_CUBLAS_LIBRARIES is actually a list, so we will make an
 # interface library similar to cudart.
 add_library(caffe2::cublas INTERFACE IMPORTED)
-set_property(
+if(CAFFE2_STATIC_LINK_CUDA)
-    TARGET caffe2::cublas PROPERTY INTERFACE_LINK_LIBRARIES
+    set_property(
-    ${CUDA_CUBLAS_LIBRARIES})
+        TARGET caffe2::cublas PROPERTY INTERFACE_LINK_LIBRARIES
+        "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcublas_static.a")
+else()
+    set_property(
+        TARGET caffe2::cublas PROPERTY INTERFACE_LINK_LIBRARIES
+        ${CUDA_CUBLAS_LIBRARIES})
+endif()
 set_property(
     TARGET caffe2::cublas PROPERTY INTERFACE_INCLUDE_DIRECTORIES
     ${CUDA_INCLUDE_DIRS})

conda/caffe2/normal/build.sh

@@ -28,7 +28,7 @@ CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$PREFIX")
 # Build Caffe2
 mkdir -p build
 cd build
-cmake "${CMAKE_ARGS[@]}" "$PYTHON_ARGS" $CONDA_CMAKE_BUILD_ARGS ..
+cmake "${CMAKE_ARGS[@]}" "$PYTHON_ARGS" $CONDA_CAFFE2_CMAKE_ARGS ..
 if [ "$(uname)" == 'Darwin' ]; then
   make "-j$(sysctl -n hw.ncpu)"
 else

conda/integrated/build.sh

@@ -32,6 +32,9 @@ export TH_BINARY_BUILD=1
 export PYTORCH_BUILD_VERSION=$PKG_VERSION
 export PYTORCH_BUILD_NUMBER=$PKG_BUILDNUM
 export NCCL_ROOT_DIR=/usr/local/cuda
+export USE_STATIC_CUDNN=1
+export USE_STATIC_NCCL=1
+export ATEN_STATIC_CUDA=1
 
 
 ###########################################################

scripts/build_anaconda.sh

@@ -284,14 +284,20 @@ fi
 if [[ -n $CUDA_VERSION ]]; then
   CAFFE2_CMAKE_ARGS+=("-DUSE_CUDA=ON")
   CAFFE2_CMAKE_ARGS+=("-DUSE_NCCL=ON")
+
+  # NCCL and GLOO don't work with static CUDA right now. Cmake changes are
+  # needed
+  #CAFFE2_CMAKE_ARGS+=("-DUSE_NCCL=OFF")
+  #CAFFE2_CMAKE_ARGS+=("-DUSE_GLOO=OFF")
+  #CAFFE2_CMAKE_ARGS+=("-DCAFFE2_STATIC_LINK_CUDA=ON")
 else
   # Flags required for CPU for Caffe2
   CAFFE2_CMAKE_ARGS+=("-DUSE_CUDA=OFF")
   CAFFE2_CMAKE_ARGS+=("-DUSE_NCCL=OFF")
   if [[ -z $BUILD_INTEGRATED ]]; then
-    CAFFE2_CMAKE_ARGS+=("-DBLAS=MKL")
+    #CAFFE2_CMAKE_ARGS+=("-DBLAS=MKL")
-    add_package 'mkl'
+    #add_package 'mkl'
-    add_package 'mkl-include'
+    #add_package 'mkl-include'
   fi
 fi
 
@@ -320,9 +326,10 @@ if [[ -z $SKIP_CONDA_TESTS && -n $UPLOAD_TO_CONDA ]]; then
   CONDA_BUILD_ARGS+=(" --token ${CAFFE2_ANACONDA_ORG_ACCESS_TOKEN}")
 
   # If building a redistributable, then package the CUDA libraries with it
-  if [[ -n $CUDA_VERSION ]]; then
+  # TODO this doesn't work on Ubuntu right now
-    export PACKAGE_CUDA_LIBS=1
+  #if [[ -n $CUDA_VERSION ]]; then
-  fi
+  #  export PACKAGE_CUDA_LIBS=1
+  #fi
 
   # Show what the final meta.yaml looks like
   echo "Finalized meta.yaml is"