Statically linking CUDA for Anaconda builds (#6680)

* Statically linking CUDA for Anaconda builds * typo * Adding a summary line * Comments * Typo fix * Fix faulty parameter passing * Removing problem CUDA modules for now * Fixing unused debugging function * Turning off static cuda linking until script changes are in * Disabling mkl
2025-12-06 12:20:52 +01:00 · 2018-04-25 18:22:54 -05:00 · 2018-04-25 18:22:54 -05:00 · 2e32e8df75
commit 2e32e8df75
parent 7599d0c3fe
9 changed files with 118 additions and 25 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -40,6 +40,7 @@ option(USE_ACL "Use ARM Compute Library" OFF)
 option(USE_ASAN "Use Address Sanitizer" OFF)
 option(USE_ATEN "Use ATen" OFF)
 option(USE_CUDA "Use Cuda" ON)
+option(CAFFE2_STATIC_LINK_CUDA "Statically link CUDA libraries" OFF)
 option(USE_FFMPEG "Use ffmpeg" OFF)
 option(USE_GFLAGS "Use GFLAGS" ON)
 option(USE_GLOG "Use GLOG" ON)
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@ -141,16 +141,25 @@ if(USE_CUDA)
  # it. We will then manually add the cudart library as interface libs.
  set(__tmp ${CUDA_LIBRARIES})
  set(CUDA_LIBRARIES PRIVATE ${CUDA_LIBRARIES})
-  CUDA_ADD_LIBRARY(caffe2_gpu ${Caffe2_GPU_SRCS})
+  if(CAFFE2_STATIC_LINK_CUDA)
+    CUDA_ADD_LIBRARY(caffe2_gpu STATIC ${Caffe2_GPU_SRCS})
+  else()
+    CUDA_ADD_LIBRARY(caffe2_gpu ${Caffe2_GPU_SRCS})
+  endif()
  set(CUDA_LIBRARIES ${__tmp})
  target_link_libraries(caffe2_gpu INTERFACE caffe2::cudart)

  target_include_directories(
      caffe2_gpu INTERFACE $<INSTALL_INTERFACE:include>)
-  target_link_libraries(
-      caffe2_gpu PUBLIC caffe2 ${Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS})
  target_link_libraries(
      caffe2_gpu PRIVATE ${Caffe2_CUDA_DEPENDENCY_LIBS})
+
+  # These public dependencies must go after the previous dependencies, as the
+  # order of the libraries in the linker call matters here when statically
+  # linking; libculibos and cublas must be last.
+  target_link_libraries(
+      caffe2_gpu PUBLIC caffe2 ${Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS})
+
  caffe2_interface_library(caffe2_gpu caffe2_gpu_library)
  list(APPEND Caffe2_MAIN_LIBS caffe2_gpu_library)
  install(TARGETS caffe2_gpu EXPORT Caffe2Targets DESTINATION lib)
--- a/cmake/Dependencies.cmake
+++ b/cmake/Dependencies.cmake
@ -378,11 +378,18 @@ endif()
 if(USE_CUDA)
  include(cmake/public/cuda.cmake)
  if(CAFFE2_FOUND_CUDA)
-    # A helper variable recording the list of Caffe2 dependent librareis
+    # A helper variable recording the list of Caffe2 dependent libraries
    # caffe2::cudart is dealt with separately, due to CUDA_ADD_LIBRARY
    # design reason (it adds CUDA_LIBRARIES itself).
    set(Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS
-        caffe2::cuda caffe2::curand caffe2::cublas caffe2::cudnn caffe2::nvrtc)
+        caffe2::cuda caffe2::curand caffe2::cudnn caffe2::nvrtc)
+    if(CAFFE2_STATIC_LINK_CUDA)
+      # When statically linking, this must be the order of the libraries
+      LIST(APPEND Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS
+          "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libculibos.a" caffe2::cublas)
+    else()
+      LIST(APPEND Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS caffe2::cublas)
+    endif()
    if(USE_TENSORRT) 
      list(APPEND Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS caffe2::tensorrt) 
    endif()
--- a/cmake/Summary.cmake
+++ b/cmake/Summary.cmake
@ -38,12 +38,22 @@ function (caffe2_print_configuration_summary)
  message(STATUS "  USE_ASAN              : ${USE_ASAN}")
  message(STATUS "  USE_CUDA              : ${USE_CUDA}")
  if(${USE_CUDA})
+    message(STATUS "    CUDA static link    : ${CAFFE2_STATIC_LINK_CUDA}")
    message(STATUS "    CUDA version        : ${CUDA_VERSION}")
    message(STATUS "    CuDNN version       : ${CUDNN_VERSION}")
    message(STATUS "    CUDA root directory : ${CUDA_TOOLKIT_ROOT_DIR}")
-    message(STATUS "    CUDA library        : ${CUDA_CUDA_LIB}")
-    message(STATUS "    CUDA NVRTC library  : ${CUDA_NVRTC_LIB}")
-    message(STATUS "    CUDA runtime library: ${CUDA_CUDART_LIBRARY}")
+    get_target_property(__tmp caffe2::cuda IMPORTED_LOCATION)
+    message(STATUS "    CUDA library        : ${__tmp}")
+    get_target_property(__tmp caffe2::cudart INTERFACE_LINK_LIBRARIES)
+    message(STATUS "    cudart library      : ${__tmp}")
+    get_target_property(__tmp caffe2::cublas INTERFACE_LINK_LIBRARIES)
+    message(STATUS "    cublas library      : ${__tmp}")
+    get_target_property(__tmp caffe2::curand IMPORTED_LOCATION)
+    message(STATUS "    curand library      : ${__tmp}")
+    get_target_property(__tmp caffe2::cudnn IMPORTED_LOCATION)
+    message(STATUS "    CuDNN library       : ${__tmp}")
+    get_target_property(__tmp caffe2::nvrtc IMPORTED_LOCATION)
+    message(STATUS "    nvrtc               : ${__tmp}")
    message(STATUS "    CUDA include path   : ${CUDA_INCLUDE_DIRS}")
    message(STATUS "    NVCC executable     : ${CUDA_NVCC_EXECUTABLE}")
    message(STATUS "    CUDA host compiler  : ${CUDA_HOST_COMPILER}")
--- a/cmake/Utils.cmake
+++ b/cmake/Utils.cmake
@ -223,3 +223,31 @@ function(pycmd outvar cmd)
  string(STRIP "${_output}" _output)
  set(${outvar} "${_output}" PARENT_SCOPE)
 endfunction()
+
+###
+# Helper function to print out everything that cmake knows about a target
+#
+# Copied from https://stackoverflow.com/questions/32183975/how-to-print-all-the-properties-of-a-target-in-cmake
+# This isn't called anywhere, but it's very useful when debugging cmake
+# NOTE: This doesn't work for INTERFACE_LIBRARY or INTERFACE_LINK_LIBRARY targets
+
+function(print_target_properties tgt)
+  if(NOT TARGET ${tgt})
+    message("There is no target named '${tgt}'")
+    return()
+  endif()
+
+  # Get a list of all cmake properties TODO cache this lazily somehow
+  execute_process(COMMAND cmake --help-property-list OUTPUT_VARIABLE CMAKE_PROPERTY_LIST)
+  STRING(REGEX REPLACE ";" "\\\\;" CMAKE_PROPERTY_LIST "${CMAKE_PROPERTY_LIST}")
+  STRING(REGEX REPLACE "\n" ";" CMAKE_PROPERTY_LIST "${CMAKE_PROPERTY_LIST}")
+
+  foreach (prop ${CMAKE_PROPERTY_LIST})
+    string(REPLACE "<CONFIG>" "${CMAKE_BUILD_TYPE}" prop ${prop})
+    get_property(propval TARGET ${tgt} PROPERTY ${prop} SET)
+    if (propval)
+      get_target_property(propval ${tgt} ${prop})
+      message ("${tgt} ${prop} = ${propval}")
+    endif()
+  endforeach(prop)
+endfunction(print_target_properties)
--- a/cmake/public/cuda.cmake
+++ b/cmake/public/cuda.cmake
@ -13,12 +13,17 @@ if(NOT CUDA_FOUND)
 endif()

 # Find cudnn.
+if(CAFFE2_STATIC_LINK_CUDA)
+  SET(CUDNN_LIBNAME "libcudnn_static.a")
+else()
+  SET(CUDNN_LIBNAME "cudnn")
+endif()
 include(FindPackageHandleStandardArgs)
 set(CUDNN_ROOT_DIR "" CACHE PATH "Folder contains NVIDIA cuDNN")
 find_path(CUDNN_INCLUDE_DIR cudnn.h
    HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}
    PATH_SUFFIXES cuda/include include)
-find_library(CUDNN_LIBRARY cudnn
+find_library(CUDNN_LIBRARY ${CUDNN_LIBNAME}
    HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR}
    PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64)
 find_package_handle_standard_args(
@ -89,6 +94,10 @@ find_library(CUDA_NVRTC_LIB nvrtc
    PATH_SUFFIXES lib lib64 lib/x64)

 # Create new style imported libraries.
+# Several of these libraries have a hardcoded path if CAFFE2_STATIC_LINK_CUDA
+# is set. This path is where sane CUDA installations have their static
+# libraries installed. This flag should only be used for binary builds, so
+# end-users should never have this flag set.

 # cuda
 add_library(caffe2::cuda UNKNOWN IMPORTED)
@ -102,14 +111,21 @@ set_property(
 # cudart. CUDA_LIBRARIES is actually a list, so we will make an interface
 # library.
 add_library(caffe2::cudart INTERFACE IMPORTED)
-set_property(
-    TARGET caffe2::cudart PROPERTY INTERFACE_LINK_LIBRARIES
-    ${CUDA_LIBRARIES})
+if(CAFFE2_STATIC_LINK_CUDA)
+    set_property(
+        TARGET caffe2::cudart PROPERTY INTERFACE_LINK_LIBRARIES
+        "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcudart_static.a")
+else()
+    set_property(
+        TARGET caffe2::cudart PROPERTY INTERFACE_LINK_LIBRARIES
+        ${CUDA_LIBRARIES})
+endif()
 set_property(
    TARGET caffe2::cudart PROPERTY INTERFACE_INCLUDE_DIRECTORIES
    ${CUDA_INCLUDE_DIRS})

 # cudnn
+# static linking is handled by USE_STATIC_CUDNN environment variable
 add_library(caffe2::cudnn UNKNOWN IMPORTED)
 set_property(
    TARGET caffe2::cudnn PROPERTY IMPORTED_LOCATION
@ -120,9 +136,15 @@ set_property(

 # curand
 add_library(caffe2::curand UNKNOWN IMPORTED)
-set_property(
-    TARGET caffe2::curand PROPERTY IMPORTED_LOCATION
-    ${CUDA_curand_LIBRARY})
+if(CAFFE2_STATIC_LINK_CUDA)
+    set_property(
+        TARGET caffe2::curand PROPERTY IMPORTED_LOCATION
+        "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcurand_static.a")
+else()
+    set_property(
+        TARGET caffe2::curand PROPERTY IMPORTED_LOCATION
+        ${CUDA_curand_LIBRARY})
+endif()
 set_property(
    TARGET caffe2::curand PROPERTY INTERFACE_INCLUDE_DIRECTORIES
    ${CUDA_INCLUDE_DIRS})
@ -141,9 +163,15 @@ endif()
 # cublas. CUDA_CUBLAS_LIBRARIES is actually a list, so we will make an
 # interface library similar to cudart.
 add_library(caffe2::cublas INTERFACE IMPORTED)
-set_property(
-    TARGET caffe2::cublas PROPERTY INTERFACE_LINK_LIBRARIES
-    ${CUDA_CUBLAS_LIBRARIES})
+if(CAFFE2_STATIC_LINK_CUDA)
+    set_property(
+        TARGET caffe2::cublas PROPERTY INTERFACE_LINK_LIBRARIES
+        "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcublas_static.a")
+else()
+    set_property(
+        TARGET caffe2::cublas PROPERTY INTERFACE_LINK_LIBRARIES
+        ${CUDA_CUBLAS_LIBRARIES})
+endif()
 set_property(
    TARGET caffe2::cublas PROPERTY INTERFACE_INCLUDE_DIRECTORIES
    ${CUDA_INCLUDE_DIRS})
--- a/conda/caffe2/normal/build.sh
+++ b/conda/caffe2/normal/build.sh
@ -28,7 +28,7 @@ CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$PREFIX")
 # Build Caffe2
 mkdir -p build
 cd build
-cmake "${CMAKE_ARGS[@]}" "$PYTHON_ARGS" $CONDA_CMAKE_BUILD_ARGS ..
+cmake "${CMAKE_ARGS[@]}" "$PYTHON_ARGS" $CONDA_CAFFE2_CMAKE_ARGS ..
 if [ "$(uname)" == 'Darwin' ]; then
  make "-j$(sysctl -n hw.ncpu)"
 else
--- a/conda/integrated/build.sh
+++ b/conda/integrated/build.sh
@ -32,6 +32,9 @@ export TH_BINARY_BUILD=1
 export PYTORCH_BUILD_VERSION=$PKG_VERSION
 export PYTORCH_BUILD_NUMBER=$PKG_BUILDNUM
 export NCCL_ROOT_DIR=/usr/local/cuda
+export USE_STATIC_CUDNN=1
+export USE_STATIC_NCCL=1
+export ATEN_STATIC_CUDA=1


 ###########################################################
--- a/scripts/build_anaconda.sh
+++ b/scripts/build_anaconda.sh
@ -284,14 +284,20 @@ fi
 if [[ -n $CUDA_VERSION ]]; then
  CAFFE2_CMAKE_ARGS+=("-DUSE_CUDA=ON")
  CAFFE2_CMAKE_ARGS+=("-DUSE_NCCL=ON")
+
+  # NCCL and GLOO don't work with static CUDA right now. Cmake changes are
+  # needed
+  #CAFFE2_CMAKE_ARGS+=("-DUSE_NCCL=OFF")
+  #CAFFE2_CMAKE_ARGS+=("-DUSE_GLOO=OFF")
+  #CAFFE2_CMAKE_ARGS+=("-DCAFFE2_STATIC_LINK_CUDA=ON")
 else
  # Flags required for CPU for Caffe2
  CAFFE2_CMAKE_ARGS+=("-DUSE_CUDA=OFF")
  CAFFE2_CMAKE_ARGS+=("-DUSE_NCCL=OFF")
  if [[ -z $BUILD_INTEGRATED ]]; then
-    CAFFE2_CMAKE_ARGS+=("-DBLAS=MKL")
-    add_package 'mkl'
-    add_package 'mkl-include'
+    #CAFFE2_CMAKE_ARGS+=("-DBLAS=MKL")
+    #add_package 'mkl'
+    #add_package 'mkl-include'
  fi
 fi

@ -320,9 +326,10 @@ if [[ -z $SKIP_CONDA_TESTS && -n $UPLOAD_TO_CONDA ]]; then
  CONDA_BUILD_ARGS+=(" --token ${CAFFE2_ANACONDA_ORG_ACCESS_TOKEN}")

  # If building a redistributable, then package the CUDA libraries with it
-  if [[ -n $CUDA_VERSION ]]; then
-    export PACKAGE_CUDA_LIBS=1
-  fi
+  # TODO this doesn't work on Ubuntu right now
+  #if [[ -n $CUDA_VERSION ]]; then
+  #  export PACKAGE_CUDA_LIBS=1
+  #fi

  # Show what the final meta.yaml looks like
  echo "Finalized meta.yaml is"