From 2e32e8df755b4b92dfbccb569812de1f1218a77d Mon Sep 17 00:00:00 2001 From: Paul Jesse Hellemn Date: Wed, 25 Apr 2018 18:22:54 -0500 Subject: [PATCH] Statically linking CUDA for Anaconda builds (#6680) * Statically linking CUDA for Anaconda builds * typo * Adding a summary line * Comments * Typo fix * Fix faulty parameter passing * Removing problem CUDA modules for now * Fixing unused debugging function * Turning off static cuda linking until script changes are in * Disabling mkl --- CMakeLists.txt | 1 + caffe2/CMakeLists.txt | 15 ++++++++--- cmake/Dependencies.cmake | 11 +++++++-- cmake/Summary.cmake | 16 +++++++++--- cmake/Utils.cmake | 28 +++++++++++++++++++++ cmake/public/cuda.cmake | 48 ++++++++++++++++++++++++++++-------- conda/caffe2/normal/build.sh | 2 +- conda/integrated/build.sh | 3 +++ scripts/build_anaconda.sh | 19 +++++++++----- 9 files changed, 118 insertions(+), 25 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 56a6494e52a..75a0b0888c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,7 @@ option(USE_ACL "Use ARM Compute Library" OFF) option(USE_ASAN "Use Address Sanitizer" OFF) option(USE_ATEN "Use ATen" OFF) option(USE_CUDA "Use Cuda" ON) +option(CAFFE2_STATIC_LINK_CUDA "Statically link CUDA libraries" OFF) option(USE_FFMPEG "Use ffmpeg" OFF) option(USE_GFLAGS "Use GFLAGS" ON) option(USE_GLOG "Use GLOG" ON) diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt index 3225e9e5355..6e8cda25203 100644 --- a/caffe2/CMakeLists.txt +++ b/caffe2/CMakeLists.txt @@ -141,16 +141,25 @@ if(USE_CUDA) # it. We will then manually add the cudart library as interface libs. set(__tmp ${CUDA_LIBRARIES}) set(CUDA_LIBRARIES PRIVATE ${CUDA_LIBRARIES}) - CUDA_ADD_LIBRARY(caffe2_gpu ${Caffe2_GPU_SRCS}) + if(CAFFE2_STATIC_LINK_CUDA) + CUDA_ADD_LIBRARY(caffe2_gpu STATIC ${Caffe2_GPU_SRCS}) + else() + CUDA_ADD_LIBRARY(caffe2_gpu ${Caffe2_GPU_SRCS}) + endif() set(CUDA_LIBRARIES ${__tmp}) target_link_libraries(caffe2_gpu INTERFACE caffe2::cudart) target_include_directories( caffe2_gpu INTERFACE $) - target_link_libraries( - caffe2_gpu PUBLIC caffe2 ${Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS}) target_link_libraries( caffe2_gpu PRIVATE ${Caffe2_CUDA_DEPENDENCY_LIBS}) + + # These public dependencies must go after the previous dependencies, as the + # order of the libraries in the linker call matters here when statically + # linking; libculibos and cublas must be last. + target_link_libraries( + caffe2_gpu PUBLIC caffe2 ${Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS}) + caffe2_interface_library(caffe2_gpu caffe2_gpu_library) list(APPEND Caffe2_MAIN_LIBS caffe2_gpu_library) install(TARGETS caffe2_gpu EXPORT Caffe2Targets DESTINATION lib) diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 3c0bbc6d967..67603d86472 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -378,11 +378,18 @@ endif() if(USE_CUDA) include(cmake/public/cuda.cmake) if(CAFFE2_FOUND_CUDA) - # A helper variable recording the list of Caffe2 dependent librareis + # A helper variable recording the list of Caffe2 dependent libraries # caffe2::cudart is dealt with separately, due to CUDA_ADD_LIBRARY # design reason (it adds CUDA_LIBRARIES itself). set(Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS - caffe2::cuda caffe2::curand caffe2::cublas caffe2::cudnn caffe2::nvrtc) + caffe2::cuda caffe2::curand caffe2::cudnn caffe2::nvrtc) + if(CAFFE2_STATIC_LINK_CUDA) + # When statically linking, this must be the order of the libraries + LIST(APPEND Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS + "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libculibos.a" caffe2::cublas) + else() + LIST(APPEND Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS caffe2::cublas) + endif() if(USE_TENSORRT) list(APPEND Caffe2_PUBLIC_CUDA_DEPENDENCY_LIBS caffe2::tensorrt) endif() diff --git a/cmake/Summary.cmake b/cmake/Summary.cmake index a6538b5fd54..7e80f61ba50 100644 --- a/cmake/Summary.cmake +++ b/cmake/Summary.cmake @@ -38,12 +38,22 @@ function (caffe2_print_configuration_summary) message(STATUS " USE_ASAN : ${USE_ASAN}") message(STATUS " USE_CUDA : ${USE_CUDA}") if(${USE_CUDA}) + message(STATUS " CUDA static link : ${CAFFE2_STATIC_LINK_CUDA}") message(STATUS " CUDA version : ${CUDA_VERSION}") message(STATUS " CuDNN version : ${CUDNN_VERSION}") message(STATUS " CUDA root directory : ${CUDA_TOOLKIT_ROOT_DIR}") - message(STATUS " CUDA library : ${CUDA_CUDA_LIB}") - message(STATUS " CUDA NVRTC library : ${CUDA_NVRTC_LIB}") - message(STATUS " CUDA runtime library: ${CUDA_CUDART_LIBRARY}") + get_target_property(__tmp caffe2::cuda IMPORTED_LOCATION) + message(STATUS " CUDA library : ${__tmp}") + get_target_property(__tmp caffe2::cudart INTERFACE_LINK_LIBRARIES) + message(STATUS " cudart library : ${__tmp}") + get_target_property(__tmp caffe2::cublas INTERFACE_LINK_LIBRARIES) + message(STATUS " cublas library : ${__tmp}") + get_target_property(__tmp caffe2::curand IMPORTED_LOCATION) + message(STATUS " curand library : ${__tmp}") + get_target_property(__tmp caffe2::cudnn IMPORTED_LOCATION) + message(STATUS " CuDNN library : ${__tmp}") + get_target_property(__tmp caffe2::nvrtc IMPORTED_LOCATION) + message(STATUS " nvrtc : ${__tmp}") message(STATUS " CUDA include path : ${CUDA_INCLUDE_DIRS}") message(STATUS " NVCC executable : ${CUDA_NVCC_EXECUTABLE}") message(STATUS " CUDA host compiler : ${CUDA_HOST_COMPILER}") diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake index a83131d932a..98f14349523 100644 --- a/cmake/Utils.cmake +++ b/cmake/Utils.cmake @@ -223,3 +223,31 @@ function(pycmd outvar cmd) string(STRIP "${_output}" _output) set(${outvar} "${_output}" PARENT_SCOPE) endfunction() + +### +# Helper function to print out everything that cmake knows about a target +# +# Copied from https://stackoverflow.com/questions/32183975/how-to-print-all-the-properties-of-a-target-in-cmake +# This isn't called anywhere, but it's very useful when debugging cmake +# NOTE: This doesn't work for INTERFACE_LIBRARY or INTERFACE_LINK_LIBRARY targets + +function(print_target_properties tgt) + if(NOT TARGET ${tgt}) + message("There is no target named '${tgt}'") + return() + endif() + + # Get a list of all cmake properties TODO cache this lazily somehow + execute_process(COMMAND cmake --help-property-list OUTPUT_VARIABLE CMAKE_PROPERTY_LIST) + STRING(REGEX REPLACE ";" "\\\\;" CMAKE_PROPERTY_LIST "${CMAKE_PROPERTY_LIST}") + STRING(REGEX REPLACE "\n" ";" CMAKE_PROPERTY_LIST "${CMAKE_PROPERTY_LIST}") + + foreach (prop ${CMAKE_PROPERTY_LIST}) + string(REPLACE "" "${CMAKE_BUILD_TYPE}" prop ${prop}) + get_property(propval TARGET ${tgt} PROPERTY ${prop} SET) + if (propval) + get_target_property(propval ${tgt} ${prop}) + message ("${tgt} ${prop} = ${propval}") + endif() + endforeach(prop) +endfunction(print_target_properties) diff --git a/cmake/public/cuda.cmake b/cmake/public/cuda.cmake index 29e27120639..3b41f2fd1a6 100644 --- a/cmake/public/cuda.cmake +++ b/cmake/public/cuda.cmake @@ -13,12 +13,17 @@ if(NOT CUDA_FOUND) endif() # Find cudnn. +if(CAFFE2_STATIC_LINK_CUDA) + SET(CUDNN_LIBNAME "libcudnn_static.a") +else() + SET(CUDNN_LIBNAME "cudnn") +endif() include(FindPackageHandleStandardArgs) set(CUDNN_ROOT_DIR "" CACHE PATH "Folder contains NVIDIA cuDNN") find_path(CUDNN_INCLUDE_DIR cudnn.h HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} PATH_SUFFIXES cuda/include include) -find_library(CUDNN_LIBRARY cudnn +find_library(CUDNN_LIBRARY ${CUDNN_LIBNAME} HINTS ${CUDNN_ROOT_DIR} ${CUDA_TOOLKIT_ROOT_DIR} PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64) find_package_handle_standard_args( @@ -89,6 +94,10 @@ find_library(CUDA_NVRTC_LIB nvrtc PATH_SUFFIXES lib lib64 lib/x64) # Create new style imported libraries. +# Several of these libraries have a hardcoded path if CAFFE2_STATIC_LINK_CUDA +# is set. This path is where sane CUDA installations have their static +# libraries installed. This flag should only be used for binary builds, so +# end-users should never have this flag set. # cuda add_library(caffe2::cuda UNKNOWN IMPORTED) @@ -102,14 +111,21 @@ set_property( # cudart. CUDA_LIBRARIES is actually a list, so we will make an interface # library. add_library(caffe2::cudart INTERFACE IMPORTED) -set_property( - TARGET caffe2::cudart PROPERTY INTERFACE_LINK_LIBRARIES - ${CUDA_LIBRARIES}) +if(CAFFE2_STATIC_LINK_CUDA) + set_property( + TARGET caffe2::cudart PROPERTY INTERFACE_LINK_LIBRARIES + "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcudart_static.a") +else() + set_property( + TARGET caffe2::cudart PROPERTY INTERFACE_LINK_LIBRARIES + ${CUDA_LIBRARIES}) +endif() set_property( TARGET caffe2::cudart PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${CUDA_INCLUDE_DIRS}) # cudnn +# static linking is handled by USE_STATIC_CUDNN environment variable add_library(caffe2::cudnn UNKNOWN IMPORTED) set_property( TARGET caffe2::cudnn PROPERTY IMPORTED_LOCATION @@ -120,9 +136,15 @@ set_property( # curand add_library(caffe2::curand UNKNOWN IMPORTED) -set_property( - TARGET caffe2::curand PROPERTY IMPORTED_LOCATION - ${CUDA_curand_LIBRARY}) +if(CAFFE2_STATIC_LINK_CUDA) + set_property( + TARGET caffe2::curand PROPERTY IMPORTED_LOCATION + "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcurand_static.a") +else() + set_property( + TARGET caffe2::curand PROPERTY IMPORTED_LOCATION + ${CUDA_curand_LIBRARY}) +endif() set_property( TARGET caffe2::curand PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${CUDA_INCLUDE_DIRS}) @@ -141,9 +163,15 @@ endif() # cublas. CUDA_CUBLAS_LIBRARIES is actually a list, so we will make an # interface library similar to cudart. add_library(caffe2::cublas INTERFACE IMPORTED) -set_property( - TARGET caffe2::cublas PROPERTY INTERFACE_LINK_LIBRARIES - ${CUDA_CUBLAS_LIBRARIES}) +if(CAFFE2_STATIC_LINK_CUDA) + set_property( + TARGET caffe2::cublas PROPERTY INTERFACE_LINK_LIBRARIES + "${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcublas_static.a") +else() + set_property( + TARGET caffe2::cublas PROPERTY INTERFACE_LINK_LIBRARIES + ${CUDA_CUBLAS_LIBRARIES}) +endif() set_property( TARGET caffe2::cublas PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${CUDA_INCLUDE_DIRS}) diff --git a/conda/caffe2/normal/build.sh b/conda/caffe2/normal/build.sh index 79b6ccf0b6a..fcb50b5c27b 100755 --- a/conda/caffe2/normal/build.sh +++ b/conda/caffe2/normal/build.sh @@ -28,7 +28,7 @@ CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$PREFIX") # Build Caffe2 mkdir -p build cd build -cmake "${CMAKE_ARGS[@]}" "$PYTHON_ARGS" $CONDA_CMAKE_BUILD_ARGS .. +cmake "${CMAKE_ARGS[@]}" "$PYTHON_ARGS" $CONDA_CAFFE2_CMAKE_ARGS .. if [ "$(uname)" == 'Darwin' ]; then make "-j$(sysctl -n hw.ncpu)" else diff --git a/conda/integrated/build.sh b/conda/integrated/build.sh index a406d29c50e..b7432428cb8 100755 --- a/conda/integrated/build.sh +++ b/conda/integrated/build.sh @@ -32,6 +32,9 @@ export TH_BINARY_BUILD=1 export PYTORCH_BUILD_VERSION=$PKG_VERSION export PYTORCH_BUILD_NUMBER=$PKG_BUILDNUM export NCCL_ROOT_DIR=/usr/local/cuda +export USE_STATIC_CUDNN=1 +export USE_STATIC_NCCL=1 +export ATEN_STATIC_CUDA=1 ########################################################### diff --git a/scripts/build_anaconda.sh b/scripts/build_anaconda.sh index f6913f8f186..75e4e37ea1f 100755 --- a/scripts/build_anaconda.sh +++ b/scripts/build_anaconda.sh @@ -284,14 +284,20 @@ fi if [[ -n $CUDA_VERSION ]]; then CAFFE2_CMAKE_ARGS+=("-DUSE_CUDA=ON") CAFFE2_CMAKE_ARGS+=("-DUSE_NCCL=ON") + + # NCCL and GLOO don't work with static CUDA right now. Cmake changes are + # needed + #CAFFE2_CMAKE_ARGS+=("-DUSE_NCCL=OFF") + #CAFFE2_CMAKE_ARGS+=("-DUSE_GLOO=OFF") + #CAFFE2_CMAKE_ARGS+=("-DCAFFE2_STATIC_LINK_CUDA=ON") else # Flags required for CPU for Caffe2 CAFFE2_CMAKE_ARGS+=("-DUSE_CUDA=OFF") CAFFE2_CMAKE_ARGS+=("-DUSE_NCCL=OFF") if [[ -z $BUILD_INTEGRATED ]]; then - CAFFE2_CMAKE_ARGS+=("-DBLAS=MKL") - add_package 'mkl' - add_package 'mkl-include' + #CAFFE2_CMAKE_ARGS+=("-DBLAS=MKL") + #add_package 'mkl' + #add_package 'mkl-include' fi fi @@ -320,9 +326,10 @@ if [[ -z $SKIP_CONDA_TESTS && -n $UPLOAD_TO_CONDA ]]; then CONDA_BUILD_ARGS+=(" --token ${CAFFE2_ANACONDA_ORG_ACCESS_TOKEN}") # If building a redistributable, then package the CUDA libraries with it - if [[ -n $CUDA_VERSION ]]; then - export PACKAGE_CUDA_LIBS=1 - fi + # TODO this doesn't work on Ubuntu right now + #if [[ -n $CUDA_VERSION ]]; then + # export PACKAGE_CUDA_LIBS=1 + #fi # Show what the final meta.yaml looks like echo "Finalized meta.yaml is"