mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-06 12:20:52 +01:00
6350dbddd1
Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/22062 ghimport-source-id: a30255d7453c4ffecf40215a785c1e06b7296368 Test Plan: USE_CUDA=0 PARALLEL_BACKEND=OPENMP BLAS=MKL USE_MKLDNN=1 MKL_SEQ=1 MKLDNN_THREADING=SEQ BUILD_BINARY=1 python setup.py develop --cmake ./build/bin/parallel_info Imported from OSS Differential Revision: D15938079 Pulled By: ilia-cher fbshipit-source-id: e7ef0c5bc75ebb845ebe66bf76a4070d45305b35
387 lines
16 KiB
Python
387 lines
16 KiB
Python
"Manages CMake."
|
|
|
|
from __future__ import print_function
|
|
|
|
import multiprocessing
|
|
import os
|
|
import re
|
|
from subprocess import check_call, check_output
|
|
import sys
|
|
import distutils
|
|
import distutils.sysconfig
|
|
from distutils.version import LooseVersion
|
|
|
|
from . import escape_path
|
|
from .env import (IS_64BIT, IS_DARWIN, IS_WINDOWS,
|
|
DEBUG, REL_WITH_DEB_INFO, USE_MKLDNN,
|
|
check_env_flag, check_negative_env_flag)
|
|
from .cuda import USE_CUDA
|
|
from .dist_check import USE_DISTRIBUTED, USE_GLOO_IBVERBS
|
|
from .nccl import (USE_SYSTEM_NCCL, NCCL_INCLUDE_DIR, NCCL_ROOT_DIR,
|
|
NCCL_SYSTEM_LIB, USE_NCCL)
|
|
from .numpy_ import USE_NUMPY, NUMPY_INCLUDE_DIR
|
|
from .rocm import USE_ROCM
|
|
from .nnpack import USE_NNPACK
|
|
from .qnnpack import USE_QNNPACK
|
|
|
|
|
|
def _which(thefile):
|
|
path = os.environ.get("PATH", os.defpath).split(os.pathsep)
|
|
for d in path:
|
|
fname = os.path.join(d, thefile)
|
|
fnames = [fname]
|
|
if IS_WINDOWS:
|
|
exts = os.environ.get('PATHEXT', '').split(os.pathsep)
|
|
fnames += [fname + ext for ext in exts]
|
|
for name in fnames:
|
|
if os.access(name, os.F_OK | os.X_OK) and not os.path.isdir(name):
|
|
return name
|
|
return None
|
|
|
|
|
|
def _mkdir_p(d):
|
|
try:
|
|
os.makedirs(d)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# Ninja
|
|
# Use ninja if it is on the PATH. Previous version of PyTorch required the
|
|
# ninja python package, but we no longer use it, so we do not have to import it
|
|
USE_NINJA = (not check_negative_env_flag('USE_NINJA') and
|
|
_which('ninja') is not None)
|
|
|
|
|
|
class CMake:
|
|
"Manages cmake."
|
|
|
|
def __init__(self, build_dir):
|
|
self._cmake_command = CMake._get_cmake_command()
|
|
self.build_dir = build_dir
|
|
|
|
if DEBUG:
|
|
self._build_type = "Debug"
|
|
elif REL_WITH_DEB_INFO:
|
|
self._build_type = "RelWithDebInfo"
|
|
else:
|
|
self._build_type = "Release"
|
|
|
|
@property
|
|
def _cmake_cache_file(self):
|
|
r"""Returns the path to CMakeCache.txt.
|
|
|
|
Returns:
|
|
string: The path to CMakeCache.txt.
|
|
"""
|
|
return os.path.join(self.build_dir, 'CMakeCache.txt')
|
|
|
|
@staticmethod
|
|
def _get_cmake_command():
|
|
"Returns cmake command."
|
|
|
|
cmake_command = 'cmake'
|
|
if IS_WINDOWS:
|
|
return cmake_command
|
|
cmake3 = _which('cmake3')
|
|
if cmake3 is not None:
|
|
cmake = _which('cmake')
|
|
if cmake is not None:
|
|
bare_version = CMake._get_version(cmake)
|
|
if (bare_version < LooseVersion("3.5.0") and
|
|
CMake._get_version(cmake3) > bare_version):
|
|
cmake_command = 'cmake3'
|
|
return cmake_command
|
|
|
|
@staticmethod
|
|
def _get_version(cmd):
|
|
"Returns cmake version."
|
|
|
|
for line in check_output([cmd, '--version']).decode('utf-8').split('\n'):
|
|
if 'version' in line:
|
|
return LooseVersion(line.strip().split(' ')[2])
|
|
raise RuntimeError('no version found')
|
|
|
|
def run(self, args, env):
|
|
"Executes cmake with arguments and an environment."
|
|
|
|
command = [self._cmake_command] + args
|
|
print(' '.join(command))
|
|
check_call(command, cwd=self.build_dir, env=env)
|
|
|
|
@staticmethod
|
|
def defines(args, **kwargs):
|
|
"Adds definitions to a cmake argument list."
|
|
for key, value in sorted(kwargs.items()):
|
|
if value is not None:
|
|
args.append('-D{}={}'.format(key, value))
|
|
|
|
@staticmethod
|
|
def convert_cmake_value_to_python_value(cmake_value, cmake_type):
|
|
r"""Convert a CMake value in a string form to a Python value.
|
|
|
|
Arguments:
|
|
cmake_value (string): The CMake value in a string form (e.g., "ON", "OFF", "1").
|
|
cmake_type (string): The CMake type of :attr:`cmake_value`.
|
|
|
|
Returns:
|
|
A Python value corresponding to :attr:`cmake_value` with type :attr:`cmake_type`.
|
|
"""
|
|
|
|
cmake_type = cmake_type.upper()
|
|
up_val = cmake_value.upper()
|
|
if cmake_type == 'BOOL':
|
|
# https://gitlab.kitware.com/cmake/community/wikis/doc/cmake/VariablesListsStrings#boolean-values-in-cmake
|
|
return not (up_val in ('FALSE', 'OFF', 'N', 'NO', '0', '', 'NOTFOUND') or up_val.endswith('-NOTFOUND'))
|
|
elif cmake_type == 'FILEPATH':
|
|
if up_val.endswith('-NOTFOUND'):
|
|
return None
|
|
else:
|
|
return cmake_value
|
|
else: # Directly return the cmake_value.
|
|
return cmake_value
|
|
|
|
@staticmethod
|
|
def _get_cmake_cache_variables(cmake_cache_file):
|
|
r"""Gets values in CMakeCache.txt into a dictionary.
|
|
|
|
Arguments:
|
|
cmake_cache_file: A CMakeCache.txt file object.
|
|
Returns:
|
|
dict: A ``dict`` containing the value of cached CMake variables.
|
|
"""
|
|
|
|
results = dict()
|
|
for line in cmake_cache_file:
|
|
line = line.strip()
|
|
if not line or line.startswith(('#', '//')):
|
|
# Blank or comment line, skip
|
|
continue
|
|
|
|
# Space can also be part of variable name and value
|
|
matched = re.match(r'(\S.*):\s*([a-zA-Z_-][a-zA-Z0-9_-]*)\s*=\s*(.*)', line)
|
|
if matched is None: # Illegal line
|
|
raise ValueError('Unexpected line in {}: {}'.format(repr(cmake_cache_file), line))
|
|
variable, type_, value = matched.groups()
|
|
if type_.upper() in ('INTERNAL', 'STATIC'):
|
|
# CMake internal variable, do not touch
|
|
continue
|
|
results[variable] = CMake.convert_cmake_value_to_python_value(value, type_)
|
|
|
|
return results
|
|
|
|
def get_cmake_cache_variables(self):
|
|
r"""Gets values in CMakeCache.txt into a dictionary.
|
|
Returns:
|
|
dict: A ``dict`` containing the value of cached CMake variables.
|
|
"""
|
|
with open(self._cmake_cache_file) as f:
|
|
return CMake._get_cmake_cache_variables(f)
|
|
|
|
def generate(self, version, cmake_python_library, build_python, build_test, my_env, rerun):
|
|
"Runs cmake to generate native build files."
|
|
|
|
if rerun and os.path.isfile(self._cmake_cache_file):
|
|
os.remove(self._cmake_cache_file)
|
|
ninja_build_file = os.path.join(self.build_dir, 'build.ninja')
|
|
if os.path.exists(self._cmake_cache_file) and not (
|
|
USE_NINJA and not os.path.exists(ninja_build_file)):
|
|
# Everything's in place. Do not rerun.
|
|
return
|
|
|
|
args = []
|
|
if USE_NINJA:
|
|
args.append('-GNinja')
|
|
elif IS_WINDOWS:
|
|
generator = os.getenv('CMAKE_GENERATOR', 'Visual Studio 15 2017')
|
|
supported = ['Visual Studio 15 2017', 'Visual Studio 16 2019']
|
|
if generator not in supported:
|
|
print('Unsupported `CMAKE_GENERATOR`: ' + generator)
|
|
print('Please set it to one of the following values: ')
|
|
print('\n'.join(supported))
|
|
sys.exit(1)
|
|
args.append('-G' + generator)
|
|
toolset_dict = {}
|
|
toolset_version = os.getenv('CMAKE_GENERATOR_TOOLSET_VERSION')
|
|
if toolset_version is not None:
|
|
toolset_dict['version'] = toolset_version
|
|
curr_toolset = os.getenv('VCToolsVersion')
|
|
if curr_toolset is None:
|
|
print('When you specify `CMAKE_GENERATOR_TOOLSET_VERSION`, you must also '
|
|
'activate the vs environment of this version. Please read the notes '
|
|
'in the build steps carefully.')
|
|
sys.exit(1)
|
|
if IS_64BIT:
|
|
args.append('-Ax64')
|
|
toolset_dict['host'] = 'x64'
|
|
if toolset_dict:
|
|
toolset_expr = ','.join(["{}={}".format(k, v) for k, v in toolset_dict.items()])
|
|
args.append('-T' + toolset_expr)
|
|
|
|
cflags = os.getenv('CFLAGS', "") + " " + os.getenv('CPPFLAGS', "")
|
|
ldflags = os.getenv('LDFLAGS', "")
|
|
if IS_WINDOWS:
|
|
CMake.defines(args, MSVC_Z7_OVERRIDE=not check_negative_env_flag(
|
|
'MSVC_Z7_OVERRIDE'))
|
|
cflags += " /EHa"
|
|
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.dirname(
|
|
os.path.abspath(__file__))))
|
|
install_dir = os.path.join(base_dir, "torch")
|
|
|
|
_mkdir_p(install_dir)
|
|
_mkdir_p(self.build_dir)
|
|
|
|
# Store build options that are directly stored in environment variables
|
|
build_options = {
|
|
# The default value cannot be easily obtained in CMakeLists.txt. We set it here.
|
|
'CMAKE_PREFIX_PATH': distutils.sysconfig.get_python_lib()
|
|
}
|
|
# Options that do not start with 'USE_' or 'BUILD_' and are directly controlled by env vars
|
|
additional_options = {
|
|
'BLAS',
|
|
'BUILDING_WITH_TORCH_LIBS',
|
|
'CMAKE_PREFIX_PATH',
|
|
'ONNX_ML',
|
|
'ONNX_NAMESPACE',
|
|
'WERROR'
|
|
}
|
|
for var, val in my_env.items():
|
|
# We currently pass over all environment variables that start with "BUILD_" or "USE_". This is because we
|
|
# currently have no reliable way to get the list of all build options we have specified in CMakeLists.txt.
|
|
# (`cmake -L` won't print dependent options when the dependency condition is not met.) We will possibly
|
|
# change this in the future by parsing CMakeLists.txt ourselves (then additional_options would also not be
|
|
# needed to be specified here).
|
|
if var.startswith(('USE_', 'BUILD_')) or var in additional_options:
|
|
build_options[var] = val
|
|
|
|
# Some options must be post-processed. Ideally, this list will be shrunk to only one or two options in the
|
|
# future, as CMake can detect many of these libraries pretty comfortably. We have them here for now before CMake
|
|
# integration is completed. They appear here not in the CMake.defines call below because they start with either
|
|
# "BUILD_" or "USE_" and must be overwritten here.
|
|
build_options.update({
|
|
'BUILD_PYTHON': build_python,
|
|
'BUILD_TEST': build_test,
|
|
'USE_CUDA': USE_CUDA,
|
|
'USE_DISTRIBUTED': USE_DISTRIBUTED,
|
|
'USE_FBGEMM': not (check_env_flag('NO_FBGEMM') or
|
|
check_negative_env_flag('USE_FBGEMM')),
|
|
'USE_MKLDNN': USE_MKLDNN,
|
|
'USE_NNPACK': USE_NNPACK,
|
|
'USE_QNNPACK': USE_QNNPACK,
|
|
'USE_NCCL': USE_NCCL,
|
|
'USE_SYSTEM_NCCL': USE_SYSTEM_NCCL,
|
|
'USE_NUMPY': USE_NUMPY,
|
|
'USE_ROCM': USE_ROCM,
|
|
'USE_SYSTEM_EIGEN_INSTALL': 'OFF'
|
|
})
|
|
|
|
CMake.defines(args,
|
|
PYTHON_EXECUTABLE=escape_path(sys.executable),
|
|
PYTHON_LIBRARY=escape_path(cmake_python_library),
|
|
PYTHON_INCLUDE_DIR=escape_path(distutils.sysconfig.get_python_inc()),
|
|
TORCH_BUILD_VERSION=version,
|
|
CMAKE_BUILD_TYPE=self._build_type,
|
|
INSTALL_TEST=build_test,
|
|
NAMEDTENSOR_ENABLED=(check_env_flag('USE_NAMEDTENSOR') or
|
|
check_negative_env_flag('NO_NAMEDTENSOR')),
|
|
NUMPY_INCLUDE_DIR=escape_path(NUMPY_INCLUDE_DIR),
|
|
NCCL_INCLUDE_DIR=NCCL_INCLUDE_DIR,
|
|
NCCL_ROOT_DIR=NCCL_ROOT_DIR,
|
|
NCCL_SYSTEM_LIB=NCCL_SYSTEM_LIB,
|
|
CAFFE2_STATIC_LINK_CUDA=check_env_flag('USE_CUDA_STATIC_LINK'),
|
|
NCCL_EXTERNAL=USE_NCCL,
|
|
CMAKE_INSTALL_PREFIX=install_dir,
|
|
CMAKE_C_FLAGS=cflags,
|
|
CMAKE_CXX_FLAGS=cflags,
|
|
CMAKE_EXE_LINKER_FLAGS=ldflags,
|
|
CMAKE_SHARED_LINKER_FLAGS=ldflags,
|
|
THD_SO_VERSION="1",
|
|
CUDA_NVCC_EXECUTABLE=escape_path(os.getenv('CUDA_NVCC_EXECUTABLE')),
|
|
**build_options)
|
|
|
|
if os.getenv('_GLIBCXX_USE_CXX11_ABI'):
|
|
CMake.defines(args, GLIBCXX_USE_CXX11_ABI=os.getenv('_GLIBCXX_USE_CXX11_ABI'))
|
|
|
|
if os.getenv('USE_OPENMP'):
|
|
CMake.defines(args, USE_OPENMP=check_env_flag('USE_OPENMP'))
|
|
|
|
if os.getenv('USE_TBB'):
|
|
CMake.defines(args, USE_TBB=check_env_flag('USE_TBB'))
|
|
|
|
mkl_threading = os.getenv('MKL_THREADING')
|
|
if mkl_threading:
|
|
CMake.defines(args, MKL_THREADING=mkl_threading)
|
|
|
|
mkldnn_threading = os.getenv('MKLDNN_THREADING')
|
|
if mkldnn_threading:
|
|
CMake.defines(args, MKLDNN_THREADING=mkldnn_threading)
|
|
|
|
parallel_backend = os.getenv('PARALLEL_BACKEND')
|
|
if parallel_backend:
|
|
CMake.defines(args, PARALLEL_BACKEND=parallel_backend)
|
|
single_thread_pool = os.getenv('EXPERIMENTAL_SINGLE_THREAD_POOL')
|
|
if single_thread_pool:
|
|
CMake.defines(args, EXPERIMENTAL_SINGLE_THREAD_POOL=single_thread_pool)
|
|
|
|
if USE_GLOO_IBVERBS:
|
|
CMake.defines(args, USE_IBVERBS="1", USE_GLOO_IBVERBS="1")
|
|
|
|
if USE_MKLDNN:
|
|
CMake.defines(args, MKLDNN_ENABLE_CONCURRENT_EXEC="ON")
|
|
|
|
expected_wrapper = '/usr/local/opt/ccache/libexec'
|
|
if IS_DARWIN and os.path.exists(expected_wrapper):
|
|
CMake.defines(args,
|
|
CMAKE_C_COMPILER="{}/gcc".format(expected_wrapper),
|
|
CMAKE_CXX_COMPILER="{}/g++".format(expected_wrapper))
|
|
for env_var_name in my_env:
|
|
if env_var_name.startswith('gh'):
|
|
# github env vars use utf-8, on windows, non-ascii code may
|
|
# cause problem, so encode first
|
|
try:
|
|
my_env[env_var_name] = str(my_env[env_var_name].encode("utf-8"))
|
|
except UnicodeDecodeError as e:
|
|
shex = ':'.join('{:02x}'.format(ord(c)) for c in my_env[env_var_name])
|
|
print('Invalid ENV[{}] = {}'.format(env_var_name, shex), file=sys.stderr)
|
|
print(e, file=sys.stderr)
|
|
# According to the CMake manual, we should pass the arguments first,
|
|
# and put the directory as the last element. Otherwise, these flags
|
|
# may not be passed correctly.
|
|
# Reference:
|
|
# 1. https://cmake.org/cmake/help/latest/manual/cmake.1.html#synopsis
|
|
# 2. https://stackoverflow.com/a/27169347
|
|
args.append(base_dir)
|
|
self.run(args, env=my_env)
|
|
|
|
def build(self, my_env):
|
|
"Runs cmake to build binaries."
|
|
|
|
max_jobs = os.getenv('MAX_JOBS', str(multiprocessing.cpu_count()))
|
|
build_args = ['--build', '.', '--target',
|
|
'install', '--config', self._build_type]
|
|
# This ``if-else'' clause would be unnecessary when cmake 3.12 becomes
|
|
# minimum, which provides a '-j' option: build_args += ['-j', max_jobs]
|
|
# would be sufficient by then.
|
|
if IS_WINDOWS and not USE_NINJA: # We are likely using msbuild here
|
|
build_args += ['--', '/maxcpucount:{}'.format(max_jobs)]
|
|
else:
|
|
build_args += ['--', '-j', max_jobs]
|
|
self.run(build_args, my_env)
|
|
|
|
# in cmake, .cu compilation involves generating certain intermediates
|
|
# such as .cu.o and .cu.depend, and these intermediates finally get compiled
|
|
# into the final .so.
|
|
# Ninja updates build.ninja's timestamp after all dependent files have been built,
|
|
# and re-kicks cmake on incremental builds if any of the dependent files
|
|
# have a timestamp newer than build.ninja's timestamp.
|
|
# There is a cmake bug with the Ninja backend, where the .cu.depend files
|
|
# are still compiling by the time the build.ninja timestamp is updated,
|
|
# so the .cu.depend file's newer timestamp is screwing with ninja's incremental
|
|
# build detector.
|
|
# This line works around that bug by manually updating the build.ninja timestamp
|
|
# after the entire build is finished.
|
|
ninja_build_file = os.path.join(self.build_dir, 'build.ninja')
|
|
if os.path.exists(ninja_build_file):
|
|
os.utime(ninja_build_file, None)
|