mirror of
https://github.com/zebrajr/pytorch.git
synced 2025-12-06 12:20:52 +01:00
047e68235f
Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/64733 The previous implementation was wrong when CPU scheduling affinity is set. In fact, it is still wrong if Ninja is not being used. When there is CPU scheduling affinity set, the number of processors available on the system likely exceeds the number of processors that are usable to the build. We ought to use `len(os.sched_getaffinity(0))` to determine the effective parallelism. This change is more minimal and instead just delegates to Ninja (which handles this correctly) when it is used. Test Plan: I verified this worked as correctly using Ninja on a 96-core machine with 24 cores available for scheduling by checking: * the cmake command did not specify "-j" * the number of top-level jobs in top/pstree never exceeded 26 (24 + 2) And I verified we get the legacy behavior by specifying USE_NINJA=0 on the build. Reviewed By: jbschlosser, driazati Differential Revision: D30968796 Pulled By: dagitses fbshipit-source-id: 29547dd378fea793957bcc2f7d52d5def1ecace2
396 lines
17 KiB
Python
396 lines
17 KiB
Python
"Manages CMake."
|
|
|
|
|
|
|
|
import multiprocessing
|
|
import os
|
|
import re
|
|
from subprocess import check_call, check_output, CalledProcessError
|
|
import sys
|
|
import sysconfig
|
|
from setuptools import distutils # type: ignore[import]
|
|
from typing import IO, Any, Dict, List, Optional, Union, cast
|
|
|
|
from . import which
|
|
from .env import (BUILD_DIR, IS_64BIT, IS_DARWIN, IS_WINDOWS, check_negative_env_flag)
|
|
from .numpy_ import USE_NUMPY, NUMPY_INCLUDE_DIR
|
|
|
|
|
|
def _mkdir_p(d: str) -> None:
|
|
try:
|
|
os.makedirs(d)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# Ninja
|
|
# Use ninja if it is on the PATH. Previous version of PyTorch required the
|
|
# ninja python package, but we no longer use it, so we do not have to import it
|
|
USE_NINJA = (not check_negative_env_flag('USE_NINJA') and
|
|
which('ninja') is not None)
|
|
|
|
|
|
CMakeValue = Optional[Union[bool, str]]
|
|
|
|
|
|
def convert_cmake_value_to_python_value(cmake_value: str, cmake_type: str) -> CMakeValue:
|
|
r"""Convert a CMake value in a string form to a Python value.
|
|
|
|
Args:
|
|
cmake_value (string): The CMake value in a string form (e.g., "ON", "OFF", "1").
|
|
cmake_type (string): The CMake type of :attr:`cmake_value`.
|
|
|
|
Returns:
|
|
A Python value corresponding to :attr:`cmake_value` with type :attr:`cmake_type`.
|
|
"""
|
|
|
|
cmake_type = cmake_type.upper()
|
|
up_val = cmake_value.upper()
|
|
if cmake_type == 'BOOL':
|
|
# https://gitlab.kitware.com/cmake/community/wikis/doc/cmake/VariablesListsStrings#boolean-values-in-cmake
|
|
return not (up_val in ('FALSE', 'OFF', 'N', 'NO', '0', '', 'NOTFOUND') or up_val.endswith('-NOTFOUND'))
|
|
elif cmake_type == 'FILEPATH':
|
|
if up_val.endswith('-NOTFOUND'):
|
|
return None
|
|
else:
|
|
return cmake_value
|
|
else: # Directly return the cmake_value.
|
|
return cmake_value
|
|
|
|
def get_cmake_cache_variables_from_file(cmake_cache_file: IO[str]) -> Dict[str, CMakeValue]:
|
|
r"""Gets values in CMakeCache.txt into a dictionary.
|
|
|
|
Args:
|
|
cmake_cache_file: A CMakeCache.txt file object.
|
|
Returns:
|
|
dict: A ``dict`` containing the value of cached CMake variables.
|
|
"""
|
|
|
|
results = dict()
|
|
for i, line in enumerate(cmake_cache_file, 1):
|
|
line = line.strip()
|
|
if not line or line.startswith(('#', '//')):
|
|
# Blank or comment line, skip
|
|
continue
|
|
|
|
# Almost any character can be part of variable name and value. As a practical matter, we assume the type must be
|
|
# valid if it were a C variable name. It should match the following kinds of strings:
|
|
#
|
|
# USE_CUDA:BOOL=ON
|
|
# "USE_CUDA":BOOL=ON
|
|
# USE_CUDA=ON
|
|
# USE_CUDA:=ON
|
|
# Intel(R) MKL-DNN_SOURCE_DIR:STATIC=/path/to/pytorch/third_party/ideep/mkl-dnn
|
|
# "OpenMP_COMPILE_RESULT_CXX_openmp:experimental":INTERNAL=FALSE
|
|
matched = re.match(r'("?)(.+?)\1(?::\s*([a-zA-Z_-][a-zA-Z0-9_-]*)?)?\s*=\s*(.*)', line)
|
|
if matched is None: # Illegal line
|
|
raise ValueError('Unexpected line {} in {}: {}'.format(i, repr(cmake_cache_file), line))
|
|
_, variable, type_, value = matched.groups()
|
|
if type_ is None:
|
|
type_ = ''
|
|
if type_.upper() in ('INTERNAL', 'STATIC'):
|
|
# CMake internal variable, do not touch
|
|
continue
|
|
results[variable] = convert_cmake_value_to_python_value(value, type_)
|
|
|
|
return results
|
|
|
|
class CMake:
|
|
"Manages cmake."
|
|
|
|
def __init__(self, build_dir: str = BUILD_DIR) -> None:
|
|
self._cmake_command = CMake._get_cmake_command()
|
|
self.build_dir = build_dir
|
|
|
|
@property
|
|
def _cmake_cache_file(self) -> str:
|
|
r"""Returns the path to CMakeCache.txt.
|
|
|
|
Returns:
|
|
string: The path to CMakeCache.txt.
|
|
"""
|
|
return os.path.join(self.build_dir, 'CMakeCache.txt')
|
|
|
|
@staticmethod
|
|
def _get_cmake_command() -> str:
|
|
"Returns cmake command."
|
|
|
|
cmake_command = 'cmake'
|
|
if IS_WINDOWS:
|
|
return cmake_command
|
|
cmake3 = which('cmake3')
|
|
cmake = which('cmake')
|
|
if cmake3 is not None and CMake._get_version(cmake3) >= distutils.version.LooseVersion("3.10.0"):
|
|
cmake_command = 'cmake3'
|
|
return cmake_command
|
|
elif cmake is not None and CMake._get_version(cmake) >= distutils.version.LooseVersion("3.10.0"):
|
|
return cmake_command
|
|
else:
|
|
raise RuntimeError('no cmake or cmake3 with version >= 3.10.0 found')
|
|
|
|
@staticmethod
|
|
def _get_version(cmd: str) -> Any:
|
|
"Returns cmake version."
|
|
|
|
for line in check_output([cmd, '--version']).decode('utf-8').split('\n'):
|
|
if 'version' in line:
|
|
return distutils.version.LooseVersion(line.strip().split(' ')[2])
|
|
raise RuntimeError('no version found')
|
|
|
|
def run(self, args: List[str], env: Dict[str, str]) -> None:
|
|
"Executes cmake with arguments and an environment."
|
|
|
|
command = [self._cmake_command] + args
|
|
print(' '.join(command))
|
|
try:
|
|
check_call(command, cwd=self.build_dir, env=env)
|
|
except (CalledProcessError, KeyboardInterrupt) as e:
|
|
# This error indicates that there was a problem with cmake, the
|
|
# Python backtrace adds no signal here so skip over it by catching
|
|
# the error and exiting manually
|
|
sys.exit(1)
|
|
|
|
@staticmethod
|
|
def defines(args: List[str], **kwargs: CMakeValue) -> None:
|
|
"Adds definitions to a cmake argument list."
|
|
for key, value in sorted(kwargs.items()):
|
|
if value is not None:
|
|
args.append('-D{}={}'.format(key, value))
|
|
|
|
def get_cmake_cache_variables(self) -> Dict[str, CMakeValue]:
|
|
r"""Gets values in CMakeCache.txt into a dictionary.
|
|
Returns:
|
|
dict: A ``dict`` containing the value of cached CMake variables.
|
|
"""
|
|
with open(self._cmake_cache_file) as f:
|
|
return get_cmake_cache_variables_from_file(f)
|
|
|
|
def generate(
|
|
self,
|
|
version: Optional[str],
|
|
cmake_python_library: Optional[str],
|
|
build_python: bool,
|
|
build_test: bool,
|
|
my_env: Dict[str, str],
|
|
rerun: bool,
|
|
) -> None:
|
|
"Runs cmake to generate native build files."
|
|
|
|
if rerun and os.path.isfile(self._cmake_cache_file):
|
|
os.remove(self._cmake_cache_file)
|
|
|
|
ninja_build_file = os.path.join(self.build_dir, 'build.ninja')
|
|
if os.path.exists(self._cmake_cache_file) and not (
|
|
USE_NINJA and not os.path.exists(ninja_build_file)):
|
|
# Everything's in place. Do not rerun.
|
|
return
|
|
|
|
args = []
|
|
if USE_NINJA:
|
|
# Avoid conflicts in '-G' and the `CMAKE_GENERATOR`
|
|
os.environ['CMAKE_GENERATOR'] = 'Ninja'
|
|
args.append('-GNinja')
|
|
elif IS_WINDOWS:
|
|
generator = os.getenv('CMAKE_GENERATOR', 'Visual Studio 15 2017')
|
|
supported = ['Visual Studio 15 2017', 'Visual Studio 16 2019']
|
|
if generator not in supported:
|
|
print('Unsupported `CMAKE_GENERATOR`: ' + generator)
|
|
print('Please set it to one of the following values: ')
|
|
print('\n'.join(supported))
|
|
sys.exit(1)
|
|
args.append('-G' + generator)
|
|
toolset_dict = {}
|
|
toolset_version = os.getenv('CMAKE_GENERATOR_TOOLSET_VERSION')
|
|
if toolset_version is not None:
|
|
toolset_dict['version'] = toolset_version
|
|
curr_toolset = os.getenv('VCToolsVersion')
|
|
if curr_toolset is None:
|
|
print('When you specify `CMAKE_GENERATOR_TOOLSET_VERSION`, you must also '
|
|
'activate the vs environment of this version. Please read the notes '
|
|
'in the build steps carefully.')
|
|
sys.exit(1)
|
|
if IS_64BIT:
|
|
args.append('-Ax64')
|
|
toolset_dict['host'] = 'x64'
|
|
if toolset_dict:
|
|
toolset_expr = ','.join(["{}={}".format(k, v) for k, v in toolset_dict.items()])
|
|
args.append('-T' + toolset_expr)
|
|
|
|
base_dir = os.path.dirname(os.path.dirname(os.path.dirname(
|
|
os.path.abspath(__file__))))
|
|
install_dir = os.path.join(base_dir, "torch")
|
|
|
|
_mkdir_p(install_dir)
|
|
_mkdir_p(self.build_dir)
|
|
|
|
# Store build options that are directly stored in environment variables
|
|
build_options: Dict[str, CMakeValue] = {}
|
|
|
|
# Build options that do not start with "BUILD_", "USE_", or "CMAKE_" and are directly controlled by env vars.
|
|
# This is a dict that maps environment variables to the corresponding variable name in CMake.
|
|
additional_options = {
|
|
# Key: environment variable name. Value: Corresponding variable name to be passed to CMake. If you are
|
|
# adding a new build option to this block: Consider making these two names identical and adding this option
|
|
# in the block below.
|
|
'_GLIBCXX_USE_CXX11_ABI': 'GLIBCXX_USE_CXX11_ABI',
|
|
'CUDNN_LIB_DIR': 'CUDNN_LIBRARY',
|
|
'USE_CUDA_STATIC_LINK': 'CAFFE2_STATIC_LINK_CUDA',
|
|
}
|
|
additional_options.update({
|
|
# Build options that have the same environment variable name and CMake variable name and that do not start
|
|
# with "BUILD_", "USE_", or "CMAKE_". If you are adding a new build option, also make sure you add it to
|
|
# CMakeLists.txt.
|
|
var: var for var in
|
|
('BLAS',
|
|
'BUILDING_WITH_TORCH_LIBS',
|
|
'CUDA_HOST_COMPILER',
|
|
'CUDA_NVCC_EXECUTABLE',
|
|
'CUDA_SEPARABLE_COMPILATION',
|
|
'CUDNN_LIBRARY',
|
|
'CUDNN_INCLUDE_DIR',
|
|
'CUDNN_ROOT',
|
|
'EXPERIMENTAL_SINGLE_THREAD_POOL',
|
|
'INSTALL_TEST',
|
|
'JAVA_HOME',
|
|
'INTEL_MKL_DIR',
|
|
'INTEL_OMP_DIR',
|
|
'MKL_THREADING',
|
|
'MKLDNN_CPU_RUNTIME',
|
|
'MSVC_Z7_OVERRIDE',
|
|
'CAFFE2_USE_MSVC_STATIC_RUNTIME',
|
|
'Numa_INCLUDE_DIR',
|
|
'Numa_LIBRARIES',
|
|
'ONNX_ML',
|
|
'ONNX_NAMESPACE',
|
|
'ATEN_THREADING',
|
|
'WERROR',
|
|
'OPENSSL_ROOT_DIR')
|
|
})
|
|
|
|
for var, val in my_env.items():
|
|
# We currently pass over all environment variables that start with "BUILD_", "USE_", and "CMAKE_". This is
|
|
# because we currently have no reliable way to get the list of all build options we have specified in
|
|
# CMakeLists.txt. (`cmake -L` won't print dependent options when the dependency condition is not met.) We
|
|
# will possibly change this in the future by parsing CMakeLists.txt ourselves (then additional_options would
|
|
# also not be needed to be specified here).
|
|
true_var = additional_options.get(var)
|
|
if true_var is not None:
|
|
build_options[true_var] = val
|
|
elif var.startswith(('BUILD_', 'USE_', 'CMAKE_')) or var.endswith(('EXITCODE', 'EXITCODE__TRYRUN_OUTPUT')):
|
|
build_options[var] = val
|
|
|
|
# The default value cannot be easily obtained in CMakeLists.txt. We set it here.
|
|
py_lib_path = sysconfig.get_path('purelib')
|
|
cmake_prefix_path = build_options.get('CMAKE_PREFIX_PATH', None)
|
|
if cmake_prefix_path:
|
|
build_options["CMAKE_PREFIX_PATH"] = (
|
|
cast(str, py_lib_path) + ";" + cast(str, cmake_prefix_path)
|
|
)
|
|
else:
|
|
build_options['CMAKE_PREFIX_PATH'] = py_lib_path
|
|
|
|
# Some options must be post-processed. Ideally, this list will be shrunk to only one or two options in the
|
|
# future, as CMake can detect many of these libraries pretty comfortably. We have them here for now before CMake
|
|
# integration is completed. They appear here not in the CMake.defines call below because they start with either
|
|
# "BUILD_" or "USE_" and must be overwritten here.
|
|
build_options.update({
|
|
# Note: Do not add new build options to this dict if it is directly read from environment variable -- you
|
|
# only need to add one in `CMakeLists.txt`. All build options that start with "BUILD_", "USE_", or "CMAKE_"
|
|
# are automatically passed to CMake; For other options you can add to additional_options above.
|
|
'BUILD_PYTHON': build_python,
|
|
'BUILD_TEST': build_test,
|
|
# Most library detection should go to CMake script, except this one, which Python can do a much better job
|
|
# due to NumPy's inherent Pythonic nature.
|
|
'USE_NUMPY': USE_NUMPY,
|
|
})
|
|
|
|
# Options starting with CMAKE_
|
|
cmake__options = {
|
|
'CMAKE_INSTALL_PREFIX': install_dir,
|
|
}
|
|
|
|
# We set some CMAKE_* options in our Python build code instead of relying on the user's direct settings. Emit an
|
|
# error if the user also attempts to set these CMAKE options directly.
|
|
specified_cmake__options = set(build_options).intersection(cmake__options)
|
|
if len(specified_cmake__options) > 0:
|
|
print(', '.join(specified_cmake__options) +
|
|
' should not be specified in the environment variable. They are directly set by PyTorch build script.')
|
|
sys.exit(1)
|
|
build_options.update(cmake__options)
|
|
|
|
CMake.defines(args,
|
|
PYTHON_EXECUTABLE=sys.executable,
|
|
PYTHON_LIBRARY=cmake_python_library,
|
|
PYTHON_INCLUDE_DIR=sysconfig.get_path('include'),
|
|
TORCH_BUILD_VERSION=version,
|
|
NUMPY_INCLUDE_DIR=NUMPY_INCLUDE_DIR,
|
|
**build_options)
|
|
|
|
expected_wrapper = '/usr/local/opt/ccache/libexec'
|
|
if IS_DARWIN and os.path.exists(expected_wrapper):
|
|
if 'CMAKE_C_COMPILER' not in build_options and 'CC' not in os.environ:
|
|
CMake.defines(args, CMAKE_C_COMPILER="{}/gcc".format(expected_wrapper))
|
|
if 'CMAKE_CXX_COMPILER' not in build_options and 'CXX' not in os.environ:
|
|
CMake.defines(args, CMAKE_CXX_COMPILER="{}/g++".format(expected_wrapper))
|
|
|
|
for env_var_name in my_env:
|
|
if env_var_name.startswith('gh'):
|
|
# github env vars use utf-8, on windows, non-ascii code may
|
|
# cause problem, so encode first
|
|
try:
|
|
my_env[env_var_name] = str(my_env[env_var_name].encode("utf-8"))
|
|
except UnicodeDecodeError as e:
|
|
shex = ':'.join('{:02x}'.format(ord(c)) for c in my_env[env_var_name])
|
|
print('Invalid ENV[{}] = {}'.format(env_var_name, shex), file=sys.stderr)
|
|
print(e, file=sys.stderr)
|
|
# According to the CMake manual, we should pass the arguments first,
|
|
# and put the directory as the last element. Otherwise, these flags
|
|
# may not be passed correctly.
|
|
# Reference:
|
|
# 1. https://cmake.org/cmake/help/latest/manual/cmake.1.html#synopsis
|
|
# 2. https://stackoverflow.com/a/27169347
|
|
args.append(base_dir)
|
|
self.run(args, env=my_env)
|
|
|
|
def build(self, my_env: Dict[str, str]) -> None:
|
|
"Runs cmake to build binaries."
|
|
|
|
from .env import build_type
|
|
|
|
build_args = ['--build', '.', '--target', 'install', '--config', build_type.build_type_string]
|
|
|
|
# Determine the parallelism according to the following
|
|
# priorities:
|
|
# 1) MAX_JOBS environment variable
|
|
# 2) If using the Ninja build system, delegate decision to it.
|
|
# 3) Otherwise, fall back to the number of processors.
|
|
|
|
# Allow the user to set parallelism explicitly. If unset,
|
|
# we'll try to figure it out.
|
|
max_jobs = os.getenv('MAX_JOBS')
|
|
|
|
if max_jobs is not None or not USE_NINJA:
|
|
# Ninja is capable of figuring out the parallelism on its
|
|
# own: only specify it explicitly if we are not using
|
|
# Ninja.
|
|
|
|
# This lists the number of processors available on the
|
|
# machine. This may be an overestimate of the usable
|
|
# processors if CPU scheduling affinity limits it
|
|
# further. In the future, we should check for that with
|
|
# os.sched_getaffinity(0) on platforms that support it.
|
|
max_jobs = max_jobs or str(multiprocessing.cpu_count())
|
|
|
|
# This ``if-else'' clause would be unnecessary when cmake
|
|
# 3.12 becomes minimum, which provides a '-j' option:
|
|
# build_args += ['-j', max_jobs] would be sufficient by
|
|
# then. Until then, we use "--" to pass parameters to the
|
|
# underlying build system.
|
|
build_args += ['--']
|
|
if IS_WINDOWS:
|
|
# We are likely using msbuild here
|
|
build_args += ['/p:CL_MPCount={}'.format(max_jobs)]
|
|
else:
|
|
build_args += ['-j', max_jobs]
|
|
self.run(build_args, my_env)
|