mirror of
https://github.com/zebrajr/tensorflow.git
synced 2025-12-06 00:19:58 +01:00
f7dfba3f74
Hermetic toolchains give us builds that are isolated from the host system, cutting down on unexpected dependencies and side effects.
With these changes, TensorFlow will build for Linux x86_64 architectures (both CPU and CUDA-enabled GPU) using self-contained C++ toolchains. If you need to use a non-hermetic toolchain, you can do so by adding the flag --config=clang_local. For remote builds with a non-hermetic toolchain, simply append _clang_local to your existing RBE flag. For example, if your hermetic RBE build uses --config=rbe_linux_cpu, the non-hermetic version would be --config=rbe_linux_cpu_clang_local.
Example: Run CPU tests for Linux x86_64
For hermetic tests, run following command (no env variables like CC, CXX, BAZEL_COMPILER, CLANG_COMPILER_PATH):
bazel test \
--config=avx_linux \
--config=release_linux_base \
--config=linux_cpu_pycpp_test_filters \
--repo_env=HERMETIC_PYTHON_VERSION=3.11 \
//tensorflow/... -- -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/... -//tensorflow/tools/toolchains/...
For Linux x86_64 non-hermetic tests use commands with the flag "--config=clang_local" and env variables CC, CXX, BAZEL_COMPILER, CLANG_COMPILER_PATH, etc.:
bazel test \
--config=clang_local \
--config=avx_linux \
--config=release_linux_base \
--config=linux_cpu_pycpp_test_filters \
--repo_env=HERMETIC_PYTHON_VERSION=3.11 \
--action_env=CLANG_COMPILER_PATH=/usr/lib/llvm-18/bin/clang \
--host_action_env=CLANG_COMPILER_PATH=/usr/lib/llvm-18/bin/clang \
--repo_env=CC=/usr/lib/llvm-18/bin/clang \
--repo_env=CXX=/usr/lib/llvm-18/bin/clang++ \
--repo_env=BAZEL_COMPILER=/usr/lib/llvm-18/bin/clang \
//tensorflow/... -- -//tensorflow/compiler/tf2tensorrt/... -//tensorflow/core/tpu/... -//tensorflow/lite/... -//tensorflow/tools/toolchains/...
PiperOrigin-RevId: 783911228
172 lines
4.3 KiB
Python
172 lines
4.3 KiB
Python
# buildifier: disable=load-on-top
|
|
|
|
workspace(name = "org_tensorflow")
|
|
|
|
# buildifier: disable=load-on-top
|
|
|
|
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
|
|
|
http_archive(
|
|
name = "rules_shell",
|
|
sha256 = "bc61ef94facc78e20a645726f64756e5e285a045037c7a61f65af2941f4c25e1",
|
|
strip_prefix = "rules_shell-0.4.1",
|
|
url = "https://github.com/bazelbuild/rules_shell/releases/download/v0.4.1/rules_shell-v0.4.1.tar.gz",
|
|
)
|
|
|
|
# Initialize the TensorFlow repository and all dependencies.
|
|
#
|
|
# The cascade of load() statements and tf_workspace?() calls works around the
|
|
# restriction that load() statements need to be at the top of .bzl files.
|
|
# E.g. we can not retrieve a new repository with http_archive and then load()
|
|
# a macro from that repository in the same file.
|
|
load("@//tensorflow:workspace3.bzl", "tf_workspace3")
|
|
|
|
tf_workspace3()
|
|
|
|
load("@rules_shell//shell:repositories.bzl", "rules_shell_dependencies", "rules_shell_toolchains")
|
|
|
|
rules_shell_dependencies()
|
|
|
|
rules_shell_toolchains()
|
|
|
|
# Initialize hermetic Python
|
|
load("@local_xla//third_party/py:python_init_rules.bzl", "python_init_rules")
|
|
|
|
python_init_rules()
|
|
|
|
load("@local_xla//third_party/py:python_init_repositories.bzl", "python_init_repositories")
|
|
|
|
python_init_repositories(
|
|
default_python_version = "system",
|
|
local_wheel_dist_folder = "dist",
|
|
local_wheel_inclusion_list = [
|
|
"tensorflow*",
|
|
"tf_nightly*",
|
|
],
|
|
local_wheel_workspaces = ["//:WORKSPACE"],
|
|
requirements = {
|
|
"3.9": "//:requirements_lock_3_9.txt",
|
|
"3.10": "//:requirements_lock_3_10.txt",
|
|
"3.11": "//:requirements_lock_3_11.txt",
|
|
"3.12": "//:requirements_lock_3_12.txt",
|
|
"3.13": "//:requirements_lock_3_13.txt",
|
|
},
|
|
)
|
|
|
|
load("@local_xla//third_party/py:python_init_toolchains.bzl", "python_init_toolchains")
|
|
|
|
python_init_toolchains()
|
|
|
|
load("@local_xla//third_party/py:python_init_pip.bzl", "python_init_pip")
|
|
|
|
python_init_pip()
|
|
|
|
load("@pypi//:requirements.bzl", "install_deps")
|
|
|
|
install_deps()
|
|
# End hermetic Python initialization
|
|
|
|
load("@//tensorflow:workspace2.bzl", "tf_workspace2")
|
|
|
|
tf_workspace2()
|
|
|
|
load("@//tensorflow:workspace1.bzl", "tf_workspace1")
|
|
|
|
tf_workspace1()
|
|
|
|
load("@//tensorflow:workspace0.bzl", "tf_workspace0")
|
|
|
|
tf_workspace0()
|
|
|
|
load(
|
|
"@local_xla//third_party/py:python_wheel.bzl",
|
|
"python_wheel_version_suffix_repository",
|
|
)
|
|
|
|
python_wheel_version_suffix_repository(name = "tf_wheel_version_suffix")
|
|
|
|
load(
|
|
"@rules_ml_toolchain//cc_toolchain/deps:cc_toolchain_deps.bzl",
|
|
"cc_toolchain_deps",
|
|
)
|
|
|
|
cc_toolchain_deps()
|
|
|
|
register_toolchains("@rules_ml_toolchain//cc_toolchain:lx64_lx64")
|
|
|
|
register_toolchains("@rules_ml_toolchain//cc_toolchain:lx64_lx64_cuda")
|
|
|
|
load(
|
|
"@rules_ml_toolchain//third_party/gpus/cuda/hermetic:cuda_json_init_repository.bzl",
|
|
"cuda_json_init_repository",
|
|
)
|
|
|
|
cuda_json_init_repository()
|
|
|
|
load(
|
|
"@cuda_redist_json//:distributions.bzl",
|
|
"CUDA_REDISTRIBUTIONS",
|
|
"CUDNN_REDISTRIBUTIONS",
|
|
)
|
|
load(
|
|
"@rules_ml_toolchain//third_party/gpus/cuda/hermetic:cuda_redist_init_repositories.bzl",
|
|
"cuda_redist_init_repositories",
|
|
"cudnn_redist_init_repository",
|
|
)
|
|
|
|
cuda_redist_init_repositories(
|
|
cuda_redistributions = CUDA_REDISTRIBUTIONS,
|
|
)
|
|
|
|
cudnn_redist_init_repository(
|
|
cudnn_redistributions = CUDNN_REDISTRIBUTIONS,
|
|
)
|
|
|
|
load(
|
|
"@rules_ml_toolchain//third_party/gpus/cuda/hermetic:cuda_configure.bzl",
|
|
"cuda_configure",
|
|
)
|
|
|
|
cuda_configure(name = "local_config_cuda")
|
|
|
|
load(
|
|
"@rules_ml_toolchain//third_party/nccl/hermetic:nccl_redist_init_repository.bzl",
|
|
"nccl_redist_init_repository",
|
|
)
|
|
|
|
nccl_redist_init_repository()
|
|
|
|
load(
|
|
"@rules_ml_toolchain//third_party/nccl/hermetic:nccl_configure.bzl",
|
|
"nccl_configure",
|
|
)
|
|
|
|
nccl_configure(name = "local_config_nccl")
|
|
|
|
load(
|
|
"@rules_ml_toolchain//third_party/nvshmem/hermetic:nvshmem_json_init_repository.bzl",
|
|
"nvshmem_json_init_repository",
|
|
)
|
|
|
|
nvshmem_json_init_repository()
|
|
|
|
load(
|
|
"@nvshmem_redist_json//:distributions.bzl",
|
|
"NVSHMEM_REDISTRIBUTIONS",
|
|
)
|
|
load(
|
|
"@rules_ml_toolchain//third_party/nvshmem/hermetic:nvshmem_redist_init_repository.bzl",
|
|
"nvshmem_redist_init_repository",
|
|
)
|
|
|
|
nvshmem_redist_init_repository(
|
|
nvshmem_redistributions = NVSHMEM_REDISTRIBUTIONS,
|
|
)
|
|
|
|
load(
|
|
"@rules_ml_toolchain//third_party/nvshmem/hermetic:nvshmem_configure.bzl",
|
|
"nvshmem_configure",
|
|
)
|
|
|
|
nvshmem_configure(name = "local_config_nvshmem")
|