Merge pull request #61871 from Intel-tensorflow:mraunak/Clang_TF_Win

PiperOrigin-RevId: 572096891

.bazelrc

@@ -420,7 +420,18 @@ build:verbose_logs --output_filter=
 #   avoid having to define linux/win separately.
 build:avx_linux --copt=-mavx
 build:avx_linux --host_copt=-mavx
-build:avx_win --copt=/arch=AVX
+build:avx_win --copt=/arch:AVX
+
+# Use Clang-cl compiler on Windows
+build:win_clang --copt=/clang:-Weverything
+build:win_clang --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl
+build:win_clang --extra_execution_platforms=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
+build:win_clang --host_platform=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
+build:win_clang --compiler=clang-cl
+build:win_clang --linkopt=/FORCE:MULTIPLE
+build:win_clang --host_linkopt=/FORCE:MULTIPLE
+test:win_clang --linkopt=/FORCE:MULTIPLE
+test:win_clang --host_linkopt=/FORCE:MULTIPLE
 
 # Options to build TensorFlow 1.x or 2.x.
 # TODO(kanglan): Change v2's define to default behavior

tensorflow/core/ir/utils/shape_inference_utils.h

@@ -27,7 +27,7 @@ limitations under the License.
 #include "tensorflow/core/framework/shape_inference.h"
 
 namespace tensorflow {
-class OpRegistrationData;
+struct OpRegistrationData;
 }  // namespace tensorflow
 
 namespace mlir {

tensorflow/opensource_only.files

@@ -350,6 +350,7 @@ tf_staging/third_party/tensorrt/tensorrt/include/tensorrt_config.h.tpl:
 tf_staging/third_party/tensorrt/tensorrt/tensorrt_config.py.tpl:
 tf_staging/third_party/tensorrt/tensorrt_configure.bzl:
 tf_staging/third_party/tensorrt/workspace.bzl:
+tf_staging/third_party/tf_runtime/BUILD:
 tf_staging/third_party/tf_toolchains.BUILD:
 tf_staging/third_party/tflite_mobilenet.BUILD:
 tf_staging/third_party/tflite_mobilenet_float.BUILD:

tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh

@@ -160,4 +160,3 @@ create_python_test_dir "${PY_TEST_DIR}"
 if [[ "$TF_NIGHTLY" == 1 ]]; then
   exit 0
 fi
-

tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows_clang-cl.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+#
+# This script assumes the standard setup on tensorflow Jenkins windows machines.
+# It is NOT guaranteed to work on any other machine. Use at your own risk!
+#
+# REQUIREMENTS:
+# * All installed in standard locations:
+#   - JDK8, and JAVA_HOME set.
+#   - Microsoft Visual Studio 2015 Community Edition
+#   - Msys2
+#   - Anaconda3
+# * Bazel windows executable copied as "bazel.exe" and included in PATH.
+# change
+
+# All commands shall pass, and all should be visible.
+set -x
+set -e
+
+# This script is under <repo_root>/tensorflow/tools/ci_build/windows/cpu/pip/
+# Change into repository root.
+script_dir=$(dirname $0)
+cd ${script_dir%%tensorflow/tools/ci_build/windows/cpu/pip}.
+
+# Setting up the environment variables Bazel and ./configure needs
+source "tensorflow/tools/ci_build/windows/bazel/common_env.sh" \
+  || { echo "Failed to source common_env.sh" >&2; exit 1; }
+
+# load bazel_test_lib.sh
+source "tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh" \
+  || { echo "Failed to source bazel_test_lib.sh" >&2; exit 1; }
+
+# Recreate an empty bazelrc file under source root
+export TMP_BAZELRC=.tmp.bazelrc
+rm -f "${TMP_BAZELRC}"
+touch "${TMP_BAZELRC}"
+
+function cleanup {
+  # Remove all options in .tmp.bazelrc
+  echo "" > "${TMP_BAZELRC}"
+}
+trap cleanup EXIT
+
+PY_TEST_DIR="py_test_dir"
+
+SKIP_TEST=0
+RELEASE_BUILD=0
+TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..."
+PROJECT_NAME=""
+EXTRA_BUILD_FLAGS=""
+EXTRA_TEST_FLAGS=""
+
+# --skip_test            Skip running tests
+# --enable_remote_cache  Add options to enable remote cache for build and test
+# --release_build        Build for release, compilation time will be longer to
+#                        ensure performance
+# --test_core_only       Use tensorflow/python/... as test target
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --tf_nightly) TF_NIGHTLY=1 ;;
+    --skip_test) SKIP_TEST=1 ;;
+    --enable_remote_cache) set_remote_cache_options ;;
+    --release_build) RELEASE_BUILD=1 ;;
+    --test_core_only) TEST_TARGET="//${PY_TEST_DIR}/tensorflow/python/..." ;;
+    --extra_build_flags)
+      shift
+      if [[ -z "$1" ]]; then
+        break
+      fi
+      EXTRA_BUILD_FLAGS="$1"
+      ;;
+    --project_name)
+      shift
+      if [[ -z "$1" ]]; then
+        break
+      fi
+      PROJECT_NAME="$1"
+      ;;
+    --extra_test_flags)
+      shift
+      if [[ -z "$1" ]]; then
+        break
+      fi
+      EXTRA_TEST_FLAGS="$1"
+      ;;
+    *)
+  esac
+  shift
+done
+
+if [[ "$RELEASE_BUILD" == 1 ]]; then
+  # Overriding eigen strong inline speeds up the compiling of conv_grad_ops_3d.cc and conv_ops_3d.cc
+  # by 20 minutes. See https://github.com/tensorflow/tensorflow/issues/10521
+  # Because this hurts the performance of TF, we don't override it in release build.
+  export TF_OVERRIDE_EIGEN_STRONG_INLINE=0
+else
+  export TF_OVERRIDE_EIGEN_STRONG_INLINE=1
+fi
+
+if [[ "$TF_NIGHTLY" == 1 ]]; then
+  if [[ ${PROJECT_NAME} == *"2.0_preview"* ]]; then
+    python tensorflow/tools/ci_build/update_version.py --version=2.0.0 --nightly
+  else
+    python tensorflow/tools/ci_build/update_version.py --nightly
+  fi
+  if [ -z ${PROJECT_NAME} ]; then
+    EXTRA_PIP_FLAGS="--nightly_flag"
+  else
+    EXTRA_PIP_FLAGS="--project_name ${PROJECT_NAME} --nightly_flag"
+  fi
+else
+  if [[ -v PROJECT_NAME  ]]; then
+    EXTRA_PIP_FLAGS="--project_name ${PROJECT_NAME}"
+  fi
+fi
+
+# Enable short object file path to avoid long path issue on Windows.
+echo "startup --output_user_root=${TMPDIR}" >> "${TMP_BAZELRC}"
+
+if ! grep -q "import %workspace%/${TMP_BAZELRC}" .bazelrc; then
+  echo "import %workspace%/${TMP_BAZELRC}" >> .bazelrc
+fi
+
+run_configure_for_cpu_build
+
+bazel build ${EXTRA_BUILD_FLAGS}  \
+  --experimental_cc_shared_library \
+  --build_tag_filters=-no_pip,-no_windows,-windows_excluded,-no_oss,-oss_excluded,-gpu,-tpu \
+  --config=win_clang \
+  --output_filter=^$ \
+  tensorflow/lite:framework tensorflow/lite/examples/minimal:minimal || exit $?
+
+bazel build \
+  --experimental_cc_shared_library \
+  --config=release_cpu_windows ${EXTRA_BUILD_FLAGS} \
+  --config=win_clang \
+  --output_filter=^$ \
+  tensorflow/tools/pip_package:build_pip_package || exit $?
+
+if [[ "$SKIP_TEST" == 1 ]]; then
+  exit 0
+fi
+
+# Create a python test directory to avoid package name conflict
+create_python_test_dir "${PY_TEST_DIR}"
+
+./bazel-bin/tensorflow/tools/pip_package/build_pip_package "$PWD/${PY_TEST_DIR}" ${EXTRA_PIP_FLAGS}
+
+if [[ "$TF_NIGHTLY" == 1 ]]; then
+  exit 0
+fi
+

tensorflow/tools/def_file_filter/def_file_filter.py.tpl

@@ -306,7 +306,6 @@ def main():
     def_fp.write("\t ??_7DeviceProperties@tensorflow@@6B@\n")
     def_fp.write("\t ??_7MetaGraphDef@tensorflow@@6B@\n")
     def_fp.write("\t ??_7SavedModel@tensorflow@@6B@\n")
-    def_fp.write("\t ??0CoordinatedTask@tensorflow@@QEAA@XZ\n") # for _pywrap_tfe
     def_fp.write("\t ?Set@ArenaStringPtr@internal@protobuf@google@@QEAAXAEBV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@std@@PEAVArena@34@@Z\n") # _pywrap_tfe
     def_fp.write("\t ??1CoordinatedTask@tensorflow@@UEAA@XZ\n") # for _pywrap_tfe
     def_fp.write("\t ?CopyFrom@CoordinatedTask@tensorflow@@QEAAXAEBV12@@Z\n") # for _pywrap_tfe
@@ -362,3 +361,4 @@ def main():
 
 if __name__ == "__main__":
   sys.exit(main())
+  

tensorflow/tools/toolchains/win/BUILD

@@ -31,3 +31,13 @@ platform(
         }
         """,
 )
+
+#Register platform to support clang-cl on Windows
+platform(
+    name = "x64_windows-clang-cl",
+    constraint_values = [
+        "@platforms//cpu:x86_64",
+        "@platforms//os:windows",
+        "@bazel_tools//tools/cpp:clang-cl",
+    ],
+)

tensorflow/workspace3.bzl

@@ -1,7 +1,7 @@
 """TensorFlow workspace initialization. Consult the WORKSPACE on how to use it."""
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-load("//third_party:tf_runtime/workspace.bzl", tf_runtime = "repo")
+load("//third_party/tf_runtime:workspace.bzl", tf_runtime = "repo")
 load("//third_party/llvm:workspace.bzl", llvm = "repo")
 load("//third_party:repo.bzl", "tf_vendored")
 

third_party/tf_runtime/tf_runtime_clangcl.patch

@@ -0,0 +1,14 @@
+diff --git a/include/tfrt/support/std_mutex.h b/include/tfrt/support/std_mutex.h
+index 6238d097..9fb24279 100644
+--- a/include/tfrt/support/std_mutex.h
++++ b/include/tfrt/support/std_mutex.h
+@@ -50,7 +50,7 @@ class TFRT_CAPABILITY("mutex") mutex {
+ 
+  private:
+   friend class mutex_lock;
+-  std::mutex mu_;
++  std::mutex mu_{};
+ };
+
+ // Wrap std::unique_lock<std::mutex> with support for thread annotations.
+ 

third_party/tf_runtime/workspace.bzl

@@ -16,5 +16,5 @@ def repo():
         urls = tf_mirror_urls("https://github.com/tensorflow/runtime/archive/{commit}.tar.gz".format(commit = TFRT_COMMIT)),
         # A patch file can be provided for atomic commits to both TF and TFRT.
         # The job that bumps the TFRT_COMMIT also resets patch_file to 'None'.
-        patch_file = None,
+        patch_file = ["//third_party/tf_runtime:tf_runtime_clangcl.patch"],
     )

third_party/xla/.bazelrc

@@ -420,7 +420,18 @@ build:verbose_logs --output_filter=
 #   avoid having to define linux/win separately.
 build:avx_linux --copt=-mavx
 build:avx_linux --host_copt=-mavx
-build:avx_win --copt=/arch=AVX
+build:avx_win --copt=/arch:AVX
+
+# Use Clang-cl compiler on Windows
+build:win_clang --copt=/clang:-Weverything
+build:win_clang --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl
+build:win_clang --extra_execution_platforms=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
+build:win_clang --host_platform=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
+build:win_clang --compiler=clang-cl
+build:win_clang --linkopt=/FORCE:MULTIPLE
+build:win_clang --host_linkopt=/FORCE:MULTIPLE
+test:win_clang --linkopt=/FORCE:MULTIPLE
+test:win_clang --host_linkopt=/FORCE:MULTIPLE
 
 # Options to build TensorFlow 1.x or 2.x.
 # TODO(kanglan): Change v2's define to default behavior

third_party/xla/opensource_only.files

@@ -135,6 +135,7 @@ third_party/tensorrt/tensorrt/include/tensorrt_config.h.tpl:
 third_party/tensorrt/tensorrt/tensorrt_config.py.tpl:
 third_party/tensorrt/tensorrt_configure.bzl:
 third_party/tensorrt/workspace.bzl:
+third_party/tf_runtime/BUILD:
 third_party/zlib.BUILD:
 tools/toolchains/BUILD:
 tools/toolchains/clang6/BUILD:

third_party/xla/third_party/tf_runtime/tf_runtime_clangcl.patch

@@ -0,0 +1,14 @@
+diff --git a/include/tfrt/support/std_mutex.h b/include/tfrt/support/std_mutex.h
+index 6238d097..9fb24279 100644
+--- a/include/tfrt/support/std_mutex.h
++++ b/include/tfrt/support/std_mutex.h
+@@ -50,7 +50,7 @@ class TFRT_CAPABILITY("mutex") mutex {
+ 
+  private:
+   friend class mutex_lock;
+-  std::mutex mu_;
++  std::mutex mu_{};
+ };
+
+ // Wrap std::unique_lock<std::mutex> with support for thread annotations.
+ 

third_party/xla/third_party/tf_runtime/workspace.bzl

@@ -16,5 +16,5 @@ def repo():
         urls = tf_mirror_urls("https://github.com/tensorflow/runtime/archive/{commit}.tar.gz".format(commit = TFRT_COMMIT)),
         # A patch file can be provided for atomic commits to both TF and TFRT.
         # The job that bumps the TFRT_COMMIT also resets patch_file to 'None'.
-        patch_file = None,
+        patch_file = ["//third_party/tf_runtime:tf_runtime_clangcl.patch"],
     )

third_party/xla/third_party/tsl/.bazelrc

@@ -420,7 +420,18 @@ build:verbose_logs --output_filter=
 #   avoid having to define linux/win separately.
 build:avx_linux --copt=-mavx
 build:avx_linux --host_copt=-mavx
-build:avx_win --copt=/arch=AVX
+build:avx_win --copt=/arch:AVX
+
+# Use Clang-cl compiler on Windows
+build:win_clang --copt=/clang:-Weverything
+build:win_clang --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl
+build:win_clang --extra_execution_platforms=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
+build:win_clang --host_platform=//tensorflow/tools/toolchains/win:x64_windows-clang-cl
+build:win_clang --compiler=clang-cl
+build:win_clang --linkopt=/FORCE:MULTIPLE
+build:win_clang --host_linkopt=/FORCE:MULTIPLE
+test:win_clang --linkopt=/FORCE:MULTIPLE
+test:win_clang --host_linkopt=/FORCE:MULTIPLE
 
 # Options to build TensorFlow 1.x or 2.x.
 # TODO(kanglan): Change v2's define to default behavior

third_party/xla/third_party/tsl/opensource_only.files

@@ -128,6 +128,7 @@ third_party/tensorrt/tensorrt/include/tensorrt_config.h.tpl:
 third_party/tensorrt/tensorrt/tensorrt_config.py.tpl:
 third_party/tensorrt/tensorrt_configure.bzl:
 third_party/tensorrt/workspace.bzl:
+third_party/tf_runtime/BUILD:
 third_party/zlib.BUILD:
 tools/def_file_filter/BUILD.tpl:
 tools/def_file_filter/BUILD:

third_party/xla/third_party/tsl/third_party/tf_runtime/tf_runtime_clangcl.patch

@@ -0,0 +1,14 @@
+diff --git a/include/tfrt/support/std_mutex.h b/include/tfrt/support/std_mutex.h
+index 6238d097..9fb24279 100644
+--- a/include/tfrt/support/std_mutex.h
++++ b/include/tfrt/support/std_mutex.h
+@@ -50,7 +50,7 @@ class TFRT_CAPABILITY("mutex") mutex {
+ 
+  private:
+   friend class mutex_lock;
+-  std::mutex mu_;
++  std::mutex mu_{};
+ };
+
+ // Wrap std::unique_lock<std::mutex> with support for thread annotations.
+ 

third_party/xla/third_party/tsl/third_party/tf_runtime/workspace.bzl

@@ -16,5 +16,5 @@ def repo():
         urls = tf_mirror_urls("https://github.com/tensorflow/runtime/archive/{commit}.tar.gz".format(commit = TFRT_COMMIT)),
         # A patch file can be provided for atomic commits to both TF and TFRT.
         # The job that bumps the TFRT_COMMIT also resets patch_file to 'None'.
-        patch_file = None,
+        patch_file = ["//third_party/tf_runtime:tf_runtime_clangcl.patch"],
     )

third_party/xla/third_party/tsl/tools/def_file_filter/def_file_filter.py.tpl

@@ -306,7 +306,6 @@ def main():
     def_fp.write("\t ??_7DeviceProperties@tensorflow@@6B@\n")
     def_fp.write("\t ??_7MetaGraphDef@tensorflow@@6B@\n")
     def_fp.write("\t ??_7SavedModel@tensorflow@@6B@\n")
-    def_fp.write("\t ??0CoordinatedTask@tensorflow@@QEAA@XZ\n") # for _pywrap_tfe
     def_fp.write("\t ?Set@ArenaStringPtr@internal@protobuf@google@@QEAAXAEBV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@std@@PEAVArena@34@@Z\n") # _pywrap_tfe
     def_fp.write("\t ??1CoordinatedTask@tensorflow@@UEAA@XZ\n") # for _pywrap_tfe
     def_fp.write("\t ?CopyFrom@CoordinatedTask@tensorflow@@QEAAXAEBV12@@Z\n") # for _pywrap_tfe
@@ -362,3 +361,4 @@ def main():
 
 if __name__ == "__main__":
   sys.exit(main())
+  

third_party/xla/third_party/tsl/tools/toolchains/win/BUILD

@@ -31,3 +31,13 @@ platform(
         }
         """,
 )
+
+#Register platform to support clang-cl on Windows
+platform(
+    name = "x64_windows-clang-cl",
+    constraint_values = [
+        "@platforms//cpu:x86_64",
+        "@platforms//os:windows",
+        "@bazel_tools//tools/cpp:clang-cl",
+    ],
+)

third_party/xla/third_party/tsl/workspace3.bzl

@@ -1,7 +1,7 @@
 """TensorFlow workspace initialization. Consult the WORKSPACE on how to use it."""
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-load("//third_party:tf_runtime/workspace.bzl", tf_runtime = "repo")
+load("//third_party/tf_runtime:workspace.bzl", tf_runtime = "repo")
 load("//third_party/llvm:workspace.bzl", llvm = "repo")
 
 def workspace():

third_party/xla/tools/toolchains/win/BUILD

@@ -31,3 +31,13 @@ platform(
         }
         """,
 )
+
+#Register platform to support clang-cl on Windows
+platform(
+    name = "x64_windows-clang-cl",
+    constraint_values = [
+        "@platforms//cpu:x86_64",
+        "@platforms//os:windows",
+        "@bazel_tools//tools/cpp:clang-cl",
+    ],
+)

third_party/xla/workspace3.bzl

@@ -2,7 +2,7 @@
 
 load("@local_tsl//:workspace3.bzl", "tsl_workspace3")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-load("//third_party:tf_runtime/workspace.bzl", tf_runtime = "repo")
+load("//third_party/tf_runtime:workspace.bzl", tf_runtime = "repo")
 load("//third_party/llvm:workspace.bzl", llvm = "repo")
 
 # buildifier: disable=function-docstring