OSSForks/tensorflow
1
0
Fork 0
mirror of https://github.com/zebrajr/tensorflow.git synced 2025-12-06 12:20:11 +01:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Merge pull request #48222 from tensorflow/mm-fix-fileystem-on-r2.5

Browse Source 
Fix GCS filesystem
This commit is contained in:
Goldie Gadde 2021-03-31 21:24:10 -07:00 committed by GitHub
commit a8b6d5ff93
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 18 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
tensorflow/core/platform/env.h
View File

@ -636,21 +636,22 @@ namespace register_file_system {
template <typename Factory> template <typename Factory>
struct Register { struct Register {
Register(Env* env, const std::string& scheme, bool legacy) { Register(Env* env, const std::string& scheme, bool try_modular_filesystems) {
// TODO(yongtang): Remove legacy file system registration for hdfs/s3/gcs // TODO(yongtang): Remove legacy file system registration for hdfs/s3/gcs
// after TF 2.6+. // after TF 2.6+.
if (legacy) { if (try_modular_filesystems) {
const char* enable_legacy_env = getenv("TF_ENABLE_LEGACY_FILESYSTEM"); const char* env_value = getenv("TF_USE_MODULAR_FILESYSTEM");
string enable_legacy = string load_plugin = env_value ? absl::AsciiStrToLower(env_value) : "";
enable_legacy_env ? absl::AsciiStrToLower(enable_legacy_env) : ""; if (load_plugin == "true" || load_plugin == "1") {
if (!(enable_legacy == "true" || enable_legacy == "1")) { // We don't register the static filesystem and wait for SIG IO one
LOG(WARNING) << "Using modular file system for '" << scheme << "."
<< " Please switch to tensorflow-io"
<< " (https://github.com/tensorflow/io) for file system"
<< " support of '" << scheme << "'.";
return; return;
} }
LOG(WARNING) << "Legacy file system for '" << scheme << "' is deprecated" // If the envvar is missing or not "true"/"1", then fall back to legacy
<< " and will be removed in tensorflow 2.6 or higher." // implementation to be backwards compatible.
<< " Please switch to tensorflow-io"
<< " (https://github.com/tensorflow/io) for file system"
<< " support of '" << scheme << "'.";
} }
// TODO(b/32704451): Don't just ignore the ::tensorflow::Status object! // TODO(b/32704451): Don't just ignore the ::tensorflow::Status object!
env->RegisterFileSystem(scheme, []() -> FileSystem* { return new Factory; }) env->RegisterFileSystem(scheme, []() -> FileSystem* { return new Factory; })
@ -666,15 +667,15 @@ struct Register {
// Register a FileSystem implementation for a scheme. Files with names that have // Register a FileSystem implementation for a scheme. Files with names that have
// "scheme://" prefixes are routed to use this implementation. // "scheme://" prefixes are routed to use this implementation.
#define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory, legacy) \ #define REGISTER_FILE_SYSTEM_ENV(env, scheme, factory, modular) \
REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory, legacy) REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, env, scheme, factory, modular)
#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory, legacy) \ #define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, env, scheme, factory, modular) \
REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy) REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, modular)
#define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, legacy) \ #define REGISTER_FILE_SYSTEM_UNIQ(ctr, env, scheme, factory, modular) \
static ::tensorflow::register_file_system::Register<factory> \ static ::tensorflow::register_file_system::Register<factory> \
register_ff##ctr TF_ATTRIBUTE_UNUSED = \ register_ff##ctr TF_ATTRIBUTE_UNUSED = \
::tensorflow::register_file_system::Register<factory>(env, scheme, \ ::tensorflow::register_file_system::Register<factory>(env, scheme, \
legacy) modular)
#define REGISTER_FILE_SYSTEM(scheme, factory) \ #define REGISTER_FILE_SYSTEM(scheme, factory) \
REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, \ REGISTER_FILE_SYSTEM_ENV(::tensorflow::Env::Default(), scheme, factory, \