OSSForks/tensorflow
1
0
Fork 0
mirror of https://github.com/zebrajr/tensorflow.git synced 2025-12-06 12:20:11 +01:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Drop LMDB dependency, replace kernels/ops with error Status.

Browse Source 
This is because LMDB has vulnerabilities and there are very few users of this feature, so it does not make sense to include it in TF.

PiperOrigin-RevId: 514551603
This commit is contained in:
Mihai Maruseac 2023-03-06 15:55:06 -08:00 committed by TensorFlower Gardener
parent 6a527ee5ce
commit dc57d03368
24 changed files with 17 additions and 791 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
RELEASE.md
View File

@ -20,6 +20,11 @@
If this breaks you, simply add `save_format="h5"` to your `.save()` call
to revert back to the prior behavior.
* The LMDB kernels have been changed to return an error. This is in preparation
for completely removing them from TensorFlow. The LMDB dependency that these
kernels are bringing to TensorFlow has been dropped, thus making the build
slightly faster and more secure.
## Known Caveats
* <CAVEATS REGARDING THE RELEASE (BUT NOT BREAKING CHANGES).>

1
tensorflow/BUILD
View File

@ -97,7 +97,6 @@ PACKAGE_STATIC_DEPS = [
"@llvm_openmp//:__subpackages__",
"@llvm_terminfo//:__subpackages__",
"@llvm_zlib//:__subpackages__",
"@lmdb//:__subpackages__",
"@local_config_cuda//:__subpackages__",
"@local_config_git//:__subpackages__",
"@local_config_nccl//:__subpackages__",

1
tensorflow/compiler/mlir/lite/experimental/tac/py_wrapper/BUILD
View File

@ -73,7 +73,6 @@ pybind_extension(
"@llvm-project//:__subpackages__",
"@llvm_terminfo//:__subpackages__",
"@llvm_zlib//:__subpackages__",
"@lmdb//:__subpackages__",
"@local_config_cuda//:__subpackages__",
"@local_config_git//:__subpackages__",
"@local_config_nccl//:__subpackages__",

5
tensorflow/core/kernels/BUILD
View File

@ -3092,9 +3092,7 @@ tf_kernel_library(
tf_kernel_library(
name = "lmdb_reader_op",
prefix = "lmdb_reader_op",
deps = IO_DEPS + [
"@lmdb",
],
deps = IO_DEPS,
)
tf_kernel_library(
@ -7770,7 +7768,6 @@ tf_cc_shared_library(
"@llvm-project//:__subpackages__",
"@llvm_terminfo//:__subpackages__",
"@llvm_zlib//:__subpackages__",
"@lmdb//:__subpackages__",
"@local_config_cuda//:__subpackages__",
"@local_config_git//:__subpackages__",
"@local_config_nccl//:__subpackages__",

20
tensorflow/core/kernels/data/experimental/BUILD
View File

@ -470,26 +470,6 @@ tf_kernel_library(
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//third_party/eigen3",
"@lmdb",
],
)
tf_cc_test(
name = "lmdb_dataset_op_test",
size = "small",
srcs = ["lmdb_dataset_op_test.cc"],
data = ["//tensorflow/core/lib/lmdb:lmdb_testdata"],
deps = [
":lmdb_dataset_op",
"//tensorflow/core:experimental_dataset_ops_op_lib",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core:testlib",
"//tensorflow/core/data:dataset_test_base",
"//tensorflow/core/platform:env",
"//third_party/eigen3",
],
)

202
tensorflow/core/kernels/data/experimental/lmdb_dataset_op.cc
View File

@ -16,10 +16,7 @@ limitations under the License.
#include <sys/stat.h>
#include "lmdb.h" // NOLINT(build/include)
#include "tensorflow/core/framework/dataset.h"
#include "tensorflow/core/lib/io/buffered_inputstream.h"
#include "tensorflow/core/platform/file_system.h"
namespace tensorflow {
namespace data {
@ -30,202 +27,13 @@ namespace experimental {
/* static */ constexpr const char* const LMDBDatasetOp::kOutputTypes;
/* static */ constexpr const char* const LMDBDatasetOp::kOutputShapes;
class LMDBDatasetOp::Dataset : public DatasetBase {
public:
Dataset(OpKernelContext* ctx, const std::vector<string>& filenames)
: DatasetBase(DatasetContext(ctx)), filenames_(filenames) {}
std::unique_ptr<IteratorBase> MakeIteratorInternal(
const string& prefix) const override {
return std::make_unique<Iterator>(
Iterator::Params{this, strings::StrCat(prefix, "::LMDB")});
}
const DataTypeVector& output_dtypes() const override {
static DataTypeVector* dtypes = new DataTypeVector({DT_STRING, DT_STRING});
return *dtypes;
}
const std::vector<PartialTensorShape>& output_shapes() const override {
static std::vector<PartialTensorShape>* shapes =
new std::vector<PartialTensorShape>({{}, {}});
return *shapes;
}
string DebugString() const override { return "LMDBDatasetOp::Dataset"; }
Status InputDatasets(std::vector<const DatasetBase*>* inputs) const override {
return OkStatus();
}
Status CheckExternalState() const override { return OkStatus(); }
protected:
Status AsGraphDefInternal(SerializationContext* ctx,
DatasetGraphDefBuilder* b,
Node** output) const override {
Node* filenames = nullptr;
TF_RETURN_IF_ERROR(b->AddVector(filenames_, &filenames));
TF_RETURN_IF_ERROR(b->AddDataset(this, {filenames}, output));
return OkStatus();
}
private:
class Iterator : public DatasetIterator<Dataset> {
public:
explicit Iterator(const Params& params)
: DatasetIterator<Dataset>(params) {}
~Iterator() override {
// Close any open database connections.
ResetStreamsLocked();
}
Status GetNextInternal(IteratorContext* ctx,
std::vector<Tensor>* out_tensors,
bool* end_of_sequence) override {
mutex_lock l(mu_);
do {
if (mdb_cursor_) {
out_tensors->emplace_back(ctx->allocator({}), DT_STRING,
TensorShape({}));
Tensor& key_tensor = out_tensors->back();
key_tensor.scalar<tstring>()() = string(
static_cast<const char*>(mdb_key_.mv_data), mdb_key_.mv_size);
out_tensors->emplace_back(ctx->allocator({}), DT_STRING,
TensorShape({}));
Tensor& value_tensor = out_tensors->back();
value_tensor.scalar<tstring>()() = string(
static_cast<const char*>(mdb_value_.mv_data), mdb_value_.mv_size);
int val;
val = mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT);
if (val != MDB_SUCCESS && val != MDB_NOTFOUND) {
return errors::InvalidArgument(mdb_strerror(val));
}
if (val == MDB_NOTFOUND) {
ResetStreamsLocked();
++current_file_index_;
}
*end_of_sequence = false;
return OkStatus();
}
if (current_file_index_ == dataset()->filenames_.size()) {
*end_of_sequence = true;
ResetStreamsLocked();
return OkStatus();
}
TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env()));
} while (true);
}
protected:
std::shared_ptr<model::Node> CreateNode(
IteratorContext* ctx, model::Node::Args args) const override {
return model::MakeSourceNode(std::move(args));
}
Status SaveInternal(SerializationContext* ctx,
IteratorStateWriter* writer) override {
return errors::Unimplemented(
"Checkpointing is currently not supported for LMDBDataset.");
}
Status RestoreInternal(IteratorContext* ctx,
IteratorStateReader* reader) override {
return errors::Unimplemented(
"Checkpointing is currently not supported for LMDBDataset.");
}
private:
Status SetupStreamsLocked(Env* env) TF_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
if (current_file_index_ >= dataset()->filenames_.size()) {
return errors::InvalidArgument(
"current_file_index_:", current_file_index_,
" >= filenames_.size():", dataset()->filenames_.size());
}
const string& filename = dataset()->filenames_[current_file_index_];
int val = mdb_env_create(&mdb_env_);
if (val != MDB_SUCCESS) {
return errors::InvalidArgument(mdb_strerror(val));
}
int flags = MDB_RDONLY | MDB_NOTLS | MDB_NOLOCK;
struct stat source_stat;
if (stat(filename.c_str(), &source_stat) == 0 &&
(source_stat.st_mode & S_IFREG)) {
flags |= MDB_NOSUBDIR;
}
val = mdb_env_open(mdb_env_, filename.c_str(), flags, 0664);
if (val != MDB_SUCCESS) {
return errors::InvalidArgument(mdb_strerror(val));
}
val = mdb_txn_begin(mdb_env_, nullptr, MDB_RDONLY, &mdb_txn_);
if (val != MDB_SUCCESS) {
return errors::InvalidArgument(mdb_strerror(val));
}
val = mdb_dbi_open(mdb_txn_, nullptr, 0, &mdb_dbi_);
if (val != MDB_SUCCESS) {
return errors::InvalidArgument(mdb_strerror(val));
}
val = mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_);
if (val != MDB_SUCCESS) {
return errors::InvalidArgument(mdb_strerror(val));
}
val = mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST);
if (val != MDB_SUCCESS && val != MDB_NOTFOUND) {
return errors::InvalidArgument(mdb_strerror(val));
}
if (val == MDB_NOTFOUND) {
ResetStreamsLocked();
}
return OkStatus();
}
void ResetStreamsLocked() TF_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
if (mdb_env_ != nullptr) {
if (mdb_cursor_) {
mdb_cursor_close(mdb_cursor_);
mdb_cursor_ = nullptr;
}
mdb_dbi_close(mdb_env_, mdb_dbi_);
mdb_txn_abort(mdb_txn_);
mdb_env_close(mdb_env_);
mdb_txn_ = nullptr;
mdb_dbi_ = 0;
mdb_env_ = nullptr;
}
}
mutex mu_;
size_t current_file_index_ TF_GUARDED_BY(mu_) = 0;
MDB_env* mdb_env_ TF_GUARDED_BY(mu_) = nullptr;
MDB_txn* mdb_txn_ TF_GUARDED_BY(mu_) = nullptr;
MDB_dbi mdb_dbi_ TF_GUARDED_BY(mu_) = 0;
MDB_cursor* mdb_cursor_ TF_GUARDED_BY(mu_) = nullptr;
MDB_val mdb_key_ TF_GUARDED_BY(mu_);
MDB_val mdb_value_ TF_GUARDED_BY(mu_);
};
const std::vector<string> filenames_;
};
void LMDBDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase** output) {
const Tensor* filenames_tensor;
OP_REQUIRES_OK(ctx, ctx->input("filenames", &filenames_tensor));
OP_REQUIRES(
ctx, filenames_tensor->dims() <= 1,
errors::InvalidArgument("`filenames` must be a scalar or a vector."));
std::vector<string> filenames;
filenames.reserve(filenames_tensor->NumElements());
for (int i = 0; i < filenames_tensor->NumElements(); ++i) {
filenames.push_back(filenames_tensor->flat<tstring>()(i));
}
*output = new Dataset(ctx, filenames);
ctx, false,
errors::Unimplemented(
"LMDB support is removed from TensorFlow. This API will be deleted "
"in the next TensorFlow release. If you need LMDB support, please "
"file a GitHub issue."));
}
namespace {

283
tensorflow/core/kernels/data/experimental/lmdb_dataset_op_test.cc
View File

@ -1,283 +0,0 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/kernels/data/experimental/lmdb_dataset_op.h"
#include "tensorflow/core/data/dataset_test_base.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/test.h"
namespace tensorflow {
namespace data {
namespace experimental {
namespace {
constexpr char kNodeName[] = "lmdb_dataset";
constexpr char kIteratorPrefix[] = "Iterator";
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
constexpr char kDataFileName[] = "data_bigendian.mdb";
#else
constexpr char kDataFileName[] = "data.mdb";
#endif
constexpr char kDataFileLoc[] = "core/lib/lmdb/testdata";
class LMDBDatasetParams : public DatasetParams {
public:
LMDBDatasetParams(std::vector<tstring> filenames,
DataTypeVector output_dtypes,
std::vector<PartialTensorShape> output_shapes,
string node_name)
: DatasetParams(std::move(output_dtypes), std::move(output_shapes),
kNodeName),
filenames_(CreateTensor<tstring>(
TensorShape({static_cast<int>(filenames.size())}), filenames)) {}
std::vector<Tensor> GetInputTensors() const override { return {filenames_}; }
Status GetInputNames(std::vector<string>* input_names) const override {
*input_names = {LMDBDatasetOp::kFileNames};
return OkStatus();
}
Status GetAttributes(AttributeVector* attributes) const override {
*attributes = {{LMDBDatasetOp::kOutputTypes, output_dtypes_},
{LMDBDatasetOp::kOutputShapes, output_shapes_}};
return OkStatus();
}
string dataset_type() const override { return LMDBDatasetOp::kDatasetType; }
private:
// Names of binary database files to read, boxed up inside a Tensor of
// strings
Tensor filenames_;
};
class LMDBDatasetOpTest : public DatasetOpsTestBase {};
// Copy our test data file to the current test program's temporary
// directory, and return the full path to the copied file.
// This copying is necessary because LMDB creates lock files adjacent
// to files that it reads.
tstring MaybeCopyDataFile() {
tstring src_loc =
io::JoinPath(testing::TensorFlowSrcRoot(), kDataFileLoc, kDataFileName);
tstring dest_loc = io::JoinPath(testing::TmpDir(), kDataFileName);
FileSystem* fs; // Pointer to singleton
TF_EXPECT_OK(Env::Default()->GetFileSystemForFile(src_loc, &fs));
// FileSystem::FileExists currently returns OkStatus() if the file
// exists and errors::NotFound() if the file doesn't exist. There's no
// indication in the code or docs about whether other error codes may be
// added in the future, so we code defensively here.
Status exists_status = fs->FileExists(dest_loc);
if (exists_status.code() == error::NOT_FOUND) {
TF_EXPECT_OK(fs->CopyFile(src_loc, dest_loc));
} else {
TF_EXPECT_OK(exists_status);
}
return dest_loc;
}
LMDBDatasetParams SingleValidInput() {
return {/*filenames=*/{MaybeCopyDataFile()},
/*output_dtypes=*/{DT_STRING, DT_STRING},
/*output_shapes=*/{PartialTensorShape({}), PartialTensorShape({})},
/*node_name=*/kNodeName};
}
LMDBDatasetParams TwoValidInputs() {
return {/*filenames*/ {MaybeCopyDataFile(), MaybeCopyDataFile()},
/*output_dtypes*/ {DT_STRING, DT_STRING},
/*output_shapes*/ {PartialTensorShape({}), PartialTensorShape({})},
/*node_name=*/kNodeName};
}
LMDBDatasetParams EmptyInput() {
return {/*filenames*/ {},
/*output_dtypes*/ {DT_STRING, DT_STRING},
/*output_shapes*/ {PartialTensorShape({}), PartialTensorShape({})},
/*node_name=*/kNodeName};
}
LMDBDatasetParams InvalidPathAtStart() {
return {/*filenames*/ {"This is not a valid filename", MaybeCopyDataFile()},
/*output_dtypes*/ {DT_STRING, DT_STRING},
/*output_shapes*/ {PartialTensorShape({}), PartialTensorShape({})},
/*node_name=*/kNodeName};
}
LMDBDatasetParams InvalidPathInMiddle() {
return {/*filenames*/ {MaybeCopyDataFile(), "This is not a valid filename",
MaybeCopyDataFile()},
/*output_dtypes*/ {DT_STRING, DT_STRING},
/*output_shapes*/ {PartialTensorShape({}), PartialTensorShape({})},
/*node_name=*/kNodeName};
}
// The tensors we expect to see each time we read through the input data file.
std::vector<GetNextTestCase<LMDBDatasetParams>> GetNextTestCases() {
const std::vector<Tensor> kFileOutput = CreateTensors<tstring>(
TensorShape({}),
{
// Each call to GetNext() produces two scalar string tensors, but the
// test harness expects to receive a flat vector
{"0"}, {"a"}, //
{"1"}, {"b"}, //
{"2"}, {"c"}, //
{"3"}, {"d"}, //
{"4"}, {"e"}, //
{"5"}, {"f"}, //
{"6"}, {"g"}, //
{"7"}, {"h"}, //
{"8"}, {"i"}, //
{"9"}, {"j"}, //
});
// STL vectors don't have a "concatenate two vectors into a new vector"
// operation, so...
std::vector<Tensor> output_twice;
output_twice.insert(output_twice.end(), kFileOutput.cbegin(),
kFileOutput.cend());
output_twice.insert(output_twice.end(), kFileOutput.cbegin(),
kFileOutput.cend());
return {
{/*dataset_params=*/SingleValidInput(), /*expected_outputs=*/kFileOutput},
{/*dataset_params=*/TwoValidInputs(), /*expected_outputs=*/output_twice},
{/*dataset_params=*/EmptyInput(), /*expected_outputs=*/{}}};
}
ITERATOR_GET_NEXT_TEST_P(LMDBDatasetOpTest, LMDBDatasetParams,
GetNextTestCases());
TEST_F(LMDBDatasetOpTest, InvalidPathAtStart) {
auto dataset_params = InvalidPathAtStart();
TF_ASSERT_OK(Initialize(dataset_params));
// Errors about invalid files are only raised when attempting to read data.
bool end_of_sequence = false;
std::vector<Tensor> out_tensors;
std::vector<Tensor> next;
Status get_next_status =
iterator_->GetNext(iterator_ctx_.get(), &next, &end_of_sequence);
EXPECT_EQ(get_next_status.code(), error::INVALID_ARGUMENT);
}
TEST_F(LMDBDatasetOpTest, InvalidPathInMiddle) {
auto dataset_params = InvalidPathInMiddle();
TF_ASSERT_OK(Initialize(dataset_params));
bool end_of_sequence = false;
std::vector<Tensor> out_tensors;
// First 10 rows should be ok
for (int i = 0; i < 10; ++i) {
std::vector<Tensor> next;
TF_ASSERT_OK(
iterator_->GetNext(iterator_ctx_.get(), &next, &end_of_sequence));
EXPECT_FALSE(end_of_sequence);
}
// Next read operation should raise an error
std::vector<Tensor> next;
Status get_next_status =
iterator_->GetNext(iterator_ctx_.get(), &next, &end_of_sequence);
EXPECT_EQ(get_next_status.code(), error::INVALID_ARGUMENT);
}
std::vector<DatasetNodeNameTestCase<LMDBDatasetParams>>
DatasetNodeNameTestCases() {
return {{/*dataset_params=*/SingleValidInput(),
/*expected_node_name=*/kNodeName}};
}
DATASET_NODE_NAME_TEST_P(LMDBDatasetOpTest, LMDBDatasetParams,
DatasetNodeNameTestCases());
std::vector<DatasetTypeStringTestCase<LMDBDatasetParams>>
DatasetTypeStringTestCases() {
return {{/*dataset_params=*/SingleValidInput(),
/*expected_dataset_type_string=*/name_utils::OpName(
LMDBDatasetOp::kDatasetType)}};
}
DATASET_TYPE_STRING_TEST_P(LMDBDatasetOpTest, LMDBDatasetParams,
DatasetTypeStringTestCases());
std::vector<DatasetOutputDtypesTestCase<LMDBDatasetParams>>
DatasetOutputDtypesTestCases() {
return {{/*dataset_params=*/SingleValidInput(),
/*expected_output_dtypes=*/{DT_STRING, DT_STRING}}};
}
DATASET_OUTPUT_DTYPES_TEST_P(LMDBDatasetOpTest, LMDBDatasetParams,
DatasetOutputDtypesTestCases());
std::vector<DatasetOutputShapesTestCase<LMDBDatasetParams>>
DatasetOutputShapesTestCases() {
return {{/*dataset_params=*/SingleValidInput(),
/*expected_output_shapes=*/{PartialTensorShape({}),
PartialTensorShape({})}}};
}
DATASET_OUTPUT_SHAPES_TEST_P(LMDBDatasetOpTest, LMDBDatasetParams,
DatasetOutputShapesTestCases());
std::vector<CardinalityTestCase<LMDBDatasetParams>> CardinalityTestCases() {
return {{/*dataset_params=*/SingleValidInput(),
/*expected_cardinality=*/kUnknownCardinality}};
}
DATASET_CARDINALITY_TEST_P(LMDBDatasetOpTest, LMDBDatasetParams,
CardinalityTestCases());
std::vector<IteratorOutputDtypesTestCase<LMDBDatasetParams>>
IteratorOutputDtypesTestCases() {
return {{/*dataset_params=*/SingleValidInput(),
/*expected_output_dtypes=*/{DT_STRING, DT_STRING}}};
}
ITERATOR_OUTPUT_DTYPES_TEST_P(LMDBDatasetOpTest, LMDBDatasetParams,
IteratorOutputDtypesTestCases());
std::vector<IteratorOutputShapesTestCase<LMDBDatasetParams>>
IteratorOutputShapesTestCases() {
return {{/*dataset_params=*/SingleValidInput(),
/*expected_output_shapes=*/{PartialTensorShape({}),
PartialTensorShape({})}}};
}
ITERATOR_OUTPUT_SHAPES_TEST_P(LMDBDatasetOpTest, LMDBDatasetParams,
IteratorOutputShapesTestCases());
std::vector<IteratorPrefixTestCase<LMDBDatasetParams>>
IteratorOutputPrefixTestCases() {
return {{/*dataset_params=*/SingleValidInput(),
/*expected_iterator_prefix=*/name_utils::IteratorPrefix(
LMDBDatasetOp::kDatasetType, kIteratorPrefix)}};
}
ITERATOR_PREFIX_TEST_P(LMDBDatasetOpTest, LMDBDatasetParams,
IteratorOutputPrefixTestCases());
// No test of save and restore; save/restore functionality is not implemented
// for this dataset.
} // namespace
} // namespace experimental
} // namespace data
} // namespace tensorflow

100
tensorflow/core/kernels/lmdb_reader_op.cc
View File

@ -18,109 +18,21 @@ limitations under the License.
#include "tensorflow/core/lib/core/errors.h"
#include <sys/stat.h>
#include "lmdb.h"
namespace tensorflow {
#define MDB_CHECK(val) CHECK_EQ(val, MDB_SUCCESS) << mdb_strerror(val)
class LMDBReader : public ReaderBase {
public:
LMDBReader(const string& node_name, Env* /*unused*/)
: ReaderBase(strings::StrCat("LMDBReader '", node_name, "'")),
mdb_env_(nullptr),
mdb_dbi_(0),
mdb_txn_(nullptr),
mdb_cursor_(nullptr) {}
Status OnWorkStartedLocked() override {
MDB_CHECK(mdb_env_create(&mdb_env_));
int flags = MDB_RDONLY | MDB_NOTLS | MDB_NOLOCK;
// Check if the LMDB filename is actually a file instead of a directory.
// If so, set appropriate flags so we can open it.
struct stat source_stat;
if (stat(current_work().c_str(), &source_stat) == 0 &&
(source_stat.st_mode & S_IFREG)) {
flags |= MDB_NOSUBDIR;
}
MDB_CHECK(mdb_env_open(mdb_env_, current_work().c_str(), flags, 0664));
MDB_CHECK(mdb_txn_begin(mdb_env_, nullptr, MDB_RDONLY, &mdb_txn_));
MDB_CHECK(mdb_dbi_open(mdb_txn_, nullptr, 0, &mdb_dbi_));
return OkStatus();
}
Status OnWorkFinishedLocked() override {
if (mdb_env_ != nullptr) {
if (mdb_cursor_) {
mdb_cursor_close(mdb_cursor_);
mdb_cursor_ = nullptr;
}
mdb_dbi_close(mdb_env_, mdb_dbi_);
mdb_txn_abort(mdb_txn_);
mdb_env_close(mdb_env_);
mdb_txn_ = nullptr;
mdb_dbi_ = 0;
mdb_env_ = nullptr;
}
return OkStatus();
}
Status ReadLocked(tstring* key, tstring* value, bool* produced,
bool* at_end) override {
if (mdb_cursor_ == nullptr) {
MDB_CHECK(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_));
if (Seek(MDB_FIRST) == false) {
*at_end = true;
return OkStatus();
}
} else {
if (Seek(MDB_NEXT) == false) {
*at_end = true;
return OkStatus();
}
}
*key =
tstring(static_cast<const char*>(mdb_key_.mv_data), mdb_key_.mv_size);
*value = tstring(static_cast<const char*>(mdb_value_.mv_data),
mdb_value_.mv_size);
*produced = true;
return OkStatus();
}
Status ResetLocked() override {
CHECK_EQ(Seek(MDB_FIRST), true);
return ReaderBase::ResetLocked();
}
private:
bool Seek(MDB_cursor_op op) {
CHECK_NOTNULL(mdb_cursor_);
int mdb_status = mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, op);
if (mdb_status == MDB_NOTFOUND) {
return false;
} else {
MDB_CHECK(mdb_status);
return true;
}
}
MDB_env* mdb_env_;
MDB_dbi mdb_dbi_;
MDB_txn* mdb_txn_;
MDB_cursor* mdb_cursor_;
MDB_val mdb_key_, mdb_value_;
};
class LMDBReaderOp : public ReaderOpKernel {
public:
explicit LMDBReaderOp(OpKernelConstruction* context)
: ReaderOpKernel(context) {
Env* env = context->env();
SetReaderFactory([this, env]() { return new LMDBReader(name(), env); });
OP_REQUIRES(
context, false,
errors::Unimplemented(
"LMDB support is removed from TensorFlow. This API will be deleted "
"in the next TensorFlow release. If you need LMDB support, please "
"file a GitHub issue."));
}
};

13
tensorflow/core/lib/lmdb/BUILD
View File

@ -1,13 +0,0 @@
# Description:
# lmdb test data packages alias.
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
alias(
name = "lmdb_testdata",
actual = "//tensorflow/core/lib/lmdb/testdata:lmdb_testdata",
visibility = ["//visibility:public"],
)

31
tensorflow/core/lib/lmdb/testdata/BUILD vendored
View File

@ -1,31 +0,0 @@
# Description:
# lmdb test data packages.
load("//tensorflow:tensorflow.default.bzl", "filegroup")
package(
# copybara:uncomment default_applicable_licenses = ["//tensorflow:license"],
licenses = ["notice"],
)
filegroup(
name = "lmdb_testdata",
testonly = 1,
srcs = [
# A simple key-value store:
# 0 : 'b'
# 1 : 'b'
# ...
# 9 : 'b'
# Which is then overwritten with:
# 0 : 'a'
# 1 : 'b'
# ...
# 9 : 'j'
"data.mdb",
# LMDB, being a memory-mapped database, uses a different file format on
# big-endian systems.
"data_bigendian.mdb",
],
visibility = ["//visibility:public"],
)

BIN
tensorflow/core/lib/lmdb/testdata/data.mdb vendored
View File

Binary file not shown.

BIN
tensorflow/core/lib/lmdb/testdata/data_bigendian.mdb vendored
View File

Binary file not shown.

1
tensorflow/opensource_only.files
View File

@ -304,7 +304,6 @@ third_party/llvm_openmp/BUILD:
third_party/llvm_openmp/cmake_vars.bzl:
third_party/llvm_openmp/expand_cmake_vars:.py
third_party/llvm_openmp/openmp.bzl:
third_party/lmdb.BUILD:
third_party/mkl/BUILD:
third_party/mkl/build_defs.bzl:
third_party/mkl_dnn/LICENSE:

1
tensorflow/python/BUILD
View File

@ -3461,7 +3461,6 @@ pywrap_tensorflow_macro(
"@llvm-project//:__subpackages__",
"@llvm_terminfo//:__subpackages__",
"@llvm_zlib//:__subpackages__",
"@lmdb//:__subpackages__",
"@local_config_cuda//:__subpackages__",
"@local_config_git//:__subpackages__",
"@local_config_nccl//:__subpackages__",

1
tensorflow/python/kernel_tests/io_ops/BUILD
View File

@ -92,7 +92,6 @@ tf_py_test(
name = "reader_ops_test",
size = "small",
srcs = ["reader_ops_test.py"],
data = ["//tensorflow/core/lib/lmdb:lmdb_testdata"],
deps = [
"//tensorflow/core:protos_all_py",
"//tensorflow/python:client_testlib",

95
tensorflow/python/kernel_tests/io_ops/reader_ops_test.py
View File

@ -17,8 +17,6 @@
import collections
import gzip
import os
import shutil
import sys
import threading
import zlib
@ -31,9 +29,6 @@ from tensorflow.python.ops import data_flow_ops
from tensorflow.python.ops import io_ops
from tensorflow.python.ops import variables
from tensorflow.python.platform import test
from tensorflow.python.training import coordinator
from tensorflow.python.training import input as input_lib
from tensorflow.python.training import queue_runner_impl
from tensorflow.python.util import compat
prefix_path = "tensorflow/core/lib"
@ -746,95 +741,5 @@ class AsyncReaderTest(test.TestCase):
output.append(sess.run(args))
class LMDBReaderTest(test.TestCase):
def setUp(self):
super(LMDBReaderTest, self).setUp()
# Copy database out because we need the path to be writable to use locks.
# The on-disk format of an LMDB file is different on big-endian machines,
# because LMDB is a memory-mapped database.
db_file = "data.mdb" if sys.byteorder == "little" else "data_bigendian.mdb"
path = os.path.join(prefix_path, "lmdb", "testdata", db_file)
self.db_path = os.path.join(self.get_temp_dir(), "data.mdb")
shutil.copy(path, self.db_path)
@test_util.run_deprecated_v1
def testReadFromFile(self):
reader = io_ops.LMDBReader(name="test_read_from_file")
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
key, value = reader.read(queue)
self.evaluate(queue.enqueue([self.db_path]))
self.evaluate(queue.close())
for i in range(10):
k, v = self.evaluate([key, value])
self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(i)))
self.assertAllEqual(
compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + i))))
with self.assertRaisesOpError("is closed and has insufficient elements "
"\\(requested 1, current size 0\\)"):
k, v = self.evaluate([key, value])
@test_util.run_deprecated_v1
def testReadFromSameFile(self):
with self.cached_session() as sess:
reader1 = io_ops.LMDBReader(name="test_read_from_same_file1")
reader2 = io_ops.LMDBReader(name="test_read_from_same_file2")
filename_queue = input_lib.string_input_producer(
[self.db_path], num_epochs=None)
key1, value1 = reader1.read(filename_queue)
key2, value2 = reader2.read(filename_queue)
coord = coordinator.Coordinator()
threads = queue_runner_impl.start_queue_runners(sess, coord=coord)
for _ in range(3):
for _ in range(10):
k1, v1, k2, v2 = self.evaluate([key1, value1, key2, value2])
self.assertAllEqual(compat.as_bytes(k1), compat.as_bytes(k2))
self.assertAllEqual(compat.as_bytes(v1), compat.as_bytes(v2))
coord.request_stop()
coord.join(threads)
@test_util.run_deprecated_v1
def testReadFromFolder(self):
reader = io_ops.LMDBReader(name="test_read_from_folder")
queue = data_flow_ops.FIFOQueue(99, [dtypes.string], shapes=())
key, value = reader.read(queue)
self.evaluate(queue.enqueue([self.db_path]))
self.evaluate(queue.close())
for i in range(10):
k, v = self.evaluate([key, value])
self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(i)))
self.assertAllEqual(
compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + i))))
with self.assertRaisesOpError("is closed and has insufficient elements "
"\\(requested 1, current size 0\\)"):
k, v = self.evaluate([key, value])
@test_util.run_deprecated_v1
def testReadFromFileRepeatedly(self):
with self.cached_session() as sess:
reader = io_ops.LMDBReader(name="test_read_from_file_repeated")
filename_queue = input_lib.string_input_producer(
[self.db_path], num_epochs=None)
key, value = reader.read(filename_queue)
coord = coordinator.Coordinator()
threads = queue_runner_impl.start_queue_runners(sess, coord=coord)
# Iterate over the lmdb 3 times.
for _ in range(3):
# Go over all 10 records each time.
for j in range(10):
k, v = self.evaluate([key, value])
self.assertAllEqual(compat.as_bytes(k), compat.as_bytes(str(j)))
self.assertAllEqual(
compat.as_bytes(v), compat.as_bytes(str(chr(ord("a") + j))))
coord.request_stop()
coord.join(threads)
if __name__ == "__main__":
test.main()

1
tensorflow/tensorflow.bzl
View File

@ -3219,7 +3219,6 @@ def tf_python_pybind_static_deps(testonly = False):
"@llvm_openmp//:__subpackages__",
"@llvm_terminfo//:__subpackages__",
"@llvm_zlib//:__subpackages__",
"@lmdb//:__subpackages__",
"@local_config_cuda//:__subpackages__",
"@local_config_git//:__subpackages__",
"@local_config_nccl//:__subpackages__",

2
tensorflow/tools/lib_package/BUILD
View File

@ -155,7 +155,6 @@ genrule(
"@libjpeg_turbo//:LICENSE.md",
"@llvm-project//llvm:LICENSE.TXT",
"@llvm-project//mlir:LICENSE.TXT",
"@lmdb//:LICENSE",
"@local_config_tensorrt//:LICENSE",
"@nsync//:LICENSE",
"@png//:LICENSE",
@ -221,7 +220,6 @@ genrule(
"@libjpeg_turbo//:LICENSE.md",
"@llvm-project//llvm:LICENSE.TXT",
"@llvm-project//mlir:LICENSE.TXT",
"@lmdb//:LICENSE",
"@local_config_tensorrt//:LICENSE",
"@nsync//:LICENSE",
"@png//:LICENSE",

1
tensorflow/tools/pip_package/BUILD
View File

@ -217,7 +217,6 @@ filegroup(
"@libjpeg_turbo//:LICENSE.md",
"@llvm-project//llvm:LICENSE.TXT",
"@llvm-project//mlir:LICENSE.TXT",
"@lmdb//:LICENSE",
"@local_config_tensorrt//:LICENSE",
"@nsync//:LICENSE",
"@opt_einsum_archive//:LICENSE",

2
tensorflow/tools/pip_package/pip_smoke_test.py
View File

@ -80,8 +80,6 @@ DEPENDENCY_DENYLIST = [
"//tensorflow:no_tensorflow_py_deps",
"//tensorflow/tools/pip_package:win_pip_package_marker",
"//tensorflow/core:image_testdata",
"//tensorflow/core/lib/lmdb:lmdb_testdata",
"//tensorflow/core/lib/lmdb/testdata:lmdb_testdata",
"//tensorflow/core/kernels/cloud:bigquery_reader_ops",
"//tensorflow/python:extra_py_tests_deps",
"//tensorflow/python:mixed_precision",

2
tensorflow/tools/tf_sig_build_dockerfiles/devel.usertools/code_check_full.bats
View File

@ -151,8 +151,6 @@ _test_lib$
//tensorflow:no_tensorflow_py_deps
//tensorflow/tools/pip_package:win_pip_package_marker
//tensorflow/core:image_testdata
//tensorflow/core/lib/lmdb:lmdb_testdata
//tensorflow/core/lib/lmdb/testdata:lmdb_testdata
//tensorflow/core/kernels/cloud:bigquery_reader_ops
//tensorflow/python:extra_py_tests_deps
//tensorflow/python:mixed_precision

9
tensorflow/workspace2.bzl
View File

@ -548,15 +548,6 @@ def _tf_repositories():
urls = tf_mirror_urls("https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.1/openmp-10.0.1.src.tar.xz"),
)
tf_http_archive(
name = "lmdb",
build_file = "//third_party:lmdb.BUILD",
sha256 = "22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb",
strip_prefix = "lmdb-LMDB_0.9.29/libraries/liblmdb",
system_build_file = "//third_party/systemlibs:lmdb.BUILD",
urls = tf_mirror_urls("https://github.com/LMDB/lmdb/archive/refs/tags/LMDB_0.9.29.tar.gz"),
)
tf_http_archive(
name = "jsoncpp_git",
sha256 = "f409856e5920c18d0c2fb85276e24ee607d2a09b5e7d5f0a371368903c275da2",

31
third_party/lmdb.BUILD vendored
View File

@ -1,31 +0,0 @@
# Description:
# LMDB is the Lightning Memory-mapped Database.
licenses(["notice"]) # OpenLDAP Public License
exports_files(["LICENSE"])
cc_library(
name = "lmdb",
srcs = [
"mdb.c",
"midl.c",
],
hdrs = [
"lmdb.h",
"midl.h",
],
copts = [
"-w",
],
linkopts = select({
":windows": ["-DEFAULTLIB:advapi32.lib"], # InitializeSecurityDescriptor, SetSecurityDescriptorDacl
"//conditions:default": ["-lpthread"],
}),
visibility = ["//visibility:public"],
)
config_setting(
name = "windows",
values = {"cpu": "x64_windows"},
)

1
third_party/systemlibs/syslibs_configure.bzl vendored
View File

@ -30,7 +30,6 @@ VALID_LIBS = [
"icu",
"jsoncpp_git",
"libjpeg_turbo",
"lmdb",
"nasm",
"nsync",
"opt_einsum_archive",