You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by je...@apache.org on 2017/10/09 16:24:41 UTC
[01/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Repository: nifi-minifi-cpp
Updated Branches:
refs/heads/master 380a98bb8 -> 488677321
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/env/env_chroot.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/env/env_chroot.cc b/thirdparty/rocksdb/env/env_chroot.cc
new file mode 100644
index 0000000..6a1fda8
--- /dev/null
+++ b/thirdparty/rocksdb/env/env_chroot.cc
@@ -0,0 +1,324 @@
+// Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#if !defined(ROCKSDB_LITE) && !defined(OS_WIN)
+
+#include "env/env_chroot.h"
+
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "rocksdb/status.h"
+
+namespace rocksdb {
+
+class ChrootEnv : public EnvWrapper {
+ public:
+ ChrootEnv(Env* base_env, const std::string& chroot_dir)
+ : EnvWrapper(base_env) {
+#if defined(OS_AIX)
+ char resolvedName[PATH_MAX];
+ char* real_chroot_dir = realpath(chroot_dir.c_str(), resolvedName);
+#else
+ char* real_chroot_dir = realpath(chroot_dir.c_str(), nullptr);
+#endif
+ // chroot_dir must exist so realpath() returns non-nullptr.
+ assert(real_chroot_dir != nullptr);
+ chroot_dir_ = real_chroot_dir;
+#if !defined(OS_AIX)
+ free(real_chroot_dir);
+#endif
+ }
+
+ virtual Status NewSequentialFile(const std::string& fname,
+ std::unique_ptr<SequentialFile>* result,
+ const EnvOptions& options) override {
+ auto status_and_enc_path = EncodePathWithNewBasename(fname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::NewSequentialFile(status_and_enc_path.second, result,
+ options);
+ }
+
+ virtual Status NewRandomAccessFile(const std::string& fname,
+ unique_ptr<RandomAccessFile>* result,
+ const EnvOptions& options) override {
+ auto status_and_enc_path = EncodePathWithNewBasename(fname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::NewRandomAccessFile(status_and_enc_path.second, result,
+ options);
+ }
+
+ virtual Status NewWritableFile(const std::string& fname,
+ unique_ptr<WritableFile>* result,
+ const EnvOptions& options) override {
+ auto status_and_enc_path = EncodePathWithNewBasename(fname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::NewWritableFile(status_and_enc_path.second, result,
+ options);
+ }
+
+ virtual Status ReuseWritableFile(const std::string& fname,
+ const std::string& old_fname,
+ unique_ptr<WritableFile>* result,
+ const EnvOptions& options) override {
+ auto status_and_enc_path = EncodePathWithNewBasename(fname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ auto status_and_old_enc_path = EncodePath(old_fname);
+ if (!status_and_old_enc_path.first.ok()) {
+ return status_and_old_enc_path.first;
+ }
+ return EnvWrapper::ReuseWritableFile(status_and_old_enc_path.second,
+ status_and_old_enc_path.second, result,
+ options);
+ }
+
+ virtual Status NewRandomRWFile(const std::string& fname,
+ unique_ptr<RandomRWFile>* result,
+ const EnvOptions& options) override {
+ auto status_and_enc_path = EncodePathWithNewBasename(fname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::NewRandomRWFile(status_and_enc_path.second, result,
+ options);
+ }
+
+ virtual Status NewDirectory(const std::string& dir,
+ unique_ptr<Directory>* result) override {
+ auto status_and_enc_path = EncodePathWithNewBasename(dir);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::NewDirectory(status_and_enc_path.second, result);
+ }
+
+ virtual Status FileExists(const std::string& fname) override {
+ auto status_and_enc_path = EncodePathWithNewBasename(fname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::FileExists(status_and_enc_path.second);
+ }
+
+ virtual Status GetChildren(const std::string& dir,
+ std::vector<std::string>* result) override {
+ auto status_and_enc_path = EncodePath(dir);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::GetChildren(status_and_enc_path.second, result);
+ }
+
+ virtual Status GetChildrenFileAttributes(
+ const std::string& dir, std::vector<FileAttributes>* result) override {
+ auto status_and_enc_path = EncodePath(dir);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::GetChildrenFileAttributes(status_and_enc_path.second,
+ result);
+ }
+
+ virtual Status DeleteFile(const std::string& fname) override {
+ auto status_and_enc_path = EncodePath(fname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::DeleteFile(status_and_enc_path.second);
+ }
+
+ virtual Status CreateDir(const std::string& dirname) override {
+ auto status_and_enc_path = EncodePathWithNewBasename(dirname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::CreateDir(status_and_enc_path.second);
+ }
+
+ virtual Status CreateDirIfMissing(const std::string& dirname) override {
+ auto status_and_enc_path = EncodePathWithNewBasename(dirname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::CreateDirIfMissing(status_and_enc_path.second);
+ }
+
+ virtual Status DeleteDir(const std::string& dirname) override {
+ auto status_and_enc_path = EncodePath(dirname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::DeleteDir(status_and_enc_path.second);
+ }
+
+ virtual Status GetFileSize(const std::string& fname,
+ uint64_t* file_size) override {
+ auto status_and_enc_path = EncodePath(fname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::GetFileSize(status_and_enc_path.second, file_size);
+ }
+
+ virtual Status GetFileModificationTime(const std::string& fname,
+ uint64_t* file_mtime) override {
+ auto status_and_enc_path = EncodePath(fname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::GetFileModificationTime(status_and_enc_path.second,
+ file_mtime);
+ }
+
+ virtual Status RenameFile(const std::string& src,
+ const std::string& dest) override {
+ auto status_and_src_enc_path = EncodePath(src);
+ if (!status_and_src_enc_path.first.ok()) {
+ return status_and_src_enc_path.first;
+ }
+ auto status_and_dest_enc_path = EncodePathWithNewBasename(dest);
+ if (!status_and_dest_enc_path.first.ok()) {
+ return status_and_dest_enc_path.first;
+ }
+ return EnvWrapper::RenameFile(status_and_src_enc_path.second,
+ status_and_dest_enc_path.second);
+ }
+
+ virtual Status LinkFile(const std::string& src,
+ const std::string& dest) override {
+ auto status_and_src_enc_path = EncodePath(src);
+ if (!status_and_src_enc_path.first.ok()) {
+ return status_and_src_enc_path.first;
+ }
+ auto status_and_dest_enc_path = EncodePathWithNewBasename(dest);
+ if (!status_and_dest_enc_path.first.ok()) {
+ return status_and_dest_enc_path.first;
+ }
+ return EnvWrapper::LinkFile(status_and_src_enc_path.second,
+ status_and_dest_enc_path.second);
+ }
+
+ virtual Status LockFile(const std::string& fname, FileLock** lock) override {
+ auto status_and_enc_path = EncodePathWithNewBasename(fname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ // FileLock subclasses may store path (e.g., PosixFileLock stores it). We
+ // can skip stripping the chroot directory from this path because callers
+ // shouldn't use it.
+ return EnvWrapper::LockFile(status_and_enc_path.second, lock);
+ }
+
+ virtual Status GetTestDirectory(std::string* path) override {
+ // Adapted from PosixEnv's implementation since it doesn't provide a way to
+ // create directory in the chroot.
+ char buf[256];
+ snprintf(buf, sizeof(buf), "/rocksdbtest-%d", static_cast<int>(geteuid()));
+ *path = buf;
+
+ // Directory may already exist, so ignore return
+ CreateDir(*path);
+ return Status::OK();
+ }
+
+ virtual Status NewLogger(const std::string& fname,
+ shared_ptr<Logger>* result) override {
+ auto status_and_enc_path = EncodePathWithNewBasename(fname);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::NewLogger(status_and_enc_path.second, result);
+ }
+
+ virtual Status GetAbsolutePath(const std::string& db_path,
+ std::string* output_path) override {
+ auto status_and_enc_path = EncodePath(db_path);
+ if (!status_and_enc_path.first.ok()) {
+ return status_and_enc_path.first;
+ }
+ return EnvWrapper::GetAbsolutePath(status_and_enc_path.second, output_path);
+ }
+
+ private:
+ // Returns status and expanded absolute path including the chroot directory.
+ // Checks whether the provided path breaks out of the chroot. If it returns
+ // non-OK status, the returned path should not be used.
+ std::pair<Status, std::string> EncodePath(const std::string& path) {
+ if (path.empty() || path[0] != '/') {
+ return {Status::InvalidArgument(path, "Not an absolute path"), ""};
+ }
+ std::pair<Status, std::string> res;
+ res.second = chroot_dir_ + path;
+#if defined(OS_AIX)
+ char resolvedName[PATH_MAX];
+ char* normalized_path = realpath(res.second.c_str(), resolvedName);
+#else
+ char* normalized_path = realpath(res.second.c_str(), nullptr);
+#endif
+ if (normalized_path == nullptr) {
+ res.first = Status::NotFound(res.second, strerror(errno));
+ } else if (strlen(normalized_path) < chroot_dir_.size() ||
+ strncmp(normalized_path, chroot_dir_.c_str(),
+ chroot_dir_.size()) != 0) {
+ res.first = Status::IOError(res.second,
+ "Attempted to access path outside chroot");
+ } else {
+ res.first = Status::OK();
+ }
+#if !defined(OS_AIX)
+ free(normalized_path);
+#endif
+ return res;
+ }
+
+ // Similar to EncodePath() except assumes the basename in the path hasn't been
+ // created yet.
+ std::pair<Status, std::string> EncodePathWithNewBasename(
+ const std::string& path) {
+ if (path.empty() || path[0] != '/') {
+ return {Status::InvalidArgument(path, "Not an absolute path"), ""};
+ }
+ // Basename may be followed by trailing slashes
+ size_t final_idx = path.find_last_not_of('/');
+ if (final_idx == std::string::npos) {
+ // It's only slashes so no basename to extract
+ return EncodePath(path);
+ }
+
+ // Pull off the basename temporarily since realname(3) (used by
+ // EncodePath()) requires a path that exists
+ size_t base_sep = path.rfind('/', final_idx);
+ auto status_and_enc_path = EncodePath(path.substr(0, base_sep + 1));
+ status_and_enc_path.second.append(path.substr(base_sep + 1));
+ return status_and_enc_path;
+ }
+
+ std::string chroot_dir_;
+};
+
+Env* NewChrootEnv(Env* base_env, const std::string& chroot_dir) {
+ if (!base_env->FileExists(chroot_dir).ok()) {
+ return nullptr;
+ }
+ return new ChrootEnv(base_env, chroot_dir);
+}
+
+} // namespace rocksdb
+
+#endif // !defined(ROCKSDB_LITE) && !defined(OS_WIN)
[51/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
MINIFI-372: Replace leveldb with RocksDB
MINIFI-372: Update Cmake
MINIFI-372: Fix tests
MINIFI-372: Include deps
MINIFI-372: Rename data
MINIFI-372: Change cmake cxx flags
MINIFI-372: Remove LevelDB reference. Not including RocksDB because
it is dual licensed under Apache License, Version2.0
MINIFI-372: Remove level db cmake fles and edit comments
MINIFI-372: Add test to cover stream and content repo
MINIFI-372: Update readme
MINIFI-372: Remove rocksdb tests and tools that aren't needed from the source
MINIFI-372: Update feature names and how they are validated
MINIFI_372: Update travis to install rocksdb
MINIFI-372: Resolve issue with inheritance across the class loader
MINIFI-372: Fix volatile repos and polymorphic issues
MINIFI-372: Remove cout statement
MINIFI-372: Remove extraneous RocksDB
This closes #142
Signed-off-by: Jeremy Dyer <je...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/commit/48867732
Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/tree/48867732
Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/diff/48867732
Branch: refs/heads/master
Commit: 488677321cdf32c196fdaae6fb31a2b38ef10461
Parents: 380a98b
Author: Marc Parisi <ph...@apache.org>
Authored: Tue Aug 15 11:34:12 2017 -0400
Committer: Jeremy Dyer <je...@apache.org>
Committed: Mon Oct 9 12:23:07 2017 -0400
----------------------------------------------------------------------
.travis.yml | 5 +-
CMakeLists.txt | 50 +-
LICENSE | 30 -
README.md | 18 +-
cmake/BuildTests.cmake | 30 +-
cmake/FindLeveldb.cmake | 50 -
cmake/FindRocksDB.cmake | 51 +
docker/Dockerfile | 12 +-
extensions/http-curl/CMakeLists.txt | 2 +-
extensions/rocksdb-repos/CMakeLists.txt | 84 +
.../rocksdb-repos/DatabaseContentRepository.cpp | 112 +
.../rocksdb-repos/DatabaseContentRepository.h | 132 +
extensions/rocksdb-repos/FlowFileRepository.cpp | 140 +
extensions/rocksdb-repos/FlowFileRepository.h | 168 +
.../rocksdb-repos/ProvenanceRepository.cpp | 93 +
extensions/rocksdb-repos/ProvenanceRepository.h | 256 +
extensions/rocksdb-repos/RocksDBLoader.cpp | 29 +
extensions/rocksdb-repos/RocksDBLoader.h | 80 +
extensions/rocksdb-repos/RocksDbStream.cpp | 124 +
extensions/rocksdb-repos/RocksDbStream.h | 185 +
libminifi/CMakeLists.txt | 17 +-
libminifi/cmake/FindLeveldb.cmake | 50 -
libminifi/cmake/FindRocksDB.cmake | 51 +
libminifi/include/Connection.h | 8 +-
libminifi/include/SchedulingAgent.h | 1 -
libminifi/include/core/ClassLoader.h | 37 +-
libminifi/include/core/Connectable.h | 4 +
libminifi/include/core/ContentRepository.h | 2 +
libminifi/include/core/Core.h | 6 +-
libminifi/include/core/FlowConfiguration.h | 16 +-
libminifi/include/core/FlowFile.h | 31 +-
libminifi/include/core/ProcessGroup.h | 2 +
libminifi/include/core/Repository.h | 10 +-
libminifi/include/core/SerializableComponent.h | 24 +-
.../core/repository/FlowFileRepository.h | 167 -
.../core/repository/VolatileContentRepository.h | 7 +-
.../repository/VolatileFlowFileRepository.h | 18 +-
.../repository/VolatileProvenanceRepository.h | 6 +-
.../core/repository/VolatileRepository.h | 17 +-
libminifi/include/properties/Configure.h | 1 +
.../include/provenance/ProvenanceRepository.h | 250 -
libminifi/src/Configure.cpp | 1 +
libminifi/src/FlowController.cpp | 9 +-
libminifi/src/FlowFileRecord.cpp | 22 +-
libminifi/src/core/FlowConfiguration.cpp | 3 +-
libminifi/src/core/FlowFile.cpp | 13 +-
libminifi/src/core/ProcessGroup.cpp | 10 +
libminifi/src/core/Repository.cpp | 1 -
libminifi/src/core/RepositoryFactory.cpp | 57 +-
.../src/core/repository/FlowFileRepository.cpp | 140 -
.../repository/VolatileContentRepository.cpp | 19 +-
.../src/provenance/ProvenanceRepository.cpp | 92 -
.../test/curl-tests/C2NullConfiguration.cpp | 4 +-
libminifi/test/nodefs/NoLevelDB.cpp | 32 -
libminifi/test/nodefs/NoRocksDB.cpp | 32 +
.../rocksdb-tests/DBContentRepositoryTests.cpp | 246 +
.../test/rocksdb-tests/ProvenanceTests.cpp | 165 +
libminifi/test/rocksdb-tests/RepoTests.cpp | 162 +
libminifi/test/unit/ProvenanceTestHelper.h | 7 +-
libminifi/test/unit/ProvenanceTests.cpp | 164 -
libminifi/test/unit/RepoTests.cpp | 162 -
main/CMakeLists.txt | 36 +-
main/MiNiFiMain.cpp | 3 +
thirdparty/leveldb-1.18/.gitignore | 33 -
thirdparty/leveldb-1.18/AUTHORS | 12 -
thirdparty/leveldb-1.18/CMakeLists.txt | 186 -
thirdparty/leveldb-1.18/LICENSE | 27 -
thirdparty/leveldb-1.18/NEWS | 17 -
thirdparty/leveldb-1.18/README | 51 -
thirdparty/leveldb-1.18/README.md | 20 -
thirdparty/leveldb-1.18/TODO | 14 -
thirdparty/leveldb-1.18/WINDOWS | 42 -
thirdparty/leveldb-1.18/build_detect_platform | 228 -
.../leveldb-1.18/build_detect_platform.cmake | 7 -
thirdparty/leveldb-1.18/db/autocompact_test.cc | 118 -
thirdparty/leveldb-1.18/db/builder.cc | 88 -
thirdparty/leveldb-1.18/db/builder.h | 34 -
thirdparty/leveldb-1.18/db/c.cc | 595 -
thirdparty/leveldb-1.18/db/c_test.c | 390 -
thirdparty/leveldb-1.18/db/corruption_test.cc | 388 -
thirdparty/leveldb-1.18/db/db_bench.cc | 979 -
thirdparty/leveldb-1.18/db/db_impl.cc | 1529 --
thirdparty/leveldb-1.18/db/db_impl.h | 211 -
thirdparty/leveldb-1.18/db/db_iter.cc | 317 -
thirdparty/leveldb-1.18/db/db_iter.h | 28 -
thirdparty/leveldb-1.18/db/db_test.cc | 2129 --
thirdparty/leveldb-1.18/db/dbformat.cc | 140 -
thirdparty/leveldb-1.18/db/dbformat.h | 230 -
thirdparty/leveldb-1.18/db/dbformat_test.cc | 112 -
thirdparty/leveldb-1.18/db/dumpfile.cc | 225 -
thirdparty/leveldb-1.18/db/filename.cc | 144 -
thirdparty/leveldb-1.18/db/filename.h | 85 -
thirdparty/leveldb-1.18/db/filename_test.cc | 123 -
thirdparty/leveldb-1.18/db/leveldb_main.cc | 64 -
thirdparty/leveldb-1.18/db/log_format.h | 35 -
thirdparty/leveldb-1.18/db/log_reader.cc | 266 -
thirdparty/leveldb-1.18/db/log_reader.h | 108 -
thirdparty/leveldb-1.18/db/log_test.cc | 530 -
thirdparty/leveldb-1.18/db/log_writer.cc | 103 -
thirdparty/leveldb-1.18/db/log_writer.h | 48 -
thirdparty/leveldb-1.18/db/memtable.cc | 145 -
thirdparty/leveldb-1.18/db/memtable.h | 91 -
thirdparty/leveldb-1.18/db/repair.cc | 461 -
thirdparty/leveldb-1.18/db/skiplist.h | 384 -
thirdparty/leveldb-1.18/db/skiplist_test.cc | 378 -
thirdparty/leveldb-1.18/db/snapshot.h | 66 -
thirdparty/leveldb-1.18/db/table_cache.cc | 127 -
thirdparty/leveldb-1.18/db/table_cache.h | 61 -
thirdparty/leveldb-1.18/db/version_edit.cc | 266 -
thirdparty/leveldb-1.18/db/version_edit.h | 107 -
thirdparty/leveldb-1.18/db/version_edit_test.cc | 46 -
thirdparty/leveldb-1.18/db/version_set.cc | 1484 --
thirdparty/leveldb-1.18/db/version_set.h | 396 -
thirdparty/leveldb-1.18/db/version_set_test.cc | 179 -
thirdparty/leveldb-1.18/db/write_batch.cc | 147 -
.../leveldb-1.18/db/write_batch_internal.h | 49 -
thirdparty/leveldb-1.18/db/write_batch_test.cc | 125 -
.../leveldb-1.18/doc/bench/db_bench_sqlite3.cc | 718 -
.../leveldb-1.18/doc/bench/db_bench_tree_db.cc | 528 -
thirdparty/leveldb-1.18/doc/benchmark.html | 459 -
thirdparty/leveldb-1.18/doc/doc.css | 89 -
thirdparty/leveldb-1.18/doc/impl.html | 213 -
thirdparty/leveldb-1.18/doc/index.html | 549 -
thirdparty/leveldb-1.18/doc/log_format.txt | 75 -
thirdparty/leveldb-1.18/doc/table_format.txt | 104 -
.../leveldb-1.18/helpers/memenv/memenv.cc | 385 -
thirdparty/leveldb-1.18/helpers/memenv/memenv.h | 20 -
.../leveldb-1.18/helpers/memenv/memenv_test.cc | 232 -
thirdparty/leveldb-1.18/include/leveldb/c.h | 290 -
thirdparty/leveldb-1.18/include/leveldb/cache.h | 99 -
.../leveldb-1.18/include/leveldb/comparator.h | 66 -
thirdparty/leveldb-1.18/include/leveldb/db.h | 161 -
.../leveldb-1.18/include/leveldb/dumpfile.h | 25 -
thirdparty/leveldb-1.18/include/leveldb/env.h | 339 -
.../include/leveldb/filter_policy.h | 70 -
.../leveldb-1.18/include/leveldb/iterator.h | 100 -
.../leveldb-1.18/include/leveldb/options.h | 195 -
thirdparty/leveldb-1.18/include/leveldb/slice.h | 109 -
.../leveldb-1.18/include/leveldb/status.h | 107 -
thirdparty/leveldb-1.18/include/leveldb/table.h | 85 -
.../include/leveldb/table_builder.h | 92 -
.../leveldb-1.18/include/leveldb/write_batch.h | 64 -
thirdparty/leveldb-1.18/issues/issue178_test.cc | 92 -
thirdparty/leveldb-1.18/issues/issue200_test.cc | 59 -
thirdparty/leveldb-1.18/port/README | 10 -
thirdparty/leveldb-1.18/port/atomic_pointer.h | 223 -
thirdparty/leveldb-1.18/port/port.h | 16 -
thirdparty/leveldb-1.18/port/port_android.cc | 64 -
thirdparty/leveldb-1.18/port/port_android.h | 159 -
thirdparty/leveldb-1.18/port/port_example.h | 135 -
thirdparty/leveldb-1.18/port/port_posix.cc | 54 -
thirdparty/leveldb-1.18/port/port_posix.h | 154 -
thirdparty/leveldb-1.18/port/port_win.cc | 182 -
thirdparty/leveldb-1.18/port/port_win.h | 164 -
thirdparty/leveldb-1.18/port/sha1_portable.cc | 298 -
thirdparty/leveldb-1.18/port/sha1_portable.h | 25 -
thirdparty/leveldb-1.18/port/sha1_test.cc | 39 -
.../leveldb-1.18/port/thread_annotations.h | 60 -
thirdparty/leveldb-1.18/port/win/stdint.h | 24 -
thirdparty/leveldb-1.18/table/block.cc | 268 -
thirdparty/leveldb-1.18/table/block.h | 44 -
thirdparty/leveldb-1.18/table/block_builder.cc | 109 -
thirdparty/leveldb-1.18/table/block_builder.h | 57 -
thirdparty/leveldb-1.18/table/filter_block.cc | 111 -
thirdparty/leveldb-1.18/table/filter_block.h | 68 -
.../leveldb-1.18/table/filter_block_test.cc | 128 -
thirdparty/leveldb-1.18/table/format.cc | 145 -
thirdparty/leveldb-1.18/table/format.h | 108 -
thirdparty/leveldb-1.18/table/iterator.cc | 67 -
.../leveldb-1.18/table/iterator_wrapper.h | 63 -
thirdparty/leveldb-1.18/table/merger.cc | 198 -
thirdparty/leveldb-1.18/table/merger.h | 28 -
thirdparty/leveldb-1.18/table/table.cc | 285 -
thirdparty/leveldb-1.18/table/table_builder.cc | 270 -
thirdparty/leveldb-1.18/table/table_test.cc | 868 -
.../leveldb-1.18/table/two_level_iterator.cc | 182 -
.../leveldb-1.18/table/two_level_iterator.h | 34 -
thirdparty/leveldb-1.18/util/arena.cc | 68 -
thirdparty/leveldb-1.18/util/arena.h | 68 -
thirdparty/leveldb-1.18/util/arena_test.cc | 68 -
thirdparty/leveldb-1.18/util/bloom.cc | 95 -
thirdparty/leveldb-1.18/util/bloom_test.cc | 161 -
thirdparty/leveldb-1.18/util/cache.cc | 327 -
thirdparty/leveldb-1.18/util/cache_test.cc | 186 -
thirdparty/leveldb-1.18/util/coding.cc | 194 -
thirdparty/leveldb-1.18/util/coding.h | 107 -
thirdparty/leveldb-1.18/util/coding_test.cc | 196 -
thirdparty/leveldb-1.18/util/comparator.cc | 88 -
thirdparty/leveldb-1.18/util/crc32c.cc | 332 -
thirdparty/leveldb-1.18/util/crc32c.h | 45 -
thirdparty/leveldb-1.18/util/crc32c_test.cc | 72 -
thirdparty/leveldb-1.18/util/env.cc | 96 -
thirdparty/leveldb-1.18/util/env_boost.cc | 726 -
thirdparty/leveldb-1.18/util/env_posix.cc | 654 -
thirdparty/leveldb-1.18/util/env_test.cc | 71 -
thirdparty/leveldb-1.18/util/filter_policy.cc | 11 -
thirdparty/leveldb-1.18/util/hash.cc | 55 -
thirdparty/leveldb-1.18/util/hash.h | 19 -
thirdparty/leveldb-1.18/util/hash_test.cc | 54 -
thirdparty/leveldb-1.18/util/histogram.cc | 139 -
thirdparty/leveldb-1.18/util/histogram.h | 42 -
thirdparty/leveldb-1.18/util/logging.cc | 72 -
thirdparty/leveldb-1.18/util/logging.h | 43 -
thirdparty/leveldb-1.18/util/mutexlock.h | 41 -
thirdparty/leveldb-1.18/util/options.cc | 29 -
thirdparty/leveldb-1.18/util/posix_logger.h | 98 -
thirdparty/leveldb-1.18/util/random.h | 64 -
thirdparty/leveldb-1.18/util/status.cc | 75 -
thirdparty/leveldb-1.18/util/testharness.cc | 77 -
thirdparty/leveldb-1.18/util/testharness.h | 138 -
thirdparty/leveldb-1.18/util/testutil.cc | 51 -
thirdparty/leveldb-1.18/util/testutil.h | 53 -
thirdparty/leveldb-1.18/util/win_logger.cc | 79 -
thirdparty/leveldb-1.18/util/win_logger.h | 28 -
thirdparty/rocksdb/.clang-format | 5 +
thirdparty/rocksdb/.gitignore | 75 +
thirdparty/rocksdb/.travis.yml | 72 +
thirdparty/rocksdb/AUTHORS | 11 +
thirdparty/rocksdb/CMakeLists.txt | 664 +
thirdparty/rocksdb/CONTRIBUTING.md | 14 +
thirdparty/rocksdb/COPYING | 339 +
thirdparty/rocksdb/DEFAULT_OPTIONS_HISTORY.md | 24 +
thirdparty/rocksdb/DUMP_FORMAT.md | 16 +
thirdparty/rocksdb/HISTORY.md | 568 +
thirdparty/rocksdb/INSTALL.md | 144 +
thirdparty/rocksdb/LANGUAGE-BINDINGS.md | 13 +
thirdparty/rocksdb/LICENSE.Apache | 202 +
thirdparty/rocksdb/LICENSE.leveldb | 29 +
thirdparty/rocksdb/README.md | 27 +
thirdparty/rocksdb/ROCKSDB_LITE.md | 21 +
thirdparty/rocksdb/TARGETS | 533 +
thirdparty/rocksdb/USERS.md | 85 +
thirdparty/rocksdb/Vagrantfile | 34 +
thirdparty/rocksdb/WINDOWS_PORT.md | 228 +
thirdparty/rocksdb/appveyor.yml | 15 +
.../rocksdb/arcanist_util/INTERNAL_ONLY_DIR | 2 +
thirdparty/rocksdb/buckifier/buckify_rocksdb.py | 172 +
thirdparty/rocksdb/buckifier/targets_builder.py | 65 +
thirdparty/rocksdb/buckifier/targets_cfg.py | 124 +
thirdparty/rocksdb/buckifier/util.py | 107 +
.../rocksdb/build_tools/RocksDBCommonHelper.php | 365 +
thirdparty/rocksdb/build_tools/amalgamate.py | 110 +
.../rocksdb/build_tools/build_detect_platform | 532 +
.../rocksdb/build_tools/cont_integration.sh | 135 +
thirdparty/rocksdb/build_tools/dependencies.sh | 18 +
.../rocksdb/build_tools/dependencies_4.8.1.sh | 18 +
thirdparty/rocksdb/build_tools/dockerbuild.sh | 2 +
thirdparty/rocksdb/build_tools/error_filter.py | 167 +
.../rocksdb/build_tools/fb_compile_mongo.sh | 55 +
thirdparty/rocksdb/build_tools/fbcode_config.sh | 156 +
.../rocksdb/build_tools/fbcode_config4.8.1.sh | 115 +
thirdparty/rocksdb/build_tools/format-diff.sh | 122 +
thirdparty/rocksdb/build_tools/gnu_parallel | 7936 +++++++
thirdparty/rocksdb/build_tools/make_package.sh | 128 +
.../rocksdb/build_tools/precommit_checker.py | 208 +
.../build_tools/rocksdb-lego-determinator | 805 +
.../rocksdb/build_tools/update_dependencies.sh | 131 +
thirdparty/rocksdb/build_tools/version.sh | 22 +
thirdparty/rocksdb/cache/cache_bench.cc | 284 +
thirdparty/rocksdb/cache/clock_cache.cc | 729 +
thirdparty/rocksdb/cache/clock_cache.h | 16 +
thirdparty/rocksdb/cache/lru_cache.cc | 524 +
thirdparty/rocksdb/cache/lru_cache.h | 298 +
thirdparty/rocksdb/cache/sharded_cache.cc | 161 +
thirdparty/rocksdb/cache/sharded_cache.h | 102 +
.../rocksdb/coverage/parse_gcov_output.py | 118 +
thirdparty/rocksdb/db/builder.cc | 228 +
thirdparty/rocksdb/db/builder.h | 82 +
thirdparty/rocksdb/db/c.cc | 3549 +++
thirdparty/rocksdb/db/column_family.cc | 1137 +
thirdparty/rocksdb/db/column_family.h | 577 +
thirdparty/rocksdb/db/compacted_db_impl.cc | 166 +
thirdparty/rocksdb/db/compacted_db_impl.h | 102 +
thirdparty/rocksdb/db/compaction.cc | 480 +
thirdparty/rocksdb/db/compaction.h | 325 +
.../rocksdb/db/compaction_iteration_stats.h | 33 +
thirdparty/rocksdb/db/compaction_iterator.cc | 576 +
thirdparty/rocksdb/db/compaction_iterator.h | 197 +
thirdparty/rocksdb/db/compaction_job.cc | 1467 ++
thirdparty/rocksdb/db/compaction_job.h | 165 +
thirdparty/rocksdb/db/compaction_picker.cc | 1591 ++
thirdparty/rocksdb/db/compaction_picker.h | 298 +
.../rocksdb/db/compaction_picker_universal.cc | 747 +
.../rocksdb/db/compaction_picker_universal.h | 91 +
thirdparty/rocksdb/db/convenience.cc | 58 +
thirdparty/rocksdb/db/db_filesnapshot.cc | 149 +
thirdparty/rocksdb/db/db_impl.cc | 2824 +++
thirdparty/rocksdb/db/db_impl.h | 1284 +
.../rocksdb/db/db_impl_compaction_flush.cc | 1910 ++
thirdparty/rocksdb/db/db_impl_debug.cc | 200 +
thirdparty/rocksdb/db/db_impl_experimental.cc | 152 +
thirdparty/rocksdb/db/db_impl_files.cc | 548 +
thirdparty/rocksdb/db/db_impl_open.cc | 1129 +
thirdparty/rocksdb/db/db_impl_readonly.cc | 197 +
thirdparty/rocksdb/db/db_impl_readonly.h | 123 +
thirdparty/rocksdb/db/db_impl_write.cc | 1240 +
thirdparty/rocksdb/db/db_info_dumper.cc | 127 +
thirdparty/rocksdb/db/db_info_dumper.h | 14 +
thirdparty/rocksdb/db/db_iter.cc | 1256 +
thirdparty/rocksdb/db/db_iter.h | 100 +
thirdparty/rocksdb/db/db_test_util.cc | 1395 ++
thirdparty/rocksdb/db/db_test_util.h | 939 +
thirdparty/rocksdb/db/dbformat.cc | 177 +
thirdparty/rocksdb/db/dbformat.h | 596 +
thirdparty/rocksdb/db/event_helpers.cc | 155 +
thirdparty/rocksdb/db/event_helpers.h | 52 +
thirdparty/rocksdb/db/experimental.cc | 49 +
.../db/external_sst_file_ingestion_job.cc | 665 +
.../db/external_sst_file_ingestion_job.h | 171 +
thirdparty/rocksdb/db/file_indexer.cc | 214 +
thirdparty/rocksdb/db/file_indexer.h | 142 +
thirdparty/rocksdb/db/flush_job.cc | 359 +
thirdparty/rocksdb/db/flush_job.h | 110 +
thirdparty/rocksdb/db/flush_scheduler.cc | 88 +
thirdparty/rocksdb/db/flush_scheduler.h | 48 +
thirdparty/rocksdb/db/forward_iterator.cc | 905 +
thirdparty/rocksdb/db/forward_iterator.h | 153 +
thirdparty/rocksdb/db/forward_iterator_bench.cc | 375 +
thirdparty/rocksdb/db/internal_stats.cc | 1174 +
thirdparty/rocksdb/db/internal_stats.h | 588 +
thirdparty/rocksdb/db/job_context.h | 129 +
thirdparty/rocksdb/db/log_format.h | 45 +
thirdparty/rocksdb/db/log_reader.cc | 432 +
thirdparty/rocksdb/db/log_reader.h | 160 +
thirdparty/rocksdb/db/log_writer.cc | 142 +
thirdparty/rocksdb/db/log_writer.h | 111 +
thirdparty/rocksdb/db/malloc_stats.cc | 52 +
thirdparty/rocksdb/db/malloc_stats.h | 22 +
thirdparty/rocksdb/db/managed_iterator.cc | 262 +
thirdparty/rocksdb/db/managed_iterator.h | 85 +
thirdparty/rocksdb/db/memtable.cc | 885 +
thirdparty/rocksdb/db/memtable.h | 427 +
thirdparty/rocksdb/db/memtable_list.cc | 482 +
thirdparty/rocksdb/db/memtable_list.h | 258 +
thirdparty/rocksdb/db/merge_context.h | 116 +
thirdparty/rocksdb/db/merge_helper.cc | 365 +
thirdparty/rocksdb/db/merge_helper.h | 209 +
thirdparty/rocksdb/db/merge_operator.cc | 86 +
.../rocksdb/db/pinned_iterators_manager.h | 87 +
thirdparty/rocksdb/db/range_del_aggregator.cc | 519 +
thirdparty/rocksdb/db/range_del_aggregator.h | 161 +
thirdparty/rocksdb/db/repair.cc | 650 +
thirdparty/rocksdb/db/snapshot_impl.cc | 26 +
thirdparty/rocksdb/db/snapshot_impl.h | 130 +
thirdparty/rocksdb/db/table_cache.cc | 472 +
thirdparty/rocksdb/db/table_cache.h | 146 +
.../rocksdb/db/table_properties_collector.cc | 134 +
.../rocksdb/db/table_properties_collector.h | 137 +
thirdparty/rocksdb/db/transaction_log_impl.cc | 272 +
thirdparty/rocksdb/db/transaction_log_impl.h | 124 +
thirdparty/rocksdb/db/version_builder.cc | 412 +
thirdparty/rocksdb/db/version_builder.h | 44 +
thirdparty/rocksdb/db/version_edit.cc | 591 +
thirdparty/rocksdb/db/version_edit.h | 309 +
thirdparty/rocksdb/db/version_set.cc | 3801 +++
thirdparty/rocksdb/db/version_set.h | 862 +
thirdparty/rocksdb/db/wal_manager.cc | 476 +
thirdparty/rocksdb/db/wal_manager.h | 95 +
thirdparty/rocksdb/db/write_batch.cc | 1396 ++
thirdparty/rocksdb/db/write_batch_base.cc | 94 +
thirdparty/rocksdb/db/write_batch_internal.h | 227 +
thirdparty/rocksdb/db/write_callback.h | 27 +
thirdparty/rocksdb/db/write_controller.cc | 128 +
thirdparty/rocksdb/db/write_controller.h | 144 +
thirdparty/rocksdb/db/write_thread.cc | 656 +
thirdparty/rocksdb/db/write_thread.h | 391 +
thirdparty/rocksdb/docs/.gitignore | 9 +
thirdparty/rocksdb/docs/CNAME | 1 +
thirdparty/rocksdb/docs/CONTRIBUTING.md | 115 +
thirdparty/rocksdb/docs/Gemfile | 2 +
thirdparty/rocksdb/docs/Gemfile.lock | 145 +
thirdparty/rocksdb/docs/LICENSE-DOCUMENTATION | 385 +
thirdparty/rocksdb/docs/README.md | 80 +
thirdparty/rocksdb/docs/TEMPLATE-INFORMATION.md | 17 +
thirdparty/rocksdb/docs/_config.yml | 85 +
thirdparty/rocksdb/docs/_data/authors.yml | 62 +
thirdparty/rocksdb/docs/_data/features.yml | 19 +
thirdparty/rocksdb/docs/_data/nav.yml | 30 +
thirdparty/rocksdb/docs/_data/nav_docs.yml | 3 +
thirdparty/rocksdb/docs/_data/powered_by.yml | 1 +
.../rocksdb/docs/_data/powered_by_highlight.yml | 1 +
thirdparty/rocksdb/docs/_data/promo.yml | 6 +
thirdparty/rocksdb/docs/_docs/faq.md | 48 +
.../rocksdb/docs/_docs/getting-started.md | 78 +
.../rocksdb/docs/_includes/blog_pagination.html | 28 +
.../docs/_includes/content/gridblocks.html | 5 +
.../docs/_includes/content/items/gridblock.html | 37 +
thirdparty/rocksdb/docs/_includes/doc.html | 25 +
.../rocksdb/docs/_includes/doc_paging.html | 0
thirdparty/rocksdb/docs/_includes/footer.html | 33 +
thirdparty/rocksdb/docs/_includes/head.html | 23 +
thirdparty/rocksdb/docs/_includes/header.html | 19 +
thirdparty/rocksdb/docs/_includes/hero.html | 0
.../rocksdb/docs/_includes/home_header.html | 22 +
.../rocksdb/docs/_includes/katex_import.html | 3 +
.../rocksdb/docs/_includes/katex_render.html | 210 +
thirdparty/rocksdb/docs/_includes/nav.html | 37 +
.../docs/_includes/nav/collection_nav.html | 64 +
.../_includes/nav/collection_nav_group.html | 19 +
.../nav/collection_nav_group_item.html | 1 +
.../rocksdb/docs/_includes/nav/header_nav.html | 30 +
.../rocksdb/docs/_includes/nav_search.html | 15 +
.../docs/_includes/plugins/all_share.html | 3 +
.../docs/_includes/plugins/ascii_cinema.html | 2 +
.../rocksdb/docs/_includes/plugins/button.html | 6 +
.../docs/_includes/plugins/github_star.html | 4 +
.../docs/_includes/plugins/github_watch.html | 4 +
.../docs/_includes/plugins/google_share.html | 5 +
.../rocksdb/docs/_includes/plugins/iframe.html | 6 +
.../docs/_includes/plugins/like_button.html | 18 +
.../docs/_includes/plugins/plugin_row.html | 5 +
.../_includes/plugins/post_social_plugins.html | 34 +
.../docs/_includes/plugins/slideshow.html | 88 +
.../docs/_includes/plugins/twitter_follow.html | 5 +
.../docs/_includes/plugins/twitter_share.html | 4 +
thirdparty/rocksdb/docs/_includes/post.html | 35 +
.../rocksdb/docs/_includes/powered_by.html | 28 +
.../rocksdb/docs/_includes/social_plugins.html | 24 +
.../rocksdb/docs/_includes/ui/button.html | 1 +
thirdparty/rocksdb/docs/_layouts/basic.html | 12 +
thirdparty/rocksdb/docs/_layouts/blog.html | 11 +
.../rocksdb/docs/_layouts/blog_default.html | 14 +
thirdparty/rocksdb/docs/_layouts/default.html | 12 +
.../rocksdb/docs/_layouts/doc_default.html | 14 +
thirdparty/rocksdb/docs/_layouts/doc_page.html | 10 +
thirdparty/rocksdb/docs/_layouts/docs.html | 5 +
thirdparty/rocksdb/docs/_layouts/home.html | 17 +
thirdparty/rocksdb/docs/_layouts/page.html | 3 +
thirdparty/rocksdb/docs/_layouts/plain.html | 10 +
thirdparty/rocksdb/docs/_layouts/post.html | 8 +
thirdparty/rocksdb/docs/_layouts/redirect.html | 6 +
thirdparty/rocksdb/docs/_layouts/top-level.html | 10 +
.../2014-03-27-how-to-backup-rocksdb.markdown | 135 +
...-persist-in-memory-rocksdb-database.markdown | 54 +
...-local-meetup-held-on-march-27-2014.markdown | 53 +
.../2014-04-07-rocksdb-2-8-release.markdown | 40 +
...files-for-better-lookup-performance.markdown | 28 +
.../docs/_posts/2014-05-14-lock.markdown | 88 +
.../2014-05-19-rocksdb-3-0-release.markdown | 24 +
.../2014-05-22-rocksdb-3-1-release.markdown | 20 +
...-06-23-plaintable-a-new-file-format.markdown | 47 +
...-06-27-avoid-expensive-locks-in-get.markdown | 89 +
.../2014-06-27-rocksdb-3-2-release.markdown | 30 +
.../2014-07-29-rocksdb-3-3-release.markdown | 34 +
.../docs/_posts/2014-09-12-cuckoo.markdown | 74 +
.../2014-09-12-new-bloom-filter-format.markdown | 52 +
.../2014-09-15-rocksdb-3-5-release.markdown | 38 +
...migrating-from-leveldb-to-rocksdb-2.markdown | 112 +
...reading-rocksdb-options-from-a-file.markdown | 41 +
.../2015-02-27-write-batch-with-index.markdown | 20 +
...-integrating-rocksdb-with-mongodb-2.markdown | 16 +
.../2015-06-12-rocksdb-in-osquery.markdown | 10 +
.../2015-07-15-rocksdb-2015-h2-roadmap.markdown | 92 +
...5-07-17-spatial-indexing-in-rocksdb.markdown | 78 +
...s-now-available-in-windows-platform.markdown | 30 +
.../_posts/2015-07-23-dynamic-level.markdown | 29 +
.../_posts/2015-10-27-getthreadlist.markdown | 193 +
...checkpoints-for-efficient-snapshots.markdown | 45 +
...analysis-file-read-latency-by-level.markdown | 244 +
.../_posts/2016-01-29-compaction_pri.markdown | 51 +
.../2016-02-24-rocksdb-4-2-release.markdown | 41 +
.../docs/_posts/2016-02-25-rocksdb-ama.markdown | 20 +
.../2016-03-07-rocksdb-options-file.markdown | 24 +
.../2016-04-26-rocksdb-4-5-1-released.markdown | 60 +
.../2016-07-26-rocksdb-4-8-released.markdown | 48 +
.../2016-09-28-rocksdb-4-11-2-released.markdown | 49 +
.../2017-01-06-rocksdb-5-0-1-released.markdown | 26 +
.../2017-02-07-rocksdb-5-1-2-released.markdown | 15 +
.../2017-02-17-bulkoad-ingest-sst-file.markdown | 50 +
.../2017-03-02-rocksdb-5-2-1-released.markdown | 22 +
...2017-05-12-partitioned-index-filter.markdown | 34 +
.../_posts/2017-05-14-core-local-stats.markdown | 106 +
.../2017-05-26-rocksdb-5-4-5-released.markdown | 39 +
.../2017-06-26-17-level-based-changes.markdown | 60 +
.../2017-06-29-rocksdb-5-5-1-released.markdown | 22 +
.../2017-07-25-rocksdb-5-6-1-released.markdown | 22 +
thirdparty/rocksdb/docs/_sass/_base.scss | 492 +
thirdparty/rocksdb/docs/_sass/_blog.scss | 45 +
thirdparty/rocksdb/docs/_sass/_buttons.scss | 47 +
thirdparty/rocksdb/docs/_sass/_footer.scss | 82 +
thirdparty/rocksdb/docs/_sass/_gridBlock.scss | 115 +
thirdparty/rocksdb/docs/_sass/_header.scss | 138 +
thirdparty/rocksdb/docs/_sass/_poweredby.scss | 69 +
thirdparty/rocksdb/docs/_sass/_promo.scss | 55 +
.../rocksdb/docs/_sass/_react_docs_nav.scss | 332 +
.../rocksdb/docs/_sass/_react_header_nav.scss | 141 +
thirdparty/rocksdb/docs/_sass/_reset.scss | 43 +
thirdparty/rocksdb/docs/_sass/_search.scss | 142 +
thirdparty/rocksdb/docs/_sass/_slideshow.scss | 48 +
.../docs/_sass/_syntax-highlighting.scss | 129 +
thirdparty/rocksdb/docs/_sass/_tables.scss | 47 +
thirdparty/rocksdb/docs/_top-level/support.md | 22 +
thirdparty/rocksdb/docs/blog/all.html | 20 +
thirdparty/rocksdb/docs/blog/index.html | 12 +
thirdparty/rocksdb/docs/css/main.scss | 149 +
.../2016-04-07-blog-post-example.md | 21 +
.../docs/doc-type-examples/docs-hello-world.md | 12 +
.../docs/doc-type-examples/top-level-example.md | 8 +
thirdparty/rocksdb/docs/docs/index.html | 6 +
thirdparty/rocksdb/docs/feed.xml | 30 +
thirdparty/rocksdb/docs/index.md | 9 +
thirdparty/rocksdb/docs/static/favicon.png | Bin 0 -> 3927 bytes
.../docs/static/fonts/LatoLatin-Black.woff | Bin 0 -> 70460 bytes
.../docs/static/fonts/LatoLatin-Black.woff2 | Bin 0 -> 43456 bytes
.../static/fonts/LatoLatin-BlackItalic.woff | Bin 0 -> 72372 bytes
.../static/fonts/LatoLatin-BlackItalic.woff2 | Bin 0 -> 44316 bytes
.../docs/static/fonts/LatoLatin-Italic.woff | Bin 0 -> 74708 bytes
.../docs/static/fonts/LatoLatin-Italic.woff2 | Bin 0 -> 45388 bytes
.../docs/static/fonts/LatoLatin-Light.woff | Bin 0 -> 72604 bytes
.../docs/static/fonts/LatoLatin-Light.woff2 | Bin 0 -> 43468 bytes
.../docs/static/fonts/LatoLatin-Regular.woff | Bin 0 -> 72456 bytes
.../docs/static/fonts/LatoLatin-Regular.woff2 | Bin 0 -> 43760 bytes
.../Resize-of-20140327_200754-300x225.jpg | Bin 0 -> 26670 bytes
.../static/images/compaction/full-range.png | Bin 0 -> 193353 bytes
.../static/images/compaction/l0-l1-contend.png | Bin 0 -> 203828 bytes
.../static/images/compaction/l1-l2-contend.png | Bin 0 -> 230195 bytes
.../static/images/compaction/part-range-old.png | Bin 0 -> 165547 bytes
.../docs/static/images/pcache-blockindex.jpg | Bin 0 -> 55324 bytes
.../docs/static/images/pcache-fileindex.jpg | Bin 0 -> 54922 bytes
.../docs/static/images/pcache-filelayout.jpg | Bin 0 -> 47197 bytes
.../docs/static/images/pcache-readiopath.jpg | Bin 0 -> 16381 bytes
.../docs/static/images/pcache-tieredstorage.jpg | Bin 0 -> 78208 bytes
.../docs/static/images/pcache-writeiopath.jpg | Bin 0 -> 22616 bytes
.../rocksdb/docs/static/images/promo-adapt.svg | 8 +
.../rocksdb/docs/static/images/promo-flash.svg | 28 +
.../docs/static/images/promo-operations.svg | 6 +
.../docs/static/images/promo-performance.svg | 134 +
.../docs/static/images/tree_example1.png | Bin 0 -> 17804 bytes
thirdparty/rocksdb/docs/static/logo.svg | 76 +
thirdparty/rocksdb/docs/static/og_image.png | Bin 0 -> 17639 bytes
thirdparty/rocksdb/env/env.cc | 381 +
thirdparty/rocksdb/env/env_chroot.cc | 324 +
thirdparty/rocksdb/env/env_chroot.h | 22 +
thirdparty/rocksdb/env/env_encryption.cc | 909 +
thirdparty/rocksdb/env/env_hdfs.cc | 612 +
thirdparty/rocksdb/env/env_posix.cc | 976 +
thirdparty/rocksdb/env/io_posix.cc | 1028 +
thirdparty/rocksdb/env/io_posix.h | 248 +
thirdparty/rocksdb/env/mock_env.cc | 799 +
thirdparty/rocksdb/env/mock_env.h | 115 +
thirdparty/rocksdb/env/posix_logger.h | 169 +
thirdparty/rocksdb/examples/.gitignore | 8 +
thirdparty/rocksdb/examples/README.md | 2 +
thirdparty/rocksdb/examples/c_simple_example.c | 79 +
.../rocksdb/examples/column_families_example.cc | 72 +
.../rocksdb/examples/compact_files_example.cc | 171 +
.../examples/compaction_filter_example.cc | 87 +
.../examples/optimistic_transaction_example.cc | 142 +
.../rocksdb/examples/options_file_example.cc | 113 +
.../examples/rocksdb_option_file_example.ini | 143 +
thirdparty/rocksdb/examples/simple_example.cc | 56 +
.../rocksdb/examples/transaction_example.cc | 144 +
thirdparty/rocksdb/hdfs/README | 23 +
thirdparty/rocksdb/hdfs/env_hdfs.h | 374 +
thirdparty/rocksdb/hdfs/setup.sh | 7 +
.../rocksdb/include/rocksdb/advanced_options.h | 578 +
thirdparty/rocksdb/include/rocksdb/c.h | 1427 ++
thirdparty/rocksdb/include/rocksdb/cache.h | 200 +
thirdparty/rocksdb/include/rocksdb/cleanable.h | 73 +
.../rocksdb/include/rocksdb/compaction_filter.h | 209 +
.../include/rocksdb/compaction_job_stats.h | 91 +
thirdparty/rocksdb/include/rocksdb/comparator.h | 84 +
.../rocksdb/include/rocksdb/convenience.h | 339 +
thirdparty/rocksdb/include/rocksdb/db.h | 1155 +
.../rocksdb/include/rocksdb/db_bench_tool.h | 9 +
.../rocksdb/include/rocksdb/db_dump_tool.h | 45 +
thirdparty/rocksdb/include/rocksdb/env.h | 1137 +
.../rocksdb/include/rocksdb/env_encryption.h | 196 +
.../rocksdb/include/rocksdb/experimental.h | 29 +
.../rocksdb/include/rocksdb/filter_policy.h | 144 +
.../include/rocksdb/flush_block_policy.h | 62 +
.../rocksdb/include/rocksdb/iostats_context.h | 52 +
thirdparty/rocksdb/include/rocksdb/iterator.h | 116 +
thirdparty/rocksdb/include/rocksdb/ldb_tool.h | 43 +
thirdparty/rocksdb/include/rocksdb/listener.h | 390 +
.../rocksdb/include/rocksdb/memtablerep.h | 353 +
.../rocksdb/include/rocksdb/merge_operator.h | 225 +
thirdparty/rocksdb/include/rocksdb/metadata.h | 94 +
thirdparty/rocksdb/include/rocksdb/options.h | 1196 +
.../rocksdb/include/rocksdb/perf_context.h | 159 +
thirdparty/rocksdb/include/rocksdb/perf_level.h | 33 +
.../rocksdb/include/rocksdb/persistent_cache.h | 67 +
.../rocksdb/include/rocksdb/rate_limiter.h | 135 +
thirdparty/rocksdb/include/rocksdb/slice.h | 247 +
.../rocksdb/include/rocksdb/slice_transform.h | 99 +
thirdparty/rocksdb/include/rocksdb/snapshot.h | 48 +
.../rocksdb/include/rocksdb/sst_dump_tool.h | 17 +
.../rocksdb/include/rocksdb/sst_file_manager.h | 85 +
.../rocksdb/include/rocksdb/sst_file_writer.h | 115 +
thirdparty/rocksdb/include/rocksdb/statistics.h | 478 +
thirdparty/rocksdb/include/rocksdb/status.h | 316 +
thirdparty/rocksdb/include/rocksdb/table.h | 513 +
.../rocksdb/include/rocksdb/table_properties.h | 218 +
.../rocksdb/include/rocksdb/thread_status.h | 193 +
thirdparty/rocksdb/include/rocksdb/threadpool.h | 57 +
.../rocksdb/include/rocksdb/transaction_log.h | 125 +
thirdparty/rocksdb/include/rocksdb/types.h | 20 +
.../include/rocksdb/universal_compaction.h | 90 +
.../include/rocksdb/utilities/backupable_db.h | 328 +
.../include/rocksdb/utilities/checkpoint.h | 43 +
.../include/rocksdb/utilities/convenience.h | 10 +
.../include/rocksdb/utilities/date_tiered_db.h | 108 +
.../rocksdb/include/rocksdb/utilities/db_ttl.h | 68 +
.../rocksdb/include/rocksdb/utilities/debug.h | 39 +
.../include/rocksdb/utilities/document_db.h | 149 +
.../include/rocksdb/utilities/env_librados.h | 176 +
.../include/rocksdb/utilities/env_mirror.h | 176 +
.../rocksdb/include/rocksdb/utilities/geo_db.h | 114 +
.../include/rocksdb/utilities/info_log_finder.h | 19 +
.../include/rocksdb/utilities/json_document.h | 195 +
.../rocksdb/include/rocksdb/utilities/ldb_cmd.h | 260 +
.../rocksdb/utilities/ldb_cmd_execute_result.h | 75 +
.../include/rocksdb/utilities/leveldb_options.h | 144 +
.../utilities/lua/rocks_lua_compaction_filter.h | 189 +
.../utilities/lua/rocks_lua_custom_library.h | 43 +
.../rocksdb/utilities/lua/rocks_lua_util.h | 55 +
.../include/rocksdb/utilities/memory_util.h | 50 +
.../include/rocksdb/utilities/object_registry.h | 90 +
.../utilities/optimistic_transaction_db.h | 76 +
.../rocksdb/utilities/option_change_migration.h | 19 +
.../include/rocksdb/utilities/options_util.h | 93 +
.../include/rocksdb/utilities/sim_cache.h | 89 +
.../include/rocksdb/utilities/spatial_db.h | 261 +
.../include/rocksdb/utilities/stackable_db.h | 374 +
.../utilities/table_properties_collectors.h | 29 +
.../include/rocksdb/utilities/transaction.h | 445 +
.../include/rocksdb/utilities/transaction_db.h | 197 +
.../rocksdb/utilities/transaction_db_mutex.h | 92 +
.../include/rocksdb/utilities/utility_db.h | 35 +
.../rocksdb/utilities/write_batch_with_index.h | 225 +
thirdparty/rocksdb/include/rocksdb/version.h | 16 +
thirdparty/rocksdb/include/rocksdb/wal_filter.h | 101 +
.../rocksdb/include/rocksdb/write_batch.h | 344 +
.../rocksdb/include/rocksdb/write_batch_base.h | 125 +
.../include/rocksdb/write_buffer_manager.h | 100 +
thirdparty/rocksdb/java/CMakeLists.txt | 207 +
thirdparty/rocksdb/java/HISTORY-JAVA.md | 86 +
thirdparty/rocksdb/java/RELEASE.md | 54 +
.../java/org/rocksdb/benchmark/DbBenchmark.java | 1648 ++
thirdparty/rocksdb/java/crossbuild/Vagrantfile | 27 +
.../java/crossbuild/build-linux-centos.sh | 32 +
.../rocksdb/java/crossbuild/build-linux.sh | 14 +
.../crossbuild/docker-build-linux-centos.sh | 11 +
thirdparty/rocksdb/java/jdb_bench.sh | 10 +
thirdparty/rocksdb/java/rocksjni.pom | 150 +
.../rocksdb/java/rocksjni/backupablejni.cc | 313 +
.../rocksdb/java/rocksjni/backupenginejni.cc | 236 +
.../rocksjni/cassandra_compactionfilterjni.cc | 22 +
.../java/rocksjni/cassandra_value_operator.cc | 45 +
thirdparty/rocksdb/java/rocksjni/checkpoint.cc | 68 +
thirdparty/rocksdb/java/rocksjni/clock_cache.cc | 40 +
.../rocksdb/java/rocksjni/columnfamilyhandle.cc | 26 +
.../rocksdb/java/rocksjni/compaction_filter.cc | 27 +
.../java/rocksjni/compaction_options_fifo.cc | 55 +
.../rocksjni/compaction_options_universal.cc | 194 +
thirdparty/rocksdb/java/rocksjni/comparator.cc | 68 +
.../java/rocksjni/comparatorjnicallback.cc | 366 +
.../java/rocksjni/comparatorjnicallback.h | 94 +
.../java/rocksjni/compression_options.cc | 121 +
thirdparty/rocksdb/java/rocksjni/env.cc | 81 +
thirdparty/rocksdb/java/rocksjni/env_options.cc | 297 +
thirdparty/rocksdb/java/rocksjni/filter.cc | 43 +
.../rocksjni/ingest_external_file_options.cc | 149 +
thirdparty/rocksdb/java/rocksjni/iterator.cc | 158 +
.../rocksdb/java/rocksjni/loggerjnicallback.cc | 313 +
.../rocksdb/java/rocksjni/loggerjnicallback.h | 50 +
thirdparty/rocksdb/java/rocksjni/lru_cache.cc | 41 +
thirdparty/rocksdb/java/rocksjni/memtablejni.cc | 90 +
.../rocksdb/java/rocksjni/merge_operator.cc | 48 +
thirdparty/rocksdb/java/rocksjni/options.cc | 6101 +++++
thirdparty/rocksdb/java/rocksjni/portal.h | 3341 +++
.../rocksdb/java/rocksjni/ratelimiterjni.cc | 96 +
.../remove_emptyvalue_compactionfilterjni.cc | 24 +
thirdparty/rocksdb/java/rocksjni/restorejni.cc | 39 +
thirdparty/rocksdb/java/rocksjni/rocksjni.cc | 2198 ++
thirdparty/rocksdb/java/rocksjni/slice.cc | 349 +
thirdparty/rocksdb/java/rocksjni/snapshot.cc | 26 +
.../rocksdb/java/rocksjni/sst_file_writerjni.cc | 148 +
thirdparty/rocksdb/java/rocksjni/statistics.cc | 244 +
.../rocksdb/java/rocksjni/statisticsjni.cc | 33 +
.../rocksdb/java/rocksjni/statisticsjni.h | 33 +
thirdparty/rocksdb/java/rocksjni/table.cc | 92 +
.../rocksdb/java/rocksjni/transaction_log.cc | 71 +
thirdparty/rocksdb/java/rocksjni/ttl.cc | 183 +
thirdparty/rocksdb/java/rocksjni/write_batch.cc | 313 +
.../java/rocksjni/write_batch_with_index.cc | 582 +
.../rocksjni/writebatchhandlerjnicallback.cc | 306 +
.../rocksjni/writebatchhandlerjnicallback.h | 48 +
.../main/java/RocksDBColumnFamilySample.java | 78 +
.../samples/src/main/java/RocksDBSample.java | 303 +
.../org/rocksdb/AbstractCompactionFilter.java | 30 +
.../java/org/rocksdb/AbstractComparator.java | 106 +
.../AbstractImmutableNativeReference.java | 66 +
.../org/rocksdb/AbstractNativeReference.java | 76 +
.../java/org/rocksdb/AbstractRocksIterator.java | 101 +
.../main/java/org/rocksdb/AbstractSlice.java | 191 +
.../java/org/rocksdb/AbstractWriteBatch.java | 119 +
.../src/main/java/org/rocksdb/AccessHint.java | 53 +
.../AdvancedColumnFamilyOptionsInterface.java | 465 +
...ncedMutableColumnFamilyOptionsInterface.java | 437 +
.../src/main/java/org/rocksdb/BackupEngine.java | 221 +
.../src/main/java/org/rocksdb/BackupInfo.java | 66 +
.../java/org/rocksdb/BackupableDBOptions.java | 465 +
.../java/org/rocksdb/BlockBasedTableConfig.java | 452 +
.../src/main/java/org/rocksdb/BloomFilter.java | 79 +
.../java/org/rocksdb/BuiltinComparator.java | 20 +
.../java/src/main/java/org/rocksdb/Cache.java | 13 +
.../org/rocksdb/CassandraCompactionFilter.java | 18 +
.../rocksdb/CassandraValueMergeOperator.java | 20 +
.../src/main/java/org/rocksdb/Checkpoint.java | 66 +
.../src/main/java/org/rocksdb/ChecksumType.java | 39 +
.../src/main/java/org/rocksdb/ClockCache.java | 59 +
.../org/rocksdb/ColumnFamilyDescriptor.java | 61 +
.../java/org/rocksdb/ColumnFamilyHandle.java | 42 +
.../java/org/rocksdb/ColumnFamilyOptions.java | 909 +
.../rocksdb/ColumnFamilyOptionsInterface.java | 375 +
.../java/org/rocksdb/CompactionOptionsFIFO.java | 50 +
.../org/rocksdb/CompactionOptionsUniversal.java | 273 +
.../java/org/rocksdb/CompactionPriority.java | 73 +
.../java/org/rocksdb/CompactionStopStyle.java | 54 +
.../main/java/org/rocksdb/CompactionStyle.java | 52 +
.../src/main/java/org/rocksdb/Comparator.java | 32 +
.../java/org/rocksdb/ComparatorOptions.java | 51 +
.../java/org/rocksdb/CompressionOptions.java | 85 +
.../main/java/org/rocksdb/CompressionType.java | 99 +
.../src/main/java/org/rocksdb/DBOptions.java | 1120 +
.../java/org/rocksdb/DBOptionsInterface.java | 1549 ++
.../java/src/main/java/org/rocksdb/DbPath.java | 47 +
.../main/java/org/rocksdb/DirectComparator.java | 33 +
.../src/main/java/org/rocksdb/DirectSlice.java | 132 +
.../src/main/java/org/rocksdb/EncodingType.java | 55 +
.../java/src/main/java/org/rocksdb/Env.java | 92 +
.../src/main/java/org/rocksdb/EnvOptions.java | 207 +
.../src/main/java/org/rocksdb/Experimental.java | 23 +
.../java/src/main/java/org/rocksdb/Filter.java | 35 +
.../src/main/java/org/rocksdb/FlushOptions.java | 49 +
.../rocksdb/HashLinkedListMemTableConfig.java | 173 +
.../org/rocksdb/HashSkipListMemTableConfig.java | 105 +
.../main/java/org/rocksdb/HistogramData.java | 44 +
.../main/java/org/rocksdb/HistogramType.java | 98 +
.../src/main/java/org/rocksdb/IndexType.java | 41 +
.../src/main/java/org/rocksdb/InfoLogLevel.java | 48 +
.../org/rocksdb/IngestExternalFileOptions.java | 125 +
.../src/main/java/org/rocksdb/LRUCache.java | 82 +
.../java/src/main/java/org/rocksdb/Logger.java | 111 +
.../main/java/org/rocksdb/MemTableConfig.java | 29 +
.../main/java/org/rocksdb/MergeOperator.java | 17 +
.../org/rocksdb/MutableColumnFamilyOptions.java | 997 +
.../MutableColumnFamilyOptionsInterface.java | 159 +
.../java/org/rocksdb/NativeLibraryLoader.java | 124 +
.../java/src/main/java/org/rocksdb/Options.java | 1864 ++
.../main/java/org/rocksdb/PlainTableConfig.java | 251 +
.../src/main/java/org/rocksdb/RateLimiter.java | 119 +
.../src/main/java/org/rocksdb/ReadOptions.java | 397 +
.../src/main/java/org/rocksdb/ReadTier.java | 48 +
.../RemoveEmptyValueCompactionFilter.java | 18 +
.../main/java/org/rocksdb/RestoreOptions.java | 32 +
.../java/src/main/java/org/rocksdb/RocksDB.java | 2384 ++
.../main/java/org/rocksdb/RocksDBException.java | 44 +
.../src/main/java/org/rocksdb/RocksEnv.java | 43 +
.../main/java/org/rocksdb/RocksIterator.java | 64 +
.../org/rocksdb/RocksIteratorInterface.java | 80 +
.../src/main/java/org/rocksdb/RocksMemEnv.java | 27 +
.../java/org/rocksdb/RocksMutableObject.java | 87 +
.../src/main/java/org/rocksdb/RocksObject.java | 41 +
.../org/rocksdb/SkipListMemTableConfig.java | 50 +
.../java/src/main/java/org/rocksdb/Slice.java | 112 +
.../src/main/java/org/rocksdb/Snapshot.java | 37 +
.../main/java/org/rocksdb/SstFileWriter.java | 206 +
.../src/main/java/org/rocksdb/Statistics.java | 149 +
.../java/org/rocksdb/StatisticsCollector.java | 111 +
.../rocksdb/StatisticsCollectorCallback.java | 32 +
.../java/org/rocksdb/StatsCollectorInput.java | 35 +
.../src/main/java/org/rocksdb/StatsLevel.java | 65 +
.../java/src/main/java/org/rocksdb/Status.java | 113 +
.../java/org/rocksdb/StringAppendOperator.java | 19 +
.../java/org/rocksdb/TableFormatConfig.java | 22 +
.../src/main/java/org/rocksdb/TickerType.java | 480 +
.../org/rocksdb/TransactionLogIterator.java | 111 +
.../java/src/main/java/org/rocksdb/TtlDB.java | 211 +
.../java/org/rocksdb/VectorMemTableConfig.java | 45 +
.../main/java/org/rocksdb/WALRecoveryMode.java | 83 +
.../java/org/rocksdb/WBWIRocksIterator.java | 184 +
.../src/main/java/org/rocksdb/WriteBatch.java | 153 +
.../java/org/rocksdb/WriteBatchInterface.java | 146 +
.../java/org/rocksdb/WriteBatchWithIndex.java | 282 +
.../src/main/java/org/rocksdb/WriteOptions.java | 159 +
.../org/rocksdb/util/BytewiseComparator.java | 91 +
.../rocksdb/util/DirectBytewiseComparator.java | 88 +
.../main/java/org/rocksdb/util/Environment.java | 90 +
.../rocksdb/util/ReverseBytewiseComparator.java | 37 +
.../main/java/org/rocksdb/util/SizeUnit.java | 16 +
.../org/rocksdb/AbstractComparatorTest.java | 199 +
.../test/java/org/rocksdb/BackupEngineTest.java | 240 +
.../org/rocksdb/BackupableDBOptionsTest.java | 351 +
.../org/rocksdb/BlockBasedTableConfigTest.java | 171 +
.../test/java/org/rocksdb/CheckPointTest.java | 82 +
.../test/java/org/rocksdb/ClockCacheTest.java | 26 +
.../org/rocksdb/ColumnFamilyOptionsTest.java | 567 +
.../test/java/org/rocksdb/ColumnFamilyTest.java | 606 +
.../org/rocksdb/CompactionOptionsFIFOTest.java | 26 +
.../rocksdb/CompactionOptionsUniversalTest.java | 80 +
.../org/rocksdb/CompactionPriorityTest.java | 31 +
.../org/rocksdb/CompactionStopStyleTest.java | 31 +
.../java/org/rocksdb/ComparatorOptionsTest.java | 32 +
.../test/java/org/rocksdb/ComparatorTest.java | 200 +
.../org/rocksdb/CompressionOptionsTest.java | 53 +
.../java/org/rocksdb/CompressionTypesTest.java | 20 +
.../test/java/org/rocksdb/DBOptionsTest.java | 637 +
.../java/org/rocksdb/DirectComparatorTest.java | 52 +
.../test/java/org/rocksdb/DirectSliceTest.java | 93 +
.../test/java/org/rocksdb/EnvOptionsTest.java | 133 +
.../src/test/java/org/rocksdb/FilterTest.java | 39 +
.../src/test/java/org/rocksdb/FlushTest.java | 49 +
.../test/java/org/rocksdb/InfoLogLevelTest.java | 107 +
.../rocksdb/IngestExternalFileOptionsTest.java | 87 +
.../test/java/org/rocksdb/KeyMayExistTest.java | 87 +
.../src/test/java/org/rocksdb/LRUCacheTest.java | 27 +
.../src/test/java/org/rocksdb/LoggerTest.java | 238 +
.../src/test/java/org/rocksdb/MemTableTest.java | 111 +
.../src/test/java/org/rocksdb/MergeTest.java | 240 +
.../test/java/org/rocksdb/MixedOptionsTest.java | 55 +
.../rocksdb/MutableColumnFamilyOptionsTest.java | 88 +
.../org/rocksdb/NativeLibraryLoaderTest.java | 41 +
.../src/test/java/org/rocksdb/OptionsTest.java | 1095 +
.../java/org/rocksdb/PlainTableConfigTest.java | 89 +
.../java/org/rocksdb/PlatformRandomHelper.java | 58 +
.../test/java/org/rocksdb/RateLimiterTest.java | 49 +
.../src/test/java/org/rocksdb/ReadOnlyTest.java | 305 +
.../test/java/org/rocksdb/ReadOptionsTest.java | 201 +
.../java/org/rocksdb/RocksDBExceptionTest.java | 115 +
.../src/test/java/org/rocksdb/RocksDBTest.java | 766 +
.../src/test/java/org/rocksdb/RocksEnvTest.java | 39 +
.../java/org/rocksdb/RocksIteratorTest.java | 58 +
.../test/java/org/rocksdb/RocksMemEnvTest.java | 148 +
.../java/org/rocksdb/RocksMemoryResource.java | 24 +
.../src/test/java/org/rocksdb/SliceTest.java | 80 +
.../src/test/java/org/rocksdb/SnapshotTest.java | 169 +
.../java/org/rocksdb/SstFileWriterTest.java | 208 +
.../org/rocksdb/StatisticsCollectorTest.java | 55 +
.../test/java/org/rocksdb/StatisticsTest.java | 160 +
.../java/org/rocksdb/StatsCallbackMock.java | 20 +
.../org/rocksdb/TransactionLogIteratorTest.java | 138 +
.../src/test/java/org/rocksdb/TtlDBTest.java | 112 +
.../java/src/test/java/org/rocksdb/Types.java | 43 +
.../java/org/rocksdb/WALRecoveryModeTest.java | 22 +
.../java/org/rocksdb/WriteBatchHandlerTest.java | 169 +
.../test/java/org/rocksdb/WriteBatchTest.java | 296 +
.../org/rocksdb/WriteBatchThreadedTest.java | 104 +
.../org/rocksdb/WriteBatchWithIndexTest.java | 410 +
.../test/java/org/rocksdb/WriteOptionsTest.java | 45 +
.../java/org/rocksdb/test/RocksJunitRunner.java | 70 +
.../rocksdb/util/BytewiseComparatorTest.java | 480 +
.../java/org/rocksdb/util/EnvironmentTest.java | 172 +
.../java/org/rocksdb/util/SizeUnitTest.java | 27 +
thirdparty/rocksdb/memtable/alloc_tracker.cc | 59 +
thirdparty/rocksdb/memtable/hash_cuckoo_rep.cc | 660 +
thirdparty/rocksdb/memtable/hash_cuckoo_rep.h | 44 +
.../rocksdb/memtable/hash_linklist_rep.cc | 847 +
thirdparty/rocksdb/memtable/hash_linklist_rep.h | 49 +
.../rocksdb/memtable/hash_skiplist_rep.cc | 351 +
thirdparty/rocksdb/memtable/hash_skiplist_rep.h | 44 +
thirdparty/rocksdb/memtable/inlineskiplist.h | 899 +
.../rocksdb/memtable/memtablerep_bench.cc | 698 +
thirdparty/rocksdb/memtable/skiplist.h | 495 +
thirdparty/rocksdb/memtable/skiplistrep.cc | 277 +
thirdparty/rocksdb/memtable/stl_wrappers.h | 34 +
thirdparty/rocksdb/memtable/vectorrep.cc | 303 +
.../rocksdb/memtable/write_buffer_manager.cc | 124 +
.../rocksdb/monitoring/file_read_sample.h | 23 +
thirdparty/rocksdb/monitoring/histogram.cc | 302 +
thirdparty/rocksdb/monitoring/histogram.h | 149 +
.../rocksdb/monitoring/histogram_windowing.cc | 197 +
.../rocksdb/monitoring/histogram_windowing.h | 80 +
.../rocksdb/monitoring/instrumented_mutex.cc | 91 +
.../rocksdb/monitoring/instrumented_mutex.h | 98 +
.../rocksdb/monitoring/iostats_context.cc | 61 +
.../rocksdb/monitoring/iostats_context_imp.h | 54 +
thirdparty/rocksdb/monitoring/perf_context.cc | 179 +
.../rocksdb/monitoring/perf_context_imp.h | 52 +
thirdparty/rocksdb/monitoring/perf_level.cc | 28 +
thirdparty/rocksdb/monitoring/perf_level_imp.h | 18 +
thirdparty/rocksdb/monitoring/perf_step_timer.h | 54 +
thirdparty/rocksdb/monitoring/statistics.cc | 191 +
thirdparty/rocksdb/monitoring/statistics.h | 115 +
.../rocksdb/monitoring/thread_status_impl.cc | 167 +
.../rocksdb/monitoring/thread_status_updater.cc | 349 +
.../rocksdb/monitoring/thread_status_updater.h | 234 +
.../monitoring/thread_status_updater_debug.cc | 46 +
.../rocksdb/monitoring/thread_status_util.cc | 222 +
.../rocksdb/monitoring/thread_status_util.h | 134 +
.../monitoring/thread_status_util_debug.cc | 32 +
thirdparty/rocksdb/options/cf_options.cc | 179 +
thirdparty/rocksdb/options/cf_options.h | 239 +
thirdparty/rocksdb/options/db_options.cc | 275 +
thirdparty/rocksdb/options/db_options.h | 104 +
thirdparty/rocksdb/options/options.cc | 636 +
thirdparty/rocksdb/options/options_helper.cc | 1133 +
thirdparty/rocksdb/options/options_helper.h | 650 +
thirdparty/rocksdb/options/options_parser.cc | 792 +
thirdparty/rocksdb/options/options_parser.h | 146 +
.../rocksdb/options/options_sanity_check.cc | 38 +
.../rocksdb/options/options_sanity_check.h | 48 +
thirdparty/rocksdb/port/README | 10 +
thirdparty/rocksdb/port/dirent.h | 47 +
thirdparty/rocksdb/port/likely.h | 21 +
thirdparty/rocksdb/port/port.h | 21 +
thirdparty/rocksdb/port/port_example.h | 104 +
thirdparty/rocksdb/port/port_posix.cc | 205 +
thirdparty/rocksdb/port/port_posix.h | 210 +
thirdparty/rocksdb/port/stack_trace.cc | 138 +
thirdparty/rocksdb/port/stack_trace.h | 19 +
thirdparty/rocksdb/port/sys_time.h | 48 +
thirdparty/rocksdb/port/util_logger.h | 23 +
thirdparty/rocksdb/port/win/env_default.cc | 42 +
thirdparty/rocksdb/port/win/env_win.cc | 1127 +
thirdparty/rocksdb/port/win/env_win.h | 310 +
thirdparty/rocksdb/port/win/io_win.cc | 1029 +
thirdparty/rocksdb/port/win/io_win.h | 441 +
thirdparty/rocksdb/port/win/port_win.cc | 307 +
thirdparty/rocksdb/port/win/port_win.h | 329 +
thirdparty/rocksdb/port/win/win_logger.cc | 160 +
thirdparty/rocksdb/port/win/win_logger.h | 63 +
thirdparty/rocksdb/port/win/win_thread.cc | 166 +
thirdparty/rocksdb/port/win/win_thread.h | 121 +
thirdparty/rocksdb/port/win/xpress_win.cc | 267 +
thirdparty/rocksdb/port/win/xpress_win.h | 26 +
thirdparty/rocksdb/port/xpress.h | 17 +
thirdparty/rocksdb/src.mk | 399 +
.../rocksdb/table/adaptive_table_factory.cc | 123 +
.../rocksdb/table/adaptive_table_factory.h | 62 +
thirdparty/rocksdb/table/block.cc | 477 +
thirdparty/rocksdb/table/block.h | 384 +
.../rocksdb/table/block_based_filter_block.cc | 255 +
.../rocksdb/table/block_based_filter_block.h | 112 +
.../rocksdb/table/block_based_table_builder.cc | 866 +
.../rocksdb/table/block_based_table_builder.h | 128 +
.../rocksdb/table/block_based_table_factory.cc | 416 +
.../rocksdb/table/block_based_table_factory.h | 157 +
.../rocksdb/table/block_based_table_reader.cc | 2378 ++
.../rocksdb/table/block_based_table_reader.h | 479 +
thirdparty/rocksdb/table/block_builder.cc | 131 +
thirdparty/rocksdb/table/block_builder.h | 62 +
thirdparty/rocksdb/table/block_prefix_index.cc | 237 +
thirdparty/rocksdb/table/block_prefix_index.h | 68 +
thirdparty/rocksdb/table/bloom_block.cc | 23 +
thirdparty/rocksdb/table/bloom_block.h | 38 +
.../rocksdb/table/cuckoo_table_builder.cc | 515 +
thirdparty/rocksdb/table/cuckoo_table_builder.h | 127 +
.../rocksdb/table/cuckoo_table_factory.cc | 72 +
thirdparty/rocksdb/table/cuckoo_table_factory.h | 89 +
thirdparty/rocksdb/table/cuckoo_table_reader.cc | 385 +
thirdparty/rocksdb/table/cuckoo_table_reader.h | 86 +
thirdparty/rocksdb/table/filter_block.h | 132 +
thirdparty/rocksdb/table/flush_block_policy.cc | 80 +
thirdparty/rocksdb/table/format.cc | 593 +
thirdparty/rocksdb/table/format.h | 258 +
.../rocksdb/table/full_filter_bits_builder.h | 73 +
thirdparty/rocksdb/table/full_filter_block.cc | 113 +
thirdparty/rocksdb/table/full_filter_block.h | 117 +
thirdparty/rocksdb/table/get_context.cc | 205 +
thirdparty/rocksdb/table/get_context.h | 92 +
thirdparty/rocksdb/table/index_builder.cc | 187 +
thirdparty/rocksdb/table/index_builder.h | 342 +
thirdparty/rocksdb/table/internal_iterator.h | 121 +
thirdparty/rocksdb/table/iter_heap.h | 42 +
thirdparty/rocksdb/table/iterator.cc | 180 +
thirdparty/rocksdb/table/iterator_wrapper.h | 106 +
thirdparty/rocksdb/table/merging_iterator.cc | 404 +
thirdparty/rocksdb/table/merging_iterator.h | 62 +
thirdparty/rocksdb/table/meta_blocks.cc | 411 +
thirdparty/rocksdb/table/meta_blocks.h | 132 +
thirdparty/rocksdb/table/mock_table.cc | 142 +
thirdparty/rocksdb/table/mock_table.h | 194 +
.../rocksdb/table/partitioned_filter_block.cc | 247 +
.../rocksdb/table/partitioned_filter_block.h | 107 +
.../rocksdb/table/persistent_cache_helper.cc | 114 +
.../rocksdb/table/persistent_cache_helper.h | 44 +
.../rocksdb/table/persistent_cache_options.h | 34 +
thirdparty/rocksdb/table/plain_table_builder.cc | 296 +
thirdparty/rocksdb/table/plain_table_builder.h | 137 +
thirdparty/rocksdb/table/plain_table_factory.cc | 238 +
thirdparty/rocksdb/table/plain_table_factory.h | 210 +
thirdparty/rocksdb/table/plain_table_index.cc | 215 +
thirdparty/rocksdb/table/plain_table_index.h | 226 +
.../rocksdb/table/plain_table_key_coding.cc | 498 +
.../rocksdb/table/plain_table_key_coding.h | 183 +
thirdparty/rocksdb/table/plain_table_reader.cc | 747 +
thirdparty/rocksdb/table/plain_table_reader.h | 234 +
.../rocksdb/table/scoped_arena_iterator.h | 61 +
thirdparty/rocksdb/table/sst_file_writer.cc | 262 +
.../rocksdb/table/sst_file_writer_collectors.h | 84 +
thirdparty/rocksdb/table/table_builder.h | 128 +
thirdparty/rocksdb/table/table_properties.cc | 222 +
.../rocksdb/table/table_properties_internal.h | 30 +
thirdparty/rocksdb/table/table_reader.h | 109 +
thirdparty/rocksdb/table/table_reader_bench.cc | 341 +
thirdparty/rocksdb/table/two_level_iterator.cc | 267 +
thirdparty/rocksdb/table/two_level_iterator.h | 53 +
thirdparty/rocksdb/third-party/fbson/COMMIT.md | 5 +
.../rocksdb/third-party/fbson/FbsonDocument.h | 893 +
.../rocksdb/third-party/fbson/FbsonJsonParser.h | 741 +
.../rocksdb/third-party/fbson/FbsonStream.h | 182 +
.../rocksdb/third-party/fbson/FbsonUtil.h | 163 +
.../rocksdb/third-party/fbson/FbsonWriter.h | 430 +
.../gtest-1.7.0/fused-src/gtest/CMakeLists.txt | 1 +
.../gtest-1.7.0/fused-src/gtest/gtest-all.cc | 10261 ++++++++
.../gtest-1.7.0/fused-src/gtest/gtest.h | 20725 +++++++++++++++++
thirdparty/rocksdb/thirdparty.inc | 265 +
thirdparty/rocksdb/tools/CMakeLists.txt | 14 +
thirdparty/rocksdb/tools/Dockerfile | 5 +
thirdparty/rocksdb/tools/benchmark.sh | 511 +
thirdparty/rocksdb/tools/benchmark_leveldb.sh | 185 +
thirdparty/rocksdb/tools/blob_dump.cc | 89 +
.../rocksdb/tools/check_format_compatible.sh | 111 +
thirdparty/rocksdb/tools/db_bench.cc | 23 +
thirdparty/rocksdb/tools/db_bench_tool.cc | 5328 +++++
thirdparty/rocksdb/tools/db_repl_stress.cc | 158 +
thirdparty/rocksdb/tools/db_stress.cc | 2456 ++
thirdparty/rocksdb/tools/dbench_monitor | 102 +
thirdparty/rocksdb/tools/dump/db_dump_tool.cc | 261 +
thirdparty/rocksdb/tools/dump/rocksdb_dump.cc | 63 +
thirdparty/rocksdb/tools/dump/rocksdb_undump.cc | 62 +
thirdparty/rocksdb/tools/generate_random_db.sh | 30 +
thirdparty/rocksdb/tools/ldb.cc | 21 +
thirdparty/rocksdb/tools/ldb_cmd.cc | 2898 +++
thirdparty/rocksdb/tools/ldb_cmd_impl.h | 523 +
thirdparty/rocksdb/tools/ldb_tool.cc | 131 +
thirdparty/rocksdb/tools/pflag | 217 +
thirdparty/rocksdb/tools/rdb/.gitignore | 1 +
thirdparty/rocksdb/tools/rdb/API.md | 178 +
thirdparty/rocksdb/tools/rdb/README.md | 40 +
thirdparty/rocksdb/tools/rdb/binding.gyp | 25 +
thirdparty/rocksdb/tools/rdb/db_wrapper.cc | 525 +
thirdparty/rocksdb/tools/rdb/db_wrapper.h | 58 +
thirdparty/rocksdb/tools/rdb/rdb | 3 +
thirdparty/rocksdb/tools/rdb/rdb.cc | 15 +
thirdparty/rocksdb/tools/run_flash_bench.sh | 358 +
thirdparty/rocksdb/tools/run_leveldb.sh | 174 +
thirdparty/rocksdb/tools/sample-dump.dmp | Bin 0 -> 100 bytes
thirdparty/rocksdb/tools/sst_dump.cc | 21 +
thirdparty/rocksdb/tools/sst_dump_tool.cc | 689 +
thirdparty/rocksdb/tools/sst_dump_tool_imp.h | 83 +
thirdparty/rocksdb/tools/verify_random_db.sh | 33 +
thirdparty/rocksdb/tools/write_stress.cc | 307 +
thirdparty/rocksdb/tools/write_stress_runner.py | 73 +
thirdparty/rocksdb/util/aligned_buffer.h | 179 +
thirdparty/rocksdb/util/allocator.h | 57 +
thirdparty/rocksdb/util/arena.cc | 224 +
thirdparty/rocksdb/util/arena.h | 141 +
thirdparty/rocksdb/util/arena_test.cc | 203 +
thirdparty/rocksdb/util/auto_roll_logger.cc | 177 +
thirdparty/rocksdb/util/auto_roll_logger.h | 133 +
.../rocksdb/util/auto_roll_logger_test.cc | 490 +
thirdparty/rocksdb/util/autovector.h | 338 +
thirdparty/rocksdb/util/autovector_test.cc | 327 +
thirdparty/rocksdb/util/bloom.cc | 353 +
thirdparty/rocksdb/util/bloom_test.cc | 321 +
thirdparty/rocksdb/util/build_version.cc.in | 4 +
thirdparty/rocksdb/util/build_version.h | 15 +
thirdparty/rocksdb/util/cast_util.h | 21 +
thirdparty/rocksdb/util/channel.h | 67 +
thirdparty/rocksdb/util/coding.cc | 81 +
thirdparty/rocksdb/util/coding.h | 388 +
thirdparty/rocksdb/util/coding_test.cc | 204 +
.../rocksdb/util/compaction_job_stats_impl.cc | 86 +
thirdparty/rocksdb/util/comparator.cc | 130 +
thirdparty/rocksdb/util/compression.h | 791 +
thirdparty/rocksdb/util/concurrent_arena.cc | 39 +
thirdparty/rocksdb/util/concurrent_arena.h | 215 +
thirdparty/rocksdb/util/core_local.h | 83 +
thirdparty/rocksdb/util/crc32c.cc | 408 +
thirdparty/rocksdb/util/crc32c.h | 48 +
thirdparty/rocksdb/util/crc32c_test.cc | 78 +
thirdparty/rocksdb/util/delete_scheduler.cc | 234 +
thirdparty/rocksdb/util/delete_scheduler.h | 111 +
.../rocksdb/util/delete_scheduler_test.cc | 563 +
thirdparty/rocksdb/util/dynamic_bloom.cc | 82 +
thirdparty/rocksdb/util/dynamic_bloom.h | 191 +
thirdparty/rocksdb/util/dynamic_bloom_test.cc | 340 +
thirdparty/rocksdb/util/event_logger.cc | 67 +
thirdparty/rocksdb/util/event_logger.h | 196 +
thirdparty/rocksdb/util/event_logger_test.cc | 43 +
.../rocksdb/util/fault_injection_test_env.cc | 313 +
.../rocksdb/util/fault_injection_test_env.h | 159 +
thirdparty/rocksdb/util/file_reader_writer.cc | 649 +
thirdparty/rocksdb/util/file_reader_writer.h | 213 +
.../rocksdb/util/file_reader_writer_test.cc | 325 +
thirdparty/rocksdb/util/file_util.cc | 102 +
thirdparty/rocksdb/util/file_util.h | 27 +
thirdparty/rocksdb/util/filelock_test.cc | 59 +
thirdparty/rocksdb/util/filename.cc | 403 +
thirdparty/rocksdb/util/filename.h | 169 +
thirdparty/rocksdb/util/filter_policy.cc | 16 +
thirdparty/rocksdb/util/hash.cc | 56 +
thirdparty/rocksdb/util/hash.h | 35 +
thirdparty/rocksdb/util/hash_map.h | 67 +
thirdparty/rocksdb/util/hash_test.cc | 73 +
thirdparty/rocksdb/util/heap.h | 166 +
thirdparty/rocksdb/util/heap_test.cc | 139 +
thirdparty/rocksdb/util/kv_map.h | 34 +
thirdparty/rocksdb/util/log_buffer.cc | 93 +
thirdparty/rocksdb/util/log_buffer.h | 55 +
thirdparty/rocksdb/util/log_write_bench.cc | 84 +
thirdparty/rocksdb/util/logging.h | 50 +
thirdparty/rocksdb/util/memory_usage.h | 25 +
thirdparty/rocksdb/util/mpsc.h | 158 +
thirdparty/rocksdb/util/murmurhash.cc | 183 +
thirdparty/rocksdb/util/murmurhash.h | 42 +
thirdparty/rocksdb/util/mutexlock.h | 131 +
thirdparty/rocksdb/util/random.cc | 38 +
thirdparty/rocksdb/util/random.h | 109 +
thirdparty/rocksdb/util/rate_limiter.cc | 270 +
thirdparty/rocksdb/util/rate_limiter.h | 104 +
thirdparty/rocksdb/util/rate_limiter_test.cc | 180 +
thirdparty/rocksdb/util/slice.cc | 202 +
thirdparty/rocksdb/util/slice_transform_test.cc | 153 +
.../rocksdb/util/sst_file_manager_impl.cc | 183 +
thirdparty/rocksdb/util/sst_file_manager_impl.h | 105 +
thirdparty/rocksdb/util/status.cc | 106 +
thirdparty/rocksdb/util/status_message.cc | 21 +
thirdparty/rocksdb/util/stderr_logger.h | 31 +
thirdparty/rocksdb/util/stop_watch.h | 85 +
thirdparty/rocksdb/util/string_util.cc | 368 +
thirdparty/rocksdb/util/string_util.h | 128 +
thirdparty/rocksdb/util/sync_point.cc | 170 +
thirdparty/rocksdb/util/sync_point.h | 140 +
thirdparty/rocksdb/util/testharness.cc | 42 +
thirdparty/rocksdb/util/testharness.h | 40 +
thirdparty/rocksdb/util/testutil.cc | 401 +
thirdparty/rocksdb/util/testutil.h | 744 +
thirdparty/rocksdb/util/thread_list_test.cc | 352 +
thirdparty/rocksdb/util/thread_local.cc | 546 +
thirdparty/rocksdb/util/thread_local.h | 91 +
thirdparty/rocksdb/util/thread_local_test.cc | 582 +
thirdparty/rocksdb/util/thread_operation.h | 121 +
thirdparty/rocksdb/util/threadpool_imp.cc | 460 +
thirdparty/rocksdb/util/threadpool_imp.h | 109 +
thirdparty/rocksdb/util/timer_queue.h | 220 +
thirdparty/rocksdb/util/timer_queue_test.cc | 72 +
.../rocksdb/util/transaction_test_util.cc | 246 +
thirdparty/rocksdb/util/transaction_test_util.h | 114 +
thirdparty/rocksdb/util/xxhash.cc | 479 +
thirdparty/rocksdb/util/xxhash.h | 164 +
.../utilities/backupable/backupable_db.cc | 1789 ++
.../utilities/backupable/backupable_db_test.cc | 1548 ++
thirdparty/rocksdb/utilities/blob_db/blob_db.cc | 198 +
thirdparty/rocksdb/utilities/blob_db/blob_db.h | 215 +
.../rocksdb/utilities/blob_db/blob_db_impl.cc | 2302 ++
.../rocksdb/utilities/blob_db/blob_db_impl.h | 736 +
.../rocksdb/utilities/blob_db/blob_db_test.cc | 708 +
.../rocksdb/utilities/blob_db/blob_dump_tool.cc | 258 +
.../rocksdb/utilities/blob_db/blob_dump_tool.h | 52 +
.../rocksdb/utilities/blob_db/blob_file.cc | 229 +
.../utilities/blob_db/blob_log_format.cc | 318 +
.../rocksdb/utilities/blob_db/blob_log_format.h | 263 +
.../utilities/blob_db/blob_log_reader.cc | 168 +
.../rocksdb/utilities/blob_db/blob_log_reader.h | 95 +
.../utilities/blob_db/blob_log_writer.cc | 168 +
.../rocksdb/utilities/blob_db/blob_log_writer.h | 98 +
.../rocksdb/utilities/blob_db/ttl_extractor.cc | 34 +
.../cassandra/cassandra_compaction_filter.cc | 47 +
.../cassandra/cassandra_compaction_filter.h | 39 +
.../cassandra/cassandra_format_test.cc | 353 +
.../cassandra/cassandra_functional_test.cc | 251 +
.../cassandra/cassandra_row_merge_test.cc | 112 +
.../cassandra/cassandra_serialize_test.cc | 188 +
.../rocksdb/utilities/cassandra/format.cc | 362 +
thirdparty/rocksdb/utilities/cassandra/format.h | 194 +
.../utilities/cassandra/merge_operator.cc | 101 +
.../utilities/cassandra/merge_operator.h | 37 +
.../rocksdb/utilities/cassandra/serialize.h | 75 +
.../rocksdb/utilities/cassandra/test_utils.cc | 67 +
.../rocksdb/utilities/cassandra/test_utils.h | 43 +
.../utilities/checkpoint/checkpoint_impl.cc | 304 +
.../utilities/checkpoint/checkpoint_impl.h | 55 +
.../utilities/checkpoint/checkpoint_test.cc | 595 +
thirdparty/rocksdb/utilities/col_buf_decoder.cc | 240 +
thirdparty/rocksdb/utilities/col_buf_decoder.h | 116 +
thirdparty/rocksdb/utilities/col_buf_encoder.cc | 217 +
thirdparty/rocksdb/utilities/col_buf_encoder.h | 219 +
.../utilities/column_aware_encoding_exp.cc | 176 +
.../utilities/column_aware_encoding_test.cc | 254 +
.../utilities/column_aware_encoding_util.cc | 490 +
.../utilities/column_aware_encoding_util.h | 80 +
.../remove_emptyvalue_compactionfilter.cc | 30 +
.../remove_emptyvalue_compactionfilter.h | 27 +
.../utilities/convenience/info_log_finder.cc | 48 +
.../date_tiered/date_tiered_db_impl.cc | 396 +
.../utilities/date_tiered/date_tiered_db_impl.h | 89 +
.../utilities/date_tiered/date_tiered_test.cc | 468 +
thirdparty/rocksdb/utilities/debug.cc | 55 +
.../rocksdb/utilities/document/document_db.cc | 1193 +
.../utilities/document/document_db_test.cc | 336 +
.../rocksdb/utilities/document/json_document.cc | 609 +
.../utilities/document/json_document_builder.cc | 120 +
.../utilities/document/json_document_test.cc | 341 +
thirdparty/rocksdb/utilities/env_librados.cc | 1488 ++
thirdparty/rocksdb/utilities/env_librados.md | 122 +
.../rocksdb/utilities/env_librados_test.cc | 1146 +
thirdparty/rocksdb/utilities/env_mirror.cc | 259 +
thirdparty/rocksdb/utilities/env_mirror_test.cc | 222 +
thirdparty/rocksdb/utilities/env_timed.cc | 148 +
thirdparty/rocksdb/utilities/env_timed_test.cc | 44 +
.../rocksdb/utilities/geodb/geodb_impl.cc | 477 +
thirdparty/rocksdb/utilities/geodb/geodb_impl.h | 185 +
.../rocksdb/utilities/geodb/geodb_test.cc | 140 +
.../leveldb_options/leveldb_options.cc | 56 +
.../lua/rocks_lua_compaction_filter.cc | 242 +
.../rocksdb/utilities/lua/rocks_lua_test.cc | 498 +
.../rocksdb/utilities/memory/memory_test.cc | 276 +
.../rocksdb/utilities/memory/memory_util.cc | 52 +
thirdparty/rocksdb/utilities/merge_operators.h | 54 +
.../rocksdb/utilities/merge_operators/max.cc | 75 +
.../rocksdb/utilities/merge_operators/put.cc | 89 +
.../string_append/stringappend.cc | 57 +
.../string_append/stringappend.h | 31 +
.../string_append/stringappend2.cc | 117 +
.../string_append/stringappend2.h | 48 +
.../string_append/stringappend_test.cc | 600 +
.../utilities/merge_operators/uint64add.cc | 73 +
.../rocksdb/utilities/object_registry_test.cc | 72 +
.../option_change_migration.cc | 165 +
.../option_change_migration_test.cc | 425 +
.../rocksdb/utilities/options/options_util.cc | 102 +
.../utilities/options/options_util_test.cc | 317 +
.../persistent_cache/block_cache_tier.cc | 425 +
.../persistent_cache/block_cache_tier.h | 156 +
.../persistent_cache/block_cache_tier_file.cc | 593 +
.../persistent_cache/block_cache_tier_file.h | 293 +
.../block_cache_tier_file_buffer.h | 127 +
.../block_cache_tier_metadata.cc | 85 +
.../block_cache_tier_metadata.h | 125 +
.../utilities/persistent_cache/hash_table.h | 238 +
.../persistent_cache/hash_table_bench.cc | 303 +
.../persistent_cache/hash_table_evictable.h | 168 +
.../persistent_cache/hash_table_test.cc | 158 +
.../utilities/persistent_cache/lrulist.h | 174 +
.../persistent_cache/persistent_cache_bench.cc | 360 +
.../persistent_cache/persistent_cache_test.cc | 471 +
.../persistent_cache/persistent_cache_test.h | 285 +
.../persistent_cache/persistent_cache_tier.cc | 168 +
.../persistent_cache/persistent_cache_tier.h | 336 +
.../persistent_cache/persistent_cache_util.h | 67 +
.../persistent_cache/volatile_tier_impl.cc | 138 +
.../persistent_cache/volatile_tier_impl.h | 142 +
thirdparty/rocksdb/utilities/redis/README | 14 +
.../utilities/redis/redis_list_exception.h | 22 +
.../utilities/redis/redis_list_iterator.h | 309 +
.../rocksdb/utilities/redis/redis_lists.cc | 552 +
.../rocksdb/utilities/redis/redis_lists.h | 108 +
.../rocksdb/utilities/redis/redis_lists_test.cc | 894 +
.../utilities/simulator_cache/sim_cache.cc | 341 +
.../utilities/simulator_cache/sim_cache_test.cc | 218 +
.../rocksdb/utilities/spatialdb/spatial_db.cc | 919 +
.../utilities/spatialdb/spatial_db_test.cc | 307 +
thirdparty/rocksdb/utilities/spatialdb/utils.h | 95 +
.../compact_on_deletion_collector.cc | 94 +
.../compact_on_deletion_collector.h | 103 +
.../compact_on_deletion_collector_test.cc | 178 +
.../transactions/optimistic_transaction.cc | 134 +
.../transactions/optimistic_transaction.h | 98 +
.../optimistic_transaction_db_impl.cc | 91 +
.../optimistic_transaction_db_impl.h | 43 +
.../transactions/optimistic_transaction_test.cc | 1401 ++
.../transactions/pessimistic_transaction.cc | 596 +
.../transactions/pessimistic_transaction.h | 230 +
.../transactions/pessimistic_transaction_db.cc | 503 +
.../transactions/pessimistic_transaction_db.h | 167 +
.../utilities/transactions/transaction_base.cc | 672 +
.../utilities/transactions/transaction_base.h | 329 +
.../transactions/transaction_db_mutex_impl.cc | 135 +
.../transactions/transaction_db_mutex_impl.h | 26 +
.../transactions/transaction_lock_mgr.cc | 651 +
.../transactions/transaction_lock_mgr.h | 131 +
.../utilities/transactions/transaction_test.cc | 4578 ++++
.../utilities/transactions/transaction_util.cc | 162 +
.../utilities/transactions/transaction_util.h | 78 +
.../transactions/write_prepared_txn.cc | 63 +
.../utilities/transactions/write_prepared_txn.h | 72 +
thirdparty/rocksdb/utilities/ttl/db_ttl_impl.cc | 313 +
thirdparty/rocksdb/utilities/ttl/db_ttl_impl.h | 347 +
thirdparty/rocksdb/utilities/ttl/ttl_test.cc | 645 +
.../utilities/util_merge_operators_test.cc | 99 +
.../write_batch_with_index.cc | 842 +
.../write_batch_with_index_internal.cc | 269 +
.../write_batch_with_index_internal.h | 114 +
.../write_batch_with_index_test.cc | 1805 ++
1288 files changed, 298360 insertions(+), 30167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index b74f40d..13ada18 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,7 +30,6 @@ matrix:
- libboost-all-dev
- uuid-dev
- doxygen
- - libleveldb-dev
- openssl
- libcurl4-openssl-dev
before_install:
@@ -42,7 +41,7 @@ matrix:
# https://docs.travis-ci.com/user/osx-ci-environment/#Homebrew
before_install:
- brew update
- - package='leveldb'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
+ - package='rocksdb'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
- package='ossp-uuid'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
- package='boost'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
- package='cmake'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
@@ -54,7 +53,7 @@ matrix:
# https://docs.travis-ci.com/user/osx-ci-environment/#Homebrew
before_install:
- brew update
- - package='leveldb'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
+ - package='rocksdb'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
- package='ossp-uuid'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
- package='boost'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
- package='cmake'; [[ $(brew ls --versions ${package}) ]] && { brew outdated ${package} || brew upgrade ${package}; } || brew install ${package}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 81ff472..0732fa5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -17,13 +17,15 @@
# under the License.
#
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 3.0)
project(nifi-minifi-cpp)
set(PROJECT_NAME "nifi-minifi-cpp")
set(PROJECT_VERSION_MAJOR 0)
set(PROJECT_VERSION_MINOR 3)
set(PROJECT_VERSION_PATCH 0)
-option(test "Build all tests." ON)
+option(SKIP_TESTS "Skips building all tests." OFF)
+
+include(FeatureSummary)
#### Establish Project Configuration ####
# Enable usage of the VERSION specifier
@@ -50,10 +52,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
# Enable asan in DEBUG for compatibility with civet
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fsanitize=address")
-
# Search for threads
find_package(Threads REQUIRED)
-find_package(CURL)
# Set the right openssl root path
if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
@@ -73,18 +73,12 @@ endif (OPENSSL_FOUND)
# Provide custom modules for the project
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
-# Include LevelDB
-find_package (Leveldb REQUIRED)
-if (LEVELDB_FOUND)
- include_directories(${LEVELDB_INCLUDE_DIRS})
-else ()
- message( FATAL_ERROR "LevelDB was not found. Please install LevelDB" )
-endif (LEVELDB_FOUND)
find_package(UUID REQUIRED)
file(GLOB SPD_SOURCES "thirdparty/spdlog-20170710/include/spdlog/*")
+
include(ExternalProject)
# Setup Jsoncpp as an external project
@@ -109,10 +103,34 @@ add_subdirectory(thirdparty/yaml-cpp-yaml-cpp-0.5.3)
add_subdirectory(thirdparty/civetweb-1.9.1 EXCLUDE_FROM_ALL)
include_directories(thirdparty/concurrentqueue)
add_subdirectory(libminifi)
-if (CURL_FOUND AND NOT DISABLE_CURL)
+if (NOT DISABLE_CURL)
+ find_package(CURL)
add_subdirectory(extensions/http-curl)
mark_as_advanced(HTTP-CURL)
-endif(CURL_FOUND)
+
+endif()
+
+if (NOT DISABLE_ROCKSDB)
+ find_package(RocksDB)
+ if (NOT ROCKSDB_FOUND)
+ add_subdirectory(thirdparty/rocksdb/)
+ include_directories(thirdparty/rocksdb/include)
+ else()
+ include_directories(${ROCKSDB_INCLUDE_DIR})
+ endif()
+
+ add_subdirectory(extensions/rocksdb-repos)
+ mark_as_advanced(ROCKSDB-REPOS)
+
+
+endif()
+
+## ENABLE FEATURE INFORMATION
+ADD_FEATURE_INFO("HTTP CURL" HTTP-CURL "This enables RESTProtocol, InvokeHTTP, and the HTTPClient for Site to Site")
+ADD_FEATURE_INFO("ROCKSDB REPOS" ROCKSDB-REPOS "This Enables persistent provenance, flowfile, and content repositories using RocksDB")
+
+## NOW WE CAN ADD LIBRARIES AND EXTENSIONS TO MAIN
+
add_subdirectory(main)
@@ -151,7 +169,9 @@ set(CPACK_COMPONENTS_ALL bin)
include(CPack)
-include(BuildTests)
+if (NOT SKIP_TESTS)
+ include(BuildTests)
+endif()
include(BuildDocs)
@@ -168,3 +188,5 @@ add_custom_target(
apache-rat
${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/cmake/RunApacheRAT.cmake
COMMAND java -jar ${CMAKE_SOURCE_DIR}/thirdparty/apache-rat/apache-rat-0.12/apache-rat-0.12.jar -E ${CMAKE_SOURCE_DIR}/thirdparty/apache-rat/.rat-excludes -d ${CMAKE_SOURCE_DIR} | grep -B 1 -A 15 Summary )
+
+feature_summary(WHAT ALL FILENAME ${CMAKE_BINARY_DIR}/all.log)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index e76566c..0978892 100644
--- a/LICENSE
+++ b/LICENSE
@@ -322,36 +322,6 @@ This product bundles 'yaml-cpp' which is available under an MIT license.
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-This product bundles 'LevelDB' which is available under a 3-Clause BSD License.
-
-Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
This product bundles 'Formatting Library for C++' which is available under a 2-Clause BSD License.
Copyright (c) 2012 - 2016, Victor Zverovich
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index d3c8851..08e12b4 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,7 @@ Perspectives of the role of MiNiFi should be from the perspective of the agent a
* PutFile
* TailFile
* MergeContent
-* Provenance events generation is supported and are persisted using levelDB.
+* Provenance events generation is supported and are persisted using RocksDB.
## System Requirements
@@ -75,15 +75,19 @@ Perspectives of the role of MiNiFi should be from the perspective of the agent a
* libboost and boost-devel
* 1.48.0 or greater
* libcurl
-* libleveldb and libleveldb-devel
+* librocksdb4.1 and librocksdb-dev
* libuuid and uuid-dev
* openssl
+
+** NOTE: IF ROCKSDB IS NOT INSTALLED, IT WILL BE BUILT FROM THE THIRD PARTY
+DIRECTORY UNLESS YOU SPECIFY -DDISABLE_ROCKSDB=true WITH CMAKE ***
+
### To run
#### Libraries
* libuuid
-* libleveldb
+* librocksdb *** IF NOT INSTALLED, WILL BE BUILT FROM THIRD PARTY DIRECTORY ***
* libcurl
* libssl and libcrypto from openssl
@@ -95,7 +99,7 @@ Yum based Linux Distributions
$ yum install cmake \
gcc gcc-c++ \
libcurl-devel \
- leveldb-devel leveldb \
+ rocksdb-dev rocksdb \
libuuid libuuid-devel \
boost-devel \
libssl-dev \
@@ -113,7 +117,7 @@ Aptitude based Linux Distributions
$ apt-get install cmake \
gcc g++ \
libcurl-dev \
- libleveldb-dev libleveldb1v5 \
+ librocksdb-dev librocksdb4.1 \
uuid-dev uuid \
libboost-all-dev libssl-dev \
doxygen
@@ -127,7 +131,7 @@ OS X Using Homebrew (with XCode Command Line Tools installed)
```
# ~/Development/code/apache/nifi-minifi-cpp on git:master
$ brew install cmake \
- leveldb \
+ rocksdb \
ossp-uuid \
boost \
openssl \
@@ -345,7 +349,7 @@ https://cwiki.apache.org/confluence/display/MINIFI/C2+Design+Proposal
### Configuring Volatile and NO-OP Repositories
Each of the repositories can be configured to be volatile ( state kept in memory and flushed
upon restart ) or persistent. Currently, the flow file and provenance repositories can persist
- to LevelDB. The content repository will persist to the local file system if a volatile repo
+ to RocksDB. The content repository will persist to the local file system if a volatile repo
is not configured.
To configure the repositories:
[36/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/testharness.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/testharness.cc b/thirdparty/leveldb-1.18/util/testharness.cc
deleted file mode 100755
index 402fab3..0000000
--- a/thirdparty/leveldb-1.18/util/testharness.cc
+++ /dev/null
@@ -1,77 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "util/testharness.h"
-
-#include <string>
-#include <stdlib.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-
-namespace leveldb {
-namespace test {
-
-namespace {
-struct Test {
- const char* base;
- const char* name;
- void (*func)();
-};
-std::vector<Test>* tests;
-}
-
-bool RegisterTest(const char* base, const char* name, void (*func)()) {
- if (tests == NULL) {
- tests = new std::vector<Test>;
- }
- Test t;
- t.base = base;
- t.name = name;
- t.func = func;
- tests->push_back(t);
- return true;
-}
-
-int RunAllTests() {
- const char* matcher = getenv("LEVELDB_TESTS");
-
- int num = 0;
- if (tests != NULL) {
- for (size_t i = 0; i < tests->size(); i++) {
- const Test& t = (*tests)[i];
- if (matcher != NULL) {
- std::string name = t.base;
- name.push_back('.');
- name.append(t.name);
- if (strstr(name.c_str(), matcher) == NULL) {
- continue;
- }
- }
- fprintf(stderr, "==== Test %s.%s\n", t.base, t.name);
- (*t.func)();
- ++num;
- }
- }
- fprintf(stderr, "==== PASSED %d tests\n", num);
- return 0;
-}
-
-std::string TmpDir() {
- std::string dir;
- Status s = Env::Default()->GetTestDirectory(&dir);
- ASSERT_TRUE(s.ok()) << s.ToString();
- return dir;
-}
-
-int RandomSeed() {
- const char* env = getenv("TEST_RANDOM_SEED");
- int result = (env != NULL ? atoi(env) : 301);
- if (result <= 0) {
- result = 301;
- }
- return result;
-}
-
-} // namespace test
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/testharness.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/testharness.h b/thirdparty/leveldb-1.18/util/testharness.h
deleted file mode 100755
index da4fe68..0000000
--- a/thirdparty/leveldb-1.18/util/testharness.h
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_UTIL_TESTHARNESS_H_
-#define STORAGE_LEVELDB_UTIL_TESTHARNESS_H_
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <sstream>
-#include "leveldb/env.h"
-#include "leveldb/slice.h"
-#include "util/random.h"
-
-namespace leveldb {
-namespace test {
-
-// Run some of the tests registered by the TEST() macro. If the
-// environment variable "LEVELDB_TESTS" is not set, runs all tests.
-// Otherwise, runs only the tests whose name contains the value of
-// "LEVELDB_TESTS" as a substring. E.g., suppose the tests are:
-// TEST(Foo, Hello) { ... }
-// TEST(Foo, World) { ... }
-// LEVELDB_TESTS=Hello will run the first test
-// LEVELDB_TESTS=o will run both tests
-// LEVELDB_TESTS=Junk will run no tests
-//
-// Returns 0 if all tests pass.
-// Dies or returns a non-zero value if some test fails.
-extern int RunAllTests();
-
-// Return the directory to use for temporary storage.
-extern std::string TmpDir();
-
-// Return a randomization seed for this run. Typically returns the
-// same number on repeated invocations of this binary, but automated
-// runs may be able to vary the seed.
-extern int RandomSeed();
-
-// An instance of Tester is allocated to hold temporary state during
-// the execution of an assertion.
-class Tester {
- private:
- bool ok_;
- const char* fname_;
- int line_;
- std::stringstream ss_;
-
- public:
- Tester(const char* f, int l)
- : ok_(true), fname_(f), line_(l) {
- }
-
- ~Tester() {
- if (!ok_) {
- fprintf(stderr, "%s:%d:%s\n", fname_, line_, ss_.str().c_str());
- exit(1);
- }
- }
-
- Tester& Is(bool b, const char* msg) {
- if (!b) {
- ss_ << " Assertion failure " << msg;
- ok_ = false;
- }
- return *this;
- }
-
- Tester& IsOk(const Status& s) {
- if (!s.ok()) {
- ss_ << " " << s.ToString();
- ok_ = false;
- }
- return *this;
- }
-
-#define BINARY_OP(name,op) \
- template <class X, class Y> \
- Tester& name(const X& x, const Y& y) { \
- if (! (x op y)) { \
- ss_ << " failed: " << x << (" " #op " ") << y; \
- ok_ = false; \
- } \
- return *this; \
- }
-
- BINARY_OP(IsEq, ==)
- BINARY_OP(IsNe, !=)
- BINARY_OP(IsGe, >=)
- BINARY_OP(IsGt, >)
- BINARY_OP(IsLe, <=)
- BINARY_OP(IsLt, <)
-#undef BINARY_OP
-
- // Attach the specified value to the error message if an error has occurred
- template <class V>
- Tester& operator<<(const V& value) {
- if (!ok_) {
- ss_ << " " << value;
- }
- return *this;
- }
-};
-
-#define ASSERT_TRUE(c) ::leveldb::test::Tester(__FILE__, __LINE__).Is((c), #c)
-#define ASSERT_OK(s) ::leveldb::test::Tester(__FILE__, __LINE__).IsOk((s))
-#define ASSERT_EQ(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsEq((a),(b))
-#define ASSERT_NE(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsNe((a),(b))
-#define ASSERT_GE(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsGe((a),(b))
-#define ASSERT_GT(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsGt((a),(b))
-#define ASSERT_LE(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsLe((a),(b))
-#define ASSERT_LT(a,b) ::leveldb::test::Tester(__FILE__, __LINE__).IsLt((a),(b))
-
-#define TCONCAT(a,b) TCONCAT1(a,b)
-#define TCONCAT1(a,b) a##b
-
-#define TEST(base,name) \
-class TCONCAT(_Test_,name) : public base { \
- public: \
- void _Run(); \
- static void _RunIt() { \
- TCONCAT(_Test_,name) t; \
- t._Run(); \
- } \
-}; \
-bool TCONCAT(_Test_ignored_,name) = \
- ::leveldb::test::RegisterTest(#base, #name, &TCONCAT(_Test_,name)::_RunIt); \
-void TCONCAT(_Test_,name)::_Run()
-
-// Register the specified test. Typically not used directly, but
-// invoked via the macro expansion of TEST.
-extern bool RegisterTest(const char* base, const char* name, void (*func)());
-
-
-} // namespace test
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_UTIL_TESTHARNESS_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/testutil.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/testutil.cc b/thirdparty/leveldb-1.18/util/testutil.cc
deleted file mode 100755
index bee56bf..0000000
--- a/thirdparty/leveldb-1.18/util/testutil.cc
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "util/testutil.h"
-
-#include "util/random.h"
-
-namespace leveldb {
-namespace test {
-
-Slice RandomString(Random* rnd, int len, std::string* dst) {
- dst->resize(len);
- for (int i = 0; i < len; i++) {
- (*dst)[i] = static_cast<char>(' ' + rnd->Uniform(95)); // ' ' .. '~'
- }
- return Slice(*dst);
-}
-
-std::string RandomKey(Random* rnd, int len) {
- // Make sure to generate a wide variety of characters so we
- // test the boundary conditions for short-key optimizations.
- static const char kTestChars[] = {
- '\0', '\1', 'a', 'b', 'c', 'd', 'e', '\xfd', '\xfe', '\xff'
- };
- std::string result;
- for (int i = 0; i < len; i++) {
- result += kTestChars[rnd->Uniform(sizeof(kTestChars))];
- }
- return result;
-}
-
-
-extern Slice CompressibleString(Random* rnd, double compressed_fraction,
- size_t len, std::string* dst) {
- int raw = static_cast<int>(len * compressed_fraction);
- if (raw < 1) raw = 1;
- std::string raw_data;
- RandomString(rnd, raw, &raw_data);
-
- // Duplicate the random data until we have filled "len" bytes
- dst->clear();
- while (dst->size() < len) {
- dst->append(raw_data);
- }
- dst->resize(len);
- return Slice(*dst);
-}
-
-} // namespace test
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/testutil.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/testutil.h b/thirdparty/leveldb-1.18/util/testutil.h
deleted file mode 100755
index adad3fc..0000000
--- a/thirdparty/leveldb-1.18/util/testutil.h
+++ /dev/null
@@ -1,53 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_UTIL_TESTUTIL_H_
-#define STORAGE_LEVELDB_UTIL_TESTUTIL_H_
-
-#include "leveldb/env.h"
-#include "leveldb/slice.h"
-#include "util/random.h"
-
-namespace leveldb {
-namespace test {
-
-// Store in *dst a random string of length "len" and return a Slice that
-// references the generated data.
-extern Slice RandomString(Random* rnd, int len, std::string* dst);
-
-// Return a random key with the specified length that may contain interesting
-// characters (e.g. \x00, \xff, etc.).
-extern std::string RandomKey(Random* rnd, int len);
-
-// Store in *dst a string of length "len" that will compress to
-// "N*compressed_fraction" bytes and return a Slice that references
-// the generated data.
-extern Slice CompressibleString(Random* rnd, double compressed_fraction,
- size_t len, std::string* dst);
-
-// A wrapper that allows injection of errors.
-class ErrorEnv : public EnvWrapper {
- public:
- bool writable_file_error_;
- int num_writable_file_errors_;
-
- ErrorEnv() : EnvWrapper(Env::Default()),
- writable_file_error_(false),
- num_writable_file_errors_(0) { }
-
- virtual Status NewWritableFile(const std::string& fname,
- WritableFile** result) {
- if (writable_file_error_) {
- ++num_writable_file_errors_;
- *result = NULL;
- return Status::IOError(fname, "fake error");
- }
- return target()->NewWritableFile(fname, result);
- }
-};
-
-} // namespace test
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_UTIL_TESTUTIL_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/win_logger.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/win_logger.cc b/thirdparty/leveldb-1.18/util/win_logger.cc
deleted file mode 100755
index 47ffd8e..0000000
--- a/thirdparty/leveldb-1.18/util/win_logger.cc
+++ /dev/null
@@ -1,79 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "util/win_logger.h"
-
-#include <windows.h>
-
-namespace leveldb {
-
-void WinLogger::Logv(const char* format, va_list ap) {
- const uint64_t thread_id = static_cast<uint64_t>(::GetCurrentThreadId());
-
- // We try twice: the first time with a fixed-size stack allocated buffer,
- // and the second time with a much larger dynamically allocated buffer.
- char buffer[500];
-
- for (int iter = 0; iter < 2; iter++) {
- char* base;
- int bufsize;
- if (iter == 0) {
- bufsize = sizeof(buffer);
- base = buffer;
- } else {
- bufsize = 30000;
- base = new char[bufsize];
- }
-
- char* p = base;
- char* limit = base + bufsize;
-
- SYSTEMTIME st;
-
- // GetSystemTime returns UTC time, we want local time!
- ::GetLocalTime(&st);
-
- p += _snprintf_s(p, limit - p, _TRUNCATE,
- "%04d/%02d/%02d-%02d:%02d:%02d.%03d %llx ",
- st.wYear,
- st.wMonth,
- st.wDay,
- st.wHour,
- st.wMinute,
- st.wSecond,
- st.wMilliseconds,
- static_cast<long long unsigned int>(thread_id));
-
- // Print the message
- if (p < limit) {
- va_list backup_ap = ap;
- p += vsnprintf(p, limit - p, format, backup_ap);
- va_end(backup_ap);
- }
-
- // Truncate to available space if necessary
- if (p >= limit) {
- if (iter == 0) {
- continue; // Try again with larger buffer
- } else {
- p = limit - 1;
- }
- }
-
- // Add newline if necessary
- if (p == base || p[-1] != '\n') {
- *p++ = '\n';
- }
-
- assert(p <= limit);
- fwrite(base, 1, p - base, file_);
- fflush(file_);
- if (base != buffer) {
- delete[] base;
- }
- break;
- }
-}
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/win_logger.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/win_logger.h b/thirdparty/leveldb-1.18/util/win_logger.h
deleted file mode 100755
index b155d5c..0000000
--- a/thirdparty/leveldb-1.18/util/win_logger.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-// Logger implementation for Windows
-
-#ifndef STORAGE_LEVELDB_UTIL_WIN_LOGGER_H_
-#define STORAGE_LEVELDB_UTIL_WIN_LOGGER_H_
-
-#include <stdio.h>
-#include "leveldb/env.h"
-
-namespace leveldb {
-
-class WinLogger : public Logger {
- private:
- FILE* file_;
- public:
- explicit WinLogger(FILE* f) : file_(f) { assert(file_); }
- virtual ~WinLogger() {
- fclose(file_);
- }
- virtual void Logv(const char* format, va_list ap);
-
-};
-
-}
-#endif // STORAGE_LEVELDB_UTIL_WIN_LOGGER_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/.clang-format
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/.clang-format b/thirdparty/rocksdb/.clang-format
new file mode 100644
index 0000000..7c27981
--- /dev/null
+++ b/thirdparty/rocksdb/.clang-format
@@ -0,0 +1,5 @@
+# Complete list of style options can be found at:
+# http://clang.llvm.org/docs/ClangFormatStyleOptions.html
+---
+BasedOnStyle: Google
+...
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/.gitignore
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/.gitignore b/thirdparty/rocksdb/.gitignore
new file mode 100644
index 0000000..87d5b98
--- /dev/null
+++ b/thirdparty/rocksdb/.gitignore
@@ -0,0 +1,75 @@
+make_config.mk
+
+*.a
+*.arc
+*.d
+*.dylib*
+*.gcda
+*.gcno
+*.o
+*.so
+*.so.*
+*_test
+*_bench
+*_stress
+*.out
+*.class
+*.jar
+*.*jnilib*
+*.d-e
+*.o-*
+*.swp
+*~
+*.vcxproj
+*.vcxproj.filters
+*.sln
+*.cmake
+CMakeCache.txt
+CMakeFiles/
+build/
+
+ldb
+manifest_dump
+sst_dump
+blob_dump
+column_aware_encoding_exp
+util/build_version.cc
+build_tools/VALGRIND_LOGS/
+coverage/COVERAGE_REPORT
+.gdbhistory
+.gdb_history
+package/
+.phutil_module_cache
+unity.a
+tags
+etags
+rocksdb_dump
+rocksdb_undump
+db_test2
+
+java/out
+java/target
+java/test-libs
+java/*.log
+java/include/org_rocksdb_*.h
+
+.idea/
+*.iml
+
+rocksdb.cc
+rocksdb.h
+unity.cc
+java/crossbuild/.vagrant
+.vagrant/
+java/**/*.asc
+java/javadoc
+
+scan_build_report/
+t
+LOG
+
+db_logs/
+tp2/
+fbcode/
+fbcode
+buckifier/*.pyc
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/.travis.yml
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/.travis.yml b/thirdparty/rocksdb/.travis.yml
new file mode 100644
index 0000000..78e5191
--- /dev/null
+++ b/thirdparty/rocksdb/.travis.yml
@@ -0,0 +1,72 @@
+sudo: false
+dist: trusty
+language: cpp
+os:
+ - linux
+ - osx
+compiler:
+ - clang
+ - gcc
+osx_image: xcode8.3
+jdk:
+ - oraclejdk7
+cache:
+ - ccache
+ - apt
+
+addons:
+ apt:
+ packages: ['zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev', 'mingw-w64']
+env:
+ - TEST_GROUP=platform_dependent # 16-18 minutes
+ - TEST_GROUP=1 # 33-35 minutes
+ - TEST_GROUP=2 # 30-32 minutes
+ # Run java tests
+ - JOB_NAME=java_test # 4-11 minutes
+ # Build ROCKSDB_LITE
+ - JOB_NAME=lite_build # 3-4 minutes
+ # Build examples
+ - JOB_NAME=examples # 5-7 minutes
+ - JOB_NAME=cmake # 3-5 minutes
+ - JOB_NAME=cmake-mingw # 3 minutes
+
+matrix:
+ exclude:
+ - os: osx
+ env: TEST_GROUP=1
+ - os: osx
+ env: TEST_GROUP=2
+ - os : osx
+ env: JOB_NAME=cmake-mingw
+ - os : linux
+ compiler: clang
+ - os : osx
+ compiler: gcc
+
+# https://docs.travis-ci.com/user/caching/#ccache-cache
+install:
+ - if [ "${TRAVIS_OS_NAME}" == osx ]; then
+ brew install ccache;
+ PATH=$PATH:/usr/local/opt/ccache/libexec;
+ fi
+
+before_script:
+ # Increase the maximum number of open file descriptors, since some tests use
+ # more FDs than the default limit.
+ - ulimit -n 8192
+
+script:
+ - ${CXX} --version
+ - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then ccache -C && OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some; fi
+ - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
+ - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
+ - if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
+ - if [ "${JOB_NAME}" == 'lite_build' ]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib; fi
+ - if [ "${JOB_NAME}" == 'examples' ]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
+ - if [ "${JOB_NAME}" == 'cmake' ]; then mkdir build && cd build && cmake .. && make -j4 rocksdb; fi
+ - if [ "${JOB_NAME}" == 'cmake-mingw' ]; then mkdir build && cd build && cmake .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb; fi
+notifications:
+ email:
+ - leveldb@fb.com
+ webhooks:
+ - https://buildtimetrend.herokuapp.com/travis
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/AUTHORS
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/AUTHORS b/thirdparty/rocksdb/AUTHORS
new file mode 100644
index 0000000..e644f55
--- /dev/null
+++ b/thirdparty/rocksdb/AUTHORS
@@ -0,0 +1,11 @@
+Facebook Inc.
+Facebook Engineering Team
+
+Google Inc.
+# Initial version authors:
+Jeffrey Dean <je...@google.com>
+Sanjay Ghemawat <sa...@google.com>
+
+# Partial list of contributors:
+Kevin Regan <ke...@gmail.com>
+Johan Bilien <jo...@litl.com>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/CMakeLists.txt b/thirdparty/rocksdb/CMakeLists.txt
new file mode 100644
index 0000000..496df64
--- /dev/null
+++ b/thirdparty/rocksdb/CMakeLists.txt
@@ -0,0 +1,664 @@
+# Prerequisites for Windows:
+# This cmake build is for Windows 64-bit only.
+#
+# Prerequisites:
+# You must have at least Visual Studio 2015 Update 3. Start the Developer Command Prompt window that is a part of Visual Studio installation.
+# Run the build commands from within the Developer Command Prompt window to have paths to the compiler and runtime libraries set.
+# You must have git.exe in your %PATH% environment variable.
+#
+# To build Rocksdb for Windows is as easy as 1-2-3-4-5:
+#
+# 1. Update paths to third-party libraries in thirdparty.inc file
+# 2. Create a new directory for build artifacts
+# mkdir build
+# cd build
+# 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
+# See thirdparty.inc for more information.
+# sample command: cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
+# 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
+# or simply /m ot use all avail cores)
+# msbuild rocksdb.sln
+#
+# rocksdb.sln build features exclusions of test only code in Release. If you build ALL_BUILD then everything
+# will be attempted but test only code does not build in Release mode.
+#
+# 5. And release mode (/m[:<N>] is also supported)
+# msbuild rocksdb.sln /p:Configuration=Release
+#
+# Linux:
+#
+# 1. Install a recent toolchain such as devtoolset-3 if you're on a older distro. C++11 required.
+# 2. mkdir build; cd build
+# 3. cmake ..
+# 4. make -j
+
+cmake_minimum_required(VERSION 2.6)
+project(rocksdb)
+
+if(POLICY CMP0042)
+ cmake_policy(SET CMP0042 NEW)
+endif()
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/")
+
+if(MSVC)
+ include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
+else()
+ option(WITH_JEMALLOC "build with JeMalloc" OFF)
+ if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+ # FreeBSD has jemaloc as default malloc
+ # but it does not have all the jemalloc files in include/...
+ set(WITH_JEMALLOC ON)
+ else()
+ if(WITH_JEMALLOC)
+ find_package(JeMalloc REQUIRED)
+ add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE)
+ include_directories(${JEMALLOC_INCLUDE_DIR})
+ endif()
+ endif()
+
+ option(WITH_SNAPPY "build with SNAPPY" OFF)
+ if(WITH_SNAPPY)
+ find_package(snappy REQUIRED)
+ add_definitions(-DSNAPPY)
+ include_directories(${SNAPPY_INCLUDE_DIR})
+ list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES})
+ endif()
+
+ option(WITH_ZLIB "build with zlib" OFF)
+ if(WITH_ZLIB)
+ find_package(zlib REQUIRED)
+ add_definitions(-DZLIB)
+ include_directories(${ZLIB_INCLUDE_DIR})
+ list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARIES})
+ endif()
+
+ option(WITH_BZ2 "build with bzip2" OFF)
+ if(WITH_BZ2)
+ find_package(bzip2 REQUIRED)
+ add_definitions(-DBZIP2)
+ include_directories(${BZIP2_INCLUDE_DIR})
+ list(APPEND THIRDPARTY_LIBS ${BZIP2_LIBRARIES})
+ endif()
+
+ option(WITH_LZ4 "build with lz4" OFF)
+ if(WITH_LZ4)
+ find_package(lz4 REQUIRED)
+ add_definitions(-DLZ4)
+ include_directories(${LZ4_INCLUDE_DIR})
+ list(APPEND THIRDPARTY_LIBS ${LZ4_LIBRARIES})
+ endif()
+
+ option(WITH_ZSTD "build with zstd" OFF)
+ if(WITH_ZSTD)
+ find_package(zstd REQUIRED)
+ add_definitions(-DZSTD)
+ include_directories(${ZSTD_INCLUDE_DIR})
+ list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARIES})
+ endif()
+endif()
+
+if(WIN32)
+ execute_process(COMMAND powershell -noprofile -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
+ execute_process(COMMAND powershell -noprofile -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
+ string(REGEX REPLACE "(..)_(..)_..(..).*" "\\1/\\2/\\3" DATE "${DATE}")
+ string(REGEX REPLACE "(..):(.....).*" " \\1:\\2" TIME "${TIME}")
+ set(GIT_DATE_TIME "${DATE} ${TIME}")
+else()
+ execute_process(COMMAND date "+%Y/%m/%d %H:%M:%S" OUTPUT_VARIABLE DATETIME)
+ string(REGEX REPLACE "\n" "" DATETIME ${DATETIME})
+ set(GIT_DATE_TIME "${DATETIME}")
+endif()
+
+find_package(Git)
+
+if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
+ if(WIN32)
+ execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
+ else()
+ execute_process(COMMAND ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
+ endif()
+else()
+ set(GIT_SHA 0)
+endif()
+
+string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
+
+if(NOT WIN32)
+ execute_process(COMMAND
+ "./build_tools/version.sh" "full"
+ WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+ OUTPUT_VARIABLE ROCKSDB_VERSION
+ )
+ string(STRIP "${ROCKSDB_VERSION}" ROCKSDB_VERSION)
+ execute_process(COMMAND
+ "./build_tools/version.sh" "major"
+ WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+ OUTPUT_VARIABLE ROCKSDB_VERSION_MAJOR
+ )
+ string(STRIP "${ROCKSDB_VERSION_MAJOR}" ROCKSDB_VERSION_MAJOR)
+endif()
+
+option(WITH_MD_LIBRARY "build with MD" ON)
+if(WIN32 AND MSVC)
+ if(WITH_MD_LIBRARY)
+ set(RUNTIME_LIBRARY "MD")
+ else()
+ set(RUNTIME_LIBRARY "MT")
+ endif()
+endif()
+
+set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
+configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
+add_library(build_version OBJECT ${BUILD_VERSION_CC})
+target_include_directories(build_version PRIVATE
+ ${CMAKE_CURRENT_SOURCE_DIR}/util)
+if(MSVC)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /wd4127 /wd4800 /wd4996 /wd4351")
+else()
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers")
+ if(MINGW)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
+ endif()
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+ if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
+ include(CheckCXXCompilerFlag)
+ CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
+ if(HAVE_OMIT_LEAF_FRAME_POINTER)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
+ endif()
+ endif()
+endif()
+
+option(PORTABLE "build a portable binary" OFF)
+option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
+if(PORTABLE)
+ # MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
+ # is available, it is available by default.
+ if(FORCE_SSE42 AND NOT MSVC)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
+ endif()
+else()
+ if(MSVC)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
+ else()
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
+ endif()
+endif()
+
+set(CMAKE_REQUIRED_FLAGS ${CMAKE_CXX_FLAGS})
+include(CheckCXXSourceCompiles)
+CHECK_CXX_SOURCE_COMPILES("
+#include <cstdint>
+#include <nmmintrin.h>
+int main() {
+ volatile uint32_t x = _mm_crc32_u32(0, 0);
+}
+" HAVE_SSE42)
+if(HAVE_SSE42)
+ add_definitions(-DHAVE_SSE42)
+elseif(FORCE_SSE42)
+ message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
+endif()
+
+CHECK_CXX_SOURCE_COMPILES("
+#if defined(_MSC_VER) && !defined(__thread)
+#define __thread __declspec(thread)
+#endif
+int main() {
+ static __thread int tls;
+}
+" HAVE_THREAD_LOCAL)
+if(HAVE_THREAD_LOCAL)
+ add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
+endif()
+
+option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON)
+if(FAIL_ON_WARNINGS)
+ if(MSVC)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX")
+ else() # assume GCC
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
+ endif()
+endif()
+
+option(WITH_ASAN "build with ASAN" OFF)
+if(WITH_ASAN)
+ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address")
+ if(WITH_JEMALLOC)
+ message(FATAL "ASAN does not work well with JeMalloc")
+ endif()
+endif()
+
+option(WITH_TSAN "build with TSAN" OFF)
+if(WITH_TSAN)
+ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread -pie")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fPIC")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread -fPIC")
+ if(WITH_JEMALLOC)
+ message(FATAL "TSAN does not work well with JeMalloc")
+ endif()
+endif()
+
+option(WITH_UBSAN "build with UBSAN" OFF)
+if(WITH_UBSAN)
+ add_definitions(-DROCKSDB_UBSAN_RUN)
+ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined")
+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined")
+ if(WITH_JEMALLOC)
+ message(FATAL "UBSAN does not work well with JeMalloc")
+ endif()
+endif()
+
+# Used to run CI build and tests so we can run faster
+set(OPTIMIZE_DEBUG_DEFAULT 0) # Debug build is unoptimized by default use -DOPTDBG=1 to optimize
+
+if(DEFINED OPTDBG)
+ set(OPTIMIZE_DEBUG ${OPTDBG})
+else()
+ set(OPTIMIZE_DEBUG ${OPTIMIZE_DEBUG_DEFAULT})
+endif()
+
+if(MSVC)
+ if((${OPTIMIZE_DEBUG} EQUAL 1))
+ message(STATUS "Debug optimization is enabled")
+ set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /${RUNTIME_LIBRARY}d")
+ else()
+ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /${RUNTIME_LIBRARY}d")
+ endif()
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}")
+
+ set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
+ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
+endif()
+
+if(CMAKE_COMPILER_IS_GNUCXX)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-builtin-memcmp")
+endif()
+
+option(ROCKSDB_LITE "Build RocksDBLite version" OFF)
+if(ROCKSDB_LITE)
+ add_definitions(-DROCKSDB_LITE)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
+endif()
+
+if(CMAKE_SYSTEM_NAME MATCHES "Cygwin")
+ add_definitions(-fno-builtin-memcmp -DCYGWIN)
+elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
+ add_definitions(-DOS_MACOSX)
+ if(CMAKE_SYSTEM_PROCESSOR MATCHES arm)
+ add_definitions(-DIOS_CROSS_COMPILE -DROCKSDB_LITE)
+ # no debug info for IOS, that will make our library big
+ add_definitions(-DNDEBUG)
+ endif()
+elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
+ add_definitions(-DOS_LINUX)
+elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")
+ add_definitions(-DOS_SOLARIS)
+elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+ add_definitions(-DOS_FREEBSD)
+elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
+ add_definitions(-DOS_NETBSD)
+elseif(CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
+ add_definitions(-DOS_OPENBSD)
+elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly")
+ add_definitions(-DOS_DRAGONFLYBSD)
+elseif(CMAKE_SYSTEM_NAME MATCHES "Android")
+ add_definitions(-DOS_ANDROID)
+elseif(CMAKE_SYSTEM_NAME MATCHES "Windows")
+ add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)
+ if(MINGW)
+ add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_VISTA)
+ endif()
+endif()
+
+if(NOT WIN32)
+ add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
+endif()
+
+option(WITH_FALLOCATE "build with fallocate" ON)
+
+if(WITH_FALLOCATE)
+ set(CMAKE_REQUIRED_FLAGS ${CMAKE_C_FLAGS})
+ include(CheckCSourceCompiles)
+ CHECK_C_SOURCE_COMPILES("
+#include <fcntl.h>
+#include <linux/falloc.h>
+int main() {
+ int fd = open(\"/dev/null\", 0);
+ fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
+}
+" HAVE_FALLOCATE)
+ if(HAVE_FALLOCATE)
+ add_definitions(-DROCKSDB_FALLOCATE_PRESENT)
+ endif()
+endif()
+
+include(CheckFunctionExists)
+CHECK_FUNCTION_EXISTS(malloc_usable_size HAVE_MALLOC_USABLE_SIZE)
+if(HAVE_MALLOC_USABLE_SIZE)
+ add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE)
+endif()
+
+include_directories(${PROJECT_SOURCE_DIR})
+include_directories(${PROJECT_SOURCE_DIR}/include)
+include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
+find_package(Threads REQUIRED)
+
+add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest)
+
+# Main library source code
+
+set(SOURCES
+ cache/clock_cache.cc
+ cache/lru_cache.cc
+ cache/sharded_cache.cc
+ db/builder.cc
+ db/c.cc
+ db/column_family.cc
+ db/compacted_db_impl.cc
+ db/compaction.cc
+ db/compaction_iterator.cc
+ db/compaction_job.cc
+ db/compaction_picker.cc
+ db/compaction_picker_universal.cc
+ db/convenience.cc
+ db/db_filesnapshot.cc
+ db/db_impl.cc
+ db/db_impl_write.cc
+ db/db_impl_compaction_flush.cc
+ db/db_impl_files.cc
+ db/db_impl_open.cc
+ db/db_impl_debug.cc
+ db/db_impl_experimental.cc
+ db/db_impl_readonly.cc
+ db/db_info_dumper.cc
+ db/db_iter.cc
+ db/dbformat.cc
+ db/event_helpers.cc
+ db/experimental.cc
+ db/external_sst_file_ingestion_job.cc
+ db/file_indexer.cc
+ db/flush_job.cc
+ db/flush_scheduler.cc
+ db/forward_iterator.cc
+ db/internal_stats.cc
+ db/log_reader.cc
+ db/log_writer.cc
+ db/malloc_stats.cc
+ db/managed_iterator.cc
+ db/memtable.cc
+ db/memtable_list.cc
+ db/merge_helper.cc
+ db/merge_operator.cc
+ db/range_del_aggregator.cc
+ db/repair.cc
+ db/snapshot_impl.cc
+ db/table_cache.cc
+ db/table_properties_collector.cc
+ db/transaction_log_impl.cc
+ db/version_builder.cc
+ db/version_edit.cc
+ db/version_set.cc
+ db/wal_manager.cc
+ db/write_batch.cc
+ db/write_batch_base.cc
+ db/write_controller.cc
+ db/write_thread.cc
+ env/env.cc
+ env/env_chroot.cc
+ env/env_encryption.cc
+ env/env_hdfs.cc
+ env/mock_env.cc
+ memtable/alloc_tracker.cc
+ memtable/hash_cuckoo_rep.cc
+ memtable/hash_linklist_rep.cc
+ memtable/hash_skiplist_rep.cc
+ memtable/skiplistrep.cc
+ memtable/vectorrep.cc
+ memtable/write_buffer_manager.cc
+ monitoring/histogram.cc
+ monitoring/histogram_windowing.cc
+ monitoring/instrumented_mutex.cc
+ monitoring/iostats_context.cc
+ monitoring/perf_context.cc
+ monitoring/perf_level.cc
+ monitoring/statistics.cc
+ monitoring/thread_status_impl.cc
+ monitoring/thread_status_updater.cc
+ monitoring/thread_status_util.cc
+ monitoring/thread_status_util_debug.cc
+ options/cf_options.cc
+ options/db_options.cc
+ options/options.cc
+ options/options_helper.cc
+ options/options_parser.cc
+ options/options_sanity_check.cc
+ port/stack_trace.cc
+ table/adaptive_table_factory.cc
+ table/block.cc
+ table/block_based_filter_block.cc
+ table/block_based_table_builder.cc
+ table/block_based_table_factory.cc
+ table/block_based_table_reader.cc
+ table/block_builder.cc
+ table/block_prefix_index.cc
+ table/bloom_block.cc
+ table/cuckoo_table_builder.cc
+ table/cuckoo_table_factory.cc
+ table/cuckoo_table_reader.cc
+ table/flush_block_policy.cc
+ table/format.cc
+ table/full_filter_block.cc
+ table/get_context.cc
+ table/index_builder.cc
+ table/iterator.cc
+ table/merging_iterator.cc
+ table/meta_blocks.cc
+ table/partitioned_filter_block.cc
+ table/persistent_cache_helper.cc
+ table/plain_table_builder.cc
+ table/plain_table_factory.cc
+ table/plain_table_index.cc
+ table/plain_table_key_coding.cc
+ table/plain_table_reader.cc
+ table/sst_file_writer.cc
+ table/table_properties.cc
+ table/two_level_iterator.cc
+ util/arena.cc
+ util/auto_roll_logger.cc
+ util/bloom.cc
+ util/coding.cc
+ util/compaction_job_stats_impl.cc
+ util/comparator.cc
+ util/concurrent_arena.cc
+ util/crc32c.cc
+ util/delete_scheduler.cc
+ util/dynamic_bloom.cc
+ util/event_logger.cc
+ util/file_reader_writer.cc
+ util/file_util.cc
+ util/filename.cc
+ util/filter_policy.cc
+ util/hash.cc
+ util/log_buffer.cc
+ util/murmurhash.cc
+ util/random.cc
+ util/rate_limiter.cc
+ util/slice.cc
+ util/sst_file_manager_impl.cc
+ util/status.cc
+ util/status_message.cc
+ util/string_util.cc
+ util/sync_point.cc
+ util/testutil.cc
+ util/thread_local.cc
+ util/threadpool_imp.cc
+ util/transaction_test_util.cc
+ util/xxhash.cc
+ utilities/backupable/backupable_db.cc
+ utilities/blob_db/blob_db.cc
+ utilities/blob_db/blob_db_impl.cc
+ utilities/blob_db/blob_dump_tool.cc
+ utilities/blob_db/blob_file.cc
+ utilities/blob_db/blob_log_reader.cc
+ utilities/blob_db/blob_log_writer.cc
+ utilities/blob_db/blob_log_format.cc
+ utilities/blob_db/ttl_extractor.cc
+ utilities/cassandra/cassandra_compaction_filter.cc
+ utilities/cassandra/format.cc
+ utilities/cassandra/merge_operator.cc
+ utilities/checkpoint/checkpoint_impl.cc
+ utilities/col_buf_decoder.cc
+ utilities/col_buf_encoder.cc
+ utilities/column_aware_encoding_util.cc
+ utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
+ utilities/date_tiered/date_tiered_db_impl.cc
+ utilities/debug.cc
+ utilities/document/document_db.cc
+ utilities/document/json_document.cc
+ utilities/document/json_document_builder.cc
+ utilities/env_mirror.cc
+ utilities/env_timed.cc
+ utilities/geodb/geodb_impl.cc
+ utilities/leveldb_options/leveldb_options.cc
+ utilities/lua/rocks_lua_compaction_filter.cc
+ utilities/memory/memory_util.cc
+ utilities/merge_operators/max.cc
+ utilities/merge_operators/put.cc
+ utilities/merge_operators/string_append/stringappend.cc
+ utilities/merge_operators/string_append/stringappend2.cc
+ utilities/merge_operators/uint64add.cc
+ utilities/option_change_migration/option_change_migration.cc
+ utilities/options/options_util.cc
+ utilities/persistent_cache/block_cache_tier.cc
+ utilities/persistent_cache/block_cache_tier_file.cc
+ utilities/persistent_cache/block_cache_tier_metadata.cc
+ utilities/persistent_cache/persistent_cache_tier.cc
+ utilities/persistent_cache/volatile_tier_impl.cc
+ utilities/redis/redis_lists.cc
+ utilities/simulator_cache/sim_cache.cc
+ utilities/spatialdb/spatial_db.cc
+ utilities/table_properties_collectors/compact_on_deletion_collector.cc
+ utilities/transactions/optimistic_transaction_db_impl.cc
+ utilities/transactions/optimistic_transaction.cc
+ utilities/transactions/transaction_base.cc
+ utilities/transactions/pessimistic_transaction_db.cc
+ utilities/transactions/transaction_db_mutex_impl.cc
+ utilities/transactions/pessimistic_transaction.cc
+ utilities/transactions/transaction_lock_mgr.cc
+ utilities/transactions/transaction_util.cc
+ utilities/transactions/write_prepared_txn.cc
+ utilities/ttl/db_ttl_impl.cc
+ utilities/write_batch_with_index/write_batch_with_index.cc
+ utilities/write_batch_with_index/write_batch_with_index_internal.cc
+ $<TARGET_OBJECTS:build_version>)
+
+if(WIN32)
+ list(APPEND SOURCES
+ port/win/io_win.cc
+ port/win/env_win.cc
+ port/win/env_default.cc
+ port/win/port_win.cc
+ port/win/win_logger.cc
+ port/win/win_thread.cc
+ port/win/xpress_win.cc)
+else()
+ list(APPEND SOURCES
+ port/port_posix.cc
+ env/env_posix.cc
+ env/io_posix.cc)
+endif()
+
+set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
+set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
+set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
+if(WIN32)
+ set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
+ set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+else()
+ set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
+ set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+
+ add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
+ target_link_libraries(${ROCKSDB_SHARED_LIB}
+ ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+ set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
+ LINKER_LANGUAGE CXX
+ VERSION ${ROCKSDB_VERSION}
+ SOVERSION ${ROCKSDB_VERSION_MAJOR}
+ CXX_STANDARD 11
+ OUTPUT_NAME "rocksdb")
+endif()
+
+option(WITH_LIBRADOS "Build with librados" OFF)
+if(WITH_LIBRADOS)
+ list(APPEND SOURCES
+ utilities/env_librados.cc)
+ list(APPEND THIRDPARTY_LIBS rados)
+endif()
+
+add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
+target_link_libraries(${ROCKSDB_STATIC_LIB}
+ ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+
+if(WIN32)
+ add_library(${ROCKSDB_IMPORT_LIB} SHARED ${SOURCES})
+ target_link_libraries(${ROCKSDB_IMPORT_LIB}
+ ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+ set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
+ COMPILE_DEFINITIONS "ROCKSDB_DLL;ROCKSDB_LIBRARY_EXPORTS")
+ if(MSVC)
+ set_target_properties(${ROCKSDB_STATIC_LIB} PROPERTIES
+ COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_STATIC_LIB}.pdb")
+ set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
+ COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_IMPORT_LIB}.pdb")
+ endif()
+endif()
+
+option(WITH_JNI "build with JNI" OFF)
+if(WITH_JNI OR JNI)
+ message(STATUS "JNI library is enabled")
+ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
+else()
+ message(STATUS "JNI library is disabled")
+endif()
+
+if(WITH_LIBRADOS)
+ list(APPEND TESTS utilities/env_librados_test.cc)
+endif()
+
+set(BENCHMARKS
+ cache/cache_bench.cc
+ memtable/memtablerep_bench.cc
+ tools/db_bench.cc
+ table/table_reader_bench.cc
+ utilities/column_aware_encoding_exp.cc
+ utilities/persistent_cache/hash_table_bench.cc)
+add_library(testharness OBJECT util/testharness.cc)
+foreach(sourcefile ${BENCHMARKS})
+ get_filename_component(exename ${sourcefile} NAME_WE)
+ add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
+ $<TARGET_OBJECTS:testharness>)
+ target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
+endforeach(sourcefile ${BENCHMARKS})
+
+add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
+
+add_subdirectory(tools)
+
+# Installation and packaging for Linux
+if(NOT WIN32)
+install(TARGETS ${ROCKSDB_STATIC_LIB} COMPONENT devel ARCHIVE DESTINATION lib64)
+install(TARGETS ${ROCKSDB_SHARED_LIB} COMPONENT runtime DESTINATION lib64)
+install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/rocksdb/"
+ COMPONENT devel
+ DESTINATION include/rocksdb)
+set(CMAKE_INSTALL_PREFIX /usr)
+endif()
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/CONTRIBUTING.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/CONTRIBUTING.md b/thirdparty/rocksdb/CONTRIBUTING.md
new file mode 100644
index 0000000..b8b1a41
--- /dev/null
+++ b/thirdparty/rocksdb/CONTRIBUTING.md
@@ -0,0 +1,14 @@
+# Contributing to RocksDB
+
+## Contributor License Agreement ("CLA")
+
+In order to accept your pull request, we need you to submit a CLA. You
+only need to do this once, so if you've done this for another Facebook
+open source project, you're good to go. If you are submitting a pull
+request for the first time, just let us know that you have completed
+the CLA and we can cross-check with your GitHub username.
+
+Complete your CLA here: <https://code.facebook.com/cla>
+
+If you prefer to sign a paper copy, we can send you a PDF. Send us an
+e-mail or create a new github issue to request the CLA in PDF format.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/COPYING
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/COPYING b/thirdparty/rocksdb/COPYING
new file mode 100644
index 0000000..d159169
--- /dev/null
+++ b/thirdparty/rocksdb/COPYING
@@ -0,0 +1,339 @@
+ GNU GENERAL PUBLIC LICENSE
+ Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.) You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+ To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+ We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+ Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+ Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+ 1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+ 2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+ a) You must cause the modified files to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in
+ whole or in part contains or is derived from the Program or any
+ part thereof, to be licensed as a whole at no charge to all third
+ parties under the terms of this License.
+
+ c) If the modified program normally reads commands interactively
+ when run, you must cause it, when started running for such
+ interactive use in the most ordinary way, to print or display an
+ announcement including an appropriate copyright notice and a
+ notice that there is no warranty (or else, saying that you provide
+ a warranty) and that users may redistribute the program under
+ these conditions, and telling the user how to view a copy of this
+ License. (Exception: if the Program itself is interactive but
+ does not normally print such an announcement, your work based on
+ the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+ 3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable
+ source code, which must be distributed under the terms of Sections
+ 1 and 2 above on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three
+ years, to give any third party, for a charge no more than your
+ cost of physically performing source distribution, a complete
+ machine-readable copy of the corresponding source code, to be
+ distributed under the terms of Sections 1 and 2 above on a medium
+ customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer
+ to distribute corresponding source code. (This alternative is
+ allowed only for noncommercial distribution and only if you
+ received the program in object code or executable form with such
+ an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+ 4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+ 5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+ 6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+ 7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+ 8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+ 9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+ 10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission. For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this. Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+ NO WARRANTY
+
+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+ Gnomovision version 69, Copyright (C) year name of author
+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ <signature of Ty Coon>, 1 April 1989
+ Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/DEFAULT_OPTIONS_HISTORY.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/DEFAULT_OPTIONS_HISTORY.md b/thirdparty/rocksdb/DEFAULT_OPTIONS_HISTORY.md
new file mode 100644
index 0000000..26280ee
--- /dev/null
+++ b/thirdparty/rocksdb/DEFAULT_OPTIONS_HISTORY.md
@@ -0,0 +1,24 @@
+# RocksDB default options change log
+## Unreleased
+* delayed_write_rate takes the rate given by rate_limiter if not specified.
+
+## 5.2
+* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
+
+## 5.0 (11/17/2016)
+* Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default
+* Options.level0_stop_writes_trigger default value changes from 24 to 32.
+
+## 4.8.0 (5/2/2016)
+* options.max_open_files changes from 5000 to -1. It improves performance, but users need to set file descriptor limit to be large enough and watch memory usage for index and bloom filters.
+* options.base_background_compactions changes from max_background_compactions to 1. When users set higher max_background_compactions but the write throughput is not high, the writes are less spiky to disks.
+* options.wal_recovery_mode changes from kTolerateCorruptedTailRecords to kPointInTimeRecovery. Avoid some false positive when file system or hardware reorder the writes for file data and metadata.
+
+## 4.7.0 (4/8/2016)
+* options.write_buffer_size changes from 4MB to 64MB.
+* options.target_file_size_base changes from 2MB to 64MB.
+* options.max_bytes_for_level_base changes from 10MB to 256MB.
+* options.soft_pending_compaction_bytes_limit changes from 0 (disabled) to 64GB.
+* options.hard_pending_compaction_bytes_limit changes from 0 (disabled) to 256GB.
+* table_cache_numshardbits changes from 4 to 6.
+* max_file_opening_threads changes from 1 to 16.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/DUMP_FORMAT.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/DUMP_FORMAT.md b/thirdparty/rocksdb/DUMP_FORMAT.md
new file mode 100644
index 0000000..009daba
--- /dev/null
+++ b/thirdparty/rocksdb/DUMP_FORMAT.md
@@ -0,0 +1,16 @@
+## RocksDB dump format
+
+The version 1 RocksDB dump format is fairly simple:
+
+1) The dump starts with the magic 8 byte identifier "ROCKDUMP"
+
+2) The magic is followed by an 8 byte big-endian version which is 0x00000001.
+
+3) Next are arbitrarily sized chunks of bytes prepended by 4 byte little endian number indicating how large each chunk is.
+
+4) The first chunk is special and is a json string indicating some things about the creation of this dump. It contains the following keys:
+* database-path: The path of the database this dump was created from.
+* hostname: The hostname of the machine where the dump was created.
+* creation-time: Unix seconds since epoc when this dump was created.
+
+5) Following the info dump the slices paired into are key/value pairs.
[41/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/doc/index.html
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/doc/index.html b/thirdparty/leveldb-1.18/doc/index.html
deleted file mode 100755
index 3ed0ed9..0000000
--- a/thirdparty/leveldb-1.18/doc/index.html
+++ /dev/null
@@ -1,549 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-<link rel="stylesheet" type="text/css" href="doc.css" />
-<title>Leveldb</title>
-</head>
-
-<body>
-<h1>Leveldb</h1>
-<address>Jeff Dean, Sanjay Ghemawat</address>
-<p>
-The <code>leveldb</code> library provides a persistent key value store. Keys and
-values are arbitrary byte arrays. The keys are ordered within the key
-value store according to a user-specified comparator function.
-
-<p>
-<h1>Opening A Database</h1>
-<p>
-A <code>leveldb</code> database has a name which corresponds to a file system
-directory. All of the contents of database are stored in this
-directory. The following example shows how to open a database,
-creating it if necessary:
-<p>
-<pre>
- #include <assert>
- #include "leveldb/db.h"
-
- leveldb::DB* db;
- leveldb::Options options;
- options.create_if_missing = true;
- leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db);
- assert(status.ok());
- ...
-</pre>
-If you want to raise an error if the database already exists, add
-the following line before the <code>leveldb::DB::Open</code> call:
-<pre>
- options.error_if_exists = true;
-</pre>
-<h1>Status</h1>
-<p>
-You may have noticed the <code>leveldb::Status</code> type above. Values of this
-type are returned by most functions in <code>leveldb</code> that may encounter an
-error. You can check if such a result is ok, and also print an
-associated error message:
-<p>
-<pre>
- leveldb::Status s = ...;
- if (!s.ok()) cerr << s.ToString() << endl;
-</pre>
-<h1>Closing A Database</h1>
-<p>
-When you are done with a database, just delete the database object.
-Example:
-<p>
-<pre>
- ... open the db as described above ...
- ... do something with db ...
- delete db;
-</pre>
-<h1>Reads And Writes</h1>
-<p>
-The database provides <code>Put</code>, <code>Delete</code>, and <code>Get</code> methods to
-modify/query the database. For example, the following code
-moves the value stored under key1 to key2.
-<pre>
- std::string value;
- leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value);
- if (s.ok()) s = db->Put(leveldb::WriteOptions(), key2, value);
- if (s.ok()) s = db->Delete(leveldb::WriteOptions(), key1);
-</pre>
-
-<h1>Atomic Updates</h1>
-<p>
-Note that if the process dies after the Put of key2 but before the
-delete of key1, the same value may be left stored under multiple keys.
-Such problems can be avoided by using the <code>WriteBatch</code> class to
-atomically apply a set of updates:
-<p>
-<pre>
- #include "leveldb/write_batch.h"
- ...
- std::string value;
- leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value);
- if (s.ok()) {
- leveldb::WriteBatch batch;
- batch.Delete(key1);
- batch.Put(key2, value);
- s = db->Write(leveldb::WriteOptions(), &batch);
- }
-</pre>
-The <code>WriteBatch</code> holds a sequence of edits to be made to the database,
-and these edits within the batch are applied in order. Note that we
-called <code>Delete</code> before <code>Put</code> so that if <code>key1</code> is identical to <code>key2</code>,
-we do not end up erroneously dropping the value entirely.
-<p>
-Apart from its atomicity benefits, <code>WriteBatch</code> may also be used to
-speed up bulk updates by placing lots of individual mutations into the
-same batch.
-
-<h1>Synchronous Writes</h1>
-By default, each write to <code>leveldb</code> is asynchronous: it
-returns after pushing the write from the process into the operating
-system. The transfer from operating system memory to the underlying
-persistent storage happens asynchronously. The <code>sync</code> flag
-can be turned on for a particular write to make the write operation
-not return until the data being written has been pushed all the way to
-persistent storage. (On Posix systems, this is implemented by calling
-either <code>fsync(...)</code> or <code>fdatasync(...)</code> or
-<code>msync(..., MS_SYNC)</code> before the write operation returns.)
-<pre>
- leveldb::WriteOptions write_options;
- write_options.sync = true;
- db->Put(write_options, ...);
-</pre>
-Asynchronous writes are often more than a thousand times as fast as
-synchronous writes. The downside of asynchronous writes is that a
-crash of the machine may cause the last few updates to be lost. Note
-that a crash of just the writing process (i.e., not a reboot) will not
-cause any loss since even when <code>sync</code> is false, an update
-is pushed from the process memory into the operating system before it
-is considered done.
-
-<p>
-Asynchronous writes can often be used safely. For example, when
-loading a large amount of data into the database you can handle lost
-updates by restarting the bulk load after a crash. A hybrid scheme is
-also possible where every Nth write is synchronous, and in the event
-of a crash, the bulk load is restarted just after the last synchronous
-write finished by the previous run. (The synchronous write can update
-a marker that describes where to restart on a crash.)
-
-<p>
-<code>WriteBatch</code> provides an alternative to asynchronous writes.
-Multiple updates may be placed in the same <code>WriteBatch</code> and
-applied together using a synchronous write (i.e.,
-<code>write_options.sync</code> is set to true). The extra cost of
-the synchronous write will be amortized across all of the writes in
-the batch.
-
-<p>
-<h1>Concurrency</h1>
-<p>
-A database may only be opened by one process at a time.
-The <code>leveldb</code> implementation acquires a lock from the
-operating system to prevent misuse. Within a single process, the
-same <code>leveldb::DB</code> object may be safely shared by multiple
-concurrent threads. I.e., different threads may write into or fetch
-iterators or call <code>Get</code> on the same database without any
-external synchronization (the leveldb implementation will
-automatically do the required synchronization). However other objects
-(like Iterator and WriteBatch) may require external synchronization.
-If two threads share such an object, they must protect access to it
-using their own locking protocol. More details are available in
-the public header files.
-<p>
-<h1>Iteration</h1>
-<p>
-The following example demonstrates how to print all key,value pairs
-in a database.
-<p>
-<pre>
- leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions());
- for (it->SeekToFirst(); it->Valid(); it->Next()) {
- cout << it->key().ToString() << ": " << it->value().ToString() << endl;
- }
- assert(it->status().ok()); // Check for any errors found during the scan
- delete it;
-</pre>
-The following variation shows how to process just the keys in the
-range <code>[start,limit)</code>:
-<p>
-<pre>
- for (it->Seek(start);
- it->Valid() && it->key().ToString() < limit;
- it->Next()) {
- ...
- }
-</pre>
-You can also process entries in reverse order. (Caveat: reverse
-iteration may be somewhat slower than forward iteration.)
-<p>
-<pre>
- for (it->SeekToLast(); it->Valid(); it->Prev()) {
- ...
- }
-</pre>
-<h1>Snapshots</h1>
-<p>
-Snapshots provide consistent read-only views over the entire state of
-the key-value store. <code>ReadOptions::snapshot</code> may be non-NULL to indicate
-that a read should operate on a particular version of the DB state.
-If <code>ReadOptions::snapshot</code> is NULL, the read will operate on an
-implicit snapshot of the current state.
-<p>
-Snapshots are created by the DB::GetSnapshot() method:
-<p>
-<pre>
- leveldb::ReadOptions options;
- options.snapshot = db->GetSnapshot();
- ... apply some updates to db ...
- leveldb::Iterator* iter = db->NewIterator(options);
- ... read using iter to view the state when the snapshot was created ...
- delete iter;
- db->ReleaseSnapshot(options.snapshot);
-</pre>
-Note that when a snapshot is no longer needed, it should be released
-using the DB::ReleaseSnapshot interface. This allows the
-implementation to get rid of state that was being maintained just to
-support reading as of that snapshot.
-<h1>Slice</h1>
-<p>
-The return value of the <code>it->key()</code> and <code>it->value()</code> calls above
-are instances of the <code>leveldb::Slice</code> type. <code>Slice</code> is a simple
-structure that contains a length and a pointer to an external byte
-array. Returning a <code>Slice</code> is a cheaper alternative to returning a
-<code>std::string</code> since we do not need to copy potentially large keys and
-values. In addition, <code>leveldb</code> methods do not return null-terminated
-C-style strings since <code>leveldb</code> keys and values are allowed to
-contain '\0' bytes.
-<p>
-C++ strings and null-terminated C-style strings can be easily converted
-to a Slice:
-<p>
-<pre>
- leveldb::Slice s1 = "hello";
-
- std::string str("world");
- leveldb::Slice s2 = str;
-</pre>
-A Slice can be easily converted back to a C++ string:
-<pre>
- std::string str = s1.ToString();
- assert(str == std::string("hello"));
-</pre>
-Be careful when using Slices since it is up to the caller to ensure that
-the external byte array into which the Slice points remains live while
-the Slice is in use. For example, the following is buggy:
-<p>
-<pre>
- leveldb::Slice slice;
- if (...) {
- std::string str = ...;
- slice = str;
- }
- Use(slice);
-</pre>
-When the <code>if</code> statement goes out of scope, <code>str</code> will be destroyed and the
-backing storage for <code>slice</code> will disappear.
-<p>
-<h1>Comparators</h1>
-<p>
-The preceding examples used the default ordering function for key,
-which orders bytes lexicographically. You can however supply a custom
-comparator when opening a database. For example, suppose each
-database key consists of two numbers and we should sort by the first
-number, breaking ties by the second number. First, define a proper
-subclass of <code>leveldb::Comparator</code> that expresses these rules:
-<p>
-<pre>
- class TwoPartComparator : public leveldb::Comparator {
- public:
- // Three-way comparison function:
- // if a < b: negative result
- // if a > b: positive result
- // else: zero result
- int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const {
- int a1, a2, b1, b2;
- ParseKey(a, &a1, &a2);
- ParseKey(b, &b1, &b2);
- if (a1 < b1) return -1;
- if (a1 > b1) return +1;
- if (a2 < b2) return -1;
- if (a2 > b2) return +1;
- return 0;
- }
-
- // Ignore the following methods for now:
- const char* Name() const { return "TwoPartComparator"; }
- void FindShortestSeparator(std::string*, const leveldb::Slice&) const { }
- void FindShortSuccessor(std::string*) const { }
- };
-</pre>
-Now create a database using this custom comparator:
-<p>
-<pre>
- TwoPartComparator cmp;
- leveldb::DB* db;
- leveldb::Options options;
- options.create_if_missing = true;
- options.comparator = &cmp;
- leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db);
- ...
-</pre>
-<h2>Backwards compatibility</h2>
-<p>
-The result of the comparator's <code>Name</code> method is attached to the
-database when it is created, and is checked on every subsequent
-database open. If the name changes, the <code>leveldb::DB::Open</code> call will
-fail. Therefore, change the name if and only if the new key format
-and comparison function are incompatible with existing databases, and
-it is ok to discard the contents of all existing databases.
-<p>
-You can however still gradually evolve your key format over time with
-a little bit of pre-planning. For example, you could store a version
-number at the end of each key (one byte should suffice for most uses).
-When you wish to switch to a new key format (e.g., adding an optional
-third part to the keys processed by <code>TwoPartComparator</code>),
-(a) keep the same comparator name (b) increment the version number
-for new keys (c) change the comparator function so it uses the
-version numbers found in the keys to decide how to interpret them.
-<p>
-<h1>Performance</h1>
-<p>
-Performance can be tuned by changing the default values of the
-types defined in <code>include/leveldb/options.h</code>.
-
-<p>
-<h2>Block size</h2>
-<p>
-<code>leveldb</code> groups adjacent keys together into the same block and such a
-block is the unit of transfer to and from persistent storage. The
-default block size is approximately 4096 uncompressed bytes.
-Applications that mostly do bulk scans over the contents of the
-database may wish to increase this size. Applications that do a lot
-of point reads of small values may wish to switch to a smaller block
-size if performance measurements indicate an improvement. There isn't
-much benefit in using blocks smaller than one kilobyte, or larger than
-a few megabytes. Also note that compression will be more effective
-with larger block sizes.
-<p>
-<h2>Compression</h2>
-<p>
-Each block is individually compressed before being written to
-persistent storage. Compression is on by default since the default
-compression method is very fast, and is automatically disabled for
-uncompressible data. In rare cases, applications may want to disable
-compression entirely, but should only do so if benchmarks show a
-performance improvement:
-<p>
-<pre>
- leveldb::Options options;
- options.compression = leveldb::kNoCompression;
- ... leveldb::DB::Open(options, name, ...) ....
-</pre>
-<h2>Cache</h2>
-<p>
-The contents of the database are stored in a set of files in the
-filesystem and each file stores a sequence of compressed blocks. If
-<code>options.cache</code> is non-NULL, it is used to cache frequently used
-uncompressed block contents.
-<p>
-<pre>
- #include "leveldb/cache.h"
-
- leveldb::Options options;
- options.cache = leveldb::NewLRUCache(100 * 1048576); // 100MB cache
- leveldb::DB* db;
- leveldb::DB::Open(options, name, &db);
- ... use the db ...
- delete db
- delete options.cache;
-</pre>
-Note that the cache holds uncompressed data, and therefore it should
-be sized according to application level data sizes, without any
-reduction from compression. (Caching of compressed blocks is left to
-the operating system buffer cache, or any custom <code>Env</code>
-implementation provided by the client.)
-<p>
-When performing a bulk read, the application may wish to disable
-caching so that the data processed by the bulk read does not end up
-displacing most of the cached contents. A per-iterator option can be
-used to achieve this:
-<p>
-<pre>
- leveldb::ReadOptions options;
- options.fill_cache = false;
- leveldb::Iterator* it = db->NewIterator(options);
- for (it->SeekToFirst(); it->Valid(); it->Next()) {
- ...
- }
-</pre>
-<h2>Key Layout</h2>
-<p>
-Note that the unit of disk transfer and caching is a block. Adjacent
-keys (according to the database sort order) will usually be placed in
-the same block. Therefore the application can improve its performance
-by placing keys that are accessed together near each other and placing
-infrequently used keys in a separate region of the key space.
-<p>
-For example, suppose we are implementing a simple file system on top
-of <code>leveldb</code>. The types of entries we might wish to store are:
-<p>
-<pre>
- filename -> permission-bits, length, list of file_block_ids
- file_block_id -> data
-</pre>
-We might want to prefix <code>filename</code> keys with one letter (say '/') and the
-<code>file_block_id</code> keys with a different letter (say '0') so that scans
-over just the metadata do not force us to fetch and cache bulky file
-contents.
-<p>
-<h2>Filters</h2>
-<p>
-Because of the way <code>leveldb</code> data is organized on disk,
-a single <code>Get()</code> call may involve multiple reads from disk.
-The optional <code>FilterPolicy</code> mechanism can be used to reduce
-the number of disk reads substantially.
-<pre>
- leveldb::Options options;
- options.filter_policy = NewBloomFilterPolicy(10);
- leveldb::DB* db;
- leveldb::DB::Open(options, "/tmp/testdb", &db);
- ... use the database ...
- delete db;
- delete options.filter_policy;
-</pre>
-The preceding code associates a
-<a href="http://en.wikipedia.org/wiki/Bloom_filter">Bloom filter</a>
-based filtering policy with the database. Bloom filter based
-filtering relies on keeping some number of bits of data in memory per
-key (in this case 10 bits per key since that is the argument we passed
-to NewBloomFilterPolicy). This filter will reduce the number of unnecessary
-disk reads needed for <code>Get()</code> calls by a factor of
-approximately a 100. Increasing the bits per key will lead to a
-larger reduction at the cost of more memory usage. We recommend that
-applications whose working set does not fit in memory and that do a
-lot of random reads set a filter policy.
-<p>
-If you are using a custom comparator, you should ensure that the filter
-policy you are using is compatible with your comparator. For example,
-consider a comparator that ignores trailing spaces when comparing keys.
-<code>NewBloomFilterPolicy</code> must not be used with such a comparator.
-Instead, the application should provide a custom filter policy that
-also ignores trailing spaces. For example:
-<pre>
- class CustomFilterPolicy : public leveldb::FilterPolicy {
- private:
- FilterPolicy* builtin_policy_;
- public:
- CustomFilterPolicy() : builtin_policy_(NewBloomFilterPolicy(10)) { }
- ~CustomFilterPolicy() { delete builtin_policy_; }
-
- const char* Name() const { return "IgnoreTrailingSpacesFilter"; }
-
- void CreateFilter(const Slice* keys, int n, std::string* dst) const {
- // Use builtin bloom filter code after removing trailing spaces
- std::vector<Slice> trimmed(n);
- for (int i = 0; i < n; i++) {
- trimmed[i] = RemoveTrailingSpaces(keys[i]);
- }
- return builtin_policy_->CreateFilter(&trimmed[i], n, dst);
- }
-
- bool KeyMayMatch(const Slice& key, const Slice& filter) const {
- // Use builtin bloom filter code after removing trailing spaces
- return builtin_policy_->KeyMayMatch(RemoveTrailingSpaces(key), filter);
- }
- };
-</pre>
-<p>
-Advanced applications may provide a filter policy that does not use
-a bloom filter but uses some other mechanism for summarizing a set
-of keys. See <code>leveldb/filter_policy.h</code> for detail.
-<p>
-<h1>Checksums</h1>
-<p>
-<code>leveldb</code> associates checksums with all data it stores in the file system.
-There are two separate controls provided over how aggressively these
-checksums are verified:
-<p>
-<ul>
-<li> <code>ReadOptions::verify_checksums</code> may be set to true to force
- checksum verification of all data that is read from the file system on
- behalf of a particular read. By default, no such verification is
- done.
-<p>
-<li> <code>Options::paranoid_checks</code> may be set to true before opening a
- database to make the database implementation raise an error as soon as
- it detects an internal corruption. Depending on which portion of the
- database has been corrupted, the error may be raised when the database
- is opened, or later by another database operation. By default,
- paranoid checking is off so that the database can be used even if
- parts of its persistent storage have been corrupted.
-<p>
- If a database is corrupted (perhaps it cannot be opened when
- paranoid checking is turned on), the <code>leveldb::RepairDB</code> function
- may be used to recover as much of the data as possible
-<p>
-</ul>
-<h1>Approximate Sizes</h1>
-<p>
-The <code>GetApproximateSizes</code> method can used to get the approximate
-number of bytes of file system space used by one or more key ranges.
-<p>
-<pre>
- leveldb::Range ranges[2];
- ranges[0] = leveldb::Range("a", "c");
- ranges[1] = leveldb::Range("x", "z");
- uint64_t sizes[2];
- leveldb::Status s = db->GetApproximateSizes(ranges, 2, sizes);
-</pre>
-The preceding call will set <code>sizes[0]</code> to the approximate number of
-bytes of file system space used by the key range <code>[a..c)</code> and
-<code>sizes[1]</code> to the approximate number of bytes used by the key range
-<code>[x..z)</code>.
-<p>
-<h1>Environment</h1>
-<p>
-All file operations (and other operating system calls) issued by the
-<code>leveldb</code> implementation are routed through a <code>leveldb::Env</code> object.
-Sophisticated clients may wish to provide their own <code>Env</code>
-implementation to get better control. For example, an application may
-introduce artificial delays in the file IO paths to limit the impact
-of <code>leveldb</code> on other activities in the system.
-<p>
-<pre>
- class SlowEnv : public leveldb::Env {
- .. implementation of the Env interface ...
- };
-
- SlowEnv env;
- leveldb::Options options;
- options.env = &env;
- Status s = leveldb::DB::Open(options, ...);
-</pre>
-<h1>Porting</h1>
-<p>
-<code>leveldb</code> may be ported to a new platform by providing platform
-specific implementations of the types/methods/functions exported by
-<code>leveldb/port/port.h</code>. See <code>leveldb/port/port_example.h</code> for more
-details.
-<p>
-In addition, the new platform may need a new default <code>leveldb::Env</code>
-implementation. See <code>leveldb/util/env_posix.h</code> for an example.
-
-<h1>Other Information</h1>
-
-<p>
-Details about the <code>leveldb</code> implementation may be found in
-the following documents:
-<ul>
-<li> <a href="impl.html">Implementation notes</a>
-<li> <a href="table_format.txt">Format of an immutable Table file</a>
-<li> <a href="log_format.txt">Format of a log file</a>
-</ul>
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/doc/log_format.txt
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/doc/log_format.txt b/thirdparty/leveldb-1.18/doc/log_format.txt
deleted file mode 100755
index 4cca5ef..0000000
--- a/thirdparty/leveldb-1.18/doc/log_format.txt
+++ /dev/null
@@ -1,75 +0,0 @@
-The log file contents are a sequence of 32KB blocks. The only
-exception is that the tail of the file may contain a partial block.
-
-Each block consists of a sequence of records:
- block := record* trailer?
- record :=
- checksum: uint32 // crc32c of type and data[] ; little-endian
- length: uint16 // little-endian
- type: uint8 // One of FULL, FIRST, MIDDLE, LAST
- data: uint8[length]
-
-A record never starts within the last six bytes of a block (since it
-won't fit). Any leftover bytes here form the trailer, which must
-consist entirely of zero bytes and must be skipped by readers.
-
-Aside: if exactly seven bytes are left in the current block, and a new
-non-zero length record is added, the writer must emit a FIRST record
-(which contains zero bytes of user data) to fill up the trailing seven
-bytes of the block and then emit all of the user data in subsequent
-blocks.
-
-More types may be added in the future. Some Readers may skip record
-types they do not understand, others may report that some data was
-skipped.
-
-FULL == 1
-FIRST == 2
-MIDDLE == 3
-LAST == 4
-
-The FULL record contains the contents of an entire user record.
-
-FIRST, MIDDLE, LAST are types used for user records that have been
-split into multiple fragments (typically because of block boundaries).
-FIRST is the type of the first fragment of a user record, LAST is the
-type of the last fragment of a user record, and MIDDLE is the type of
-all interior fragments of a user record.
-
-Example: consider a sequence of user records:
- A: length 1000
- B: length 97270
- C: length 8000
-A will be stored as a FULL record in the first block.
-
-B will be split into three fragments: first fragment occupies the rest
-of the first block, second fragment occupies the entirety of the
-second block, and the third fragment occupies a prefix of the third
-block. This will leave six bytes free in the third block, which will
-be left empty as the trailer.
-
-C will be stored as a FULL record in the fourth block.
-
-===================
-
-Some benefits over the recordio format:
-
-(1) We do not need any heuristics for resyncing - just go to next
-block boundary and scan. If there is a corruption, skip to the next
-block. As a side-benefit, we do not get confused when part of the
-contents of one log file are embedded as a record inside another log
-file.
-
-(2) Splitting at approximate boundaries (e.g., for mapreduce) is
-simple: find the next block boundary and skip records until we
-hit a FULL or FIRST record.
-
-(3) We do not need extra buffering for large records.
-
-Some downsides compared to recordio format:
-
-(1) No packing of tiny records. This could be fixed by adding a new
-record type, so it is a shortcoming of the current implementation,
-not necessarily the format.
-
-(2) No compression. Again, this could be fixed by adding new record types.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/doc/table_format.txt
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/doc/table_format.txt b/thirdparty/leveldb-1.18/doc/table_format.txt
deleted file mode 100755
index ca8f9b4..0000000
--- a/thirdparty/leveldb-1.18/doc/table_format.txt
+++ /dev/null
@@ -1,104 +0,0 @@
-File format
-===========
-
- <beginning_of_file>
- [data block 1]
- [data block 2]
- ...
- [data block N]
- [meta block 1]
- ...
- [meta block K]
- [metaindex block]
- [index block]
- [Footer] (fixed size; starts at file_size - sizeof(Footer))
- <end_of_file>
-
-The file contains internal pointers. Each such pointer is called
-a BlockHandle and contains the following information:
- offset: varint64
- size: varint64
-See https://developers.google.com/protocol-buffers/docs/encoding#varints
-for an explanation of varint64 format.
-
-(1) The sequence of key/value pairs in the file are stored in sorted
-order and partitioned into a sequence of data blocks. These blocks
-come one after another at the beginning of the file. Each data block
-is formatted according to the code in block_builder.cc, and then
-optionally compressed.
-
-(2) After the data blocks we store a bunch of meta blocks. The
-supported meta block types are described below. More meta block types
-may be added in the future. Each meta block is again formatted using
-block_builder.cc and then optionally compressed.
-
-(3) A "metaindex" block. It contains one entry for every other meta
-block where the key is the name of the meta block and the value is a
-BlockHandle pointing to that meta block.
-
-(4) An "index" block. This block contains one entry per data block,
-where the key is a string >= last key in that data block and before
-the first key in the successive data block. The value is the
-BlockHandle for the data block.
-
-(6) At the very end of the file is a fixed length footer that contains
-the BlockHandle of the metaindex and index blocks as well as a magic number.
- metaindex_handle: char[p]; // Block handle for metaindex
- index_handle: char[q]; // Block handle for index
- padding: char[40-p-q]; // zeroed bytes to make fixed length
- // (40==2*BlockHandle::kMaxEncodedLength)
- magic: fixed64; // == 0xdb4775248b80fb57 (little-endian)
-
-"filter" Meta Block
--------------------
-
-If a "FilterPolicy" was specified when the database was opened, a
-filter block is stored in each table. The "metaindex" block contains
-an entry that maps from "filter.<N>" to the BlockHandle for the filter
-block where "<N>" is the string returned by the filter policy's
-"Name()" method.
-
-The filter block stores a sequence of filters, where filter i contains
-the output of FilterPolicy::CreateFilter() on all keys that are stored
-in a block whose file offset falls within the range
-
- [ i*base ... (i+1)*base-1 ]
-
-Currently, "base" is 2KB. So for example, if blocks X and Y start in
-the range [ 0KB .. 2KB-1 ], all of the keys in X and Y will be
-converted to a filter by calling FilterPolicy::CreateFilter(), and the
-resulting filter will be stored as the first filter in the filter
-block.
-
-The filter block is formatted as follows:
-
- [filter 0]
- [filter 1]
- [filter 2]
- ...
- [filter N-1]
-
- [offset of filter 0] : 4 bytes
- [offset of filter 1] : 4 bytes
- [offset of filter 2] : 4 bytes
- ...
- [offset of filter N-1] : 4 bytes
-
- [offset of beginning of offset array] : 4 bytes
- lg(base) : 1 byte
-
-The offset array at the end of the filter block allows efficient
-mapping from a data block offset to the corresponding filter.
-
-"stats" Meta Block
-------------------
-
-This meta block contains a bunch of stats. The key is the name
-of the statistic. The value contains the statistic.
-TODO(postrelease): record following stats.
- data size
- index size
- key size (uncompressed)
- value size (uncompressed)
- number of entries
- number of data blocks
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/helpers/memenv/memenv.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/helpers/memenv/memenv.cc b/thirdparty/leveldb-1.18/helpers/memenv/memenv.cc
deleted file mode 100755
index 43ef2e0..0000000
--- a/thirdparty/leveldb-1.18/helpers/memenv/memenv.cc
+++ /dev/null
@@ -1,385 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "helpers/memenv/memenv.h"
-
-#include "leveldb/env.h"
-#include "leveldb/status.h"
-#include "port/port.h"
-#include "util/mutexlock.h"
-#include <map>
-#include <string.h>
-#include <string>
-#include <vector>
-
-namespace leveldb {
-
-namespace {
-
-class FileState {
- public:
- // FileStates are reference counted. The initial reference count is zero
- // and the caller must call Ref() at least once.
- FileState() : refs_(0), size_(0) {}
-
- // Increase the reference count.
- void Ref() {
- MutexLock lock(&refs_mutex_);
- ++refs_;
- }
-
- // Decrease the reference count. Delete if this is the last reference.
- void Unref() {
- bool do_delete = false;
-
- {
- MutexLock lock(&refs_mutex_);
- --refs_;
- assert(refs_ >= 0);
- if (refs_ <= 0) {
- do_delete = true;
- }
- }
-
- if (do_delete) {
- delete this;
- }
- }
-
- uint64_t Size() const { return size_; }
-
- Status Read(uint64_t offset, size_t n, Slice* result, char* scratch) const {
- if (offset > size_) {
- return Status::IOError("Offset greater than file size.");
- }
- const uint64_t available = size_ - offset;
- if (n > available) {
- n = static_cast<size_t>(available);
- }
- if (n == 0) {
- *result = Slice();
- return Status::OK();
- }
-
- assert(offset / kBlockSize <= SIZE_MAX);
- size_t block = static_cast<size_t>(offset / kBlockSize);
- size_t block_offset = offset % kBlockSize;
-
- if (n <= kBlockSize - block_offset) {
- // The requested bytes are all in the first block.
- *result = Slice(blocks_[block] + block_offset, n);
- return Status::OK();
- }
-
- size_t bytes_to_copy = n;
- char* dst = scratch;
-
- while (bytes_to_copy > 0) {
- size_t avail = kBlockSize - block_offset;
- if (avail > bytes_to_copy) {
- avail = bytes_to_copy;
- }
- memcpy(dst, blocks_[block] + block_offset, avail);
-
- bytes_to_copy -= avail;
- dst += avail;
- block++;
- block_offset = 0;
- }
-
- *result = Slice(scratch, n);
- return Status::OK();
- }
-
- Status Append(const Slice& data) {
- const char* src = data.data();
- size_t src_len = data.size();
-
- while (src_len > 0) {
- size_t avail;
- size_t offset = size_ % kBlockSize;
-
- if (offset != 0) {
- // There is some room in the last block.
- avail = kBlockSize - offset;
- } else {
- // No room in the last block; push new one.
- blocks_.push_back(new char[kBlockSize]);
- avail = kBlockSize;
- }
-
- if (avail > src_len) {
- avail = src_len;
- }
- memcpy(blocks_.back() + offset, src, avail);
- src_len -= avail;
- src += avail;
- size_ += avail;
- }
-
- return Status::OK();
- }
-
- private:
- // Private since only Unref() should be used to delete it.
- ~FileState() {
- for (std::vector<char*>::iterator i = blocks_.begin(); i != blocks_.end();
- ++i) {
- delete [] *i;
- }
- }
-
- // No copying allowed.
- FileState(const FileState&);
- void operator=(const FileState&);
-
- port::Mutex refs_mutex_;
- int refs_; // Protected by refs_mutex_;
-
- // The following fields are not protected by any mutex. They are only mutable
- // while the file is being written, and concurrent access is not allowed
- // to writable files.
- std::vector<char*> blocks_;
- uint64_t size_;
-
- enum { kBlockSize = 8 * 1024 };
-};
-
-class SequentialFileImpl : public SequentialFile {
- public:
- explicit SequentialFileImpl(FileState* file) : file_(file), pos_(0) {
- file_->Ref();
- }
-
- ~SequentialFileImpl() {
- file_->Unref();
- }
-
- virtual Status Read(size_t n, Slice* result, char* scratch) {
- Status s = file_->Read(pos_, n, result, scratch);
- if (s.ok()) {
- pos_ += result->size();
- }
- return s;
- }
-
- virtual Status Skip(uint64_t n) {
- if (pos_ > file_->Size()) {
- return Status::IOError("pos_ > file_->Size()");
- }
- const uint64_t available = file_->Size() - pos_;
- if (n > available) {
- n = available;
- }
- pos_ += n;
- return Status::OK();
- }
-
- private:
- FileState* file_;
- uint64_t pos_;
-};
-
-class RandomAccessFileImpl : public RandomAccessFile {
- public:
- explicit RandomAccessFileImpl(FileState* file) : file_(file) {
- file_->Ref();
- }
-
- ~RandomAccessFileImpl() {
- file_->Unref();
- }
-
- virtual Status Read(uint64_t offset, size_t n, Slice* result,
- char* scratch) const {
- return file_->Read(offset, n, result, scratch);
- }
-
- private:
- FileState* file_;
-};
-
-class WritableFileImpl : public WritableFile {
- public:
- WritableFileImpl(FileState* file) : file_(file) {
- file_->Ref();
- }
-
- ~WritableFileImpl() {
- file_->Unref();
- }
-
- virtual Status Append(const Slice& data) {
- return file_->Append(data);
- }
-
- virtual Status Close() { return Status::OK(); }
- virtual Status Flush() { return Status::OK(); }
- virtual Status Sync() { return Status::OK(); }
-
- private:
- FileState* file_;
-};
-
-class NoOpLogger : public Logger {
- public:
- virtual void Logv(const char* format, va_list ap) { }
-};
-
-class InMemoryEnv : public EnvWrapper {
- public:
- explicit InMemoryEnv(Env* base_env) : EnvWrapper(base_env) { }
-
- virtual ~InMemoryEnv() {
- for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
- i->second->Unref();
- }
- }
-
- // Partial implementation of the Env interface.
- virtual Status NewSequentialFile(const std::string& fname,
- SequentialFile** result) {
- MutexLock lock(&mutex_);
- if (file_map_.find(fname) == file_map_.end()) {
- *result = NULL;
- return Status::IOError(fname, "File not found");
- }
-
- *result = new SequentialFileImpl(file_map_[fname]);
- return Status::OK();
- }
-
- virtual Status NewRandomAccessFile(const std::string& fname,
- RandomAccessFile** result) {
- MutexLock lock(&mutex_);
- if (file_map_.find(fname) == file_map_.end()) {
- *result = NULL;
- return Status::IOError(fname, "File not found");
- }
-
- *result = new RandomAccessFileImpl(file_map_[fname]);
- return Status::OK();
- }
-
- virtual Status NewWritableFile(const std::string& fname,
- WritableFile** result) {
- MutexLock lock(&mutex_);
- if (file_map_.find(fname) != file_map_.end()) {
- DeleteFileInternal(fname);
- }
-
- FileState* file = new FileState();
- file->Ref();
- file_map_[fname] = file;
-
- *result = new WritableFileImpl(file);
- return Status::OK();
- }
-
- virtual bool FileExists(const std::string& fname) {
- MutexLock lock(&mutex_);
- return file_map_.find(fname) != file_map_.end();
- }
-
- virtual Status GetChildren(const std::string& dir,
- std::vector<std::string>* result) {
- MutexLock lock(&mutex_);
- result->clear();
-
- for (FileSystem::iterator i = file_map_.begin(); i != file_map_.end(); ++i){
- const std::string& filename = i->first;
-
- if (filename.size() >= dir.size() + 1 && filename[dir.size()] == '/' &&
- Slice(filename).starts_with(Slice(dir))) {
- result->push_back(filename.substr(dir.size() + 1));
- }
- }
-
- return Status::OK();
- }
-
- void DeleteFileInternal(const std::string& fname) {
- if (file_map_.find(fname) == file_map_.end()) {
- return;
- }
-
- file_map_[fname]->Unref();
- file_map_.erase(fname);
- }
-
- virtual Status DeleteFile(const std::string& fname) {
- MutexLock lock(&mutex_);
- if (file_map_.find(fname) == file_map_.end()) {
- return Status::IOError(fname, "File not found");
- }
-
- DeleteFileInternal(fname);
- return Status::OK();
- }
-
- virtual Status CreateDir(const std::string& dirname) {
- return Status::OK();
- }
-
- virtual Status DeleteDir(const std::string& dirname) {
- return Status::OK();
- }
-
- virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) {
- MutexLock lock(&mutex_);
- if (file_map_.find(fname) == file_map_.end()) {
- return Status::IOError(fname, "File not found");
- }
-
- *file_size = file_map_[fname]->Size();
- return Status::OK();
- }
-
- virtual Status RenameFile(const std::string& src,
- const std::string& target) {
- MutexLock lock(&mutex_);
- if (file_map_.find(src) == file_map_.end()) {
- return Status::IOError(src, "File not found");
- }
-
- DeleteFileInternal(target);
- file_map_[target] = file_map_[src];
- file_map_.erase(src);
- return Status::OK();
- }
-
- virtual Status LockFile(const std::string& fname, FileLock** lock) {
- *lock = new FileLock;
- return Status::OK();
- }
-
- virtual Status UnlockFile(FileLock* lock) {
- delete lock;
- return Status::OK();
- }
-
- virtual Status GetTestDirectory(std::string* path) {
- *path = "/test";
- return Status::OK();
- }
-
- virtual Status NewLogger(const std::string& fname, Logger** result) {
- *result = new NoOpLogger;
- return Status::OK();
- }
-
- private:
- // Map from filenames to FileState objects, representing a simple file system.
- typedef std::map<std::string, FileState*> FileSystem;
- port::Mutex mutex_;
- FileSystem file_map_; // Protected by mutex_.
-};
-
-} // namespace
-
-Env* NewMemEnv(Env* base_env) {
- return new InMemoryEnv(base_env);
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/helpers/memenv/memenv.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/helpers/memenv/memenv.h b/thirdparty/leveldb-1.18/helpers/memenv/memenv.h
deleted file mode 100755
index 03b88de..0000000
--- a/thirdparty/leveldb-1.18/helpers/memenv/memenv.h
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
-#define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
-
-namespace leveldb {
-
-class Env;
-
-// Returns a new environment that stores its data in memory and delegates
-// all non-file-storage tasks to base_env. The caller must delete the result
-// when it is no longer needed.
-// *base_env must remain live while the result is in use.
-Env* NewMemEnv(Env* base_env);
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/helpers/memenv/memenv_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/helpers/memenv/memenv_test.cc b/thirdparty/leveldb-1.18/helpers/memenv/memenv_test.cc
deleted file mode 100755
index a44310f..0000000
--- a/thirdparty/leveldb-1.18/helpers/memenv/memenv_test.cc
+++ /dev/null
@@ -1,232 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "helpers/memenv/memenv.h"
-
-#include "db/db_impl.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "util/testharness.h"
-#include <string>
-#include <vector>
-
-namespace leveldb {
-
-class MemEnvTest {
- public:
- Env* env_;
-
- MemEnvTest()
- : env_(NewMemEnv(Env::Default())) {
- }
- ~MemEnvTest() {
- delete env_;
- }
-};
-
-TEST(MemEnvTest, Basics) {
- uint64_t file_size;
- WritableFile* writable_file;
- std::vector<std::string> children;
-
- ASSERT_OK(env_->CreateDir("/dir"));
-
- // Check that the directory is empty.
- ASSERT_TRUE(!env_->FileExists("/dir/non_existent"));
- ASSERT_TRUE(!env_->GetFileSize("/dir/non_existent", &file_size).ok());
- ASSERT_OK(env_->GetChildren("/dir", &children));
- ASSERT_EQ(0, children.size());
-
- // Create a file.
- ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
- delete writable_file;
-
- // Check that the file exists.
- ASSERT_TRUE(env_->FileExists("/dir/f"));
- ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
- ASSERT_EQ(0, file_size);
- ASSERT_OK(env_->GetChildren("/dir", &children));
- ASSERT_EQ(1, children.size());
- ASSERT_EQ("f", children[0]);
-
- // Write to the file.
- ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
- ASSERT_OK(writable_file->Append("abc"));
- delete writable_file;
-
- // Check for expected size.
- ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
- ASSERT_EQ(3, file_size);
-
- // Check that renaming works.
- ASSERT_TRUE(!env_->RenameFile("/dir/non_existent", "/dir/g").ok());
- ASSERT_OK(env_->RenameFile("/dir/f", "/dir/g"));
- ASSERT_TRUE(!env_->FileExists("/dir/f"));
- ASSERT_TRUE(env_->FileExists("/dir/g"));
- ASSERT_OK(env_->GetFileSize("/dir/g", &file_size));
- ASSERT_EQ(3, file_size);
-
- // Check that opening non-existent file fails.
- SequentialFile* seq_file;
- RandomAccessFile* rand_file;
- ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file).ok());
- ASSERT_TRUE(!seq_file);
- ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file).ok());
- ASSERT_TRUE(!rand_file);
-
- // Check that deleting works.
- ASSERT_TRUE(!env_->DeleteFile("/dir/non_existent").ok());
- ASSERT_OK(env_->DeleteFile("/dir/g"));
- ASSERT_TRUE(!env_->FileExists("/dir/g"));
- ASSERT_OK(env_->GetChildren("/dir", &children));
- ASSERT_EQ(0, children.size());
- ASSERT_OK(env_->DeleteDir("/dir"));
-}
-
-TEST(MemEnvTest, ReadWrite) {
- WritableFile* writable_file;
- SequentialFile* seq_file;
- RandomAccessFile* rand_file;
- Slice result;
- char scratch[100];
-
- ASSERT_OK(env_->CreateDir("/dir"));
-
- ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
- ASSERT_OK(writable_file->Append("hello "));
- ASSERT_OK(writable_file->Append("world"));
- delete writable_file;
-
- // Read sequentially.
- ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
- ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
- ASSERT_EQ(0, result.compare("hello"));
- ASSERT_OK(seq_file->Skip(1));
- ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Read "world".
- ASSERT_EQ(0, result.compare("world"));
- ASSERT_OK(seq_file->Read(1000, &result, scratch)); // Try reading past EOF.
- ASSERT_EQ(0, result.size());
- ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file.
- ASSERT_OK(seq_file->Read(1000, &result, scratch));
- ASSERT_EQ(0, result.size());
- delete seq_file;
-
- // Random reads.
- ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file));
- ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
- ASSERT_EQ(0, result.compare("world"));
- ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
- ASSERT_EQ(0, result.compare("hello"));
- ASSERT_OK(rand_file->Read(10, 100, &result, scratch)); // Read "d".
- ASSERT_EQ(0, result.compare("d"));
-
- // Too high offset.
- ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok());
- delete rand_file;
-}
-
-TEST(MemEnvTest, Locks) {
- FileLock* lock;
-
- // These are no-ops, but we test they return success.
- ASSERT_OK(env_->LockFile("some file", &lock));
- ASSERT_OK(env_->UnlockFile(lock));
-}
-
-TEST(MemEnvTest, Misc) {
- std::string test_dir;
- ASSERT_OK(env_->GetTestDirectory(&test_dir));
- ASSERT_TRUE(!test_dir.empty());
-
- WritableFile* writable_file;
- ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file));
-
- // These are no-ops, but we test they return success.
- ASSERT_OK(writable_file->Sync());
- ASSERT_OK(writable_file->Flush());
- ASSERT_OK(writable_file->Close());
- delete writable_file;
-}
-
-TEST(MemEnvTest, LargeWrite) {
- const size_t kWriteSize = 300 * 1024;
- char* scratch = new char[kWriteSize * 2];
-
- std::string write_data;
- for (size_t i = 0; i < kWriteSize; ++i) {
- write_data.append(1, static_cast<char>(i));
- }
-
- WritableFile* writable_file;
- ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
- ASSERT_OK(writable_file->Append("foo"));
- ASSERT_OK(writable_file->Append(write_data));
- delete writable_file;
-
- SequentialFile* seq_file;
- Slice result;
- ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
- ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
- ASSERT_EQ(0, result.compare("foo"));
-
- size_t read = 0;
- std::string read_data;
- while (read < kWriteSize) {
- ASSERT_OK(seq_file->Read(kWriteSize - read, &result, scratch));
- read_data.append(result.data(), result.size());
- read += result.size();
- }
- ASSERT_TRUE(write_data == read_data);
- delete seq_file;
- delete [] scratch;
-}
-
-TEST(MemEnvTest, DBTest) {
- Options options;
- options.create_if_missing = true;
- options.env = env_;
- DB* db;
-
- const Slice keys[] = {Slice("aaa"), Slice("bbb"), Slice("ccc")};
- const Slice vals[] = {Slice("foo"), Slice("bar"), Slice("baz")};
-
- ASSERT_OK(DB::Open(options, "/dir/db", &db));
- for (size_t i = 0; i < 3; ++i) {
- ASSERT_OK(db->Put(WriteOptions(), keys[i], vals[i]));
- }
-
- for (size_t i = 0; i < 3; ++i) {
- std::string res;
- ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
- ASSERT_TRUE(res == vals[i]);
- }
-
- Iterator* iterator = db->NewIterator(ReadOptions());
- iterator->SeekToFirst();
- for (size_t i = 0; i < 3; ++i) {
- ASSERT_TRUE(iterator->Valid());
- ASSERT_TRUE(keys[i] == iterator->key());
- ASSERT_TRUE(vals[i] == iterator->value());
- iterator->Next();
- }
- ASSERT_TRUE(!iterator->Valid());
- delete iterator;
-
- DBImpl* dbi = reinterpret_cast<DBImpl*>(db);
- ASSERT_OK(dbi->TEST_CompactMemTable());
-
- for (size_t i = 0; i < 3; ++i) {
- std::string res;
- ASSERT_OK(db->Get(ReadOptions(), keys[i], &res));
- ASSERT_TRUE(res == vals[i]);
- }
-
- delete db;
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/c.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/c.h b/thirdparty/leveldb-1.18/include/leveldb/c.h
deleted file mode 100755
index 1048fe3..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/c.h
+++ /dev/null
@@ -1,290 +0,0 @@
-/* Copyright (c) 2011 The LevelDB Authors. All rights reserved.
- Use of this source code is governed by a BSD-style license that can be
- found in the LICENSE file. See the AUTHORS file for names of contributors.
-
- C bindings for leveldb. May be useful as a stable ABI that can be
- used by programs that keep leveldb in a shared library, or for
- a JNI api.
-
- Does not support:
- . getters for the option types
- . custom comparators that implement key shortening
- . custom iter, db, env, cache implementations using just the C bindings
-
- Some conventions:
-
- (1) We expose just opaque struct pointers and functions to clients.
- This allows us to change internal representations without having to
- recompile clients.
-
- (2) For simplicity, there is no equivalent to the Slice type. Instead,
- the caller has to pass the pointer and length as separate
- arguments.
-
- (3) Errors are represented by a null-terminated c string. NULL
- means no error. All operations that can raise an error are passed
- a "char** errptr" as the last argument. One of the following must
- be true on entry:
- *errptr == NULL
- *errptr points to a malloc()ed null-terminated error message
- (On Windows, *errptr must have been malloc()-ed by this library.)
- On success, a leveldb routine leaves *errptr unchanged.
- On failure, leveldb frees the old value of *errptr and
- set *errptr to a malloc()ed error message.
-
- (4) Bools have the type unsigned char (0 == false; rest == true)
-
- (5) All of the pointer arguments must be non-NULL.
-*/
-
-#ifndef STORAGE_LEVELDB_INCLUDE_C_H_
-#define STORAGE_LEVELDB_INCLUDE_C_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdarg.h>
-#include <stddef.h>
-#include <stdint.h>
-
-/* Exported types */
-
-typedef struct leveldb_t leveldb_t;
-typedef struct leveldb_cache_t leveldb_cache_t;
-typedef struct leveldb_comparator_t leveldb_comparator_t;
-typedef struct leveldb_env_t leveldb_env_t;
-typedef struct leveldb_filelock_t leveldb_filelock_t;
-typedef struct leveldb_filterpolicy_t leveldb_filterpolicy_t;
-typedef struct leveldb_iterator_t leveldb_iterator_t;
-typedef struct leveldb_logger_t leveldb_logger_t;
-typedef struct leveldb_options_t leveldb_options_t;
-typedef struct leveldb_randomfile_t leveldb_randomfile_t;
-typedef struct leveldb_readoptions_t leveldb_readoptions_t;
-typedef struct leveldb_seqfile_t leveldb_seqfile_t;
-typedef struct leveldb_snapshot_t leveldb_snapshot_t;
-typedef struct leveldb_writablefile_t leveldb_writablefile_t;
-typedef struct leveldb_writebatch_t leveldb_writebatch_t;
-typedef struct leveldb_writeoptions_t leveldb_writeoptions_t;
-
-/* DB operations */
-
-extern leveldb_t* leveldb_open(
- const leveldb_options_t* options,
- const char* name,
- char** errptr);
-
-extern void leveldb_close(leveldb_t* db);
-
-extern void leveldb_put(
- leveldb_t* db,
- const leveldb_writeoptions_t* options,
- const char* key, size_t keylen,
- const char* val, size_t vallen,
- char** errptr);
-
-extern void leveldb_delete(
- leveldb_t* db,
- const leveldb_writeoptions_t* options,
- const char* key, size_t keylen,
- char** errptr);
-
-extern void leveldb_write(
- leveldb_t* db,
- const leveldb_writeoptions_t* options,
- leveldb_writebatch_t* batch,
- char** errptr);
-
-/* Returns NULL if not found. A malloc()ed array otherwise.
- Stores the length of the array in *vallen. */
-extern char* leveldb_get(
- leveldb_t* db,
- const leveldb_readoptions_t* options,
- const char* key, size_t keylen,
- size_t* vallen,
- char** errptr);
-
-extern leveldb_iterator_t* leveldb_create_iterator(
- leveldb_t* db,
- const leveldb_readoptions_t* options);
-
-extern const leveldb_snapshot_t* leveldb_create_snapshot(
- leveldb_t* db);
-
-extern void leveldb_release_snapshot(
- leveldb_t* db,
- const leveldb_snapshot_t* snapshot);
-
-/* Returns NULL if property name is unknown.
- Else returns a pointer to a malloc()-ed null-terminated value. */
-extern char* leveldb_property_value(
- leveldb_t* db,
- const char* propname);
-
-extern void leveldb_approximate_sizes(
- leveldb_t* db,
- int num_ranges,
- const char* const* range_start_key, const size_t* range_start_key_len,
- const char* const* range_limit_key, const size_t* range_limit_key_len,
- uint64_t* sizes);
-
-extern void leveldb_compact_range(
- leveldb_t* db,
- const char* start_key, size_t start_key_len,
- const char* limit_key, size_t limit_key_len);
-
-/* Management operations */
-
-extern void leveldb_destroy_db(
- const leveldb_options_t* options,
- const char* name,
- char** errptr);
-
-extern void leveldb_repair_db(
- const leveldb_options_t* options,
- const char* name,
- char** errptr);
-
-/* Iterator */
-
-extern void leveldb_iter_destroy(leveldb_iterator_t*);
-extern unsigned char leveldb_iter_valid(const leveldb_iterator_t*);
-extern void leveldb_iter_seek_to_first(leveldb_iterator_t*);
-extern void leveldb_iter_seek_to_last(leveldb_iterator_t*);
-extern void leveldb_iter_seek(leveldb_iterator_t*, const char* k, size_t klen);
-extern void leveldb_iter_next(leveldb_iterator_t*);
-extern void leveldb_iter_prev(leveldb_iterator_t*);
-extern const char* leveldb_iter_key(const leveldb_iterator_t*, size_t* klen);
-extern const char* leveldb_iter_value(const leveldb_iterator_t*, size_t* vlen);
-extern void leveldb_iter_get_error(const leveldb_iterator_t*, char** errptr);
-
-/* Write batch */
-
-extern leveldb_writebatch_t* leveldb_writebatch_create();
-extern void leveldb_writebatch_destroy(leveldb_writebatch_t*);
-extern void leveldb_writebatch_clear(leveldb_writebatch_t*);
-extern void leveldb_writebatch_put(
- leveldb_writebatch_t*,
- const char* key, size_t klen,
- const char* val, size_t vlen);
-extern void leveldb_writebatch_delete(
- leveldb_writebatch_t*,
- const char* key, size_t klen);
-extern void leveldb_writebatch_iterate(
- leveldb_writebatch_t*,
- void* state,
- void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
- void (*deleted)(void*, const char* k, size_t klen));
-
-/* Options */
-
-extern leveldb_options_t* leveldb_options_create();
-extern void leveldb_options_destroy(leveldb_options_t*);
-extern void leveldb_options_set_comparator(
- leveldb_options_t*,
- leveldb_comparator_t*);
-extern void leveldb_options_set_filter_policy(
- leveldb_options_t*,
- leveldb_filterpolicy_t*);
-extern void leveldb_options_set_create_if_missing(
- leveldb_options_t*, unsigned char);
-extern void leveldb_options_set_error_if_exists(
- leveldb_options_t*, unsigned char);
-extern void leveldb_options_set_paranoid_checks(
- leveldb_options_t*, unsigned char);
-extern void leveldb_options_set_env(leveldb_options_t*, leveldb_env_t*);
-extern void leveldb_options_set_info_log(leveldb_options_t*, leveldb_logger_t*);
-extern void leveldb_options_set_write_buffer_size(leveldb_options_t*, size_t);
-extern void leveldb_options_set_max_open_files(leveldb_options_t*, int);
-extern void leveldb_options_set_cache(leveldb_options_t*, leveldb_cache_t*);
-extern void leveldb_options_set_block_size(leveldb_options_t*, size_t);
-extern void leveldb_options_set_block_restart_interval(leveldb_options_t*, int);
-
-enum {
- leveldb_no_compression = 0,
- leveldb_snappy_compression = 1
-};
-extern void leveldb_options_set_compression(leveldb_options_t*, int);
-
-/* Comparator */
-
-extern leveldb_comparator_t* leveldb_comparator_create(
- void* state,
- void (*destructor)(void*),
- int (*compare)(
- void*,
- const char* a, size_t alen,
- const char* b, size_t blen),
- const char* (*name)(void*));
-extern void leveldb_comparator_destroy(leveldb_comparator_t*);
-
-/* Filter policy */
-
-extern leveldb_filterpolicy_t* leveldb_filterpolicy_create(
- void* state,
- void (*destructor)(void*),
- char* (*create_filter)(
- void*,
- const char* const* key_array, const size_t* key_length_array,
- int num_keys,
- size_t* filter_length),
- unsigned char (*key_may_match)(
- void*,
- const char* key, size_t length,
- const char* filter, size_t filter_length),
- const char* (*name)(void*));
-extern void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t*);
-
-extern leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(
- int bits_per_key);
-
-/* Read options */
-
-extern leveldb_readoptions_t* leveldb_readoptions_create();
-extern void leveldb_readoptions_destroy(leveldb_readoptions_t*);
-extern void leveldb_readoptions_set_verify_checksums(
- leveldb_readoptions_t*,
- unsigned char);
-extern void leveldb_readoptions_set_fill_cache(
- leveldb_readoptions_t*, unsigned char);
-extern void leveldb_readoptions_set_snapshot(
- leveldb_readoptions_t*,
- const leveldb_snapshot_t*);
-
-/* Write options */
-
-extern leveldb_writeoptions_t* leveldb_writeoptions_create();
-extern void leveldb_writeoptions_destroy(leveldb_writeoptions_t*);
-extern void leveldb_writeoptions_set_sync(
- leveldb_writeoptions_t*, unsigned char);
-
-/* Cache */
-
-extern leveldb_cache_t* leveldb_cache_create_lru(size_t capacity);
-extern void leveldb_cache_destroy(leveldb_cache_t* cache);
-
-/* Env */
-
-extern leveldb_env_t* leveldb_create_default_env();
-extern void leveldb_env_destroy(leveldb_env_t*);
-
-/* Utility */
-
-/* Calls free(ptr).
- REQUIRES: ptr was malloc()-ed and returned by one of the routines
- in this file. Note that in certain cases (typically on Windows), you
- may need to call this routine instead of free(ptr) to dispose of
- malloc()-ed memory returned by this library. */
-extern void leveldb_free(void* ptr);
-
-/* Return the major version number for this release. */
-extern int leveldb_major_version();
-
-/* Return the minor version number for this release. */
-extern int leveldb_minor_version();
-
-#ifdef __cplusplus
-} /* end extern "C" */
-#endif
-
-#endif /* STORAGE_LEVELDB_INCLUDE_C_H_ */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/cache.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/cache.h b/thirdparty/leveldb-1.18/include/leveldb/cache.h
deleted file mode 100755
index 1a201e5..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/cache.h
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A Cache is an interface that maps keys to values. It has internal
-// synchronization and may be safely accessed concurrently from
-// multiple threads. It may automatically evict entries to make room
-// for new entries. Values have a specified charge against the cache
-// capacity. For example, a cache where the values are variable
-// length strings, may use the length of the string as the charge for
-// the string.
-//
-// A builtin cache implementation with a least-recently-used eviction
-// policy is provided. Clients may use their own implementations if
-// they want something more sophisticated (like scan-resistance, a
-// custom eviction policy, variable cache sizing, etc.)
-
-#ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_
-#define STORAGE_LEVELDB_INCLUDE_CACHE_H_
-
-#include <stdint.h>
-#include "leveldb/slice.h"
-
-namespace leveldb {
-
-class Cache;
-
-// Create a new cache with a fixed size capacity. This implementation
-// of Cache uses a least-recently-used eviction policy.
-extern Cache* NewLRUCache(size_t capacity);
-
-class Cache {
- public:
- Cache() { }
-
- // Destroys all existing entries by calling the "deleter"
- // function that was passed to the constructor.
- virtual ~Cache();
-
- // Opaque handle to an entry stored in the cache.
- struct Handle { };
-
- // Insert a mapping from key->value into the cache and assign it
- // the specified charge against the total cache capacity.
- //
- // Returns a handle that corresponds to the mapping. The caller
- // must call this->Release(handle) when the returned mapping is no
- // longer needed.
- //
- // When the inserted entry is no longer needed, the key and
- // value will be passed to "deleter".
- virtual Handle* Insert(const Slice& key, void* value, size_t charge,
- void (*deleter)(const Slice& key, void* value)) = 0;
-
- // If the cache has no mapping for "key", returns NULL.
- //
- // Else return a handle that corresponds to the mapping. The caller
- // must call this->Release(handle) when the returned mapping is no
- // longer needed.
- virtual Handle* Lookup(const Slice& key) = 0;
-
- // Release a mapping returned by a previous Lookup().
- // REQUIRES: handle must not have been released yet.
- // REQUIRES: handle must have been returned by a method on *this.
- virtual void Release(Handle* handle) = 0;
-
- // Return the value encapsulated in a handle returned by a
- // successful Lookup().
- // REQUIRES: handle must not have been released yet.
- // REQUIRES: handle must have been returned by a method on *this.
- virtual void* Value(Handle* handle) = 0;
-
- // If the cache contains entry for key, erase it. Note that the
- // underlying entry will be kept around until all existing handles
- // to it have been released.
- virtual void Erase(const Slice& key) = 0;
-
- // Return a new numeric id. May be used by multiple clients who are
- // sharing the same cache to partition the key space. Typically the
- // client will allocate a new id at startup and prepend the id to
- // its cache keys.
- virtual uint64_t NewId() = 0;
-
- private:
- void LRU_Remove(Handle* e);
- void LRU_Append(Handle* e);
- void Unref(Handle* e);
-
- struct Rep;
- Rep* rep_;
-
- // No copying allowed
- Cache(const Cache&);
- void operator=(const Cache&);
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_INCLUDE_CACHE_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/comparator.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/comparator.h b/thirdparty/leveldb-1.18/include/leveldb/comparator.h
deleted file mode 100755
index a8d43ae..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/comparator.h
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
-#define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
-
-#include <string>
-
-namespace leveldb {
-
-class Slice;
-
-// A Comparator object provides a total order across slices that are
-// used as keys in an sstable or a database. A Comparator implementation
-// must be thread-safe since leveldb may invoke its methods concurrently
-// from multiple threads.
-class Comparator {
- public:
- virtual ~Comparator();
-
- // Three-way comparison. Returns value:
- // < 0 iff "a" < "b",
- // == 0 iff "a" == "b",
- // > 0 iff "a" > "b"
- virtual int Compare(const Slice& a, const Slice& b) const = 0;
-
- // The name of the comparator. Used to check for comparator
- // mismatches (i.e., a DB created with one comparator is
- // accessed using a different comparator.
- //
- // The client of this package should switch to a new name whenever
- // the comparator implementation changes in a way that will cause
- // the relative ordering of any two keys to change.
- //
- // Names starting with "leveldb." are reserved and should not be used
- // by any clients of this package.
- virtual const char* Name() const = 0;
-
- // Advanced functions: these are used to reduce the space requirements
- // for internal data structures like index blocks.
-
- // If *start < limit, changes *start to a short string in [start,limit).
- // Simple comparator implementations may return with *start unchanged,
- // i.e., an implementation of this method that does nothing is correct.
- virtual void FindShortestSeparator(
- std::string* start,
- const Slice& limit) const = 0;
-
- // Changes *key to a short string >= *key.
- // Simple comparator implementations may return with *key unchanged,
- // i.e., an implementation of this method that does nothing is correct.
- virtual void FindShortSuccessor(std::string* key) const = 0;
-};
-
-// Return a builtin comparator that uses lexicographic byte-wise
-// ordering. The result remains the property of this module and
-// must not be deleted.
-extern const Comparator* BytewiseComparator();
-
-// memory leaks hunting
-extern void UnsafeDeallocateBytewiseComparator();
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/db.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/db.h b/thirdparty/leveldb-1.18/include/leveldb/db.h
deleted file mode 100755
index 4c169bf..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/db.h
+++ /dev/null
@@ -1,161 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_DB_H_
-#define STORAGE_LEVELDB_INCLUDE_DB_H_
-
-#include <stdint.h>
-#include <stdio.h>
-#include "leveldb/iterator.h"
-#include "leveldb/options.h"
-
-namespace leveldb {
-
-// Update Makefile if you change these
-static const int kMajorVersion = 1;
-static const int kMinorVersion = 18;
-
-struct Options;
-struct ReadOptions;
-struct WriteOptions;
-class WriteBatch;
-
-// Abstract handle to particular state of a DB.
-// A Snapshot is an immutable object and can therefore be safely
-// accessed from multiple threads without any external synchronization.
-class Snapshot {
- protected:
- virtual ~Snapshot();
-};
-
-// A range of keys
-struct Range {
- Slice start; // Included in the range
- Slice limit; // Not included in the range
-
- Range() { }
- Range(const Slice& s, const Slice& l) : start(s), limit(l) { }
-};
-
-// A DB is a persistent ordered map from keys to values.
-// A DB is safe for concurrent access from multiple threads without
-// any external synchronization.
-class DB {
- public:
- // Open the database with the specified "name".
- // Stores a pointer to a heap-allocated database in *dbptr and returns
- // OK on success.
- // Stores NULL in *dbptr and returns a non-OK status on error.
- // Caller should delete *dbptr when it is no longer needed.
- static Status Open(const Options& options,
- const std::string& name,
- DB** dbptr);
-
- DB() { }
- virtual ~DB();
-
- // Set the database entry for "key" to "value". Returns OK on success,
- // and a non-OK status on error.
- // Note: consider setting options.sync = true.
- virtual Status Put(const WriteOptions& options,
- const Slice& key,
- const Slice& value) = 0;
-
- // Remove the database entry (if any) for "key". Returns OK on
- // success, and a non-OK status on error. It is not an error if "key"
- // did not exist in the database.
- // Note: consider setting options.sync = true.
- virtual Status Delete(const WriteOptions& options, const Slice& key) = 0;
-
- // Apply the specified updates to the database.
- // Returns OK on success, non-OK on failure.
- // Note: consider setting options.sync = true.
- virtual Status Write(const WriteOptions& options, WriteBatch* updates) = 0;
-
- // If the database contains an entry for "key" store the
- // corresponding value in *value and return OK.
- //
- // If there is no entry for "key" leave *value unchanged and return
- // a status for which Status::IsNotFound() returns true.
- //
- // May return some other Status on an error.
- virtual Status Get(const ReadOptions& options,
- const Slice& key, std::string* value) = 0;
-
- // Return a heap-allocated iterator over the contents of the database.
- // The result of NewIterator() is initially invalid (caller must
- // call one of the Seek methods on the iterator before using it).
- //
- // Caller should delete the iterator when it is no longer needed.
- // The returned iterator should be deleted before this db is deleted.
- virtual Iterator* NewIterator(const ReadOptions& options) = 0;
-
- // Return a handle to the current DB state. Iterators created with
- // this handle will all observe a stable snapshot of the current DB
- // state. The caller must call ReleaseSnapshot(result) when the
- // snapshot is no longer needed.
- virtual const Snapshot* GetSnapshot() = 0;
-
- // Release a previously acquired snapshot. The caller must not
- // use "snapshot" after this call.
- virtual void ReleaseSnapshot(const Snapshot* snapshot) = 0;
-
- // DB implementations can export properties about their state
- // via this method. If "property" is a valid property understood by this
- // DB implementation, fills "*value" with its current value and returns
- // true. Otherwise returns false.
- //
- //
- // Valid property names include:
- //
- // "leveldb.num-files-at-level<N>" - return the number of files at level <N>,
- // where <N> is an ASCII representation of a level number (e.g. "0").
- // "leveldb.stats" - returns a multi-line string that describes statistics
- // about the internal operation of the DB.
- // "leveldb.sstables" - returns a multi-line string that describes all
- // of the sstables that make up the db contents.
- virtual bool GetProperty(const Slice& property, std::string* value) = 0;
-
- // For each i in [0,n-1], store in "sizes[i]", the approximate
- // file system space used by keys in "[range[i].start .. range[i].limit)".
- //
- // Note that the returned sizes measure file system space usage, so
- // if the user data compresses by a factor of ten, the returned
- // sizes will be one-tenth the size of the corresponding user data size.
- //
- // The results may not include the sizes of recently written data.
- virtual void GetApproximateSizes(const Range* range, int n,
- uint64_t* sizes) = 0;
-
- // Compact the underlying storage for the key range [*begin,*end].
- // In particular, deleted and overwritten versions are discarded,
- // and the data is rearranged to reduce the cost of operations
- // needed to access the data. This operation should typically only
- // be invoked by users who understand the underlying implementation.
- //
- // begin==NULL is treated as a key before all keys in the database.
- // end==NULL is treated as a key after all keys in the database.
- // Therefore the following call will compact the entire database:
- // db->CompactRange(NULL, NULL);
- virtual void CompactRange(const Slice* begin, const Slice* end) = 0;
-
- private:
- // No copying allowed
- DB(const DB&);
- void operator=(const DB&);
-};
-
-// Destroy the contents of the specified database.
-// Be very careful using this method.
-Status DestroyDB(const std::string& name, const Options& options);
-
-// If a DB cannot be opened, you may attempt to call this method to
-// resurrect as much of the contents of the database as possible.
-// Some data may be lost, so be careful when calling this function
-// on a database that contains important information.
-Status RepairDB(const std::string& dbname, const Options& options);
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_INCLUDE_DB_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/dumpfile.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/dumpfile.h b/thirdparty/leveldb-1.18/include/leveldb/dumpfile.h
deleted file mode 100755
index 3f97fda..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/dumpfile.h
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright (c) 2014 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
-#define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
-
-#include <string>
-#include "leveldb/env.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-// Dump the contents of the file named by fname in text format to
-// *dst. Makes a sequence of dst->Append() calls; each call is passed
-// the newline-terminated text corresponding to a single item found
-// in the file.
-//
-// Returns a non-OK result if fname does not name a leveldb storage
-// file, or if the file cannot be read.
-Status DumpFile(Env* env, const std::string& fname, WritableFile* dst);
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/env.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/env.h b/thirdparty/leveldb-1.18/include/leveldb/env.h
deleted file mode 100755
index 6c25c7c..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/env.h
+++ /dev/null
@@ -1,339 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// An Env is an interface used by the leveldb implementation to access
-// operating system functionality like the filesystem etc. Callers
-// may wish to provide a custom Env object when opening a database to
-// get fine gain control; e.g., to rate limit file system operations.
-//
-// All Env implementations are safe for concurrent access from
-// multiple threads without any external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_ENV_H_
-#define STORAGE_LEVELDB_INCLUDE_ENV_H_
-
-#include <string>
-#include <vector>
-#include <thread>
-#include <stdarg.h>
-#include <stdint.h>
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class FileLock;
-class Logger;
-class RandomAccessFile;
-class SequentialFile;
-class Slice;
-class WritableFile;
-
-class Env {
- public:
- Env() { }
- virtual ~Env();
-
- // Return a default environment suitable for the current operating
- // system. Sophisticated users may wish to provide their own Env
- // implementation instead of relying on this default environment.
- //
- // The result of Default() belongs to leveldb and must never be deleted.
- static Env* Default();
-
- // use this call to deallocate memory when you know you no longer need it
- // totally not thread safe but needed to prevent memory analyzers from crying
- static void UnsafeDeallocate();
-
- // needed if you want to set affinity or adjust thread priority
- virtual std::thread::native_handle_type GetBackgroundThreadHandle() = 0;
-
- // Create a brand new sequentially-readable file with the specified name.
- // On success, stores a pointer to the new file in *result and returns OK.
- // On failure stores NULL in *result and returns non-OK. If the file does
- // not exist, returns a non-OK status.
- //
- // The returned file will only be accessed by one thread at a time.
- virtual Status NewSequentialFile(const std::string& fname,
- SequentialFile** result) = 0;
-
- // Create a brand new random access read-only file with the
- // specified name. On success, stores a pointer to the new file in
- // *result and returns OK. On failure stores NULL in *result and
- // returns non-OK. If the file does not exist, returns a non-OK
- // status.
- //
- // The returned file may be concurrently accessed by multiple threads.
- virtual Status NewRandomAccessFile(const std::string& fname,
- RandomAccessFile** result) = 0;
-
- // Create an object that writes to a new file with the specified
- // name. Deletes any existing file with the same name and creates a
- // new file. On success, stores a pointer to the new file in
- // *result and returns OK. On failure stores NULL in *result and
- // returns non-OK.
- //
- // The returned file will only be accessed by one thread at a time.
- virtual Status NewWritableFile(const std::string& fname,
- WritableFile** result) = 0;
-
- // Returns true iff the named file exists.
- virtual bool FileExists(const std::string& fname) = 0;
-
- // Store in *result the names of the children of the specified directory.
- // The names are relative to "dir".
- // Original contents of *results are dropped.
- virtual Status GetChildren(const std::string& dir,
- std::vector<std::string>* result) = 0;
-
- // Delete the named file.
- virtual Status DeleteFile(const std::string& fname) = 0;
-
- // Create the specified directory.
- virtual Status CreateDir(const std::string& dirname) = 0;
-
- // Delete the specified directory.
- virtual Status DeleteDir(const std::string& dirname) = 0;
-
- // Store the size of fname in *file_size.
- virtual Status GetFileSize(const std::string& fname, uint64_t* file_size) = 0;
-
- // Rename file src to target.
- virtual Status RenameFile(const std::string& src,
- const std::string& target) = 0;
-
- // Lock the specified file. Used to prevent concurrent access to
- // the same db by multiple processes. On failure, stores NULL in
- // *lock and returns non-OK.
- //
- // On success, stores a pointer to the object that represents the
- // acquired lock in *lock and returns OK. The caller should call
- // UnlockFile(*lock) to release the lock. If the process exits,
- // the lock will be automatically released.
- //
- // If somebody else already holds the lock, finishes immediately
- // with a failure. I.e., this call does not wait for existing locks
- // to go away.
- //
- // May create the named file if it does not already exist.
- virtual Status LockFile(const std::string& fname, FileLock** lock) = 0;
-
- // Release the lock acquired by a previous successful call to LockFile.
- // REQUIRES: lock was returned by a successful LockFile() call
- // REQUIRES: lock has not already been unlocked.
- virtual Status UnlockFile(FileLock* lock) = 0;
-
- // Arrange to run "(*function)(arg)" once in a background thread.
- //
- // "function" may run in an unspecified thread. Multiple functions
- // added to the same Env may run concurrently in different threads.
- // I.e., the caller may not assume that background work items are
- // serialized.
- virtual void Schedule(
- void (*function)(void* arg),
- void* arg) = 0;
-
- // *path is set to a temporary directory that can be used for testing. It may
- // or many not have just been created. The directory may or may not differ
- // between runs of the same process, but subsequent calls will return the
- // same directory.
- virtual Status GetTestDirectory(std::string* path) = 0;
-
- // Create and return a log file for storing informational messages.
- virtual Status NewLogger(const std::string& fname, Logger** result) = 0;
-
- // Returns the number of micro-seconds since some fixed point in time. Only
- // useful for computing deltas of time.
- virtual uint64_t NowMicros() = 0;
-
- // Sleep/delay the thread for the prescribed number of micro-seconds.
- virtual void SleepForMicroseconds(int micros) = 0;
-
- private:
- // No copying allowed
- Env(const Env&);
- void operator=(const Env&);
-};
-
-// A file abstraction for reading sequentially through a file
-class SequentialFile {
- public:
- SequentialFile() { }
- virtual ~SequentialFile();
-
- // Read up to "n" bytes from the file. "scratch[0..n-1]" may be
- // written by this routine. Sets "*result" to the data that was
- // read (including if fewer than "n" bytes were successfully read).
- // May set "*result" to point at data in "scratch[0..n-1]", so
- // "scratch[0..n-1]" must be live when "*result" is used.
- // If an error was encountered, returns a non-OK status.
- //
- // REQUIRES: External synchronization
- virtual Status Read(size_t n, Slice* result, char* scratch) = 0;
-
- // Skip "n" bytes from the file. This is guaranteed to be no
- // slower that reading the same data, but may be faster.
- //
- // If end of file is reached, skipping will stop at the end of the
- // file, and Skip will return OK.
- //
- // REQUIRES: External synchronization
- virtual Status Skip(uint64_t n) = 0;
-
- private:
- // No copying allowed
- SequentialFile(const SequentialFile&);
- void operator=(const SequentialFile&);
-};
-
-// A file abstraction for randomly reading the contents of a file.
-class RandomAccessFile {
- public:
- RandomAccessFile() { }
- virtual ~RandomAccessFile();
-
- // Read up to "n" bytes from the file starting at "offset".
- // "scratch[0..n-1]" may be written by this routine. Sets "*result"
- // to the data that was read (including if fewer than "n" bytes were
- // successfully read). May set "*result" to point at data in
- // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
- // "*result" is used. If an error was encountered, returns a non-OK
- // status.
- //
- // Safe for concurrent use by multiple threads.
- virtual Status Read(uint64_t offset, size_t n, Slice* result,
- char* scratch) const = 0;
-
- private:
- // No copying allowed
- RandomAccessFile(const RandomAccessFile&);
- void operator=(const RandomAccessFile&);
-};
-
-// A file abstraction for sequential writing. The implementation
-// must provide buffering since callers may append small fragments
-// at a time to the file.
-class WritableFile {
- public:
- WritableFile() { }
- virtual ~WritableFile();
-
- virtual Status Append(const Slice& data) = 0;
- virtual Status Close() = 0;
- virtual Status Flush() = 0;
- virtual Status Sync() = 0;
-
- private:
- // No copying allowed
- WritableFile(const WritableFile&);
- void operator=(const WritableFile&);
-};
-
-// An interface for writing log messages.
-class Logger {
- public:
- Logger() { }
- virtual ~Logger();
-
- // Write an entry to the log file with the specified format.
- virtual void Logv(const char* format, va_list ap) = 0;
-
- private:
- // No copying allowed
- Logger(const Logger&);
- void operator=(const Logger&);
-};
-
-
-// Identifies a locked file.
-class FileLock {
- public:
- FileLock() { }
- virtual ~FileLock();
- private:
- // No copying allowed
- FileLock(const FileLock&);
- void operator=(const FileLock&);
-};
-
-// Log the specified data to *info_log if info_log is non-NULL.
-extern void Log(Logger* info_log, const char* format, ...)
-# if defined(__GNUC__) || defined(__clang__)
- __attribute__((__format__ (__printf__, 2, 3)))
-# endif
- ;
-
-// A utility routine: write "data" to the named file.
-extern Status WriteStringToFile(Env* env, const Slice& data,
- const std::string& fname);
-
-// A utility routine: read contents of named file into *data
-extern Status ReadFileToString(Env* env, const std::string& fname,
- std::string* data);
-
-// An implementation of Env that forwards all calls to another Env.
-// May be useful to clients who wish to override just part of the
-// functionality of another Env.
-class EnvWrapper : public Env {
- public:
- // Initialize an EnvWrapper that delegates all calls to *t
- explicit EnvWrapper(Env* t) : target_(t) { }
- virtual ~EnvWrapper();
-
- // Return the target to which this Env forwards all calls
- Env* target() const { return target_; }
-
- std::thread::native_handle_type GetBackgroundThreadHandle() {
- return target_->GetBackgroundThreadHandle();
- }
-
- // The following text is boilerplate that forwards all methods to target()
- Status NewSequentialFile(const std::string& f, SequentialFile** r) {
- return target_->NewSequentialFile(f, r);
- }
- Status NewRandomAccessFile(const std::string& f, RandomAccessFile** r) {
- return target_->NewRandomAccessFile(f, r);
- }
- Status NewWritableFile(const std::string& f, WritableFile** r) {
- return target_->NewWritableFile(f, r);
- }
- bool FileExists(const std::string& f) { return target_->FileExists(f); }
- Status GetChildren(const std::string& dir, std::vector<std::string>* r) {
- return target_->GetChildren(dir, r);
- }
- Status DeleteFile(const std::string& f) { return target_->DeleteFile(f); }
- Status CreateDir(const std::string& d) { return target_->CreateDir(d); }
- Status DeleteDir(const std::string& d) { return target_->DeleteDir(d); }
- Status GetFileSize(const std::string& f, uint64_t* s) {
- return target_->GetFileSize(f, s);
- }
- Status RenameFile(const std::string& s, const std::string& t) {
- return target_->RenameFile(s, t);
- }
- Status LockFile(const std::string& f, FileLock** l) {
- return target_->LockFile(f, l);
- }
- Status UnlockFile(FileLock* l) { return target_->UnlockFile(l); }
- void Schedule(void (*f)(void*), void* a) {
- return target_->Schedule(f, a);
- }
-
- virtual Status GetTestDirectory(std::string* path) {
- return target_->GetTestDirectory(path);
- }
- virtual Status NewLogger(const std::string& fname, Logger** result) {
- return target_->NewLogger(fname, result);
- }
- uint64_t NowMicros() {
- return target_->NowMicros();
- }
- void SleepForMicroseconds(int micros) {
- target_->SleepForMicroseconds(micros);
- }
- private:
- Env* target_;
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_INCLUDE_ENV_H_
[42/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/doc/bench/db_bench_sqlite3.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/doc/bench/db_bench_sqlite3.cc b/thirdparty/leveldb-1.18/doc/bench/db_bench_sqlite3.cc
deleted file mode 100755
index e63aaa8..0000000
--- a/thirdparty/leveldb-1.18/doc/bench/db_bench_sqlite3.cc
+++ /dev/null
@@ -1,718 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <sqlite3.h>
-#include "util/histogram.h"
-#include "util/random.h"
-#include "util/testutil.h"
-
-// Comma-separated list of operations to run in the specified order
-// Actual benchmarks:
-//
-// fillseq -- write N values in sequential key order in async mode
-// fillseqsync -- write N/100 values in sequential key order in sync mode
-// fillseqbatch -- batch write N values in sequential key order in async mode
-// fillrandom -- write N values in random key order in async mode
-// fillrandsync -- write N/100 values in random key order in sync mode
-// fillrandbatch -- batch write N values in sequential key order in async mode
-// overwrite -- overwrite N values in random key order in async mode
-// fillrand100K -- write N/1000 100K values in random order in async mode
-// fillseq100K -- write N/1000 100K values in sequential order in async mode
-// readseq -- read N times sequentially
-// readrandom -- read N times in random order
-// readrand100K -- read N/1000 100K values in sequential order in async mode
-static const char* FLAGS_benchmarks =
- "fillseq,"
- "fillseqsync,"
- "fillseqbatch,"
- "fillrandom,"
- "fillrandsync,"
- "fillrandbatch,"
- "overwrite,"
- "overwritebatch,"
- "readrandom,"
- "readseq,"
- "fillrand100K,"
- "fillseq100K,"
- "readseq,"
- "readrand100K,"
- ;
-
-// Number of key/values to place in database
-static int FLAGS_num = 1000000;
-
-// Number of read operations to do. If negative, do FLAGS_num reads.
-static int FLAGS_reads = -1;
-
-// Size of each value
-static int FLAGS_value_size = 100;
-
-// Print histogram of operation timings
-static bool FLAGS_histogram = false;
-
-// Arrange to generate values that shrink to this fraction of
-// their original size after compression
-static double FLAGS_compression_ratio = 0.5;
-
-// Page size. Default 1 KB.
-static int FLAGS_page_size = 1024;
-
-// Number of pages.
-// Default cache size = FLAGS_page_size * FLAGS_num_pages = 4 MB.
-static int FLAGS_num_pages = 4096;
-
-// If true, do not destroy the existing database. If you set this
-// flag and also specify a benchmark that wants a fresh database, that
-// benchmark will fail.
-static bool FLAGS_use_existing_db = false;
-
-// If true, we allow batch writes to occur
-static bool FLAGS_transaction = true;
-
-// If true, we enable Write-Ahead Logging
-static bool FLAGS_WAL_enabled = true;
-
-// Use the db with the following name.
-static const char* FLAGS_db = NULL;
-
-inline
-static void ExecErrorCheck(int status, char *err_msg) {
- if (status != SQLITE_OK) {
- fprintf(stderr, "SQL error: %s\n", err_msg);
- sqlite3_free(err_msg);
- exit(1);
- }
-}
-
-inline
-static void StepErrorCheck(int status) {
- if (status != SQLITE_DONE) {
- fprintf(stderr, "SQL step error: status = %d\n", status);
- exit(1);
- }
-}
-
-inline
-static void ErrorCheck(int status) {
- if (status != SQLITE_OK) {
- fprintf(stderr, "sqlite3 error: status = %d\n", status);
- exit(1);
- }
-}
-
-inline
-static void WalCheckpoint(sqlite3* db_) {
- // Flush all writes to disk
- if (FLAGS_WAL_enabled) {
- sqlite3_wal_checkpoint_v2(db_, NULL, SQLITE_CHECKPOINT_FULL, NULL, NULL);
- }
-}
-
-namespace leveldb {
-
-// Helper for quickly generating random data.
-namespace {
-class RandomGenerator {
- private:
- std::string data_;
- int pos_;
-
- public:
- RandomGenerator() {
- // We use a limited amount of data over and over again and ensure
- // that it is larger than the compression window (32KB), and also
- // large enough to serve all typical value sizes we want to write.
- Random rnd(301);
- std::string piece;
- while (data_.size() < 1048576) {
- // Add a short fragment that is as compressible as specified
- // by FLAGS_compression_ratio.
- test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece);
- data_.append(piece);
- }
- pos_ = 0;
- }
-
- Slice Generate(int len) {
- if (pos_ + len > data_.size()) {
- pos_ = 0;
- assert(len < data_.size());
- }
- pos_ += len;
- return Slice(data_.data() + pos_ - len, len);
- }
-};
-
-static Slice TrimSpace(Slice s) {
- int start = 0;
- while (start < s.size() && isspace(s[start])) {
- start++;
- }
- int limit = s.size();
- while (limit > start && isspace(s[limit-1])) {
- limit--;
- }
- return Slice(s.data() + start, limit - start);
-}
-
-} // namespace
-
-class Benchmark {
- private:
- sqlite3* db_;
- int db_num_;
- int num_;
- int reads_;
- double start_;
- double last_op_finish_;
- int64_t bytes_;
- std::string message_;
- Histogram hist_;
- RandomGenerator gen_;
- Random rand_;
-
- // State kept for progress messages
- int done_;
- int next_report_; // When to report next
-
- void PrintHeader() {
- const int kKeySize = 16;
- PrintEnvironment();
- fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
- fprintf(stdout, "Values: %d bytes each\n", FLAGS_value_size);
- fprintf(stdout, "Entries: %d\n", num_);
- fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
- ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_)
- / 1048576.0));
- PrintWarnings();
- fprintf(stdout, "------------------------------------------------\n");
- }
-
- void PrintWarnings() {
-#if defined(__GNUC__) && !defined(__OPTIMIZE__)
- fprintf(stdout,
- "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
- );
-#endif
-#ifndef NDEBUG
- fprintf(stdout,
- "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
-#endif
- }
-
- void PrintEnvironment() {
- fprintf(stderr, "SQLite: version %s\n", SQLITE_VERSION);
-
-#if defined(__linux)
- time_t now = time(NULL);
- fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline
-
- FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
- if (cpuinfo != NULL) {
- char line[1000];
- int num_cpus = 0;
- std::string cpu_type;
- std::string cache_size;
- while (fgets(line, sizeof(line), cpuinfo) != NULL) {
- const char* sep = strchr(line, ':');
- if (sep == NULL) {
- continue;
- }
- Slice key = TrimSpace(Slice(line, sep - 1 - line));
- Slice val = TrimSpace(Slice(sep + 1));
- if (key == "model name") {
- ++num_cpus;
- cpu_type = val.ToString();
- } else if (key == "cache size") {
- cache_size = val.ToString();
- }
- }
- fclose(cpuinfo);
- fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
- fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
- }
-#endif
- }
-
- void Start() {
- start_ = Env::Default()->NowMicros() * 1e-6;
- bytes_ = 0;
- message_.clear();
- last_op_finish_ = start_;
- hist_.Clear();
- done_ = 0;
- next_report_ = 100;
- }
-
- void FinishedSingleOp() {
- if (FLAGS_histogram) {
- double now = Env::Default()->NowMicros() * 1e-6;
- double micros = (now - last_op_finish_) * 1e6;
- hist_.Add(micros);
- if (micros > 20000) {
- fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
- fflush(stderr);
- }
- last_op_finish_ = now;
- }
-
- done_++;
- if (done_ >= next_report_) {
- if (next_report_ < 1000) next_report_ += 100;
- else if (next_report_ < 5000) next_report_ += 500;
- else if (next_report_ < 10000) next_report_ += 1000;
- else if (next_report_ < 50000) next_report_ += 5000;
- else if (next_report_ < 100000) next_report_ += 10000;
- else if (next_report_ < 500000) next_report_ += 50000;
- else next_report_ += 100000;
- fprintf(stderr, "... finished %d ops%30s\r", done_, "");
- fflush(stderr);
- }
- }
-
- void Stop(const Slice& name) {
- double finish = Env::Default()->NowMicros() * 1e-6;
-
- // Pretend at least one op was done in case we are running a benchmark
- // that does not call FinishedSingleOp().
- if (done_ < 1) done_ = 1;
-
- if (bytes_ > 0) {
- char rate[100];
- snprintf(rate, sizeof(rate), "%6.1f MB/s",
- (bytes_ / 1048576.0) / (finish - start_));
- if (!message_.empty()) {
- message_ = std::string(rate) + " " + message_;
- } else {
- message_ = rate;
- }
- }
-
- fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
- name.ToString().c_str(),
- (finish - start_) * 1e6 / done_,
- (message_.empty() ? "" : " "),
- message_.c_str());
- if (FLAGS_histogram) {
- fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
- }
- fflush(stdout);
- }
-
- public:
- enum Order {
- SEQUENTIAL,
- RANDOM
- };
- enum DBState {
- FRESH,
- EXISTING
- };
-
- Benchmark()
- : db_(NULL),
- db_num_(0),
- num_(FLAGS_num),
- reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
- bytes_(0),
- rand_(301) {
- std::vector<std::string> files;
- std::string test_dir;
- Env::Default()->GetTestDirectory(&test_dir);
- Env::Default()->GetChildren(test_dir, &files);
- if (!FLAGS_use_existing_db) {
- for (int i = 0; i < files.size(); i++) {
- if (Slice(files[i]).starts_with("dbbench_sqlite3")) {
- std::string file_name(test_dir);
- file_name += "/";
- file_name += files[i];
- Env::Default()->DeleteFile(file_name.c_str());
- }
- }
- }
- }
-
- ~Benchmark() {
- int status = sqlite3_close(db_);
- ErrorCheck(status);
- }
-
- void Run() {
- PrintHeader();
- Open();
-
- const char* benchmarks = FLAGS_benchmarks;
- while (benchmarks != NULL) {
- const char* sep = strchr(benchmarks, ',');
- Slice name;
- if (sep == NULL) {
- name = benchmarks;
- benchmarks = NULL;
- } else {
- name = Slice(benchmarks, sep - benchmarks);
- benchmarks = sep + 1;
- }
-
- bytes_ = 0;
- Start();
-
- bool known = true;
- bool write_sync = false;
- if (name == Slice("fillseq")) {
- Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1);
- WalCheckpoint(db_);
- } else if (name == Slice("fillseqbatch")) {
- Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1000);
- WalCheckpoint(db_);
- } else if (name == Slice("fillrandom")) {
- Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1);
- WalCheckpoint(db_);
- } else if (name == Slice("fillrandbatch")) {
- Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1000);
- WalCheckpoint(db_);
- } else if (name == Slice("overwrite")) {
- Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1);
- WalCheckpoint(db_);
- } else if (name == Slice("overwritebatch")) {
- Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1000);
- WalCheckpoint(db_);
- } else if (name == Slice("fillrandsync")) {
- write_sync = true;
- Write(write_sync, RANDOM, FRESH, num_ / 100, FLAGS_value_size, 1);
- WalCheckpoint(db_);
- } else if (name == Slice("fillseqsync")) {
- write_sync = true;
- Write(write_sync, SEQUENTIAL, FRESH, num_ / 100, FLAGS_value_size, 1);
- WalCheckpoint(db_);
- } else if (name == Slice("fillrand100K")) {
- Write(write_sync, RANDOM, FRESH, num_ / 1000, 100 * 1000, 1);
- WalCheckpoint(db_);
- } else if (name == Slice("fillseq100K")) {
- Write(write_sync, SEQUENTIAL, FRESH, num_ / 1000, 100 * 1000, 1);
- WalCheckpoint(db_);
- } else if (name == Slice("readseq")) {
- ReadSequential();
- } else if (name == Slice("readrandom")) {
- Read(RANDOM, 1);
- } else if (name == Slice("readrand100K")) {
- int n = reads_;
- reads_ /= 1000;
- Read(RANDOM, 1);
- reads_ = n;
- } else {
- known = false;
- if (name != Slice()) { // No error message for empty name
- fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
- }
- }
- if (known) {
- Stop(name);
- }
- }
- }
-
- void Open() {
- assert(db_ == NULL);
-
- int status;
- char file_name[100];
- char* err_msg = NULL;
- db_num_++;
-
- // Open database
- std::string tmp_dir;
- Env::Default()->GetTestDirectory(&tmp_dir);
- snprintf(file_name, sizeof(file_name),
- "%s/dbbench_sqlite3-%d.db",
- tmp_dir.c_str(),
- db_num_);
- status = sqlite3_open(file_name, &db_);
- if (status) {
- fprintf(stderr, "open error: %s\n", sqlite3_errmsg(db_));
- exit(1);
- }
-
- // Change SQLite cache size
- char cache_size[100];
- snprintf(cache_size, sizeof(cache_size), "PRAGMA cache_size = %d",
- FLAGS_num_pages);
- status = sqlite3_exec(db_, cache_size, NULL, NULL, &err_msg);
- ExecErrorCheck(status, err_msg);
-
- // FLAGS_page_size is defaulted to 1024
- if (FLAGS_page_size != 1024) {
- char page_size[100];
- snprintf(page_size, sizeof(page_size), "PRAGMA page_size = %d",
- FLAGS_page_size);
- status = sqlite3_exec(db_, page_size, NULL, NULL, &err_msg);
- ExecErrorCheck(status, err_msg);
- }
-
- // Change journal mode to WAL if WAL enabled flag is on
- if (FLAGS_WAL_enabled) {
- std::string WAL_stmt = "PRAGMA journal_mode = WAL";
-
- // LevelDB's default cache size is a combined 4 MB
- std::string WAL_checkpoint = "PRAGMA wal_autocheckpoint = 4096";
- status = sqlite3_exec(db_, WAL_stmt.c_str(), NULL, NULL, &err_msg);
- ExecErrorCheck(status, err_msg);
- status = sqlite3_exec(db_, WAL_checkpoint.c_str(), NULL, NULL, &err_msg);
- ExecErrorCheck(status, err_msg);
- }
-
- // Change locking mode to exclusive and create tables/index for database
- std::string locking_stmt = "PRAGMA locking_mode = EXCLUSIVE";
- std::string create_stmt =
- "CREATE TABLE test (key blob, value blob, PRIMARY KEY(key))";
- std::string stmt_array[] = { locking_stmt, create_stmt };
- int stmt_array_length = sizeof(stmt_array) / sizeof(std::string);
- for (int i = 0; i < stmt_array_length; i++) {
- status = sqlite3_exec(db_, stmt_array[i].c_str(), NULL, NULL, &err_msg);
- ExecErrorCheck(status, err_msg);
- }
- }
-
- void Write(bool write_sync, Order order, DBState state,
- int num_entries, int value_size, int entries_per_batch) {
- // Create new database if state == FRESH
- if (state == FRESH) {
- if (FLAGS_use_existing_db) {
- message_ = "skipping (--use_existing_db is true)";
- return;
- }
- sqlite3_close(db_);
- db_ = NULL;
- Open();
- Start();
- }
-
- if (num_entries != num_) {
- char msg[100];
- snprintf(msg, sizeof(msg), "(%d ops)", num_entries);
- message_ = msg;
- }
-
- char* err_msg = NULL;
- int status;
-
- sqlite3_stmt *replace_stmt, *begin_trans_stmt, *end_trans_stmt;
- std::string replace_str = "REPLACE INTO test (key, value) VALUES (?, ?)";
- std::string begin_trans_str = "BEGIN TRANSACTION;";
- std::string end_trans_str = "END TRANSACTION;";
-
- // Check for synchronous flag in options
- std::string sync_stmt = (write_sync) ? "PRAGMA synchronous = FULL" :
- "PRAGMA synchronous = OFF";
- status = sqlite3_exec(db_, sync_stmt.c_str(), NULL, NULL, &err_msg);
- ExecErrorCheck(status, err_msg);
-
- // Preparing sqlite3 statements
- status = sqlite3_prepare_v2(db_, replace_str.c_str(), -1,
- &replace_stmt, NULL);
- ErrorCheck(status);
- status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1,
- &begin_trans_stmt, NULL);
- ErrorCheck(status);
- status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1,
- &end_trans_stmt, NULL);
- ErrorCheck(status);
-
- bool transaction = (entries_per_batch > 1);
- for (int i = 0; i < num_entries; i += entries_per_batch) {
- // Begin write transaction
- if (FLAGS_transaction && transaction) {
- status = sqlite3_step(begin_trans_stmt);
- StepErrorCheck(status);
- status = sqlite3_reset(begin_trans_stmt);
- ErrorCheck(status);
- }
-
- // Create and execute SQL statements
- for (int j = 0; j < entries_per_batch; j++) {
- const char* value = gen_.Generate(value_size).data();
-
- // Create values for key-value pair
- const int k = (order == SEQUENTIAL) ? i + j :
- (rand_.Next() % num_entries);
- char key[100];
- snprintf(key, sizeof(key), "%016d", k);
-
- // Bind KV values into replace_stmt
- status = sqlite3_bind_blob(replace_stmt, 1, key, 16, SQLITE_STATIC);
- ErrorCheck(status);
- status = sqlite3_bind_blob(replace_stmt, 2, value,
- value_size, SQLITE_STATIC);
- ErrorCheck(status);
-
- // Execute replace_stmt
- bytes_ += value_size + strlen(key);
- status = sqlite3_step(replace_stmt);
- StepErrorCheck(status);
-
- // Reset SQLite statement for another use
- status = sqlite3_clear_bindings(replace_stmt);
- ErrorCheck(status);
- status = sqlite3_reset(replace_stmt);
- ErrorCheck(status);
-
- FinishedSingleOp();
- }
-
- // End write transaction
- if (FLAGS_transaction && transaction) {
- status = sqlite3_step(end_trans_stmt);
- StepErrorCheck(status);
- status = sqlite3_reset(end_trans_stmt);
- ErrorCheck(status);
- }
- }
-
- status = sqlite3_finalize(replace_stmt);
- ErrorCheck(status);
- status = sqlite3_finalize(begin_trans_stmt);
- ErrorCheck(status);
- status = sqlite3_finalize(end_trans_stmt);
- ErrorCheck(status);
- }
-
- void Read(Order order, int entries_per_batch) {
- int status;
- sqlite3_stmt *read_stmt, *begin_trans_stmt, *end_trans_stmt;
-
- std::string read_str = "SELECT * FROM test WHERE key = ?";
- std::string begin_trans_str = "BEGIN TRANSACTION;";
- std::string end_trans_str = "END TRANSACTION;";
-
- // Preparing sqlite3 statements
- status = sqlite3_prepare_v2(db_, begin_trans_str.c_str(), -1,
- &begin_trans_stmt, NULL);
- ErrorCheck(status);
- status = sqlite3_prepare_v2(db_, end_trans_str.c_str(), -1,
- &end_trans_stmt, NULL);
- ErrorCheck(status);
- status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &read_stmt, NULL);
- ErrorCheck(status);
-
- bool transaction = (entries_per_batch > 1);
- for (int i = 0; i < reads_; i += entries_per_batch) {
- // Begin read transaction
- if (FLAGS_transaction && transaction) {
- status = sqlite3_step(begin_trans_stmt);
- StepErrorCheck(status);
- status = sqlite3_reset(begin_trans_stmt);
- ErrorCheck(status);
- }
-
- // Create and execute SQL statements
- for (int j = 0; j < entries_per_batch; j++) {
- // Create key value
- char key[100];
- int k = (order == SEQUENTIAL) ? i + j : (rand_.Next() % reads_);
- snprintf(key, sizeof(key), "%016d", k);
-
- // Bind key value into read_stmt
- status = sqlite3_bind_blob(read_stmt, 1, key, 16, SQLITE_STATIC);
- ErrorCheck(status);
-
- // Execute read statement
- while ((status = sqlite3_step(read_stmt)) == SQLITE_ROW) {}
- StepErrorCheck(status);
-
- // Reset SQLite statement for another use
- status = sqlite3_clear_bindings(read_stmt);
- ErrorCheck(status);
- status = sqlite3_reset(read_stmt);
- ErrorCheck(status);
- FinishedSingleOp();
- }
-
- // End read transaction
- if (FLAGS_transaction && transaction) {
- status = sqlite3_step(end_trans_stmt);
- StepErrorCheck(status);
- status = sqlite3_reset(end_trans_stmt);
- ErrorCheck(status);
- }
- }
-
- status = sqlite3_finalize(read_stmt);
- ErrorCheck(status);
- status = sqlite3_finalize(begin_trans_stmt);
- ErrorCheck(status);
- status = sqlite3_finalize(end_trans_stmt);
- ErrorCheck(status);
- }
-
- void ReadSequential() {
- int status;
- sqlite3_stmt *pStmt;
- std::string read_str = "SELECT * FROM test ORDER BY key";
-
- status = sqlite3_prepare_v2(db_, read_str.c_str(), -1, &pStmt, NULL);
- ErrorCheck(status);
- for (int i = 0; i < reads_ && SQLITE_ROW == sqlite3_step(pStmt); i++) {
- bytes_ += sqlite3_column_bytes(pStmt, 1) + sqlite3_column_bytes(pStmt, 2);
- FinishedSingleOp();
- }
-
- status = sqlite3_finalize(pStmt);
- ErrorCheck(status);
- }
-
-};
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- std::string default_db_path;
- for (int i = 1; i < argc; i++) {
- double d;
- int n;
- char junk;
- if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
- FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
- } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
- (n == 0 || n == 1)) {
- FLAGS_histogram = n;
- } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
- FLAGS_compression_ratio = d;
- } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
- (n == 0 || n == 1)) {
- FLAGS_use_existing_db = n;
- } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
- FLAGS_num = n;
- } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
- FLAGS_reads = n;
- } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
- FLAGS_value_size = n;
- } else if (leveldb::Slice(argv[i]) == leveldb::Slice("--no_transaction")) {
- FLAGS_transaction = false;
- } else if (sscanf(argv[i], "--page_size=%d%c", &n, &junk) == 1) {
- FLAGS_page_size = n;
- } else if (sscanf(argv[i], "--num_pages=%d%c", &n, &junk) == 1) {
- FLAGS_num_pages = n;
- } else if (sscanf(argv[i], "--WAL_enabled=%d%c", &n, &junk) == 1 &&
- (n == 0 || n == 1)) {
- FLAGS_WAL_enabled = n;
- } else if (strncmp(argv[i], "--db=", 5) == 0) {
- FLAGS_db = argv[i] + 5;
- } else {
- fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
- exit(1);
- }
- }
-
- // Choose a location for the test database if none given with --db=<path>
- if (FLAGS_db == NULL) {
- leveldb::Env::Default()->GetTestDirectory(&default_db_path);
- default_db_path += "/dbbench";
- FLAGS_db = default_db_path.c_str();
- }
-
- leveldb::Benchmark benchmark;
- benchmark.Run();
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/doc/bench/db_bench_tree_db.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/doc/bench/db_bench_tree_db.cc b/thirdparty/leveldb-1.18/doc/bench/db_bench_tree_db.cc
deleted file mode 100755
index 4ca381f..0000000
--- a/thirdparty/leveldb-1.18/doc/bench/db_bench_tree_db.cc
+++ /dev/null
@@ -1,528 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <kcpolydb.h>
-#include "util/histogram.h"
-#include "util/random.h"
-#include "util/testutil.h"
-
-// Comma-separated list of operations to run in the specified order
-// Actual benchmarks:
-//
-// fillseq -- write N values in sequential key order in async mode
-// fillrandom -- write N values in random key order in async mode
-// overwrite -- overwrite N values in random key order in async mode
-// fillseqsync -- write N/100 values in sequential key order in sync mode
-// fillrandsync -- write N/100 values in random key order in sync mode
-// fillrand100K -- write N/1000 100K values in random order in async mode
-// fillseq100K -- write N/1000 100K values in seq order in async mode
-// readseq -- read N times sequentially
-// readseq100K -- read N/1000 100K values in sequential order in async mode
-// readrand100K -- read N/1000 100K values in sequential order in async mode
-// readrandom -- read N times in random order
-static const char* FLAGS_benchmarks =
- "fillseq,"
- "fillseqsync,"
- "fillrandsync,"
- "fillrandom,"
- "overwrite,"
- "readrandom,"
- "readseq,"
- "fillrand100K,"
- "fillseq100K,"
- "readseq100K,"
- "readrand100K,"
- ;
-
-// Number of key/values to place in database
-static int FLAGS_num = 1000000;
-
-// Number of read operations to do. If negative, do FLAGS_num reads.
-static int FLAGS_reads = -1;
-
-// Size of each value
-static int FLAGS_value_size = 100;
-
-// Arrange to generate values that shrink to this fraction of
-// their original size after compression
-static double FLAGS_compression_ratio = 0.5;
-
-// Print histogram of operation timings
-static bool FLAGS_histogram = false;
-
-// Cache size. Default 4 MB
-static int FLAGS_cache_size = 4194304;
-
-// Page size. Default 1 KB
-static int FLAGS_page_size = 1024;
-
-// If true, do not destroy the existing database. If you set this
-// flag and also specify a benchmark that wants a fresh database, that
-// benchmark will fail.
-static bool FLAGS_use_existing_db = false;
-
-// Compression flag. If true, compression is on. If false, compression
-// is off.
-static bool FLAGS_compression = true;
-
-// Use the db with the following name.
-static const char* FLAGS_db = NULL;
-
-inline
-static void DBSynchronize(kyotocabinet::TreeDB* db_)
-{
- // Synchronize will flush writes to disk
- if (!db_->synchronize()) {
- fprintf(stderr, "synchronize error: %s\n", db_->error().name());
- }
-}
-
-namespace leveldb {
-
-// Helper for quickly generating random data.
-namespace {
-class RandomGenerator {
- private:
- std::string data_;
- int pos_;
-
- public:
- RandomGenerator() {
- // We use a limited amount of data over and over again and ensure
- // that it is larger than the compression window (32KB), and also
- // large enough to serve all typical value sizes we want to write.
- Random rnd(301);
- std::string piece;
- while (data_.size() < 1048576) {
- // Add a short fragment that is as compressible as specified
- // by FLAGS_compression_ratio.
- test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece);
- data_.append(piece);
- }
- pos_ = 0;
- }
-
- Slice Generate(int len) {
- if (pos_ + len > data_.size()) {
- pos_ = 0;
- assert(len < data_.size());
- }
- pos_ += len;
- return Slice(data_.data() + pos_ - len, len);
- }
-};
-
-static Slice TrimSpace(Slice s) {
- int start = 0;
- while (start < s.size() && isspace(s[start])) {
- start++;
- }
- int limit = s.size();
- while (limit > start && isspace(s[limit-1])) {
- limit--;
- }
- return Slice(s.data() + start, limit - start);
-}
-
-} // namespace
-
-class Benchmark {
- private:
- kyotocabinet::TreeDB* db_;
- int db_num_;
- int num_;
- int reads_;
- double start_;
- double last_op_finish_;
- int64_t bytes_;
- std::string message_;
- Histogram hist_;
- RandomGenerator gen_;
- Random rand_;
- kyotocabinet::LZOCompressor<kyotocabinet::LZO::RAW> comp_;
-
- // State kept for progress messages
- int done_;
- int next_report_; // When to report next
-
- void PrintHeader() {
- const int kKeySize = 16;
- PrintEnvironment();
- fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
- fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n",
- FLAGS_value_size,
- static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
- fprintf(stdout, "Entries: %d\n", num_);
- fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
- ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_)
- / 1048576.0));
- fprintf(stdout, "FileSize: %.1f MB (estimated)\n",
- (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_)
- / 1048576.0));
- PrintWarnings();
- fprintf(stdout, "------------------------------------------------\n");
- }
-
- void PrintWarnings() {
-#if defined(__GNUC__) && !defined(__OPTIMIZE__)
- fprintf(stdout,
- "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
- );
-#endif
-#ifndef NDEBUG
- fprintf(stdout,
- "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
-#endif
- }
-
- void PrintEnvironment() {
- fprintf(stderr, "Kyoto Cabinet: version %s, lib ver %d, lib rev %d\n",
- kyotocabinet::VERSION, kyotocabinet::LIBVER, kyotocabinet::LIBREV);
-
-#if defined(__linux)
- time_t now = time(NULL);
- fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline
-
- FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
- if (cpuinfo != NULL) {
- char line[1000];
- int num_cpus = 0;
- std::string cpu_type;
- std::string cache_size;
- while (fgets(line, sizeof(line), cpuinfo) != NULL) {
- const char* sep = strchr(line, ':');
- if (sep == NULL) {
- continue;
- }
- Slice key = TrimSpace(Slice(line, sep - 1 - line));
- Slice val = TrimSpace(Slice(sep + 1));
- if (key == "model name") {
- ++num_cpus;
- cpu_type = val.ToString();
- } else if (key == "cache size") {
- cache_size = val.ToString();
- }
- }
- fclose(cpuinfo);
- fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
- fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
- }
-#endif
- }
-
- void Start() {
- start_ = Env::Default()->NowMicros() * 1e-6;
- bytes_ = 0;
- message_.clear();
- last_op_finish_ = start_;
- hist_.Clear();
- done_ = 0;
- next_report_ = 100;
- }
-
- void FinishedSingleOp() {
- if (FLAGS_histogram) {
- double now = Env::Default()->NowMicros() * 1e-6;
- double micros = (now - last_op_finish_) * 1e6;
- hist_.Add(micros);
- if (micros > 20000) {
- fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
- fflush(stderr);
- }
- last_op_finish_ = now;
- }
-
- done_++;
- if (done_ >= next_report_) {
- if (next_report_ < 1000) next_report_ += 100;
- else if (next_report_ < 5000) next_report_ += 500;
- else if (next_report_ < 10000) next_report_ += 1000;
- else if (next_report_ < 50000) next_report_ += 5000;
- else if (next_report_ < 100000) next_report_ += 10000;
- else if (next_report_ < 500000) next_report_ += 50000;
- else next_report_ += 100000;
- fprintf(stderr, "... finished %d ops%30s\r", done_, "");
- fflush(stderr);
- }
- }
-
- void Stop(const Slice& name) {
- double finish = Env::Default()->NowMicros() * 1e-6;
-
- // Pretend at least one op was done in case we are running a benchmark
- // that does not call FinishedSingleOp().
- if (done_ < 1) done_ = 1;
-
- if (bytes_ > 0) {
- char rate[100];
- snprintf(rate, sizeof(rate), "%6.1f MB/s",
- (bytes_ / 1048576.0) / (finish - start_));
- if (!message_.empty()) {
- message_ = std::string(rate) + " " + message_;
- } else {
- message_ = rate;
- }
- }
-
- fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
- name.ToString().c_str(),
- (finish - start_) * 1e6 / done_,
- (message_.empty() ? "" : " "),
- message_.c_str());
- if (FLAGS_histogram) {
- fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
- }
- fflush(stdout);
- }
-
- public:
- enum Order {
- SEQUENTIAL,
- RANDOM
- };
- enum DBState {
- FRESH,
- EXISTING
- };
-
- Benchmark()
- : db_(NULL),
- num_(FLAGS_num),
- reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
- bytes_(0),
- rand_(301) {
- std::vector<std::string> files;
- std::string test_dir;
- Env::Default()->GetTestDirectory(&test_dir);
- Env::Default()->GetChildren(test_dir.c_str(), &files);
- if (!FLAGS_use_existing_db) {
- for (int i = 0; i < files.size(); i++) {
- if (Slice(files[i]).starts_with("dbbench_polyDB")) {
- std::string file_name(test_dir);
- file_name += "/";
- file_name += files[i];
- Env::Default()->DeleteFile(file_name.c_str());
- }
- }
- }
- }
-
- ~Benchmark() {
- if (!db_->close()) {
- fprintf(stderr, "close error: %s\n", db_->error().name());
- }
- }
-
- void Run() {
- PrintHeader();
- Open(false);
-
- const char* benchmarks = FLAGS_benchmarks;
- while (benchmarks != NULL) {
- const char* sep = strchr(benchmarks, ',');
- Slice name;
- if (sep == NULL) {
- name = benchmarks;
- benchmarks = NULL;
- } else {
- name = Slice(benchmarks, sep - benchmarks);
- benchmarks = sep + 1;
- }
-
- Start();
-
- bool known = true;
- bool write_sync = false;
- if (name == Slice("fillseq")) {
- Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1);
- DBSynchronize(db_);
- } else if (name == Slice("fillrandom")) {
- Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1);
- DBSynchronize(db_);
- } else if (name == Slice("overwrite")) {
- Write(write_sync, RANDOM, EXISTING, num_, FLAGS_value_size, 1);
- DBSynchronize(db_);
- } else if (name == Slice("fillrandsync")) {
- write_sync = true;
- Write(write_sync, RANDOM, FRESH, num_ / 100, FLAGS_value_size, 1);
- DBSynchronize(db_);
- } else if (name == Slice("fillseqsync")) {
- write_sync = true;
- Write(write_sync, SEQUENTIAL, FRESH, num_ / 100, FLAGS_value_size, 1);
- DBSynchronize(db_);
- } else if (name == Slice("fillrand100K")) {
- Write(write_sync, RANDOM, FRESH, num_ / 1000, 100 * 1000, 1);
- DBSynchronize(db_);
- } else if (name == Slice("fillseq100K")) {
- Write(write_sync, SEQUENTIAL, FRESH, num_ / 1000, 100 * 1000, 1);
- DBSynchronize(db_);
- } else if (name == Slice("readseq")) {
- ReadSequential();
- } else if (name == Slice("readrandom")) {
- ReadRandom();
- } else if (name == Slice("readrand100K")) {
- int n = reads_;
- reads_ /= 1000;
- ReadRandom();
- reads_ = n;
- } else if (name == Slice("readseq100K")) {
- int n = reads_;
- reads_ /= 1000;
- ReadSequential();
- reads_ = n;
- } else {
- known = false;
- if (name != Slice()) { // No error message for empty name
- fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
- }
- }
- if (known) {
- Stop(name);
- }
- }
- }
-
- private:
- void Open(bool sync) {
- assert(db_ == NULL);
-
- // Initialize db_
- db_ = new kyotocabinet::TreeDB();
- char file_name[100];
- db_num_++;
- std::string test_dir;
- Env::Default()->GetTestDirectory(&test_dir);
- snprintf(file_name, sizeof(file_name),
- "%s/dbbench_polyDB-%d.kct",
- test_dir.c_str(),
- db_num_);
-
- // Create tuning options and open the database
- int open_options = kyotocabinet::PolyDB::OWRITER |
- kyotocabinet::PolyDB::OCREATE;
- int tune_options = kyotocabinet::TreeDB::TSMALL |
- kyotocabinet::TreeDB::TLINEAR;
- if (FLAGS_compression) {
- tune_options |= kyotocabinet::TreeDB::TCOMPRESS;
- db_->tune_compressor(&comp_);
- }
- db_->tune_options(tune_options);
- db_->tune_page_cache(FLAGS_cache_size);
- db_->tune_page(FLAGS_page_size);
- db_->tune_map(256LL<<20);
- if (sync) {
- open_options |= kyotocabinet::PolyDB::OAUTOSYNC;
- }
- if (!db_->open(file_name, open_options)) {
- fprintf(stderr, "open error: %s\n", db_->error().name());
- }
- }
-
- void Write(bool sync, Order order, DBState state,
- int num_entries, int value_size, int entries_per_batch) {
- // Create new database if state == FRESH
- if (state == FRESH) {
- if (FLAGS_use_existing_db) {
- message_ = "skipping (--use_existing_db is true)";
- return;
- }
- delete db_;
- db_ = NULL;
- Open(sync);
- Start(); // Do not count time taken to destroy/open
- }
-
- if (num_entries != num_) {
- char msg[100];
- snprintf(msg, sizeof(msg), "(%d ops)", num_entries);
- message_ = msg;
- }
-
- // Write to database
- for (int i = 0; i < num_entries; i++)
- {
- const int k = (order == SEQUENTIAL) ? i : (rand_.Next() % num_entries);
- char key[100];
- snprintf(key, sizeof(key), "%016d", k);
- bytes_ += value_size + strlen(key);
- std::string cpp_key = key;
- if (!db_->set(cpp_key, gen_.Generate(value_size).ToString())) {
- fprintf(stderr, "set error: %s\n", db_->error().name());
- }
- FinishedSingleOp();
- }
- }
-
- void ReadSequential() {
- kyotocabinet::DB::Cursor* cur = db_->cursor();
- cur->jump();
- std::string ckey, cvalue;
- while (cur->get(&ckey, &cvalue, true)) {
- bytes_ += ckey.size() + cvalue.size();
- FinishedSingleOp();
- }
- delete cur;
- }
-
- void ReadRandom() {
- std::string value;
- for (int i = 0; i < reads_; i++) {
- char key[100];
- const int k = rand_.Next() % reads_;
- snprintf(key, sizeof(key), "%016d", k);
- db_->get(key, &value);
- FinishedSingleOp();
- }
- }
-};
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- std::string default_db_path;
- for (int i = 1; i < argc; i++) {
- double d;
- int n;
- char junk;
- if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
- FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
- } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
- FLAGS_compression_ratio = d;
- } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
- (n == 0 || n == 1)) {
- FLAGS_histogram = n;
- } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
- FLAGS_num = n;
- } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
- FLAGS_reads = n;
- } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
- FLAGS_value_size = n;
- } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) {
- FLAGS_cache_size = n;
- } else if (sscanf(argv[i], "--page_size=%d%c", &n, &junk) == 1) {
- FLAGS_page_size = n;
- } else if (sscanf(argv[i], "--compression=%d%c", &n, &junk) == 1 &&
- (n == 0 || n == 1)) {
- FLAGS_compression = (n == 1) ? true : false;
- } else if (strncmp(argv[i], "--db=", 5) == 0) {
- FLAGS_db = argv[i] + 5;
- } else {
- fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
- exit(1);
- }
- }
-
- // Choose a location for the test database if none given with --db=<path>
- if (FLAGS_db == NULL) {
- leveldb::Env::Default()->GetTestDirectory(&default_db_path);
- default_db_path += "/dbbench";
- FLAGS_db = default_db_path.c_str();
- }
-
- leveldb::Benchmark benchmark;
- benchmark.Run();
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/doc/benchmark.html
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/doc/benchmark.html b/thirdparty/leveldb-1.18/doc/benchmark.html
deleted file mode 100755
index c463977..0000000
--- a/thirdparty/leveldb-1.18/doc/benchmark.html
+++ /dev/null
@@ -1,459 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-<title>LevelDB Benchmarks</title>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-<style>
-body {
- font-family:Helvetica,sans-serif;
- padding:20px;
-}
-
-h2 {
- padding-top:30px;
-}
-
-table.bn {
- width:800px;
- border-collapse:collapse;
- border:0;
- padding:0;
-}
-
-table.bnbase {
- width:650px;
-}
-
-table.bn td {
- padding:2px 0;
-}
-
-table.bn td.c1 {
- font-weight:bold;
- width:150px;
-}
-
-table.bn td.c1 div.e {
- float:right;
- font-weight:normal;
-}
-
-table.bn td.c2 {
- width:150px;
- text-align:right;
- padding:2px;
-}
-
-table.bn td.c3 {
- width:350px;
-}
-
-table.bn td.c4 {
- width:150px;
- font-size:small;
- padding-left:4px;
-}
-
-/* chart bars */
-div.bldb {
- background-color:#0255df;
-}
-
-div.bkct {
- background-color:#df5555;
-}
-
-div.bsql {
- background-color:#aadf55;
-}
-
-.code {
- font-family:monospace;
- font-size:large;
-}
-
-.todo {
- color: red;
-}
-
-</style>
-</head>
-<body>
-<h1>LevelDB Benchmarks</h1>
-<p>Google, July 2011</p>
-<hr>
-
-<p>In order to test LevelDB's performance, we benchmark it against other well-established database implementations. We compare LevelDB (revision 39) against <a href="http://www.sqlite.org/">SQLite3</a> (version 3.7.6.3) and <a href="http://fallabs.com/kyotocabinet/spex.html">Kyoto Cabinet's</a> (version 1.2.67) TreeDB (a B+Tree based key-value store). We would like to acknowledge Scott Hess and Mikio Hirabayashi for their suggestions and contributions to the SQLite3 and Kyoto Cabinet benchmarks, respectively.</p>
-
-<p>Benchmarks were all performed on a six-core Intel(R) Xeon(R) CPU X5650 @ 2.67GHz, with 12288 KB of total L3 cache and 12 GB of DDR3 RAM at 1333 MHz. (Note that LevelDB uses at most two CPUs since the benchmarks are single threaded: one to run the benchmark, and one for background compactions.) We ran the benchmarks on two machines (with identical processors), one with an Ext3 file system and one with an Ext4 file system. The machine with the Ext3 file system has a SATA Hitachi HDS721050CLA362 hard drive. The machine with the Ext4 file system has a SATA Samsung HD502HJ hard drive. Both hard drives spin at 7200 RPM and have hard drive write-caching enabled (using `hdparm -W 1 [device]`). The numbers reported below are the median of three measurements.</p>
-
-<h4>Benchmark Source Code</h4>
-<p>We wrote benchmark tools for SQLite and Kyoto TreeDB based on LevelDB's <span class="code">db_bench</span>. The code for each of the benchmarks resides here:</p>
-<ul>
- <li> <b>LevelDB:</b> <a href="http://code.google.com/p/leveldb/source/browse/trunk/db/db_bench.cc">db/db_bench.cc</a>.</li>
- <li> <b>SQLite:</b> <a href="http://code.google.com/p/leveldb/source/browse/#svn%2Ftrunk%2Fdoc%2Fbench%2Fdb_bench_sqlite3.cc">doc/bench/db_bench_sqlite3.cc</a>.</li>
- <li> <b>Kyoto TreeDB:</b> <a href="http://code.google.com/p/leveldb/source/browse/#svn%2Ftrunk%2Fdoc%2Fbench%2Fdb_bench_tree_db.cc">doc/bench/db_bench_tree_db.cc</a>.</li>
-</ul>
-
-<h4>Custom Build Specifications</h4>
-<ul>
-<li>LevelDB: LevelDB was compiled with the <a href="http://code.google.com/p/google-perftools">tcmalloc</a> library and the <a href="http://code.google.com/p/snappy/">Snappy</a> compression library (revision 33). Assertions were disabled.</li>
-<li>TreeDB: TreeDB was compiled using the <a href="http://www.oberhumer.com/opensource/lzo/">LZO</a> compression library (version 2.03). Furthermore, we enabled the TSMALL and TLINEAR options when opening the database in order to reduce the footprint of each record.</li>
-<li>SQLite: We tuned SQLite's performance, by setting its locking mode to exclusive. We also enabled SQLite's <a href="http://www.sqlite.org/draft/wal.html">write-ahead logging</a>.</li>
-</ul>
-
-<h2>1. Baseline Performance</h2>
-<p>This section gives the baseline performance of all the
-databases. Following sections show how performance changes as various
-parameters are varied. For the baseline:</p>
-<ul>
- <li> Each database is allowed 4 MB of cache memory.</li>
- <li> Databases are opened in <em>asynchronous</em> write mode.
- (LevelDB's sync option, TreeDB's OAUTOSYNC option, and
- SQLite3's synchronous options are all turned off). I.e.,
- every write is pushed to the operating system, but the
- benchmark does not wait for the write to reach the disk.</li>
- <li> Keys are 16 bytes each.</li>
- <li> Value are 100 bytes each (with enough redundancy so that
- a simple compressor shrinks them to 50% of their original
- size).</li>
- <li> Sequential reads/writes traverse the key space in increasing order.</li>
- <li> Random reads/writes traverse the key space in random order.</li>
-</ul>
-
-<h3>A. Sequential Reads</h3>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">4,030,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">1,010,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:95px"> </div></td>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">383,000 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:33px"> </div></td>
-</table>
-<h3>B. Random Reads</h3>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">129,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:298px"> </div></td>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">151,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:350px"> </div></td>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">134,000 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:310px"> </div></td>
-</table>
-<h3>C. Sequential Writes</h3>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">779,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">342,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:154px"> </div></td>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">48,600 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:22px"> </div></td>
-</table>
-<h3>D. Random Writes</h3>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">164,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">88,500 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:188px"> </div></td>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">9,860 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:21px"> </div></td>
-</table>
-
-<p>LevelDB outperforms both SQLite3 and TreeDB in sequential and random write operations and sequential read operations. Kyoto Cabinet has the fastest random read operations.</p>
-
-<h2>2. Write Performance under Different Configurations</h2>
-<h3>A. Large Values </h3>
-<p>For this benchmark, we start with an empty database, and write 100,000 byte values (~50% compressible). To keep the benchmark running time reasonable, we stop after writing 1000 values.</p>
-<h4>Sequential Writes</h4>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">1,100 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:234px"> </div></td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">1,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:224px"> </div></td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">1,600 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:350px"> </div></td></tr>
-</table>
-<h4>Random Writes</h4>
-<table class="bn bnbase">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">480 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:105px"> </div></td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">1,100 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:240px"> </div></td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">1,600 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:350px"> </div></td></tr>
-</table>
-<p>LevelDB doesn't perform as well with large values of 100,000 bytes each. This is because LevelDB writes keys and values at least twice: first time to the transaction log, and second time (during a compaction) to a sorted file.
-With larger values, LevelDB's per-operation efficiency is swamped by the
-cost of extra copies of large values.</p>
-<h3>B. Batch Writes</h3>
-<p>A batch write is a set of writes that are applied atomically to the underlying database. A single batch of N writes may be significantly faster than N individual writes. The following benchmark writes one thousand batches where each batch contains one thousand 100-byte values. TreeDB does not support batch writes and is omitted from this benchmark.</p>
-<h4>Sequential Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">840,000 entries/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
- <td class="c4">(1.08x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">124,000 entries/sec</td>
- <td class="c3"><div class="bsql" style="width:52px"> </div></td>
- <td class="c4">(2.55x baseline)</td></tr>
-</table>
-<h4>Random Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">221,000 entries/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
- <td class="c4">(1.35x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">22,000 entries/sec</td>
- <td class="c3"><div class="bsql" style="width:34px"> </div></td>
- <td class="c4">(2.23x baseline)</td></tr>
-</table>
-
-<p>Because of the way LevelDB persistent storage is organized, batches of
-random writes are not much slower (only a factor of 4x) than batches
-of sequential writes.</p>
-
-<h3>C. Synchronous Writes</h3>
-<p>In the following benchmark, we enable the synchronous writing modes
-of all of the databases. Since this change significantly slows down the
-benchmark, we stop after 10,000 writes. For synchronous write tests, we've
-disabled hard drive write-caching (using `hdparm -W 0 [device]`).</p>
-<ul>
- <li>For LevelDB, we set WriteOptions.sync = true.</li>
- <li>In TreeDB, we enabled TreeDB's OAUTOSYNC option.</li>
- <li>For SQLite3, we set "PRAGMA synchronous = FULL".</li>
-</ul>
-<h4>Sequential Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">100 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
- <td class="c4">(0.003x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">7 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:27px"> </div></td>
- <td class="c4">(0.0004x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">88 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:315px"> </div></td>
- <td class="c4">(0.002x baseline)</td></tr>
-</table>
-<h4>Random Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">100 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
- <td class="c4">(0.015x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">8 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:29px"> </div></td>
- <td class="c4">(0.001x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">88 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:314px"> </div></td>
- <td class="c4">(0.009x baseline)</td></tr>
-</table>
-
-<p>Also see the <code>ext4</code> performance numbers below
-since synchronous writes behave significantly differently
-on <code>ext3</code> and <code>ext4</code>.</p>
-
-<h3>D. Turning Compression Off</h3>
-
-<p>In the baseline measurements, LevelDB and TreeDB were using
-light-weight compression
-(<a href="http://code.google.com/p/snappy/">Snappy</a> for LevelDB,
-and <a href="http://www.oberhumer.com/opensource/lzo/">LZO</a> for
-TreeDB). SQLite3, by default does not use compression. The
-experiments below show what happens when compression is disabled in
-all of the databases (the SQLite3 numbers are just a copy of
-its baseline measurements):</p>
-
-<h4>Sequential Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">594,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
- <td class="c4">(0.76x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">485,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:239px"> </div></td>
- <td class="c4">(1.42x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">48,600 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:29px"> </div></td>
- <td class="c4">(1.00x baseline)</td></tr>
-</table>
-<h4>Random Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">135,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:296px"> </div></td>
- <td class="c4">(0.82x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">159,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:350px"> </div></td>
- <td class="c4">(1.80x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">9,860 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:22px"> </div></td>
- <td class="c4">(1.00x baseline)</td></tr>
-</table>
-
-<p>LevelDB's write performance is better with compression than without
-since compression decreases the amount of data that has to be written
-to disk. Therefore LevelDB users can leave compression enabled in
-most scenarios without having worry about a tradeoff between space
-usage and performance. TreeDB's performance on the other hand is
-better without compression than with compression. Presumably this is
-because TreeDB's compression library (LZO) is more expensive than
-LevelDB's compression library (Snappy).<p>
-
-<h3>E. Using More Memory</h3>
-<p>We increased the overall cache size for each database to 128 MB. For LevelDB, we partitioned 128 MB into a 120 MB write buffer and 8 MB of cache (up from 2 MB of write buffer and 2 MB of cache). For SQLite3, we kept the page size at 1024 bytes, but increased the number of pages to 131,072 (up from 4096). For TreeDB, we also kept the page size at 1024 bytes, but increased the cache size to 128 MB (up from 4 MB).</p>
-<h4>Sequential Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">812,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
- <td class="c4">(1.04x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">321,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:138px"> </div></td>
- <td class="c4">(0.94x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">48,500 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:21px"> </div></td>
- <td class="c4">(1.00x baseline)</td></tr>
-</table>
-<h4>Random Writes</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">355,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
- <td class="c4">(2.16x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">284,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:280px"> </div></td>
- <td class="c4">(3.21x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">9,670 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:10px"> </div></td>
- <td class="c4">(0.98x baseline)</td></tr>
-</table>
-
-<p>SQLite's performance does not change substantially when compared to
-the baseline, but the random write performance for both LevelDB and
-TreeDB increases significantly. LevelDB's performance improves
-because a larger write buffer reduces the need to merge sorted files
-(since it creates a smaller number of larger sorted files). TreeDB's
-performance goes up because the entire database is available in memory
-for fast in-place updates.</p>
-
- <h2>3. Read Performance under Different Configurations</h2>
-<h3>A. Larger Caches</h3>
-<p>We increased the overall memory usage to 128 MB for each database.
-For LevelDB, we allocated 8 MB to LevelDB's write buffer and 120 MB
-to LevelDB's cache. The other databases don't differentiate between a
-write buffer and a cache, so we simply set their cache size to 128
-MB.</p>
-<h4>Sequential Reads</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">5,210,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
- <td class="c4">(1.29x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">1,070,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:72px"> </div></td>
- <td class="c4">(1.06x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">609,000 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:41px"> </div></td>
- <td class="c4">(1.59x baseline)</td></tr>
-</table>
-
-<h4>Random Reads</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">190,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:144px"> </div></td>
- <td class="c4">(1.47x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">463,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:350px"> </div></td>
- <td class="c4">(3.07x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">186,000 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:141px"> </div></td>
- <td class="c4">(1.39x baseline)</td></tr>
-</table>
-
-<p>As expected, the read performance of all of the databases increases
-when the caches are enlarged. In particular, TreeDB seems to make
-very effective use of a cache that is large enough to hold the entire
-database.</p>
-
-<h3>B. No Compression Reads </h3>
-<p>For this benchmark, we populated a database with 1 million entries consisting of 16 byte keys and 100 byte values. We compiled LevelDB and Kyoto Cabinet without compression support, so results that are read out from the database are already uncompressed. We've listed the SQLite3 baseline read performance as a point of comparison.</p>
-<h4>Sequential Reads</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">4,880,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:350px"> </div></td>
- <td class="c4">(1.21x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">1,230,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:88px"> </div></td>
- <td class="c4">(3.60x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">383,000 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:27px"> </div></td>
- <td class="c4">(1.00x baseline)</td></tr>
-</table>
-<h4>Random Reads</h4>
-<table class="bn">
-<tr><td class="c1">LevelDB</td>
- <td class="c2">149,000 ops/sec</td>
- <td class="c3"><div class="bldb" style="width:300px"> </div></td>
- <td class="c4">(1.16x baseline)</td></tr>
-<tr><td class="c1">Kyoto TreeDB</td>
- <td class="c2">175,000 ops/sec</td>
- <td class="c3"><div class="bkct" style="width:350px"> </div></td>
- <td class="c4">(1.16x baseline)</td></tr>
-<tr><td class="c1">SQLite3</td>
- <td class="c2">134,000 ops/sec</td>
- <td class="c3"><div class="bsql" style="width:268px"> </div></td>
- <td class="c4">(1.00x baseline)</td></tr>
-</table>
-
-<p>Performance of both LevelDB and TreeDB improves a small amount when
-compression is disabled. Note however that under different workloads,
-performance may very well be better with compression if it allows more
-of the working set to fit in memory.</p>
-
-<h2>Note about Ext4 Filesystems</h2>
-<p>The preceding numbers are for an ext3 file system. Synchronous writes are much slower under <a href="http://en.wikipedia.org/wiki/Ext4">ext4</a> (LevelDB drops to ~31 writes / second and TreeDB drops to ~5 writes / second; SQLite3's synchronous writes do not noticeably drop) due to ext4's different handling of <span class="code">fsync</span> / <span class="code">msync</span> calls. Even LevelDB's asynchronous write performance drops somewhat since it spreads its storage across multiple files and issues <span class="code">fsync</span> calls when switching to a new file.</p>
-
-<h2>Acknowledgements</h2>
-<p>Jeff Dean and Sanjay Ghemawat wrote LevelDB. Kevin Tseng wrote and compiled these benchmarks. Mikio Hirabayashi, Scott Hess, and Gabor Cselle provided help and advice.</p>
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/doc/doc.css
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/doc/doc.css b/thirdparty/leveldb-1.18/doc/doc.css
deleted file mode 100755
index 700c564..0000000
--- a/thirdparty/leveldb-1.18/doc/doc.css
+++ /dev/null
@@ -1,89 +0,0 @@
-body {
- margin-left: 0.5in;
- margin-right: 0.5in;
- background: white;
- color: black;
-}
-
-h1 {
- margin-left: -0.2in;
- font-size: 14pt;
-}
-h2 {
- margin-left: -0in;
- font-size: 12pt;
-}
-h3 {
- margin-left: -0in;
-}
-h4 {
- margin-left: -0in;
-}
-hr {
- margin-left: -0in;
-}
-
-/* Definition lists: definition term bold */
-dt {
- font-weight: bold;
-}
-
-address {
- text-align: center;
-}
-code,samp,var {
- color: blue;
-}
-kbd {
- color: #600000;
-}
-div.note p {
- float: right;
- width: 3in;
- margin-right: 0%;
- padding: 1px;
- border: 2px solid #6060a0;
- background-color: #fffff0;
-}
-
-ul {
- margin-top: -0em;
- margin-bottom: -0em;
-}
-
-ol {
- margin-top: -0em;
- margin-bottom: -0em;
-}
-
-UL.nobullets {
- list-style-type: none;
- list-style-image: none;
- margin-left: -1em;
-}
-
-p {
- margin: 1em 0 1em 0;
- padding: 0 0 0 0;
-}
-
-pre {
- line-height: 1.3em;
- padding: 0.4em 0 0.8em 0;
- margin: 0 0 0 0;
- border: 0 0 0 0;
- color: blue;
-}
-
-.datatable {
- margin-left: auto;
- margin-right: auto;
- margin-top: 2em;
- margin-bottom: 2em;
- border: 1px solid;
-}
-
-.datatable td,th {
- padding: 0 0.5em 0 0.5em;
- text-align: right;
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/doc/impl.html
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/doc/impl.html b/thirdparty/leveldb-1.18/doc/impl.html
deleted file mode 100755
index 6a468be..0000000
--- a/thirdparty/leveldb-1.18/doc/impl.html
+++ /dev/null
@@ -1,213 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-<link rel="stylesheet" type="text/css" href="doc.css" />
-<title>Leveldb file layout and compactions</title>
-</head>
-
-<body>
-
-<h1>Files</h1>
-
-The implementation of leveldb is similar in spirit to the
-representation of a single
-<a href="http://research.google.com/archive/bigtable.html">
-Bigtable tablet (section 5.3)</a>.
-However the organization of the files that make up the representation
-is somewhat different and is explained below.
-
-<p>
-Each database is represented by a set of files stored in a directory.
-There are several different types of files as documented below:
-<p>
-<h2>Log files</h2>
-<p>
-A log file (*.log) stores a sequence of recent updates. Each update
-is appended to the current log file. When the log file reaches a
-pre-determined size (approximately 4MB by default), it is converted
-to a sorted table (see below) and a new log file is created for future
-updates.
-<p>
-A copy of the current log file is kept in an in-memory structure (the
-<code>memtable</code>). This copy is consulted on every read so that read
-operations reflect all logged updates.
-<p>
-<h2>Sorted tables</h2>
-<p>
-A sorted table (*.sst) stores a sequence of entries sorted by key.
-Each entry is either a value for the key, or a deletion marker for the
-key. (Deletion markers are kept around to hide obsolete values
-present in older sorted tables).
-<p>
-The set of sorted tables are organized into a sequence of levels. The
-sorted table generated from a log file is placed in a special <code>young</code>
-level (also called level-0). When the number of young files exceeds a
-certain threshold (currently four), all of the young files are merged
-together with all of the overlapping level-1 files to produce a
-sequence of new level-1 files (we create a new level-1 file for every
-2MB of data.)
-<p>
-Files in the young level may contain overlapping keys. However files
-in other levels have distinct non-overlapping key ranges. Consider
-level number L where L >= 1. When the combined size of files in
-level-L exceeds (10^L) MB (i.e., 10MB for level-1, 100MB for level-2,
-...), one file in level-L, and all of the overlapping files in
-level-(L+1) are merged to form a set of new files for level-(L+1).
-These merges have the effect of gradually migrating new updates from
-the young level to the largest level using only bulk reads and writes
-(i.e., minimizing expensive seeks).
-
-<h2>Manifest</h2>
-<p>
-A MANIFEST file lists the set of sorted tables that make up each
-level, the corresponding key ranges, and other important metadata.
-A new MANIFEST file (with a new number embedded in the file name)
-is created whenever the database is reopened. The MANIFEST file is
-formatted as a log, and changes made to the serving state (as files
-are added or removed) are appended to this log.
-<p>
-<h2>Current</h2>
-<p>
-CURRENT is a simple text file that contains the name of the latest
-MANIFEST file.
-<p>
-<h2>Info logs</h2>
-<p>
-Informational messages are printed to files named LOG and LOG.old.
-<p>
-<h2>Others</h2>
-<p>
-Other files used for miscellaneous purposes may also be present
-(LOCK, *.dbtmp).
-
-<h1>Level 0</h1>
-When the log file grows above a certain size (1MB by default):
-<ul>
-<li>Create a brand new memtable and log file and direct future updates here
-<li>In the background:
-<ul>
-<li>Write the contents of the previous memtable to an sstable
-<li>Discard the memtable
-<li>Delete the old log file and the old memtable
-<li>Add the new sstable to the young (level-0) level.
-</ul>
-</ul>
-
-<h1>Compactions</h1>
-
-<p>
-When the size of level L exceeds its limit, we compact it in a
-background thread. The compaction picks a file from level L and all
-overlapping files from the next level L+1. Note that if a level-L
-file overlaps only part of a level-(L+1) file, the entire file at
-level-(L+1) is used as an input to the compaction and will be
-discarded after the compaction. Aside: because level-0 is special
-(files in it may overlap each other), we treat compactions from
-level-0 to level-1 specially: a level-0 compaction may pick more than
-one level-0 file in case some of these files overlap each other.
-
-<p>
-A compaction merges the contents of the picked files to produce a
-sequence of level-(L+1) files. We switch to producing a new
-level-(L+1) file after the current output file has reached the target
-file size (2MB). We also switch to a new output file when the key
-range of the current output file has grown enough to overlap more than
-ten level-(L+2) files. This last rule ensures that a later compaction
-of a level-(L+1) file will not pick up too much data from level-(L+2).
-
-<p>
-The old files are discarded and the new files are added to the serving
-state.
-
-<p>
-Compactions for a particular level rotate through the key space. In
-more detail, for each level L, we remember the ending key of the last
-compaction at level L. The next compaction for level L will pick the
-first file that starts after this key (wrapping around to the
-beginning of the key space if there is no such file).
-
-<p>
-Compactions drop overwritten values. They also drop deletion markers
-if there are no higher numbered levels that contain a file whose range
-overlaps the current key.
-
-<h2>Timing</h2>
-
-Level-0 compactions will read up to four 1MB files from level-0, and
-at worst all the level-1 files (10MB). I.e., we will read 14MB and
-write 14MB.
-
-<p>
-Other than the special level-0 compactions, we will pick one 2MB file
-from level L. In the worst case, this will overlap ~ 12 files from
-level L+1 (10 because level-(L+1) is ten times the size of level-L,
-and another two at the boundaries since the file ranges at level-L
-will usually not be aligned with the file ranges at level-L+1). The
-compaction will therefore read 26MB and write 26MB. Assuming a disk
-IO rate of 100MB/s (ballpark range for modern drives), the worst
-compaction cost will be approximately 0.5 second.
-
-<p>
-If we throttle the background writing to something small, say 10% of
-the full 100MB/s speed, a compaction may take up to 5 seconds. If the
-user is writing at 10MB/s, we might build up lots of level-0 files
-(~50 to hold the 5*10MB). This may significantly increase the cost of
-reads due to the overhead of merging more files together on every
-read.
-
-<p>
-Solution 1: To reduce this problem, we might want to increase the log
-switching threshold when the number of level-0 files is large. Though
-the downside is that the larger this threshold, the more memory we will
-need to hold the corresponding memtable.
-
-<p>
-Solution 2: We might want to decrease write rate artificially when the
-number of level-0 files goes up.
-
-<p>
-Solution 3: We work on reducing the cost of very wide merges.
-Perhaps most of the level-0 files will have their blocks sitting
-uncompressed in the cache and we will only need to worry about the
-O(N) complexity in the merging iterator.
-
-<h2>Number of files</h2>
-
-Instead of always making 2MB files, we could make larger files for
-larger levels to reduce the total file count, though at the expense of
-more bursty compactions. Alternatively, we could shard the set of
-files into multiple directories.
-
-<p>
-An experiment on an <code>ext3</code> filesystem on Feb 04, 2011 shows
-the following timings to do 100K file opens in directories with
-varying number of files:
-<table class="datatable">
-<tr><th>Files in directory</th><th>Microseconds to open a file</th></tr>
-<tr><td>1000</td><td>9</td>
-<tr><td>10000</td><td>10</td>
-<tr><td>100000</td><td>16</td>
-</table>
-So maybe even the sharding is not necessary on modern filesystems?
-
-<h1>Recovery</h1>
-
-<ul>
-<li> Read CURRENT to find name of the latest committed MANIFEST
-<li> Read the named MANIFEST file
-<li> Clean up stale files
-<li> We could open all sstables here, but it is probably better to be lazy...
-<li> Convert log chunk to a new level-0 sstable
-<li> Start directing new writes to a new log file with recovered sequence#
-</ul>
-
-<h1>Garbage collection of files</h1>
-
-<code>DeleteObsoleteFiles()</code> is called at the end of every
-compaction and at the end of recovery. It finds the names of all
-files in the database. It deletes all log files that are not the
-current log file. It deletes all table files that are not referenced
-from some level and are not the output of an active compaction.
-
-</body>
-</html>
[24/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compaction_picker_universal.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compaction_picker_universal.cc b/thirdparty/rocksdb/db/compaction_picker_universal.cc
new file mode 100644
index 0000000..ce48026
--- /dev/null
+++ b/thirdparty/rocksdb/db/compaction_picker_universal.cc
@@ -0,0 +1,747 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/compaction_picker_universal.h"
+#ifndef ROCKSDB_LITE
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <limits>
+#include <queue>
+#include <string>
+#include <utility>
+#include "db/column_family.h"
+#include "monitoring/statistics.h"
+#include "util/filename.h"
+#include "util/log_buffer.h"
+#include "util/random.h"
+#include "util/string_util.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+namespace {
+// Used in universal compaction when trivial move is enabled.
+// This structure is used for the construction of min heap
+// that contains the file meta data, the level of the file
+// and the index of the file in that level
+
+struct InputFileInfo {
+ InputFileInfo() : f(nullptr) {}
+
+ FileMetaData* f;
+ size_t level;
+ size_t index;
+};
+
+// Used in universal compaction when trivial move is enabled.
+// This comparator is used for the construction of min heap
+// based on the smallest key of the file.
+struct SmallestKeyHeapComparator {
+ explicit SmallestKeyHeapComparator(const Comparator* ucmp) { ucmp_ = ucmp; }
+
+ bool operator()(InputFileInfo i1, InputFileInfo i2) const {
+ return (ucmp_->Compare(i1.f->smallest.user_key(),
+ i2.f->smallest.user_key()) > 0);
+ }
+
+ private:
+ const Comparator* ucmp_;
+};
+
+typedef std::priority_queue<InputFileInfo, std::vector<InputFileInfo>,
+ SmallestKeyHeapComparator>
+ SmallestKeyHeap;
+
+// This function creates the heap that is used to find if the files are
+// overlapping during universal compaction when the allow_trivial_move
+// is set.
+SmallestKeyHeap create_level_heap(Compaction* c, const Comparator* ucmp) {
+ SmallestKeyHeap smallest_key_priority_q =
+ SmallestKeyHeap(SmallestKeyHeapComparator(ucmp));
+
+ InputFileInfo input_file;
+
+ for (size_t l = 0; l < c->num_input_levels(); l++) {
+ if (c->num_input_files(l) != 0) {
+ if (l == 0 && c->start_level() == 0) {
+ for (size_t i = 0; i < c->num_input_files(0); i++) {
+ input_file.f = c->input(0, i);
+ input_file.level = 0;
+ input_file.index = i;
+ smallest_key_priority_q.push(std::move(input_file));
+ }
+ } else {
+ input_file.f = c->input(l, 0);
+ input_file.level = l;
+ input_file.index = 0;
+ smallest_key_priority_q.push(std::move(input_file));
+ }
+ }
+ }
+ return smallest_key_priority_q;
+}
+
+#ifndef NDEBUG
+// smallest_seqno and largest_seqno are set iff. `files` is not empty.
+void GetSmallestLargestSeqno(const std::vector<FileMetaData*>& files,
+ SequenceNumber* smallest_seqno,
+ SequenceNumber* largest_seqno) {
+ bool is_first = true;
+ for (FileMetaData* f : files) {
+ assert(f->smallest_seqno <= f->largest_seqno);
+ if (is_first) {
+ is_first = false;
+ *smallest_seqno = f->smallest_seqno;
+ *largest_seqno = f->largest_seqno;
+ } else {
+ if (f->smallest_seqno < *smallest_seqno) {
+ *smallest_seqno = f->smallest_seqno;
+ }
+ if (f->largest_seqno > *largest_seqno) {
+ *largest_seqno = f->largest_seqno;
+ }
+ }
+ }
+}
+#endif
+} // namespace
+
+// Algorithm that checks to see if there are any overlapping
+// files in the input
+bool UniversalCompactionPicker::IsInputFilesNonOverlapping(Compaction* c) {
+ auto comparator = icmp_->user_comparator();
+ int first_iter = 1;
+
+ InputFileInfo prev, curr, next;
+
+ SmallestKeyHeap smallest_key_priority_q =
+ create_level_heap(c, icmp_->user_comparator());
+
+ while (!smallest_key_priority_q.empty()) {
+ curr = smallest_key_priority_q.top();
+ smallest_key_priority_q.pop();
+
+ if (first_iter) {
+ prev = curr;
+ first_iter = 0;
+ } else {
+ if (comparator->Compare(prev.f->largest.user_key(),
+ curr.f->smallest.user_key()) >= 0) {
+ // found overlapping files, return false
+ return false;
+ }
+ assert(comparator->Compare(curr.f->largest.user_key(),
+ prev.f->largest.user_key()) > 0);
+ prev = curr;
+ }
+
+ next.f = nullptr;
+
+ if (curr.level != 0 && curr.index < c->num_input_files(curr.level) - 1) {
+ next.f = c->input(curr.level, curr.index + 1);
+ next.level = curr.level;
+ next.index = curr.index + 1;
+ }
+
+ if (next.f) {
+ smallest_key_priority_q.push(std::move(next));
+ }
+ }
+ return true;
+}
+
+bool UniversalCompactionPicker::NeedsCompaction(
+ const VersionStorageInfo* vstorage) const {
+ const int kLevel0 = 0;
+ return vstorage->CompactionScore(kLevel0) >= 1;
+}
+
+void UniversalCompactionPicker::SortedRun::Dump(char* out_buf,
+ size_t out_buf_size,
+ bool print_path) const {
+ if (level == 0) {
+ assert(file != nullptr);
+ if (file->fd.GetPathId() == 0 || !print_path) {
+ snprintf(out_buf, out_buf_size, "file %" PRIu64, file->fd.GetNumber());
+ } else {
+ snprintf(out_buf, out_buf_size, "file %" PRIu64
+ "(path "
+ "%" PRIu32 ")",
+ file->fd.GetNumber(), file->fd.GetPathId());
+ }
+ } else {
+ snprintf(out_buf, out_buf_size, "level %d", level);
+ }
+}
+
+void UniversalCompactionPicker::SortedRun::DumpSizeInfo(
+ char* out_buf, size_t out_buf_size, size_t sorted_run_count) const {
+ if (level == 0) {
+ assert(file != nullptr);
+ snprintf(out_buf, out_buf_size,
+ "file %" PRIu64 "[%" ROCKSDB_PRIszt
+ "] "
+ "with size %" PRIu64 " (compensated size %" PRIu64 ")",
+ file->fd.GetNumber(), sorted_run_count, file->fd.GetFileSize(),
+ file->compensated_file_size);
+ } else {
+ snprintf(out_buf, out_buf_size,
+ "level %d[%" ROCKSDB_PRIszt
+ "] "
+ "with size %" PRIu64 " (compensated size %" PRIu64 ")",
+ level, sorted_run_count, size, compensated_file_size);
+ }
+}
+
+std::vector<UniversalCompactionPicker::SortedRun>
+UniversalCompactionPicker::CalculateSortedRuns(
+ const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions) {
+ std::vector<UniversalCompactionPicker::SortedRun> ret;
+ for (FileMetaData* f : vstorage.LevelFiles(0)) {
+ ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size,
+ f->being_compacted);
+ }
+ for (int level = 1; level < vstorage.num_levels(); level++) {
+ uint64_t total_compensated_size = 0U;
+ uint64_t total_size = 0U;
+ bool being_compacted = false;
+ bool is_first = true;
+ for (FileMetaData* f : vstorage.LevelFiles(level)) {
+ total_compensated_size += f->compensated_file_size;
+ total_size += f->fd.GetFileSize();
+ if (ioptions.compaction_options_universal.allow_trivial_move == true) {
+ if (f->being_compacted) {
+ being_compacted = f->being_compacted;
+ }
+ } else {
+ // Compaction always includes all files for a non-zero level, so for a
+ // non-zero level, all the files should share the same being_compacted
+ // value.
+ // This assumption is only valid when
+ // ioptions.compaction_options_universal.allow_trivial_move is false
+ assert(is_first || f->being_compacted == being_compacted);
+ }
+ if (is_first) {
+ being_compacted = f->being_compacted;
+ is_first = false;
+ }
+ }
+ if (total_compensated_size > 0) {
+ ret.emplace_back(level, nullptr, total_size, total_compensated_size,
+ being_compacted);
+ }
+ }
+ return ret;
+}
+
+// Universal style of compaction. Pick files that are contiguous in
+// time-range to compact.
+//
+Compaction* UniversalCompactionPicker::PickCompaction(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
+ const int kLevel0 = 0;
+ double score = vstorage->CompactionScore(kLevel0);
+ std::vector<SortedRun> sorted_runs =
+ CalculateSortedRuns(*vstorage, ioptions_);
+
+ if (sorted_runs.size() == 0 ||
+ sorted_runs.size() <
+ (unsigned int)mutable_cf_options.level0_file_num_compaction_trigger) {
+ ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: nothing to do\n",
+ cf_name.c_str());
+ TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
+ nullptr);
+ return nullptr;
+ }
+ VersionStorageInfo::LevelSummaryStorage tmp;
+ ROCKS_LOG_BUFFER_MAX_SZ(
+ log_buffer, 3072,
+ "[%s] Universal: sorted runs files(%" ROCKSDB_PRIszt "): %s\n",
+ cf_name.c_str(), sorted_runs.size(), vstorage->LevelSummary(&tmp));
+
+ // Check for size amplification first.
+ Compaction* c;
+ if ((c = PickCompactionToReduceSizeAmp(cf_name, mutable_cf_options, vstorage,
+ score, sorted_runs, log_buffer)) !=
+ nullptr) {
+ ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: compacting for size amp\n",
+ cf_name.c_str());
+ } else {
+ // Size amplification is within limits. Try reducing read
+ // amplification while maintaining file size ratios.
+ unsigned int ratio = ioptions_.compaction_options_universal.size_ratio;
+
+ if ((c = PickCompactionToReduceSortedRuns(
+ cf_name, mutable_cf_options, vstorage, score, ratio, UINT_MAX,
+ sorted_runs, log_buffer)) != nullptr) {
+ ROCKS_LOG_BUFFER(log_buffer,
+ "[%s] Universal: compacting for size ratio\n",
+ cf_name.c_str());
+ } else {
+ // Size amplification and file size ratios are within configured limits.
+ // If max read amplification is exceeding configured limits, then force
+ // compaction without looking at filesize ratios and try to reduce
+ // the number of files to fewer than level0_file_num_compaction_trigger.
+ // This is guaranteed by NeedsCompaction()
+ assert(sorted_runs.size() >=
+ static_cast<size_t>(
+ mutable_cf_options.level0_file_num_compaction_trigger));
+ // Get the total number of sorted runs that are not being compacted
+ int num_sr_not_compacted = 0;
+ for (size_t i = 0; i < sorted_runs.size(); i++) {
+ if (sorted_runs[i].being_compacted == false) {
+ num_sr_not_compacted++;
+ }
+ }
+
+ // The number of sorted runs that are not being compacted is greater than
+ // the maximum allowed number of sorted runs
+ if (num_sr_not_compacted >
+ mutable_cf_options.level0_file_num_compaction_trigger) {
+ unsigned int num_files =
+ num_sr_not_compacted -
+ mutable_cf_options.level0_file_num_compaction_trigger + 1;
+ if ((c = PickCompactionToReduceSortedRuns(
+ cf_name, mutable_cf_options, vstorage, score, UINT_MAX,
+ num_files, sorted_runs, log_buffer)) != nullptr) {
+ ROCKS_LOG_BUFFER(log_buffer,
+ "[%s] Universal: compacting for file num -- %u\n",
+ cf_name.c_str(), num_files);
+ }
+ }
+ }
+ }
+ if (c == nullptr) {
+ TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
+ nullptr);
+ return nullptr;
+ }
+
+ if (ioptions_.compaction_options_universal.allow_trivial_move == true) {
+ c->set_is_trivial_move(IsInputFilesNonOverlapping(c));
+ }
+
+// validate that all the chosen files of L0 are non overlapping in time
+#ifndef NDEBUG
+ SequenceNumber prev_smallest_seqno = 0U;
+ bool is_first = true;
+
+ size_t level_index = 0U;
+ if (c->start_level() == 0) {
+ for (auto f : *c->inputs(0)) {
+ assert(f->smallest_seqno <= f->largest_seqno);
+ if (is_first) {
+ is_first = false;
+ }
+ prev_smallest_seqno = f->smallest_seqno;
+ }
+ level_index = 1U;
+ }
+ for (; level_index < c->num_input_levels(); level_index++) {
+ if (c->num_input_files(level_index) != 0) {
+ SequenceNumber smallest_seqno = 0U;
+ SequenceNumber largest_seqno = 0U;
+ GetSmallestLargestSeqno(*(c->inputs(level_index)), &smallest_seqno,
+ &largest_seqno);
+ if (is_first) {
+ is_first = false;
+ } else if (prev_smallest_seqno > 0) {
+ // A level is considered as the bottommost level if there are
+ // no files in higher levels or if files in higher levels do
+ // not overlap with the files being compacted. Sequence numbers
+ // of files in bottommost level can be set to 0 to help
+ // compression. As a result, the following assert may not hold
+ // if the prev_smallest_seqno is 0.
+ assert(prev_smallest_seqno > largest_seqno);
+ }
+ prev_smallest_seqno = smallest_seqno;
+ }
+ }
+#endif
+ // update statistics
+ MeasureTime(ioptions_.statistics, NUM_FILES_IN_SINGLE_COMPACTION,
+ c->inputs(0)->size());
+
+ RegisterCompaction(c);
+
+ TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
+ c);
+ return c;
+}
+
+uint32_t UniversalCompactionPicker::GetPathId(
+ const ImmutableCFOptions& ioptions, uint64_t file_size) {
+ // Two conditions need to be satisfied:
+ // (1) the target path needs to be able to hold the file's size
+ // (2) Total size left in this and previous paths need to be not
+ // smaller than expected future file size before this new file is
+ // compacted, which is estimated based on size_ratio.
+ // For example, if now we are compacting files of size (1, 1, 2, 4, 8),
+ // we will make sure the target file, probably with size of 16, will be
+ // placed in a path so that eventually when new files are generated and
+ // compacted to (1, 1, 2, 4, 8, 16), all those files can be stored in or
+ // before the path we chose.
+ //
+ // TODO(sdong): now the case of multiple column families is not
+ // considered in this algorithm. So the target size can be violated in
+ // that case. We need to improve it.
+ uint64_t accumulated_size = 0;
+ uint64_t future_size =
+ file_size * (100 - ioptions.compaction_options_universal.size_ratio) /
+ 100;
+ uint32_t p = 0;
+ assert(!ioptions.db_paths.empty());
+ for (; p < ioptions.db_paths.size() - 1; p++) {
+ uint64_t target_size = ioptions.db_paths[p].target_size;
+ if (target_size > file_size &&
+ accumulated_size + (target_size - file_size) > future_size) {
+ return p;
+ }
+ accumulated_size += target_size;
+ }
+ return p;
+}
+
+//
+// Consider compaction files based on their size differences with
+// the next file in time order.
+//
+Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, double score, unsigned int ratio,
+ unsigned int max_number_of_files_to_compact,
+ const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
+ unsigned int min_merge_width =
+ ioptions_.compaction_options_universal.min_merge_width;
+ unsigned int max_merge_width =
+ ioptions_.compaction_options_universal.max_merge_width;
+
+ const SortedRun* sr = nullptr;
+ bool done = false;
+ size_t start_index = 0;
+ unsigned int candidate_count = 0;
+
+ unsigned int max_files_to_compact =
+ std::min(max_merge_width, max_number_of_files_to_compact);
+ min_merge_width = std::max(min_merge_width, 2U);
+
+ // Caller checks the size before executing this function. This invariant is
+ // important because otherwise we may have a possible integer underflow when
+ // dealing with unsigned types.
+ assert(sorted_runs.size() > 0);
+
+ // Considers a candidate file only if it is smaller than the
+ // total size accumulated so far.
+ for (size_t loop = 0; loop < sorted_runs.size(); loop++) {
+ candidate_count = 0;
+
+ // Skip files that are already being compacted
+ for (sr = nullptr; loop < sorted_runs.size(); loop++) {
+ sr = &sorted_runs[loop];
+
+ if (!sr->being_compacted) {
+ candidate_count = 1;
+ break;
+ }
+ char file_num_buf[kFormatFileNumberBufSize];
+ sr->Dump(file_num_buf, sizeof(file_num_buf));
+ ROCKS_LOG_BUFFER(log_buffer,
+ "[%s] Universal: %s"
+ "[%d] being compacted, skipping",
+ cf_name.c_str(), file_num_buf, loop);
+
+ sr = nullptr;
+ }
+
+ // This file is not being compacted. Consider it as the
+ // first candidate to be compacted.
+ uint64_t candidate_size = sr != nullptr ? sr->compensated_file_size : 0;
+ if (sr != nullptr) {
+ char file_num_buf[kFormatFileNumberBufSize];
+ sr->Dump(file_num_buf, sizeof(file_num_buf), true);
+ ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Possible candidate %s[%d].",
+ cf_name.c_str(), file_num_buf, loop);
+ }
+
+ // Check if the succeeding files need compaction.
+ for (size_t i = loop + 1;
+ candidate_count < max_files_to_compact && i < sorted_runs.size();
+ i++) {
+ const SortedRun* succeeding_sr = &sorted_runs[i];
+ if (succeeding_sr->being_compacted) {
+ break;
+ }
+ // Pick files if the total/last candidate file size (increased by the
+ // specified ratio) is still larger than the next candidate file.
+ // candidate_size is the total size of files picked so far with the
+ // default kCompactionStopStyleTotalSize; with
+ // kCompactionStopStyleSimilarSize, it's simply the size of the last
+ // picked file.
+ double sz = candidate_size * (100.0 + ratio) / 100.0;
+ if (sz < static_cast<double>(succeeding_sr->size)) {
+ break;
+ }
+ if (ioptions_.compaction_options_universal.stop_style ==
+ kCompactionStopStyleSimilarSize) {
+ // Similar-size stopping rule: also check the last picked file isn't
+ // far larger than the next candidate file.
+ sz = (succeeding_sr->size * (100.0 + ratio)) / 100.0;
+ if (sz < static_cast<double>(candidate_size)) {
+ // If the small file we've encountered begins a run of similar-size
+ // files, we'll pick them up on a future iteration of the outer
+ // loop. If it's some lonely straggler, it'll eventually get picked
+ // by the last-resort read amp strategy which disregards size ratios.
+ break;
+ }
+ candidate_size = succeeding_sr->compensated_file_size;
+ } else { // default kCompactionStopStyleTotalSize
+ candidate_size += succeeding_sr->compensated_file_size;
+ }
+ candidate_count++;
+ }
+
+ // Found a series of consecutive files that need compaction.
+ if (candidate_count >= (unsigned int)min_merge_width) {
+ start_index = loop;
+ done = true;
+ break;
+ } else {
+ for (size_t i = loop;
+ i < loop + candidate_count && i < sorted_runs.size(); i++) {
+ const SortedRun* skipping_sr = &sorted_runs[i];
+ char file_num_buf[256];
+ skipping_sr->DumpSizeInfo(file_num_buf, sizeof(file_num_buf), loop);
+ ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Skipping %s",
+ cf_name.c_str(), file_num_buf);
+ }
+ }
+ }
+ if (!done || candidate_count <= 1) {
+ return nullptr;
+ }
+ size_t first_index_after = start_index + candidate_count;
+ // Compression is enabled if files compacted earlier already reached
+ // size ratio of compression.
+ bool enable_compression = true;
+ int ratio_to_compress =
+ ioptions_.compaction_options_universal.compression_size_percent;
+ if (ratio_to_compress >= 0) {
+ uint64_t total_size = 0;
+ for (auto& sorted_run : sorted_runs) {
+ total_size += sorted_run.compensated_file_size;
+ }
+
+ uint64_t older_file_size = 0;
+ for (size_t i = sorted_runs.size() - 1; i >= first_index_after; i--) {
+ older_file_size += sorted_runs[i].size;
+ if (older_file_size * 100L >= total_size * (long)ratio_to_compress) {
+ enable_compression = false;
+ break;
+ }
+ }
+ }
+
+ uint64_t estimated_total_size = 0;
+ for (unsigned int i = 0; i < first_index_after; i++) {
+ estimated_total_size += sorted_runs[i].size;
+ }
+ uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
+ int start_level = sorted_runs[start_index].level;
+ int output_level;
+ if (first_index_after == sorted_runs.size()) {
+ output_level = vstorage->num_levels() - 1;
+ } else if (sorted_runs[first_index_after].level == 0) {
+ output_level = 0;
+ } else {
+ output_level = sorted_runs[first_index_after].level - 1;
+ }
+
+ // last level is reserved for the files ingested behind
+ if (ioptions_.allow_ingest_behind &&
+ (output_level == vstorage->num_levels() - 1)) {
+ assert(output_level > 1);
+ output_level--;
+ }
+
+ std::vector<CompactionInputFiles> inputs(vstorage->num_levels());
+ for (size_t i = 0; i < inputs.size(); ++i) {
+ inputs[i].level = start_level + static_cast<int>(i);
+ }
+ for (size_t i = start_index; i < first_index_after; i++) {
+ auto& picking_sr = sorted_runs[i];
+ if (picking_sr.level == 0) {
+ FileMetaData* picking_file = picking_sr.file;
+ inputs[0].files.push_back(picking_file);
+ } else {
+ auto& files = inputs[picking_sr.level - start_level].files;
+ for (auto* f : vstorage->LevelFiles(picking_sr.level)) {
+ files.push_back(f);
+ }
+ }
+ char file_num_buf[256];
+ picking_sr.DumpSizeInfo(file_num_buf, sizeof(file_num_buf), i);
+ ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Picking %s", cf_name.c_str(),
+ file_num_buf);
+ }
+
+ CompactionReason compaction_reason;
+ if (max_number_of_files_to_compact == UINT_MAX) {
+ compaction_reason = CompactionReason::kUniversalSortedRunNum;
+ } else {
+ compaction_reason = CompactionReason::kUniversalSizeRatio;
+ }
+ return new Compaction(
+ vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level,
+ mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id,
+ GetCompressionType(ioptions_, vstorage, mutable_cf_options, start_level,
+ 1, enable_compression),
+ /* grandparents */ {}, /* is manual */ false, score,
+ false /* deletion_compaction */, compaction_reason);
+}
+
+// Look at overall size amplification. If size amplification
+// exceeeds the configured value, then do a compaction
+// of the candidate files all the way upto the earliest
+// base file (overrides configured values of file-size ratios,
+// min_merge_width and max_merge_width).
+//
+Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, double score,
+ const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
+ // percentage flexibility while reducing size amplification
+ uint64_t ratio =
+ ioptions_.compaction_options_universal.max_size_amplification_percent;
+
+ unsigned int candidate_count = 0;
+ uint64_t candidate_size = 0;
+ size_t start_index = 0;
+ const SortedRun* sr = nullptr;
+
+ // Skip files that are already being compacted
+ for (size_t loop = 0; loop < sorted_runs.size() - 1; loop++) {
+ sr = &sorted_runs[loop];
+ if (!sr->being_compacted) {
+ start_index = loop; // Consider this as the first candidate.
+ break;
+ }
+ char file_num_buf[kFormatFileNumberBufSize];
+ sr->Dump(file_num_buf, sizeof(file_num_buf), true);
+ ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: skipping %s[%d] compacted %s",
+ cf_name.c_str(), file_num_buf, loop,
+ " cannot be a candidate to reduce size amp.\n");
+ sr = nullptr;
+ }
+
+ if (sr == nullptr) {
+ return nullptr; // no candidate files
+ }
+ {
+ char file_num_buf[kFormatFileNumberBufSize];
+ sr->Dump(file_num_buf, sizeof(file_num_buf), true);
+ ROCKS_LOG_BUFFER(
+ log_buffer,
+ "[%s] Universal: First candidate %s[%" ROCKSDB_PRIszt "] %s",
+ cf_name.c_str(), file_num_buf, start_index, " to reduce size amp.\n");
+ }
+
+ // keep adding up all the remaining files
+ for (size_t loop = start_index; loop < sorted_runs.size() - 1; loop++) {
+ sr = &sorted_runs[loop];
+ if (sr->being_compacted) {
+ char file_num_buf[kFormatFileNumberBufSize];
+ sr->Dump(file_num_buf, sizeof(file_num_buf), true);
+ ROCKS_LOG_BUFFER(
+ log_buffer, "[%s] Universal: Possible candidate %s[%d] %s",
+ cf_name.c_str(), file_num_buf, start_index,
+ " is already being compacted. No size amp reduction possible.\n");
+ return nullptr;
+ }
+ candidate_size += sr->compensated_file_size;
+ candidate_count++;
+ }
+ if (candidate_count == 0) {
+ return nullptr;
+ }
+
+ // size of earliest file
+ uint64_t earliest_file_size = sorted_runs.back().size;
+
+ // size amplification = percentage of additional size
+ if (candidate_size * 100 < ratio * earliest_file_size) {
+ ROCKS_LOG_BUFFER(
+ log_buffer,
+ "[%s] Universal: size amp not needed. newer-files-total-size %" PRIu64
+ " earliest-file-size %" PRIu64,
+ cf_name.c_str(), candidate_size, earliest_file_size);
+ return nullptr;
+ } else {
+ ROCKS_LOG_BUFFER(
+ log_buffer,
+ "[%s] Universal: size amp needed. newer-files-total-size %" PRIu64
+ " earliest-file-size %" PRIu64,
+ cf_name.c_str(), candidate_size, earliest_file_size);
+ }
+ assert(start_index < sorted_runs.size() - 1);
+
+ // Estimate total file size
+ uint64_t estimated_total_size = 0;
+ for (size_t loop = start_index; loop < sorted_runs.size(); loop++) {
+ estimated_total_size += sorted_runs[loop].size;
+ }
+ uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
+ int start_level = sorted_runs[start_index].level;
+
+ std::vector<CompactionInputFiles> inputs(vstorage->num_levels());
+ for (size_t i = 0; i < inputs.size(); ++i) {
+ inputs[i].level = start_level + static_cast<int>(i);
+ }
+ // We always compact all the files, so always compress.
+ for (size_t loop = start_index; loop < sorted_runs.size(); loop++) {
+ auto& picking_sr = sorted_runs[loop];
+ if (picking_sr.level == 0) {
+ FileMetaData* f = picking_sr.file;
+ inputs[0].files.push_back(f);
+ } else {
+ auto& files = inputs[picking_sr.level - start_level].files;
+ for (auto* f : vstorage->LevelFiles(picking_sr.level)) {
+ files.push_back(f);
+ }
+ }
+ char file_num_buf[256];
+ picking_sr.DumpSizeInfo(file_num_buf, sizeof(file_num_buf), loop);
+ ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: size amp picking %s",
+ cf_name.c_str(), file_num_buf);
+ }
+
+ // output files at the bottom most level, unless it's reserved
+ int output_level = vstorage->num_levels() - 1;
+ // last level is reserved for the files ingested behind
+ if (ioptions_.allow_ingest_behind) {
+ assert(output_level > 1);
+ output_level--;
+ }
+
+ return new Compaction(
+ vstorage, ioptions_, mutable_cf_options, std::move(inputs),
+ output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
+ /* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
+ GetCompressionType(ioptions_, vstorage, mutable_cf_options,
+ output_level, 1),
+ /* grandparents */ {}, /* is manual */ false, score,
+ false /* deletion_compaction */,
+ CompactionReason::kUniversalSizeAmplification);
+}
+} // namespace rocksdb
+
+#endif // !ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compaction_picker_universal.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compaction_picker_universal.h b/thirdparty/rocksdb/db/compaction_picker_universal.h
new file mode 100644
index 0000000..3f2bed3
--- /dev/null
+++ b/thirdparty/rocksdb/db/compaction_picker_universal.h
@@ -0,0 +1,91 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#ifndef ROCKSDB_LITE
+
+#include "db/compaction_picker.h"
+
+namespace rocksdb {
+class UniversalCompactionPicker : public CompactionPicker {
+ public:
+ UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
+ const InternalKeyComparator* icmp)
+ : CompactionPicker(ioptions, icmp) {}
+ virtual Compaction* PickCompaction(const std::string& cf_name,
+ const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage,
+ LogBuffer* log_buffer) override;
+
+ virtual int MaxOutputLevel() const override { return NumberLevels() - 1; }
+
+ virtual bool NeedsCompaction(
+ const VersionStorageInfo* vstorage) const override;
+
+ private:
+ struct SortedRun {
+ SortedRun(int _level, FileMetaData* _file, uint64_t _size,
+ uint64_t _compensated_file_size, bool _being_compacted)
+ : level(_level),
+ file(_file),
+ size(_size),
+ compensated_file_size(_compensated_file_size),
+ being_compacted(_being_compacted) {
+ assert(compensated_file_size > 0);
+ assert(level != 0 || file != nullptr);
+ }
+
+ void Dump(char* out_buf, size_t out_buf_size,
+ bool print_path = false) const;
+
+ // sorted_run_count is added into the string to print
+ void DumpSizeInfo(char* out_buf, size_t out_buf_size,
+ size_t sorted_run_count) const;
+
+ int level;
+ // `file` Will be null for level > 0. For level = 0, the sorted run is
+ // for this file.
+ FileMetaData* file;
+ // For level > 0, `size` and `compensated_file_size` are sum of sizes all
+ // files in the level. `being_compacted` should be the same for all files
+ // in a non-zero level. Use the value here.
+ uint64_t size;
+ uint64_t compensated_file_size;
+ bool being_compacted;
+ };
+
+ // Pick Universal compaction to limit read amplification
+ Compaction* PickCompactionToReduceSortedRuns(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, double score, unsigned int ratio,
+ unsigned int num_files, const std::vector<SortedRun>& sorted_runs,
+ LogBuffer* log_buffer);
+
+ // Pick Universal compaction to limit space amplification.
+ Compaction* PickCompactionToReduceSizeAmp(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, double score,
+ const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer);
+
+ // Used in universal compaction when the enabled_trivial_move
+ // option is set. Checks whether there are any overlapping files
+ // in the input. Returns true if the input files are non
+ // overlapping.
+ bool IsInputFilesNonOverlapping(Compaction* c);
+
+ static std::vector<SortedRun> CalculateSortedRuns(
+ const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions);
+
+ // Pick a path ID to place a newly generated file, with its estimated file
+ // size.
+ static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
+ uint64_t file_size);
+};
+} // namespace rocksdb
+#endif // !ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/convenience.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/convenience.cc b/thirdparty/rocksdb/db/convenience.cc
new file mode 100644
index 0000000..8ee31ca
--- /dev/null
+++ b/thirdparty/rocksdb/db/convenience.cc
@@ -0,0 +1,58 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+
+#ifndef ROCKSDB_LITE
+
+#include "rocksdb/convenience.h"
+
+#include "db/db_impl.h"
+#include "util/cast_util.h"
+
+namespace rocksdb {
+
+void CancelAllBackgroundWork(DB* db, bool wait) {
+ (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
+ ->CancelAllBackgroundWork(wait);
+}
+
+Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
+ const Slice* begin, const Slice* end) {
+ return (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
+ ->DeleteFilesInRange(column_family, begin, end);
+}
+
+Status VerifySstFileChecksum(const Options& options,
+ const EnvOptions& env_options,
+ const std::string& file_path) {
+ unique_ptr<RandomAccessFile> file;
+ uint64_t file_size;
+ InternalKeyComparator internal_comparator(options.comparator);
+ ImmutableCFOptions ioptions(options);
+
+ Status s = ioptions.env->NewRandomAccessFile(file_path, &file, env_options);
+ if (s.ok()) {
+ s = ioptions.env->GetFileSize(file_path, &file_size);
+ } else {
+ return s;
+ }
+ unique_ptr<TableReader> table_reader;
+ std::unique_ptr<RandomAccessFileReader> file_reader(
+ new RandomAccessFileReader(std::move(file), file_path));
+ s = ioptions.table_factory->NewTableReader(
+ TableReaderOptions(ioptions, env_options, internal_comparator,
+ false /* skip_filters */, -1 /* level */),
+ std::move(file_reader), file_size, &table_reader,
+ false /* prefetch_index_and_filter_in_cache */);
+ if (!s.ok()) {
+ return s;
+ }
+ s = table_reader->VerifyChecksum();
+ return s;
+}
+
+} // namespace rocksdb
+
+#endif // ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_filesnapshot.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_filesnapshot.cc b/thirdparty/rocksdb/db/db_filesnapshot.cc
new file mode 100644
index 0000000..e266bf1
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_filesnapshot.cc
@@ -0,0 +1,149 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+
+#ifndef ROCKSDB_LITE
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <stdint.h>
+#include <algorithm>
+#include <string>
+#include "db/db_impl.h"
+#include "db/job_context.h"
+#include "db/version_set.h"
+#include "port/port.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "util/file_util.h"
+#include "util/filename.h"
+#include "util/mutexlock.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+Status DBImpl::DisableFileDeletions() {
+ InstrumentedMutexLock l(&mutex_);
+ ++disable_delete_obsolete_files_;
+ if (disable_delete_obsolete_files_ == 1) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Disabled");
+ } else {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "File Deletions Disabled, but already disabled. Counter: %d",
+ disable_delete_obsolete_files_);
+ }
+ return Status::OK();
+}
+
+Status DBImpl::EnableFileDeletions(bool force) {
+ // Job id == 0 means that this is not our background process, but rather
+ // user thread
+ JobContext job_context(0);
+ bool should_purge_files = false;
+ {
+ InstrumentedMutexLock l(&mutex_);
+ if (force) {
+ // if force, we need to enable file deletions right away
+ disable_delete_obsolete_files_ = 0;
+ } else if (disable_delete_obsolete_files_ > 0) {
+ --disable_delete_obsolete_files_;
+ }
+ if (disable_delete_obsolete_files_ == 0) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Enabled");
+ should_purge_files = true;
+ FindObsoleteFiles(&job_context, true);
+ } else {
+ ROCKS_LOG_WARN(
+ immutable_db_options_.info_log,
+ "File Deletions Enable, but not really enabled. Counter: %d",
+ disable_delete_obsolete_files_);
+ }
+ }
+ if (should_purge_files) {
+ PurgeObsoleteFiles(job_context);
+ }
+ job_context.Clean();
+ LogFlush(immutable_db_options_.info_log);
+ return Status::OK();
+}
+
+int DBImpl::IsFileDeletionsEnabled() const {
+ return disable_delete_obsolete_files_;
+}
+
+Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
+ uint64_t* manifest_file_size,
+ bool flush_memtable) {
+ *manifest_file_size = 0;
+
+ mutex_.Lock();
+
+ if (flush_memtable) {
+ // flush all dirty data to disk.
+ Status status;
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ if (cfd->IsDropped()) {
+ continue;
+ }
+ cfd->Ref();
+ mutex_.Unlock();
+ status = FlushMemTable(cfd, FlushOptions());
+ TEST_SYNC_POINT("DBImpl::GetLiveFiles:1");
+ TEST_SYNC_POINT("DBImpl::GetLiveFiles:2");
+ mutex_.Lock();
+ cfd->Unref();
+ if (!status.ok()) {
+ break;
+ }
+ }
+ versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();
+
+ if (!status.ok()) {
+ mutex_.Unlock();
+ ROCKS_LOG_ERROR(immutable_db_options_.info_log, "Cannot Flush data %s\n",
+ status.ToString().c_str());
+ return status;
+ }
+ }
+
+ // Make a set of all of the live *.sst files
+ std::vector<FileDescriptor> live;
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ if (cfd->IsDropped()) {
+ continue;
+ }
+ cfd->current()->AddLiveFiles(&live);
+ }
+
+ ret.clear();
+ ret.reserve(live.size() + 3); // *.sst + CURRENT + MANIFEST + OPTIONS
+
+ // create names of the live files. The names are not absolute
+ // paths, instead they are relative to dbname_;
+ for (auto live_file : live) {
+ ret.push_back(MakeTableFileName("", live_file.GetNumber()));
+ }
+
+ ret.push_back(CurrentFileName(""));
+ ret.push_back(DescriptorFileName("", versions_->manifest_file_number()));
+ ret.push_back(OptionsFileName("", versions_->options_file_number()));
+
+ // find length of manifest file while holding the mutex lock
+ *manifest_file_size = versions_->manifest_file_size();
+
+ mutex_.Unlock();
+ return Status::OK();
+}
+
+Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
+ return wal_manager_.GetSortedWalFiles(files);
+}
+
+}
+
+#endif // ROCKSDB_LITE
[43/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/version_set.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/version_set.cc b/thirdparty/leveldb-1.18/db/version_set.cc
deleted file mode 100755
index c59aef1..0000000
--- a/thirdparty/leveldb-1.18/db/version_set.cc
+++ /dev/null
@@ -1,1484 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/version_set.h"
-
-#include <algorithm>
-#include <stdio.h>
-#include "db/filename.h"
-#include "db/log_reader.h"
-#include "db/log_writer.h"
-#include "db/memtable.h"
-#include "db/table_cache.h"
-#include "leveldb/env.h"
-#include "leveldb/table_builder.h"
-#include "table/merger.h"
-#include "table/two_level_iterator.h"
-#include "util/coding.h"
-#include "util/logging.h"
-
-namespace leveldb {
-
-static const int kTargetFileSize = 2 * 1048576;
-
-// Maximum bytes of overlaps in grandparent (i.e., level+2) before we
-// stop building a single file in a level->level+1 compaction.
-static const int64_t kMaxGrandParentOverlapBytes = 10 * kTargetFileSize;
-
-// Maximum number of bytes in all compacted files. We avoid expanding
-// the lower level file set of a compaction if it would make the
-// total compaction cover more than this many bytes.
-static const int64_t kExpandedCompactionByteSizeLimit = 25 * kTargetFileSize;
-
-static double MaxBytesForLevel(int level) {
- // Note: the result for level zero is not really used since we set
- // the level-0 compaction threshold based on number of files.
- double result = 10 * 1048576.0; // Result for both level-0 and level-1
- while (level > 1) {
- result *= 10;
- level--;
- }
- return result;
-}
-
-static uint64_t MaxFileSizeForLevel(int level) {
- return kTargetFileSize; // We could vary per level to reduce number of files?
-}
-
-static int64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
- int64_t sum = 0;
- for (size_t i = 0; i < files.size(); i++) {
- sum += files[i]->file_size;
- }
- return sum;
-}
-
-Version::~Version() {
- assert(refs_ == 0);
-
- // Remove from linked list
- prev_->next_ = next_;
- next_->prev_ = prev_;
-
- // Drop references to files
- for (int level = 0; level < config::kNumLevels; level++) {
- for (size_t i = 0; i < files_[level].size(); i++) {
- FileMetaData* f = files_[level][i];
- assert(f->refs > 0);
- f->refs--;
- if (f->refs <= 0) {
- delete f;
- }
- }
- }
-}
-
-size_t FindFile(const InternalKeyComparator& icmp,
- const std::vector<FileMetaData*>& files,
- const Slice& key) {
- size_t left = 0;
- size_t right = files.size();
- while (left < right) {
- size_t mid = (left + right) / 2;
- const FileMetaData* f = files[mid];
- if (icmp.InternalKeyComparator::Compare(f->largest.Encode(), key) < 0) {
- // Key at "mid.largest" is < "target". Therefore all
- // files at or before "mid" are uninteresting.
- left = mid + 1;
- } else {
- // Key at "mid.largest" is >= "target". Therefore all files
- // after "mid" are uninteresting.
- right = mid;
- }
- }
- return right;
-}
-
-static bool AfterFile(const Comparator* ucmp,
- const Slice* user_key, const FileMetaData* f) {
- // NULL user_key occurs before all keys and is therefore never after *f
- return (user_key != NULL &&
- ucmp->Compare(*user_key, f->largest.user_key()) > 0);
-}
-
-static bool BeforeFile(const Comparator* ucmp,
- const Slice* user_key, const FileMetaData* f) {
- // NULL user_key occurs after all keys and is therefore never before *f
- return (user_key != NULL &&
- ucmp->Compare(*user_key, f->smallest.user_key()) < 0);
-}
-
-bool SomeFileOverlapsRange(
- const InternalKeyComparator& icmp,
- bool disjoint_sorted_files,
- const std::vector<FileMetaData*>& files,
- const Slice* smallest_user_key,
- const Slice* largest_user_key) {
- const Comparator* ucmp = icmp.user_comparator();
- if (!disjoint_sorted_files) {
- // Need to check against all files
- for (size_t i = 0; i < files.size(); i++) {
- const FileMetaData* f = files[i];
- if (AfterFile(ucmp, smallest_user_key, f) ||
- BeforeFile(ucmp, largest_user_key, f)) {
- // No overlap
- } else {
- return true; // Overlap
- }
- }
- return false;
- }
-
- // Binary search over file list
- size_t index = 0;
- if (smallest_user_key != NULL) {
- // Find the earliest possible internal key for smallest_user_key
- InternalKey small(*smallest_user_key, kMaxSequenceNumber,kValueTypeForSeek);
- index = FindFile(icmp, files, small.Encode());
- }
-
- if (index >= files.size()) {
- // beginning of range is after all files, so no overlap.
- return false;
- }
-
- return !BeforeFile(ucmp, largest_user_key, files[index]);
-}
-
-// An internal iterator. For a given version/level pair, yields
-// information about the files in the level. For a given entry, key()
-// is the largest key that occurs in the file, and value() is an
-// 16-byte value containing the file number and file size, both
-// encoded using EncodeFixed64.
-class Version::LevelFileNumIterator : public Iterator {
- public:
- LevelFileNumIterator(const InternalKeyComparator& icmp,
- const std::vector<FileMetaData*>* flist)
- : icmp_(icmp),
- flist_(flist),
- index_(flist->size()) { // Marks as invalid
- }
- virtual bool Valid() const {
- return index_ < flist_->size();
- }
- virtual void Seek(const Slice& target) {
- index_ = FindFile(icmp_, *flist_, target);
- }
- virtual void SeekToFirst() { index_ = 0; }
- virtual void SeekToLast() {
- index_ = flist_->empty() ? 0 : flist_->size() - 1;
- }
- virtual void Next() {
- assert(Valid());
- index_++;
- }
- virtual void Prev() {
- assert(Valid());
- if (index_ == 0) {
- index_ = flist_->size(); // Marks as invalid
- } else {
- index_--;
- }
- }
- Slice key() const {
- assert(Valid());
- return (*flist_)[index_]->largest.Encode();
- }
- Slice value() const {
- assert(Valid());
- EncodeFixed64(value_buf_, (*flist_)[index_]->number);
- EncodeFixed64(value_buf_+8, (*flist_)[index_]->file_size);
- return Slice(value_buf_, sizeof(value_buf_));
- }
- virtual Status status() const { return Status::OK(); }
- private:
- const InternalKeyComparator icmp_;
- const std::vector<FileMetaData*>* const flist_;
- size_t index_;
-
- // Backing store for value(). Holds the file number and size.
- mutable char value_buf_[16];
-};
-
-static Iterator* GetFileIterator(void* arg,
- const ReadOptions& options,
- const Slice& file_value) {
- TableCache* cache = reinterpret_cast<TableCache*>(arg);
- if (file_value.size() != 16) {
- return NewErrorIterator(
- Status::Corruption("FileReader invoked with unexpected value"));
- } else {
- return cache->NewIterator(options,
- DecodeFixed64(file_value.data()),
- DecodeFixed64(file_value.data() + 8));
- }
-}
-
-Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
- int level) const {
- return NewTwoLevelIterator(
- new LevelFileNumIterator(vset_->icmp_, &files_[level]),
- &GetFileIterator, vset_->table_cache_, options);
-}
-
-void Version::AddIterators(const ReadOptions& options,
- std::vector<Iterator*>* iters) {
- // Merge all level zero files together since they may overlap
- for (size_t i = 0; i < files_[0].size(); i++) {
- iters->push_back(
- vset_->table_cache_->NewIterator(
- options, files_[0][i]->number, files_[0][i]->file_size));
- }
-
- // For levels > 0, we can use a concatenating iterator that sequentially
- // walks through the non-overlapping files in the level, opening them
- // lazily.
- for (int level = 1; level < config::kNumLevels; level++) {
- if (!files_[level].empty()) {
- iters->push_back(NewConcatenatingIterator(options, level));
- }
- }
-}
-
-// Callback from TableCache::Get()
-namespace {
-enum SaverState {
- kNotFound,
- kFound,
- kDeleted,
- kCorrupt,
-};
-struct Saver {
- SaverState state;
- const Comparator* ucmp;
- Slice user_key;
- std::string* value;
-};
-}
-static void SaveValue(void* arg, const Slice& ikey, const Slice& v) {
- Saver* s = reinterpret_cast<Saver*>(arg);
- ParsedInternalKey parsed_key;
- if (!ParseInternalKey(ikey, &parsed_key)) {
- s->state = kCorrupt;
- } else {
- if (s->ucmp->Compare(parsed_key.user_key, s->user_key) == 0) {
- s->state = (parsed_key.type == kTypeValue) ? kFound : kDeleted;
- if (s->state == kFound) {
- s->value->assign(v.data(), v.size());
- }
- }
- }
-}
-
-static bool NewestFirst(FileMetaData* a, FileMetaData* b) {
- return a->number > b->number;
-}
-
-void Version::ForEachOverlapping(Slice user_key, Slice internal_key,
- void* arg,
- bool (*func)(void*, int, FileMetaData*)) {
- // TODO(sanjay): Change Version::Get() to use this function.
- const Comparator* ucmp = vset_->icmp_.user_comparator();
-
- // Search level-0 in order from newest to oldest.
- std::vector<FileMetaData*> tmp;
- tmp.reserve(files_[0].size());
- for (uint32_t i = 0; i < files_[0].size(); i++) {
- FileMetaData* f = files_[0][i];
- if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 &&
- ucmp->Compare(user_key, f->largest.user_key()) <= 0) {
- tmp.push_back(f);
- }
- }
- if (!tmp.empty()) {
- std::sort(tmp.begin(), tmp.end(), NewestFirst);
- for (uint32_t i = 0; i < tmp.size(); i++) {
- if (!(*func)(arg, 0, tmp[i])) {
- return;
- }
- }
- }
-
- // Search other levels.
- for (int level = 1; level < config::kNumLevels; level++) {
- size_t num_files = files_[level].size();
- if (num_files == 0) continue;
-
- // Binary search to find earliest index whose largest key >= internal_key.
- size_t index = FindFile(vset_->icmp_, files_[level], internal_key);
- if (index < num_files) {
- FileMetaData* f = files_[level][index];
- if (ucmp->Compare(user_key, f->smallest.user_key()) < 0) {
- // All of "f" is past any data for user_key
- } else {
- if (!(*func)(arg, level, f)) {
- return;
- }
- }
- }
- }
-}
-
-Status Version::Get(const ReadOptions& options,
- const LookupKey& k,
- std::string* value,
- GetStats* stats) {
- Slice ikey = k.internal_key();
- Slice user_key = k.user_key();
- const Comparator* ucmp = vset_->icmp_.user_comparator();
- Status s;
-
- stats->seek_file = NULL;
- stats->seek_file_level = -1;
- FileMetaData* last_file_read = NULL;
- int last_file_read_level = -1;
-
- // We can search level-by-level since entries never hop across
- // levels. Therefore we are guaranteed that if we find data
- // in an smaller level, later levels are irrelevant.
- std::vector<FileMetaData*> tmp;
- FileMetaData* tmp2;
- for (int level = 0; level < config::kNumLevels; level++) {
- size_t num_files = files_[level].size();
- if (num_files == 0) continue;
-
- // Get the list of files to search in this level
- FileMetaData* const* files = &files_[level][0];
- if (level == 0) {
- // Level-0 files may overlap each other. Find all files that
- // overlap user_key and process them in order from newest to oldest.
- tmp.reserve(num_files);
- for (uint32_t i = 0; i < num_files; i++) {
- FileMetaData* f = files[i];
- if (ucmp->Compare(user_key, f->smallest.user_key()) >= 0 &&
- ucmp->Compare(user_key, f->largest.user_key()) <= 0) {
- tmp.push_back(f);
- }
- }
- if (tmp.empty()) continue;
-
- std::sort(tmp.begin(), tmp.end(), NewestFirst);
- files = &tmp[0];
- num_files = tmp.size();
- } else {
- // Binary search to find earliest index whose largest key >= ikey.
- size_t index = FindFile(vset_->icmp_, files_[level], ikey);
- if (index >= num_files) {
- files = NULL;
- num_files = 0;
- } else {
- tmp2 = files[index];
- if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) {
- // All of "tmp2" is past any data for user_key
- files = NULL;
- num_files = 0;
- } else {
- files = &tmp2;
- num_files = 1;
- }
- }
- }
-
- for (uint32_t i = 0; i < num_files; ++i) {
- if (last_file_read != NULL && stats->seek_file == NULL) {
- // We have had more than one seek for this read. Charge the 1st file.
- stats->seek_file = last_file_read;
- stats->seek_file_level = last_file_read_level;
- }
-
- FileMetaData* f = files[i];
- last_file_read = f;
- last_file_read_level = level;
-
- Saver saver;
- saver.state = kNotFound;
- saver.ucmp = ucmp;
- saver.user_key = user_key;
- saver.value = value;
- s = vset_->table_cache_->Get(options, f->number, f->file_size,
- ikey, &saver, SaveValue);
- if (!s.ok()) {
- return s;
- }
- switch (saver.state) {
- case kNotFound:
- break; // Keep searching in other files
- case kFound:
- return s;
- case kDeleted:
- s = Status::NotFound(Slice()); // Use empty error message for speed
- return s;
- case kCorrupt:
- s = Status::Corruption("corrupted key for ", user_key);
- return s;
- }
- }
- }
-
- return Status::NotFound(Slice()); // Use an empty error message for speed
-}
-
-bool Version::UpdateStats(const GetStats& stats) {
- FileMetaData* f = stats.seek_file;
- if (f != NULL) {
- f->allowed_seeks--;
- if (f->allowed_seeks <= 0 && file_to_compact_ == NULL) {
- file_to_compact_ = f;
- file_to_compact_level_ = stats.seek_file_level;
- return true;
- }
- }
- return false;
-}
-
-bool Version::RecordReadSample(Slice internal_key) {
- ParsedInternalKey ikey;
- if (!ParseInternalKey(internal_key, &ikey)) {
- return false;
- }
-
- struct State {
- GetStats stats; // Holds first matching file
- int matches;
-
- static bool Match(void* arg, int level, FileMetaData* f) {
- State* state = reinterpret_cast<State*>(arg);
- state->matches++;
- if (state->matches == 1) {
- // Remember first match.
- state->stats.seek_file = f;
- state->stats.seek_file_level = level;
- }
- // We can stop iterating once we have a second match.
- return state->matches < 2;
- }
- };
-
- State state;
- state.matches = 0;
- ForEachOverlapping(ikey.user_key, internal_key, &state, &State::Match);
-
- // Must have at least two matches since we want to merge across
- // files. But what if we have a single file that contains many
- // overwrites and deletions? Should we have another mechanism for
- // finding such files?
- if (state.matches >= 2) {
- // 1MB cost is about 1 seek (see comment in Builder::Apply).
- return UpdateStats(state.stats);
- }
- return false;
-}
-
-void Version::Ref() {
- ++refs_;
-}
-
-void Version::Unref() {
- assert(this != &vset_->dummy_versions_);
- assert(refs_ >= 1);
- --refs_;
- if (refs_ == 0) {
- delete this;
- }
-}
-
-bool Version::OverlapInLevel(int level,
- const Slice* smallest_user_key,
- const Slice* largest_user_key) {
- return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level],
- smallest_user_key, largest_user_key);
-}
-
-int Version::PickLevelForMemTableOutput(
- const Slice& smallest_user_key,
- const Slice& largest_user_key) {
- int level = 0;
- if (!OverlapInLevel(0, &smallest_user_key, &largest_user_key)) {
- // Push to next level if there is no overlap in next level,
- // and the #bytes overlapping in the level after that are limited.
- InternalKey start(smallest_user_key, kMaxSequenceNumber, kValueTypeForSeek);
- InternalKey limit(largest_user_key, 0, static_cast<ValueType>(0));
- std::vector<FileMetaData*> overlaps;
- while (level < config::kMaxMemCompactLevel) {
- if (OverlapInLevel(level + 1, &smallest_user_key, &largest_user_key)) {
- break;
- }
- if (level + 2 < config::kNumLevels) {
- // Check that file does not overlap too many grandparent bytes.
- GetOverlappingInputs(level + 2, &start, &limit, &overlaps);
- const int64_t sum = TotalFileSize(overlaps);
- if (sum > kMaxGrandParentOverlapBytes) {
- break;
- }
- }
- level++;
- }
- }
- return level;
-}
-
-// Store in "*inputs" all files in "level" that overlap [begin,end]
-void Version::GetOverlappingInputs(
- int level,
- const InternalKey* begin,
- const InternalKey* end,
- std::vector<FileMetaData*>* inputs) {
- assert(level >= 0);
- assert(level < config::kNumLevels);
- inputs->clear();
- Slice user_begin, user_end;
- if (begin != NULL) {
- user_begin = begin->user_key();
- }
- if (end != NULL) {
- user_end = end->user_key();
- }
- const Comparator* user_cmp = vset_->icmp_.user_comparator();
- for (size_t i = 0; i < files_[level].size(); ) {
- FileMetaData* f = files_[level][i++];
- const Slice file_start = f->smallest.user_key();
- const Slice file_limit = f->largest.user_key();
- if (begin != NULL && user_cmp->Compare(file_limit, user_begin) < 0) {
- // "f" is completely before specified range; skip it
- } else if (end != NULL && user_cmp->Compare(file_start, user_end) > 0) {
- // "f" is completely after specified range; skip it
- } else {
- inputs->push_back(f);
- if (level == 0) {
- // Level-0 files may overlap each other. So check if the newly
- // added file has expanded the range. If so, restart search.
- if (begin != NULL && user_cmp->Compare(file_start, user_begin) < 0) {
- user_begin = file_start;
- inputs->clear();
- i = 0;
- } else if (end != NULL && user_cmp->Compare(file_limit, user_end) > 0) {
- user_end = file_limit;
- inputs->clear();
- i = 0;
- }
- }
- }
- }
-}
-
-std::string Version::DebugString() const {
- std::string r;
- for (int level = 0; level < config::kNumLevels; level++) {
- // E.g.,
- // --- level 1 ---
- // 17:123['a' .. 'd']
- // 20:43['e' .. 'g']
- r.append("--- level ");
- AppendNumberTo(&r, level);
- r.append(" ---\n");
- const std::vector<FileMetaData*>& files = files_[level];
- for (size_t i = 0; i < files.size(); i++) {
- r.push_back(' ');
- AppendNumberTo(&r, files[i]->number);
- r.push_back(':');
- AppendNumberTo(&r, files[i]->file_size);
- r.append("[");
- r.append(files[i]->smallest.DebugString());
- r.append(" .. ");
- r.append(files[i]->largest.DebugString());
- r.append("]\n");
- }
- }
- return r;
-}
-
-// A helper class so we can efficiently apply a whole sequence
-// of edits to a particular state without creating intermediate
-// Versions that contain full copies of the intermediate state.
-class VersionSet::Builder {
- private:
- // Helper to sort by v->files_[file_number].smallest
- struct BySmallestKey {
- const InternalKeyComparator* internal_comparator;
-
- bool operator()(FileMetaData* f1, FileMetaData* f2) const {
- int r = internal_comparator->Compare(f1->smallest, f2->smallest);
- if (r != 0) {
- return (r < 0);
- } else {
- // Break ties by file number
- return (f1->number < f2->number);
- }
- }
- };
-
- typedef std::set<FileMetaData*, BySmallestKey> FileSet;
- struct LevelState {
- std::set<uint64_t> deleted_files;
- FileSet* added_files;
- };
-
- VersionSet* vset_;
- Version* base_;
- LevelState levels_[config::kNumLevels];
-
- public:
- // Initialize a builder with the files from *base and other info from *vset
- Builder(VersionSet* vset, Version* base)
- : vset_(vset),
- base_(base) {
- base_->Ref();
- BySmallestKey cmp;
- cmp.internal_comparator = &vset_->icmp_;
- for (int level = 0; level < config::kNumLevels; level++) {
- levels_[level].added_files = new FileSet(cmp);
- }
- }
-
- ~Builder() {
- for (int level = 0; level < config::kNumLevels; level++) {
- const FileSet* added = levels_[level].added_files;
- std::vector<FileMetaData*> to_unref;
- to_unref.reserve(added->size());
- for (FileSet::const_iterator it = added->begin();
- it != added->end(); ++it) {
- to_unref.push_back(*it);
- }
- delete added;
- for (uint32_t i = 0; i < to_unref.size(); i++) {
- FileMetaData* f = to_unref[i];
- f->refs--;
- if (f->refs <= 0) {
- delete f;
- }
- }
- }
- base_->Unref();
- }
-
- // Apply all of the edits in *edit to the current state.
- void Apply(VersionEdit* edit) {
- // Update compaction pointers
- for (size_t i = 0; i < edit->compact_pointers_.size(); i++) {
- const int level = edit->compact_pointers_[i].first;
- vset_->compact_pointer_[level] =
- edit->compact_pointers_[i].second.Encode().ToString();
- }
-
- // Delete files
- const VersionEdit::DeletedFileSet& del = edit->deleted_files_;
- for (VersionEdit::DeletedFileSet::const_iterator iter = del.begin();
- iter != del.end();
- ++iter) {
- const int level = iter->first;
- const uint64_t number = iter->second;
- levels_[level].deleted_files.insert(number);
- }
-
- // Add new files
- for (size_t i = 0; i < edit->new_files_.size(); i++) {
- const int level = edit->new_files_[i].first;
- FileMetaData* f = new FileMetaData(edit->new_files_[i].second);
- f->refs = 1;
-
- // We arrange to automatically compact this file after
- // a certain number of seeks. Let's assume:
- // (1) One seek costs 10ms
- // (2) Writing or reading 1MB costs 10ms (100MB/s)
- // (3) A compaction of 1MB does 25MB of IO:
- // 1MB read from this level
- // 10-12MB read from next level (boundaries may be misaligned)
- // 10-12MB written to next level
- // This implies that 25 seeks cost the same as the compaction
- // of 1MB of data. I.e., one seek costs approximately the
- // same as the compaction of 40KB of data. We are a little
- // conservative and allow approximately one seek for every 16KB
- // of data before triggering a compaction.
- f->allowed_seeks = (f->file_size / 16384);
- if (f->allowed_seeks < 100) f->allowed_seeks = 100;
-
- levels_[level].deleted_files.erase(f->number);
- levels_[level].added_files->insert(f);
- }
- }
-
- // Save the current state in *v.
- void SaveTo(Version* v) {
- BySmallestKey cmp;
- cmp.internal_comparator = &vset_->icmp_;
- for (int level = 0; level < config::kNumLevels; level++) {
- // Merge the set of added files with the set of pre-existing files.
- // Drop any deleted files. Store the result in *v.
- const std::vector<FileMetaData*>& base_files = base_->files_[level];
- std::vector<FileMetaData*>::const_iterator base_iter = base_files.begin();
- std::vector<FileMetaData*>::const_iterator base_end = base_files.end();
- const FileSet* added = levels_[level].added_files;
- v->files_[level].reserve(base_files.size() + added->size());
- for (FileSet::const_iterator added_iter = added->begin();
- added_iter != added->end();
- ++added_iter) {
- // Add all smaller files listed in base_
- for (std::vector<FileMetaData*>::const_iterator bpos
- = std::upper_bound(base_iter, base_end, *added_iter, cmp);
- base_iter != bpos;
- ++base_iter) {
- MaybeAddFile(v, level, *base_iter);
- }
-
- MaybeAddFile(v, level, *added_iter);
- }
-
- // Add remaining base files
- for (; base_iter != base_end; ++base_iter) {
- MaybeAddFile(v, level, *base_iter);
- }
-
-#ifndef NDEBUG
- // Make sure there is no overlap in levels > 0
- if (level > 0) {
- for (uint32_t i = 1; i < v->files_[level].size(); i++) {
- const InternalKey& prev_end = v->files_[level][i-1]->largest;
- const InternalKey& this_begin = v->files_[level][i]->smallest;
- if (vset_->icmp_.Compare(prev_end, this_begin) >= 0) {
- fprintf(stderr, "overlapping ranges in same level %s vs. %s\n",
- prev_end.DebugString().c_str(),
- this_begin.DebugString().c_str());
- abort();
- }
- }
- }
-#endif
- }
- }
-
- void MaybeAddFile(Version* v, int level, FileMetaData* f) {
- if (levels_[level].deleted_files.count(f->number) > 0) {
- // File is deleted: do nothing
- } else {
- std::vector<FileMetaData*>* files = &v->files_[level];
- if (level > 0 && !files->empty()) {
- // Must not overlap
- assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest,
- f->smallest) < 0);
- }
- f->refs++;
- files->push_back(f);
- }
- }
-};
-
-VersionSet::VersionSet(const std::string& dbname,
- const Options* options,
- TableCache* table_cache,
- const InternalKeyComparator* cmp)
- : env_(options->env),
- dbname_(dbname),
- options_(options),
- table_cache_(table_cache),
- icmp_(*cmp),
- next_file_number_(2),
- manifest_file_number_(0), // Filled by Recover()
- last_sequence_(0),
- log_number_(0),
- prev_log_number_(0),
- descriptor_file_(NULL),
- descriptor_log_(NULL),
- dummy_versions_(this),
- current_(NULL) {
- AppendVersion(new Version(this));
-}
-
-VersionSet::~VersionSet() {
- current_->Unref();
- assert(dummy_versions_.next_ == &dummy_versions_); // List must be empty
- delete descriptor_log_;
- delete descriptor_file_;
-}
-
-void VersionSet::AppendVersion(Version* v) {
- // Make "v" current
- assert(v->refs_ == 0);
- assert(v != current_);
- if (current_ != NULL) {
- current_->Unref();
- }
- current_ = v;
- v->Ref();
-
- // Append to linked list
- v->prev_ = dummy_versions_.prev_;
- v->next_ = &dummy_versions_;
- v->prev_->next_ = v;
- v->next_->prev_ = v;
-}
-
-Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu) {
- if (edit->has_log_number_) {
- assert(edit->log_number_ >= log_number_);
- assert(edit->log_number_ < next_file_number_);
- } else {
- edit->SetLogNumber(log_number_);
- }
-
- if (!edit->has_prev_log_number_) {
- edit->SetPrevLogNumber(prev_log_number_);
- }
-
- edit->SetNextFile(next_file_number_);
- edit->SetLastSequence(last_sequence_);
-
- Version* v = new Version(this);
- {
- Builder builder(this, current_);
- builder.Apply(edit);
- builder.SaveTo(v);
- }
- Finalize(v);
-
- // Initialize new descriptor log file if necessary by creating
- // a temporary file that contains a snapshot of the current version.
- std::string new_manifest_file;
- Status s;
- if (descriptor_log_ == NULL) {
- // No reason to unlock *mu here since we only hit this path in the
- // first call to LogAndApply (when opening the database).
- assert(descriptor_file_ == NULL);
- new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_);
- edit->SetNextFile(next_file_number_);
- s = env_->NewWritableFile(new_manifest_file, &descriptor_file_);
- if (s.ok()) {
- descriptor_log_ = new log::Writer(descriptor_file_);
- s = WriteSnapshot(descriptor_log_);
- }
- }
-
- // Unlock during expensive MANIFEST log write
- {
- mu->Unlock();
-
- // Write new record to MANIFEST log
- if (s.ok()) {
- std::string record;
- edit->EncodeTo(&record);
- s = descriptor_log_->AddRecord(record);
- if (s.ok()) {
- s = descriptor_file_->Sync();
- }
- if (!s.ok()) {
- Log(options_->info_log, "MANIFEST write: %s\n", s.ToString().c_str());
- }
- }
-
- // If we just created a new descriptor file, install it by writing a
- // new CURRENT file that points to it.
- if (s.ok() && !new_manifest_file.empty()) {
- s = SetCurrentFile(env_, dbname_, manifest_file_number_);
- }
-
- mu->Lock();
- }
-
- // Install the new version
- if (s.ok()) {
- AppendVersion(v);
- log_number_ = edit->log_number_;
- prev_log_number_ = edit->prev_log_number_;
- } else {
- delete v;
- if (!new_manifest_file.empty()) {
- delete descriptor_log_;
- delete descriptor_file_;
- descriptor_log_ = NULL;
- descriptor_file_ = NULL;
- env_->DeleteFile(new_manifest_file);
- }
- }
-
- return s;
-}
-
-Status VersionSet::Recover() {
- struct LogReporter : public log::Reader::Reporter {
- Status* status;
- virtual void Corruption(size_t bytes, const Status& s) {
- if (this->status->ok()) *this->status = s;
- }
- };
-
- // Read "CURRENT" file, which contains a pointer to the current manifest file
- std::string current;
- Status s = ReadFileToString(env_, CurrentFileName(dbname_), ¤t);
- if (!s.ok()) {
- return s;
- }
- if (current.empty() || current[current.size()-1] != '\n') {
- return Status::Corruption("CURRENT file does not end with newline");
- }
- current.resize(current.size() - 1);
-
- std::string dscname = dbname_ + "/" + current;
- SequentialFile* file;
- s = env_->NewSequentialFile(dscname, &file);
- if (!s.ok()) {
- return s;
- }
-
- bool have_log_number = false;
- bool have_prev_log_number = false;
- bool have_next_file = false;
- bool have_last_sequence = false;
- uint64_t next_file = 0;
- uint64_t last_sequence = 0;
- uint64_t log_number = 0;
- uint64_t prev_log_number = 0;
- Builder builder(this, current_);
-
- {
- LogReporter reporter;
- reporter.status = &s;
- log::Reader reader(file, &reporter, true/*checksum*/, 0/*initial_offset*/);
- Slice record;
- std::string scratch;
- while (reader.ReadRecord(&record, &scratch) && s.ok()) {
- VersionEdit edit;
- s = edit.DecodeFrom(record);
- if (s.ok()) {
- if (edit.has_comparator_ &&
- edit.comparator_ != icmp_.user_comparator()->Name()) {
- s = Status::InvalidArgument(
- edit.comparator_ + " does not match existing comparator ",
- icmp_.user_comparator()->Name());
- }
- }
-
- if (s.ok()) {
- builder.Apply(&edit);
- }
-
- if (edit.has_log_number_) {
- log_number = edit.log_number_;
- have_log_number = true;
- }
-
- if (edit.has_prev_log_number_) {
- prev_log_number = edit.prev_log_number_;
- have_prev_log_number = true;
- }
-
- if (edit.has_next_file_number_) {
- next_file = edit.next_file_number_;
- have_next_file = true;
- }
-
- if (edit.has_last_sequence_) {
- last_sequence = edit.last_sequence_;
- have_last_sequence = true;
- }
- }
- }
- delete file;
- file = NULL;
-
- if (s.ok()) {
- if (!have_next_file) {
- s = Status::Corruption("no meta-nextfile entry in descriptor");
- } else if (!have_log_number) {
- s = Status::Corruption("no meta-lognumber entry in descriptor");
- } else if (!have_last_sequence) {
- s = Status::Corruption("no last-sequence-number entry in descriptor");
- }
-
- if (!have_prev_log_number) {
- prev_log_number = 0;
- }
-
- MarkFileNumberUsed(prev_log_number);
- MarkFileNumberUsed(log_number);
- }
-
- if (s.ok()) {
- Version* v = new Version(this);
- builder.SaveTo(v);
- // Install recovered version
- Finalize(v);
- AppendVersion(v);
- manifest_file_number_ = next_file;
- next_file_number_ = next_file + 1;
- last_sequence_ = last_sequence;
- log_number_ = log_number;
- prev_log_number_ = prev_log_number;
- }
-
- return s;
-}
-
-void VersionSet::MarkFileNumberUsed(uint64_t number) {
- if (next_file_number_ <= number) {
- next_file_number_ = number + 1;
- }
-}
-
-void VersionSet::Finalize(Version* v) {
- // Precomputed best level for next compaction
- int best_level = -1;
- double best_score = -1;
-
- for (int level = 0; level < config::kNumLevels-1; level++) {
- double score;
- if (level == 0) {
- // We treat level-0 specially by bounding the number of files
- // instead of number of bytes for two reasons:
- //
- // (1) With larger write-buffer sizes, it is nice not to do too
- // many level-0 compactions.
- //
- // (2) The files in level-0 are merged on every read and
- // therefore we wish to avoid too many files when the individual
- // file size is small (perhaps because of a small write-buffer
- // setting, or very high compression ratios, or lots of
- // overwrites/deletions).
- score = v->files_[level].size() /
- static_cast<double>(config::kL0_CompactionTrigger);
- } else {
- // Compute the ratio of current size to size limit.
- const uint64_t level_bytes = TotalFileSize(v->files_[level]);
- score = static_cast<double>(level_bytes) / MaxBytesForLevel(level);
- }
-
- if (score > best_score) {
- best_level = level;
- best_score = score;
- }
- }
-
- v->compaction_level_ = best_level;
- v->compaction_score_ = best_score;
-}
-
-Status VersionSet::WriteSnapshot(log::Writer* log) {
- // TODO: Break up into multiple records to reduce memory usage on recovery?
-
- // Save metadata
- VersionEdit edit;
- edit.SetComparatorName(icmp_.user_comparator()->Name());
-
- // Save compaction pointers
- for (int level = 0; level < config::kNumLevels; level++) {
- if (!compact_pointer_[level].empty()) {
- InternalKey key;
- key.DecodeFrom(compact_pointer_[level]);
- edit.SetCompactPointer(level, key);
- }
- }
-
- // Save files
- for (int level = 0; level < config::kNumLevels; level++) {
- const std::vector<FileMetaData*>& files = current_->files_[level];
- for (size_t i = 0; i < files.size(); i++) {
- const FileMetaData* f = files[i];
- edit.AddFile(level, f->number, f->file_size, f->smallest, f->largest);
- }
- }
-
- std::string record;
- edit.EncodeTo(&record);
- return log->AddRecord(record);
-}
-
-size_t VersionSet::NumLevelFiles(int level) const {
- assert(level >= 0);
- assert(level < config::kNumLevels);
- return current_->files_[level].size();
-}
-
-const char* VersionSet::LevelSummary(LevelSummaryStorage* scratch) const {
- // Update code if kNumLevels changes
- assert(config::kNumLevels == 7);
- snprintf(scratch->buffer, sizeof(scratch->buffer),
- "files[ %d %d %d %d %d %d %d ]",
- int(current_->files_[0].size()),
- int(current_->files_[1].size()),
- int(current_->files_[2].size()),
- int(current_->files_[3].size()),
- int(current_->files_[4].size()),
- int(current_->files_[5].size()),
- int(current_->files_[6].size()));
- return scratch->buffer;
-}
-
-uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
- uint64_t result = 0;
- for (int level = 0; level < config::kNumLevels; level++) {
- const std::vector<FileMetaData*>& files = v->files_[level];
- for (size_t i = 0; i < files.size(); i++) {
- if (icmp_.Compare(files[i]->largest, ikey) <= 0) {
- // Entire file is before "ikey", so just add the file size
- result += files[i]->file_size;
- } else if (icmp_.Compare(files[i]->smallest, ikey) > 0) {
- // Entire file is after "ikey", so ignore
- if (level > 0) {
- // Files other than level 0 are sorted by meta->smallest, so
- // no further files in this level will contain data for
- // "ikey".
- break;
- }
- } else {
- // "ikey" falls in the range for this table. Add the
- // approximate offset of "ikey" within the table.
- Table* tableptr;
- Iterator* iter = table_cache_->NewIterator(
- ReadOptions(), files[i]->number, files[i]->file_size, &tableptr);
- if (tableptr != NULL) {
- result += tableptr->ApproximateOffsetOf(ikey.Encode());
- }
- delete iter;
- }
- }
- }
- return result;
-}
-
-void VersionSet::AddLiveFiles(std::set<uint64_t>* live) {
- for (Version* v = dummy_versions_.next_;
- v != &dummy_versions_;
- v = v->next_) {
- for (int level = 0; level < config::kNumLevels; level++) {
- const std::vector<FileMetaData*>& files = v->files_[level];
- for (size_t i = 0; i < files.size(); i++) {
- live->insert(files[i]->number);
- }
- }
- }
-}
-
-int64_t VersionSet::NumLevelBytes(int level) const {
- assert(level >= 0);
- assert(level < config::kNumLevels);
- return TotalFileSize(current_->files_[level]);
-}
-
-int64_t VersionSet::MaxNextLevelOverlappingBytes() {
- int64_t result = 0;
- std::vector<FileMetaData*> overlaps;
- for (int level = 1; level < config::kNumLevels - 1; level++) {
- for (size_t i = 0; i < current_->files_[level].size(); i++) {
- const FileMetaData* f = current_->files_[level][i];
- current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest,
- &overlaps);
- const int64_t sum = TotalFileSize(overlaps);
- if (sum > result) {
- result = sum;
- }
- }
- }
- return result;
-}
-
-// Stores the minimal range that covers all entries in inputs in
-// *smallest, *largest.
-// REQUIRES: inputs is not empty
-void VersionSet::GetRange(const std::vector<FileMetaData*>& inputs,
- InternalKey* smallest,
- InternalKey* largest) {
- assert(!inputs.empty());
- smallest->Clear();
- largest->Clear();
- for (size_t i = 0; i < inputs.size(); i++) {
- FileMetaData* f = inputs[i];
- if (i == 0) {
- *smallest = f->smallest;
- *largest = f->largest;
- } else {
- if (icmp_.Compare(f->smallest, *smallest) < 0) {
- *smallest = f->smallest;
- }
- if (icmp_.Compare(f->largest, *largest) > 0) {
- *largest = f->largest;
- }
- }
- }
-}
-
-// Stores the minimal range that covers all entries in inputs1 and inputs2
-// in *smallest, *largest.
-// REQUIRES: inputs is not empty
-void VersionSet::GetRange2(const std::vector<FileMetaData*>& inputs1,
- const std::vector<FileMetaData*>& inputs2,
- InternalKey* smallest,
- InternalKey* largest) {
- std::vector<FileMetaData*> all = inputs1;
- all.insert(all.end(), inputs2.begin(), inputs2.end());
- GetRange(all, smallest, largest);
-}
-
-Iterator* VersionSet::MakeInputIterator(Compaction* c) {
- ReadOptions options;
- options.verify_checksums = options_->paranoid_checks;
- options.fill_cache = false;
-
- // Level-0 files have to be merged together. For other levels,
- // we will make a concatenating iterator per level.
- // TODO(opt): use concatenating iterator for level-0 if there is no overlap
- const size_t space = (c->level() == 0 ? c->inputs_[0].size() + 1 : 2);
- Iterator** list = new Iterator*[space];
- int num = 0;
- for (int which = 0; which < 2; which++) {
- if (!c->inputs_[which].empty()) {
- if (c->level() + which == 0) {
- const std::vector<FileMetaData*>& files = c->inputs_[which];
- for (size_t i = 0; i < files.size(); i++) {
- list[num++] = table_cache_->NewIterator(
- options, files[i]->number, files[i]->file_size);
- }
- } else {
- // Create concatenating iterator for the files from this level
- list[num++] = NewTwoLevelIterator(
- new Version::LevelFileNumIterator(icmp_, &c->inputs_[which]),
- &GetFileIterator, table_cache_, options);
- }
- }
- }
- assert(num <= space);
- Iterator* result = NewMergingIterator(&icmp_, list, num);
- delete[] list;
- return result;
-}
-
-Compaction* VersionSet::PickCompaction() {
- Compaction* c;
- int level;
-
- // We prefer compactions triggered by too much data in a level over
- // the compactions triggered by seeks.
- const bool size_compaction = (current_->compaction_score_ >= 1);
- const bool seek_compaction = (current_->file_to_compact_ != NULL);
- if (size_compaction) {
- level = current_->compaction_level_;
- assert(level >= 0);
- assert(level+1 < config::kNumLevels);
- c = new Compaction(level);
-
- // Pick the first file that comes after compact_pointer_[level]
- for (size_t i = 0; i < current_->files_[level].size(); i++) {
- FileMetaData* f = current_->files_[level][i];
- if (compact_pointer_[level].empty() ||
- icmp_.Compare(f->largest.Encode(), compact_pointer_[level]) > 0) {
- c->inputs_[0].push_back(f);
- break;
- }
- }
- if (c->inputs_[0].empty()) {
- // Wrap-around to the beginning of the key space
- c->inputs_[0].push_back(current_->files_[level][0]);
- }
- } else if (seek_compaction) {
- level = current_->file_to_compact_level_;
- c = new Compaction(level);
- c->inputs_[0].push_back(current_->file_to_compact_);
- } else {
- return NULL;
- }
-
- c->input_version_ = current_;
- c->input_version_->Ref();
-
- // Files in level 0 may overlap each other, so pick up all overlapping ones
- if (level == 0) {
- InternalKey smallest, largest;
- GetRange(c->inputs_[0], &smallest, &largest);
- // Note that the next call will discard the file we placed in
- // c->inputs_[0] earlier and replace it with an overlapping set
- // which will include the picked file.
- current_->GetOverlappingInputs(0, &smallest, &largest, &c->inputs_[0]);
- assert(!c->inputs_[0].empty());
- }
-
- SetupOtherInputs(c);
-
- return c;
-}
-
-void VersionSet::SetupOtherInputs(Compaction* c) {
- const int level = c->level();
- InternalKey smallest, largest;
- GetRange(c->inputs_[0], &smallest, &largest);
-
- current_->GetOverlappingInputs(level+1, &smallest, &largest, &c->inputs_[1]);
-
- // Get entire range covered by compaction
- InternalKey all_start, all_limit;
- GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
-
- // See if we can grow the number of inputs in "level" without
- // changing the number of "level+1" files we pick up.
- if (!c->inputs_[1].empty()) {
- std::vector<FileMetaData*> expanded0;
- current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0);
- const int64_t inputs0_size = TotalFileSize(c->inputs_[0]);
- const int64_t inputs1_size = TotalFileSize(c->inputs_[1]);
- const int64_t expanded0_size = TotalFileSize(expanded0);
- if (expanded0.size() > c->inputs_[0].size() &&
- inputs1_size + expanded0_size < kExpandedCompactionByteSizeLimit) {
- InternalKey new_start, new_limit;
- GetRange(expanded0, &new_start, &new_limit);
- std::vector<FileMetaData*> expanded1;
- current_->GetOverlappingInputs(level+1, &new_start, &new_limit,
- &expanded1);
- if (expanded1.size() == c->inputs_[1].size()) {
- Log(options_->info_log,
- "Expanding@%d %d+%d (%ld+%ld bytes) to %d+%d (%ld+%ld bytes)\n",
- level,
- int(c->inputs_[0].size()),
- int(c->inputs_[1].size()),
- long(inputs0_size), long(inputs1_size),
- int(expanded0.size()),
- int(expanded1.size()),
- long(expanded0_size), long(inputs1_size));
- smallest = new_start;
- largest = new_limit;
- c->inputs_[0] = expanded0;
- c->inputs_[1] = expanded1;
- GetRange2(c->inputs_[0], c->inputs_[1], &all_start, &all_limit);
- }
- }
- }
-
- // Compute the set of grandparent files that overlap this compaction
- // (parent == level+1; grandparent == level+2)
- if (level + 2 < config::kNumLevels) {
- current_->GetOverlappingInputs(level + 2, &all_start, &all_limit,
- &c->grandparents_);
- }
-
- if (false) {
- Log(options_->info_log, "Compacting %d '%s' .. '%s'",
- level,
- smallest.DebugString().c_str(),
- largest.DebugString().c_str());
- }
-
- // Update the place where we will do the next compaction for this level.
- // We update this immediately instead of waiting for the VersionEdit
- // to be applied so that if the compaction fails, we will try a different
- // key range next time.
- compact_pointer_[level] = largest.Encode().ToString();
- c->edit_.SetCompactPointer(level, largest);
-}
-
-Compaction* VersionSet::CompactRange(
- int level,
- const InternalKey* begin,
- const InternalKey* end) {
- std::vector<FileMetaData*> inputs;
- current_->GetOverlappingInputs(level, begin, end, &inputs);
- if (inputs.empty()) {
- return NULL;
- }
-
- // Avoid compacting too much in one shot in case the range is large.
- // But we cannot do this for level-0 since level-0 files can overlap
- // and we must not pick one file and drop another older file if the
- // two files overlap.
- if (level > 0) {
- const uint64_t limit = MaxFileSizeForLevel(level);
- uint64_t total = 0;
- for (size_t i = 0; i < inputs.size(); i++) {
- uint64_t s = inputs[i]->file_size;
- total += s;
- if (total >= limit) {
- inputs.resize(i + 1);
- break;
- }
- }
- }
-
- Compaction* c = new Compaction(level);
- c->input_version_ = current_;
- c->input_version_->Ref();
- c->inputs_[0] = inputs;
- SetupOtherInputs(c);
- return c;
-}
-
-Compaction::Compaction(int level)
- : level_(level),
- max_output_file_size_(MaxFileSizeForLevel(level)),
- input_version_(NULL),
- grandparent_index_(0),
- seen_key_(false),
- overlapped_bytes_(0) {
- for (int i = 0; i < config::kNumLevels; i++) {
- level_ptrs_[i] = 0;
- }
-}
-
-Compaction::~Compaction() {
- if (input_version_ != NULL) {
- input_version_->Unref();
- }
-}
-
-bool Compaction::IsTrivialMove() const {
- // Avoid a move if there is lots of overlapping grandparent data.
- // Otherwise, the move could create a parent file that will require
- // a very expensive merge later on.
- return (num_input_files(0) == 1 &&
- num_input_files(1) == 0 &&
- TotalFileSize(grandparents_) <= kMaxGrandParentOverlapBytes);
-}
-
-void Compaction::AddInputDeletions(VersionEdit* edit) {
- for (int which = 0; which < 2; which++) {
- for (size_t i = 0; i < inputs_[which].size(); i++) {
- edit->DeleteFile(level_ + which, inputs_[which][i]->number);
- }
- }
-}
-
-bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
- // Maybe use binary search to find right entry instead of linear search?
- const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator();
- for (int lvl = level_ + 2; lvl < config::kNumLevels; lvl++) {
- const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
- for (; level_ptrs_[lvl] < files.size(); ) {
- FileMetaData* f = files[level_ptrs_[lvl]];
- if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
- // We've advanced far enough
- if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
- // Key falls in this file's range, so definitely not base level
- return false;
- }
- break;
- }
- level_ptrs_[lvl]++;
- }
- }
- return true;
-}
-
-bool Compaction::ShouldStopBefore(const Slice& internal_key) {
- // Scan to find earliest grandparent file that contains key.
- const InternalKeyComparator* icmp = &input_version_->vset_->icmp_;
- while (grandparent_index_ < grandparents_.size() &&
- icmp->Compare(internal_key,
- grandparents_[grandparent_index_]->largest.Encode()) > 0) {
- if (seen_key_) {
- overlapped_bytes_ += grandparents_[grandparent_index_]->file_size;
- }
- grandparent_index_++;
- }
- seen_key_ = true;
-
- if (overlapped_bytes_ > kMaxGrandParentOverlapBytes) {
- // Too much overlap for current output; start new output
- overlapped_bytes_ = 0;
- return true;
- } else {
- return false;
- }
-}
-
-void Compaction::ReleaseInputs() {
- if (input_version_ != NULL) {
- input_version_->Unref();
- input_version_ = NULL;
- }
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/version_set.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/version_set.h b/thirdparty/leveldb-1.18/db/version_set.h
deleted file mode 100755
index 208c5e4..0000000
--- a/thirdparty/leveldb-1.18/db/version_set.h
+++ /dev/null
@@ -1,396 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// The representation of a DBImpl consists of a set of Versions. The
-// newest version is called "current". Older versions may be kept
-// around to provide a consistent view to live iterators.
-//
-// Each Version keeps track of a set of Table files per level. The
-// entire set of versions is maintained in a VersionSet.
-//
-// Version,VersionSet are thread-compatible, but require external
-// synchronization on all accesses.
-
-#ifndef STORAGE_LEVELDB_DB_VERSION_SET_H_
-#define STORAGE_LEVELDB_DB_VERSION_SET_H_
-
-#include <map>
-#include <set>
-#include <vector>
-#include "db/dbformat.h"
-#include "db/version_edit.h"
-#include "port/port.h"
-#include "port/thread_annotations.h"
-
-namespace leveldb {
-
-namespace log { class Writer; }
-
-class Compaction;
-class Iterator;
-class MemTable;
-class TableBuilder;
-class TableCache;
-class Version;
-class VersionSet;
-class WritableFile;
-
-// Return the smallest index i such that files[i]->largest >= key.
-// Return files.size() if there is no such file.
-// REQUIRES: "files" contains a sorted list of non-overlapping files.
-extern size_t FindFile(const InternalKeyComparator& icmp,
- const std::vector<FileMetaData*>& files,
- const Slice& key);
-
-// Returns true iff some file in "files" overlaps the user key range
-// [*smallest,*largest].
-// smallest==NULL represents a key smaller than all keys in the DB.
-// largest==NULL represents a key largest than all keys in the DB.
-// REQUIRES: If disjoint_sorted_files, files[] contains disjoint ranges
-// in sorted order.
-extern bool SomeFileOverlapsRange(
- const InternalKeyComparator& icmp,
- bool disjoint_sorted_files,
- const std::vector<FileMetaData*>& files,
- const Slice* smallest_user_key,
- const Slice* largest_user_key);
-
-class Version {
- public:
- // Append to *iters a sequence of iterators that will
- // yield the contents of this Version when merged together.
- // REQUIRES: This version has been saved (see VersionSet::SaveTo)
- void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters);
-
- // Lookup the value for key. If found, store it in *val and
- // return OK. Else return a non-OK status. Fills *stats.
- // REQUIRES: lock is not held
- struct GetStats {
- FileMetaData* seek_file;
- int seek_file_level;
- };
- Status Get(const ReadOptions&, const LookupKey& key, std::string* val,
- GetStats* stats);
-
- // Adds "stats" into the current state. Returns true if a new
- // compaction may need to be triggered, false otherwise.
- // REQUIRES: lock is held
- bool UpdateStats(const GetStats& stats);
-
- // Record a sample of bytes read at the specified internal key.
- // Samples are taken approximately once every config::kReadBytesPeriod
- // bytes. Returns true if a new compaction may need to be triggered.
- // REQUIRES: lock is held
- bool RecordReadSample(Slice key);
-
- // Reference count management (so Versions do not disappear out from
- // under live iterators)
- void Ref();
- void Unref();
-
- void GetOverlappingInputs(
- int level,
- const InternalKey* begin, // NULL means before all keys
- const InternalKey* end, // NULL means after all keys
- std::vector<FileMetaData*>* inputs);
-
- // Returns true iff some file in the specified level overlaps
- // some part of [*smallest_user_key,*largest_user_key].
- // smallest_user_key==NULL represents a key smaller than all keys in the DB.
- // largest_user_key==NULL represents a key largest than all keys in the DB.
- bool OverlapInLevel(int level,
- const Slice* smallest_user_key,
- const Slice* largest_user_key);
-
- // Return the level at which we should place a new memtable compaction
- // result that covers the range [smallest_user_key,largest_user_key].
- int PickLevelForMemTableOutput(const Slice& smallest_user_key,
- const Slice& largest_user_key);
-
- size_t NumFiles(int level) const { return files_[level].size(); }
-
- // Return a human readable string that describes this version's contents.
- std::string DebugString() const;
-
- private:
- friend class Compaction;
- friend class VersionSet;
-
- class LevelFileNumIterator;
- Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const;
-
- // Call func(arg, level, f) for every file that overlaps user_key in
- // order from newest to oldest. If an invocation of func returns
- // false, makes no more calls.
- //
- // REQUIRES: user portion of internal_key == user_key.
- void ForEachOverlapping(Slice user_key, Slice internal_key,
- void* arg,
- bool (*func)(void*, int, FileMetaData*));
-
- VersionSet* vset_; // VersionSet to which this Version belongs
- Version* next_; // Next version in linked list
- Version* prev_; // Previous version in linked list
- int refs_; // Number of live refs to this version
-
- // List of files per level
- std::vector<FileMetaData*> files_[config::kNumLevels];
-
- // Next file to compact based on seek stats.
- FileMetaData* file_to_compact_;
- int file_to_compact_level_;
-
- // Level that should be compacted next and its compaction score.
- // Score < 1 means compaction is not strictly needed. These fields
- // are initialized by Finalize().
- double compaction_score_;
- int compaction_level_;
-
- explicit Version(VersionSet* vset)
- : vset_(vset), next_(this), prev_(this), refs_(0),
- file_to_compact_(NULL),
- file_to_compact_level_(-1),
- compaction_score_(-1),
- compaction_level_(-1) {
- }
-
- ~Version();
-
- // No copying allowed
- Version(const Version&);
- void operator=(const Version&);
-};
-
-class VersionSet {
- public:
- VersionSet(const std::string& dbname,
- const Options* options,
- TableCache* table_cache,
- const InternalKeyComparator*);
- ~VersionSet();
-
- // Apply *edit to the current version to form a new descriptor that
- // is both saved to persistent state and installed as the new
- // current version. Will release *mu while actually writing to the file.
- // REQUIRES: *mu is held on entry.
- // REQUIRES: no other thread concurrently calls LogAndApply()
- Status LogAndApply(VersionEdit* edit, port::Mutex* mu)
- EXCLUSIVE_LOCKS_REQUIRED(mu);
-
- // Recover the last saved descriptor from persistent storage.
- Status Recover();
-
- // Return the current version.
- Version* current() const { return current_; }
-
- // Return the current manifest file number
- uint64_t ManifestFileNumber() const { return manifest_file_number_; }
-
- // Allocate and return a new file number
- uint64_t NewFileNumber() { return next_file_number_++; }
-
- // Arrange to reuse "file_number" unless a newer file number has
- // already been allocated.
- // REQUIRES: "file_number" was returned by a call to NewFileNumber().
- void ReuseFileNumber(uint64_t file_number) {
- if (next_file_number_ == file_number + 1) {
- next_file_number_ = file_number;
- }
- }
-
- // Return the number of Table files at the specified level.
- size_t NumLevelFiles(int level) const;
-
- // Return the combined file size of all files at the specified level.
- int64_t NumLevelBytes(int level) const;
-
- // Return the last sequence number.
- uint64_t LastSequence() const { return last_sequence_; }
-
- // Set the last sequence number to s.
- void SetLastSequence(uint64_t s) {
- assert(s >= last_sequence_);
- last_sequence_ = s;
- }
-
- // Mark the specified file number as used.
- void MarkFileNumberUsed(uint64_t number);
-
- // Return the current log file number.
- uint64_t LogNumber() const { return log_number_; }
-
- // Return the log file number for the log file that is currently
- // being compacted, or zero if there is no such log file.
- uint64_t PrevLogNumber() const { return prev_log_number_; }
-
- // Pick level and inputs for a new compaction.
- // Returns NULL if there is no compaction to be done.
- // Otherwise returns a pointer to a heap-allocated object that
- // describes the compaction. Caller should delete the result.
- Compaction* PickCompaction();
-
- // Return a compaction object for compacting the range [begin,end] in
- // the specified level. Returns NULL if there is nothing in that
- // level that overlaps the specified range. Caller should delete
- // the result.
- Compaction* CompactRange(
- int level,
- const InternalKey* begin,
- const InternalKey* end);
-
- // Return the maximum overlapping data (in bytes) at next level for any
- // file at a level >= 1.
- int64_t MaxNextLevelOverlappingBytes();
-
- // Create an iterator that reads over the compaction inputs for "*c".
- // The caller should delete the iterator when no longer needed.
- Iterator* MakeInputIterator(Compaction* c);
-
- // Returns true iff some level needs a compaction.
- bool NeedsCompaction() const {
- Version* v = current_;
- return (v->compaction_score_ >= 1) || (v->file_to_compact_ != NULL);
- }
-
- // Add all files listed in any live version to *live.
- // May also mutate some internal state.
- void AddLiveFiles(std::set<uint64_t>* live);
-
- // Return the approximate offset in the database of the data for
- // "key" as of version "v".
- uint64_t ApproximateOffsetOf(Version* v, const InternalKey& key);
-
- // Return a human-readable short (single-line) summary of the number
- // of files per level. Uses *scratch as backing store.
- struct LevelSummaryStorage {
- char buffer[100];
- };
- const char* LevelSummary(LevelSummaryStorage* scratch) const;
-
- private:
- class Builder;
-
- friend class Compaction;
- friend class Version;
-
- void Finalize(Version* v);
-
- void GetRange(const std::vector<FileMetaData*>& inputs,
- InternalKey* smallest,
- InternalKey* largest);
-
- void GetRange2(const std::vector<FileMetaData*>& inputs1,
- const std::vector<FileMetaData*>& inputs2,
- InternalKey* smallest,
- InternalKey* largest);
-
- void SetupOtherInputs(Compaction* c);
-
- // Save current contents to *log
- Status WriteSnapshot(log::Writer* log);
-
- void AppendVersion(Version* v);
-
- Env* const env_;
- const std::string dbname_;
- const Options* const options_;
- TableCache* const table_cache_;
- const InternalKeyComparator icmp_;
- uint64_t next_file_number_;
- uint64_t manifest_file_number_;
- uint64_t last_sequence_;
- uint64_t log_number_;
- uint64_t prev_log_number_; // 0 or backing store for memtable being compacted
-
- // Opened lazily
- WritableFile* descriptor_file_;
- log::Writer* descriptor_log_;
- Version dummy_versions_; // Head of circular doubly-linked list of versions.
- Version* current_; // == dummy_versions_.prev_
-
- // Per-level key at which the next compaction at that level should start.
- // Either an empty string, or a valid InternalKey.
- std::string compact_pointer_[config::kNumLevels];
-
- // No copying allowed
- VersionSet(const VersionSet&);
- void operator=(const VersionSet&);
-};
-
-// A Compaction encapsulates information about a compaction.
-class Compaction {
- public:
- ~Compaction();
-
- // Return the level that is being compacted. Inputs from "level"
- // and "level+1" will be merged to produce a set of "level+1" files.
- int level() const { return level_; }
-
- // Return the object that holds the edits to the descriptor done
- // by this compaction.
- VersionEdit* edit() { return &edit_; }
-
- // "which" must be either 0 or 1
- size_t num_input_files(int which) const { return inputs_[which].size(); }
-
- // Return the ith input file at "level()+which" ("which" must be 0 or 1).
- FileMetaData* input(int which, size_t i) const { return inputs_[which][i]; }
-
- // Maximum size of files to build during this compaction.
- uint64_t MaxOutputFileSize() const { return max_output_file_size_; }
-
- // Is this a trivial compaction that can be implemented by just
- // moving a single input file to the next level (no merging or splitting)
- bool IsTrivialMove() const;
-
- // Add all inputs to this compaction as delete operations to *edit.
- void AddInputDeletions(VersionEdit* edit);
-
- // Returns true if the information we have available guarantees that
- // the compaction is producing data in "level+1" for which no data exists
- // in levels greater than "level+1".
- bool IsBaseLevelForKey(const Slice& user_key);
-
- // Returns true iff we should stop building the current output
- // before processing "internal_key".
- bool ShouldStopBefore(const Slice& internal_key);
-
- // Release the input version for the compaction, once the compaction
- // is successful.
- void ReleaseInputs();
-
- private:
- friend class Version;
- friend class VersionSet;
-
- explicit Compaction(int level);
-
- int level_;
- uint64_t max_output_file_size_;
- Version* input_version_;
- VersionEdit edit_;
-
- // Each compaction reads inputs from "level_" and "level_+1"
- std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
-
- // State used to check for number of of overlapping grandparent files
- // (parent == level_ + 1, grandparent == level_ + 2)
- std::vector<FileMetaData*> grandparents_;
- size_t grandparent_index_; // Index in grandparent_starts_
- bool seen_key_; // Some output key has been seen
- int64_t overlapped_bytes_; // Bytes of overlap between current output
- // and grandparent files
-
- // State for implementing IsBaseLevelForKey
-
- // level_ptrs_ holds indices into input_version_->levels_: our state
- // is that we are positioned at one of the file ranges for each
- // higher level than the ones involved in this compaction (i.e. for
- // all L >= level_ + 2).
- size_t level_ptrs_[config::kNumLevels];
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_VERSION_SET_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/version_set_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/version_set_test.cc b/thirdparty/leveldb-1.18/db/version_set_test.cc
deleted file mode 100755
index 2f207a5..0000000
--- a/thirdparty/leveldb-1.18/db/version_set_test.cc
+++ /dev/null
@@ -1,179 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/version_set.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-class FindFileTest {
- public:
- std::vector<FileMetaData*> files_;
- bool disjoint_sorted_files_;
-
- FindFileTest() : disjoint_sorted_files_(true) { }
-
- ~FindFileTest() {
- for (int i = 0; i < files_.size(); i++) {
- delete files_[i];
- }
- }
-
- void Add(const char* smallest, const char* largest,
- SequenceNumber smallest_seq = 100,
- SequenceNumber largest_seq = 100) {
- FileMetaData* f = new FileMetaData;
- f->number = files_.size() + 1;
- f->smallest = InternalKey(smallest, smallest_seq, kTypeValue);
- f->largest = InternalKey(largest, largest_seq, kTypeValue);
- files_.push_back(f);
- }
-
- int Find(const char* key) {
- InternalKey target(key, 100, kTypeValue);
- InternalKeyComparator cmp(BytewiseComparator());
- return static_cast<int>(FindFile(cmp, files_, target.Encode()));
- }
-
- bool Overlaps(const char* smallest, const char* largest) {
- InternalKeyComparator cmp(BytewiseComparator());
- Slice s(smallest != NULL ? smallest : "");
- Slice l(largest != NULL ? largest : "");
- return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, files_,
- (smallest != NULL ? &s : NULL),
- (largest != NULL ? &l : NULL));
- }
-};
-
-TEST(FindFileTest, Empty) {
- ASSERT_EQ(0, Find("foo"));
- ASSERT_TRUE(! Overlaps("a", "z"));
- ASSERT_TRUE(! Overlaps(NULL, "z"));
- ASSERT_TRUE(! Overlaps("a", NULL));
- ASSERT_TRUE(! Overlaps(NULL, NULL));
-}
-
-TEST(FindFileTest, Single) {
- Add("p", "q");
- ASSERT_EQ(0, Find("a"));
- ASSERT_EQ(0, Find("p"));
- ASSERT_EQ(0, Find("p1"));
- ASSERT_EQ(0, Find("q"));
- ASSERT_EQ(1, Find("q1"));
- ASSERT_EQ(1, Find("z"));
-
- ASSERT_TRUE(! Overlaps("a", "b"));
- ASSERT_TRUE(! Overlaps("z1", "z2"));
- ASSERT_TRUE(Overlaps("a", "p"));
- ASSERT_TRUE(Overlaps("a", "q"));
- ASSERT_TRUE(Overlaps("a", "z"));
- ASSERT_TRUE(Overlaps("p", "p1"));
- ASSERT_TRUE(Overlaps("p", "q"));
- ASSERT_TRUE(Overlaps("p", "z"));
- ASSERT_TRUE(Overlaps("p1", "p2"));
- ASSERT_TRUE(Overlaps("p1", "z"));
- ASSERT_TRUE(Overlaps("q", "q"));
- ASSERT_TRUE(Overlaps("q", "q1"));
-
- ASSERT_TRUE(! Overlaps(NULL, "j"));
- ASSERT_TRUE(! Overlaps("r", NULL));
- ASSERT_TRUE(Overlaps(NULL, "p"));
- ASSERT_TRUE(Overlaps(NULL, "p1"));
- ASSERT_TRUE(Overlaps("q", NULL));
- ASSERT_TRUE(Overlaps(NULL, NULL));
-}
-
-
-TEST(FindFileTest, Multiple) {
- Add("150", "200");
- Add("200", "250");
- Add("300", "350");
- Add("400", "450");
- ASSERT_EQ(0, Find("100"));
- ASSERT_EQ(0, Find("150"));
- ASSERT_EQ(0, Find("151"));
- ASSERT_EQ(0, Find("199"));
- ASSERT_EQ(0, Find("200"));
- ASSERT_EQ(1, Find("201"));
- ASSERT_EQ(1, Find("249"));
- ASSERT_EQ(1, Find("250"));
- ASSERT_EQ(2, Find("251"));
- ASSERT_EQ(2, Find("299"));
- ASSERT_EQ(2, Find("300"));
- ASSERT_EQ(2, Find("349"));
- ASSERT_EQ(2, Find("350"));
- ASSERT_EQ(3, Find("351"));
- ASSERT_EQ(3, Find("400"));
- ASSERT_EQ(3, Find("450"));
- ASSERT_EQ(4, Find("451"));
-
- ASSERT_TRUE(! Overlaps("100", "149"));
- ASSERT_TRUE(! Overlaps("251", "299"));
- ASSERT_TRUE(! Overlaps("451", "500"));
- ASSERT_TRUE(! Overlaps("351", "399"));
-
- ASSERT_TRUE(Overlaps("100", "150"));
- ASSERT_TRUE(Overlaps("100", "200"));
- ASSERT_TRUE(Overlaps("100", "300"));
- ASSERT_TRUE(Overlaps("100", "400"));
- ASSERT_TRUE(Overlaps("100", "500"));
- ASSERT_TRUE(Overlaps("375", "400"));
- ASSERT_TRUE(Overlaps("450", "450"));
- ASSERT_TRUE(Overlaps("450", "500"));
-}
-
-TEST(FindFileTest, MultipleNullBoundaries) {
- Add("150", "200");
- Add("200", "250");
- Add("300", "350");
- Add("400", "450");
- ASSERT_TRUE(! Overlaps(NULL, "149"));
- ASSERT_TRUE(! Overlaps("451", NULL));
- ASSERT_TRUE(Overlaps(NULL, NULL));
- ASSERT_TRUE(Overlaps(NULL, "150"));
- ASSERT_TRUE(Overlaps(NULL, "199"));
- ASSERT_TRUE(Overlaps(NULL, "200"));
- ASSERT_TRUE(Overlaps(NULL, "201"));
- ASSERT_TRUE(Overlaps(NULL, "400"));
- ASSERT_TRUE(Overlaps(NULL, "800"));
- ASSERT_TRUE(Overlaps("100", NULL));
- ASSERT_TRUE(Overlaps("200", NULL));
- ASSERT_TRUE(Overlaps("449", NULL));
- ASSERT_TRUE(Overlaps("450", NULL));
-}
-
-TEST(FindFileTest, OverlapSequenceChecks) {
- Add("200", "200", 5000, 3000);
- ASSERT_TRUE(! Overlaps("199", "199"));
- ASSERT_TRUE(! Overlaps("201", "300"));
- ASSERT_TRUE(Overlaps("200", "200"));
- ASSERT_TRUE(Overlaps("190", "200"));
- ASSERT_TRUE(Overlaps("200", "210"));
-}
-
-TEST(FindFileTest, OverlappingFiles) {
- Add("150", "600");
- Add("400", "500");
- disjoint_sorted_files_ = false;
- ASSERT_TRUE(! Overlaps("100", "149"));
- ASSERT_TRUE(! Overlaps("601", "700"));
- ASSERT_TRUE(Overlaps("100", "150"));
- ASSERT_TRUE(Overlaps("100", "200"));
- ASSERT_TRUE(Overlaps("100", "300"));
- ASSERT_TRUE(Overlaps("100", "400"));
- ASSERT_TRUE(Overlaps("100", "500"));
- ASSERT_TRUE(Overlaps("375", "400"));
- ASSERT_TRUE(Overlaps("450", "450"));
- ASSERT_TRUE(Overlaps("450", "500"));
- ASSERT_TRUE(Overlaps("450", "700"));
- ASSERT_TRUE(Overlaps("600", "700"));
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/write_batch.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/write_batch.cc b/thirdparty/leveldb-1.18/db/write_batch.cc
deleted file mode 100755
index 33f4a42..0000000
--- a/thirdparty/leveldb-1.18/db/write_batch.cc
+++ /dev/null
@@ -1,147 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// WriteBatch::rep_ :=
-// sequence: fixed64
-// count: fixed32
-// data: record[count]
-// record :=
-// kTypeValue varstring varstring |
-// kTypeDeletion varstring
-// varstring :=
-// len: varint32
-// data: uint8[len]
-
-#include "leveldb/write_batch.h"
-
-#include "leveldb/db.h"
-#include "db/dbformat.h"
-#include "db/memtable.h"
-#include "db/write_batch_internal.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
-static const size_t kHeader = 12;
-
-WriteBatch::WriteBatch() {
- Clear();
-}
-
-WriteBatch::~WriteBatch() { }
-
-WriteBatch::Handler::~Handler() { }
-
-void WriteBatch::Clear() {
- rep_.clear();
- rep_.resize(kHeader);
-}
-
-Status WriteBatch::Iterate(Handler* handler) const {
- Slice input(rep_);
- if (input.size() < kHeader) {
- return Status::Corruption("malformed WriteBatch (too small)");
- }
-
- input.remove_prefix(kHeader);
- Slice key, value;
- int found = 0;
- while (!input.empty()) {
- found++;
- char tag = input[0];
- input.remove_prefix(1);
- switch (tag) {
- case kTypeValue:
- if (GetLengthPrefixedSlice(&input, &key) &&
- GetLengthPrefixedSlice(&input, &value)) {
- handler->Put(key, value);
- } else {
- return Status::Corruption("bad WriteBatch Put");
- }
- break;
- case kTypeDeletion:
- if (GetLengthPrefixedSlice(&input, &key)) {
- handler->Delete(key);
- } else {
- return Status::Corruption("bad WriteBatch Delete");
- }
- break;
- default:
- return Status::Corruption("unknown WriteBatch tag");
- }
- }
- if (found != WriteBatchInternal::Count(this)) {
- return Status::Corruption("WriteBatch has wrong count");
- } else {
- return Status::OK();
- }
-}
-
-int WriteBatchInternal::Count(const WriteBatch* b) {
- return DecodeFixed32(b->rep_.data() + 8);
-}
-
-void WriteBatchInternal::SetCount(WriteBatch* b, int n) {
- EncodeFixed32(&b->rep_[8], n);
-}
-
-SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) {
- return SequenceNumber(DecodeFixed64(b->rep_.data()));
-}
-
-void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) {
- EncodeFixed64(&b->rep_[0], seq);
-}
-
-void WriteBatch::Put(const Slice& key, const Slice& value) {
- WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
- rep_.push_back(static_cast<char>(kTypeValue));
- PutLengthPrefixedSlice(&rep_, key);
- PutLengthPrefixedSlice(&rep_, value);
-}
-
-void WriteBatch::Delete(const Slice& key) {
- WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
- rep_.push_back(static_cast<char>(kTypeDeletion));
- PutLengthPrefixedSlice(&rep_, key);
-}
-
-namespace {
-class MemTableInserter : public WriteBatch::Handler {
- public:
- SequenceNumber sequence_;
- MemTable* mem_;
-
- virtual void Put(const Slice& key, const Slice& value) {
- mem_->Add(sequence_, kTypeValue, key, value);
- sequence_++;
- }
- virtual void Delete(const Slice& key) {
- mem_->Add(sequence_, kTypeDeletion, key, Slice());
- sequence_++;
- }
-};
-} // namespace
-
-Status WriteBatchInternal::InsertInto(const WriteBatch* b,
- MemTable* memtable) {
- MemTableInserter inserter;
- inserter.sequence_ = WriteBatchInternal::Sequence(b);
- inserter.mem_ = memtable;
- return b->Iterate(&inserter);
-}
-
-void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) {
- assert(contents.size() >= kHeader);
- b->rep_.assign(contents.data(), contents.size());
-}
-
-void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) {
- SetCount(dst, Count(dst) + Count(src));
- assert(src->rep_.size() >= kHeader);
- dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader);
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/write_batch_internal.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/write_batch_internal.h b/thirdparty/leveldb-1.18/db/write_batch_internal.h
deleted file mode 100755
index 4423a7f..0000000
--- a/thirdparty/leveldb-1.18/db/write_batch_internal.h
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
-#define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
-
-#include "leveldb/write_batch.h"
-
-namespace leveldb {
-
-class MemTable;
-
-// WriteBatchInternal provides static methods for manipulating a
-// WriteBatch that we don't want in the public WriteBatch interface.
-class WriteBatchInternal {
- public:
- // Return the number of entries in the batch.
- static int Count(const WriteBatch* batch);
-
- // Set the count for the number of entries in the batch.
- static void SetCount(WriteBatch* batch, int n);
-
- // Return the seqeunce number for the start of this batch.
- static SequenceNumber Sequence(const WriteBatch* batch);
-
- // Store the specified number as the seqeunce number for the start of
- // this batch.
- static void SetSequence(WriteBatch* batch, SequenceNumber seq);
-
- static Slice Contents(const WriteBatch* batch) {
- return Slice(batch->rep_);
- }
-
- static size_t ByteSize(const WriteBatch* batch) {
- return batch->rep_.size();
- }
-
- static void SetContents(WriteBatch* batch, const Slice& contents);
-
- static Status InsertInto(const WriteBatch* batch, MemTable* memtable);
-
- static void Append(WriteBatch* dst, const WriteBatch* src);
-};
-
-} // namespace leveldb
-
-
-#endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/write_batch_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/write_batch_test.cc b/thirdparty/leveldb-1.18/db/write_batch_test.cc
deleted file mode 100755
index 4cf03b7..0000000
--- a/thirdparty/leveldb-1.18/db/write_batch_test.cc
+++ /dev/null
@@ -1,125 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/db.h"
-
-#include "db/memtable.h"
-#include "db/write_batch_internal.h"
-#include "leveldb/env.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-static std::string PrintContents(WriteBatch* b) {
- InternalKeyComparator cmp(BytewiseComparator());
- MemTable* mem = new MemTable(cmp);
- mem->Ref();
- std::string state;
- Status s = WriteBatchInternal::InsertInto(b, mem);
- int count = 0;
- Iterator* iter = mem->NewIterator();
- for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
- ParsedInternalKey ikey;
-
- // removes gcc warning about "maybe" uninitialized variable
- ikey.type = kTypeDeletion;
- ikey.sequence = 0;
-
- ASSERT_TRUE(ParseInternalKey(iter->key(), &ikey));
- switch (ikey.type) {
- case kTypeValue:
- state.append("Put(");
- state.append(ikey.user_key.ToString());
- state.append(", ");
- state.append(iter->value().ToString());
- state.append(")");
- count++;
- break;
- case kTypeDeletion:
- state.append("Delete(");
- state.append(ikey.user_key.ToString());
- state.append(")");
- count++;
- break;
- }
- state.append("@");
- state.append(NumberToString(ikey.sequence));
- }
- delete iter;
- if (!s.ok()) {
- state.append("ParseError()");
- } else if (count != WriteBatchInternal::Count(b)) {
- state.append("CountMismatch()");
- }
- mem->Unref();
- return state;
-}
-
-class WriteBatchTest { };
-
-TEST(WriteBatchTest, Empty) {
- WriteBatch batch;
- ASSERT_EQ("", PrintContents(&batch));
- ASSERT_EQ(0, WriteBatchInternal::Count(&batch));
-}
-
-TEST(WriteBatchTest, Multiple) {
- WriteBatch batch;
- batch.Put(Slice("foo"), Slice("bar"));
- batch.Delete(Slice("box"));
- batch.Put(Slice("baz"), Slice("boo"));
- WriteBatchInternal::SetSequence(&batch, 100);
- ASSERT_EQ(100, WriteBatchInternal::Sequence(&batch));
- ASSERT_EQ(3, WriteBatchInternal::Count(&batch));
- ASSERT_EQ("Put(baz, boo)@102"
- "Delete(box)@101"
- "Put(foo, bar)@100",
- PrintContents(&batch));
-}
-
-TEST(WriteBatchTest, Corruption) {
- WriteBatch batch;
- batch.Put(Slice("foo"), Slice("bar"));
- batch.Delete(Slice("box"));
- WriteBatchInternal::SetSequence(&batch, 200);
- Slice contents = WriteBatchInternal::Contents(&batch);
- WriteBatchInternal::SetContents(&batch,
- Slice(contents.data(),contents.size()-1));
- ASSERT_EQ("Put(foo, bar)@200"
- "ParseError()",
- PrintContents(&batch));
-}
-
-TEST(WriteBatchTest, Append) {
- WriteBatch b1, b2;
- WriteBatchInternal::SetSequence(&b1, 200);
- WriteBatchInternal::SetSequence(&b2, 300);
- WriteBatchInternal::Append(&b1, &b2);
- ASSERT_EQ("",
- PrintContents(&b1));
- b2.Put("a", "va");
- WriteBatchInternal::Append(&b1, &b2);
- ASSERT_EQ("Put(a, va)@200",
- PrintContents(&b1));
- b2.Clear();
- b2.Put("b", "vb");
- WriteBatchInternal::Append(&b1, &b2);
- ASSERT_EQ("Put(a, va)@200"
- "Put(b, vb)@201",
- PrintContents(&b1));
- b2.Delete("foo");
- WriteBatchInternal::Append(&b1, &b2);
- ASSERT_EQ("Put(a, va)@200"
- "Put(b, vb)@202"
- "Put(b, vb)@201"
- "Delete(foo)@203",
- PrintContents(&b1));
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
[15/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/flush_job.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/flush_job.cc b/thirdparty/rocksdb/db/flush_job.cc
new file mode 100644
index 0000000..846edb4
--- /dev/null
+++ b/thirdparty/rocksdb/db/flush_job.cc
@@ -0,0 +1,359 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/flush_job.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+
+#include <algorithm>
+#include <vector>
+
+#include "db/builder.h"
+#include "db/db_iter.h"
+#include "db/dbformat.h"
+#include "db/event_helpers.h"
+#include "db/log_reader.h"
+#include "db/log_writer.h"
+#include "db/memtable_list.h"
+#include "db/merge_context.h"
+#include "db/version_set.h"
+#include "monitoring/iostats_context_imp.h"
+#include "monitoring/perf_context_imp.h"
+#include "monitoring/thread_status_util.h"
+#include "port/likely.h"
+#include "port/port.h"
+#include "db/memtable.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/status.h"
+#include "rocksdb/table.h"
+#include "table/block.h"
+#include "table/block_based_table_factory.h"
+#include "table/merging_iterator.h"
+#include "table/table_builder.h"
+#include "table/two_level_iterator.h"
+#include "util/coding.h"
+#include "util/event_logger.h"
+#include "util/file_util.h"
+#include "util/filename.h"
+#include "util/log_buffer.h"
+#include "util/logging.h"
+#include "util/mutexlock.h"
+#include "util/stop_watch.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
+ const ImmutableDBOptions& db_options,
+ const MutableCFOptions& mutable_cf_options,
+ const EnvOptions& env_options, VersionSet* versions,
+ InstrumentedMutex* db_mutex,
+ std::atomic<bool>* shutting_down,
+ std::vector<SequenceNumber> existing_snapshots,
+ SequenceNumber earliest_write_conflict_snapshot,
+ JobContext* job_context, LogBuffer* log_buffer,
+ Directory* db_directory, Directory* output_file_directory,
+ CompressionType output_compression, Statistics* stats,
+ EventLogger* event_logger, bool measure_io_stats)
+ : dbname_(dbname),
+ cfd_(cfd),
+ db_options_(db_options),
+ mutable_cf_options_(mutable_cf_options),
+ env_options_(env_options),
+ versions_(versions),
+ db_mutex_(db_mutex),
+ shutting_down_(shutting_down),
+ existing_snapshots_(std::move(existing_snapshots)),
+ earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
+ job_context_(job_context),
+ log_buffer_(log_buffer),
+ db_directory_(db_directory),
+ output_file_directory_(output_file_directory),
+ output_compression_(output_compression),
+ stats_(stats),
+ event_logger_(event_logger),
+ measure_io_stats_(measure_io_stats),
+ pick_memtable_called(false) {
+ // Update the thread status to indicate flush.
+ ReportStartedFlush();
+ TEST_SYNC_POINT("FlushJob::FlushJob()");
+}
+
+FlushJob::~FlushJob() {
+ ThreadStatusUtil::ResetThreadStatus();
+}
+
+void FlushJob::ReportStartedFlush() {
+ ThreadStatusUtil::SetColumnFamily(cfd_, cfd_->ioptions()->env,
+ db_options_.enable_thread_tracking);
+ ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH);
+ ThreadStatusUtil::SetThreadOperationProperty(
+ ThreadStatus::COMPACTION_JOB_ID,
+ job_context_->job_id);
+ IOSTATS_RESET(bytes_written);
+}
+
+void FlushJob::ReportFlushInputSize(const autovector<MemTable*>& mems) {
+ uint64_t input_size = 0;
+ for (auto* mem : mems) {
+ input_size += mem->ApproximateMemoryUsage();
+ }
+ ThreadStatusUtil::IncreaseThreadOperationProperty(
+ ThreadStatus::FLUSH_BYTES_MEMTABLES,
+ input_size);
+}
+
+void FlushJob::RecordFlushIOStats() {
+ RecordTick(stats_, FLUSH_WRITE_BYTES, IOSTATS(bytes_written));
+ ThreadStatusUtil::IncreaseThreadOperationProperty(
+ ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written));
+ IOSTATS_RESET(bytes_written);
+}
+
+void FlushJob::PickMemTable() {
+ db_mutex_->AssertHeld();
+ assert(!pick_memtable_called);
+ pick_memtable_called = true;
+ // Save the contents of the earliest memtable as a new Table
+ cfd_->imm()->PickMemtablesToFlush(&mems_);
+ if (mems_.empty()) {
+ return;
+ }
+
+ ReportFlushInputSize(mems_);
+
+ // entries mems are (implicitly) sorted in ascending order by their created
+ // time. We will use the first memtable's `edit` to keep the meta info for
+ // this flush.
+ MemTable* m = mems_[0];
+ edit_ = m->GetEdits();
+ edit_->SetPrevLogNumber(0);
+ // SetLogNumber(log_num) indicates logs with number smaller than log_num
+ // will no longer be picked up for recovery.
+ edit_->SetLogNumber(mems_.back()->GetNextLogNumber());
+ edit_->SetColumnFamily(cfd_->GetID());
+
+ // path 0 for level 0 file.
+ meta_.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0);
+
+ base_ = cfd_->current();
+ base_->Ref(); // it is likely that we do not need this reference
+}
+
+Status FlushJob::Run(FileMetaData* file_meta) {
+ db_mutex_->AssertHeld();
+ assert(pick_memtable_called);
+ AutoThreadOperationStageUpdater stage_run(
+ ThreadStatus::STAGE_FLUSH_RUN);
+ if (mems_.empty()) {
+ ROCKS_LOG_BUFFER(log_buffer_, "[%s] Nothing in memtable to flush",
+ cfd_->GetName().c_str());
+ return Status::OK();
+ }
+
+ // I/O measurement variables
+ PerfLevel prev_perf_level = PerfLevel::kEnableTime;
+ uint64_t prev_write_nanos = 0;
+ uint64_t prev_fsync_nanos = 0;
+ uint64_t prev_range_sync_nanos = 0;
+ uint64_t prev_prepare_write_nanos = 0;
+ if (measure_io_stats_) {
+ prev_perf_level = GetPerfLevel();
+ SetPerfLevel(PerfLevel::kEnableTime);
+ prev_write_nanos = IOSTATS(write_nanos);
+ prev_fsync_nanos = IOSTATS(fsync_nanos);
+ prev_range_sync_nanos = IOSTATS(range_sync_nanos);
+ prev_prepare_write_nanos = IOSTATS(prepare_write_nanos);
+ }
+
+ // This will release and re-acquire the mutex.
+ Status s = WriteLevel0Table();
+
+ if (s.ok() &&
+ (shutting_down_->load(std::memory_order_acquire) || cfd_->IsDropped())) {
+ s = Status::ShutdownInProgress(
+ "Database shutdown or Column family drop during flush");
+ }
+
+ if (!s.ok()) {
+ cfd_->imm()->RollbackMemtableFlush(mems_, meta_.fd.GetNumber());
+ } else {
+ TEST_SYNC_POINT("FlushJob::InstallResults");
+ // Replace immutable memtable with the generated Table
+ s = cfd_->imm()->InstallMemtableFlushResults(
+ cfd_, mutable_cf_options_, mems_, versions_, db_mutex_,
+ meta_.fd.GetNumber(), &job_context_->memtables_to_free, db_directory_,
+ log_buffer_);
+ }
+
+ if (s.ok() && file_meta != nullptr) {
+ *file_meta = meta_;
+ }
+ RecordFlushIOStats();
+
+ auto stream = event_logger_->LogToBuffer(log_buffer_);
+ stream << "job" << job_context_->job_id << "event"
+ << "flush_finished";
+ stream << "lsm_state";
+ stream.StartArray();
+ auto vstorage = cfd_->current()->storage_info();
+ for (int level = 0; level < vstorage->num_levels(); ++level) {
+ stream << vstorage->NumLevelFiles(level);
+ }
+ stream.EndArray();
+ stream << "immutable_memtables" << cfd_->imm()->NumNotFlushed();
+
+ if (measure_io_stats_) {
+ if (prev_perf_level != PerfLevel::kEnableTime) {
+ SetPerfLevel(prev_perf_level);
+ }
+ stream << "file_write_nanos" << (IOSTATS(write_nanos) - prev_write_nanos);
+ stream << "file_range_sync_nanos"
+ << (IOSTATS(range_sync_nanos) - prev_range_sync_nanos);
+ stream << "file_fsync_nanos" << (IOSTATS(fsync_nanos) - prev_fsync_nanos);
+ stream << "file_prepare_write_nanos"
+ << (IOSTATS(prepare_write_nanos) - prev_prepare_write_nanos);
+ }
+
+ return s;
+}
+
+void FlushJob::Cancel() {
+ db_mutex_->AssertHeld();
+ assert(base_ != nullptr);
+ base_->Unref();
+}
+
+Status FlushJob::WriteLevel0Table() {
+ AutoThreadOperationStageUpdater stage_updater(
+ ThreadStatus::STAGE_FLUSH_WRITE_L0);
+ db_mutex_->AssertHeld();
+ const uint64_t start_micros = db_options_.env->NowMicros();
+ Status s;
+ {
+ db_mutex_->Unlock();
+ if (log_buffer_) {
+ log_buffer_->FlushBufferToLog();
+ }
+ // memtables and range_del_iters store internal iterators over each data
+ // memtable and its associated range deletion memtable, respectively, at
+ // corresponding indexes.
+ std::vector<InternalIterator*> memtables;
+ std::vector<InternalIterator*> range_del_iters;
+ ReadOptions ro;
+ ro.total_order_seek = true;
+ Arena arena;
+ uint64_t total_num_entries = 0, total_num_deletes = 0;
+ size_t total_memory_usage = 0;
+ for (MemTable* m : mems_) {
+ ROCKS_LOG_INFO(
+ db_options_.info_log,
+ "[%s] [JOB %d] Flushing memtable with next log file: %" PRIu64 "\n",
+ cfd_->GetName().c_str(), job_context_->job_id, m->GetNextLogNumber());
+ memtables.push_back(m->NewIterator(ro, &arena));
+ auto* range_del_iter = m->NewRangeTombstoneIterator(ro);
+ if (range_del_iter != nullptr) {
+ range_del_iters.push_back(range_del_iter);
+ }
+ total_num_entries += m->num_entries();
+ total_num_deletes += m->num_deletes();
+ total_memory_usage += m->ApproximateMemoryUsage();
+ }
+
+ event_logger_->Log() << "job" << job_context_->job_id << "event"
+ << "flush_started"
+ << "num_memtables" << mems_.size() << "num_entries"
+ << total_num_entries << "num_deletes"
+ << total_num_deletes << "memory_usage"
+ << total_memory_usage;
+
+ {
+ ScopedArenaIterator iter(
+ NewMergingIterator(&cfd_->internal_comparator(), &memtables[0],
+ static_cast<int>(memtables.size()), &arena));
+ std::unique_ptr<InternalIterator> range_del_iter(NewMergingIterator(
+ &cfd_->internal_comparator(),
+ range_del_iters.empty() ? nullptr : &range_del_iters[0],
+ static_cast<int>(range_del_iters.size())));
+ ROCKS_LOG_INFO(db_options_.info_log,
+ "[%s] [JOB %d] Level-0 flush table #%" PRIu64 ": started",
+ cfd_->GetName().c_str(), job_context_->job_id,
+ meta_.fd.GetNumber());
+
+ TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:output_compression",
+ &output_compression_);
+ EnvOptions optimized_env_options =
+ db_options_.env->OptimizeForCompactionTableWrite(env_options_, db_options_);
+
+ int64_t _current_time = 0;
+ db_options_.env->GetCurrentTime(&_current_time); // ignore error
+ const uint64_t current_time = static_cast<uint64_t>(_current_time);
+
+ s = BuildTable(
+ dbname_, db_options_.env, *cfd_->ioptions(), mutable_cf_options_,
+ optimized_env_options, cfd_->table_cache(), iter.get(),
+ std::move(range_del_iter), &meta_, cfd_->internal_comparator(),
+ cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
+ cfd_->GetName(), existing_snapshots_,
+ earliest_write_conflict_snapshot_, output_compression_,
+ cfd_->ioptions()->compression_opts,
+ mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
+ TableFileCreationReason::kFlush, event_logger_, job_context_->job_id,
+ Env::IO_HIGH, &table_properties_, 0 /* level */, current_time);
+ LogFlush(db_options_.info_log);
+ }
+ ROCKS_LOG_INFO(db_options_.info_log,
+ "[%s] [JOB %d] Level-0 flush table #%" PRIu64 ": %" PRIu64
+ " bytes %s"
+ "%s",
+ cfd_->GetName().c_str(), job_context_->job_id,
+ meta_.fd.GetNumber(), meta_.fd.GetFileSize(),
+ s.ToString().c_str(),
+ meta_.marked_for_compaction ? " (needs compaction)" : "");
+
+ if (output_file_directory_ != nullptr) {
+ output_file_directory_->Fsync();
+ }
+ TEST_SYNC_POINT("FlushJob::WriteLevel0Table");
+ db_mutex_->Lock();
+ }
+ base_->Unref();
+
+ // Note that if file_size is zero, the file has been deleted and
+ // should not be added to the manifest.
+ if (s.ok() && meta_.fd.GetFileSize() > 0) {
+ // if we have more than 1 background thread, then we cannot
+ // insert files directly into higher levels because some other
+ // threads could be concurrently producing compacted files for
+ // that key range.
+ // Add file to L0
+ edit_->AddFile(0 /* level */, meta_.fd.GetNumber(), meta_.fd.GetPathId(),
+ meta_.fd.GetFileSize(), meta_.smallest, meta_.largest,
+ meta_.smallest_seqno, meta_.largest_seqno,
+ meta_.marked_for_compaction);
+ }
+
+ // Note that here we treat flush as level 0 compaction in internal stats
+ InternalStats::CompactionStats stats(1);
+ stats.micros = db_options_.env->NowMicros() - start_micros;
+ stats.bytes_written = meta_.fd.GetFileSize();
+ cfd_->internal_stats()->AddCompactionStats(0 /* level */, stats);
+ cfd_->internal_stats()->AddCFStats(InternalStats::BYTES_FLUSHED,
+ meta_.fd.GetFileSize());
+ RecordFlushIOStats();
+ return s;
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/flush_job.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/flush_job.h b/thirdparty/rocksdb/db/flush_job.h
new file mode 100644
index 0000000..4698ae7
--- /dev/null
+++ b/thirdparty/rocksdb/db/flush_job.h
@@ -0,0 +1,110 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#pragma once
+
+#include <atomic>
+#include <deque>
+#include <limits>
+#include <set>
+#include <utility>
+#include <vector>
+#include <string>
+
+#include "db/column_family.h"
+#include "db/dbformat.h"
+#include "db/flush_scheduler.h"
+#include "db/internal_stats.h"
+#include "db/job_context.h"
+#include "db/log_writer.h"
+#include "db/memtable_list.h"
+#include "db/snapshot_impl.h"
+#include "db/version_edit.h"
+#include "db/write_controller.h"
+#include "db/write_thread.h"
+#include "monitoring/instrumented_mutex.h"
+#include "options/db_options.h"
+#include "port/port.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/transaction_log.h"
+#include "table/scoped_arena_iterator.h"
+#include "util/autovector.h"
+#include "util/event_logger.h"
+#include "util/stop_watch.h"
+#include "util/thread_local.h"
+
+namespace rocksdb {
+
+class MemTable;
+class TableCache;
+class Version;
+class VersionEdit;
+class VersionSet;
+class Arena;
+
+class FlushJob {
+ public:
+ // TODO(icanadi) make effort to reduce number of parameters here
+ // IMPORTANT: mutable_cf_options needs to be alive while FlushJob is alive
+ FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
+ const ImmutableDBOptions& db_options,
+ const MutableCFOptions& mutable_cf_options,
+ const EnvOptions& env_options, VersionSet* versions,
+ InstrumentedMutex* db_mutex, std::atomic<bool>* shutting_down,
+ std::vector<SequenceNumber> existing_snapshots,
+ SequenceNumber earliest_write_conflict_snapshot,
+ JobContext* job_context, LogBuffer* log_buffer,
+ Directory* db_directory, Directory* output_file_directory,
+ CompressionType output_compression, Statistics* stats,
+ EventLogger* event_logger, bool measure_io_stats);
+
+ ~FlushJob();
+
+ // Require db_mutex held.
+ // Once PickMemTable() is called, either Run() or Cancel() has to be called.
+ void PickMemTable();
+ Status Run(FileMetaData* file_meta = nullptr);
+ void Cancel();
+ TableProperties GetTableProperties() const { return table_properties_; }
+
+ private:
+ void ReportStartedFlush();
+ void ReportFlushInputSize(const autovector<MemTable*>& mems);
+ void RecordFlushIOStats();
+ Status WriteLevel0Table();
+ const std::string& dbname_;
+ ColumnFamilyData* cfd_;
+ const ImmutableDBOptions& db_options_;
+ const MutableCFOptions& mutable_cf_options_;
+ const EnvOptions& env_options_;
+ VersionSet* versions_;
+ InstrumentedMutex* db_mutex_;
+ std::atomic<bool>* shutting_down_;
+ std::vector<SequenceNumber> existing_snapshots_;
+ SequenceNumber earliest_write_conflict_snapshot_;
+ JobContext* job_context_;
+ LogBuffer* log_buffer_;
+ Directory* db_directory_;
+ Directory* output_file_directory_;
+ CompressionType output_compression_;
+ Statistics* stats_;
+ EventLogger* event_logger_;
+ TableProperties table_properties_;
+ bool measure_io_stats_;
+
+ // Variables below are set by PickMemTable():
+ FileMetaData meta_;
+ autovector<MemTable*> mems_;
+ VersionEdit* edit_;
+ Version* base_;
+ bool pick_memtable_called;
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/flush_scheduler.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/flush_scheduler.cc b/thirdparty/rocksdb/db/flush_scheduler.cc
new file mode 100644
index 0000000..8735a6b
--- /dev/null
+++ b/thirdparty/rocksdb/db/flush_scheduler.cc
@@ -0,0 +1,88 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "db/flush_scheduler.h"
+
+#include <cassert>
+
+#include "db/column_family.h"
+
+namespace rocksdb {
+
+void FlushScheduler::ScheduleFlush(ColumnFamilyData* cfd) {
+#ifndef NDEBUG
+ std::lock_guard<std::mutex> lock(checking_mutex_);
+ assert(checking_set_.count(cfd) == 0);
+ checking_set_.insert(cfd);
+#endif // NDEBUG
+ cfd->Ref();
+// Suppress false positive clang analyzer warnings.
+#ifndef __clang_analyzer__
+ Node* node = new Node{cfd, head_.load(std::memory_order_relaxed)};
+ while (!head_.compare_exchange_strong(
+ node->next, node, std::memory_order_relaxed, std::memory_order_relaxed)) {
+ // failing CAS updates the first param, so we are already set for
+ // retry. TakeNextColumnFamily won't happen until after another
+ // inter-thread synchronization, so we don't even need release
+ // semantics for this CAS
+ }
+#endif // __clang_analyzer__
+}
+
+ColumnFamilyData* FlushScheduler::TakeNextColumnFamily() {
+#ifndef NDEBUG
+ std::lock_guard<std::mutex> lock(checking_mutex_);
+#endif // NDEBUG
+ while (true) {
+ if (head_.load(std::memory_order_relaxed) == nullptr) {
+ return nullptr;
+ }
+
+ // dequeue the head
+ Node* node = head_.load(std::memory_order_relaxed);
+ head_.store(node->next, std::memory_order_relaxed);
+ ColumnFamilyData* cfd = node->column_family;
+ delete node;
+
+#ifndef NDEBUG
+ auto iter = checking_set_.find(cfd);
+ assert(iter != checking_set_.end());
+ checking_set_.erase(iter);
+#endif // NDEBUG
+
+ if (!cfd->IsDropped()) {
+ // success
+ return cfd;
+ }
+
+ // no longer relevant, retry
+ if (cfd->Unref()) {
+ delete cfd;
+ }
+ }
+}
+
+bool FlushScheduler::Empty() {
+#ifndef NDEBUG
+ std::lock_guard<std::mutex> lock(checking_mutex_);
+#endif // NDEBUG
+ auto rv = head_.load(std::memory_order_relaxed) == nullptr;
+#ifndef NDEBUG
+ assert(rv == checking_set_.empty());
+#endif // NDEBUG
+ return rv;
+}
+
+void FlushScheduler::Clear() {
+ ColumnFamilyData* cfd;
+ while ((cfd = TakeNextColumnFamily()) != nullptr) {
+ if (cfd->Unref()) {
+ delete cfd;
+ }
+ }
+ assert(head_.load(std::memory_order_relaxed) == nullptr);
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/flush_scheduler.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/flush_scheduler.h b/thirdparty/rocksdb/db/flush_scheduler.h
new file mode 100644
index 0000000..cd35758
--- /dev/null
+++ b/thirdparty/rocksdb/db/flush_scheduler.h
@@ -0,0 +1,48 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <stdint.h>
+#include <atomic>
+#include <mutex>
+#include <set>
+
+namespace rocksdb {
+
+class ColumnFamilyData;
+
+// Unless otherwise noted, all methods on FlushScheduler should be called
+// only with the DB mutex held or from a single-threaded recovery context.
+class FlushScheduler {
+ public:
+ FlushScheduler() : head_(nullptr) {}
+
+ // May be called from multiple threads at once, but not concurrent with
+ // any other method calls on this instance
+ void ScheduleFlush(ColumnFamilyData* cfd);
+
+ // Removes and returns Ref()-ed column family. Client needs to Unref().
+ // Filters column families that have been dropped.
+ ColumnFamilyData* TakeNextColumnFamily();
+
+ bool Empty();
+
+ void Clear();
+
+ private:
+ struct Node {
+ ColumnFamilyData* column_family;
+ Node* next;
+ };
+
+ std::atomic<Node*> head_;
+#ifndef NDEBUG
+ std::mutex checking_mutex_;
+ std::set<ColumnFamilyData*> checking_set_;
+#endif // NDEBUG
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/forward_iterator.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/forward_iterator.cc b/thirdparty/rocksdb/db/forward_iterator.cc
new file mode 100644
index 0000000..65fff95
--- /dev/null
+++ b/thirdparty/rocksdb/db/forward_iterator.cc
@@ -0,0 +1,905 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+#include "db/forward_iterator.h"
+
+#include <limits>
+#include <string>
+#include <utility>
+
+#include "db/column_family.h"
+#include "db/db_impl.h"
+#include "db/db_iter.h"
+#include "db/dbformat.h"
+#include "db/job_context.h"
+#include "rocksdb/env.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/slice_transform.h"
+#include "table/merging_iterator.h"
+#include "util/string_util.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+// Usage:
+// LevelIterator iter;
+// iter.SetFileIndex(file_index);
+// iter.Seek(target);
+// iter.Next()
+class LevelIterator : public InternalIterator {
+ public:
+ LevelIterator(const ColumnFamilyData* const cfd,
+ const ReadOptions& read_options,
+ const std::vector<FileMetaData*>& files)
+ : cfd_(cfd),
+ read_options_(read_options),
+ files_(files),
+ valid_(false),
+ file_index_(std::numeric_limits<uint32_t>::max()),
+ file_iter_(nullptr),
+ pinned_iters_mgr_(nullptr) {}
+
+ ~LevelIterator() {
+ // Reset current pointer
+ if (pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled()) {
+ pinned_iters_mgr_->PinIterator(file_iter_);
+ } else {
+ delete file_iter_;
+ }
+ }
+
+ void SetFileIndex(uint32_t file_index) {
+ assert(file_index < files_.size());
+ if (file_index != file_index_) {
+ file_index_ = file_index;
+ Reset();
+ }
+ valid_ = false;
+ }
+ void Reset() {
+ assert(file_index_ < files_.size());
+
+ // Reset current pointer
+ if (pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled()) {
+ pinned_iters_mgr_->PinIterator(file_iter_);
+ } else {
+ delete file_iter_;
+ }
+
+ RangeDelAggregator range_del_agg(
+ cfd_->internal_comparator(), {} /* snapshots */);
+ file_iter_ = cfd_->table_cache()->NewIterator(
+ read_options_, *(cfd_->soptions()), cfd_->internal_comparator(),
+ files_[file_index_]->fd,
+ read_options_.ignore_range_deletions ? nullptr : &range_del_agg,
+ nullptr /* table_reader_ptr */, nullptr, false);
+ file_iter_->SetPinnedItersMgr(pinned_iters_mgr_);
+ if (!range_del_agg.IsEmpty()) {
+ status_ = Status::NotSupported(
+ "Range tombstones unsupported with ForwardIterator");
+ valid_ = false;
+ }
+ }
+ void SeekToLast() override {
+ status_ = Status::NotSupported("LevelIterator::SeekToLast()");
+ valid_ = false;
+ }
+ void Prev() override {
+ status_ = Status::NotSupported("LevelIterator::Prev()");
+ valid_ = false;
+ }
+ bool Valid() const override {
+ return valid_;
+ }
+ void SeekToFirst() override {
+ SetFileIndex(0);
+ file_iter_->SeekToFirst();
+ valid_ = file_iter_->Valid();
+ }
+ void Seek(const Slice& internal_key) override {
+ assert(file_iter_ != nullptr);
+ file_iter_->Seek(internal_key);
+ valid_ = file_iter_->Valid();
+ }
+ void SeekForPrev(const Slice& internal_key) override {
+ status_ = Status::NotSupported("LevelIterator::SeekForPrev()");
+ valid_ = false;
+ }
+ void Next() override {
+ assert(valid_);
+ file_iter_->Next();
+ for (;;) {
+ if (file_iter_->status().IsIncomplete() || file_iter_->Valid()) {
+ valid_ = !file_iter_->status().IsIncomplete();
+ return;
+ }
+ if (file_index_ + 1 >= files_.size()) {
+ valid_ = false;
+ return;
+ }
+ SetFileIndex(file_index_ + 1);
+ file_iter_->SeekToFirst();
+ }
+ }
+ Slice key() const override {
+ assert(valid_);
+ return file_iter_->key();
+ }
+ Slice value() const override {
+ assert(valid_);
+ return file_iter_->value();
+ }
+ Status status() const override {
+ if (!status_.ok()) {
+ return status_;
+ } else if (file_iter_ && !file_iter_->status().ok()) {
+ return file_iter_->status();
+ }
+ return Status::OK();
+ }
+ bool IsKeyPinned() const override {
+ return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() &&
+ file_iter_->IsKeyPinned();
+ }
+ bool IsValuePinned() const override {
+ return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() &&
+ file_iter_->IsValuePinned();
+ }
+ void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override {
+ pinned_iters_mgr_ = pinned_iters_mgr;
+ if (file_iter_) {
+ file_iter_->SetPinnedItersMgr(pinned_iters_mgr_);
+ }
+ }
+
+ private:
+ const ColumnFamilyData* const cfd_;
+ const ReadOptions& read_options_;
+ const std::vector<FileMetaData*>& files_;
+
+ bool valid_;
+ uint32_t file_index_;
+ Status status_;
+ InternalIterator* file_iter_;
+ PinnedIteratorsManager* pinned_iters_mgr_;
+};
+
+ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
+ ColumnFamilyData* cfd,
+ SuperVersion* current_sv)
+ : db_(db),
+ read_options_(read_options),
+ cfd_(cfd),
+ prefix_extractor_(cfd->ioptions()->prefix_extractor),
+ user_comparator_(cfd->user_comparator()),
+ immutable_min_heap_(MinIterComparator(&cfd_->internal_comparator())),
+ sv_(current_sv),
+ mutable_iter_(nullptr),
+ current_(nullptr),
+ valid_(false),
+ status_(Status::OK()),
+ immutable_status_(Status::OK()),
+ has_iter_trimmed_for_upper_bound_(false),
+ current_over_upper_bound_(false),
+ is_prev_set_(false),
+ is_prev_inclusive_(false),
+ pinned_iters_mgr_(nullptr) {
+ if (sv_) {
+ RebuildIterators(false);
+ }
+}
+
+ForwardIterator::~ForwardIterator() {
+ Cleanup(true);
+}
+
+namespace {
+// Used in PinnedIteratorsManager to release pinned SuperVersion
+static void ReleaseSuperVersionFunc(void* sv) {
+ delete reinterpret_cast<SuperVersion*>(sv);
+}
+} // namespace
+
+void ForwardIterator::SVCleanup() {
+ if (sv_ != nullptr && sv_->Unref()) {
+ // Job id == 0 means that this is not our background process, but rather
+ // user thread
+ JobContext job_context(0);
+ db_->mutex_.Lock();
+ sv_->Cleanup();
+ db_->FindObsoleteFiles(&job_context, false, true);
+ if (read_options_.background_purge_on_iterator_cleanup) {
+ db_->ScheduleBgLogWriterClose(&job_context);
+ }
+ db_->mutex_.Unlock();
+ if (pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled()) {
+ pinned_iters_mgr_->PinPtr(sv_, &ReleaseSuperVersionFunc);
+ } else {
+ delete sv_;
+ }
+ if (job_context.HaveSomethingToDelete()) {
+ db_->PurgeObsoleteFiles(
+ job_context, read_options_.background_purge_on_iterator_cleanup);
+ }
+ job_context.Clean();
+ }
+}
+
+void ForwardIterator::Cleanup(bool release_sv) {
+ if (mutable_iter_ != nullptr) {
+ DeleteIterator(mutable_iter_, true /* is_arena */);
+ }
+
+ for (auto* m : imm_iters_) {
+ DeleteIterator(m, true /* is_arena */);
+ }
+ imm_iters_.clear();
+
+ for (auto* f : l0_iters_) {
+ DeleteIterator(f);
+ }
+ l0_iters_.clear();
+
+ for (auto* l : level_iters_) {
+ DeleteIterator(l);
+ }
+ level_iters_.clear();
+
+ if (release_sv) {
+ SVCleanup();
+ }
+}
+
+bool ForwardIterator::Valid() const {
+ // See UpdateCurrent().
+ return valid_ ? !current_over_upper_bound_ : false;
+}
+
+void ForwardIterator::SeekToFirst() {
+ if (sv_ == nullptr) {
+ RebuildIterators(true);
+ } else if (sv_->version_number != cfd_->GetSuperVersionNumber()) {
+ RenewIterators();
+ } else if (immutable_status_.IsIncomplete()) {
+ ResetIncompleteIterators();
+ }
+ SeekInternal(Slice(), true);
+}
+
+bool ForwardIterator::IsOverUpperBound(const Slice& internal_key) const {
+ return !(read_options_.iterate_upper_bound == nullptr ||
+ cfd_->internal_comparator().user_comparator()->Compare(
+ ExtractUserKey(internal_key),
+ *read_options_.iterate_upper_bound) < 0);
+}
+
+void ForwardIterator::Seek(const Slice& internal_key) {
+ if (IsOverUpperBound(internal_key)) {
+ valid_ = false;
+ }
+ if (sv_ == nullptr) {
+ RebuildIterators(true);
+ } else if (sv_->version_number != cfd_->GetSuperVersionNumber()) {
+ RenewIterators();
+ } else if (immutable_status_.IsIncomplete()) {
+ ResetIncompleteIterators();
+ }
+ SeekInternal(internal_key, false);
+}
+
+void ForwardIterator::SeekInternal(const Slice& internal_key,
+ bool seek_to_first) {
+ assert(mutable_iter_);
+ // mutable
+ seek_to_first ? mutable_iter_->SeekToFirst() :
+ mutable_iter_->Seek(internal_key);
+
+ // immutable
+ // TODO(ljin): NeedToSeekImmutable has negative impact on performance
+ // if it turns to need to seek immutable often. We probably want to have
+ // an option to turn it off.
+ if (seek_to_first || NeedToSeekImmutable(internal_key)) {
+ immutable_status_ = Status::OK();
+ if (has_iter_trimmed_for_upper_bound_ &&
+ (
+ // prev_ is not set yet
+ is_prev_set_ == false ||
+ // We are doing SeekToFirst() and internal_key.size() = 0
+ seek_to_first ||
+ // prev_key_ > internal_key
+ cfd_->internal_comparator().InternalKeyComparator::Compare(
+ prev_key_.GetInternalKey(), internal_key) > 0)) {
+ // Some iterators are trimmed. Need to rebuild.
+ RebuildIterators(true);
+ // Already seeked mutable iter, so seek again
+ seek_to_first ? mutable_iter_->SeekToFirst()
+ : mutable_iter_->Seek(internal_key);
+ }
+ {
+ auto tmp = MinIterHeap(MinIterComparator(&cfd_->internal_comparator()));
+ immutable_min_heap_.swap(tmp);
+ }
+ for (size_t i = 0; i < imm_iters_.size(); i++) {
+ auto* m = imm_iters_[i];
+ seek_to_first ? m->SeekToFirst() : m->Seek(internal_key);
+ if (!m->status().ok()) {
+ immutable_status_ = m->status();
+ } else if (m->Valid()) {
+ immutable_min_heap_.push(m);
+ }
+ }
+
+ Slice user_key;
+ if (!seek_to_first) {
+ user_key = ExtractUserKey(internal_key);
+ }
+ const VersionStorageInfo* vstorage = sv_->current->storage_info();
+ const std::vector<FileMetaData*>& l0 = vstorage->LevelFiles(0);
+ for (size_t i = 0; i < l0.size(); ++i) {
+ if (!l0_iters_[i]) {
+ continue;
+ }
+ if (seek_to_first) {
+ l0_iters_[i]->SeekToFirst();
+ } else {
+ // If the target key passes over the larget key, we are sure Next()
+ // won't go over this file.
+ if (user_comparator_->Compare(user_key,
+ l0[i]->largest.user_key()) > 0) {
+ if (read_options_.iterate_upper_bound != nullptr) {
+ has_iter_trimmed_for_upper_bound_ = true;
+ DeleteIterator(l0_iters_[i]);
+ l0_iters_[i] = nullptr;
+ }
+ continue;
+ }
+ l0_iters_[i]->Seek(internal_key);
+ }
+
+ if (!l0_iters_[i]->status().ok()) {
+ immutable_status_ = l0_iters_[i]->status();
+ } else if (l0_iters_[i]->Valid()) {
+ if (!IsOverUpperBound(l0_iters_[i]->key())) {
+ immutable_min_heap_.push(l0_iters_[i]);
+ } else {
+ has_iter_trimmed_for_upper_bound_ = true;
+ DeleteIterator(l0_iters_[i]);
+ l0_iters_[i] = nullptr;
+ }
+ }
+ }
+
+ for (int32_t level = 1; level < vstorage->num_levels(); ++level) {
+ const std::vector<FileMetaData*>& level_files =
+ vstorage->LevelFiles(level);
+ if (level_files.empty()) {
+ continue;
+ }
+ if (level_iters_[level - 1] == nullptr) {
+ continue;
+ }
+ uint32_t f_idx = 0;
+ if (!seek_to_first) {
+ f_idx = FindFileInRange(level_files, internal_key, 0,
+ static_cast<uint32_t>(level_files.size()));
+ }
+
+ // Seek
+ if (f_idx < level_files.size()) {
+ level_iters_[level - 1]->SetFileIndex(f_idx);
+ seek_to_first ? level_iters_[level - 1]->SeekToFirst() :
+ level_iters_[level - 1]->Seek(internal_key);
+
+ if (!level_iters_[level - 1]->status().ok()) {
+ immutable_status_ = level_iters_[level - 1]->status();
+ } else if (level_iters_[level - 1]->Valid()) {
+ if (!IsOverUpperBound(level_iters_[level - 1]->key())) {
+ immutable_min_heap_.push(level_iters_[level - 1]);
+ } else {
+ // Nothing in this level is interesting. Remove.
+ has_iter_trimmed_for_upper_bound_ = true;
+ DeleteIterator(level_iters_[level - 1]);
+ level_iters_[level - 1] = nullptr;
+ }
+ }
+ }
+ }
+
+ if (seek_to_first) {
+ is_prev_set_ = false;
+ } else {
+ prev_key_.SetInternalKey(internal_key);
+ is_prev_set_ = true;
+ is_prev_inclusive_ = true;
+ }
+
+ TEST_SYNC_POINT_CALLBACK("ForwardIterator::SeekInternal:Immutable", this);
+ } else if (current_ && current_ != mutable_iter_) {
+ // current_ is one of immutable iterators, push it back to the heap
+ immutable_min_heap_.push(current_);
+ }
+
+ UpdateCurrent();
+ TEST_SYNC_POINT_CALLBACK("ForwardIterator::SeekInternal:Return", this);
+}
+
+void ForwardIterator::Next() {
+ assert(valid_);
+ bool update_prev_key = false;
+
+ if (sv_ == nullptr ||
+ sv_->version_number != cfd_->GetSuperVersionNumber()) {
+ std::string current_key = key().ToString();
+ Slice old_key(current_key.data(), current_key.size());
+
+ if (sv_ == nullptr) {
+ RebuildIterators(true);
+ } else {
+ RenewIterators();
+ }
+ SeekInternal(old_key, false);
+ if (!valid_ || key().compare(old_key) != 0) {
+ return;
+ }
+ } else if (current_ != mutable_iter_) {
+ // It is going to advance immutable iterator
+
+ if (is_prev_set_ && prefix_extractor_) {
+ // advance prev_key_ to current_ only if they share the same prefix
+ update_prev_key =
+ prefix_extractor_->Transform(prev_key_.GetUserKey())
+ .compare(prefix_extractor_->Transform(current_->key())) == 0;
+ } else {
+ update_prev_key = true;
+ }
+
+
+ if (update_prev_key) {
+ prev_key_.SetInternalKey(current_->key());
+ is_prev_set_ = true;
+ is_prev_inclusive_ = false;
+ }
+ }
+
+ current_->Next();
+ if (current_ != mutable_iter_) {
+ if (!current_->status().ok()) {
+ immutable_status_ = current_->status();
+ } else if ((current_->Valid()) && (!IsOverUpperBound(current_->key()))) {
+ immutable_min_heap_.push(current_);
+ } else {
+ if ((current_->Valid()) && (IsOverUpperBound(current_->key()))) {
+ // remove the current iterator
+ DeleteCurrentIter();
+ current_ = nullptr;
+ }
+ if (update_prev_key) {
+ mutable_iter_->Seek(prev_key_.GetInternalKey());
+ }
+ }
+ }
+ UpdateCurrent();
+ TEST_SYNC_POINT_CALLBACK("ForwardIterator::Next:Return", this);
+}
+
+Slice ForwardIterator::key() const {
+ assert(valid_);
+ return current_->key();
+}
+
+Slice ForwardIterator::value() const {
+ assert(valid_);
+ return current_->value();
+}
+
+Status ForwardIterator::status() const {
+ if (!status_.ok()) {
+ return status_;
+ } else if (!mutable_iter_->status().ok()) {
+ return mutable_iter_->status();
+ }
+
+ return immutable_status_;
+}
+
+Status ForwardIterator::GetProperty(std::string prop_name, std::string* prop) {
+ assert(prop != nullptr);
+ if (prop_name == "rocksdb.iterator.super-version-number") {
+ *prop = ToString(sv_->version_number);
+ return Status::OK();
+ }
+ return Status::InvalidArgument();
+}
+
+void ForwardIterator::SetPinnedItersMgr(
+ PinnedIteratorsManager* pinned_iters_mgr) {
+ pinned_iters_mgr_ = pinned_iters_mgr;
+ UpdateChildrenPinnedItersMgr();
+}
+
+void ForwardIterator::UpdateChildrenPinnedItersMgr() {
+ // Set PinnedIteratorsManager for mutable memtable iterator.
+ if (mutable_iter_) {
+ mutable_iter_->SetPinnedItersMgr(pinned_iters_mgr_);
+ }
+
+ // Set PinnedIteratorsManager for immutable memtable iterators.
+ for (InternalIterator* child_iter : imm_iters_) {
+ if (child_iter) {
+ child_iter->SetPinnedItersMgr(pinned_iters_mgr_);
+ }
+ }
+
+ // Set PinnedIteratorsManager for L0 files iterators.
+ for (InternalIterator* child_iter : l0_iters_) {
+ if (child_iter) {
+ child_iter->SetPinnedItersMgr(pinned_iters_mgr_);
+ }
+ }
+
+ // Set PinnedIteratorsManager for L1+ levels iterators.
+ for (LevelIterator* child_iter : level_iters_) {
+ if (child_iter) {
+ child_iter->SetPinnedItersMgr(pinned_iters_mgr_);
+ }
+ }
+}
+
+bool ForwardIterator::IsKeyPinned() const {
+ return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() &&
+ current_->IsKeyPinned();
+}
+
+bool ForwardIterator::IsValuePinned() const {
+ return pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled() &&
+ current_->IsValuePinned();
+}
+
+void ForwardIterator::RebuildIterators(bool refresh_sv) {
+ // Clean up
+ Cleanup(refresh_sv);
+ if (refresh_sv) {
+ // New
+ sv_ = cfd_->GetReferencedSuperVersion(&(db_->mutex_));
+ }
+ RangeDelAggregator range_del_agg(
+ InternalKeyComparator(cfd_->internal_comparator()), {} /* snapshots */);
+ mutable_iter_ = sv_->mem->NewIterator(read_options_, &arena_);
+ sv_->imm->AddIterators(read_options_, &imm_iters_, &arena_);
+ if (!read_options_.ignore_range_deletions) {
+ std::unique_ptr<InternalIterator> range_del_iter(
+ sv_->mem->NewRangeTombstoneIterator(read_options_));
+ range_del_agg.AddTombstones(std::move(range_del_iter));
+ sv_->imm->AddRangeTombstoneIterators(read_options_, &arena_,
+ &range_del_agg);
+ }
+ has_iter_trimmed_for_upper_bound_ = false;
+
+ const auto* vstorage = sv_->current->storage_info();
+ const auto& l0_files = vstorage->LevelFiles(0);
+ l0_iters_.reserve(l0_files.size());
+ for (const auto* l0 : l0_files) {
+ if ((read_options_.iterate_upper_bound != nullptr) &&
+ cfd_->internal_comparator().user_comparator()->Compare(
+ l0->smallest.user_key(), *read_options_.iterate_upper_bound) > 0) {
+ has_iter_trimmed_for_upper_bound_ = true;
+ l0_iters_.push_back(nullptr);
+ continue;
+ }
+ l0_iters_.push_back(cfd_->table_cache()->NewIterator(
+ read_options_, *cfd_->soptions(), cfd_->internal_comparator(), l0->fd,
+ read_options_.ignore_range_deletions ? nullptr : &range_del_agg));
+ }
+ BuildLevelIterators(vstorage);
+ current_ = nullptr;
+ is_prev_set_ = false;
+
+ UpdateChildrenPinnedItersMgr();
+ if (!range_del_agg.IsEmpty()) {
+ status_ = Status::NotSupported(
+ "Range tombstones unsupported with ForwardIterator");
+ valid_ = false;
+ }
+}
+
+void ForwardIterator::RenewIterators() {
+ SuperVersion* svnew;
+ assert(sv_);
+ svnew = cfd_->GetReferencedSuperVersion(&(db_->mutex_));
+
+ if (mutable_iter_ != nullptr) {
+ DeleteIterator(mutable_iter_, true /* is_arena */);
+ }
+ for (auto* m : imm_iters_) {
+ DeleteIterator(m, true /* is_arena */);
+ }
+ imm_iters_.clear();
+
+ mutable_iter_ = svnew->mem->NewIterator(read_options_, &arena_);
+ svnew->imm->AddIterators(read_options_, &imm_iters_, &arena_);
+ RangeDelAggregator range_del_agg(
+ InternalKeyComparator(cfd_->internal_comparator()), {} /* snapshots */);
+ if (!read_options_.ignore_range_deletions) {
+ std::unique_ptr<InternalIterator> range_del_iter(
+ svnew->mem->NewRangeTombstoneIterator(read_options_));
+ range_del_agg.AddTombstones(std::move(range_del_iter));
+ sv_->imm->AddRangeTombstoneIterators(read_options_, &arena_,
+ &range_del_agg);
+ }
+
+ const auto* vstorage = sv_->current->storage_info();
+ const auto& l0_files = vstorage->LevelFiles(0);
+ const auto* vstorage_new = svnew->current->storage_info();
+ const auto& l0_files_new = vstorage_new->LevelFiles(0);
+ size_t iold, inew;
+ bool found;
+ std::vector<InternalIterator*> l0_iters_new;
+ l0_iters_new.reserve(l0_files_new.size());
+
+ for (inew = 0; inew < l0_files_new.size(); inew++) {
+ found = false;
+ for (iold = 0; iold < l0_files.size(); iold++) {
+ if (l0_files[iold] == l0_files_new[inew]) {
+ found = true;
+ break;
+ }
+ }
+ if (found) {
+ if (l0_iters_[iold] == nullptr) {
+ l0_iters_new.push_back(nullptr);
+ TEST_SYNC_POINT_CALLBACK("ForwardIterator::RenewIterators:Null", this);
+ } else {
+ l0_iters_new.push_back(l0_iters_[iold]);
+ l0_iters_[iold] = nullptr;
+ TEST_SYNC_POINT_CALLBACK("ForwardIterator::RenewIterators:Copy", this);
+ }
+ continue;
+ }
+ l0_iters_new.push_back(cfd_->table_cache()->NewIterator(
+ read_options_, *cfd_->soptions(), cfd_->internal_comparator(),
+ l0_files_new[inew]->fd,
+ read_options_.ignore_range_deletions ? nullptr : &range_del_agg));
+ }
+
+ for (auto* f : l0_iters_) {
+ DeleteIterator(f);
+ }
+ l0_iters_.clear();
+ l0_iters_ = l0_iters_new;
+
+ for (auto* l : level_iters_) {
+ DeleteIterator(l);
+ }
+ level_iters_.clear();
+ BuildLevelIterators(vstorage_new);
+ current_ = nullptr;
+ is_prev_set_ = false;
+ SVCleanup();
+ sv_ = svnew;
+
+ UpdateChildrenPinnedItersMgr();
+ if (!range_del_agg.IsEmpty()) {
+ status_ = Status::NotSupported(
+ "Range tombstones unsupported with ForwardIterator");
+ valid_ = false;
+ }
+}
+
+void ForwardIterator::BuildLevelIterators(const VersionStorageInfo* vstorage) {
+ level_iters_.reserve(vstorage->num_levels() - 1);
+ for (int32_t level = 1; level < vstorage->num_levels(); ++level) {
+ const auto& level_files = vstorage->LevelFiles(level);
+ if ((level_files.empty()) ||
+ ((read_options_.iterate_upper_bound != nullptr) &&
+ (user_comparator_->Compare(*read_options_.iterate_upper_bound,
+ level_files[0]->smallest.user_key()) <
+ 0))) {
+ level_iters_.push_back(nullptr);
+ if (!level_files.empty()) {
+ has_iter_trimmed_for_upper_bound_ = true;
+ }
+ } else {
+ level_iters_.push_back(
+ new LevelIterator(cfd_, read_options_, level_files));
+ }
+ }
+}
+
+void ForwardIterator::ResetIncompleteIterators() {
+ const auto& l0_files = sv_->current->storage_info()->LevelFiles(0);
+ for (size_t i = 0; i < l0_iters_.size(); ++i) {
+ assert(i < l0_files.size());
+ if (!l0_iters_[i] || !l0_iters_[i]->status().IsIncomplete()) {
+ continue;
+ }
+ DeleteIterator(l0_iters_[i]);
+ l0_iters_[i] = cfd_->table_cache()->NewIterator(
+ read_options_, *cfd_->soptions(), cfd_->internal_comparator(),
+ l0_files[i]->fd, nullptr /* range_del_agg */);
+ l0_iters_[i]->SetPinnedItersMgr(pinned_iters_mgr_);
+ }
+
+ for (auto* level_iter : level_iters_) {
+ if (level_iter && level_iter->status().IsIncomplete()) {
+ level_iter->Reset();
+ }
+ }
+
+ current_ = nullptr;
+ is_prev_set_ = false;
+}
+
+void ForwardIterator::UpdateCurrent() {
+ if (immutable_min_heap_.empty() && !mutable_iter_->Valid()) {
+ current_ = nullptr;
+ } else if (immutable_min_heap_.empty()) {
+ current_ = mutable_iter_;
+ } else if (!mutable_iter_->Valid()) {
+ current_ = immutable_min_heap_.top();
+ immutable_min_heap_.pop();
+ } else {
+ current_ = immutable_min_heap_.top();
+ assert(current_ != nullptr);
+ assert(current_->Valid());
+ int cmp = cfd_->internal_comparator().InternalKeyComparator::Compare(
+ mutable_iter_->key(), current_->key());
+ assert(cmp != 0);
+ if (cmp > 0) {
+ immutable_min_heap_.pop();
+ } else {
+ current_ = mutable_iter_;
+ }
+ }
+ valid_ = (current_ != nullptr);
+ if (!status_.ok()) {
+ status_ = Status::OK();
+ }
+
+ // Upper bound doesn't apply to the memtable iterator. We want Valid() to
+ // return false when all iterators are over iterate_upper_bound, but can't
+ // just set valid_ to false, as that would effectively disable the tailing
+ // optimization (Seek() would be called on all immutable iterators regardless
+ // of whether the target key is greater than prev_key_).
+ current_over_upper_bound_ = valid_ && IsOverUpperBound(current_->key());
+}
+
+bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
+ // We maintain the interval (prev_key_, immutable_min_heap_.top()->key())
+ // such that there are no records with keys within that range in
+ // immutable_min_heap_. Since immutable structures (SST files and immutable
+ // memtables) can't change in this version, we don't need to do a seek if
+ // 'target' belongs to that interval (immutable_min_heap_.top() is already
+ // at the correct position).
+
+ if (!valid_ || !current_ || !is_prev_set_ || !immutable_status_.ok()) {
+ return true;
+ }
+ Slice prev_key = prev_key_.GetInternalKey();
+ if (prefix_extractor_ && prefix_extractor_->Transform(target).compare(
+ prefix_extractor_->Transform(prev_key)) != 0) {
+ return true;
+ }
+ if (cfd_->internal_comparator().InternalKeyComparator::Compare(
+ prev_key, target) >= (is_prev_inclusive_ ? 1 : 0)) {
+ return true;
+ }
+
+ if (immutable_min_heap_.empty() && current_ == mutable_iter_) {
+ // Nothing to seek on.
+ return false;
+ }
+ if (cfd_->internal_comparator().InternalKeyComparator::Compare(
+ target, current_ == mutable_iter_ ? immutable_min_heap_.top()->key()
+ : current_->key()) > 0) {
+ return true;
+ }
+ return false;
+}
+
+void ForwardIterator::DeleteCurrentIter() {
+ const VersionStorageInfo* vstorage = sv_->current->storage_info();
+ const std::vector<FileMetaData*>& l0 = vstorage->LevelFiles(0);
+ for (size_t i = 0; i < l0.size(); ++i) {
+ if (!l0_iters_[i]) {
+ continue;
+ }
+ if (l0_iters_[i] == current_) {
+ has_iter_trimmed_for_upper_bound_ = true;
+ DeleteIterator(l0_iters_[i]);
+ l0_iters_[i] = nullptr;
+ return;
+ }
+ }
+
+ for (int32_t level = 1; level < vstorage->num_levels(); ++level) {
+ if (level_iters_[level - 1] == nullptr) {
+ continue;
+ }
+ if (level_iters_[level - 1] == current_) {
+ has_iter_trimmed_for_upper_bound_ = true;
+ DeleteIterator(level_iters_[level - 1]);
+ level_iters_[level - 1] = nullptr;
+ }
+ }
+}
+
+bool ForwardIterator::TEST_CheckDeletedIters(int* pdeleted_iters,
+ int* pnum_iters) {
+ bool retval = false;
+ int deleted_iters = 0;
+ int num_iters = 0;
+
+ const VersionStorageInfo* vstorage = sv_->current->storage_info();
+ const std::vector<FileMetaData*>& l0 = vstorage->LevelFiles(0);
+ for (size_t i = 0; i < l0.size(); ++i) {
+ if (!l0_iters_[i]) {
+ retval = true;
+ deleted_iters++;
+ } else {
+ num_iters++;
+ }
+ }
+
+ for (int32_t level = 1; level < vstorage->num_levels(); ++level) {
+ if ((level_iters_[level - 1] == nullptr) &&
+ (!vstorage->LevelFiles(level).empty())) {
+ retval = true;
+ deleted_iters++;
+ } else if (!vstorage->LevelFiles(level).empty()) {
+ num_iters++;
+ }
+ }
+ if ((!retval) && num_iters <= 1) {
+ retval = true;
+ }
+ if (pdeleted_iters) {
+ *pdeleted_iters = deleted_iters;
+ }
+ if (pnum_iters) {
+ *pnum_iters = num_iters;
+ }
+ return retval;
+}
+
+uint32_t ForwardIterator::FindFileInRange(
+ const std::vector<FileMetaData*>& files, const Slice& internal_key,
+ uint32_t left, uint32_t right) {
+ while (left < right) {
+ uint32_t mid = (left + right) / 2;
+ const FileMetaData* f = files[mid];
+ if (cfd_->internal_comparator().InternalKeyComparator::Compare(
+ f->largest.Encode(), internal_key) < 0) {
+ // Key at "mid.largest" is < "target". Therefore all
+ // files at or before "mid" are uninteresting.
+ left = mid + 1;
+ } else {
+ // Key at "mid.largest" is >= "target". Therefore all files
+ // after "mid" are uninteresting.
+ right = mid;
+ }
+ }
+ return right;
+}
+
+void ForwardIterator::DeleteIterator(InternalIterator* iter, bool is_arena) {
+ if (iter == nullptr) {
+ return;
+ }
+
+ if (pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled()) {
+ pinned_iters_mgr_->PinIterator(iter, is_arena);
+ } else {
+ if (is_arena) {
+ iter->~InternalIterator();
+ } else {
+ delete iter;
+ }
+ }
+}
+
+} // namespace rocksdb
+
+#endif // ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/forward_iterator.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/forward_iterator.h b/thirdparty/rocksdb/db/forward_iterator.h
new file mode 100644
index 0000000..d4f32cb
--- /dev/null
+++ b/thirdparty/rocksdb/db/forward_iterator.h
@@ -0,0 +1,153 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+#pragma once
+
+#ifndef ROCKSDB_LITE
+
+#include <string>
+#include <vector>
+#include <queue>
+
+#include "rocksdb/db.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/options.h"
+#include "db/dbformat.h"
+#include "table/internal_iterator.h"
+#include "util/arena.h"
+
+namespace rocksdb {
+
+class DBImpl;
+class Env;
+struct SuperVersion;
+class ColumnFamilyData;
+class LevelIterator;
+class VersionStorageInfo;
+struct FileMetaData;
+
+class MinIterComparator {
+ public:
+ explicit MinIterComparator(const Comparator* comparator) :
+ comparator_(comparator) {}
+
+ bool operator()(InternalIterator* a, InternalIterator* b) {
+ return comparator_->Compare(a->key(), b->key()) > 0;
+ }
+ private:
+ const Comparator* comparator_;
+};
+
+typedef std::priority_queue<InternalIterator*, std::vector<InternalIterator*>,
+ MinIterComparator> MinIterHeap;
+
+/**
+ * ForwardIterator is a special type of iterator that only supports Seek()
+ * and Next(). It is expected to perform better than TailingIterator by
+ * removing the encapsulation and making all information accessible within
+ * the iterator. At the current implementation, snapshot is taken at the
+ * time Seek() is called. The Next() followed do not see new values after.
+ */
+class ForwardIterator : public InternalIterator {
+ public:
+ ForwardIterator(DBImpl* db, const ReadOptions& read_options,
+ ColumnFamilyData* cfd, SuperVersion* current_sv = nullptr);
+ virtual ~ForwardIterator();
+
+ void SeekForPrev(const Slice& target) override {
+ status_ = Status::NotSupported("ForwardIterator::SeekForPrev()");
+ valid_ = false;
+ }
+ void SeekToLast() override {
+ status_ = Status::NotSupported("ForwardIterator::SeekToLast()");
+ valid_ = false;
+ }
+ void Prev() override {
+ status_ = Status::NotSupported("ForwardIterator::Prev");
+ valid_ = false;
+ }
+
+ virtual bool Valid() const override;
+ void SeekToFirst() override;
+ virtual void Seek(const Slice& target) override;
+ virtual void Next() override;
+ virtual Slice key() const override;
+ virtual Slice value() const override;
+ virtual Status status() const override;
+ virtual Status GetProperty(std::string prop_name, std::string* prop) override;
+ virtual void SetPinnedItersMgr(
+ PinnedIteratorsManager* pinned_iters_mgr) override;
+ virtual bool IsKeyPinned() const override;
+ virtual bool IsValuePinned() const override;
+
+ bool TEST_CheckDeletedIters(int* deleted_iters, int* num_iters);
+
+ private:
+ void Cleanup(bool release_sv);
+ void SVCleanup();
+ void RebuildIterators(bool refresh_sv);
+ void RenewIterators();
+ void BuildLevelIterators(const VersionStorageInfo* vstorage);
+ void ResetIncompleteIterators();
+ void SeekInternal(const Slice& internal_key, bool seek_to_first);
+ void UpdateCurrent();
+ bool NeedToSeekImmutable(const Slice& internal_key);
+ void DeleteCurrentIter();
+ uint32_t FindFileInRange(
+ const std::vector<FileMetaData*>& files, const Slice& internal_key,
+ uint32_t left, uint32_t right);
+
+ bool IsOverUpperBound(const Slice& internal_key) const;
+
+ // Set PinnedIteratorsManager for all children Iterators, this function should
+ // be called whenever we update children Iterators or pinned_iters_mgr_.
+ void UpdateChildrenPinnedItersMgr();
+
+ // A helper function that will release iter in the proper manner, or pass it
+ // to pinned_iters_mgr_ to release it later if pinning is enabled.
+ void DeleteIterator(InternalIterator* iter, bool is_arena = false);
+
+ DBImpl* const db_;
+ const ReadOptions read_options_;
+ ColumnFamilyData* const cfd_;
+ const SliceTransform* const prefix_extractor_;
+ const Comparator* user_comparator_;
+ MinIterHeap immutable_min_heap_;
+
+ SuperVersion* sv_;
+ InternalIterator* mutable_iter_;
+ std::vector<InternalIterator*> imm_iters_;
+ std::vector<InternalIterator*> l0_iters_;
+ std::vector<LevelIterator*> level_iters_;
+ InternalIterator* current_;
+ bool valid_;
+
+ // Internal iterator status; set only by one of the unsupported methods.
+ Status status_;
+ // Status of immutable iterators, maintained here to avoid iterating over
+ // all of them in status().
+ Status immutable_status_;
+ // Indicates that at least one of the immutable iterators pointed to a key
+ // larger than iterate_upper_bound and was therefore destroyed. Seek() may
+ // need to rebuild such iterators.
+ bool has_iter_trimmed_for_upper_bound_;
+ // Is current key larger than iterate_upper_bound? If so, makes Valid()
+ // return false.
+ bool current_over_upper_bound_;
+
+ // Left endpoint of the range of keys that immutable iterators currently
+ // cover. When Seek() is called with a key that's within that range, immutable
+ // iterators don't need to be moved; see NeedToSeekImmutable(). This key is
+ // included in the range after a Seek(), but excluded when advancing the
+ // iterator using Next().
+ IterKey prev_key_;
+ bool is_prev_set_;
+ bool is_prev_inclusive_;
+
+ PinnedIteratorsManager* pinned_iters_mgr_;
+ Arena arena_;
+};
+
+} // namespace rocksdb
+#endif // ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/forward_iterator_bench.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/forward_iterator_bench.cc b/thirdparty/rocksdb/db/forward_iterator_bench.cc
new file mode 100644
index 0000000..e9ae770
--- /dev/null
+++ b/thirdparty/rocksdb/db/forward_iterator_bench.cc
@@ -0,0 +1,375 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#if !defined(GFLAGS) || defined(ROCKSDB_LITE)
+#include <cstdio>
+int main() {
+ fprintf(stderr, "Please install gflags to run rocksdb tools\n");
+ return 1;
+}
+#elif defined(OS_MACOSX) || defined(OS_WIN)
+// Block forward_iterator_bench under MAC and Windows
+int main() { return 0; }
+#else
+#include <gflags/gflags.h>
+#include <semaphore.h>
+#include <atomic>
+#include <bitset>
+#include <chrono>
+#include <climits>
+#include <condition_variable>
+#include <limits>
+#include <mutex>
+#include <queue>
+#include <random>
+#include <thread>
+
+#include "rocksdb/cache.h"
+#include "rocksdb/db.h"
+#include "rocksdb/status.h"
+#include "rocksdb/table.h"
+#include "port/port.h"
+#include "util/testharness.h"
+
+const int MAX_SHARDS = 100000;
+
+DEFINE_int32(writers, 8, "");
+DEFINE_int32(readers, 8, "");
+DEFINE_int64(rate, 100000, "");
+DEFINE_int64(value_size, 300, "");
+DEFINE_int64(shards, 1000, "");
+DEFINE_int64(memtable_size, 500000000, "");
+DEFINE_int64(block_cache_size, 300000000, "");
+DEFINE_int64(block_size, 65536, "");
+DEFINE_double(runtime, 300.0, "");
+DEFINE_bool(cache_only_first, true, "");
+DEFINE_bool(iterate_upper_bound, true, "");
+
+struct Stats {
+ char pad1[128] __attribute__((__unused__));
+ std::atomic<uint64_t> written{0};
+ char pad2[128] __attribute__((__unused__));
+ std::atomic<uint64_t> read{0};
+ std::atomic<uint64_t> cache_misses{0};
+ char pad3[128] __attribute__((__unused__));
+} stats;
+
+struct Key {
+ Key() {}
+ Key(uint64_t shard_in, uint64_t seqno_in)
+ : shard_be(htobe64(shard_in)), seqno_be(htobe64(seqno_in)) {}
+
+ uint64_t shard() const { return be64toh(shard_be); }
+ uint64_t seqno() const { return be64toh(seqno_be); }
+
+ private:
+ uint64_t shard_be;
+ uint64_t seqno_be;
+} __attribute__((__packed__));
+
+struct Reader;
+struct Writer;
+
+struct ShardState {
+ char pad1[128] __attribute__((__unused__));
+ std::atomic<uint64_t> last_written{0};
+ Writer* writer;
+ Reader* reader;
+ char pad2[128] __attribute__((__unused__));
+ std::atomic<uint64_t> last_read{0};
+ std::unique_ptr<rocksdb::Iterator> it;
+ std::unique_ptr<rocksdb::Iterator> it_cacheonly;
+ Key upper_bound;
+ rocksdb::Slice upper_bound_slice;
+ char pad3[128] __attribute__((__unused__));
+};
+
+struct Reader {
+ public:
+ explicit Reader(std::vector<ShardState>* shard_states, rocksdb::DB* db)
+ : shard_states_(shard_states), db_(db) {
+ sem_init(&sem_, 0, 0);
+ thread_ = port::Thread(&Reader::run, this);
+ }
+
+ void run() {
+ while (1) {
+ sem_wait(&sem_);
+ if (done_.load()) {
+ break;
+ }
+
+ uint64_t shard;
+ {
+ std::lock_guard<std::mutex> guard(queue_mutex_);
+ assert(!shards_pending_queue_.empty());
+ shard = shards_pending_queue_.front();
+ shards_pending_queue_.pop();
+ shards_pending_set_.reset(shard);
+ }
+ readOnceFromShard(shard);
+ }
+ }
+
+ void readOnceFromShard(uint64_t shard) {
+ ShardState& state = (*shard_states_)[shard];
+ if (!state.it) {
+ // Initialize iterators
+ rocksdb::ReadOptions options;
+ options.tailing = true;
+ if (FLAGS_iterate_upper_bound) {
+ state.upper_bound = Key(shard, std::numeric_limits<uint64_t>::max());
+ state.upper_bound_slice = rocksdb::Slice(
+ (const char*)&state.upper_bound, sizeof(state.upper_bound));
+ options.iterate_upper_bound = &state.upper_bound_slice;
+ }
+
+ state.it.reset(db_->NewIterator(options));
+
+ if (FLAGS_cache_only_first) {
+ options.read_tier = rocksdb::ReadTier::kBlockCacheTier;
+ state.it_cacheonly.reset(db_->NewIterator(options));
+ }
+ }
+
+ const uint64_t upto = state.last_written.load();
+ for (rocksdb::Iterator* it : {state.it_cacheonly.get(), state.it.get()}) {
+ if (it == nullptr) {
+ continue;
+ }
+ if (state.last_read.load() >= upto) {
+ break;
+ }
+ bool need_seek = true;
+ for (uint64_t seq = state.last_read.load() + 1; seq <= upto; ++seq) {
+ if (need_seek) {
+ Key from(shard, state.last_read.load() + 1);
+ it->Seek(rocksdb::Slice((const char*)&from, sizeof(from)));
+ need_seek = false;
+ } else {
+ it->Next();
+ }
+ if (it->status().IsIncomplete()) {
+ ++::stats.cache_misses;
+ break;
+ }
+ assert(it->Valid());
+ assert(it->key().size() == sizeof(Key));
+ Key key;
+ memcpy(&key, it->key().data(), it->key().size());
+ // fprintf(stderr, "Expecting (%ld, %ld) read (%ld, %ld)\n",
+ // shard, seq, key.shard(), key.seqno());
+ assert(key.shard() == shard);
+ assert(key.seqno() == seq);
+ state.last_read.store(seq);
+ ++::stats.read;
+ }
+ }
+ }
+
+ void onWrite(uint64_t shard) {
+ {
+ std::lock_guard<std::mutex> guard(queue_mutex_);
+ if (!shards_pending_set_.test(shard)) {
+ shards_pending_queue_.push(shard);
+ shards_pending_set_.set(shard);
+ sem_post(&sem_);
+ }
+ }
+ }
+
+ ~Reader() {
+ done_.store(true);
+ sem_post(&sem_);
+ thread_.join();
+ }
+
+ private:
+ char pad1[128] __attribute__((__unused__));
+ std::vector<ShardState>* shard_states_;
+ rocksdb::DB* db_;
+ rocksdb::port::Thread thread_;
+ sem_t sem_;
+ std::mutex queue_mutex_;
+ std::bitset<MAX_SHARDS + 1> shards_pending_set_;
+ std::queue<uint64_t> shards_pending_queue_;
+ std::atomic<bool> done_{false};
+ char pad2[128] __attribute__((__unused__));
+};
+
+struct Writer {
+ explicit Writer(std::vector<ShardState>* shard_states, rocksdb::DB* db)
+ : shard_states_(shard_states), db_(db) {}
+
+ void start() { thread_ = port::Thread(&Writer::run, this); }
+
+ void run() {
+ std::queue<std::chrono::steady_clock::time_point> workq;
+ std::chrono::steady_clock::time_point deadline(
+ std::chrono::steady_clock::now() +
+ std::chrono::nanoseconds((uint64_t)(1000000000 * FLAGS_runtime)));
+ std::vector<uint64_t> my_shards;
+ for (int i = 1; i <= FLAGS_shards; ++i) {
+ if ((*shard_states_)[i].writer == this) {
+ my_shards.push_back(i);
+ }
+ }
+
+ std::mt19937 rng{std::random_device()()};
+ std::uniform_int_distribution<int> shard_dist(
+ 0, static_cast<int>(my_shards.size()) - 1);
+ std::string value(FLAGS_value_size, '*');
+
+ while (1) {
+ auto now = std::chrono::steady_clock::now();
+ if (FLAGS_runtime >= 0 && now >= deadline) {
+ break;
+ }
+ if (workq.empty()) {
+ for (int i = 0; i < FLAGS_rate; i += FLAGS_writers) {
+ std::chrono::nanoseconds offset(1000000000LL * i / FLAGS_rate);
+ workq.push(now + offset);
+ }
+ }
+ while (!workq.empty() && workq.front() < now) {
+ workq.pop();
+ uint64_t shard = my_shards[shard_dist(rng)];
+ ShardState& state = (*shard_states_)[shard];
+ uint64_t seqno = state.last_written.load() + 1;
+ Key key(shard, seqno);
+ // fprintf(stderr, "Writing (%ld, %ld)\n", shard, seqno);
+ rocksdb::Status status =
+ db_->Put(rocksdb::WriteOptions(),
+ rocksdb::Slice((const char*)&key, sizeof(key)),
+ rocksdb::Slice(value));
+ assert(status.ok());
+ state.last_written.store(seqno);
+ state.reader->onWrite(shard);
+ ++::stats.written;
+ }
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ }
+ // fprintf(stderr, "Writer done\n");
+ }
+
+ ~Writer() { thread_.join(); }
+
+ private:
+ char pad1[128] __attribute__((__unused__));
+ std::vector<ShardState>* shard_states_;
+ rocksdb::DB* db_;
+ rocksdb::port::Thread thread_;
+ char pad2[128] __attribute__((__unused__));
+};
+
+struct StatsThread {
+ explicit StatsThread(rocksdb::DB* db)
+ : db_(db), thread_(&StatsThread::run, this) {}
+
+ void run() {
+ // using namespace std::chrono;
+ auto tstart = std::chrono::steady_clock::now(), tlast = tstart;
+ uint64_t wlast = 0, rlast = 0;
+ while (!done_.load()) {
+ {
+ std::unique_lock<std::mutex> lock(cvm_);
+ cv_.wait_for(lock, std::chrono::seconds(1));
+ }
+ auto now = std::chrono::steady_clock::now();
+ double elapsed =
+ std::chrono::duration_cast<std::chrono::duration<double> >(
+ now - tlast).count();
+ uint64_t w = ::stats.written.load();
+ uint64_t r = ::stats.read.load();
+ fprintf(stderr,
+ "%s elapsed %4lds | written %10ld | w/s %10.0f | read %10ld | "
+ "r/s %10.0f | cache misses %10ld\n",
+ db_->GetEnv()->TimeToString(time(nullptr)).c_str(),
+ std::chrono::duration_cast<std::chrono::seconds>(now - tstart)
+ .count(),
+ w, (w - wlast) / elapsed, r, (r - rlast) / elapsed,
+ ::stats.cache_misses.load());
+ wlast = w;
+ rlast = r;
+ tlast = now;
+ }
+ }
+
+ ~StatsThread() {
+ {
+ std::lock_guard<std::mutex> guard(cvm_);
+ done_.store(true);
+ }
+ cv_.notify_all();
+ thread_.join();
+ }
+
+ private:
+ rocksdb::DB* db_;
+ std::mutex cvm_;
+ std::condition_variable cv_;
+ rocksdb::port::Thread thread_;
+ std::atomic<bool> done_{false};
+};
+
+int main(int argc, char** argv) {
+ GFLAGS::ParseCommandLineFlags(&argc, &argv, true);
+
+ std::mt19937 rng{std::random_device()()};
+ rocksdb::Status status;
+ std::string path = rocksdb::test::TmpDir() + "/forward_iterator_test";
+ fprintf(stderr, "db path is %s\n", path.c_str());
+ rocksdb::Options options;
+ options.create_if_missing = true;
+ options.compression = rocksdb::CompressionType::kNoCompression;
+ options.compaction_style = rocksdb::CompactionStyle::kCompactionStyleNone;
+ options.level0_slowdown_writes_trigger = 99999;
+ options.level0_stop_writes_trigger = 99999;
+ options.use_direct_io_for_flush_and_compaction = true;
+ options.write_buffer_size = FLAGS_memtable_size;
+ rocksdb::BlockBasedTableOptions table_options;
+ table_options.block_cache = rocksdb::NewLRUCache(FLAGS_block_cache_size);
+ table_options.block_size = FLAGS_block_size;
+ options.table_factory.reset(
+ rocksdb::NewBlockBasedTableFactory(table_options));
+
+ status = rocksdb::DestroyDB(path, options);
+ assert(status.ok());
+ rocksdb::DB* db_raw;
+ status = rocksdb::DB::Open(options, path, &db_raw);
+ assert(status.ok());
+ std::unique_ptr<rocksdb::DB> db(db_raw);
+
+ std::vector<ShardState> shard_states(FLAGS_shards + 1);
+ std::deque<Reader> readers;
+ while (static_cast<int>(readers.size()) < FLAGS_readers) {
+ readers.emplace_back(&shard_states, db_raw);
+ }
+ std::deque<Writer> writers;
+ while (static_cast<int>(writers.size()) < FLAGS_writers) {
+ writers.emplace_back(&shard_states, db_raw);
+ }
+
+ // Each shard gets a random reader and random writer assigned to it
+ for (int i = 1; i <= FLAGS_shards; ++i) {
+ std::uniform_int_distribution<int> reader_dist(0, FLAGS_readers - 1);
+ std::uniform_int_distribution<int> writer_dist(0, FLAGS_writers - 1);
+ shard_states[i].reader = &readers[reader_dist(rng)];
+ shard_states[i].writer = &writers[writer_dist(rng)];
+ }
+
+ StatsThread stats_thread(db_raw);
+ for (Writer& w : writers) {
+ w.start();
+ }
+
+ writers.clear();
+ readers.clear();
+}
+#endif // !defined(GFLAGS) || defined(ROCKSDB_LITE)
[21/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_impl_compaction_flush.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_impl_compaction_flush.cc b/thirdparty/rocksdb/db/db_impl_compaction_flush.cc
new file mode 100644
index 0000000..3e686fe
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_impl_compaction_flush.cc
@@ -0,0 +1,1910 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include "db/db_impl.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+#include <inttypes.h>
+
+#include "db/builder.h"
+#include "db/event_helpers.h"
+#include "monitoring/iostats_context_imp.h"
+#include "monitoring/perf_context_imp.h"
+#include "monitoring/thread_status_updater.h"
+#include "monitoring/thread_status_util.h"
+#include "util/sst_file_manager_impl.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+Status DBImpl::SyncClosedLogs(JobContext* job_context) {
+ TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Start");
+ mutex_.AssertHeld();
+ autovector<log::Writer*, 1> logs_to_sync;
+ uint64_t current_log_number = logfile_number_;
+ while (logs_.front().number < current_log_number &&
+ logs_.front().getting_synced) {
+ log_sync_cv_.Wait();
+ }
+ for (auto it = logs_.begin();
+ it != logs_.end() && it->number < current_log_number; ++it) {
+ auto& log = *it;
+ assert(!log.getting_synced);
+ log.getting_synced = true;
+ logs_to_sync.push_back(log.writer);
+ }
+
+ Status s;
+ if (!logs_to_sync.empty()) {
+ mutex_.Unlock();
+
+ for (log::Writer* log : logs_to_sync) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "[JOB %d] Syncing log #%" PRIu64, job_context->job_id,
+ log->get_log_number());
+ s = log->file()->Sync(immutable_db_options_.use_fsync);
+ }
+ if (s.ok()) {
+ s = directories_.GetWalDir()->Fsync();
+ }
+
+ mutex_.Lock();
+
+ // "number <= current_log_number - 1" is equivalent to
+ // "number < current_log_number".
+ MarkLogsSynced(current_log_number - 1, true, s);
+ if (!s.ok()) {
+ Status new_bg_error = s;
+ // may temporarily unlock and lock the mutex.
+ EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
+ BackgroundErrorReason::kFlush,
+ &new_bg_error, &mutex_);
+ if (!new_bg_error.ok()) {
+ bg_error_ = new_bg_error;
+ }
+ TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Failed");
+ return s;
+ }
+ }
+ return s;
+}
+
+Status DBImpl::FlushMemTableToOutputFile(
+ ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
+ bool* made_progress, JobContext* job_context, LogBuffer* log_buffer) {
+ mutex_.AssertHeld();
+ assert(cfd->imm()->NumNotFlushed() != 0);
+ assert(cfd->imm()->IsFlushPending());
+
+ SequenceNumber earliest_write_conflict_snapshot;
+ std::vector<SequenceNumber> snapshot_seqs =
+ snapshots_.GetAll(&earliest_write_conflict_snapshot);
+
+ FlushJob flush_job(
+ dbname_, cfd, immutable_db_options_, mutable_cf_options, env_options_,
+ versions_.get(), &mutex_, &shutting_down_, snapshot_seqs,
+ earliest_write_conflict_snapshot, job_context, log_buffer,
+ directories_.GetDbDir(), directories_.GetDataDir(0U),
+ GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
+ &event_logger_, mutable_cf_options.report_bg_io_stats);
+
+ FileMetaData file_meta;
+
+ flush_job.PickMemTable();
+
+#ifndef ROCKSDB_LITE
+ // may temporarily unlock and lock the mutex.
+ NotifyOnFlushBegin(cfd, &file_meta, mutable_cf_options, job_context->job_id,
+ flush_job.GetTableProperties());
+#endif // ROCKSDB_LITE
+
+ Status s;
+ if (logfile_number_ > 0 &&
+ versions_->GetColumnFamilySet()->NumberOfColumnFamilies() > 0) {
+ // If there are more than one column families, we need to make sure that
+ // all the log files except the most recent one are synced. Otherwise if
+ // the host crashes after flushing and before WAL is persistent, the
+ // flushed SST may contain data from write batches whose updates to
+ // other column families are missing.
+ // SyncClosedLogs() may unlock and re-lock the db_mutex.
+ s = SyncClosedLogs(job_context);
+ }
+
+ // Within flush_job.Run, rocksdb may call event listener to notify
+ // file creation and deletion.
+ //
+ // Note that flush_job.Run will unlock and lock the db_mutex,
+ // and EventListener callback will be called when the db_mutex
+ // is unlocked by the current thread.
+ if (s.ok()) {
+ s = flush_job.Run(&file_meta);
+ } else {
+ flush_job.Cancel();
+ }
+
+ if (s.ok()) {
+ InstallSuperVersionAndScheduleWorkWrapper(cfd, job_context,
+ mutable_cf_options);
+ if (made_progress) {
+ *made_progress = 1;
+ }
+ VersionStorageInfo::LevelSummaryStorage tmp;
+ ROCKS_LOG_BUFFER(log_buffer, "[%s] Level summary: %s\n",
+ cfd->GetName().c_str(),
+ cfd->current()->storage_info()->LevelSummary(&tmp));
+ }
+
+ if (!s.ok() && !s.IsShutdownInProgress() &&
+ immutable_db_options_.paranoid_checks && bg_error_.ok()) {
+ Status new_bg_error = s;
+ // may temporarily unlock and lock the mutex.
+ EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
+ BackgroundErrorReason::kFlush,
+ &new_bg_error, &mutex_);
+ if (!new_bg_error.ok()) {
+ // if a bad error happened (not ShutdownInProgress), paranoid_checks is
+ // true, and the error isn't handled by callback, mark DB read-only
+ bg_error_ = new_bg_error;
+ }
+ }
+ if (s.ok()) {
+#ifndef ROCKSDB_LITE
+ // may temporarily unlock and lock the mutex.
+ NotifyOnFlushCompleted(cfd, &file_meta, mutable_cf_options,
+ job_context->job_id, flush_job.GetTableProperties());
+ auto sfm = static_cast<SstFileManagerImpl*>(
+ immutable_db_options_.sst_file_manager.get());
+ if (sfm) {
+ // Notify sst_file_manager that a new file was added
+ std::string file_path = MakeTableFileName(
+ immutable_db_options_.db_paths[0].path, file_meta.fd.GetNumber());
+ sfm->OnAddFile(file_path);
+ if (sfm->IsMaxAllowedSpaceReached() && bg_error_.ok()) {
+ Status new_bg_error = Status::IOError("Max allowed space was reached");
+ TEST_SYNC_POINT_CALLBACK(
+ "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached",
+ &new_bg_error);
+ // may temporarily unlock and lock the mutex.
+ EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
+ BackgroundErrorReason::kFlush,
+ &new_bg_error, &mutex_);
+ if (!new_bg_error.ok()) {
+ bg_error_ = new_bg_error;
+ }
+ }
+ }
+#endif // ROCKSDB_LITE
+ }
+ return s;
+}
+
+void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
+ const MutableCFOptions& mutable_cf_options,
+ int job_id, TableProperties prop) {
+#ifndef ROCKSDB_LITE
+ if (immutable_db_options_.listeners.size() == 0U) {
+ return;
+ }
+ mutex_.AssertHeld();
+ if (shutting_down_.load(std::memory_order_acquire)) {
+ return;
+ }
+ bool triggered_writes_slowdown =
+ (cfd->current()->storage_info()->NumLevelFiles(0) >=
+ mutable_cf_options.level0_slowdown_writes_trigger);
+ bool triggered_writes_stop =
+ (cfd->current()->storage_info()->NumLevelFiles(0) >=
+ mutable_cf_options.level0_stop_writes_trigger);
+ // release lock while notifying events
+ mutex_.Unlock();
+ {
+ FlushJobInfo info;
+ info.cf_name = cfd->GetName();
+ // TODO(yhchiang): make db_paths dynamic in case flush does not
+ // go to L0 in the future.
+ info.file_path = MakeTableFileName(immutable_db_options_.db_paths[0].path,
+ file_meta->fd.GetNumber());
+ info.thread_id = env_->GetThreadID();
+ info.job_id = job_id;
+ info.triggered_writes_slowdown = triggered_writes_slowdown;
+ info.triggered_writes_stop = triggered_writes_stop;
+ info.smallest_seqno = file_meta->smallest_seqno;
+ info.largest_seqno = file_meta->largest_seqno;
+ info.table_properties = prop;
+ for (auto listener : immutable_db_options_.listeners) {
+ listener->OnFlushBegin(this, info);
+ }
+ }
+ mutex_.Lock();
+// no need to signal bg_cv_ as it will be signaled at the end of the
+// flush process.
+#endif // ROCKSDB_LITE
+}
+
+void DBImpl::NotifyOnFlushCompleted(ColumnFamilyData* cfd,
+ FileMetaData* file_meta,
+ const MutableCFOptions& mutable_cf_options,
+ int job_id, TableProperties prop) {
+#ifndef ROCKSDB_LITE
+ if (immutable_db_options_.listeners.size() == 0U) {
+ return;
+ }
+ mutex_.AssertHeld();
+ if (shutting_down_.load(std::memory_order_acquire)) {
+ return;
+ }
+ bool triggered_writes_slowdown =
+ (cfd->current()->storage_info()->NumLevelFiles(0) >=
+ mutable_cf_options.level0_slowdown_writes_trigger);
+ bool triggered_writes_stop =
+ (cfd->current()->storage_info()->NumLevelFiles(0) >=
+ mutable_cf_options.level0_stop_writes_trigger);
+ // release lock while notifying events
+ mutex_.Unlock();
+ {
+ FlushJobInfo info;
+ info.cf_name = cfd->GetName();
+ // TODO(yhchiang): make db_paths dynamic in case flush does not
+ // go to L0 in the future.
+ info.file_path = MakeTableFileName(immutable_db_options_.db_paths[0].path,
+ file_meta->fd.GetNumber());
+ info.thread_id = env_->GetThreadID();
+ info.job_id = job_id;
+ info.triggered_writes_slowdown = triggered_writes_slowdown;
+ info.triggered_writes_stop = triggered_writes_stop;
+ info.smallest_seqno = file_meta->smallest_seqno;
+ info.largest_seqno = file_meta->largest_seqno;
+ info.table_properties = prop;
+ for (auto listener : immutable_db_options_.listeners) {
+ listener->OnFlushCompleted(this, info);
+ }
+ }
+ mutex_.Lock();
+ // no need to signal bg_cv_ as it will be signaled at the end of the
+ // flush process.
+#endif // ROCKSDB_LITE
+}
+
+Status DBImpl::CompactRange(const CompactRangeOptions& options,
+ ColumnFamilyHandle* column_family,
+ const Slice* begin, const Slice* end) {
+ if (options.target_path_id >= immutable_db_options_.db_paths.size()) {
+ return Status::InvalidArgument("Invalid target path ID");
+ }
+
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ auto cfd = cfh->cfd();
+ bool exclusive = options.exclusive_manual_compaction;
+
+ Status s = FlushMemTable(cfd, FlushOptions());
+ if (!s.ok()) {
+ LogFlush(immutable_db_options_.info_log);
+ return s;
+ }
+
+ int max_level_with_files = 0;
+ {
+ InstrumentedMutexLock l(&mutex_);
+ Version* base = cfd->current();
+ for (int level = 1; level < base->storage_info()->num_non_empty_levels();
+ level++) {
+ if (base->storage_info()->OverlapInLevel(level, begin, end)) {
+ max_level_with_files = level;
+ }
+ }
+ }
+
+ int final_output_level = 0;
+ if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal &&
+ cfd->NumberLevels() > 1) {
+ // Always compact all files together.
+ final_output_level = cfd->NumberLevels() - 1;
+ // if bottom most level is reserved
+ if (immutable_db_options_.allow_ingest_behind) {
+ final_output_level--;
+ }
+ s = RunManualCompaction(cfd, ColumnFamilyData::kCompactAllLevels,
+ final_output_level, options.target_path_id,
+ begin, end, exclusive);
+ } else {
+ for (int level = 0; level <= max_level_with_files; level++) {
+ int output_level;
+ // in case the compaction is universal or if we're compacting the
+ // bottom-most level, the output level will be the same as input one.
+ // level 0 can never be the bottommost level (i.e. if all files are in
+ // level 0, we will compact to level 1)
+ if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
+ cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
+ output_level = level;
+ } else if (level == max_level_with_files && level > 0) {
+ if (options.bottommost_level_compaction ==
+ BottommostLevelCompaction::kSkip) {
+ // Skip bottommost level compaction
+ continue;
+ } else if (options.bottommost_level_compaction ==
+ BottommostLevelCompaction::kIfHaveCompactionFilter &&
+ cfd->ioptions()->compaction_filter == nullptr &&
+ cfd->ioptions()->compaction_filter_factory == nullptr) {
+ // Skip bottommost level compaction since we don't have a compaction
+ // filter
+ continue;
+ }
+ output_level = level;
+ } else {
+ output_level = level + 1;
+ if (cfd->ioptions()->compaction_style == kCompactionStyleLevel &&
+ cfd->ioptions()->level_compaction_dynamic_level_bytes &&
+ level == 0) {
+ output_level = ColumnFamilyData::kCompactToBaseLevel;
+ }
+ }
+ s = RunManualCompaction(cfd, level, output_level, options.target_path_id,
+ begin, end, exclusive);
+ if (!s.ok()) {
+ break;
+ }
+ if (output_level == ColumnFamilyData::kCompactToBaseLevel) {
+ final_output_level = cfd->NumberLevels() - 1;
+ } else if (output_level > final_output_level) {
+ final_output_level = output_level;
+ }
+ TEST_SYNC_POINT("DBImpl::RunManualCompaction()::1");
+ TEST_SYNC_POINT("DBImpl::RunManualCompaction()::2");
+ }
+ }
+ if (!s.ok()) {
+ LogFlush(immutable_db_options_.info_log);
+ return s;
+ }
+
+ if (options.change_level) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "[RefitLevel] waiting for background threads to stop");
+ s = PauseBackgroundWork();
+ if (s.ok()) {
+ s = ReFitLevel(cfd, final_output_level, options.target_level);
+ }
+ ContinueBackgroundWork();
+ }
+ LogFlush(immutable_db_options_.info_log);
+
+ {
+ InstrumentedMutexLock l(&mutex_);
+ // an automatic compaction that has been scheduled might have been
+ // preempted by the manual compactions. Need to schedule it back.
+ MaybeScheduleFlushOrCompaction();
+ }
+
+ return s;
+}
+
+Status DBImpl::CompactFiles(
+ const CompactionOptions& compact_options,
+ ColumnFamilyHandle* column_family,
+ const std::vector<std::string>& input_file_names,
+ const int output_level, const int output_path_id) {
+#ifdef ROCKSDB_LITE
+ // not supported in lite version
+ return Status::NotSupported("Not supported in ROCKSDB LITE");
+#else
+ if (column_family == nullptr) {
+ return Status::InvalidArgument("ColumnFamilyHandle must be non-null.");
+ }
+
+ auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
+ assert(cfd);
+
+ Status s;
+ JobContext job_context(0, true);
+ LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL,
+ immutable_db_options_.info_log.get());
+
+ // Perform CompactFiles
+ SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+ {
+ InstrumentedMutexLock l(&mutex_);
+
+ // This call will unlock/lock the mutex to wait for current running
+ // IngestExternalFile() calls to finish.
+ WaitForIngestFile();
+
+ s = CompactFilesImpl(compact_options, cfd, sv->current,
+ input_file_names, output_level,
+ output_path_id, &job_context, &log_buffer);
+ }
+ if (sv->Unref()) {
+ mutex_.Lock();
+ sv->Cleanup();
+ mutex_.Unlock();
+ delete sv;
+ }
+
+ // Find and delete obsolete files
+ {
+ InstrumentedMutexLock l(&mutex_);
+ // If !s.ok(), this means that Compaction failed. In that case, we want
+ // to delete all obsolete files we might have created and we force
+ // FindObsoleteFiles(). This is because job_context does not
+ // catch all created files if compaction failed.
+ FindObsoleteFiles(&job_context, !s.ok());
+ } // release the mutex
+
+ // delete unnecessary files if any, this is done outside the mutex
+ if (job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) {
+ // Have to flush the info logs before bg_compaction_scheduled_--
+ // because if bg_flush_scheduled_ becomes 0 and the lock is
+ // released, the deconstructor of DB can kick in and destroy all the
+ // states of DB so info_log might not be available after that point.
+ // It also applies to access other states that DB owns.
+ log_buffer.FlushBufferToLog();
+ if (job_context.HaveSomethingToDelete()) {
+ // no mutex is locked here. No need to Unlock() and Lock() here.
+ PurgeObsoleteFiles(job_context);
+ }
+ job_context.Clean();
+ }
+
+ return s;
+#endif // ROCKSDB_LITE
+}
+
+#ifndef ROCKSDB_LITE
+Status DBImpl::CompactFilesImpl(
+ const CompactionOptions& compact_options, ColumnFamilyData* cfd,
+ Version* version, const std::vector<std::string>& input_file_names,
+ const int output_level, int output_path_id, JobContext* job_context,
+ LogBuffer* log_buffer) {
+ mutex_.AssertHeld();
+
+ if (shutting_down_.load(std::memory_order_acquire)) {
+ return Status::ShutdownInProgress();
+ }
+
+ std::unordered_set<uint64_t> input_set;
+ for (auto file_name : input_file_names) {
+ input_set.insert(TableFileNameToNumber(file_name));
+ }
+
+ ColumnFamilyMetaData cf_meta;
+ // TODO(yhchiang): can directly use version here if none of the
+ // following functions call is pluggable to external developers.
+ version->GetColumnFamilyMetaData(&cf_meta);
+
+ if (output_path_id < 0) {
+ if (immutable_db_options_.db_paths.size() == 1U) {
+ output_path_id = 0;
+ } else {
+ return Status::NotSupported(
+ "Automatic output path selection is not "
+ "yet supported in CompactFiles()");
+ }
+ }
+
+ Status s = cfd->compaction_picker()->SanitizeCompactionInputFiles(
+ &input_set, cf_meta, output_level);
+ if (!s.ok()) {
+ return s;
+ }
+
+ std::vector<CompactionInputFiles> input_files;
+ s = cfd->compaction_picker()->GetCompactionInputsFromFileNumbers(
+ &input_files, &input_set, version->storage_info(), compact_options);
+ if (!s.ok()) {
+ return s;
+ }
+
+ for (auto inputs : input_files) {
+ if (cfd->compaction_picker()->AreFilesInCompaction(inputs.files)) {
+ return Status::Aborted(
+ "Some of the necessary compaction input "
+ "files are already being compacted");
+ }
+ }
+
+ // At this point, CompactFiles will be run.
+ bg_compaction_scheduled_++;
+
+ unique_ptr<Compaction> c;
+ assert(cfd->compaction_picker());
+ c.reset(cfd->compaction_picker()->CompactFiles(
+ compact_options, input_files, output_level, version->storage_info(),
+ *cfd->GetLatestMutableCFOptions(), output_path_id));
+ if (!c) {
+ return Status::Aborted("Another Level 0 compaction is running");
+ }
+ c->SetInputVersion(version);
+ // deletion compaction currently not allowed in CompactFiles.
+ assert(!c->deletion_compaction());
+
+ SequenceNumber earliest_write_conflict_snapshot;
+ std::vector<SequenceNumber> snapshot_seqs =
+ snapshots_.GetAll(&earliest_write_conflict_snapshot);
+
+ auto pending_outputs_inserted_elem =
+ CaptureCurrentFileNumberInPendingOutputs();
+
+ assert(is_snapshot_supported_ || snapshots_.empty());
+ CompactionJob compaction_job(
+ job_context->job_id, c.get(), immutable_db_options_, env_options_,
+ versions_.get(), &shutting_down_, log_buffer, directories_.GetDbDir(),
+ directories_.GetDataDir(c->output_path_id()), stats_, &mutex_, &bg_error_,
+ snapshot_seqs, earliest_write_conflict_snapshot, table_cache_,
+ &event_logger_, c->mutable_cf_options()->paranoid_file_checks,
+ c->mutable_cf_options()->report_bg_io_stats, dbname_,
+ nullptr); // Here we pass a nullptr for CompactionJobStats because
+ // CompactFiles does not trigger OnCompactionCompleted(),
+ // which is the only place where CompactionJobStats is
+ // returned. The idea of not triggering OnCompationCompleted()
+ // is that CompactFiles runs in the caller thread, so the user
+ // should always know when it completes. As a result, it makes
+ // less sense to notify the users something they should already
+ // know.
+ //
+ // In the future, if we would like to add CompactionJobStats
+ // support for CompactFiles, we should have CompactFiles API
+ // pass a pointer of CompactionJobStats as the out-value
+ // instead of using EventListener.
+
+ // Creating a compaction influences the compaction score because the score
+ // takes running compactions into account (by skipping files that are already
+ // being compacted). Since we just changed compaction score, we recalculate it
+ // here.
+ version->storage_info()->ComputeCompactionScore(*cfd->ioptions(),
+ *c->mutable_cf_options());
+
+ compaction_job.Prepare();
+
+ mutex_.Unlock();
+ TEST_SYNC_POINT("CompactFilesImpl:0");
+ TEST_SYNC_POINT("CompactFilesImpl:1");
+ compaction_job.Run();
+ TEST_SYNC_POINT("CompactFilesImpl:2");
+ TEST_SYNC_POINT("CompactFilesImpl:3");
+ mutex_.Lock();
+
+ Status status = compaction_job.Install(*c->mutable_cf_options());
+ if (status.ok()) {
+ InstallSuperVersionAndScheduleWorkWrapper(
+ c->column_family_data(), job_context, *c->mutable_cf_options());
+ }
+ c->ReleaseCompactionFiles(s);
+
+ ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
+
+ if (status.ok()) {
+ // Done
+ } else if (status.IsShutdownInProgress()) {
+ // Ignore compaction errors found during shutting down
+ } else {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "[%s] [JOB %d] Compaction error: %s",
+ c->column_family_data()->GetName().c_str(),
+ job_context->job_id, status.ToString().c_str());
+ if (immutable_db_options_.paranoid_checks && bg_error_.ok()) {
+ Status new_bg_error = status;
+ // may temporarily unlock and lock the mutex.
+ EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
+ BackgroundErrorReason::kCompaction,
+ &new_bg_error, &mutex_);
+ if (!new_bg_error.ok()) {
+ bg_error_ = new_bg_error;
+ }
+ }
+ }
+
+ c.reset();
+
+ bg_compaction_scheduled_--;
+ if (bg_compaction_scheduled_ == 0) {
+ bg_cv_.SignalAll();
+ }
+
+ return status;
+}
+#endif // ROCKSDB_LITE
+
+Status DBImpl::PauseBackgroundWork() {
+ InstrumentedMutexLock guard_lock(&mutex_);
+ bg_compaction_paused_++;
+ while (bg_bottom_compaction_scheduled_ > 0 || bg_compaction_scheduled_ > 0 ||
+ bg_flush_scheduled_ > 0) {
+ bg_cv_.Wait();
+ }
+ bg_work_paused_++;
+ return Status::OK();
+}
+
+Status DBImpl::ContinueBackgroundWork() {
+ InstrumentedMutexLock guard_lock(&mutex_);
+ if (bg_work_paused_ == 0) {
+ return Status::InvalidArgument();
+ }
+ assert(bg_work_paused_ > 0);
+ assert(bg_compaction_paused_ > 0);
+ bg_compaction_paused_--;
+ bg_work_paused_--;
+ // It's sufficient to check just bg_work_paused_ here since
+ // bg_work_paused_ is always no greater than bg_compaction_paused_
+ if (bg_work_paused_ == 0) {
+ MaybeScheduleFlushOrCompaction();
+ }
+ return Status::OK();
+}
+
+void DBImpl::NotifyOnCompactionCompleted(
+ ColumnFamilyData* cfd, Compaction *c, const Status &st,
+ const CompactionJobStats& compaction_job_stats,
+ const int job_id) {
+#ifndef ROCKSDB_LITE
+ if (immutable_db_options_.listeners.size() == 0U) {
+ return;
+ }
+ mutex_.AssertHeld();
+ if (shutting_down_.load(std::memory_order_acquire)) {
+ return;
+ }
+ // release lock while notifying events
+ mutex_.Unlock();
+ TEST_SYNC_POINT("DBImpl::NotifyOnCompactionCompleted::UnlockMutex");
+ {
+ CompactionJobInfo info;
+ info.cf_name = cfd->GetName();
+ info.status = st;
+ info.thread_id = env_->GetThreadID();
+ info.job_id = job_id;
+ info.base_input_level = c->start_level();
+ info.output_level = c->output_level();
+ info.stats = compaction_job_stats;
+ info.table_properties = c->GetOutputTableProperties();
+ info.compaction_reason = c->compaction_reason();
+ info.compression = c->output_compression();
+ for (size_t i = 0; i < c->num_input_levels(); ++i) {
+ for (const auto fmd : *c->inputs(i)) {
+ auto fn = TableFileName(immutable_db_options_.db_paths,
+ fmd->fd.GetNumber(), fmd->fd.GetPathId());
+ info.input_files.push_back(fn);
+ if (info.table_properties.count(fn) == 0) {
+ std::shared_ptr<const TableProperties> tp;
+ auto s = cfd->current()->GetTableProperties(&tp, fmd, &fn);
+ if (s.ok()) {
+ info.table_properties[fn] = tp;
+ }
+ }
+ }
+ }
+ for (const auto newf : c->edit()->GetNewFiles()) {
+ info.output_files.push_back(TableFileName(immutable_db_options_.db_paths,
+ newf.second.fd.GetNumber(),
+ newf.second.fd.GetPathId()));
+ }
+ for (auto listener : immutable_db_options_.listeners) {
+ listener->OnCompactionCompleted(this, info);
+ }
+ }
+ mutex_.Lock();
+ // no need to signal bg_cv_ as it will be signaled at the end of the
+ // flush process.
+#endif // ROCKSDB_LITE
+}
+
+// REQUIREMENT: block all background work by calling PauseBackgroundWork()
+// before calling this function
+Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
+ assert(level < cfd->NumberLevels());
+ if (target_level >= cfd->NumberLevels()) {
+ return Status::InvalidArgument("Target level exceeds number of levels");
+ }
+
+ std::unique_ptr<SuperVersion> superversion_to_free;
+ std::unique_ptr<SuperVersion> new_superversion(new SuperVersion());
+
+ Status status;
+
+ InstrumentedMutexLock guard_lock(&mutex_);
+
+ // only allow one thread refitting
+ if (refitting_level_) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "[ReFitLevel] another thread is refitting");
+ return Status::NotSupported("another thread is refitting");
+ }
+ refitting_level_ = true;
+
+ const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
+ // move to a smaller level
+ int to_level = target_level;
+ if (target_level < 0) {
+ to_level = FindMinimumEmptyLevelFitting(cfd, mutable_cf_options, level);
+ }
+
+ auto* vstorage = cfd->current()->storage_info();
+ if (to_level > level) {
+ if (level == 0) {
+ return Status::NotSupported(
+ "Cannot change from level 0 to other levels.");
+ }
+ // Check levels are empty for a trivial move
+ for (int l = level + 1; l <= to_level; l++) {
+ if (vstorage->NumLevelFiles(l) > 0) {
+ return Status::NotSupported(
+ "Levels between source and target are not empty for a move.");
+ }
+ }
+ }
+ if (to_level != level) {
+ ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
+ "[%s] Before refitting:\n%s", cfd->GetName().c_str(),
+ cfd->current()->DebugString().data());
+
+ VersionEdit edit;
+ edit.SetColumnFamily(cfd->GetID());
+ for (const auto& f : vstorage->LevelFiles(level)) {
+ edit.DeleteFile(level, f->fd.GetNumber());
+ edit.AddFile(to_level, f->fd.GetNumber(), f->fd.GetPathId(),
+ f->fd.GetFileSize(), f->smallest, f->largest,
+ f->smallest_seqno, f->largest_seqno,
+ f->marked_for_compaction);
+ }
+ ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
+ "[%s] Apply version edit:\n%s", cfd->GetName().c_str(),
+ edit.DebugString().data());
+
+ status = versions_->LogAndApply(cfd, mutable_cf_options, &edit, &mutex_,
+ directories_.GetDbDir());
+ superversion_to_free.reset(InstallSuperVersionAndScheduleWork(
+ cfd, new_superversion.release(), mutable_cf_options));
+
+ ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "[%s] LogAndApply: %s\n",
+ cfd->GetName().c_str(), status.ToString().data());
+
+ if (status.ok()) {
+ ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
+ "[%s] After refitting:\n%s", cfd->GetName().c_str(),
+ cfd->current()->DebugString().data());
+ }
+ }
+
+ refitting_level_ = false;
+
+ return status;
+}
+
+int DBImpl::NumberLevels(ColumnFamilyHandle* column_family) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ return cfh->cfd()->NumberLevels();
+}
+
+int DBImpl::MaxMemCompactionLevel(ColumnFamilyHandle* column_family) {
+ return 0;
+}
+
+int DBImpl::Level0StopWriteTrigger(ColumnFamilyHandle* column_family) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ InstrumentedMutexLock l(&mutex_);
+ return cfh->cfd()->GetSuperVersion()->
+ mutable_cf_options.level0_stop_writes_trigger;
+}
+
+Status DBImpl::Flush(const FlushOptions& flush_options,
+ ColumnFamilyHandle* column_family) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ return FlushMemTable(cfh->cfd(), flush_options);
+}
+
+Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
+ int output_level, uint32_t output_path_id,
+ const Slice* begin, const Slice* end,
+ bool exclusive, bool disallow_trivial_move) {
+ assert(input_level == ColumnFamilyData::kCompactAllLevels ||
+ input_level >= 0);
+
+ InternalKey begin_storage, end_storage;
+ CompactionArg* ca;
+
+ bool scheduled = false;
+ bool manual_conflict = false;
+ ManualCompactionState manual;
+ manual.cfd = cfd;
+ manual.input_level = input_level;
+ manual.output_level = output_level;
+ manual.output_path_id = output_path_id;
+ manual.done = false;
+ manual.in_progress = false;
+ manual.incomplete = false;
+ manual.exclusive = exclusive;
+ manual.disallow_trivial_move = disallow_trivial_move;
+ // For universal compaction, we enforce every manual compaction to compact
+ // all files.
+ if (begin == nullptr ||
+ cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
+ cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
+ manual.begin = nullptr;
+ } else {
+ begin_storage.SetMaxPossibleForUserKey(*begin);
+ manual.begin = &begin_storage;
+ }
+ if (end == nullptr ||
+ cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
+ cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
+ manual.end = nullptr;
+ } else {
+ end_storage.SetMinPossibleForUserKey(*end);
+ manual.end = &end_storage;
+ }
+
+ TEST_SYNC_POINT("DBImpl::RunManualCompaction:0");
+ TEST_SYNC_POINT("DBImpl::RunManualCompaction:1");
+ InstrumentedMutexLock l(&mutex_);
+
+ // When a manual compaction arrives, temporarily disable scheduling of
+ // non-manual compactions and wait until the number of scheduled compaction
+ // jobs drops to zero. This is needed to ensure that this manual compaction
+ // can compact any range of keys/files.
+ //
+ // HasPendingManualCompaction() is true when at least one thread is inside
+ // RunManualCompaction(), i.e. during that time no other compaction will
+ // get scheduled (see MaybeScheduleFlushOrCompaction).
+ //
+ // Note that the following loop doesn't stop more that one thread calling
+ // RunManualCompaction() from getting to the second while loop below.
+ // However, only one of them will actually schedule compaction, while
+ // others will wait on a condition variable until it completes.
+
+ AddManualCompaction(&manual);
+ TEST_SYNC_POINT_CALLBACK("DBImpl::RunManualCompaction:NotScheduled", &mutex_);
+ if (exclusive) {
+ while (bg_bottom_compaction_scheduled_ > 0 ||
+ bg_compaction_scheduled_ > 0) {
+ TEST_SYNC_POINT("DBImpl::RunManualCompaction:WaitScheduled");
+ ROCKS_LOG_INFO(
+ immutable_db_options_.info_log,
+ "[%s] Manual compaction waiting for all other scheduled background "
+ "compactions to finish",
+ cfd->GetName().c_str());
+ bg_cv_.Wait();
+ }
+ }
+
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "[%s] Manual compaction starting", cfd->GetName().c_str());
+
+ // We don't check bg_error_ here, because if we get the error in compaction,
+ // the compaction will set manual.status to bg_error_ and set manual.done to
+ // true.
+ while (!manual.done) {
+ assert(HasPendingManualCompaction());
+ manual_conflict = false;
+ Compaction* compaction;
+ if (ShouldntRunManualCompaction(&manual) || (manual.in_progress == true) ||
+ scheduled ||
+ ((manual.manual_end = &manual.tmp_storage1) &&
+ ((compaction = manual.cfd->CompactRange(
+ *manual.cfd->GetLatestMutableCFOptions(), manual.input_level,
+ manual.output_level, manual.output_path_id, manual.begin,
+ manual.end, &manual.manual_end, &manual_conflict)) == nullptr) &&
+ manual_conflict)) {
+ // exclusive manual compactions should not see a conflict during
+ // CompactRange
+ assert(!exclusive || !manual_conflict);
+ // Running either this or some other manual compaction
+ bg_cv_.Wait();
+ if (scheduled && manual.incomplete == true) {
+ assert(!manual.in_progress);
+ scheduled = false;
+ manual.incomplete = false;
+ }
+ } else if (!scheduled) {
+ if (compaction == nullptr) {
+ manual.done = true;
+ bg_cv_.SignalAll();
+ continue;
+ }
+ ca = new CompactionArg;
+ ca->db = this;
+ ca->prepicked_compaction = new PrepickedCompaction;
+ ca->prepicked_compaction->manual_compaction_state = &manual;
+ ca->prepicked_compaction->compaction = compaction;
+ manual.incomplete = false;
+ bg_compaction_scheduled_++;
+ env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
+ &DBImpl::UnscheduleCallback);
+ scheduled = true;
+ }
+ }
+
+ assert(!manual.in_progress);
+ assert(HasPendingManualCompaction());
+ RemoveManualCompaction(&manual);
+ bg_cv_.SignalAll();
+ return manual.status;
+}
+
+Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
+ const FlushOptions& flush_options,
+ bool writes_stopped) {
+ Status s;
+ {
+ WriteContext context;
+ InstrumentedMutexLock guard_lock(&mutex_);
+
+ if (cfd->imm()->NumNotFlushed() == 0 && cfd->mem()->IsEmpty()) {
+ // Nothing to flush
+ return Status::OK();
+ }
+
+ WriteThread::Writer w;
+ if (!writes_stopped) {
+ write_thread_.EnterUnbatched(&w, &mutex_);
+ }
+
+ // SwitchMemtable() will release and reacquire mutex
+ // during execution
+ s = SwitchMemtable(cfd, &context);
+
+ if (!writes_stopped) {
+ write_thread_.ExitUnbatched(&w);
+ }
+
+ cfd->imm()->FlushRequested();
+
+ // schedule flush
+ SchedulePendingFlush(cfd);
+ MaybeScheduleFlushOrCompaction();
+ }
+
+ if (s.ok() && flush_options.wait) {
+ // Wait until the compaction completes
+ s = WaitForFlushMemTable(cfd);
+ }
+ return s;
+}
+
+Status DBImpl::WaitForFlushMemTable(ColumnFamilyData* cfd) {
+ Status s;
+ // Wait until the compaction completes
+ InstrumentedMutexLock l(&mutex_);
+ while (cfd->imm()->NumNotFlushed() > 0 && bg_error_.ok()) {
+ if (shutting_down_.load(std::memory_order_acquire)) {
+ return Status::ShutdownInProgress();
+ }
+ if (cfd->IsDropped()) {
+ // FlushJob cannot flush a dropped CF, if we did not break here
+ // we will loop forever since cfd->imm()->NumNotFlushed() will never
+ // drop to zero
+ return Status::InvalidArgument("Cannot flush a dropped CF");
+ }
+ bg_cv_.Wait();
+ }
+ if (!bg_error_.ok()) {
+ s = bg_error_;
+ }
+ return s;
+}
+
+Status DBImpl::EnableAutoCompaction(
+ const std::vector<ColumnFamilyHandle*>& column_family_handles) {
+ Status s;
+ for (auto cf_ptr : column_family_handles) {
+ Status status =
+ this->SetOptions(cf_ptr, {{"disable_auto_compactions", "false"}});
+ if (!status.ok()) {
+ s = status;
+ }
+ }
+
+ return s;
+}
+
+void DBImpl::MaybeScheduleFlushOrCompaction() {
+ mutex_.AssertHeld();
+ if (!opened_successfully_) {
+ // Compaction may introduce data race to DB open
+ return;
+ }
+ if (bg_work_paused_ > 0) {
+ // we paused the background work
+ return;
+ } else if (shutting_down_.load(std::memory_order_acquire)) {
+ // DB is being deleted; no more background compactions
+ return;
+ }
+ auto bg_job_limits = GetBGJobLimits();
+ bool is_flush_pool_empty =
+ env_->GetBackgroundThreads(Env::Priority::HIGH) == 0;
+ while (!is_flush_pool_empty && unscheduled_flushes_ > 0 &&
+ bg_flush_scheduled_ < bg_job_limits.max_flushes) {
+ unscheduled_flushes_--;
+ bg_flush_scheduled_++;
+ env_->Schedule(&DBImpl::BGWorkFlush, this, Env::Priority::HIGH, this);
+ }
+
+ // special case -- if high-pri (flush) thread pool is empty, then schedule
+ // flushes in low-pri (compaction) thread pool.
+ if (is_flush_pool_empty) {
+ while (unscheduled_flushes_ > 0 &&
+ bg_flush_scheduled_ + bg_compaction_scheduled_ <
+ bg_job_limits.max_flushes) {
+ unscheduled_flushes_--;
+ bg_flush_scheduled_++;
+ env_->Schedule(&DBImpl::BGWorkFlush, this, Env::Priority::LOW, this);
+ }
+ }
+
+ if (bg_compaction_paused_ > 0) {
+ // we paused the background compaction
+ return;
+ }
+
+ if (HasExclusiveManualCompaction()) {
+ // only manual compactions are allowed to run. don't schedule automatic
+ // compactions
+ return;
+ }
+
+ while (bg_compaction_scheduled_ < bg_job_limits.max_compactions &&
+ unscheduled_compactions_ > 0) {
+ CompactionArg* ca = new CompactionArg;
+ ca->db = this;
+ ca->prepicked_compaction = nullptr;
+ bg_compaction_scheduled_++;
+ unscheduled_compactions_--;
+ env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
+ &DBImpl::UnscheduleCallback);
+ }
+}
+
+DBImpl::BGJobLimits DBImpl::GetBGJobLimits() const {
+ mutex_.AssertHeld();
+ return GetBGJobLimits(immutable_db_options_.max_background_flushes,
+ mutable_db_options_.max_background_compactions,
+ mutable_db_options_.max_background_jobs,
+ write_controller_.NeedSpeedupCompaction());
+}
+
+DBImpl::BGJobLimits DBImpl::GetBGJobLimits(int max_background_flushes,
+ int max_background_compactions,
+ int max_background_jobs,
+ bool parallelize_compactions) {
+ BGJobLimits res;
+ if (max_background_flushes == -1 && max_background_compactions == -1) {
+ // for our first stab implementing max_background_jobs, simply allocate a
+ // quarter of the threads to flushes.
+ res.max_flushes = std::max(1, max_background_jobs / 4);
+ res.max_compactions = std::max(1, max_background_jobs - res.max_flushes);
+ } else {
+ // compatibility code in case users haven't migrated to max_background_jobs,
+ // which automatically computes flush/compaction limits
+ res.max_flushes = std::max(1, max_background_flushes);
+ res.max_compactions = std::max(1, max_background_compactions);
+ }
+ if (!parallelize_compactions) {
+ // throttle background compactions until we deem necessary
+ res.max_compactions = 1;
+ }
+ return res;
+}
+
+void DBImpl::AddToCompactionQueue(ColumnFamilyData* cfd) {
+ assert(!cfd->pending_compaction());
+ cfd->Ref();
+ compaction_queue_.push_back(cfd);
+ cfd->set_pending_compaction(true);
+}
+
+ColumnFamilyData* DBImpl::PopFirstFromCompactionQueue() {
+ assert(!compaction_queue_.empty());
+ auto cfd = *compaction_queue_.begin();
+ compaction_queue_.pop_front();
+ assert(cfd->pending_compaction());
+ cfd->set_pending_compaction(false);
+ return cfd;
+}
+
+void DBImpl::AddToFlushQueue(ColumnFamilyData* cfd) {
+ assert(!cfd->pending_flush());
+ cfd->Ref();
+ flush_queue_.push_back(cfd);
+ cfd->set_pending_flush(true);
+}
+
+ColumnFamilyData* DBImpl::PopFirstFromFlushQueue() {
+ assert(!flush_queue_.empty());
+ auto cfd = *flush_queue_.begin();
+ flush_queue_.pop_front();
+ assert(cfd->pending_flush());
+ cfd->set_pending_flush(false);
+ return cfd;
+}
+
+void DBImpl::SchedulePendingFlush(ColumnFamilyData* cfd) {
+ if (!cfd->pending_flush() && cfd->imm()->IsFlushPending()) {
+ AddToFlushQueue(cfd);
+ ++unscheduled_flushes_;
+ }
+}
+
+void DBImpl::SchedulePendingCompaction(ColumnFamilyData* cfd) {
+ if (!cfd->pending_compaction() && cfd->NeedsCompaction()) {
+ AddToCompactionQueue(cfd);
+ ++unscheduled_compactions_;
+ }
+}
+
+void DBImpl::SchedulePendingPurge(std::string fname, FileType type,
+ uint64_t number, uint32_t path_id,
+ int job_id) {
+ mutex_.AssertHeld();
+ PurgeFileInfo file_info(fname, type, number, path_id, job_id);
+ purge_queue_.push_back(std::move(file_info));
+}
+
+void DBImpl::BGWorkFlush(void* db) {
+ IOSTATS_SET_THREAD_POOL_ID(Env::Priority::HIGH);
+ TEST_SYNC_POINT("DBImpl::BGWorkFlush");
+ reinterpret_cast<DBImpl*>(db)->BackgroundCallFlush();
+ TEST_SYNC_POINT("DBImpl::BGWorkFlush:done");
+}
+
+void DBImpl::BGWorkCompaction(void* arg) {
+ CompactionArg ca = *(reinterpret_cast<CompactionArg*>(arg));
+ delete reinterpret_cast<CompactionArg*>(arg);
+ IOSTATS_SET_THREAD_POOL_ID(Env::Priority::LOW);
+ TEST_SYNC_POINT("DBImpl::BGWorkCompaction");
+ auto prepicked_compaction =
+ static_cast<PrepickedCompaction*>(ca.prepicked_compaction);
+ reinterpret_cast<DBImpl*>(ca.db)->BackgroundCallCompaction(
+ prepicked_compaction, Env::Priority::LOW);
+ delete prepicked_compaction;
+}
+
+void DBImpl::BGWorkBottomCompaction(void* arg) {
+ CompactionArg ca = *(static_cast<CompactionArg*>(arg));
+ delete static_cast<CompactionArg*>(arg);
+ IOSTATS_SET_THREAD_POOL_ID(Env::Priority::BOTTOM);
+ TEST_SYNC_POINT("DBImpl::BGWorkBottomCompaction");
+ auto* prepicked_compaction = ca.prepicked_compaction;
+ assert(prepicked_compaction && prepicked_compaction->compaction &&
+ !prepicked_compaction->manual_compaction_state);
+ ca.db->BackgroundCallCompaction(prepicked_compaction, Env::Priority::BOTTOM);
+ delete prepicked_compaction;
+}
+
+void DBImpl::BGWorkPurge(void* db) {
+ IOSTATS_SET_THREAD_POOL_ID(Env::Priority::HIGH);
+ TEST_SYNC_POINT("DBImpl::BGWorkPurge:start");
+ reinterpret_cast<DBImpl*>(db)->BackgroundCallPurge();
+ TEST_SYNC_POINT("DBImpl::BGWorkPurge:end");
+}
+
+void DBImpl::UnscheduleCallback(void* arg) {
+ CompactionArg ca = *(reinterpret_cast<CompactionArg*>(arg));
+ delete reinterpret_cast<CompactionArg*>(arg);
+ if (ca.prepicked_compaction != nullptr) {
+ if (ca.prepicked_compaction->compaction != nullptr) {
+ delete ca.prepicked_compaction->compaction;
+ }
+ delete ca.prepicked_compaction;
+ }
+ TEST_SYNC_POINT("DBImpl::UnscheduleCallback");
+}
+
+Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
+ LogBuffer* log_buffer) {
+ mutex_.AssertHeld();
+
+ Status status = bg_error_;
+ if (status.ok() && shutting_down_.load(std::memory_order_acquire)) {
+ status = Status::ShutdownInProgress();
+ }
+
+ if (!status.ok()) {
+ return status;
+ }
+
+ ColumnFamilyData* cfd = nullptr;
+ while (!flush_queue_.empty()) {
+ // This cfd is already referenced
+ auto first_cfd = PopFirstFromFlushQueue();
+
+ if (first_cfd->IsDropped() || !first_cfd->imm()->IsFlushPending()) {
+ // can't flush this CF, try next one
+ if (first_cfd->Unref()) {
+ delete first_cfd;
+ }
+ continue;
+ }
+
+ // found a flush!
+ cfd = first_cfd;
+ break;
+ }
+
+ if (cfd != nullptr) {
+ const MutableCFOptions mutable_cf_options =
+ *cfd->GetLatestMutableCFOptions();
+ auto bg_job_limits = GetBGJobLimits();
+ ROCKS_LOG_BUFFER(
+ log_buffer,
+ "Calling FlushMemTableToOutputFile with column "
+ "family [%s], flush slots available %d, compaction slots available %d, "
+ "flush slots scheduled %d, compaction slots scheduled %d",
+ cfd->GetName().c_str(), bg_job_limits.max_flushes,
+ bg_job_limits.max_compactions, bg_flush_scheduled_,
+ bg_compaction_scheduled_);
+ status = FlushMemTableToOutputFile(cfd, mutable_cf_options, made_progress,
+ job_context, log_buffer);
+ if (cfd->Unref()) {
+ delete cfd;
+ }
+ }
+ return status;
+}
+
+void DBImpl::BackgroundCallFlush() {
+ bool made_progress = false;
+ JobContext job_context(next_job_id_.fetch_add(1), true);
+
+ TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:start");
+
+ LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL,
+ immutable_db_options_.info_log.get());
+ {
+ InstrumentedMutexLock l(&mutex_);
+ assert(bg_flush_scheduled_);
+ num_running_flushes_++;
+
+ auto pending_outputs_inserted_elem =
+ CaptureCurrentFileNumberInPendingOutputs();
+
+ Status s = BackgroundFlush(&made_progress, &job_context, &log_buffer);
+ if (!s.ok() && !s.IsShutdownInProgress()) {
+ // Wait a little bit before retrying background flush in
+ // case this is an environmental problem and we do not want to
+ // chew up resources for failed flushes for the duration of
+ // the problem.
+ uint64_t error_cnt =
+ default_cf_internal_stats_->BumpAndGetBackgroundErrorCount();
+ bg_cv_.SignalAll(); // In case a waiter can proceed despite the error
+ mutex_.Unlock();
+ ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+ "Waiting after background flush error: %s"
+ "Accumulated background error counts: %" PRIu64,
+ s.ToString().c_str(), error_cnt);
+ log_buffer.FlushBufferToLog();
+ LogFlush(immutable_db_options_.info_log);
+ env_->SleepForMicroseconds(1000000);
+ mutex_.Lock();
+ }
+
+ ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
+
+ // If flush failed, we want to delete all temporary files that we might have
+ // created. Thus, we force full scan in FindObsoleteFiles()
+ FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress());
+ // delete unnecessary files if any, this is done outside the mutex
+ if (job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) {
+ mutex_.Unlock();
+ // Have to flush the info logs before bg_flush_scheduled_--
+ // because if bg_flush_scheduled_ becomes 0 and the lock is
+ // released, the deconstructor of DB can kick in and destroy all the
+ // states of DB so info_log might not be available after that point.
+ // It also applies to access other states that DB owns.
+ log_buffer.FlushBufferToLog();
+ if (job_context.HaveSomethingToDelete()) {
+ PurgeObsoleteFiles(job_context);
+ }
+ job_context.Clean();
+ mutex_.Lock();
+ }
+
+ assert(num_running_flushes_ > 0);
+ num_running_flushes_--;
+ bg_flush_scheduled_--;
+ // See if there's more work to be done
+ MaybeScheduleFlushOrCompaction();
+ bg_cv_.SignalAll();
+ // IMPORTANT: there should be no code after calling SignalAll. This call may
+ // signal the DB destructor that it's OK to proceed with destruction. In
+ // that case, all DB variables will be dealloacated and referencing them
+ // will cause trouble.
+ }
+}
+
+void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
+ Env::Priority bg_thread_pri) {
+ bool made_progress = false;
+ JobContext job_context(next_job_id_.fetch_add(1), true);
+ TEST_SYNC_POINT("BackgroundCallCompaction:0");
+ MaybeDumpStats();
+ LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL,
+ immutable_db_options_.info_log.get());
+ {
+ InstrumentedMutexLock l(&mutex_);
+
+ // This call will unlock/lock the mutex to wait for current running
+ // IngestExternalFile() calls to finish.
+ WaitForIngestFile();
+
+ num_running_compactions_++;
+
+ auto pending_outputs_inserted_elem =
+ CaptureCurrentFileNumberInPendingOutputs();
+
+ assert((bg_thread_pri == Env::Priority::BOTTOM &&
+ bg_bottom_compaction_scheduled_) ||
+ (bg_thread_pri == Env::Priority::LOW && bg_compaction_scheduled_));
+ Status s = BackgroundCompaction(&made_progress, &job_context, &log_buffer,
+ prepicked_compaction);
+ TEST_SYNC_POINT("BackgroundCallCompaction:1");
+ if (!s.ok() && !s.IsShutdownInProgress()) {
+ // Wait a little bit before retrying background compaction in
+ // case this is an environmental problem and we do not want to
+ // chew up resources for failed compactions for the duration of
+ // the problem.
+ uint64_t error_cnt =
+ default_cf_internal_stats_->BumpAndGetBackgroundErrorCount();
+ bg_cv_.SignalAll(); // In case a waiter can proceed despite the error
+ mutex_.Unlock();
+ log_buffer.FlushBufferToLog();
+ ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+ "Waiting after background compaction error: %s, "
+ "Accumulated background error counts: %" PRIu64,
+ s.ToString().c_str(), error_cnt);
+ LogFlush(immutable_db_options_.info_log);
+ env_->SleepForMicroseconds(1000000);
+ mutex_.Lock();
+ }
+
+ ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
+
+ // If compaction failed, we want to delete all temporary files that we might
+ // have created (they might not be all recorded in job_context in case of a
+ // failure). Thus, we force full scan in FindObsoleteFiles()
+ FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress());
+
+ // delete unnecessary files if any, this is done outside the mutex
+ if (job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) {
+ mutex_.Unlock();
+ // Have to flush the info logs before bg_compaction_scheduled_--
+ // because if bg_flush_scheduled_ becomes 0 and the lock is
+ // released, the deconstructor of DB can kick in and destroy all the
+ // states of DB so info_log might not be available after that point.
+ // It also applies to access other states that DB owns.
+ log_buffer.FlushBufferToLog();
+ if (job_context.HaveSomethingToDelete()) {
+ PurgeObsoleteFiles(job_context);
+ }
+ job_context.Clean();
+ mutex_.Lock();
+ }
+
+ assert(num_running_compactions_ > 0);
+ num_running_compactions_--;
+ if (bg_thread_pri == Env::Priority::LOW) {
+ bg_compaction_scheduled_--;
+ } else {
+ assert(bg_thread_pri == Env::Priority::BOTTOM);
+ bg_bottom_compaction_scheduled_--;
+ }
+
+ versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();
+
+ // See if there's more work to be done
+ MaybeScheduleFlushOrCompaction();
+ if (made_progress ||
+ (bg_compaction_scheduled_ == 0 &&
+ bg_bottom_compaction_scheduled_ == 0) ||
+ HasPendingManualCompaction()) {
+ // signal if
+ // * made_progress -- need to wakeup DelayWrite
+ // * bg_{bottom,}_compaction_scheduled_ == 0 -- need to wakeup ~DBImpl
+ // * HasPendingManualCompaction -- need to wakeup RunManualCompaction
+ // If none of this is true, there is no need to signal since nobody is
+ // waiting for it
+ bg_cv_.SignalAll();
+ }
+ // IMPORTANT: there should be no code after calling SignalAll. This call may
+ // signal the DB destructor that it's OK to proceed with destruction. In
+ // that case, all DB variables will be dealloacated and referencing them
+ // will cause trouble.
+ }
+}
+
+Status DBImpl::BackgroundCompaction(bool* made_progress,
+ JobContext* job_context,
+ LogBuffer* log_buffer,
+ PrepickedCompaction* prepicked_compaction) {
+ ManualCompactionState* manual_compaction =
+ prepicked_compaction == nullptr
+ ? nullptr
+ : prepicked_compaction->manual_compaction_state;
+ *made_progress = false;
+ mutex_.AssertHeld();
+ TEST_SYNC_POINT("DBImpl::BackgroundCompaction:Start");
+
+ bool is_manual = (manual_compaction != nullptr);
+ unique_ptr<Compaction> c;
+ if (prepicked_compaction != nullptr &&
+ prepicked_compaction->compaction != nullptr) {
+ c.reset(prepicked_compaction->compaction);
+ }
+ bool is_prepicked = is_manual || c;
+
+ // (manual_compaction->in_progress == false);
+ bool trivial_move_disallowed =
+ is_manual && manual_compaction->disallow_trivial_move;
+
+ CompactionJobStats compaction_job_stats;
+ Status status = bg_error_;
+ if (status.ok() && shutting_down_.load(std::memory_order_acquire)) {
+ status = Status::ShutdownInProgress();
+ }
+
+ if (!status.ok()) {
+ if (is_manual) {
+ manual_compaction->status = status;
+ manual_compaction->done = true;
+ manual_compaction->in_progress = false;
+ manual_compaction = nullptr;
+ }
+ return status;
+ }
+
+ if (is_manual) {
+ // another thread cannot pick up the same work
+ manual_compaction->in_progress = true;
+ }
+
+ // InternalKey manual_end_storage;
+ // InternalKey* manual_end = &manual_end_storage;
+ if (is_manual) {
+ ManualCompactionState* m = manual_compaction;
+ assert(m->in_progress);
+ if (!c) {
+ m->done = true;
+ m->manual_end = nullptr;
+ ROCKS_LOG_BUFFER(log_buffer,
+ "[%s] Manual compaction from level-%d from %s .. "
+ "%s; nothing to do\n",
+ m->cfd->GetName().c_str(), m->input_level,
+ (m->begin ? m->begin->DebugString().c_str() : "(begin)"),
+ (m->end ? m->end->DebugString().c_str() : "(end)"));
+ } else {
+ ROCKS_LOG_BUFFER(
+ log_buffer,
+ "[%s] Manual compaction from level-%d to level-%d from %s .. "
+ "%s; will stop at %s\n",
+ m->cfd->GetName().c_str(), m->input_level, c->output_level(),
+ (m->begin ? m->begin->DebugString().c_str() : "(begin)"),
+ (m->end ? m->end->DebugString().c_str() : "(end)"),
+ ((m->done || m->manual_end == nullptr)
+ ? "(end)"
+ : m->manual_end->DebugString().c_str()));
+ }
+ } else if (!is_prepicked && !compaction_queue_.empty()) {
+ if (HaveManualCompaction(compaction_queue_.front())) {
+ // Can't compact right now, but try again later
+ TEST_SYNC_POINT("DBImpl::BackgroundCompaction()::Conflict");
+
+ // Stay in the compaction queue.
+ unscheduled_compactions_++;
+
+ return Status::OK();
+ }
+
+ // cfd is referenced here
+ auto cfd = PopFirstFromCompactionQueue();
+ // We unreference here because the following code will take a Ref() on
+ // this cfd if it is going to use it (Compaction class holds a
+ // reference).
+ // This will all happen under a mutex so we don't have to be afraid of
+ // somebody else deleting it.
+ if (cfd->Unref()) {
+ delete cfd;
+ // This was the last reference of the column family, so no need to
+ // compact.
+ return Status::OK();
+ }
+
+ // Pick up latest mutable CF Options and use it throughout the
+ // compaction job
+ // Compaction makes a copy of the latest MutableCFOptions. It should be used
+ // throughout the compaction procedure to make sure consistency. It will
+ // eventually be installed into SuperVersion
+ auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
+ if (!mutable_cf_options->disable_auto_compactions && !cfd->IsDropped()) {
+ // NOTE: try to avoid unnecessary copy of MutableCFOptions if
+ // compaction is not necessary. Need to make sure mutex is held
+ // until we make a copy in the following code
+ TEST_SYNC_POINT("DBImpl::BackgroundCompaction():BeforePickCompaction");
+ c.reset(cfd->PickCompaction(*mutable_cf_options, log_buffer));
+ TEST_SYNC_POINT("DBImpl::BackgroundCompaction():AfterPickCompaction");
+ if (c != nullptr) {
+ // update statistics
+ MeasureTime(stats_, NUM_FILES_IN_SINGLE_COMPACTION,
+ c->inputs(0)->size());
+ // There are three things that can change compaction score:
+ // 1) When flush or compaction finish. This case is covered by
+ // InstallSuperVersionAndScheduleWork
+ // 2) When MutableCFOptions changes. This case is also covered by
+ // InstallSuperVersionAndScheduleWork, because this is when the new
+ // options take effect.
+ // 3) When we Pick a new compaction, we "remove" those files being
+ // compacted from the calculation, which then influences compaction
+ // score. Here we check if we need the new compaction even without the
+ // files that are currently being compacted. If we need another
+ // compaction, we might be able to execute it in parallel, so we add it
+ // to the queue and schedule a new thread.
+ if (cfd->NeedsCompaction()) {
+ // Yes, we need more compactions!
+ AddToCompactionQueue(cfd);
+ ++unscheduled_compactions_;
+ MaybeScheduleFlushOrCompaction();
+ }
+ }
+ }
+ }
+
+ if (!c) {
+ // Nothing to do
+ ROCKS_LOG_BUFFER(log_buffer, "Compaction nothing to do");
+ } else if (c->deletion_compaction()) {
+ // TODO(icanadi) Do we want to honor snapshots here? i.e. not delete old
+ // file if there is alive snapshot pointing to it
+ assert(c->num_input_files(1) == 0);
+ assert(c->level() == 0);
+ assert(c->column_family_data()->ioptions()->compaction_style ==
+ kCompactionStyleFIFO);
+
+ compaction_job_stats.num_input_files = c->num_input_files(0);
+
+ for (const auto& f : *c->inputs(0)) {
+ c->edit()->DeleteFile(c->level(), f->fd.GetNumber());
+ }
+ status = versions_->LogAndApply(c->column_family_data(),
+ *c->mutable_cf_options(), c->edit(),
+ &mutex_, directories_.GetDbDir());
+ InstallSuperVersionAndScheduleWorkWrapper(
+ c->column_family_data(), job_context, *c->mutable_cf_options());
+ ROCKS_LOG_BUFFER(log_buffer, "[%s] Deleted %d files\n",
+ c->column_family_data()->GetName().c_str(),
+ c->num_input_files(0));
+ *made_progress = true;
+ } else if (!trivial_move_disallowed && c->IsTrivialMove()) {
+ TEST_SYNC_POINT("DBImpl::BackgroundCompaction:TrivialMove");
+ // Instrument for event update
+ // TODO(yhchiang): add op details for showing trivial-move.
+ ThreadStatusUtil::SetColumnFamily(
+ c->column_family_data(), c->column_family_data()->ioptions()->env,
+ immutable_db_options_.enable_thread_tracking);
+ ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION);
+
+ compaction_job_stats.num_input_files = c->num_input_files(0);
+
+ // Move files to next level
+ int32_t moved_files = 0;
+ int64_t moved_bytes = 0;
+ for (unsigned int l = 0; l < c->num_input_levels(); l++) {
+ if (c->level(l) == c->output_level()) {
+ continue;
+ }
+ for (size_t i = 0; i < c->num_input_files(l); i++) {
+ FileMetaData* f = c->input(l, i);
+ c->edit()->DeleteFile(c->level(l), f->fd.GetNumber());
+ c->edit()->AddFile(c->output_level(), f->fd.GetNumber(),
+ f->fd.GetPathId(), f->fd.GetFileSize(), f->smallest,
+ f->largest, f->smallest_seqno, f->largest_seqno,
+ f->marked_for_compaction);
+
+ ROCKS_LOG_BUFFER(log_buffer, "[%s] Moving #%" PRIu64
+ " to level-%d %" PRIu64 " bytes\n",
+ c->column_family_data()->GetName().c_str(),
+ f->fd.GetNumber(), c->output_level(),
+ f->fd.GetFileSize());
+ ++moved_files;
+ moved_bytes += f->fd.GetFileSize();
+ }
+ }
+
+ status = versions_->LogAndApply(c->column_family_data(),
+ *c->mutable_cf_options(), c->edit(),
+ &mutex_, directories_.GetDbDir());
+ // Use latest MutableCFOptions
+ InstallSuperVersionAndScheduleWorkWrapper(
+ c->column_family_data(), job_context, *c->mutable_cf_options());
+
+ VersionStorageInfo::LevelSummaryStorage tmp;
+ c->column_family_data()->internal_stats()->IncBytesMoved(c->output_level(),
+ moved_bytes);
+ {
+ event_logger_.LogToBuffer(log_buffer)
+ << "job" << job_context->job_id << "event"
+ << "trivial_move"
+ << "destination_level" << c->output_level() << "files" << moved_files
+ << "total_files_size" << moved_bytes;
+ }
+ ROCKS_LOG_BUFFER(
+ log_buffer,
+ "[%s] Moved #%d files to level-%d %" PRIu64 " bytes %s: %s\n",
+ c->column_family_data()->GetName().c_str(), moved_files,
+ c->output_level(), moved_bytes, status.ToString().c_str(),
+ c->column_family_data()->current()->storage_info()->LevelSummary(&tmp));
+ *made_progress = true;
+
+ // Clear Instrument
+ ThreadStatusUtil::ResetThreadStatus();
+ } else if (c->column_family_data()->ioptions()->compaction_style ==
+ kCompactionStyleUniversal &&
+ !is_prepicked && c->output_level() > 0 &&
+ c->output_level() ==
+ c->column_family_data()
+ ->current()
+ ->storage_info()
+ ->MaxOutputLevel(
+ immutable_db_options_.allow_ingest_behind) &&
+ env_->GetBackgroundThreads(Env::Priority::BOTTOM) > 0) {
+ // Forward universal compactions involving last level to the bottom pool
+ // if it exists, such that long-running compactions can't block short-
+ // lived ones, like L0->L0s.
+ TEST_SYNC_POINT("DBImpl::BackgroundCompaction:ForwardToBottomPriPool");
+ CompactionArg* ca = new CompactionArg;
+ ca->db = this;
+ ca->prepicked_compaction = new PrepickedCompaction;
+ ca->prepicked_compaction->compaction = c.release();
+ ca->prepicked_compaction->manual_compaction_state = nullptr;
+ ++bg_bottom_compaction_scheduled_;
+ env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca, Env::Priority::BOTTOM,
+ this, &DBImpl::UnscheduleCallback);
+ } else {
+ int output_level __attribute__((unused)) = c->output_level();
+ TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:NonTrivial",
+ &output_level);
+
+ SequenceNumber earliest_write_conflict_snapshot;
+ std::vector<SequenceNumber> snapshot_seqs =
+ snapshots_.GetAll(&earliest_write_conflict_snapshot);
+
+ assert(is_snapshot_supported_ || snapshots_.empty());
+ CompactionJob compaction_job(
+ job_context->job_id, c.get(), immutable_db_options_, env_options_,
+ versions_.get(), &shutting_down_, log_buffer, directories_.GetDbDir(),
+ directories_.GetDataDir(c->output_path_id()), stats_, &mutex_,
+ &bg_error_, snapshot_seqs, earliest_write_conflict_snapshot,
+ table_cache_, &event_logger_,
+ c->mutable_cf_options()->paranoid_file_checks,
+ c->mutable_cf_options()->report_bg_io_stats, dbname_,
+ &compaction_job_stats);
+ compaction_job.Prepare();
+
+ mutex_.Unlock();
+ compaction_job.Run();
+ TEST_SYNC_POINT("DBImpl::BackgroundCompaction:NonTrivial:AfterRun");
+ mutex_.Lock();
+
+ status = compaction_job.Install(*c->mutable_cf_options());
+ if (status.ok()) {
+ InstallSuperVersionAndScheduleWorkWrapper(
+ c->column_family_data(), job_context, *c->mutable_cf_options());
+ }
+ *made_progress = true;
+ }
+ if (c != nullptr) {
+ c->ReleaseCompactionFiles(status);
+ *made_progress = true;
+ NotifyOnCompactionCompleted(
+ c->column_family_data(), c.get(), status,
+ compaction_job_stats, job_context->job_id);
+ }
+ // this will unref its input_version and column_family_data
+ c.reset();
+
+ if (status.ok()) {
+ // Done
+ } else if (status.IsShutdownInProgress()) {
+ // Ignore compaction errors found during shutting down
+ } else {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log, "Compaction error: %s",
+ status.ToString().c_str());
+ if (immutable_db_options_.paranoid_checks && bg_error_.ok()) {
+ Status new_bg_error = status;
+ // may temporarily unlock and lock the mutex.
+ EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
+ BackgroundErrorReason::kCompaction,
+ &new_bg_error, &mutex_);
+ if (!new_bg_error.ok()) {
+ bg_error_ = new_bg_error;
+ }
+ }
+ }
+
+ if (is_manual) {
+ ManualCompactionState* m = manual_compaction;
+ if (!status.ok()) {
+ m->status = status;
+ m->done = true;
+ }
+ // For universal compaction:
+ // Because universal compaction always happens at level 0, so one
+ // compaction will pick up all overlapped files. No files will be
+ // filtered out due to size limit and left for a successive compaction.
+ // So we can safely conclude the current compaction.
+ //
+ // Also note that, if we don't stop here, then the current compaction
+ // writes a new file back to level 0, which will be used in successive
+ // compaction. Hence the manual compaction will never finish.
+ //
+ // Stop the compaction if manual_end points to nullptr -- this means
+ // that we compacted the whole range. manual_end should always point
+ // to nullptr in case of universal compaction
+ if (m->manual_end == nullptr) {
+ m->done = true;
+ }
+ if (!m->done) {
+ // We only compacted part of the requested range. Update *m
+ // to the range that is left to be compacted.
+ // Universal and FIFO compactions should always compact the whole range
+ assert(m->cfd->ioptions()->compaction_style !=
+ kCompactionStyleUniversal ||
+ m->cfd->ioptions()->num_levels > 1);
+ assert(m->cfd->ioptions()->compaction_style != kCompactionStyleFIFO);
+ m->tmp_storage = *m->manual_end;
+ m->begin = &m->tmp_storage;
+ m->incomplete = true;
+ }
+ m->in_progress = false; // not being processed anymore
+ }
+ TEST_SYNC_POINT("DBImpl::BackgroundCompaction:Finish");
+ return status;
+}
+
+bool DBImpl::HasPendingManualCompaction() {
+ return (!manual_compaction_dequeue_.empty());
+}
+
+void DBImpl::AddManualCompaction(DBImpl::ManualCompactionState* m) {
+ manual_compaction_dequeue_.push_back(m);
+}
+
+void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) {
+ // Remove from queue
+ std::deque<ManualCompactionState*>::iterator it =
+ manual_compaction_dequeue_.begin();
+ while (it != manual_compaction_dequeue_.end()) {
+ if (m == (*it)) {
+ it = manual_compaction_dequeue_.erase(it);
+ return;
+ }
+ it++;
+ }
+ assert(false);
+ return;
+}
+
+bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) {
+ if (num_running_ingest_file_ > 0) {
+ // We need to wait for other IngestExternalFile() calls to finish
+ // before running a manual compaction.
+ return true;
+ }
+ if (m->exclusive) {
+ return (bg_bottom_compaction_scheduled_ > 0 ||
+ bg_compaction_scheduled_ > 0);
+ }
+ std::deque<ManualCompactionState*>::iterator it =
+ manual_compaction_dequeue_.begin();
+ bool seen = false;
+ while (it != manual_compaction_dequeue_.end()) {
+ if (m == (*it)) {
+ it++;
+ seen = true;
+ continue;
+ } else if (MCOverlap(m, (*it)) && (!seen && !(*it)->in_progress)) {
+ // Consider the other manual compaction *it, conflicts if:
+ // overlaps with m
+ // and (*it) is ahead in the queue and is not yet in progress
+ return true;
+ }
+ it++;
+ }
+ return false;
+}
+
+bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) {
+ // Remove from priority queue
+ std::deque<ManualCompactionState*>::iterator it =
+ manual_compaction_dequeue_.begin();
+ while (it != manual_compaction_dequeue_.end()) {
+ if ((*it)->exclusive) {
+ return true;
+ }
+ if ((cfd == (*it)->cfd) && (!((*it)->in_progress || (*it)->done))) {
+ // Allow automatic compaction if manual compaction is
+ // in progress
+ return true;
+ }
+ it++;
+ }
+ return false;
+}
+
+bool DBImpl::HasExclusiveManualCompaction() {
+ // Remove from priority queue
+ std::deque<ManualCompactionState*>::iterator it =
+ manual_compaction_dequeue_.begin();
+ while (it != manual_compaction_dequeue_.end()) {
+ if ((*it)->exclusive) {
+ return true;
+ }
+ it++;
+ }
+ return false;
+}
+
+bool DBImpl::MCOverlap(ManualCompactionState* m, ManualCompactionState* m1) {
+ if ((m->exclusive) || (m1->exclusive)) {
+ return true;
+ }
+ if (m->cfd != m1->cfd) {
+ return false;
+ }
+ return true;
+}
+
+// JobContext gets created and destructed outside of the lock --
+// we
+// use this convinently to:
+// * malloc one SuperVersion() outside of the lock -- new_superversion
+// * delete SuperVersion()s outside of the lock -- superversions_to_free
+//
+// However, if InstallSuperVersionAndScheduleWork() gets called twice with the
+// same job_context, we can't reuse the SuperVersion() that got
+// malloced because
+// first call already used it. In that rare case, we take a hit and create a
+// new SuperVersion() inside of the mutex. We do similar thing
+// for superversion_to_free
+void DBImpl::InstallSuperVersionAndScheduleWorkWrapper(
+ ColumnFamilyData* cfd, JobContext* job_context,
+ const MutableCFOptions& mutable_cf_options) {
+ mutex_.AssertHeld();
+ SuperVersion* old_superversion = InstallSuperVersionAndScheduleWork(
+ cfd, job_context->new_superversion, mutable_cf_options);
+ job_context->new_superversion = nullptr;
+ job_context->superversions_to_free.push_back(old_superversion);
+}
+
+SuperVersion* DBImpl::InstallSuperVersionAndScheduleWork(
+ ColumnFamilyData* cfd, SuperVersion* new_sv,
+ const MutableCFOptions& mutable_cf_options) {
+ mutex_.AssertHeld();
+
+ // Update max_total_in_memory_state_
+ size_t old_memtable_size = 0;
+ auto* old_sv = cfd->GetSuperVersion();
+ if (old_sv) {
+ old_memtable_size = old_sv->mutable_cf_options.write_buffer_size *
+ old_sv->mutable_cf_options.max_write_buffer_number;
+ }
+
+ auto* old = cfd->InstallSuperVersion(
+ new_sv ? new_sv : new SuperVersion(), &mutex_, mutable_cf_options);
+
+ // Whenever we install new SuperVersion, we might need to issue new flushes or
+ // compactions.
+ SchedulePendingFlush(cfd);
+ SchedulePendingCompaction(cfd);
+ MaybeScheduleFlushOrCompaction();
+
+ // Update max_total_in_memory_state_
+ max_total_in_memory_state_ =
+ max_total_in_memory_state_ - old_memtable_size +
+ mutable_cf_options.write_buffer_size *
+ mutable_cf_options.max_write_buffer_number;
+ return old;
+}
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_impl_debug.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_impl_debug.cc b/thirdparty/rocksdb/db/db_impl_debug.cc
new file mode 100644
index 0000000..de5b66f
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_impl_debug.cc
@@ -0,0 +1,200 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef NDEBUG
+
+#include "db/db_impl.h"
+#include "monitoring/thread_status_updater.h"
+
+namespace rocksdb {
+
+uint64_t DBImpl::TEST_GetLevel0TotalSize() {
+ InstrumentedMutexLock l(&mutex_);
+ return default_cf_handle_->cfd()->current()->storage_info()->NumLevelBytes(0);
+}
+
+void DBImpl::TEST_HandleWALFull() {
+ WriteContext write_context;
+ InstrumentedMutexLock l(&mutex_);
+ HandleWALFull(&write_context);
+}
+
+int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes(
+ ColumnFamilyHandle* column_family) {
+ ColumnFamilyData* cfd;
+ if (column_family == nullptr) {
+ cfd = default_cf_handle_->cfd();
+ } else {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ cfd = cfh->cfd();
+ }
+ InstrumentedMutexLock l(&mutex_);
+ return cfd->current()->storage_info()->MaxNextLevelOverlappingBytes();
+}
+
+void DBImpl::TEST_GetFilesMetaData(
+ ColumnFamilyHandle* column_family,
+ std::vector<std::vector<FileMetaData>>* metadata) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ auto cfd = cfh->cfd();
+ InstrumentedMutexLock l(&mutex_);
+ metadata->resize(NumberLevels());
+ for (int level = 0; level < NumberLevels(); level++) {
+ const std::vector<FileMetaData*>& files =
+ cfd->current()->storage_info()->LevelFiles(level);
+
+ (*metadata)[level].clear();
+ for (const auto& f : files) {
+ (*metadata)[level].push_back(*f);
+ }
+ }
+}
+
+uint64_t DBImpl::TEST_Current_Manifest_FileNo() {
+ return versions_->manifest_file_number();
+}
+
+Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
+ const Slice* end,
+ ColumnFamilyHandle* column_family,
+ bool disallow_trivial_move) {
+ ColumnFamilyData* cfd;
+ if (column_family == nullptr) {
+ cfd = default_cf_handle_->cfd();
+ } else {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ cfd = cfh->cfd();
+ }
+ int output_level =
+ (cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
+ cfd->ioptions()->compaction_style == kCompactionStyleFIFO)
+ ? level
+ : level + 1;
+ return RunManualCompaction(cfd, level, output_level, 0, begin, end, true,
+ disallow_trivial_move);
+}
+
+Status DBImpl::TEST_FlushMemTable(bool wait, ColumnFamilyHandle* cfh) {
+ FlushOptions fo;
+ fo.wait = wait;
+ ColumnFamilyData* cfd;
+ if (cfh == nullptr) {
+ cfd = default_cf_handle_->cfd();
+ } else {
+ auto cfhi = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh);
+ cfd = cfhi->cfd();
+ }
+ return FlushMemTable(cfd, fo);
+}
+
+Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
+ ColumnFamilyData* cfd;
+ if (column_family == nullptr) {
+ cfd = default_cf_handle_->cfd();
+ } else {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ cfd = cfh->cfd();
+ }
+ return WaitForFlushMemTable(cfd);
+}
+
+Status DBImpl::TEST_WaitForCompact() {
+ // Wait until the compaction completes
+
+ // TODO: a bug here. This function actually does not necessarily
+ // wait for compact. It actually waits for scheduled compaction
+ // OR flush to finish.
+
+ InstrumentedMutexLock l(&mutex_);
+ while ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
+ bg_flush_scheduled_) &&
+ bg_error_.ok()) {
+ bg_cv_.Wait();
+ }
+ return bg_error_;
+}
+
+void DBImpl::TEST_LockMutex() {
+ mutex_.Lock();
+}
+
+void DBImpl::TEST_UnlockMutex() {
+ mutex_.Unlock();
+}
+
+void* DBImpl::TEST_BeginWrite() {
+ auto w = new WriteThread::Writer();
+ write_thread_.EnterUnbatched(w, &mutex_);
+ return reinterpret_cast<void*>(w);
+}
+
+void DBImpl::TEST_EndWrite(void* w) {
+ auto writer = reinterpret_cast<WriteThread::Writer*>(w);
+ write_thread_.ExitUnbatched(writer);
+ delete writer;
+}
+
+size_t DBImpl::TEST_LogsToFreeSize() {
+ InstrumentedMutexLock l(&mutex_);
+ return logs_to_free_.size();
+}
+
+uint64_t DBImpl::TEST_LogfileNumber() {
+ InstrumentedMutexLock l(&mutex_);
+ return logfile_number_;
+}
+
+Status DBImpl::TEST_GetAllImmutableCFOptions(
+ std::unordered_map<std::string, const ImmutableCFOptions*>* iopts_map) {
+ std::vector<std::string> cf_names;
+ std::vector<const ImmutableCFOptions*> iopts;
+ {
+ InstrumentedMutexLock l(&mutex_);
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ cf_names.push_back(cfd->GetName());
+ iopts.push_back(cfd->ioptions());
+ }
+ }
+ iopts_map->clear();
+ for (size_t i = 0; i < cf_names.size(); ++i) {
+ iopts_map->insert({cf_names[i], iopts[i]});
+ }
+
+ return Status::OK();
+}
+
+uint64_t DBImpl::TEST_FindMinLogContainingOutstandingPrep() {
+ return FindMinLogContainingOutstandingPrep();
+}
+
+uint64_t DBImpl::TEST_FindMinPrepLogReferencedByMemTable() {
+ return FindMinPrepLogReferencedByMemTable();
+}
+
+Status DBImpl::TEST_GetLatestMutableCFOptions(
+ ColumnFamilyHandle* column_family, MutableCFOptions* mutable_cf_options) {
+ InstrumentedMutexLock l(&mutex_);
+
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ *mutable_cf_options = *cfh->cfd()->GetLatestMutableCFOptions();
+ return Status::OK();
+}
+
+int DBImpl::TEST_BGCompactionsAllowed() const {
+ InstrumentedMutexLock l(&mutex_);
+ return GetBGJobLimits().max_compactions;
+}
+
+int DBImpl::TEST_BGFlushesAllowed() const {
+ InstrumentedMutexLock l(&mutex_);
+ return GetBGJobLimits().max_flushes;
+}
+
+} // namespace rocksdb
+#endif // NDEBUG
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_impl_experimental.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_impl_experimental.cc b/thirdparty/rocksdb/db/db_impl_experimental.cc
new file mode 100644
index 0000000..0d01075
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_impl_experimental.cc
@@ -0,0 +1,152 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/db_impl.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <vector>
+
+#include "db/column_family.h"
+#include "db/job_context.h"
+#include "db/version_set.h"
+#include "rocksdb/status.h"
+
+namespace rocksdb {
+
+#ifndef ROCKSDB_LITE
+Status DBImpl::SuggestCompactRange(ColumnFamilyHandle* column_family,
+ const Slice* begin, const Slice* end) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ auto cfd = cfh->cfd();
+ InternalKey start_key, end_key;
+ if (begin != nullptr) {
+ start_key.SetMaxPossibleForUserKey(*begin);
+ }
+ if (end != nullptr) {
+ end_key.SetMinPossibleForUserKey(*end);
+ }
+ {
+ InstrumentedMutexLock l(&mutex_);
+ auto vstorage = cfd->current()->storage_info();
+ for (int level = 0; level < vstorage->num_non_empty_levels() - 1; ++level) {
+ std::vector<FileMetaData*> inputs;
+ vstorage->GetOverlappingInputs(
+ level, begin == nullptr ? nullptr : &start_key,
+ end == nullptr ? nullptr : &end_key, &inputs);
+ for (auto f : inputs) {
+ f->marked_for_compaction = true;
+ }
+ }
+ // Since we have some more files to compact, we should also recompute
+ // compaction score
+ vstorage->ComputeCompactionScore(*cfd->ioptions(),
+ *cfd->GetLatestMutableCFOptions());
+ SchedulePendingCompaction(cfd);
+ MaybeScheduleFlushOrCompaction();
+ }
+ return Status::OK();
+}
+
+Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
+ assert(column_family);
+
+ if (target_level < 1) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "PromoteL0 FAILED. Invalid target level %d\n", target_level);
+ return Status::InvalidArgument("Invalid target level");
+ }
+
+ Status status;
+ VersionEdit edit;
+ JobContext job_context(next_job_id_.fetch_add(1), true);
+ {
+ InstrumentedMutexLock l(&mutex_);
+ auto* cfd = static_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
+ const auto* vstorage = cfd->current()->storage_info();
+
+ if (target_level >= vstorage->num_levels()) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "PromoteL0 FAILED. Target level %d does not exist\n",
+ target_level);
+ job_context.Clean();
+ return Status::InvalidArgument("Target level does not exist");
+ }
+
+ // Sort L0 files by range.
+ const InternalKeyComparator* icmp = &cfd->internal_comparator();
+ auto l0_files = vstorage->LevelFiles(0);
+ std::sort(l0_files.begin(), l0_files.end(),
+ [icmp](FileMetaData* f1, FileMetaData* f2) {
+ return icmp->Compare(f1->largest, f2->largest) < 0;
+ });
+
+ // Check that no L0 file is being compacted and that they have
+ // non-overlapping ranges.
+ for (size_t i = 0; i < l0_files.size(); ++i) {
+ auto f = l0_files[i];
+ if (f->being_compacted) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "PromoteL0 FAILED. File %" PRIu64 " being compacted\n",
+ f->fd.GetNumber());
+ job_context.Clean();
+ return Status::InvalidArgument("PromoteL0 called during L0 compaction");
+ }
+
+ if (i == 0) continue;
+ auto prev_f = l0_files[i - 1];
+ if (icmp->Compare(prev_f->largest, f->smallest) >= 0) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "PromoteL0 FAILED. Files %" PRIu64 " and %" PRIu64
+ " have overlapping ranges\n",
+ prev_f->fd.GetNumber(), f->fd.GetNumber());
+ job_context.Clean();
+ return Status::InvalidArgument("L0 has overlapping files");
+ }
+ }
+
+ // Check that all levels up to target_level are empty.
+ for (int level = 1; level <= target_level; ++level) {
+ if (vstorage->NumLevelFiles(level) > 0) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "PromoteL0 FAILED. Level %d not empty\n", level);
+ job_context.Clean();
+ return Status::InvalidArgument(
+ "All levels up to target_level "
+ "must be empty");
+ }
+ }
+
+ edit.SetColumnFamily(cfd->GetID());
+ for (const auto& f : l0_files) {
+ edit.DeleteFile(0, f->fd.GetNumber());
+ edit.AddFile(target_level, f->fd.GetNumber(), f->fd.GetPathId(),
+ f->fd.GetFileSize(), f->smallest, f->largest,
+ f->smallest_seqno, f->largest_seqno,
+ f->marked_for_compaction);
+ }
+
+ status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
+ &edit, &mutex_, directories_.GetDbDir());
+ if (status.ok()) {
+ InstallSuperVersionAndScheduleWorkWrapper(
+ cfd, &job_context, *cfd->GetLatestMutableCFOptions());
+ }
+ } // lock released here
+ LogFlush(immutable_db_options_.info_log);
+ job_context.Clean();
+
+ return status;
+}
+#endif // ROCKSDB_LITE
+
+} // namespace rocksdb
[20/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_impl_files.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_impl_files.cc b/thirdparty/rocksdb/db/db_impl_files.cc
new file mode 100644
index 0000000..e44e423
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_impl_files.cc
@@ -0,0 +1,548 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include "db/db_impl.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+#include <inttypes.h>
+#include "db/event_helpers.h"
+#include "util/file_util.h"
+#include "util/sst_file_manager_impl.h"
+
+
+namespace rocksdb {
+uint64_t DBImpl::FindMinPrepLogReferencedByMemTable() {
+ if (!allow_2pc()) {
+ return 0;
+ }
+
+ uint64_t min_log = 0;
+
+ // we must look through the memtables for two phase transactions
+ // that have been committed but not yet flushed
+ for (auto loop_cfd : *versions_->GetColumnFamilySet()) {
+ if (loop_cfd->IsDropped()) {
+ continue;
+ }
+
+ auto log = loop_cfd->imm()->GetMinLogContainingPrepSection();
+
+ if (log > 0 && (min_log == 0 || log < min_log)) {
+ min_log = log;
+ }
+
+ log = loop_cfd->mem()->GetMinLogContainingPrepSection();
+
+ if (log > 0 && (min_log == 0 || log < min_log)) {
+ min_log = log;
+ }
+ }
+
+ return min_log;
+}
+
+void DBImpl::MarkLogAsHavingPrepSectionFlushed(uint64_t log) {
+ assert(log != 0);
+ std::lock_guard<std::mutex> lock(prep_heap_mutex_);
+ auto it = prepared_section_completed_.find(log);
+ assert(it != prepared_section_completed_.end());
+ it->second += 1;
+}
+
+void DBImpl::MarkLogAsContainingPrepSection(uint64_t log) {
+ assert(log != 0);
+ std::lock_guard<std::mutex> lock(prep_heap_mutex_);
+ min_log_with_prep_.push(log);
+ auto it = prepared_section_completed_.find(log);
+ if (it == prepared_section_completed_.end()) {
+ prepared_section_completed_[log] = 0;
+ }
+}
+
+uint64_t DBImpl::FindMinLogContainingOutstandingPrep() {
+
+ if (!allow_2pc()) {
+ return 0;
+ }
+
+ std::lock_guard<std::mutex> lock(prep_heap_mutex_);
+ uint64_t min_log = 0;
+
+ // first we look in the prepared heap where we keep
+ // track of transactions that have been prepared (written to WAL)
+ // but not yet committed.
+ while (!min_log_with_prep_.empty()) {
+ min_log = min_log_with_prep_.top();
+
+ auto it = prepared_section_completed_.find(min_log);
+
+ // value was marked as 'deleted' from heap
+ if (it != prepared_section_completed_.end() && it->second > 0) {
+ it->second -= 1;
+ min_log_with_prep_.pop();
+
+ // back to squere one...
+ min_log = 0;
+ continue;
+ } else {
+ // found a valid value
+ break;
+ }
+ }
+
+ return min_log;
+}
+
+uint64_t DBImpl::MinLogNumberToKeep() {
+ uint64_t log_number = versions_->MinLogNumber();
+
+ if (allow_2pc()) {
+ // if are 2pc we must consider logs containing prepared
+ // sections of outstanding transactions.
+ //
+ // We must check min logs with outstanding prep before we check
+ // logs referneces by memtables because a log referenced by the
+ // first data structure could transition to the second under us.
+ //
+ // TODO(horuff): iterating over all column families under db mutex.
+ // should find more optimial solution
+ auto min_log_in_prep_heap = FindMinLogContainingOutstandingPrep();
+
+ if (min_log_in_prep_heap != 0 && min_log_in_prep_heap < log_number) {
+ log_number = min_log_in_prep_heap;
+ }
+
+ auto min_log_refed_by_mem = FindMinPrepLogReferencedByMemTable();
+
+ if (min_log_refed_by_mem != 0 && min_log_refed_by_mem < log_number) {
+ log_number = min_log_refed_by_mem;
+ }
+ }
+ return log_number;
+}
+
+// * Returns the list of live files in 'sst_live'
+// If it's doing full scan:
+// * Returns the list of all files in the filesystem in
+// 'full_scan_candidate_files'.
+// Otherwise, gets obsolete files from VersionSet.
+// no_full_scan = true -- never do the full scan using GetChildren()
+// force = false -- don't force the full scan, except every
+// mutable_db_options_.delete_obsolete_files_period_micros
+// force = true -- force the full scan
+void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
+ bool no_full_scan) {
+ mutex_.AssertHeld();
+
+ // if deletion is disabled, do nothing
+ if (disable_delete_obsolete_files_ > 0) {
+ return;
+ }
+
+ bool doing_the_full_scan = false;
+
+ // logic for figurint out if we're doing the full scan
+ if (no_full_scan) {
+ doing_the_full_scan = false;
+ } else if (force ||
+ mutable_db_options_.delete_obsolete_files_period_micros == 0) {
+ doing_the_full_scan = true;
+ } else {
+ const uint64_t now_micros = env_->NowMicros();
+ if ((delete_obsolete_files_last_run_ +
+ mutable_db_options_.delete_obsolete_files_period_micros) <
+ now_micros) {
+ doing_the_full_scan = true;
+ delete_obsolete_files_last_run_ = now_micros;
+ }
+ }
+
+ // don't delete files that might be currently written to from compaction
+ // threads
+ // Since job_context->min_pending_output is set, until file scan finishes,
+ // mutex_ cannot be released. Otherwise, we might see no min_pending_output
+ // here but later find newer generated unfinalized files while scannint.
+ if (!pending_outputs_.empty()) {
+ job_context->min_pending_output = *pending_outputs_.begin();
+ } else {
+ // delete all of them
+ job_context->min_pending_output = std::numeric_limits<uint64_t>::max();
+ }
+
+ // Get obsolete files. This function will also update the list of
+ // pending files in VersionSet().
+ versions_->GetObsoleteFiles(&job_context->sst_delete_files,
+ &job_context->manifest_delete_files,
+ job_context->min_pending_output);
+
+ // store the current filenum, lognum, etc
+ job_context->manifest_file_number = versions_->manifest_file_number();
+ job_context->pending_manifest_file_number =
+ versions_->pending_manifest_file_number();
+ job_context->log_number = MinLogNumberToKeep();
+
+ job_context->prev_log_number = versions_->prev_log_number();
+
+ versions_->AddLiveFiles(&job_context->sst_live);
+ if (doing_the_full_scan) {
+ for (size_t path_id = 0; path_id < immutable_db_options_.db_paths.size();
+ path_id++) {
+ // set of all files in the directory. We'll exclude files that are still
+ // alive in the subsequent processings.
+ std::vector<std::string> files;
+ env_->GetChildren(immutable_db_options_.db_paths[path_id].path,
+ &files); // Ignore errors
+ for (std::string file : files) {
+ // TODO(icanadi) clean up this mess to avoid having one-off "/" prefixes
+ job_context->full_scan_candidate_files.emplace_back(
+ "/" + file, static_cast<uint32_t>(path_id));
+ }
+ }
+
+ // Add log files in wal_dir
+ if (immutable_db_options_.wal_dir != dbname_) {
+ std::vector<std::string> log_files;
+ env_->GetChildren(immutable_db_options_.wal_dir,
+ &log_files); // Ignore errors
+ for (std::string log_file : log_files) {
+ job_context->full_scan_candidate_files.emplace_back(log_file, 0);
+ }
+ }
+ // Add info log files in db_log_dir
+ if (!immutable_db_options_.db_log_dir.empty() &&
+ immutable_db_options_.db_log_dir != dbname_) {
+ std::vector<std::string> info_log_files;
+ // Ignore errors
+ env_->GetChildren(immutable_db_options_.db_log_dir, &info_log_files);
+ for (std::string log_file : info_log_files) {
+ job_context->full_scan_candidate_files.emplace_back(log_file, 0);
+ }
+ }
+ }
+
+ // logs_ is empty when called during recovery, in which case there can't yet
+ // be any tracked obsolete logs
+ if (!alive_log_files_.empty() && !logs_.empty()) {
+ uint64_t min_log_number = job_context->log_number;
+ size_t num_alive_log_files = alive_log_files_.size();
+ // find newly obsoleted log files
+ while (alive_log_files_.begin()->number < min_log_number) {
+ auto& earliest = *alive_log_files_.begin();
+ if (immutable_db_options_.recycle_log_file_num >
+ log_recycle_files.size()) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "adding log %" PRIu64 " to recycle list\n",
+ earliest.number);
+ log_recycle_files.push_back(earliest.number);
+ } else {
+ job_context->log_delete_files.push_back(earliest.number);
+ }
+ if (job_context->size_log_to_delete == 0) {
+ job_context->prev_total_log_size = total_log_size_;
+ job_context->num_alive_log_files = num_alive_log_files;
+ }
+ job_context->size_log_to_delete += earliest.size;
+ total_log_size_ -= earliest.size;
+ if (concurrent_prepare_) {
+ log_write_mutex_.Lock();
+ }
+ alive_log_files_.pop_front();
+ if (concurrent_prepare_) {
+ log_write_mutex_.Unlock();
+ }
+ // Current log should always stay alive since it can't have
+ // number < MinLogNumber().
+ assert(alive_log_files_.size());
+ }
+ while (!logs_.empty() && logs_.front().number < min_log_number) {
+ auto& log = logs_.front();
+ if (log.getting_synced) {
+ log_sync_cv_.Wait();
+ // logs_ could have changed while we were waiting.
+ continue;
+ }
+ logs_to_free_.push_back(log.ReleaseWriter());
+ {
+ InstrumentedMutexLock wl(&log_write_mutex_);
+ logs_.pop_front();
+ }
+ }
+ // Current log cannot be obsolete.
+ assert(!logs_.empty());
+ }
+
+ // We're just cleaning up for DB::Write().
+ assert(job_context->logs_to_free.empty());
+ job_context->logs_to_free = logs_to_free_;
+ job_context->log_recycle_files.assign(log_recycle_files.begin(),
+ log_recycle_files.end());
+ logs_to_free_.clear();
+}
+
+namespace {
+bool CompareCandidateFile(const JobContext::CandidateFileInfo& first,
+ const JobContext::CandidateFileInfo& second) {
+ if (first.file_name > second.file_name) {
+ return true;
+ } else if (first.file_name < second.file_name) {
+ return false;
+ } else {
+ return (first.path_id > second.path_id);
+ }
+}
+}; // namespace
+
+// Delete obsolete files and log status and information of file deletion
+void DBImpl::DeleteObsoleteFileImpl(Status file_deletion_status, int job_id,
+ const std::string& fname, FileType type,
+ uint64_t number, uint32_t path_id) {
+ if (type == kTableFile) {
+ file_deletion_status =
+ DeleteSSTFile(&immutable_db_options_, fname, path_id);
+ } else {
+ file_deletion_status = env_->DeleteFile(fname);
+ }
+ if (file_deletion_status.ok()) {
+ ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
+ "[JOB %d] Delete %s type=%d #%" PRIu64 " -- %s\n", job_id,
+ fname.c_str(), type, number,
+ file_deletion_status.ToString().c_str());
+ } else if (env_->FileExists(fname).IsNotFound()) {
+ ROCKS_LOG_INFO(
+ immutable_db_options_.info_log,
+ "[JOB %d] Tried to delete a non-existing file %s type=%d #%" PRIu64
+ " -- %s\n",
+ job_id, fname.c_str(), type, number,
+ file_deletion_status.ToString().c_str());
+ } else {
+ ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+ "[JOB %d] Failed to delete %s type=%d #%" PRIu64 " -- %s\n",
+ job_id, fname.c_str(), type, number,
+ file_deletion_status.ToString().c_str());
+ }
+ if (type == kTableFile) {
+ EventHelpers::LogAndNotifyTableFileDeletion(
+ &event_logger_, job_id, number, fname, file_deletion_status, GetName(),
+ immutable_db_options_.listeners);
+ }
+}
+
+// Diffs the files listed in filenames and those that do not
+// belong to live files are posibly removed. Also, removes all the
+// files in sst_delete_files and log_delete_files.
+// It is not necessary to hold the mutex when invoking this method.
+void DBImpl::PurgeObsoleteFiles(const JobContext& state, bool schedule_only) {
+ // we'd better have sth to delete
+ assert(state.HaveSomethingToDelete());
+
+ // this checks if FindObsoleteFiles() was run before. If not, don't do
+ // PurgeObsoleteFiles(). If FindObsoleteFiles() was run, we need to also
+ // run PurgeObsoleteFiles(), even if disable_delete_obsolete_files_ is true
+ if (state.manifest_file_number == 0) {
+ return;
+ }
+
+ // Now, convert live list to an unordered map, WITHOUT mutex held;
+ // set is slow.
+ std::unordered_map<uint64_t, const FileDescriptor*> sst_live_map;
+ for (const FileDescriptor& fd : state.sst_live) {
+ sst_live_map[fd.GetNumber()] = &fd;
+ }
+ std::unordered_set<uint64_t> log_recycle_files_set(
+ state.log_recycle_files.begin(), state.log_recycle_files.end());
+
+ auto candidate_files = state.full_scan_candidate_files;
+ candidate_files.reserve(
+ candidate_files.size() + state.sst_delete_files.size() +
+ state.log_delete_files.size() + state.manifest_delete_files.size());
+ // We may ignore the dbname when generating the file names.
+ const char* kDumbDbName = "";
+ for (auto file : state.sst_delete_files) {
+ candidate_files.emplace_back(
+ MakeTableFileName(kDumbDbName, file->fd.GetNumber()),
+ file->fd.GetPathId());
+ if (file->table_reader_handle) {
+ table_cache_->Release(file->table_reader_handle);
+ }
+ delete file;
+ }
+
+ for (auto file_num : state.log_delete_files) {
+ if (file_num > 0) {
+ candidate_files.emplace_back(LogFileName(kDumbDbName, file_num), 0);
+ }
+ }
+ for (const auto& filename : state.manifest_delete_files) {
+ candidate_files.emplace_back(filename, 0);
+ }
+
+ // dedup state.candidate_files so we don't try to delete the same
+ // file twice
+ std::sort(candidate_files.begin(), candidate_files.end(),
+ CompareCandidateFile);
+ candidate_files.erase(
+ std::unique(candidate_files.begin(), candidate_files.end()),
+ candidate_files.end());
+
+ if (state.prev_total_log_size > 0) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "[JOB %d] Try to delete WAL files size %" PRIu64
+ ", prev total WAL file size %" PRIu64
+ ", number of live WAL files %" ROCKSDB_PRIszt ".\n",
+ state.job_id, state.size_log_to_delete,
+ state.prev_total_log_size, state.num_alive_log_files);
+ }
+
+ std::vector<std::string> old_info_log_files;
+ InfoLogPrefix info_log_prefix(!immutable_db_options_.db_log_dir.empty(),
+ dbname_);
+ for (const auto& candidate_file : candidate_files) {
+ std::string to_delete = candidate_file.file_name;
+ uint32_t path_id = candidate_file.path_id;
+ uint64_t number;
+ FileType type;
+ // Ignore file if we cannot recognize it.
+ if (!ParseFileName(to_delete, &number, info_log_prefix.prefix, &type)) {
+ continue;
+ }
+
+ bool keep = true;
+ switch (type) {
+ case kLogFile:
+ keep = ((number >= state.log_number) ||
+ (number == state.prev_log_number) ||
+ (log_recycle_files_set.find(number) !=
+ log_recycle_files_set.end()));
+ break;
+ case kDescriptorFile:
+ // Keep my manifest file, and any newer incarnations'
+ // (can happen during manifest roll)
+ keep = (number >= state.manifest_file_number);
+ break;
+ case kTableFile:
+ // If the second condition is not there, this makes
+ // DontDeletePendingOutputs fail
+ keep = (sst_live_map.find(number) != sst_live_map.end()) ||
+ number >= state.min_pending_output;
+ break;
+ case kTempFile:
+ // Any temp files that are currently being written to must
+ // be recorded in pending_outputs_, which is inserted into "live".
+ // Also, SetCurrentFile creates a temp file when writing out new
+ // manifest, which is equal to state.pending_manifest_file_number. We
+ // should not delete that file
+ //
+ // TODO(yhchiang): carefully modify the third condition to safely
+ // remove the temp options files.
+ keep = (sst_live_map.find(number) != sst_live_map.end()) ||
+ (number == state.pending_manifest_file_number) ||
+ (to_delete.find(kOptionsFileNamePrefix) != std::string::npos);
+ break;
+ case kInfoLogFile:
+ keep = true;
+ if (number != 0) {
+ old_info_log_files.push_back(to_delete);
+ }
+ break;
+ case kCurrentFile:
+ case kDBLockFile:
+ case kIdentityFile:
+ case kMetaDatabase:
+ case kOptionsFile:
+ case kBlobFile:
+ keep = true;
+ break;
+ }
+
+ if (keep) {
+ continue;
+ }
+
+ std::string fname;
+ if (type == kTableFile) {
+ // evict from cache
+ TableCache::Evict(table_cache_.get(), number);
+ fname = TableFileName(immutable_db_options_.db_paths, number, path_id);
+ } else {
+ fname = ((type == kLogFile) ? immutable_db_options_.wal_dir : dbname_) +
+ "/" + to_delete;
+ }
+
+#ifndef ROCKSDB_LITE
+ if (type == kLogFile && (immutable_db_options_.wal_ttl_seconds > 0 ||
+ immutable_db_options_.wal_size_limit_mb > 0)) {
+ wal_manager_.ArchiveWALFile(fname, number);
+ continue;
+ }
+#endif // !ROCKSDB_LITE
+
+ Status file_deletion_status;
+ if (schedule_only) {
+ InstrumentedMutexLock guard_lock(&mutex_);
+ SchedulePendingPurge(fname, type, number, path_id, state.job_id);
+ } else {
+ DeleteObsoleteFileImpl(file_deletion_status, state.job_id, fname, type,
+ number, path_id);
+ }
+ }
+
+ // Delete old info log files.
+ size_t old_info_log_file_count = old_info_log_files.size();
+ if (old_info_log_file_count != 0 &&
+ old_info_log_file_count >= immutable_db_options_.keep_log_file_num) {
+ std::sort(old_info_log_files.begin(), old_info_log_files.end());
+ size_t end =
+ old_info_log_file_count - immutable_db_options_.keep_log_file_num;
+ for (unsigned int i = 0; i <= end; i++) {
+ std::string& to_delete = old_info_log_files.at(i);
+ std::string full_path_to_delete =
+ (immutable_db_options_.db_log_dir.empty()
+ ? dbname_
+ : immutable_db_options_.db_log_dir) +
+ "/" + to_delete;
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "[JOB %d] Delete info log file %s\n", state.job_id,
+ full_path_to_delete.c_str());
+ Status s = env_->DeleteFile(full_path_to_delete);
+ if (!s.ok()) {
+ if (env_->FileExists(full_path_to_delete).IsNotFound()) {
+ ROCKS_LOG_INFO(
+ immutable_db_options_.info_log,
+ "[JOB %d] Tried to delete non-existing info log file %s FAILED "
+ "-- %s\n",
+ state.job_id, to_delete.c_str(), s.ToString().c_str());
+ } else {
+ ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+ "[JOB %d] Delete info log file %s FAILED -- %s\n",
+ state.job_id, to_delete.c_str(),
+ s.ToString().c_str());
+ }
+ }
+ }
+ }
+#ifndef ROCKSDB_LITE
+ wal_manager_.PurgeObsoleteWALFiles();
+#endif // ROCKSDB_LITE
+ LogFlush(immutable_db_options_.info_log);
+}
+
+void DBImpl::DeleteObsoleteFiles() {
+ mutex_.AssertHeld();
+ JobContext job_context(next_job_id_.fetch_add(1));
+ FindObsoleteFiles(&job_context, true);
+
+ mutex_.Unlock();
+ if (job_context.HaveSomethingToDelete()) {
+ PurgeObsoleteFiles(job_context);
+ }
+ job_context.Clean();
+ mutex_.Lock();
+}
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_impl_open.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_impl_open.cc b/thirdparty/rocksdb/db/db_impl_open.cc
new file mode 100644
index 0000000..bc94b60
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_impl_open.cc
@@ -0,0 +1,1129 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include "db/db_impl.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+#include <inttypes.h>
+
+#include "db/builder.h"
+#include "options/options_helper.h"
+#include "rocksdb/wal_filter.h"
+#include "table/block_based_table_factory.h"
+#include "util/rate_limiter.h"
+#include "util/sst_file_manager_impl.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+Options SanitizeOptions(const std::string& dbname,
+ const Options& src) {
+ auto db_options = SanitizeOptions(dbname, DBOptions(src));
+ ImmutableDBOptions immutable_db_options(db_options);
+ auto cf_options =
+ SanitizeOptions(immutable_db_options, ColumnFamilyOptions(src));
+ return Options(db_options, cf_options);
+}
+
+DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
+ DBOptions result(src);
+
+ // result.max_open_files means an "infinite" open files.
+ if (result.max_open_files != -1) {
+ int max_max_open_files = port::GetMaxOpenFiles();
+ if (max_max_open_files == -1) {
+ max_max_open_files = 0x400000;
+ }
+ ClipToRange(&result.max_open_files, 20, max_max_open_files);
+ }
+
+ if (result.info_log == nullptr) {
+ Status s = CreateLoggerFromOptions(dbname, result, &result.info_log);
+ if (!s.ok()) {
+ // No place suitable for logging
+ result.info_log = nullptr;
+ }
+ }
+
+ if (!result.write_buffer_manager) {
+ result.write_buffer_manager.reset(
+ new WriteBufferManager(result.db_write_buffer_size));
+ }
+ auto bg_job_limits = DBImpl::GetBGJobLimits(result.max_background_flushes,
+ result.max_background_compactions,
+ result.max_background_jobs,
+ true /* parallelize_compactions */);
+ result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_compactions,
+ Env::Priority::LOW);
+ result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_flushes,
+ Env::Priority::HIGH);
+
+ if (result.rate_limiter.get() != nullptr) {
+ if (result.bytes_per_sync == 0) {
+ result.bytes_per_sync = 1024 * 1024;
+ }
+ }
+
+ if (result.delayed_write_rate == 0) {
+ if (result.rate_limiter.get() != nullptr) {
+ result.delayed_write_rate = result.rate_limiter->GetBytesPerSecond();
+ }
+ if (result.delayed_write_rate == 0) {
+ result.delayed_write_rate = 16 * 1024 * 1024;
+ }
+ }
+
+ if (result.WAL_ttl_seconds > 0 || result.WAL_size_limit_MB > 0) {
+ result.recycle_log_file_num = false;
+ }
+
+ if (result.recycle_log_file_num &&
+ (result.wal_recovery_mode == WALRecoveryMode::kPointInTimeRecovery ||
+ result.wal_recovery_mode == WALRecoveryMode::kAbsoluteConsistency)) {
+ // kPointInTimeRecovery is indistinguishable from
+ // kTolerateCorruptedTailRecords in recycle mode since we define
+ // the "end" of the log as the first corrupt record we encounter.
+ // kAbsoluteConsistency doesn't make sense because even a clean
+ // shutdown leaves old junk at the end of the log file.
+ result.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
+ }
+
+ if (result.wal_dir.empty()) {
+ // Use dbname as default
+ result.wal_dir = dbname;
+ }
+ if (result.wal_dir.back() == '/') {
+ result.wal_dir = result.wal_dir.substr(0, result.wal_dir.size() - 1);
+ }
+
+ if (result.db_paths.size() == 0) {
+ result.db_paths.emplace_back(dbname, std::numeric_limits<uint64_t>::max());
+ }
+
+ if (result.use_direct_io_for_flush_and_compaction &&
+ result.compaction_readahead_size == 0) {
+ TEST_SYNC_POINT_CALLBACK("SanitizeOptions:direct_io", nullptr);
+ result.compaction_readahead_size = 1024 * 1024 * 2;
+ }
+
+ if (result.compaction_readahead_size > 0 ||
+ result.use_direct_io_for_flush_and_compaction) {
+ result.new_table_reader_for_compaction_inputs = true;
+ }
+
+ // Force flush on DB open if 2PC is enabled, since with 2PC we have no
+ // guarantee that consecutive log files have consecutive sequence id, which
+ // make recovery complicated.
+ if (result.allow_2pc) {
+ result.avoid_flush_during_recovery = false;
+ }
+
+ return result;
+}
+
+namespace {
+
+Status SanitizeOptionsByTable(
+ const DBOptions& db_opts,
+ const std::vector<ColumnFamilyDescriptor>& column_families) {
+ Status s;
+ for (auto cf : column_families) {
+ s = cf.options.table_factory->SanitizeOptions(db_opts, cf.options);
+ if (!s.ok()) {
+ return s;
+ }
+ }
+ return Status::OK();
+}
+
+static Status ValidateOptions(
+ const DBOptions& db_options,
+ const std::vector<ColumnFamilyDescriptor>& column_families) {
+ Status s;
+
+ for (auto& cfd : column_families) {
+ s = CheckCompressionSupported(cfd.options);
+ if (s.ok() && db_options.allow_concurrent_memtable_write) {
+ s = CheckConcurrentWritesSupported(cfd.options);
+ }
+ if (!s.ok()) {
+ return s;
+ }
+ if (db_options.db_paths.size() > 1) {
+ if ((cfd.options.compaction_style != kCompactionStyleUniversal) &&
+ (cfd.options.compaction_style != kCompactionStyleLevel)) {
+ return Status::NotSupported(
+ "More than one DB paths are only supported in "
+ "universal and level compaction styles. ");
+ }
+ }
+ if (cfd.options.compaction_options_fifo.ttl > 0) {
+ if (db_options.max_open_files != -1) {
+ return Status::NotSupported(
+ "FIFO Compaction with TTL is only supported when files are always "
+ "kept open (set max_open_files = -1). ");
+ }
+ if (cfd.options.table_factory->Name() !=
+ BlockBasedTableFactory().Name()) {
+ return Status::NotSupported(
+ "FIFO Compaction with TTL is only supported in "
+ "Block-Based Table format. ");
+ }
+ }
+ }
+
+ if (db_options.db_paths.size() > 4) {
+ return Status::NotSupported(
+ "More than four DB paths are not supported yet. ");
+ }
+
+ if (db_options.allow_mmap_reads && db_options.use_direct_reads) {
+ // Protect against assert in PosixMMapReadableFile constructor
+ return Status::NotSupported(
+ "If memory mapped reads (allow_mmap_reads) are enabled "
+ "then direct I/O reads (use_direct_reads) must be disabled. ");
+ }
+
+ if (db_options.allow_mmap_writes &&
+ db_options.use_direct_io_for_flush_and_compaction) {
+ return Status::NotSupported(
+ "If memory mapped writes (allow_mmap_writes) are enabled "
+ "then direct I/O writes (use_direct_io_for_flush_and_compaction) must "
+ "be disabled. ");
+ }
+
+ if (db_options.keep_log_file_num == 0) {
+ return Status::InvalidArgument("keep_log_file_num must be greater than 0");
+ }
+
+ return Status::OK();
+}
+} // namespace
+Status DBImpl::NewDB() {
+ VersionEdit new_db;
+ new_db.SetLogNumber(0);
+ new_db.SetNextFile(2);
+ new_db.SetLastSequence(0);
+
+ Status s;
+
+ ROCKS_LOG_INFO(immutable_db_options_.info_log, "Creating manifest 1 \n");
+ const std::string manifest = DescriptorFileName(dbname_, 1);
+ {
+ unique_ptr<WritableFile> file;
+ EnvOptions env_options = env_->OptimizeForManifestWrite(env_options_);
+ s = NewWritableFile(env_, manifest, &file, env_options);
+ if (!s.ok()) {
+ return s;
+ }
+ file->SetPreallocationBlockSize(
+ immutable_db_options_.manifest_preallocation_size);
+ unique_ptr<WritableFileWriter> file_writer(
+ new WritableFileWriter(std::move(file), env_options));
+ log::Writer log(std::move(file_writer), 0, false);
+ std::string record;
+ new_db.EncodeTo(&record);
+ s = log.AddRecord(record);
+ if (s.ok()) {
+ s = SyncManifest(env_, &immutable_db_options_, log.file());
+ }
+ }
+ if (s.ok()) {
+ // Make "CURRENT" file that points to the new manifest file.
+ s = SetCurrentFile(env_, dbname_, 1, directories_.GetDbDir());
+ } else {
+ env_->DeleteFile(manifest);
+ }
+ return s;
+}
+
+Status DBImpl::Directories::CreateAndNewDirectory(
+ Env* env, const std::string& dirname,
+ std::unique_ptr<Directory>* directory) const {
+ // We call CreateDirIfMissing() as the directory may already exist (if we
+ // are reopening a DB), when this happens we don't want creating the
+ // directory to cause an error. However, we need to check if creating the
+ // directory fails or else we may get an obscure message about the lock
+ // file not existing. One real-world example of this occurring is if
+ // env->CreateDirIfMissing() doesn't create intermediate directories, e.g.
+ // when dbname_ is "dir/db" but when "dir" doesn't exist.
+ Status s = env->CreateDirIfMissing(dirname);
+ if (!s.ok()) {
+ return s;
+ }
+ return env->NewDirectory(dirname, directory);
+}
+
+Status DBImpl::Directories::SetDirectories(
+ Env* env, const std::string& dbname, const std::string& wal_dir,
+ const std::vector<DbPath>& data_paths) {
+ Status s = CreateAndNewDirectory(env, dbname, &db_dir_);
+ if (!s.ok()) {
+ return s;
+ }
+ if (!wal_dir.empty() && dbname != wal_dir) {
+ s = CreateAndNewDirectory(env, wal_dir, &wal_dir_);
+ if (!s.ok()) {
+ return s;
+ }
+ }
+
+ data_dirs_.clear();
+ for (auto& p : data_paths) {
+ const std::string db_path = p.path;
+ if (db_path == dbname) {
+ data_dirs_.emplace_back(nullptr);
+ } else {
+ std::unique_ptr<Directory> path_directory;
+ s = CreateAndNewDirectory(env, db_path, &path_directory);
+ if (!s.ok()) {
+ return s;
+ }
+ data_dirs_.emplace_back(path_directory.release());
+ }
+ }
+ assert(data_dirs_.size() == data_paths.size());
+ return Status::OK();
+}
+
+Status DBImpl::Recover(
+ const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
+ bool error_if_log_file_exist, bool error_if_data_exists_in_logs) {
+ mutex_.AssertHeld();
+
+ bool is_new_db = false;
+ assert(db_lock_ == nullptr);
+ if (!read_only) {
+ Status s = directories_.SetDirectories(env_, dbname_,
+ immutable_db_options_.wal_dir,
+ immutable_db_options_.db_paths);
+ if (!s.ok()) {
+ return s;
+ }
+
+ s = env_->LockFile(LockFileName(dbname_), &db_lock_);
+ if (!s.ok()) {
+ return s;
+ }
+
+ s = env_->FileExists(CurrentFileName(dbname_));
+ if (s.IsNotFound()) {
+ if (immutable_db_options_.create_if_missing) {
+ s = NewDB();
+ is_new_db = true;
+ if (!s.ok()) {
+ return s;
+ }
+ } else {
+ return Status::InvalidArgument(
+ dbname_, "does not exist (create_if_missing is false)");
+ }
+ } else if (s.ok()) {
+ if (immutable_db_options_.error_if_exists) {
+ return Status::InvalidArgument(
+ dbname_, "exists (error_if_exists is true)");
+ }
+ } else {
+ // Unexpected error reading file
+ assert(s.IsIOError());
+ return s;
+ }
+ // Check for the IDENTITY file and create it if not there
+ s = env_->FileExists(IdentityFileName(dbname_));
+ if (s.IsNotFound()) {
+ s = SetIdentityFile(env_, dbname_);
+ if (!s.ok()) {
+ return s;
+ }
+ } else if (!s.ok()) {
+ assert(s.IsIOError());
+ return s;
+ }
+ }
+
+ Status s = versions_->Recover(column_families, read_only);
+ if (immutable_db_options_.paranoid_checks && s.ok()) {
+ s = CheckConsistency();
+ }
+ if (s.ok()) {
+ SequenceNumber next_sequence(kMaxSequenceNumber);
+ default_cf_handle_ = new ColumnFamilyHandleImpl(
+ versions_->GetColumnFamilySet()->GetDefault(), this, &mutex_);
+ default_cf_internal_stats_ = default_cf_handle_->cfd()->internal_stats();
+ single_column_family_mode_ =
+ versions_->GetColumnFamilySet()->NumberOfColumnFamilies() == 1;
+
+ // Recover from all newer log files than the ones named in the
+ // descriptor (new log files may have been added by the previous
+ // incarnation without registering them in the descriptor).
+ //
+ // Note that prev_log_number() is no longer used, but we pay
+ // attention to it in case we are recovering a database
+ // produced by an older version of rocksdb.
+ std::vector<std::string> filenames;
+ s = env_->GetChildren(immutable_db_options_.wal_dir, &filenames);
+ if (!s.ok()) {
+ return s;
+ }
+
+ std::vector<uint64_t> logs;
+ for (size_t i = 0; i < filenames.size(); i++) {
+ uint64_t number;
+ FileType type;
+ if (ParseFileName(filenames[i], &number, &type) && type == kLogFile) {
+ if (is_new_db) {
+ return Status::Corruption(
+ "While creating a new Db, wal_dir contains "
+ "existing log file: ",
+ filenames[i]);
+ } else {
+ logs.push_back(number);
+ }
+ }
+ }
+
+ if (logs.size() > 0) {
+ if (error_if_log_file_exist) {
+ return Status::Corruption(
+ "The db was opened in readonly mode with error_if_log_file_exist"
+ "flag but a log file already exists");
+ } else if (error_if_data_exists_in_logs) {
+ for (auto& log : logs) {
+ std::string fname = LogFileName(immutable_db_options_.wal_dir, log);
+ uint64_t bytes;
+ s = env_->GetFileSize(fname, &bytes);
+ if (s.ok()) {
+ if (bytes > 0) {
+ return Status::Corruption(
+ "error_if_data_exists_in_logs is set but there are data "
+ " in log files.");
+ }
+ }
+ }
+ }
+ }
+
+ if (!logs.empty()) {
+ // Recover in the order in which the logs were generated
+ std::sort(logs.begin(), logs.end());
+ s = RecoverLogFiles(logs, &next_sequence, read_only);
+ if (!s.ok()) {
+ // Clear memtables if recovery failed
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ cfd->CreateNewMemtable(*cfd->GetLatestMutableCFOptions(),
+ kMaxSequenceNumber);
+ }
+ }
+ }
+ }
+
+ // Initial value
+ max_total_in_memory_state_ = 0;
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
+ max_total_in_memory_state_ += mutable_cf_options->write_buffer_size *
+ mutable_cf_options->max_write_buffer_number;
+ }
+
+ return s;
+}
+
+// REQUIRES: log_numbers are sorted in ascending order
+Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
+ SequenceNumber* next_sequence, bool read_only) {
+ struct LogReporter : public log::Reader::Reporter {
+ Env* env;
+ Logger* info_log;
+ const char* fname;
+ Status* status; // nullptr if immutable_db_options_.paranoid_checks==false
+ virtual void Corruption(size_t bytes, const Status& s) override {
+ ROCKS_LOG_WARN(info_log, "%s%s: dropping %d bytes; %s",
+ (this->status == nullptr ? "(ignoring error) " : ""),
+ fname, static_cast<int>(bytes), s.ToString().c_str());
+ if (this->status != nullptr && this->status->ok()) {
+ *this->status = s;
+ }
+ }
+ };
+
+ mutex_.AssertHeld();
+ Status status;
+ std::unordered_map<int, VersionEdit> version_edits;
+ // no need to refcount because iteration is under mutex
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ VersionEdit edit;
+ edit.SetColumnFamily(cfd->GetID());
+ version_edits.insert({cfd->GetID(), edit});
+ }
+ int job_id = next_job_id_.fetch_add(1);
+ {
+ auto stream = event_logger_.Log();
+ stream << "job" << job_id << "event"
+ << "recovery_started";
+ stream << "log_files";
+ stream.StartArray();
+ for (auto log_number : log_numbers) {
+ stream << log_number;
+ }
+ stream.EndArray();
+ }
+
+#ifndef ROCKSDB_LITE
+ if (immutable_db_options_.wal_filter != nullptr) {
+ std::map<std::string, uint32_t> cf_name_id_map;
+ std::map<uint32_t, uint64_t> cf_lognumber_map;
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ cf_name_id_map.insert(
+ std::make_pair(cfd->GetName(), cfd->GetID()));
+ cf_lognumber_map.insert(
+ std::make_pair(cfd->GetID(), cfd->GetLogNumber()));
+ }
+
+ immutable_db_options_.wal_filter->ColumnFamilyLogNumberMap(cf_lognumber_map,
+ cf_name_id_map);
+ }
+#endif
+
+ bool stop_replay_by_wal_filter = false;
+ bool stop_replay_for_corruption = false;
+ bool flushed = false;
+ for (auto log_number : log_numbers) {
+ // The previous incarnation may not have written any MANIFEST
+ // records after allocating this log number. So we manually
+ // update the file number allocation counter in VersionSet.
+ versions_->MarkFileNumberUsedDuringRecovery(log_number);
+ // Open the log file
+ std::string fname = LogFileName(immutable_db_options_.wal_dir, log_number);
+
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "Recovering log #%" PRIu64 " mode %d", log_number,
+ immutable_db_options_.wal_recovery_mode);
+ auto logFileDropped = [this, &fname]() {
+ uint64_t bytes;
+ if (env_->GetFileSize(fname, &bytes).ok()) {
+ auto info_log = immutable_db_options_.info_log.get();
+ ROCKS_LOG_WARN(info_log, "%s: dropping %d bytes", fname.c_str(),
+ static_cast<int>(bytes));
+ }
+ };
+ if (stop_replay_by_wal_filter) {
+ logFileDropped();
+ continue;
+ }
+
+ unique_ptr<SequentialFileReader> file_reader;
+ {
+ unique_ptr<SequentialFile> file;
+ status = env_->NewSequentialFile(fname, &file,
+ env_->OptimizeForLogRead(env_options_));
+ if (!status.ok()) {
+ MaybeIgnoreError(&status);
+ if (!status.ok()) {
+ return status;
+ } else {
+ // Fail with one log file, but that's ok.
+ // Try next one.
+ continue;
+ }
+ }
+ file_reader.reset(new SequentialFileReader(std::move(file)));
+ }
+
+ // Create the log reader.
+ LogReporter reporter;
+ reporter.env = env_;
+ reporter.info_log = immutable_db_options_.info_log.get();
+ reporter.fname = fname.c_str();
+ if (!immutable_db_options_.paranoid_checks ||
+ immutable_db_options_.wal_recovery_mode ==
+ WALRecoveryMode::kSkipAnyCorruptedRecords) {
+ reporter.status = nullptr;
+ } else {
+ reporter.status = &status;
+ }
+ // We intentially make log::Reader do checksumming even if
+ // paranoid_checks==false so that corruptions cause entire commits
+ // to be skipped instead of propagating bad information (like overly
+ // large sequence numbers).
+ log::Reader reader(immutable_db_options_.info_log, std::move(file_reader),
+ &reporter, true /*checksum*/, 0 /*initial_offset*/,
+ log_number);
+
+ // Determine if we should tolerate incomplete records at the tail end of the
+ // Read all the records and add to a memtable
+ std::string scratch;
+ Slice record;
+ WriteBatch batch;
+
+ while (!stop_replay_by_wal_filter &&
+ reader.ReadRecord(&record, &scratch,
+ immutable_db_options_.wal_recovery_mode) &&
+ status.ok()) {
+ if (record.size() < WriteBatchInternal::kHeader) {
+ reporter.Corruption(record.size(),
+ Status::Corruption("log record too small"));
+ continue;
+ }
+ WriteBatchInternal::SetContents(&batch, record);
+ SequenceNumber sequence = WriteBatchInternal::Sequence(&batch);
+
+ if (immutable_db_options_.wal_recovery_mode ==
+ WALRecoveryMode::kPointInTimeRecovery) {
+ // In point-in-time recovery mode, if sequence id of log files are
+ // consecutive, we continue recovery despite corruption. This could
+ // happen when we open and write to a corrupted DB, where sequence id
+ // will start from the last sequence id we recovered.
+ if (sequence == *next_sequence) {
+ stop_replay_for_corruption = false;
+ }
+ if (stop_replay_for_corruption) {
+ logFileDropped();
+ break;
+ }
+ }
+
+#ifndef ROCKSDB_LITE
+ if (immutable_db_options_.wal_filter != nullptr) {
+ WriteBatch new_batch;
+ bool batch_changed = false;
+
+ WalFilter::WalProcessingOption wal_processing_option =
+ immutable_db_options_.wal_filter->LogRecordFound(
+ log_number, fname, batch, &new_batch, &batch_changed);
+
+ switch (wal_processing_option) {
+ case WalFilter::WalProcessingOption::kContinueProcessing:
+ // do nothing, proceeed normally
+ break;
+ case WalFilter::WalProcessingOption::kIgnoreCurrentRecord:
+ // skip current record
+ continue;
+ case WalFilter::WalProcessingOption::kStopReplay:
+ // skip current record and stop replay
+ stop_replay_by_wal_filter = true;
+ continue;
+ case WalFilter::WalProcessingOption::kCorruptedRecord: {
+ status =
+ Status::Corruption("Corruption reported by Wal Filter ",
+ immutable_db_options_.wal_filter->Name());
+ MaybeIgnoreError(&status);
+ if (!status.ok()) {
+ reporter.Corruption(record.size(), status);
+ continue;
+ }
+ break;
+ }
+ default: {
+ assert(false); // unhandled case
+ status = Status::NotSupported(
+ "Unknown WalProcessingOption returned"
+ " by Wal Filter ",
+ immutable_db_options_.wal_filter->Name());
+ MaybeIgnoreError(&status);
+ if (!status.ok()) {
+ return status;
+ } else {
+ // Ignore the error with current record processing.
+ continue;
+ }
+ }
+ }
+
+ if (batch_changed) {
+ // Make sure that the count in the new batch is
+ // within the orignal count.
+ int new_count = WriteBatchInternal::Count(&new_batch);
+ int original_count = WriteBatchInternal::Count(&batch);
+ if (new_count > original_count) {
+ ROCKS_LOG_FATAL(
+ immutable_db_options_.info_log,
+ "Recovering log #%" PRIu64
+ " mode %d log filter %s returned "
+ "more records (%d) than original (%d) which is not allowed. "
+ "Aborting recovery.",
+ log_number, immutable_db_options_.wal_recovery_mode,
+ immutable_db_options_.wal_filter->Name(), new_count,
+ original_count);
+ status = Status::NotSupported(
+ "More than original # of records "
+ "returned by Wal Filter ",
+ immutable_db_options_.wal_filter->Name());
+ return status;
+ }
+ // Set the same sequence number in the new_batch
+ // as the original batch.
+ WriteBatchInternal::SetSequence(&new_batch,
+ WriteBatchInternal::Sequence(&batch));
+ batch = new_batch;
+ }
+ }
+#endif // ROCKSDB_LITE
+
+ // If column family was not found, it might mean that the WAL write
+ // batch references to the column family that was dropped after the
+ // insert. We don't want to fail the whole write batch in that case --
+ // we just ignore the update.
+ // That's why we set ignore missing column families to true
+ bool has_valid_writes = false;
+ status = WriteBatchInternal::InsertInto(
+ &batch, column_family_memtables_.get(), &flush_scheduler_, true,
+ log_number, this, false /* concurrent_memtable_writes */,
+ next_sequence, &has_valid_writes);
+ MaybeIgnoreError(&status);
+ if (!status.ok()) {
+ // We are treating this as a failure while reading since we read valid
+ // blocks that do not form coherent data
+ reporter.Corruption(record.size(), status);
+ continue;
+ }
+
+ if (has_valid_writes && !read_only) {
+ // we can do this because this is called before client has access to the
+ // DB and there is only a single thread operating on DB
+ ColumnFamilyData* cfd;
+
+ while ((cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) {
+ cfd->Unref();
+ // If this asserts, it means that InsertInto failed in
+ // filtering updates to already-flushed column families
+ assert(cfd->GetLogNumber() <= log_number);
+ auto iter = version_edits.find(cfd->GetID());
+ assert(iter != version_edits.end());
+ VersionEdit* edit = &iter->second;
+ status = WriteLevel0TableForRecovery(job_id, cfd, cfd->mem(), edit);
+ if (!status.ok()) {
+ // Reflect errors immediately so that conditions like full
+ // file-systems cause the DB::Open() to fail.
+ return status;
+ }
+ flushed = true;
+
+ cfd->CreateNewMemtable(*cfd->GetLatestMutableCFOptions(),
+ *next_sequence);
+ }
+ }
+ }
+
+ if (!status.ok()) {
+ if (immutable_db_options_.wal_recovery_mode ==
+ WALRecoveryMode::kSkipAnyCorruptedRecords) {
+ // We should ignore all errors unconditionally
+ status = Status::OK();
+ } else if (immutable_db_options_.wal_recovery_mode ==
+ WALRecoveryMode::kPointInTimeRecovery) {
+ // We should ignore the error but not continue replaying
+ status = Status::OK();
+ stop_replay_for_corruption = true;
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "Point in time recovered to log #%" PRIu64
+ " seq #%" PRIu64,
+ log_number, *next_sequence);
+ } else {
+ assert(immutable_db_options_.wal_recovery_mode ==
+ WALRecoveryMode::kTolerateCorruptedTailRecords ||
+ immutable_db_options_.wal_recovery_mode ==
+ WALRecoveryMode::kAbsoluteConsistency);
+ return status;
+ }
+ }
+
+ flush_scheduler_.Clear();
+ auto last_sequence = *next_sequence - 1;
+ if ((*next_sequence != kMaxSequenceNumber) &&
+ (versions_->LastSequence() <= last_sequence)) {
+ versions_->SetLastToBeWrittenSequence(last_sequence);
+ versions_->SetLastSequence(last_sequence);
+ }
+ }
+
+ // True if there's any data in the WALs; if not, we can skip re-processing
+ // them later
+ bool data_seen = false;
+ if (!read_only) {
+ // no need to refcount since client still doesn't have access
+ // to the DB and can not drop column families while we iterate
+ auto max_log_number = log_numbers.back();
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ auto iter = version_edits.find(cfd->GetID());
+ assert(iter != version_edits.end());
+ VersionEdit* edit = &iter->second;
+
+ if (cfd->GetLogNumber() > max_log_number) {
+ // Column family cfd has already flushed the data
+ // from all logs. Memtable has to be empty because
+ // we filter the updates based on log_number
+ // (in WriteBatch::InsertInto)
+ assert(cfd->mem()->GetFirstSequenceNumber() == 0);
+ assert(edit->NumEntries() == 0);
+ continue;
+ }
+
+ // flush the final memtable (if non-empty)
+ if (cfd->mem()->GetFirstSequenceNumber() != 0) {
+ // If flush happened in the middle of recovery (e.g. due to memtable
+ // being full), we flush at the end. Otherwise we'll need to record
+ // where we were on last flush, which make the logic complicated.
+ if (flushed || !immutable_db_options_.avoid_flush_during_recovery) {
+ status = WriteLevel0TableForRecovery(job_id, cfd, cfd->mem(), edit);
+ if (!status.ok()) {
+ // Recovery failed
+ break;
+ }
+ flushed = true;
+
+ cfd->CreateNewMemtable(*cfd->GetLatestMutableCFOptions(),
+ versions_->LastSequence());
+ }
+ data_seen = true;
+ }
+
+ // write MANIFEST with update
+ // writing log_number in the manifest means that any log file
+ // with number strongly less than (log_number + 1) is already
+ // recovered and should be ignored on next reincarnation.
+ // Since we already recovered max_log_number, we want all logs
+ // with numbers `<= max_log_number` (includes this one) to be ignored
+ if (flushed || cfd->mem()->GetFirstSequenceNumber() == 0) {
+ edit->SetLogNumber(max_log_number + 1);
+ }
+ // we must mark the next log number as used, even though it's
+ // not actually used. that is because VersionSet assumes
+ // VersionSet::next_file_number_ always to be strictly greater than any
+ // log number
+ versions_->MarkFileNumberUsedDuringRecovery(max_log_number + 1);
+ status = versions_->LogAndApply(
+ cfd, *cfd->GetLatestMutableCFOptions(), edit, &mutex_);
+ if (!status.ok()) {
+ // Recovery failed
+ break;
+ }
+ }
+ }
+
+ if (data_seen && !flushed) {
+ // Mark these as alive so they'll be considered for deletion later by
+ // FindObsoleteFiles()
+ if (concurrent_prepare_) {
+ log_write_mutex_.Lock();
+ }
+ for (auto log_number : log_numbers) {
+ alive_log_files_.push_back(LogFileNumberSize(log_number));
+ }
+ if (concurrent_prepare_) {
+ log_write_mutex_.Unlock();
+ }
+ }
+
+ event_logger_.Log() << "job" << job_id << "event"
+ << "recovery_finished";
+
+ return status;
+}
+
+Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
+ MemTable* mem, VersionEdit* edit) {
+ mutex_.AssertHeld();
+ const uint64_t start_micros = env_->NowMicros();
+ FileMetaData meta;
+ auto pending_outputs_inserted_elem =
+ CaptureCurrentFileNumberInPendingOutputs();
+ meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0);
+ ReadOptions ro;
+ ro.total_order_seek = true;
+ Arena arena;
+ Status s;
+ TableProperties table_properties;
+ {
+ ScopedArenaIterator iter(mem->NewIterator(ro, &arena));
+ ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
+ "[%s] [WriteLevel0TableForRecovery]"
+ " Level-0 table #%" PRIu64 ": started",
+ cfd->GetName().c_str(), meta.fd.GetNumber());
+
+ // Get the latest mutable cf options while the mutex is still locked
+ const MutableCFOptions mutable_cf_options =
+ *cfd->GetLatestMutableCFOptions();
+ bool paranoid_file_checks =
+ cfd->GetLatestMutableCFOptions()->paranoid_file_checks;
+
+ int64_t _current_time = 0;
+ env_->GetCurrentTime(&_current_time); // ignore error
+ const uint64_t current_time = static_cast<uint64_t>(_current_time);
+
+ {
+ mutex_.Unlock();
+
+ SequenceNumber earliest_write_conflict_snapshot;
+ std::vector<SequenceNumber> snapshot_seqs =
+ snapshots_.GetAll(&earliest_write_conflict_snapshot);
+
+ EnvOptions optimized_env_options =
+ env_->OptimizeForCompactionTableWrite(env_options_, immutable_db_options_);
+ s = BuildTable(
+ dbname_, env_, *cfd->ioptions(), mutable_cf_options,
+ optimized_env_options, cfd->table_cache(), iter.get(),
+ std::unique_ptr<InternalIterator>(mem->NewRangeTombstoneIterator(ro)),
+ &meta, cfd->internal_comparator(),
+ cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
+ snapshot_seqs, earliest_write_conflict_snapshot,
+ GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
+ cfd->ioptions()->compression_opts, paranoid_file_checks,
+ cfd->internal_stats(), TableFileCreationReason::kRecovery,
+ &event_logger_, job_id, Env::IO_HIGH, nullptr /* table_properties */,
+ -1 /* level */, current_time);
+ LogFlush(immutable_db_options_.info_log);
+ ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
+ "[%s] [WriteLevel0TableForRecovery]"
+ " Level-0 table #%" PRIu64 ": %" PRIu64 " bytes %s",
+ cfd->GetName().c_str(), meta.fd.GetNumber(),
+ meta.fd.GetFileSize(), s.ToString().c_str());
+ mutex_.Lock();
+ }
+ }
+ ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
+
+ // Note that if file_size is zero, the file has been deleted and
+ // should not be added to the manifest.
+ int level = 0;
+ if (s.ok() && meta.fd.GetFileSize() > 0) {
+ edit->AddFile(level, meta.fd.GetNumber(), meta.fd.GetPathId(),
+ meta.fd.GetFileSize(), meta.smallest, meta.largest,
+ meta.smallest_seqno, meta.largest_seqno,
+ meta.marked_for_compaction);
+ }
+
+ InternalStats::CompactionStats stats(1);
+ stats.micros = env_->NowMicros() - start_micros;
+ stats.bytes_written = meta.fd.GetFileSize();
+ stats.num_output_files = 1;
+ cfd->internal_stats()->AddCompactionStats(level, stats);
+ cfd->internal_stats()->AddCFStats(
+ InternalStats::BYTES_FLUSHED, meta.fd.GetFileSize());
+ RecordTick(stats_, COMPACT_WRITE_BYTES, meta.fd.GetFileSize());
+ return s;
+}
+
+Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
+ DBOptions db_options(options);
+ ColumnFamilyOptions cf_options(options);
+ std::vector<ColumnFamilyDescriptor> column_families;
+ column_families.push_back(
+ ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options));
+ std::vector<ColumnFamilyHandle*> handles;
+ Status s = DB::Open(db_options, dbname, column_families, &handles, dbptr);
+ if (s.ok()) {
+ assert(handles.size() == 1);
+ // i can delete the handle since DBImpl is always holding a reference to
+ // default column family
+ delete handles[0];
+ }
+ return s;
+}
+
+Status DB::Open(const DBOptions& db_options, const std::string& dbname,
+ const std::vector<ColumnFamilyDescriptor>& column_families,
+ std::vector<ColumnFamilyHandle*>* handles, DB** dbptr) {
+ Status s = SanitizeOptionsByTable(db_options, column_families);
+ if (!s.ok()) {
+ return s;
+ }
+
+ s = ValidateOptions(db_options, column_families);
+ if (!s.ok()) {
+ return s;
+ }
+
+ *dbptr = nullptr;
+ handles->clear();
+
+ size_t max_write_buffer_size = 0;
+ for (auto cf : column_families) {
+ max_write_buffer_size =
+ std::max(max_write_buffer_size, cf.options.write_buffer_size);
+ }
+
+ DBImpl* impl = new DBImpl(db_options, dbname);
+ s = impl->env_->CreateDirIfMissing(impl->immutable_db_options_.wal_dir);
+ if (s.ok()) {
+ for (auto db_path : impl->immutable_db_options_.db_paths) {
+ s = impl->env_->CreateDirIfMissing(db_path.path);
+ if (!s.ok()) {
+ break;
+ }
+ }
+ }
+
+ if (!s.ok()) {
+ delete impl;
+ return s;
+ }
+
+ s = impl->CreateArchivalDirectory();
+ if (!s.ok()) {
+ delete impl;
+ return s;
+ }
+ impl->mutex_.Lock();
+ // Handles create_if_missing, error_if_exists
+ s = impl->Recover(column_families);
+ if (s.ok()) {
+ uint64_t new_log_number = impl->versions_->NewFileNumber();
+ unique_ptr<WritableFile> lfile;
+ EnvOptions soptions(db_options);
+ EnvOptions opt_env_options =
+ impl->immutable_db_options_.env->OptimizeForLogWrite(
+ soptions, BuildDBOptions(impl->immutable_db_options_,
+ impl->mutable_db_options_));
+ s = NewWritableFile(
+ impl->immutable_db_options_.env,
+ LogFileName(impl->immutable_db_options_.wal_dir, new_log_number),
+ &lfile, opt_env_options);
+ if (s.ok()) {
+ lfile->SetPreallocationBlockSize(
+ impl->GetWalPreallocateBlockSize(max_write_buffer_size));
+ {
+ InstrumentedMutexLock wl(&impl->log_write_mutex_);
+ impl->logfile_number_ = new_log_number;
+ unique_ptr<WritableFileWriter> file_writer(
+ new WritableFileWriter(std::move(lfile), opt_env_options));
+ impl->logs_.emplace_back(
+ new_log_number,
+ new log::Writer(
+ std::move(file_writer), new_log_number,
+ impl->immutable_db_options_.recycle_log_file_num > 0));
+ }
+
+ // set column family handles
+ for (auto cf : column_families) {
+ auto cfd =
+ impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name);
+ if (cfd != nullptr) {
+ handles->push_back(
+ new ColumnFamilyHandleImpl(cfd, impl, &impl->mutex_));
+ impl->NewThreadStatusCfInfo(cfd);
+ } else {
+ if (db_options.create_missing_column_families) {
+ // missing column family, create it
+ ColumnFamilyHandle* handle;
+ impl->mutex_.Unlock();
+ s = impl->CreateColumnFamily(cf.options, cf.name, &handle);
+ impl->mutex_.Lock();
+ if (s.ok()) {
+ handles->push_back(handle);
+ } else {
+ break;
+ }
+ } else {
+ s = Status::InvalidArgument("Column family not found: ", cf.name);
+ break;
+ }
+ }
+ }
+ }
+ if (s.ok()) {
+ for (auto cfd : *impl->versions_->GetColumnFamilySet()) {
+ delete impl->InstallSuperVersionAndScheduleWork(
+ cfd, nullptr, *cfd->GetLatestMutableCFOptions());
+ }
+ if (impl->concurrent_prepare_) {
+ impl->log_write_mutex_.Lock();
+ }
+ impl->alive_log_files_.push_back(
+ DBImpl::LogFileNumberSize(impl->logfile_number_));
+ if (impl->concurrent_prepare_) {
+ impl->log_write_mutex_.Unlock();
+ }
+ impl->DeleteObsoleteFiles();
+ s = impl->directories_.GetDbDir()->Fsync();
+ }
+ }
+
+ if (s.ok()) {
+ for (auto cfd : *impl->versions_->GetColumnFamilySet()) {
+ if (cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
+ auto* vstorage = cfd->current()->storage_info();
+ for (int i = 1; i < vstorage->num_levels(); ++i) {
+ int num_files = vstorage->NumLevelFiles(i);
+ if (num_files > 0) {
+ s = Status::InvalidArgument(
+ "Not all files are at level 0. Cannot "
+ "open with FIFO compaction style.");
+ break;
+ }
+ }
+ }
+ if (!cfd->mem()->IsSnapshotSupported()) {
+ impl->is_snapshot_supported_ = false;
+ }
+ if (cfd->ioptions()->merge_operator != nullptr &&
+ !cfd->mem()->IsMergeOperatorSupported()) {
+ s = Status::InvalidArgument(
+ "The memtable of column family %s does not support merge operator "
+ "its options.merge_operator is non-null", cfd->GetName().c_str());
+ }
+ if (!s.ok()) {
+ break;
+ }
+ }
+ }
+ TEST_SYNC_POINT("DBImpl::Open:Opened");
+ Status persist_options_status;
+ if (s.ok()) {
+ // Persist RocksDB Options before scheduling the compaction.
+ // The WriteOptionsFile() will release and lock the mutex internally.
+ persist_options_status = impl->WriteOptionsFile(
+ false /*need_mutex_lock*/, false /*need_enter_write_thread*/);
+
+ *dbptr = impl;
+ impl->opened_successfully_ = true;
+ impl->MaybeScheduleFlushOrCompaction();
+ }
+ impl->mutex_.Unlock();
+
+#ifndef ROCKSDB_LITE
+ auto sfm = static_cast<SstFileManagerImpl*>(
+ impl->immutable_db_options_.sst_file_manager.get());
+ if (s.ok() && sfm) {
+ // Notify SstFileManager about all sst files that already exist in
+ // db_paths[0] when the DB is opened.
+ auto& db_path = impl->immutable_db_options_.db_paths[0];
+ std::vector<std::string> existing_files;
+ impl->immutable_db_options_.env->GetChildren(db_path.path, &existing_files);
+ for (auto& file_name : existing_files) {
+ uint64_t file_number;
+ FileType file_type;
+ std::string file_path = db_path.path + "/" + file_name;
+ if (ParseFileName(file_name, &file_number, &file_type) &&
+ file_type == kTableFile) {
+ sfm->OnAddFile(file_path);
+ }
+ }
+ }
+#endif // !ROCKSDB_LITE
+
+ if (s.ok()) {
+ ROCKS_LOG_INFO(impl->immutable_db_options_.info_log, "DB pointer %p", impl);
+ LogFlush(impl->immutable_db_options_.info_log);
+ if (!persist_options_status.ok()) {
+ s = Status::IOError(
+ "DB::Open() failed --- Unable to persist Options file",
+ persist_options_status.ToString());
+ }
+ }
+ if (!s.ok()) {
+ for (auto* h : *handles) {
+ delete h;
+ }
+ handles->clear();
+ delete impl;
+ *dbptr = nullptr;
+ }
+ return s;
+}
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_impl_readonly.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_impl_readonly.cc b/thirdparty/rocksdb/db/db_impl_readonly.cc
new file mode 100644
index 0000000..d69eecb
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_impl_readonly.cc
@@ -0,0 +1,197 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "db/db_impl_readonly.h"
+
+#include "db/compacted_db_impl.h"
+#include "db/db_impl.h"
+#include "db/db_iter.h"
+#include "db/merge_context.h"
+#include "db/range_del_aggregator.h"
+#include "monitoring/perf_context_imp.h"
+
+namespace rocksdb {
+
+#ifndef ROCKSDB_LITE
+
+DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
+ const std::string& dbname)
+ : DBImpl(db_options, dbname) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "Opening the db in read only mode");
+ LogFlush(immutable_db_options_.info_log);
+}
+
+DBImplReadOnly::~DBImplReadOnly() {
+}
+
+// Implementations of the DB interface
+Status DBImplReadOnly::Get(const ReadOptions& read_options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ PinnableSlice* pinnable_val) {
+ assert(pinnable_val != nullptr);
+ Status s;
+ SequenceNumber snapshot = versions_->LastSequence();
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ auto cfd = cfh->cfd();
+ SuperVersion* super_version = cfd->GetSuperVersion();
+ MergeContext merge_context;
+ RangeDelAggregator range_del_agg(cfd->internal_comparator(), snapshot);
+ LookupKey lkey(key, snapshot);
+ if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
+ &range_del_agg, read_options)) {
+ pinnable_val->PinSelf();
+ } else {
+ PERF_TIMER_GUARD(get_from_output_files_time);
+ super_version->current->Get(read_options, lkey, pinnable_val, &s,
+ &merge_context, &range_del_agg);
+ }
+ return s;
+}
+
+Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
+ ColumnFamilyHandle* column_family) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ auto cfd = cfh->cfd();
+ SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
+ SequenceNumber latest_snapshot = versions_->LastSequence();
+ auto db_iter = NewArenaWrappedDbIterator(
+ env_, read_options, *cfd->ioptions(),
+ (read_options.snapshot != nullptr
+ ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
+ ->number_
+ : latest_snapshot),
+ super_version->mutable_cf_options.max_sequential_skip_in_iterations,
+ super_version->version_number);
+ auto internal_iter =
+ NewInternalIterator(read_options, cfd, super_version, db_iter->GetArena(),
+ db_iter->GetRangeDelAggregator());
+ db_iter->SetIterUnderDBIter(internal_iter);
+ return db_iter;
+}
+
+Status DBImplReadOnly::NewIterators(
+ const ReadOptions& read_options,
+ const std::vector<ColumnFamilyHandle*>& column_families,
+ std::vector<Iterator*>* iterators) {
+ if (iterators == nullptr) {
+ return Status::InvalidArgument("iterators not allowed to be nullptr");
+ }
+ iterators->clear();
+ iterators->reserve(column_families.size());
+ SequenceNumber latest_snapshot = versions_->LastSequence();
+
+ for (auto cfh : column_families) {
+ auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
+ auto* sv = cfd->GetSuperVersion()->Ref();
+ auto* db_iter = NewArenaWrappedDbIterator(
+ env_, read_options, *cfd->ioptions(),
+ (read_options.snapshot != nullptr
+ ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
+ ->number_
+ : latest_snapshot),
+ sv->mutable_cf_options.max_sequential_skip_in_iterations,
+ sv->version_number);
+ auto* internal_iter =
+ NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
+ db_iter->GetRangeDelAggregator());
+ db_iter->SetIterUnderDBIter(internal_iter);
+ iterators->push_back(db_iter);
+ }
+
+ return Status::OK();
+}
+
+Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
+ DB** dbptr, bool error_if_log_file_exist) {
+ *dbptr = nullptr;
+
+ // Try to first open DB as fully compacted DB
+ Status s;
+ s = CompactedDBImpl::Open(options, dbname, dbptr);
+ if (s.ok()) {
+ return s;
+ }
+
+ DBOptions db_options(options);
+ ColumnFamilyOptions cf_options(options);
+ std::vector<ColumnFamilyDescriptor> column_families;
+ column_families.push_back(
+ ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options));
+ std::vector<ColumnFamilyHandle*> handles;
+
+ s = DB::OpenForReadOnly(db_options, dbname, column_families, &handles, dbptr);
+ if (s.ok()) {
+ assert(handles.size() == 1);
+ // i can delete the handle since DBImpl is always holding a
+ // reference to default column family
+ delete handles[0];
+ }
+ return s;
+}
+
+Status DB::OpenForReadOnly(
+ const DBOptions& db_options, const std::string& dbname,
+ const std::vector<ColumnFamilyDescriptor>& column_families,
+ std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
+ bool error_if_log_file_exist) {
+ *dbptr = nullptr;
+ handles->clear();
+
+ DBImplReadOnly* impl = new DBImplReadOnly(db_options, dbname);
+ impl->mutex_.Lock();
+ Status s = impl->Recover(column_families, true /* read only */,
+ error_if_log_file_exist);
+ if (s.ok()) {
+ // set column family handles
+ for (auto cf : column_families) {
+ auto cfd =
+ impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name);
+ if (cfd == nullptr) {
+ s = Status::InvalidArgument("Column family not found: ", cf.name);
+ break;
+ }
+ handles->push_back(new ColumnFamilyHandleImpl(cfd, impl, &impl->mutex_));
+ }
+ }
+ if (s.ok()) {
+ for (auto cfd : *impl->versions_->GetColumnFamilySet()) {
+ delete cfd->InstallSuperVersion(new SuperVersion(), &impl->mutex_);
+ }
+ }
+ impl->mutex_.Unlock();
+ if (s.ok()) {
+ *dbptr = impl;
+ for (auto* h : *handles) {
+ impl->NewThreadStatusCfInfo(
+ reinterpret_cast<ColumnFamilyHandleImpl*>(h)->cfd());
+ }
+ } else {
+ for (auto h : *handles) {
+ delete h;
+ }
+ handles->clear();
+ delete impl;
+ }
+ return s;
+}
+
+#else // !ROCKSDB_LITE
+
+Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
+ DB** dbptr, bool error_if_log_file_exist) {
+ return Status::NotSupported("Not supported in ROCKSDB_LITE.");
+}
+
+Status DB::OpenForReadOnly(
+ const DBOptions& db_options, const std::string& dbname,
+ const std::vector<ColumnFamilyDescriptor>& column_families,
+ std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
+ bool error_if_log_file_exist) {
+ return Status::NotSupported("Not supported in ROCKSDB_LITE.");
+}
+#endif // !ROCKSDB_LITE
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_impl_readonly.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_impl_readonly.h b/thirdparty/rocksdb/db/db_impl_readonly.h
new file mode 100644
index 0000000..9bdc95c
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_impl_readonly.h
@@ -0,0 +1,123 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#ifndef ROCKSDB_LITE
+
+#include "db/db_impl.h"
+#include <vector>
+#include <string>
+
+namespace rocksdb {
+
+class DBImplReadOnly : public DBImpl {
+ public:
+ DBImplReadOnly(const DBOptions& options, const std::string& dbname);
+ virtual ~DBImplReadOnly();
+
+ // Implementations of the DB interface
+ using DB::Get;
+ virtual Status Get(const ReadOptions& options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ PinnableSlice* value) override;
+
+ // TODO: Implement ReadOnly MultiGet?
+
+ using DBImpl::NewIterator;
+ virtual Iterator* NewIterator(const ReadOptions&,
+ ColumnFamilyHandle* column_family) override;
+
+ virtual Status NewIterators(
+ const ReadOptions& options,
+ const std::vector<ColumnFamilyHandle*>& column_families,
+ std::vector<Iterator*>* iterators) override;
+
+ using DBImpl::Put;
+ virtual Status Put(const WriteOptions& options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ const Slice& value) override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+ using DBImpl::Merge;
+ virtual Status Merge(const WriteOptions& options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ const Slice& value) override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+ using DBImpl::Delete;
+ virtual Status Delete(const WriteOptions& options,
+ ColumnFamilyHandle* column_family,
+ const Slice& key) override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+ using DBImpl::SingleDelete;
+ virtual Status SingleDelete(const WriteOptions& options,
+ ColumnFamilyHandle* column_family,
+ const Slice& key) override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+ virtual Status Write(const WriteOptions& options,
+ WriteBatch* updates) override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+ using DBImpl::CompactRange;
+ virtual Status CompactRange(const CompactRangeOptions& options,
+ ColumnFamilyHandle* column_family,
+ const Slice* begin, const Slice* end) override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+
+ using DBImpl::CompactFiles;
+ virtual Status CompactFiles(
+ const CompactionOptions& compact_options,
+ ColumnFamilyHandle* column_family,
+ const std::vector<std::string>& input_file_names,
+ const int output_level, const int output_path_id = -1) override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+
+ virtual Status DisableFileDeletions() override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+
+ virtual Status EnableFileDeletions(bool force) override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+ virtual Status GetLiveFiles(std::vector<std::string>&,
+ uint64_t* manifest_file_size,
+ bool flush_memtable = true) override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+
+ using DBImpl::Flush;
+ virtual Status Flush(const FlushOptions& options,
+ ColumnFamilyHandle* column_family) override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+
+ using DBImpl::SyncWAL;
+ virtual Status SyncWAL() override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+
+ using DB::IngestExternalFile;
+ virtual Status IngestExternalFile(
+ ColumnFamilyHandle* column_family,
+ const std::vector<std::string>& external_files,
+ const IngestExternalFileOptions& ingestion_options) override {
+ return Status::NotSupported("Not supported operation in read only mode.");
+ }
+
+ private:
+ friend class DB;
+
+ // No copying allowed
+ DBImplReadOnly(const DBImplReadOnly&);
+ void operator=(const DBImplReadOnly&);
+};
+}
+
+#endif // !ROCKSDB_LITE
[47/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/corruption_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/corruption_test.cc b/thirdparty/leveldb-1.18/db/corruption_test.cc
deleted file mode 100755
index 52ab7e2..0000000
--- a/thirdparty/leveldb-1.18/db/corruption_test.cc
+++ /dev/null
@@ -1,388 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/db.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include "leveldb/cache.h"
-#include "leveldb/env.h"
-#include "leveldb/table.h"
-#include "leveldb/write_batch.h"
-#include "db/db_impl.h"
-#include "db/filename.h"
-#include "db/log_format.h"
-#include "db/version_set.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-static const int kValueSize = 1000;
-
-class CorruptionTest {
- public:
- test::ErrorEnv env_;
- std::string dbname_;
- Cache* tiny_cache_;
- Options options_;
- DB* db_;
-
- CorruptionTest() {
- tiny_cache_ = NewLRUCache(100);
- options_.env = &env_;
- options_.block_cache = tiny_cache_;
- dbname_ = test::TmpDir() + "/db_test";
- DestroyDB(dbname_, options_);
-
- db_ = NULL;
- options_.create_if_missing = true;
- Reopen();
- options_.create_if_missing = false;
- }
-
- ~CorruptionTest() {
- delete db_;
- DestroyDB(dbname_, Options());
- delete tiny_cache_;
- }
-
- Status TryReopen() {
- delete db_;
- db_ = NULL;
- return DB::Open(options_, dbname_, &db_);
- }
-
- void Close() {
- delete db_;
- db_ = nullptr;
- }
-
- void Reopen() {
- ASSERT_OK(TryReopen());
- }
-
- void RepairDB() {
- delete db_;
- db_ = NULL;
- ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
- }
-
- void Build(int n) {
- std::string key_space, value_space;
- WriteBatch batch;
- for (int i = 0; i < n; i++) {
- //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
- Slice key = Key(i, &key_space);
- batch.Clear();
- batch.Put(key, Value(i, &value_space));
- WriteOptions options;
- // Corrupt() doesn't work without this sync on windows; stat reports 0 for
- // the file size.
- if (i == n - 1) {
- options.sync = true;
- }
- ASSERT_OK(db_->Write(options, &batch));
- }
- }
-
- void Check(int min_expected, int max_expected) {
- int next_expected = 0;
- int missed = 0;
- int bad_keys = 0;
- int bad_values = 0;
- int correct = 0;
- std::string value_space;
- Iterator* iter = db_->NewIterator(ReadOptions());
- for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
- uint64_t key;
- Slice in(iter->key());
- if (in == "" || in == "~") {
- // Ignore boundary keys.
- continue;
- }
- if (!ConsumeDecimalNumber(&in, &key) ||
- !in.empty() ||
- key < next_expected) {
- bad_keys++;
- continue;
- }
- missed += static_cast<int>(key - next_expected);
- next_expected = static_cast<int>(key + 1);
- if (iter->value() != Value(static_cast<int>(key), &value_space)) {
- bad_values++;
- } else {
- correct++;
- }
- }
- delete iter;
-
- fprintf(stderr,
- "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n",
- min_expected, max_expected, correct, bad_keys, bad_values, missed);
- ASSERT_LE(min_expected, correct);
- ASSERT_GE(max_expected, correct);
- }
-
- void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
- // Pick file to corrupt
- std::vector<std::string> filenames;
- ASSERT_OK(env_.GetChildren(dbname_, &filenames));
- uint64_t number;
- FileType type;
- std::string fname;
- int picked_number = -1;
- for (size_t i = 0; i < filenames.size(); i++) {
- if (ParseFileName(filenames[i], &number, &type) &&
- type == filetype &&
- int(number) > picked_number) { // Pick latest file
- fname = dbname_ + "/" + filenames[i];
- picked_number = static_cast<int>(number);
- }
- }
- ASSERT_TRUE(!fname.empty()) << filetype;
-
- struct stat sbuf;
- if (stat(fname.c_str(), &sbuf) != 0) {
- const char* msg = strerror(errno);
- ASSERT_TRUE(false) << fname << ": " << msg;
- }
-
- if (offset < 0) {
- // Relative to end of file; make it absolute
- if (-offset > sbuf.st_size) {
- offset = 0;
- } else {
- offset = sbuf.st_size + offset;
- }
- }
- if (offset > sbuf.st_size) {
- offset = sbuf.st_size;
- }
- if (offset + bytes_to_corrupt > sbuf.st_size) {
- bytes_to_corrupt = sbuf.st_size - offset;
- }
-
- // Do it
- std::string contents;
- Status s = ReadFileToString(Env::Default(), fname, &contents);
- ASSERT_TRUE(s.ok()) << s.ToString();
- for (int i = 0; i < bytes_to_corrupt; i++) {
- contents[i + offset] ^= 0x80;
- }
- s = WriteStringToFile(Env::Default(), contents, fname);
- ASSERT_TRUE(s.ok()) << s.ToString();
- }
-
- int Property(const std::string& name) {
- std::string property;
- int result;
- if (db_->GetProperty(name, &property) &&
- sscanf(property.c_str(), "%d", &result) == 1) {
- return result;
- } else {
- return -1;
- }
- }
-
- // Return the ith key
- Slice Key(int i, std::string* storage) {
- char buf[100];
- snprintf(buf, sizeof(buf), "%016d", i);
- storage->assign(buf, strlen(buf));
- return Slice(*storage);
- }
-
- // Return the value to associate with the specified key
- Slice Value(int k, std::string* storage) {
- Random r(k);
- return test::RandomString(&r, kValueSize, storage);
- }
-};
-
-TEST(CorruptionTest, Recovery) {
- Build(100);
- Check(100, 100);
- Close();
- Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record
- Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block
- Reopen();
-
- // The 64 records in the first two log blocks are completely lost.
- Check(36, 36);
-}
-
-TEST(CorruptionTest, RecoverWriteError) {
- env_.writable_file_error_ = true;
- Status s = TryReopen();
- ASSERT_TRUE(!s.ok());
-}
-
-TEST(CorruptionTest, NewFileErrorDuringWrite) {
- // Do enough writing to force minor compaction
- env_.writable_file_error_ = true;
- const int num = 3 + static_cast<int>(Options().write_buffer_size / kValueSize);
- std::string value_storage;
- Status s;
- for (int i = 0; s.ok() && i < num; i++) {
- WriteBatch batch;
- batch.Put("a", Value(100, &value_storage));
- s = db_->Write(WriteOptions(), &batch);
- }
- ASSERT_TRUE(!s.ok());
- ASSERT_GE(env_.num_writable_file_errors_, 1);
- env_.writable_file_error_ = false;
- Reopen();
-}
-
-TEST(CorruptionTest, TableFile) {
- Build(100);
- DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
- dbi->TEST_CompactMemTable();
- dbi->TEST_CompactRange(0, NULL, NULL);
- dbi->TEST_CompactRange(1, NULL, NULL);
- Close();
- Corrupt(kTableFile, 100, 1);
- Reopen();
- Check(90, 99);
-}
-
-TEST(CorruptionTest, TableFileRepair) {
- options_.block_size = 2 * kValueSize; // Limit scope of corruption
- options_.paranoid_checks = true;
- Reopen();
- Build(100);
- DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
- dbi->TEST_CompactMemTable();
- dbi->TEST_CompactRange(0, NULL, NULL);
- dbi->TEST_CompactRange(1, NULL, NULL);
- Close();
- Corrupt(kTableFile, 100, 1);
- Reopen();
- Check(95, 99);
-}
-
-TEST(CorruptionTest, TableFileIndexData) {
- Build(10000); // Enough to build multiple Tables
- DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
- dbi->TEST_CompactMemTable();
- Close();
- Corrupt(kTableFile, -2000, 500);
- Reopen();
- Check(5000, 9999);
-}
-
-TEST(CorruptionTest, MissingDescriptor) {
- Build(1000);
- RepairDB();
- Reopen();
- Check(1000, 1000);
-}
-
-TEST(CorruptionTest, SequenceNumberRecovery) {
- ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
- ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
- ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3"));
- ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4"));
- ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5"));
- RepairDB();
- Reopen();
- std::string v;
- ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
- ASSERT_EQ("v5", v);
- // Write something. If sequence number was not recovered properly,
- // it will be hidden by an earlier write.
- ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6"));
- ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
- ASSERT_EQ("v6", v);
- Reopen();
- ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
- ASSERT_EQ("v6", v);
-}
-
-TEST(CorruptionTest, CorruptedDescriptor) {
- ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
- DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
- dbi->TEST_CompactMemTable();
- dbi->TEST_CompactRange(0, NULL, NULL);
- Close();
- Corrupt(kDescriptorFile, 0, 1000);
- Status s = TryReopen();
- ASSERT_TRUE(!s.ok());
-
- RepairDB();
- Reopen();
- std::string v;
- ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
- ASSERT_EQ("hello", v);
-}
-
-TEST(CorruptionTest, CompactionInputError) {
- Build(10);
- DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
- dbi->TEST_CompactMemTable();
- const int last = config::kMaxMemCompactLevel;
- ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
- Close();
- Corrupt(kTableFile, 100, 1);
- Reopen();
- Check(5, 9);
-
- // Force compactions by writing lots of values
- Build(10000);
- Check(10000, 10000);
-
- Close();
-}
-
-TEST(CorruptionTest, CompactionInputErrorParanoid) {
- options_.paranoid_checks = true;
- options_.write_buffer_size = 512 << 10;
-
- // Make multiple inputs so we need to compact.
- for (int i = 0; i < 2; i++) {
- Reopen();
- Build(10);
- reinterpret_cast<DBImpl*>(db_)->TEST_CompactMemTable();
- Close();
- Corrupt(kTableFile, 100, 1);
- env_.SleepForMicroseconds(100000);
- }
-
- Reopen();
- reinterpret_cast<DBImpl*>(db_)->CompactRange(NULL, NULL);
-
- // Write must fail because of corrupted table
- std::string tmp1, tmp2;
- Status s = db_->Put(WriteOptions(), Key(5, &tmp1), Value(5, &tmp2));
- ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
-}
-
-TEST(CorruptionTest, UnrelatedKeys) {
- Build(10);
- DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
- dbi->TEST_CompactMemTable();
- Close();
- Corrupt(kTableFile, 100, 1);
- Reopen();
- dbi = reinterpret_cast<DBImpl*>(db_);
-
- std::string tmp1, tmp2;
- ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
- std::string v;
- ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
- ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
- dbi->TEST_CompactMemTable();
- ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
- ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/db_bench.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/db_bench.cc b/thirdparty/leveldb-1.18/db/db_bench.cc
deleted file mode 100755
index 817064a..0000000
--- a/thirdparty/leveldb-1.18/db/db_bench.cc
+++ /dev/null
@@ -1,979 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <sys/types.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include "db/db_impl.h"
-#include "db/version_set.h"
-#include "leveldb/cache.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/write_batch.h"
-#include "port/port.h"
-#include "util/crc32c.h"
-#include "util/histogram.h"
-#include "util/mutexlock.h"
-#include "util/random.h"
-#include "util/testutil.h"
-
-// Comma-separated list of operations to run in the specified order
-// Actual benchmarks:
-// fillseq -- write N values in sequential key order in async mode
-// fillrandom -- write N values in random key order in async mode
-// overwrite -- overwrite N values in random key order in async mode
-// fillsync -- write N/100 values in random key order in sync mode
-// fill100K -- write N/1000 100K values in random order in async mode
-// deleteseq -- delete N keys in sequential order
-// deleterandom -- delete N keys in random order
-// readseq -- read N times sequentially
-// readreverse -- read N times in reverse order
-// readrandom -- read N times in random order
-// readmissing -- read N missing keys in random order
-// readhot -- read N times in random order from 1% section of DB
-// seekrandom -- N random seeks
-// crc32c -- repeated crc32c of 4K of data
-// acquireload -- load N*1000 times
-// Meta operations:
-// compact -- Compact the entire DB
-// stats -- Print DB stats
-// sstables -- Print sstable info
-// heapprofile -- Dump a heap profile (if supported by this port)
-static const char* FLAGS_benchmarks =
- "fillseq,"
- "fillsync,"
- "fillrandom,"
- "overwrite,"
- "readrandom,"
- "readrandom," // Extra run to allow previous compactions to quiesce
- "readseq,"
- "readreverse,"
- "compact,"
- "readrandom,"
- "readseq,"
- "readreverse,"
- "fill100K,"
- "crc32c,"
- "snappycomp,"
- "snappyuncomp,"
- "acquireload,"
- ;
-
-// Number of key/values to place in database
-static int FLAGS_num = 1000000;
-
-// Number of read operations to do. If negative, do FLAGS_num reads.
-static int FLAGS_reads = -1;
-
-// Number of concurrent threads to run.
-static int FLAGS_threads = 1;
-
-// Size of each value
-static int FLAGS_value_size = 100;
-
-// Arrange to generate values that shrink to this fraction of
-// their original size after compression
-static double FLAGS_compression_ratio = 0.5;
-
-// Print histogram of operation timings
-static bool FLAGS_histogram = false;
-
-// Number of bytes to buffer in memtable before compacting
-// (initialized to default value by "main")
-static int FLAGS_write_buffer_size = 0;
-
-// Number of bytes to use as a cache of uncompressed data.
-// Negative means use default settings.
-static int FLAGS_cache_size = -1;
-
-// Maximum number of files to keep open at the same time (use default if == 0)
-static int FLAGS_open_files = 0;
-
-// Bloom filter bits per key.
-// Negative means use default settings.
-static int FLAGS_bloom_bits = -1;
-
-// If true, do not destroy the existing database. If you set this
-// flag and also specify a benchmark that wants a fresh database, that
-// benchmark will fail.
-static bool FLAGS_use_existing_db = false;
-
-// Use the db with the following name.
-static const char* FLAGS_db = NULL;
-
-namespace leveldb {
-
-namespace {
-
-// Helper for quickly generating random data.
-class RandomGenerator {
- private:
- std::string data_;
- int pos_;
-
- public:
- RandomGenerator() {
- // We use a limited amount of data over and over again and ensure
- // that it is larger than the compression window (32KB), and also
- // large enough to serve all typical value sizes we want to write.
- Random rnd(301);
- std::string piece;
- while (data_.size() < 1048576) {
- // Add a short fragment that is as compressible as specified
- // by FLAGS_compression_ratio.
- test::CompressibleString(&rnd, FLAGS_compression_ratio, 100, &piece);
- data_.append(piece);
- }
- pos_ = 0;
- }
-
- Slice Generate(size_t len) {
- if (pos_ + len > data_.size()) {
- pos_ = 0;
- assert(len < data_.size());
- }
- pos_ += static_cast<int>(len);
- return Slice(data_.data() + pos_ - len, len);
- }
-};
-
-static Slice TrimSpace(Slice s) {
- size_t start = 0;
- while (start < s.size() && isspace(s[start])) {
- start++;
- }
- size_t limit = s.size();
- while (limit > start && isspace(s[limit-1])) {
- limit--;
- }
- return Slice(s.data() + start, limit - start);
-}
-
-static void AppendWithSpace(std::string* str, Slice msg) {
- if (msg.empty()) return;
- if (!str->empty()) {
- str->push_back(' ');
- }
- str->append(msg.data(), msg.size());
-}
-
-class Stats {
- private:
- double start_;
- double finish_;
- double seconds_;
- int done_;
- int next_report_;
- int64_t bytes_;
- double last_op_finish_;
- Histogram hist_;
- std::string message_;
-
- public:
- Stats() { Start(); }
-
- void Start() {
- next_report_ = 100;
- last_op_finish_ = start_;
- hist_.Clear();
- done_ = 0;
- bytes_ = 0;
- seconds_ = 0;
- start_ = static_cast<double>(Env::Default()->NowMicros());
- finish_ = start_;
- message_.clear();
- }
-
- void Merge(const Stats& other) {
- hist_.Merge(other.hist_);
- done_ += other.done_;
- bytes_ += other.bytes_;
- seconds_ += other.seconds_;
- if (other.start_ < start_) start_ = other.start_;
- if (other.finish_ > finish_) finish_ = other.finish_;
-
- // Just keep the messages from one thread
- if (message_.empty()) message_ = other.message_;
- }
-
- void Stop() {
- finish_ = static_cast<double>(Env::Default()->NowMicros());
- seconds_ = (finish_ - start_) * 1e-6;
- }
-
- void AddMessage(Slice msg) {
- AppendWithSpace(&message_, msg);
- }
-
- void FinishedSingleOp() {
- if (FLAGS_histogram) {
- double now = static_cast<double>(Env::Default()->NowMicros());
- double micros = now - last_op_finish_;
- hist_.Add(micros);
- if (micros > 20000) {
- fprintf(stderr, "long op: %.1f micros%30s\r", micros, "");
- fflush(stderr);
- }
- last_op_finish_ = now;
- }
-
- done_++;
- if (done_ >= next_report_) {
- if (next_report_ < 1000) next_report_ += 100;
- else if (next_report_ < 5000) next_report_ += 500;
- else if (next_report_ < 10000) next_report_ += 1000;
- else if (next_report_ < 50000) next_report_ += 5000;
- else if (next_report_ < 100000) next_report_ += 10000;
- else if (next_report_ < 500000) next_report_ += 50000;
- else next_report_ += 100000;
- fprintf(stderr, "... finished %d ops%30s\r", done_, "");
- fflush(stderr);
- }
- }
-
- void AddBytes(int64_t n) {
- bytes_ += n;
- }
-
- void Report(const Slice& name) {
- // Pretend at least one op was done in case we are running a benchmark
- // that does not call FinishedSingleOp().
- if (done_ < 1) done_ = 1;
-
- std::string extra;
- if (bytes_ > 0) {
- // Rate is computed on actual elapsed time, not the sum of per-thread
- // elapsed times.
- double elapsed = (finish_ - start_) * 1e-6;
- char rate[100];
- snprintf(rate, sizeof(rate), "%6.1f MB/s",
- (bytes_ / 1048576.0) / elapsed);
- extra = rate;
- }
- AppendWithSpace(&extra, message_);
-
- fprintf(stdout, "%-12s : %11.3f micros/op;%s%s\n",
- name.ToString().c_str(),
- seconds_ * 1e6 / done_,
- (extra.empty() ? "" : " "),
- extra.c_str());
- if (FLAGS_histogram) {
- fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
- }
- fflush(stdout);
- }
-};
-
-// State shared by all concurrent executions of the same benchmark.
-struct SharedState {
- port::Mutex mu;
- port::CondVar cv;
- int total;
-
- // Each thread goes through the following states:
- // (1) initializing
- // (2) waiting for others to be initialized
- // (3) running
- // (4) done
-
- int num_initialized;
- int num_done;
- bool start;
-
- SharedState() : cv(&mu) { }
-};
-
-// Per-thread state for concurrent executions of the same benchmark.
-struct ThreadState {
- int tid; // 0..n-1 when running in n threads
- Random rand; // Has different seeds for different threads
- Stats stats;
- SharedState* shared;
-
- ThreadState(int index)
- : tid(index),
- rand(1000 + index) {
- }
-};
-
-} // namespace
-
-class Benchmark {
- private:
- Cache* cache_;
- const FilterPolicy* filter_policy_;
- DB* db_;
- int num_;
- int value_size_;
- int entries_per_batch_;
- WriteOptions write_options_;
- int reads_;
- int heap_counter_;
-
- void PrintHeader() {
- const int kKeySize = 16;
- PrintEnvironment();
- fprintf(stdout, "Keys: %d bytes each\n", kKeySize);
- fprintf(stdout, "Values: %d bytes each (%d bytes after compression)\n",
- FLAGS_value_size,
- static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
- fprintf(stdout, "Entries: %d\n", num_);
- fprintf(stdout, "RawSize: %.1f MB (estimated)\n",
- ((static_cast<int64_t>(kKeySize + FLAGS_value_size) * num_)
- / 1048576.0));
- fprintf(stdout, "FileSize: %.1f MB (estimated)\n",
- (((kKeySize + FLAGS_value_size * FLAGS_compression_ratio) * num_)
- / 1048576.0));
- PrintWarnings();
- fprintf(stdout, "------------------------------------------------\n");
- }
-
- void PrintWarnings() {
-#if defined(__GNUC__) && !defined(__OPTIMIZE__)
- fprintf(stdout,
- "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
- );
-#endif
-#ifndef NDEBUG
- fprintf(stdout,
- "WARNING: Assertions are enabled; benchmarks unnecessarily slow\n");
-#endif
-
- // See if snappy is working by attempting to compress a compressible string
- const char text[] = "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy";
- std::string compressed;
- if (!port::Snappy_Compress(text, sizeof(text), &compressed)) {
- fprintf(stdout, "WARNING: Snappy compression is not enabled\n");
- } else if (compressed.size() >= sizeof(text)) {
- fprintf(stdout, "WARNING: Snappy compression is not effective\n");
- }
- }
-
- void PrintEnvironment() {
- fprintf(stderr, "LevelDB: version %d.%d\n",
- kMajorVersion, kMinorVersion);
-
-#if defined(__linux)
- time_t now = time(NULL);
- fprintf(stderr, "Date: %s", ctime(&now)); // ctime() adds newline
-
- FILE* cpuinfo = fopen("/proc/cpuinfo", "r");
- if (cpuinfo != NULL) {
- char line[1000];
- int num_cpus = 0;
- std::string cpu_type;
- std::string cache_size;
- while (fgets(line, sizeof(line), cpuinfo) != NULL) {
- const char* sep = strchr(line, ':');
- if (sep == NULL) {
- continue;
- }
- Slice key = TrimSpace(Slice(line, sep - 1 - line));
- Slice val = TrimSpace(Slice(sep + 1));
- if (key == "model name") {
- ++num_cpus;
- cpu_type = val.ToString();
- } else if (key == "cache size") {
- cache_size = val.ToString();
- }
- }
- fclose(cpuinfo);
- fprintf(stderr, "CPU: %d * %s\n", num_cpus, cpu_type.c_str());
- fprintf(stderr, "CPUCache: %s\n", cache_size.c_str());
- }
-#endif
- }
-
- public:
- Benchmark()
- : cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : NULL),
- filter_policy_(FLAGS_bloom_bits >= 0
- ? NewBloomFilterPolicy(FLAGS_bloom_bits)
- : NULL),
- db_(NULL),
- num_(FLAGS_num),
- value_size_(FLAGS_value_size),
- entries_per_batch_(1),
- reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
- heap_counter_(0) {
- std::vector<std::string> files;
- Env::Default()->GetChildren(FLAGS_db, &files);
- for (size_t i = 0; i < files.size(); i++) {
- if (Slice(files[i]).starts_with("heap-")) {
- Env::Default()->DeleteFile(std::string(FLAGS_db) + "/" + files[i]);
- }
- }
- if (!FLAGS_use_existing_db) {
- DestroyDB(FLAGS_db, Options());
- }
- }
-
- ~Benchmark() {
- delete db_;
- delete cache_;
- delete filter_policy_;
- }
-
- void Run() {
- PrintHeader();
- Open();
-
- const char* benchmarks = FLAGS_benchmarks;
- while (benchmarks != NULL) {
- const char* sep = strchr(benchmarks, ',');
- Slice name;
- if (sep == NULL) {
- name = benchmarks;
- benchmarks = NULL;
- } else {
- name = Slice(benchmarks, sep - benchmarks);
- benchmarks = sep + 1;
- }
-
- // Reset parameters that may be overridden below
- num_ = FLAGS_num;
- reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads);
- value_size_ = FLAGS_value_size;
- entries_per_batch_ = 1;
- write_options_ = WriteOptions();
-
- void (Benchmark::*method)(ThreadState*) = NULL;
- bool fresh_db = false;
- int num_threads = FLAGS_threads;
-
- if (name == Slice("fillseq")) {
- fresh_db = true;
- method = &Benchmark::WriteSeq;
- } else if (name == Slice("fillbatch")) {
- fresh_db = true;
- entries_per_batch_ = 1000;
- method = &Benchmark::WriteSeq;
- } else if (name == Slice("fillrandom")) {
- fresh_db = true;
- method = &Benchmark::WriteRandom;
- } else if (name == Slice("overwrite")) {
- fresh_db = false;
- method = &Benchmark::WriteRandom;
- } else if (name == Slice("fillsync")) {
- fresh_db = true;
- num_ /= 1000;
- write_options_.sync = true;
- method = &Benchmark::WriteRandom;
- } else if (name == Slice("fill100K")) {
- fresh_db = true;
- num_ /= 1000;
- value_size_ = 100 * 1000;
- method = &Benchmark::WriteRandom;
- } else if (name == Slice("readseq")) {
- method = &Benchmark::ReadSequential;
- } else if (name == Slice("readreverse")) {
- method = &Benchmark::ReadReverse;
- } else if (name == Slice("readrandom")) {
- method = &Benchmark::ReadRandom;
- } else if (name == Slice("readmissing")) {
- method = &Benchmark::ReadMissing;
- } else if (name == Slice("seekrandom")) {
- method = &Benchmark::SeekRandom;
- } else if (name == Slice("readhot")) {
- method = &Benchmark::ReadHot;
- } else if (name == Slice("readrandomsmall")) {
- reads_ /= 1000;
- method = &Benchmark::ReadRandom;
- } else if (name == Slice("deleteseq")) {
- method = &Benchmark::DeleteSeq;
- } else if (name == Slice("deleterandom")) {
- method = &Benchmark::DeleteRandom;
- } else if (name == Slice("readwhilewriting")) {
- num_threads++; // Add extra thread for writing
- method = &Benchmark::ReadWhileWriting;
- } else if (name == Slice("compact")) {
- method = &Benchmark::Compact;
- } else if (name == Slice("crc32c")) {
- method = &Benchmark::Crc32c;
- } else if (name == Slice("acquireload")) {
- method = &Benchmark::AcquireLoad;
- } else if (name == Slice("snappycomp")) {
- method = &Benchmark::SnappyCompress;
- } else if (name == Slice("snappyuncomp")) {
- method = &Benchmark::SnappyUncompress;
- } else if (name == Slice("heapprofile")) {
- HeapProfile();
- } else if (name == Slice("stats")) {
- PrintStats("leveldb.stats");
- } else if (name == Slice("sstables")) {
- PrintStats("leveldb.sstables");
- } else {
- if (name != Slice()) { // No error message for empty name
- fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
- }
- }
-
- if (fresh_db) {
- if (FLAGS_use_existing_db) {
- fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n",
- name.ToString().c_str());
- method = NULL;
- } else {
- delete db_;
- db_ = NULL;
- DestroyDB(FLAGS_db, Options());
- Open();
- }
- }
-
- if (method != NULL) {
- RunBenchmark(num_threads, name, method);
- }
- }
- }
-
- private:
- struct ThreadArg {
- Benchmark* bm;
- SharedState* shared;
- ThreadState* thread;
- void (Benchmark::*method)(ThreadState*);
- };
-
- static void ThreadBody(void* v) {
- ThreadArg* arg = reinterpret_cast<ThreadArg*>(v);
- SharedState* shared = arg->shared;
- ThreadState* thread = arg->thread;
- {
- MutexLock l(&shared->mu);
- shared->num_initialized++;
- if (shared->num_initialized >= shared->total) {
- shared->cv.SignalAll();
- }
- while (!shared->start) {
- shared->cv.Wait();
- }
- }
-
- thread->stats.Start();
- (arg->bm->*(arg->method))(thread);
- thread->stats.Stop();
-
- {
- MutexLock l(&shared->mu);
- shared->num_done++;
- if (shared->num_done >= shared->total) {
- shared->cv.SignalAll();
- }
- }
- }
-
- void RunBenchmark(int n, Slice name,
- void (Benchmark::*method)(ThreadState*)) {
- SharedState shared;
- shared.total = n;
- shared.num_initialized = 0;
- shared.num_done = 0;
- shared.start = false;
-
- ThreadArg* arg = new ThreadArg[n];
- for (int i = 0; i < n; i++) {
- arg[i].bm = this;
- arg[i].method = method;
- arg[i].shared = &shared;
- arg[i].thread = new ThreadState(i);
- arg[i].thread->shared = &shared;
- std::thread t(std::bind(&ThreadBody, &arg[i]));
- t.detach();
- }
-
- shared.mu.Lock();
- while (shared.num_initialized < n) {
- shared.cv.Wait();
- }
-
- shared.start = true;
- shared.cv.SignalAll();
- while (shared.num_done < n) {
- shared.cv.Wait();
- }
- shared.mu.Unlock();
-
- for (int i = 1; i < n; i++) {
- arg[0].thread->stats.Merge(arg[i].thread->stats);
- }
- arg[0].thread->stats.Report(name);
-
- for (int i = 0; i < n; i++) {
- delete arg[i].thread;
- }
- delete[] arg;
- }
-
- void Crc32c(ThreadState* thread) {
- // Checksum about 500MB of data total
- const int size = 4096;
- const char* label = "(4K per op)";
- std::string data(size, 'x');
- int64_t bytes = 0;
- uint32_t crc = 0;
- while (bytes < 500 * 1048576) {
- crc = crc32c::Value(data.data(), size);
- thread->stats.FinishedSingleOp();
- bytes += size;
- }
- // Print so result is not dead
- fprintf(stderr, "... crc=0x%x\r", static_cast<unsigned int>(crc));
-
- thread->stats.AddBytes(bytes);
- thread->stats.AddMessage(label);
- }
-
- void AcquireLoad(ThreadState* thread) {
- int dummy;
- port::AtomicPointer ap(&dummy);
- int count = 0;
- void *ptr = NULL;
- thread->stats.AddMessage("(each op is 1000 loads)");
- while (count < 100000) {
- for (int i = 0; i < 1000; i++) {
- ptr = ap.Acquire_Load();
- }
- count++;
- thread->stats.FinishedSingleOp();
- }
- if (ptr == NULL) exit(1); // Disable unused variable warning.
- }
-
- void SnappyCompress(ThreadState* thread) {
- RandomGenerator gen;
- Slice input = gen.Generate(Options().block_size);
- int64_t bytes = 0;
- int64_t produced = 0;
- bool ok = true;
- std::string compressed;
- while (ok && bytes < 1024 * 1048576) { // Compress 1G
- ok = port::Snappy_Compress(input.data(), input.size(), &compressed);
- produced += compressed.size();
- bytes += input.size();
- thread->stats.FinishedSingleOp();
- }
-
- if (!ok) {
- thread->stats.AddMessage("(snappy failure)");
- } else {
- char buf[100];
- snprintf(buf, sizeof(buf), "(output: %.1f%%)",
- (produced * 100.0) / bytes);
- thread->stats.AddMessage(buf);
- thread->stats.AddBytes(bytes);
- }
- }
-
- void SnappyUncompress(ThreadState* thread) {
- RandomGenerator gen;
- Slice input = gen.Generate(Options().block_size);
- std::string compressed;
- bool ok = port::Snappy_Compress(input.data(), input.size(), &compressed);
- int64_t bytes = 0;
- char* uncompressed = new char[input.size()];
- while (ok && bytes < 1024 * 1048576) { // Compress 1G
- ok = port::Snappy_Uncompress(compressed.data(), compressed.size(),
- uncompressed);
- bytes += input.size();
- thread->stats.FinishedSingleOp();
- }
- delete[] uncompressed;
-
- if (!ok) {
- thread->stats.AddMessage("(snappy failure)");
- } else {
- thread->stats.AddBytes(bytes);
- }
- }
-
- void Open() {
- assert(db_ == NULL);
- Options options;
- options.create_if_missing = !FLAGS_use_existing_db;
- options.block_cache = cache_;
- options.write_buffer_size = FLAGS_write_buffer_size;
- options.max_open_files = FLAGS_open_files;
- options.filter_policy = filter_policy_;
- Status s = DB::Open(options, FLAGS_db, &db_);
- if (!s.ok()) {
- fprintf(stderr, "open error: %s\n", s.ToString().c_str());
- exit(1);
- }
- }
-
- void WriteSeq(ThreadState* thread) {
- DoWrite(thread, true);
- }
-
- void WriteRandom(ThreadState* thread) {
- DoWrite(thread, false);
- }
-
- void DoWrite(ThreadState* thread, bool seq) {
- if (num_ != FLAGS_num) {
- char msg[100];
- snprintf(msg, sizeof(msg), "(%d ops)", num_);
- thread->stats.AddMessage(msg);
- }
-
- RandomGenerator gen;
- WriteBatch batch;
- Status s;
- int64_t bytes = 0;
- for (int i = 0; i < num_; i += entries_per_batch_) {
- batch.Clear();
- for (int j = 0; j < entries_per_batch_; j++) {
- const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
- char key[100];
- snprintf(key, sizeof(key), "%016d", k);
- batch.Put(key, gen.Generate(value_size_));
- bytes += value_size_ + strlen(key);
- thread->stats.FinishedSingleOp();
- }
- s = db_->Write(write_options_, &batch);
- if (!s.ok()) {
- fprintf(stderr, "put error: %s\n", s.ToString().c_str());
- exit(1);
- }
- }
- thread->stats.AddBytes(bytes);
- }
-
- void ReadSequential(ThreadState* thread) {
- Iterator* iter = db_->NewIterator(ReadOptions());
- int i = 0;
- int64_t bytes = 0;
- for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) {
- bytes += iter->key().size() + iter->value().size();
- thread->stats.FinishedSingleOp();
- ++i;
- }
- delete iter;
- thread->stats.AddBytes(bytes);
- }
-
- void ReadReverse(ThreadState* thread) {
- Iterator* iter = db_->NewIterator(ReadOptions());
- int i = 0;
- int64_t bytes = 0;
- for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) {
- bytes += iter->key().size() + iter->value().size();
- thread->stats.FinishedSingleOp();
- ++i;
- }
- delete iter;
- thread->stats.AddBytes(bytes);
- }
-
- void ReadRandom(ThreadState* thread) {
- ReadOptions options;
- std::string value;
- int found = 0;
- for (int i = 0; i < reads_; i++) {
- char key[100];
- const int k = thread->rand.Next() % FLAGS_num;
- snprintf(key, sizeof(key), "%016d", k);
- if (db_->Get(options, key, &value).ok()) {
- found++;
- }
- thread->stats.FinishedSingleOp();
- }
- char msg[100];
- snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_);
- thread->stats.AddMessage(msg);
- }
-
- void ReadMissing(ThreadState* thread) {
- ReadOptions options;
- std::string value;
- for (int i = 0; i < reads_; i++) {
- char key[100];
- const int k = thread->rand.Next() % FLAGS_num;
- snprintf(key, sizeof(key), "%016d.", k);
- db_->Get(options, key, &value);
- thread->stats.FinishedSingleOp();
- }
- }
-
- void ReadHot(ThreadState* thread) {
- ReadOptions options;
- std::string value;
- const int range = (FLAGS_num + 99) / 100;
- for (int i = 0; i < reads_; i++) {
- char key[100];
- const int k = thread->rand.Next() % range;
- snprintf(key, sizeof(key), "%016d", k);
- db_->Get(options, key, &value);
- thread->stats.FinishedSingleOp();
- }
- }
-
- void SeekRandom(ThreadState* thread) {
- ReadOptions options;
- int found = 0;
- for (int i = 0; i < reads_; i++) {
- Iterator* iter = db_->NewIterator(options);
- char key[100];
- const int k = thread->rand.Next() % FLAGS_num;
- snprintf(key, sizeof(key), "%016d", k);
- iter->Seek(key);
- if (iter->Valid() && iter->key() == key) found++;
- delete iter;
- thread->stats.FinishedSingleOp();
- }
- char msg[100];
- snprintf(msg, sizeof(msg), "(%d of %d found)", found, num_);
- thread->stats.AddMessage(msg);
- }
-
- void DoDelete(ThreadState* thread, bool seq) {
- RandomGenerator gen;
- WriteBatch batch;
- Status s;
- for (int i = 0; i < num_; i += entries_per_batch_) {
- batch.Clear();
- for (int j = 0; j < entries_per_batch_; j++) {
- const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
- char key[100];
- snprintf(key, sizeof(key), "%016d", k);
- batch.Delete(key);
- thread->stats.FinishedSingleOp();
- }
- s = db_->Write(write_options_, &batch);
- if (!s.ok()) {
- fprintf(stderr, "del error: %s\n", s.ToString().c_str());
- exit(1);
- }
- }
- }
-
- void DeleteSeq(ThreadState* thread) {
- DoDelete(thread, true);
- }
-
- void DeleteRandom(ThreadState* thread) {
- DoDelete(thread, false);
- }
-
- void ReadWhileWriting(ThreadState* thread) {
- if (thread->tid > 0) {
- ReadRandom(thread);
- } else {
- // Special thread that keeps writing until other threads are done.
- RandomGenerator gen;
- while (true) {
- {
- MutexLock l(&thread->shared->mu);
- if (thread->shared->num_done + 1 >= thread->shared->num_initialized) {
- // Other threads have finished
- break;
- }
- }
-
- const int k = thread->rand.Next() % FLAGS_num;
- char key[100];
- snprintf(key, sizeof(key), "%016d", k);
- Status s = db_->Put(write_options_, key, gen.Generate(value_size_));
- if (!s.ok()) {
- fprintf(stderr, "put error: %s\n", s.ToString().c_str());
- exit(1);
- }
- }
-
- // Do not count any of the preceding work/delay in stats.
- thread->stats.Start();
- }
- }
-
- void Compact(ThreadState* thread) {
- db_->CompactRange(NULL, NULL);
- }
-
- void PrintStats(const char* key) {
- std::string stats;
- if (!db_->GetProperty(key, &stats)) {
- stats = "(failed)";
- }
- fprintf(stdout, "\n%s\n", stats.c_str());
- }
-
- static void WriteToFile(void* arg, const char* buf, int n) {
- reinterpret_cast<WritableFile*>(arg)->Append(Slice(buf, n));
- }
-
- void HeapProfile() {
- char fname[100];
- snprintf(fname, sizeof(fname), "%s/heap-%04d", FLAGS_db, ++heap_counter_);
- WritableFile* file;
- Status s = Env::Default()->NewWritableFile(fname, &file);
- if (!s.ok()) {
- fprintf(stderr, "%s\n", s.ToString().c_str());
- return;
- }
- bool ok = port::GetHeapProfile(WriteToFile, file);
- delete file;
- if (!ok) {
- fprintf(stderr, "heap profiling not supported\n");
- Env::Default()->DeleteFile(fname);
- }
- }
-};
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- FLAGS_write_buffer_size = static_cast<int>(leveldb::Options().write_buffer_size);
- FLAGS_open_files = leveldb::Options().max_open_files;
- std::string default_db_path;
-
- for (int i = 1; i < argc; i++) {
- double d;
- int n;
- char junk;
- if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
- FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
- } else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
- FLAGS_compression_ratio = d;
- } else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
- (n == 0 || n == 1)) {
- FLAGS_histogram = n;
- } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
- (n == 0 || n == 1)) {
- FLAGS_use_existing_db = n;
- } else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
- FLAGS_num = n;
- } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
- FLAGS_reads = n;
- } else if (sscanf(argv[i], "--threads=%d%c", &n, &junk) == 1) {
- FLAGS_threads = n;
- } else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
- FLAGS_value_size = n;
- } else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
- FLAGS_write_buffer_size = n;
- } else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) {
- FLAGS_cache_size = n;
- } else if (sscanf(argv[i], "--bloom_bits=%d%c", &n, &junk) == 1) {
- FLAGS_bloom_bits = n;
- } else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) {
- FLAGS_open_files = n;
- } else if (strncmp(argv[i], "--db=", 5) == 0) {
- FLAGS_db = argv[i] + 5;
- } else {
- fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
- exit(1);
- }
- }
-
- // Choose a location for the test database if none given with --db=<path>
- if (FLAGS_db == NULL) {
- leveldb::Env::Default()->GetTestDirectory(&default_db_path);
- default_db_path += "/dbbench";
- FLAGS_db = default_db_path.c_str();
- }
-
- leveldb::Benchmark benchmark;
- benchmark.Run();
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/db_impl.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/db_impl.cc b/thirdparty/leveldb-1.18/db/db_impl.cc
deleted file mode 100755
index a9a4db7..0000000
--- a/thirdparty/leveldb-1.18/db/db_impl.cc
+++ /dev/null
@@ -1,1529 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/db_impl.h"
-
-#include <algorithm>
-#include <set>
-#include <string>
-#include <stdint.h>
-#include <stdio.h>
-#include <vector>
-#include "db/builder.h"
-#include "db/db_iter.h"
-#include "db/dbformat.h"
-#include "db/filename.h"
-#include "db/log_reader.h"
-#include "db/log_writer.h"
-#include "db/memtable.h"
-#include "db/table_cache.h"
-#include "db/version_set.h"
-#include "db/write_batch_internal.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/status.h"
-#include "leveldb/table.h"
-#include "leveldb/table_builder.h"
-#include "port/port.h"
-#include "table/block.h"
-#include "table/merger.h"
-#include "table/two_level_iterator.h"
-#include "util/coding.h"
-#include "util/logging.h"
-#include "util/mutexlock.h"
-
-namespace leveldb {
-
-const int kNumNonTableCacheFiles = 10;
-
-// Information kept for every waiting writer
-struct DBImpl::Writer {
- Status status;
- WriteBatch* batch;
- bool sync;
- bool done;
- port::CondVar cv;
-
- explicit Writer(port::Mutex* mu) : cv(mu) { }
-};
-
-struct DBImpl::CompactionState {
- Compaction* const compaction;
-
- // Sequence numbers < smallest_snapshot are not significant since we
- // will never have to service a snapshot below smallest_snapshot.
- // Therefore if we have seen a sequence number S <= smallest_snapshot,
- // we can drop all entries for the same key with sequence numbers < S.
- SequenceNumber smallest_snapshot;
-
- // Files produced by compaction
- struct Output {
- uint64_t number;
- uint64_t file_size;
- InternalKey smallest, largest;
- };
- std::vector<Output> outputs;
-
- // State kept for output being generated
- WritableFile* outfile;
- TableBuilder* builder;
-
- uint64_t total_bytes;
-
- Output* current_output() { return &outputs[outputs.size()-1]; }
-
- explicit CompactionState(Compaction* c)
- : compaction(c),
- outfile(NULL),
- builder(NULL),
- total_bytes(0) {
- }
-};
-
-// Fix user-supplied options to be reasonable
-template <class T,class V>
-static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
- if (static_cast<V>(*ptr) > maxvalue) *ptr = maxvalue;
- if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
-}
-Options SanitizeOptions(const std::string& dbname,
- const InternalKeyComparator* icmp,
- const InternalFilterPolicy* ipolicy,
- const Options& src) {
- Options result = src;
- result.comparator = icmp;
- result.filter_policy = (src.filter_policy != NULL) ? ipolicy : NULL;
- ClipToRange(&result.max_open_files, 64 + kNumNonTableCacheFiles, 50000);
- ClipToRange(&result.write_buffer_size, 64<<10, 1<<30);
- ClipToRange(&result.block_size, 1<<10, 4<<20);
- if (result.info_log == NULL) {
- // Open a log file in the same directory as the db
- src.env->CreateDir(dbname); // In case it does not exist
- src.env->RenameFile(InfoLogFileName(dbname), OldInfoLogFileName(dbname));
- Status s = src.env->NewLogger(InfoLogFileName(dbname), &result.info_log);
- if (!s.ok()) {
- // No place suitable for logging
- result.info_log = NULL;
- }
- }
- if (result.block_cache == NULL) {
- result.block_cache = NewLRUCache(8 << 20);
- }
- return result;
-}
-
-DBImpl::DBImpl(const Options& raw_options, const std::string& dbname)
- : env_(raw_options.env),
- internal_comparator_(raw_options.comparator),
- internal_filter_policy_(raw_options.filter_policy),
- options_(SanitizeOptions(dbname, &internal_comparator_,
- &internal_filter_policy_, raw_options)),
- owns_info_log_(options_.info_log != raw_options.info_log),
- owns_cache_(options_.block_cache != raw_options.block_cache),
- dbname_(dbname),
- db_lock_(NULL),
- shutting_down_(NULL),
- bg_cv_(&mutex_),
- mem_(new MemTable(internal_comparator_)),
- imm_(NULL),
- logfile_(NULL),
- logfile_number_(0),
- log_(NULL),
- seed_(0),
- tmp_batch_(new WriteBatch),
- bg_compaction_scheduled_(false),
- manual_compaction_(NULL) {
- mem_->Ref();
- has_imm_.Release_Store(NULL);
-
- // Reserve ten files or so for other uses and give the rest to TableCache.
- const int table_cache_size = options_.max_open_files - kNumNonTableCacheFiles;
- table_cache_ = new TableCache(dbname_, &options_, table_cache_size);
-
- versions_ = new VersionSet(dbname_, &options_, table_cache_,
- &internal_comparator_);
-}
-
-DBImpl::~DBImpl() {
- // Wait for background work to finish
- mutex_.Lock();
- shutting_down_.Release_Store(this); // Any non-NULL value is ok
- while (bg_compaction_scheduled_) {
- bg_cv_.Wait();
- }
- mutex_.Unlock();
-
- if (db_lock_ != NULL) {
- env_->UnlockFile(db_lock_);
- }
-
- delete versions_;
- if (mem_ != NULL) mem_->Unref();
- if (imm_ != NULL) imm_->Unref();
- delete tmp_batch_;
- delete log_;
- delete logfile_;
- delete table_cache_;
-
- if (owns_info_log_) {
- delete options_.info_log;
- }
- if (owns_cache_) {
- delete options_.block_cache;
- }
-}
-
-Status DBImpl::NewDB() {
- VersionEdit new_db;
- new_db.SetComparatorName(user_comparator()->Name());
- new_db.SetLogNumber(0);
- new_db.SetNextFile(2);
- new_db.SetLastSequence(0);
-
- const std::string manifest = DescriptorFileName(dbname_, 1);
- WritableFile* file;
- Status s = env_->NewWritableFile(manifest, &file);
- if (!s.ok()) {
- return s;
- }
- {
- log::Writer log(file);
- std::string record;
- new_db.EncodeTo(&record);
- s = log.AddRecord(record);
- if (s.ok()) {
- s = file->Close();
- }
- }
- delete file;
- if (s.ok()) {
- // Make "CURRENT" file that points to the new manifest file.
- s = SetCurrentFile(env_, dbname_, 1);
- } else {
- env_->DeleteFile(manifest);
- }
- return s;
-}
-
-void DBImpl::MaybeIgnoreError(Status* s) const {
- if (s->ok() || options_.paranoid_checks) {
- // No change needed
- } else {
- Log(options_.info_log, "Ignoring error %s", s->ToString().c_str());
- *s = Status::OK();
- }
-}
-
-void DBImpl::DeleteObsoleteFiles() {
- if (!bg_error_.ok()) {
- // After a background error, we don't know whether a new version may
- // or may not have been committed, so we cannot safely garbage collect.
- return;
- }
-
- // Make a set of all of the live files
- std::set<uint64_t> live = pending_outputs_;
- versions_->AddLiveFiles(&live);
-
- std::vector<std::string> filenames;
- env_->GetChildren(dbname_, &filenames); // Ignoring errors on purpose
- uint64_t number;
- FileType type;
- for (size_t i = 0; i < filenames.size(); i++) {
- if (ParseFileName(filenames[i], &number, &type)) {
- bool keep = true;
- switch (type) {
- case kLogFile:
- keep = ((number >= versions_->LogNumber()) ||
- (number == versions_->PrevLogNumber()));
- break;
- case kDescriptorFile:
- // Keep my manifest file, and any newer incarnations'
- // (in case there is a race that allows other incarnations)
- keep = (number >= versions_->ManifestFileNumber());
- break;
- case kTableFile:
- keep = (live.find(number) != live.end());
- break;
- case kTempFile:
- // Any temp files that are currently being written to must
- // be recorded in pending_outputs_, which is inserted into "live"
- keep = (live.find(number) != live.end());
- break;
- case kCurrentFile:
- case kDBLockFile:
- case kInfoLogFile:
- keep = true;
- break;
- }
-
- if (!keep) {
- if (type == kTableFile) {
- table_cache_->Evict(number);
- }
- Log(options_.info_log, "Delete type=%d #%lld\n",
- int(type),
- static_cast<unsigned long long>(number));
- env_->DeleteFile(dbname_ + "/" + filenames[i]);
- }
- }
- }
-}
-
-Status DBImpl::Recover(VersionEdit* edit) {
- mutex_.AssertHeld();
-
- // Ignore error from CreateDir since the creation of the DB is
- // committed only when the descriptor is created, and this directory
- // may already exist from a previous failed creation attempt.
- env_->CreateDir(dbname_);
- assert(db_lock_ == NULL);
- Status s = env_->LockFile(LockFileName(dbname_), &db_lock_);
- if (!s.ok()) {
- return s;
- }
-
- if (!env_->FileExists(CurrentFileName(dbname_))) {
- if (options_.create_if_missing) {
- s = NewDB();
- if (!s.ok()) {
- return s;
- }
- } else {
- return Status::InvalidArgument(
- dbname_, "does not exist (create_if_missing is false)");
- }
- } else {
- if (options_.error_if_exists) {
- return Status::InvalidArgument(
- dbname_, "exists (error_if_exists is true)");
- }
- }
-
- s = versions_->Recover();
- if (s.ok()) {
- SequenceNumber max_sequence(0);
-
- // Recover from all newer log files than the ones named in the
- // descriptor (new log files may have been added by the previous
- // incarnation without registering them in the descriptor).
- //
- // Note that PrevLogNumber() is no longer used, but we pay
- // attention to it in case we are recovering a database
- // produced by an older version of leveldb.
- const uint64_t min_log = versions_->LogNumber();
- const uint64_t prev_log = versions_->PrevLogNumber();
- std::vector<std::string> filenames;
- s = env_->GetChildren(dbname_, &filenames);
- if (!s.ok()) {
- return s;
- }
- std::set<uint64_t> expected;
- versions_->AddLiveFiles(&expected);
- uint64_t number;
- FileType type;
- std::vector<uint64_t> logs;
- for (size_t i = 0; i < filenames.size(); i++) {
- if (ParseFileName(filenames[i], &number, &type)) {
- expected.erase(number);
- if (type == kLogFile && ((number >= min_log) || (number == prev_log)))
- logs.push_back(number);
- }
- }
- if (!expected.empty()) {
- char buf[50];
- snprintf(buf, sizeof(buf), "%d missing files; e.g.",
- static_cast<int>(expected.size()));
- return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin())));
- }
-
- // Recover in the order in which the logs were generated
- std::sort(logs.begin(), logs.end());
- for (size_t i = 0; i < logs.size(); i++) {
- s = RecoverLogFile(logs[i], edit, &max_sequence);
-
- // The previous incarnation may not have written any MANIFEST
- // records after allocating this log number. So we manually
- // update the file number allocation counter in VersionSet.
- versions_->MarkFileNumberUsed(logs[i]);
- }
-
- if (s.ok()) {
- if (versions_->LastSequence() < max_sequence) {
- versions_->SetLastSequence(max_sequence);
- }
- }
- }
-
- return s;
-}
-
-Status DBImpl::RecoverLogFile(uint64_t log_number,
- VersionEdit* edit,
- SequenceNumber* max_sequence) {
- struct LogReporter : public log::Reader::Reporter {
- Env* env;
- Logger* info_log;
- const char* fname;
- Status* status; // NULL if options_.paranoid_checks==false
- virtual void Corruption(size_t bytes, const Status& s) {
- Log(info_log, "%s%s: dropping %d bytes; %s",
- (this->status == NULL ? "(ignoring error) " : ""),
- fname, static_cast<int>(bytes), s.ToString().c_str());
- if (this->status != NULL && this->status->ok()) *this->status = s;
- }
- };
-
- mutex_.AssertHeld();
-
- // Open the log file
- std::string fname = LogFileName(dbname_, log_number);
- SequentialFile* file;
- Status status = env_->NewSequentialFile(fname, &file);
- if (!status.ok()) {
- MaybeIgnoreError(&status);
- return status;
- }
-
- // Create the log reader.
- LogReporter reporter;
- reporter.env = env_;
- reporter.info_log = options_.info_log;
- reporter.fname = fname.c_str();
- reporter.status = (options_.paranoid_checks ? &status : NULL);
- // We intentionally make log::Reader do checksumming even if
- // paranoid_checks==false so that corruptions cause entire commits
- // to be skipped instead of propagating bad information (like overly
- // large sequence numbers).
- log::Reader reader(file, &reporter, true/*checksum*/,
- 0/*initial_offset*/);
- Log(options_.info_log, "Recovering log #%llu",
- (unsigned long long) log_number);
-
- // Read all the records and add to a memtable
- std::string scratch;
- Slice record;
- WriteBatch batch;
- MemTable* mem = NULL;
- while (reader.ReadRecord(&record, &scratch) &&
- status.ok()) {
- if (record.size() < 12) {
- reporter.Corruption(
- record.size(), Status::Corruption("log record too small"));
- continue;
- }
- WriteBatchInternal::SetContents(&batch, record);
-
- if (mem == NULL) {
- mem = new MemTable(internal_comparator_);
- mem->Ref();
- }
- status = WriteBatchInternal::InsertInto(&batch, mem);
- MaybeIgnoreError(&status);
- if (!status.ok()) {
- break;
- }
- const SequenceNumber last_seq =
- WriteBatchInternal::Sequence(&batch) +
- WriteBatchInternal::Count(&batch) - 1;
- if (last_seq > *max_sequence) {
- *max_sequence = last_seq;
- }
-
- if (mem->ApproximateMemoryUsage() > options_.write_buffer_size) {
- status = WriteLevel0Table(mem, edit, NULL);
- if (!status.ok()) {
- // Reflect errors immediately so that conditions like full
- // file-systems cause the DB::Open() to fail.
- break;
- }
- mem->Unref();
- mem = NULL;
- }
- }
-
- if (status.ok() && mem != NULL) {
- status = WriteLevel0Table(mem, edit, NULL);
- // Reflect errors immediately so that conditions like full
- // file-systems cause the DB::Open() to fail.
- }
-
- if (mem != NULL) mem->Unref();
- delete file;
- return status;
-}
-
-Status DBImpl::WriteLevel0Table(MemTable* mem, VersionEdit* edit,
- Version* base) {
- mutex_.AssertHeld();
- const uint64_t start_micros = env_->NowMicros();
- FileMetaData meta;
- meta.number = versions_->NewFileNumber();
- pending_outputs_.insert(meta.number);
- Iterator* iter = mem->NewIterator();
- Log(options_.info_log, "Level-0 table #%llu: started",
- (unsigned long long) meta.number);
-
- Status s;
- {
- mutex_.Unlock();
- s = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
- mutex_.Lock();
- }
-
- Log(options_.info_log, "Level-0 table #%llu: %lld bytes %s",
- (unsigned long long) meta.number,
- (unsigned long long) meta.file_size,
- s.ToString().c_str());
- delete iter;
- pending_outputs_.erase(meta.number);
-
-
- // Note that if file_size is zero, the file has been deleted and
- // should not be added to the manifest.
- int level = 0;
- if (s.ok() && meta.file_size > 0) {
- const Slice min_user_key = meta.smallest.user_key();
- const Slice max_user_key = meta.largest.user_key();
- if (base != NULL) {
- level = base->PickLevelForMemTableOutput(min_user_key, max_user_key);
- }
- edit->AddFile(level, meta.number, meta.file_size,
- meta.smallest, meta.largest);
- }
-
- CompactionStats stats;
- stats.micros = env_->NowMicros() - start_micros;
- stats.bytes_written = meta.file_size;
- stats_[level].Add(stats);
- return s;
-}
-
-void DBImpl::CompactMemTable() {
- mutex_.AssertHeld();
- assert(imm_ != NULL);
-
- // Save the contents of the memtable as a new Table
- VersionEdit edit;
- Version* base = versions_->current();
- base->Ref();
- Status s = WriteLevel0Table(imm_, &edit, base);
- base->Unref();
-
- if (s.ok() && shutting_down_.Acquire_Load()) {
- s = Status::IOError("Deleting DB during memtable compaction");
- }
-
- // Replace immutable memtable with the generated Table
- if (s.ok()) {
- edit.SetPrevLogNumber(0);
- edit.SetLogNumber(logfile_number_); // Earlier logs no longer needed
- s = versions_->LogAndApply(&edit, &mutex_);
- }
-
- if (s.ok()) {
- // Commit to the new state
- imm_->Unref();
- imm_ = NULL;
- has_imm_.Release_Store(NULL);
- DeleteObsoleteFiles();
- } else {
- RecordBackgroundError(s);
- }
-}
-
-void DBImpl::CompactRange(const Slice* begin, const Slice* end) {
- int max_level_with_files = 1;
- {
- MutexLock l(&mutex_);
- Version* base = versions_->current();
- for (int level = 1; level < config::kNumLevels; level++) {
- if (base->OverlapInLevel(level, begin, end)) {
- max_level_with_files = level;
- }
- }
- }
- TEST_CompactMemTable(); // TODO(sanjay): Skip if memtable does not overlap
- for (int level = 0; level < max_level_with_files; level++) {
- TEST_CompactRange(level, begin, end);
- }
-}
-
-void DBImpl::TEST_CompactRange(int level, const Slice* begin,const Slice* end) {
- assert(level >= 0);
- assert(level + 1 < config::kNumLevels);
-
- InternalKey begin_storage, end_storage;
-
- ManualCompaction manual;
- manual.level = level;
- manual.done = false;
- if (begin == NULL) {
- manual.begin = NULL;
- } else {
- begin_storage = InternalKey(*begin, kMaxSequenceNumber, kValueTypeForSeek);
- manual.begin = &begin_storage;
- }
- if (end == NULL) {
- manual.end = NULL;
- } else {
- end_storage = InternalKey(*end, 0, static_cast<ValueType>(0));
- manual.end = &end_storage;
- }
-
- MutexLock l(&mutex_);
- while (!manual.done && !shutting_down_.Acquire_Load() && bg_error_.ok()) {
- if (manual_compaction_ == NULL) { // Idle
- manual_compaction_ = &manual;
- MaybeScheduleCompaction();
- } else { // Running either my compaction or another compaction.
- bg_cv_.Wait();
- }
- }
- if (manual_compaction_ == &manual) {
- // Cancel my manual compaction since we aborted early for some reason.
- manual_compaction_ = NULL;
- }
-}
-
-Status DBImpl::TEST_CompactMemTable() {
- // NULL batch means just wait for earlier writes to be done
- Status s = Write(WriteOptions(), NULL);
- if (s.ok()) {
- // Wait until the compaction completes
- MutexLock l(&mutex_);
- while (imm_ != NULL && bg_error_.ok()) {
- bg_cv_.Wait();
- }
- if (imm_ != NULL) {
- s = bg_error_;
- }
- }
- return s;
-}
-
-void DBImpl::RecordBackgroundError(const Status& s) {
- mutex_.AssertHeld();
- if (bg_error_.ok()) {
- bg_error_ = s;
- bg_cv_.SignalAll();
- }
-}
-
-void DBImpl::MaybeScheduleCompaction() {
- mutex_.AssertHeld();
- if (bg_compaction_scheduled_) {
- // Already scheduled
- } else if (shutting_down_.Acquire_Load()) {
- // DB is being deleted; no more background compactions
- } else if (!bg_error_.ok()) {
- // Already got an error; no more changes
- } else if (imm_ == NULL &&
- manual_compaction_ == NULL &&
- !versions_->NeedsCompaction()) {
- // No work to be done
- } else {
- bg_compaction_scheduled_ = true;
- env_->Schedule(&DBImpl::BGWork, this);
- }
-}
-
-void DBImpl::BGWork(void* db) {
- reinterpret_cast<DBImpl*>(db)->BackgroundCall();
-}
-
-void DBImpl::BackgroundCall() {
- MutexLock l(&mutex_);
- assert(bg_compaction_scheduled_);
- if (shutting_down_.Acquire_Load()) {
- // No more background work when shutting down.
- } else if (!bg_error_.ok()) {
- // No more background work after a background error.
- } else {
- BackgroundCompaction();
- }
-
- bg_compaction_scheduled_ = false;
-
- // Previous compaction may have produced too many files in a level,
- // so reschedule another compaction if needed.
- MaybeScheduleCompaction();
- bg_cv_.SignalAll();
-}
-
-void DBImpl::BackgroundCompaction() {
- mutex_.AssertHeld();
-
- if (imm_ != NULL) {
- CompactMemTable();
- return;
- }
-
- Compaction* c;
- bool is_manual = (manual_compaction_ != NULL);
- InternalKey manual_end;
- if (is_manual) {
- ManualCompaction* m = manual_compaction_;
- c = versions_->CompactRange(m->level, m->begin, m->end);
- m->done = (c == NULL);
- if (c != NULL) {
- manual_end = c->input(0, c->num_input_files(0) - 1)->largest;
- }
- Log(options_.info_log,
- "Manual compaction at level-%d from %s .. %s; will stop at %s\n",
- m->level,
- (m->begin ? m->begin->DebugString().c_str() : "(begin)"),
- (m->end ? m->end->DebugString().c_str() : "(end)"),
- (m->done ? "(end)" : manual_end.DebugString().c_str()));
- } else {
- c = versions_->PickCompaction();
- }
-
- Status status;
- if (c == NULL) {
- // Nothing to do
- } else if (!is_manual && c->IsTrivialMove()) {
- // Move file to next level
- assert(c->num_input_files(0) == 1);
- FileMetaData* f = c->input(0, 0);
- c->edit()->DeleteFile(c->level(), f->number);
- c->edit()->AddFile(c->level() + 1, f->number, f->file_size,
- f->smallest, f->largest);
- status = versions_->LogAndApply(c->edit(), &mutex_);
- if (!status.ok()) {
- RecordBackgroundError(status);
- }
- VersionSet::LevelSummaryStorage tmp;
- Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n",
- static_cast<unsigned long long>(f->number),
- c->level() + 1,
- static_cast<unsigned long long>(f->file_size),
- status.ToString().c_str(),
- versions_->LevelSummary(&tmp));
- } else {
- CompactionState* compact = new CompactionState(c);
- status = DoCompactionWork(compact);
- if (!status.ok()) {
- RecordBackgroundError(status);
- }
- CleanupCompaction(compact);
- c->ReleaseInputs();
- DeleteObsoleteFiles();
- }
- delete c;
-
- if (status.ok()) {
- // Done
- } else if (shutting_down_.Acquire_Load()) {
- // Ignore compaction errors found during shutting down
- } else {
- Log(options_.info_log,
- "Compaction error: %s", status.ToString().c_str());
- }
-
- if (is_manual) {
- ManualCompaction* m = manual_compaction_;
- if (!status.ok()) {
- m->done = true;
- }
- if (!m->done) {
- // We only compacted part of the requested range. Update *m
- // to the range that is left to be compacted.
- m->tmp_storage = manual_end;
- m->begin = &m->tmp_storage;
- }
- manual_compaction_ = NULL;
- }
-}
-
-void DBImpl::CleanupCompaction(CompactionState* compact) {
- mutex_.AssertHeld();
- if (compact->builder != NULL) {
- // May happen if we get a shutdown call in the middle of compaction
- compact->builder->Abandon();
- delete compact->builder;
- } else {
- assert(compact->outfile == NULL);
- }
- delete compact->outfile;
- for (size_t i = 0; i < compact->outputs.size(); i++) {
- const CompactionState::Output& out = compact->outputs[i];
- pending_outputs_.erase(out.number);
- }
- delete compact;
-}
-
-Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) {
- assert(compact != NULL);
- assert(compact->builder == NULL);
- uint64_t file_number;
- {
- mutex_.Lock();
- file_number = versions_->NewFileNumber();
- pending_outputs_.insert(file_number);
- CompactionState::Output out;
- out.number = file_number;
- out.smallest.Clear();
- out.largest.Clear();
- compact->outputs.push_back(out);
- mutex_.Unlock();
- }
-
- // Make the output file
- std::string fname = TableFileName(dbname_, file_number);
- Status s = env_->NewWritableFile(fname, &compact->outfile);
- if (s.ok()) {
- compact->builder = new TableBuilder(options_, compact->outfile);
- }
- return s;
-}
-
-Status DBImpl::FinishCompactionOutputFile(CompactionState* compact,
- Iterator* input) {
- assert(compact != NULL);
- assert(compact->outfile != NULL);
- assert(compact->builder != NULL);
-
- const uint64_t output_number = compact->current_output()->number;
- assert(output_number != 0);
-
- // Check for iterator errors
- Status s = input->status();
- const uint64_t current_entries = compact->builder->NumEntries();
- if (s.ok()) {
- s = compact->builder->Finish();
- } else {
- compact->builder->Abandon();
- }
- const uint64_t current_bytes = compact->builder->FileSize();
- compact->current_output()->file_size = current_bytes;
- compact->total_bytes += current_bytes;
- delete compact->builder;
- compact->builder = NULL;
-
- // Finish and check for file errors
- if (s.ok()) {
- s = compact->outfile->Sync();
- }
- if (s.ok()) {
- s = compact->outfile->Close();
- }
- delete compact->outfile;
- compact->outfile = NULL;
-
- if (s.ok() && current_entries > 0) {
- // Verify that the table is usable
- Iterator* iter = table_cache_->NewIterator(ReadOptions(),
- output_number,
- current_bytes);
- s = iter->status();
- delete iter;
- if (s.ok()) {
- Log(options_.info_log,
- "Generated table #%llu: %lld keys, %lld bytes",
- (unsigned long long) output_number,
- (unsigned long long) current_entries,
- (unsigned long long) current_bytes);
- }
- }
- return s;
-}
-
-
-Status DBImpl::InstallCompactionResults(CompactionState* compact) {
- mutex_.AssertHeld();
-#ifdef _WIN32
- Log(options_.info_log, "Compacted %Iu@%d + %Iu@%d files => %lld bytes",
-#else
- Log(options_.info_log, "Compacted %zu@%d + %zu@%d files => %lld bytes",
-#endif
- compact->compaction->num_input_files(0),
- compact->compaction->level(),
- compact->compaction->num_input_files(1),
- compact->compaction->level() + 1,
- static_cast<long long>(compact->total_bytes));
-
- // Add compaction outputs
- compact->compaction->AddInputDeletions(compact->compaction->edit());
- const int level = compact->compaction->level();
- for (size_t i = 0; i < compact->outputs.size(); i++) {
- const CompactionState::Output& out = compact->outputs[i];
- compact->compaction->edit()->AddFile(
- level + 1,
- out.number, out.file_size, out.smallest, out.largest);
- }
- return versions_->LogAndApply(compact->compaction->edit(), &mutex_);
-}
-
-Status DBImpl::DoCompactionWork(CompactionState* compact) {
- const uint64_t start_micros = env_->NowMicros();
- int64_t imm_micros = 0; // Micros spent doing imm_ compactions
-
-#ifdef _WIN32
- Log(options_.info_log, "Compacting %Iu@%d + %Iu@%d files",
-#else
- Log(options_.info_log, "Compacting %zu@%d + %zu@%d files",
-#endif
- compact->compaction->num_input_files(0),
- compact->compaction->level(),
- compact->compaction->num_input_files(1),
- compact->compaction->level() + 1);
-
- assert(versions_->NumLevelFiles(compact->compaction->level()) > 0);
- assert(compact->builder == NULL);
- assert(compact->outfile == NULL);
- if (snapshots_.empty()) {
- compact->smallest_snapshot = versions_->LastSequence();
- } else {
- compact->smallest_snapshot = snapshots_.oldest()->number_;
- }
-
- // Release mutex while we're actually doing the compaction work
- mutex_.Unlock();
-
- Iterator* input = versions_->MakeInputIterator(compact->compaction);
- input->SeekToFirst();
- Status status;
- ParsedInternalKey ikey;
- std::string current_user_key;
- bool has_current_user_key = false;
- SequenceNumber last_sequence_for_key = kMaxSequenceNumber;
- for (; input->Valid() && !shutting_down_.Acquire_Load(); ) {
- // Prioritize immutable compaction work
- if (has_imm_.NoBarrier_Load() != NULL) {
- const uint64_t imm_start = env_->NowMicros();
- mutex_.Lock();
- if (imm_ != NULL) {
- CompactMemTable();
- bg_cv_.SignalAll(); // Wakeup MakeRoomForWrite() if necessary
- }
- mutex_.Unlock();
- imm_micros += (env_->NowMicros() - imm_start);
- }
-
- Slice key = input->key();
- if (compact->compaction->ShouldStopBefore(key) &&
- compact->builder != NULL) {
- status = FinishCompactionOutputFile(compact, input);
- if (!status.ok()) {
- break;
- }
- }
-
- // Handle key/value, add to state, etc.
- bool drop = false;
- if (!ParseInternalKey(key, &ikey)) {
- // Do not hide error keys
- current_user_key.clear();
- has_current_user_key = false;
- last_sequence_for_key = kMaxSequenceNumber;
- } else {
- if (!has_current_user_key ||
- user_comparator()->Compare(ikey.user_key,
- Slice(current_user_key)) != 0) {
- // First occurrence of this user key
- current_user_key.assign(ikey.user_key.data(), ikey.user_key.size());
- has_current_user_key = true;
- last_sequence_for_key = kMaxSequenceNumber;
- }
-
- if (last_sequence_for_key <= compact->smallest_snapshot) {
- // Hidden by an newer entry for same user key
- drop = true; // (A)
- } else if (ikey.type == kTypeDeletion &&
- ikey.sequence <= compact->smallest_snapshot &&
- compact->compaction->IsBaseLevelForKey(ikey.user_key)) {
- // For this user key:
- // (1) there is no data in higher levels
- // (2) data in lower levels will have larger sequence numbers
- // (3) data in layers that are being compacted here and have
- // smaller sequence numbers will be dropped in the next
- // few iterations of this loop (by rule (A) above).
- // Therefore this deletion marker is obsolete and can be dropped.
- drop = true;
- }
-
- last_sequence_for_key = ikey.sequence;
- }
-#if 0
- Log(options_.info_log,
- " Compact: %s, seq %d, type: %d %d, drop: %d, is_base: %d, "
- "%d smallest_snapshot: %d",
- ikey.user_key.ToString().c_str(),
- (int)ikey.sequence, ikey.type, kTypeValue, drop,
- compact->compaction->IsBaseLevelForKey(ikey.user_key),
- (int)last_sequence_for_key, (int)compact->smallest_snapshot);
-#endif
-
- if (!drop) {
- // Open output file if necessary
- if (compact->builder == NULL) {
- status = OpenCompactionOutputFile(compact);
- if (!status.ok()) {
- break;
- }
- }
- if (compact->builder->NumEntries() == 0) {
- compact->current_output()->smallest.DecodeFrom(key);
- }
- compact->current_output()->largest.DecodeFrom(key);
- compact->builder->Add(key, input->value());
-
- // Close output file if it is big enough
- if (compact->builder->FileSize() >=
- compact->compaction->MaxOutputFileSize()) {
- status = FinishCompactionOutputFile(compact, input);
- if (!status.ok()) {
- break;
- }
- }
- }
-
- input->Next();
- }
-
- if (status.ok() && shutting_down_.Acquire_Load()) {
- status = Status::IOError("Deleting DB during compaction");
- }
- if (status.ok() && compact->builder != NULL) {
- status = FinishCompactionOutputFile(compact, input);
- }
- if (status.ok()) {
- status = input->status();
- }
- delete input;
- input = NULL;
-
- CompactionStats stats;
- stats.micros = env_->NowMicros() - start_micros - imm_micros;
- for (int which = 0; which < 2; which++) {
- for (size_t i = 0; i < compact->compaction->num_input_files(which); i++) {
- stats.bytes_read += compact->compaction->input(which, i)->file_size;
- }
- }
- for (size_t i = 0; i < compact->outputs.size(); i++) {
- stats.bytes_written += compact->outputs[i].file_size;
- }
-
- mutex_.Lock();
- stats_[compact->compaction->level() + 1].Add(stats);
-
- if (status.ok()) {
- status = InstallCompactionResults(compact);
- }
- if (!status.ok()) {
- RecordBackgroundError(status);
- }
- VersionSet::LevelSummaryStorage tmp;
- Log(options_.info_log,
- "compacted to: %s", versions_->LevelSummary(&tmp));
- return status;
-}
-
-namespace {
-struct IterState {
- port::Mutex* mu;
- Version* version;
- MemTable* mem;
- MemTable* imm;
-};
-
-static void CleanupIteratorState(void* arg1, void* arg2) {
- IterState* state = reinterpret_cast<IterState*>(arg1);
- state->mu->Lock();
- state->mem->Unref();
- if (state->imm != NULL) state->imm->Unref();
- state->version->Unref();
- state->mu->Unlock();
- delete state;
-}
-} // namespace
-
-Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
- SequenceNumber* latest_snapshot,
- uint32_t* seed) {
- IterState* cleanup = new IterState;
- mutex_.Lock();
- *latest_snapshot = versions_->LastSequence();
-
- // Collect together all needed child iterators
- std::vector<Iterator*> list;
- list.push_back(mem_->NewIterator());
- mem_->Ref();
- if (imm_ != NULL) {
- list.push_back(imm_->NewIterator());
- imm_->Ref();
- }
- versions_->current()->AddIterators(options, &list);
- Iterator* internal_iter =
- NewMergingIterator(&internal_comparator_, &list[0], list.size());
- versions_->current()->Ref();
-
- cleanup->mu = &mutex_;
- cleanup->mem = mem_;
- cleanup->imm = imm_;
- cleanup->version = versions_->current();
- internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, NULL);
-
- *seed = ++seed_;
- mutex_.Unlock();
- return internal_iter;
-}
-
-Iterator* DBImpl::TEST_NewInternalIterator() {
- SequenceNumber ignored;
- uint32_t ignored_seed;
- return NewInternalIterator(ReadOptions(), &ignored, &ignored_seed);
-}
-
-int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() {
- MutexLock l(&mutex_);
- return versions_->MaxNextLevelOverlappingBytes();
-}
-
-Status DBImpl::Get(const ReadOptions& options,
- const Slice& key,
- std::string* value) {
- Status s;
- MutexLock l(&mutex_);
- SequenceNumber snapshot;
- if (options.snapshot != NULL) {
- snapshot = reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_;
- } else {
- snapshot = versions_->LastSequence();
- }
-
- MemTable* mem = mem_;
- MemTable* imm = imm_;
- Version* current = versions_->current();
- mem->Ref();
- if (imm != NULL) imm->Ref();
- current->Ref();
-
- bool have_stat_update = false;
- Version::GetStats stats;
-
- // Unlock while reading from files and memtables
- {
- mutex_.Unlock();
- // First look in the memtable, then in the immutable memtable (if any).
- LookupKey lkey(key, snapshot);
- if (mem->Get(lkey, value, &s)) {
- // Done
- } else if (imm != NULL && imm->Get(lkey, value, &s)) {
- // Done
- } else {
- s = current->Get(options, lkey, value, &stats);
- have_stat_update = true;
- }
- mutex_.Lock();
- }
-
- if (have_stat_update && current->UpdateStats(stats)) {
- MaybeScheduleCompaction();
- }
- mem->Unref();
- if (imm != NULL) imm->Unref();
- current->Unref();
- return s;
-}
-
-Iterator* DBImpl::NewIterator(const ReadOptions& options) {
- SequenceNumber latest_snapshot;
- uint32_t seed;
- Iterator* iter = NewInternalIterator(options, &latest_snapshot, &seed);
- return NewDBIterator(
- this, user_comparator(), iter,
- (options.snapshot != NULL
- ? reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_
- : latest_snapshot),
- seed);
-}
-
-void DBImpl::RecordReadSample(Slice key) {
- MutexLock l(&mutex_);
- if (versions_->current()->RecordReadSample(key)) {
- MaybeScheduleCompaction();
- }
-}
-
-const Snapshot* DBImpl::GetSnapshot() {
- MutexLock l(&mutex_);
- return snapshots_.New(versions_->LastSequence());
-}
-
-void DBImpl::ReleaseSnapshot(const Snapshot* s) {
- MutexLock l(&mutex_);
- snapshots_.Delete(reinterpret_cast<const SnapshotImpl*>(s));
-}
-
-// Convenience methods
-Status DBImpl::Put(const WriteOptions& o, const Slice& key, const Slice& val) {
- return DB::Put(o, key, val);
-}
-
-Status DBImpl::Delete(const WriteOptions& options, const Slice& key) {
- return DB::Delete(options, key);
-}
-
-Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
- Writer w(&mutex_);
- w.batch = my_batch;
- w.sync = options.sync;
- w.done = false;
-
- MutexLock l(&mutex_);
- writers_.push_back(&w);
- while (!w.done && &w != writers_.front()) {
- w.cv.Wait();
- }
- if (w.done) {
- return w.status;
- }
-
- // May temporarily unlock and wait.
- Status status = MakeRoomForWrite(my_batch == NULL);
- uint64_t last_sequence = versions_->LastSequence();
- Writer* last_writer = &w;
- if (status.ok() && my_batch != NULL) { // NULL batch is for compactions
- WriteBatch* updates = BuildBatchGroup(&last_writer);
- WriteBatchInternal::SetSequence(updates, last_sequence + 1);
- last_sequence += WriteBatchInternal::Count(updates);
-
- // Add to log and apply to memtable. We can release the lock
- // during this phase since &w is currently responsible for logging
- // and protects against concurrent loggers and concurrent writes
- // into mem_.
- {
- mutex_.Unlock();
- status = log_->AddRecord(WriteBatchInternal::Contents(updates));
- bool sync_error = false;
- if (status.ok() && options.sync) {
- status = logfile_->Sync();
- if (!status.ok()) {
- sync_error = true;
- }
- }
- if (status.ok()) {
- status = WriteBatchInternal::InsertInto(updates, mem_);
- }
- mutex_.Lock();
- if (sync_error) {
- // The state of the log file is indeterminate: the log record we
- // just added may or may not show up when the DB is re-opened.
- // So we force the DB into a mode where all future writes fail.
- RecordBackgroundError(status);
- }
- }
- if (updates == tmp_batch_) tmp_batch_->Clear();
-
- versions_->SetLastSequence(last_sequence);
- }
-
- while (true) {
- Writer* ready = writers_.front();
- writers_.pop_front();
- if (ready != &w) {
- ready->status = status;
- ready->done = true;
- ready->cv.Signal();
- }
- if (ready == last_writer) break;
- }
-
- // Notify new head of write queue
- if (!writers_.empty()) {
- writers_.front()->cv.Signal();
- }
-
- return status;
-}
-
-// REQUIRES: Writer list must be non-empty
-// REQUIRES: First writer must have a non-NULL batch
-WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) {
- assert(!writers_.empty());
- Writer* first = writers_.front();
- WriteBatch* result = first->batch;
- assert(result != NULL);
-
- size_t size = WriteBatchInternal::ByteSize(first->batch);
-
- // Allow the group to grow up to a maximum size, but if the
- // original write is small, limit the growth so we do not slow
- // down the small write too much.
- size_t max_size = 1 << 20;
- if (size <= (128<<10)) {
- max_size = size + (128<<10);
- }
-
- *last_writer = first;
- std::deque<Writer*>::iterator iter = writers_.begin();
- ++iter; // Advance past "first"
- for (; iter != writers_.end(); ++iter) {
- Writer* w = *iter;
- if (w->sync && !first->sync) {
- // Do not include a sync write into a batch handled by a non-sync write.
- break;
- }
-
- if (w->batch != NULL) {
- size += WriteBatchInternal::ByteSize(w->batch);
- if (size > max_size) {
- // Do not make batch too big
- break;
- }
-
- // Append to *result
- if (result == first->batch) {
- // Switch to temporary batch instead of disturbing caller's batch
- result = tmp_batch_;
- assert(WriteBatchInternal::Count(result) == 0);
- WriteBatchInternal::Append(result, first->batch);
- }
- WriteBatchInternal::Append(result, w->batch);
- }
- *last_writer = w;
- }
- return result;
-}
-
-// REQUIRES: mutex_ is held
-// REQUIRES: this thread is currently at the front of the writer queue
-Status DBImpl::MakeRoomForWrite(bool force) {
- mutex_.AssertHeld();
- assert(!writers_.empty());
- bool allow_delay = !force;
- Status s;
- while (true) {
- if (!bg_error_.ok()) {
- // Yield previous error
- s = bg_error_;
- break;
- } else if (
- allow_delay &&
- versions_->NumLevelFiles(0) >= config::kL0_SlowdownWritesTrigger) {
- // We are getting close to hitting a hard limit on the number of
- // L0 files. Rather than delaying a single write by several
- // seconds when we hit the hard limit, start delaying each
- // individual write by 1ms to reduce latency variance. Also,
- // this delay hands over some CPU to the compaction thread in
- // case it is sharing the same core as the writer.
- mutex_.Unlock();
- env_->SleepForMicroseconds(1000);
- allow_delay = false; // Do not delay a single write more than once
- mutex_.Lock();
- } else if (!force &&
- (mem_->ApproximateMemoryUsage() <= options_.write_buffer_size)) {
- // There is room in current memtable
- break;
- } else if (imm_ != NULL) {
- // We have filled up the current memtable, but the previous
- // one is still being compacted, so we wait.
- Log(options_.info_log, "Current memtable full; waiting...\n");
- bg_cv_.Wait();
- } else if (versions_->NumLevelFiles(0) >= config::kL0_StopWritesTrigger) {
- // There are too many level-0 files.
- Log(options_.info_log, "Too many L0 files; waiting...\n");
- bg_cv_.Wait();
- } else {
- // Attempt to switch to a new memtable and trigger compaction of old
- assert(versions_->PrevLogNumber() == 0);
- uint64_t new_log_number = versions_->NewFileNumber();
- WritableFile* lfile = NULL;
- s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile);
- if (!s.ok()) {
- // Avoid chewing through file number space in a tight loop.
- versions_->ReuseFileNumber(new_log_number);
- break;
- }
- delete log_;
- delete logfile_;
- logfile_ = lfile;
- logfile_number_ = new_log_number;
- log_ = new log::Writer(lfile);
- imm_ = mem_;
- has_imm_.Release_Store(imm_);
- mem_ = new MemTable(internal_comparator_);
- mem_->Ref();
- force = false; // Do not force another compaction if have room
- MaybeScheduleCompaction();
- }
- }
- return s;
-}
-
-bool DBImpl::GetProperty(const Slice& property, std::string* value) {
- value->clear();
-
- MutexLock l(&mutex_);
- Slice in = property;
- Slice prefix("leveldb.");
- if (!in.starts_with(prefix)) return false;
- in.remove_prefix(prefix.size());
-
- if (in.starts_with("num-files-at-level")) {
- in.remove_prefix(strlen("num-files-at-level"));
- uint64_t level;
- bool ok = ConsumeDecimalNumber(&in, &level) && in.empty();
- if (!ok || level >= config::kNumLevels) {
- return false;
- } else {
- char buf[100];
-#ifdef _WIN32
- snprintf(buf, sizeof(buf), "%Iu",
-#else
- snprintf(buf, sizeof(buf), "%zu",
-#endif
- versions_->NumLevelFiles(static_cast<int>(level)));
- *value = buf;
- return true;
- }
- } else if (in == "stats") {
- char buf[200];
- snprintf(buf, sizeof(buf),
- " Compactions\n"
- "Level Files Size(MB) Time(sec) Read(MB) Write(MB)\n"
- "--------------------------------------------------\n"
- );
- value->append(buf);
- for (int level = 0; level < config::kNumLevels; level++) {
- size_t files = versions_->NumLevelFiles(level);
- if (stats_[level].micros > 0 || files > 0) {
- snprintf(
- buf, sizeof(buf),
-#ifdef _WIN32
- "%3d %8Iu %8.0f %9.0f %8.0f %9.0f\n",
-#else
- "%3d %8zu %8.0f %9.0f %8.0f %9.0f\n",
-#endif
- level,
- files,
- versions_->NumLevelBytes(level) / 1048576.0,
- stats_[level].micros / 1e6,
- stats_[level].bytes_read / 1048576.0,
- stats_[level].bytes_written / 1048576.0);
- value->append(buf);
- }
- }
- return true;
- } else if (in == "sstables") {
- *value = versions_->current()->DebugString();
- return true;
- }
-
- return false;
-}
-
-void DBImpl::GetApproximateSizes(
- const Range* range, int n,
- uint64_t* sizes) {
- // TODO(opt): better implementation
- Version* v;
- {
- MutexLock l(&mutex_);
- versions_->current()->Ref();
- v = versions_->current();
- }
-
- for (int i = 0; i < n; i++) {
- // Convert user_key into a corresponding internal key.
- InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek);
- InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek);
- uint64_t start = versions_->ApproximateOffsetOf(v, k1);
- uint64_t limit = versions_->ApproximateOffsetOf(v, k2);
- sizes[i] = (limit >= start ? limit - start : 0);
- }
-
- {
- MutexLock l(&mutex_);
- v->Unref();
- }
-}
-
-// Default implementations of convenience methods that subclasses of DB
-// can call if they wish
-Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) {
- WriteBatch batch;
- batch.Put(key, value);
- return Write(opt, &batch);
-}
-
-Status DB::Delete(const WriteOptions& opt, const Slice& key) {
- WriteBatch batch;
- batch.Delete(key);
- return Write(opt, &batch);
-}
-
-DB::~DB() { }
-
-Status DB::Open(const Options& options, const std::string& dbname,
- DB** dbptr) {
- *dbptr = NULL;
-
- DBImpl* impl = new DBImpl(options, dbname);
- impl->mutex_.Lock();
- VersionEdit edit;
- Status s = impl->Recover(&edit); // Handles create_if_missing, error_if_exists
- if (s.ok()) {
- uint64_t new_log_number = impl->versions_->NewFileNumber();
- WritableFile* lfile;
- s = options.env->NewWritableFile(LogFileName(dbname, new_log_number),
- &lfile);
- if (s.ok()) {
- edit.SetLogNumber(new_log_number);
- impl->logfile_ = lfile;
- impl->logfile_number_ = new_log_number;
- impl->log_ = new log::Writer(lfile);
- s = impl->versions_->LogAndApply(&edit, &impl->mutex_);
- }
- if (s.ok()) {
- impl->DeleteObsoleteFiles();
- impl->MaybeScheduleCompaction();
- }
- }
- impl->mutex_.Unlock();
- if (s.ok()) {
- *dbptr = impl;
- } else {
- delete impl;
- }
- return s;
-}
-
-Snapshot::~Snapshot() {
-}
-
-Status DestroyDB(const std::string& dbname, const Options& options) {
- Env* env = options.env;
- std::vector<std::string> filenames;
- // Ignore error in case directory does not exist
- env->GetChildren(dbname, &filenames);
- if (filenames.empty()) {
- return Status::OK();
- }
-
- FileLock* lock;
- const std::string lockname = LockFileName(dbname);
- Status result = env->LockFile(lockname, &lock);
- if (result.ok()) {
- uint64_t number;
- FileType type;
- for (size_t i = 0; i < filenames.size(); i++) {
- if (ParseFileName(filenames[i], &number, &type) &&
- type != kDBLockFile) { // Lock file will be deleted at end
- Status del = env->DeleteFile(dbname + "/" + filenames[i]);
- if (result.ok() && !del.ok()) {
- result = del;
- }
- }
- }
- env->UnlockFile(lock); // Ignore error since state is already gone
- env->DeleteFile(lockname);
- env->DeleteDir(dbname); // Ignore error in case dir contains other files
- }
- return result;
-}
-
-} // namespace leveldb
[30/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/cache/lru_cache.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/cache/lru_cache.h b/thirdparty/rocksdb/cache/lru_cache.h
new file mode 100644
index 0000000..2fd44bb
--- /dev/null
+++ b/thirdparty/rocksdb/cache/lru_cache.h
@@ -0,0 +1,298 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#pragma once
+
+#include <string>
+
+#include "cache/sharded_cache.h"
+
+#include "port/port.h"
+#include "util/autovector.h"
+
+namespace rocksdb {
+
+// LRU cache implementation
+
+// An entry is a variable length heap-allocated structure.
+// Entries are referenced by cache and/or by any external entity.
+// The cache keeps all its entries in table. Some elements
+// are also stored on LRU list.
+//
+// LRUHandle can be in these states:
+// 1. Referenced externally AND in hash table.
+// In that case the entry is *not* in the LRU. (refs > 1 && in_cache == true)
+// 2. Not referenced externally and in hash table. In that case the entry is
+// in the LRU and can be freed. (refs == 1 && in_cache == true)
+// 3. Referenced externally and not in hash table. In that case the entry is
+// in not on LRU and not in table. (refs >= 1 && in_cache == false)
+//
+// All newly created LRUHandles are in state 1. If you call
+// LRUCacheShard::Release
+// on entry in state 1, it will go into state 2. To move from state 1 to
+// state 3, either call LRUCacheShard::Erase or LRUCacheShard::Insert with the
+// same key.
+// To move from state 2 to state 1, use LRUCacheShard::Lookup.
+// Before destruction, make sure that no handles are in state 1. This means
+// that any successful LRUCacheShard::Lookup/LRUCacheShard::Insert have a
+// matching
+// RUCache::Release (to move into state 2) or LRUCacheShard::Erase (for state 3)
+
+struct LRUHandle {
+ void* value;
+ void (*deleter)(const Slice&, void* value);
+ LRUHandle* next_hash;
+ LRUHandle* next;
+ LRUHandle* prev;
+ size_t charge; // TODO(opt): Only allow uint32_t?
+ size_t key_length;
+ uint32_t refs; // a number of refs to this entry
+ // cache itself is counted as 1
+
+ // Include the following flags:
+ // in_cache: whether this entry is referenced by the hash table.
+ // is_high_pri: whether this entry is high priority entry.
+ // in_high_pro_pool: whether this entry is in high-pri pool.
+ char flags;
+
+ uint32_t hash; // Hash of key(); used for fast sharding and comparisons
+
+ char key_data[1]; // Beginning of key
+
+ Slice key() const {
+ // For cheaper lookups, we allow a temporary Handle object
+ // to store a pointer to a key in "value".
+ if (next == this) {
+ return *(reinterpret_cast<Slice*>(value));
+ } else {
+ return Slice(key_data, key_length);
+ }
+ }
+
+ bool InCache() { return flags & 1; }
+ bool IsHighPri() { return flags & 2; }
+ bool InHighPriPool() { return flags & 4; }
+
+ void SetInCache(bool in_cache) {
+ if (in_cache) {
+ flags |= 1;
+ } else {
+ flags &= ~1;
+ }
+ }
+
+ void SetPriority(Cache::Priority priority) {
+ if (priority == Cache::Priority::HIGH) {
+ flags |= 2;
+ } else {
+ flags &= ~2;
+ }
+ }
+
+ void SetInHighPriPool(bool in_high_pri_pool) {
+ if (in_high_pri_pool) {
+ flags |= 4;
+ } else {
+ flags &= ~4;
+ }
+ }
+
+ void Free() {
+ assert((refs == 1 && InCache()) || (refs == 0 && !InCache()));
+ if (deleter) {
+ (*deleter)(key(), value);
+ }
+ delete[] reinterpret_cast<char*>(this);
+ }
+};
+
+// We provide our own simple hash table since it removes a whole bunch
+// of porting hacks and is also faster than some of the built-in hash
+// table implementations in some of the compiler/runtime combinations
+// we have tested. E.g., readrandom speeds up by ~5% over the g++
+// 4.4.3's builtin hashtable.
+class LRUHandleTable {
+ public:
+ LRUHandleTable();
+ ~LRUHandleTable();
+
+ LRUHandle* Lookup(const Slice& key, uint32_t hash);
+ LRUHandle* Insert(LRUHandle* h);
+ LRUHandle* Remove(const Slice& key, uint32_t hash);
+
+ template <typename T>
+ void ApplyToAllCacheEntries(T func) {
+ for (uint32_t i = 0; i < length_; i++) {
+ LRUHandle* h = list_[i];
+ while (h != nullptr) {
+ auto n = h->next_hash;
+ assert(h->InCache());
+ func(h);
+ h = n;
+ }
+ }
+ }
+
+ private:
+ // Return a pointer to slot that points to a cache entry that
+ // matches key/hash. If there is no such cache entry, return a
+ // pointer to the trailing slot in the corresponding linked list.
+ LRUHandle** FindPointer(const Slice& key, uint32_t hash);
+
+ void Resize();
+
+ // The table consists of an array of buckets where each bucket is
+ // a linked list of cache entries that hash into the bucket.
+ LRUHandle** list_;
+ uint32_t length_;
+ uint32_t elems_;
+};
+
+// A single shard of sharded cache.
+class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
+ public:
+ LRUCacheShard();
+ virtual ~LRUCacheShard();
+
+ // Separate from constructor so caller can easily make an array of LRUCache
+ // if current usage is more than new capacity, the function will attempt to
+ // free the needed space
+ virtual void SetCapacity(size_t capacity) override;
+
+ // Set the flag to reject insertion if cache if full.
+ virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
+
+ // Set percentage of capacity reserved for high-pri cache entries.
+ void SetHighPriorityPoolRatio(double high_pri_pool_ratio);
+
+ // Like Cache methods, but with an extra "hash" parameter.
+ virtual Status Insert(const Slice& key, uint32_t hash, void* value,
+ size_t charge,
+ void (*deleter)(const Slice& key, void* value),
+ Cache::Handle** handle,
+ Cache::Priority priority) override;
+ virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
+ virtual bool Ref(Cache::Handle* handle) override;
+ virtual bool Release(Cache::Handle* handle,
+ bool force_erase = false) override;
+ virtual void Erase(const Slice& key, uint32_t hash) override;
+
+ // Although in some platforms the update of size_t is atomic, to make sure
+ // GetUsage() and GetPinnedUsage() work correctly under any platform, we'll
+ // protect them with mutex_.
+
+ virtual size_t GetUsage() const override;
+ virtual size_t GetPinnedUsage() const override;
+
+ virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+ bool thread_safe) override;
+
+ virtual void EraseUnRefEntries() override;
+
+ virtual std::string GetPrintableOptions() const override;
+
+ void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri);
+
+ // Retrieves number of elements in LRU, for unit test purpose only
+ // not threadsafe
+ size_t TEST_GetLRUSize();
+
+ // Overloading to aligned it to cache line size
+ void* operator new(size_t);
+
+ void operator delete(void *);
+
+ private:
+ void LRU_Remove(LRUHandle* e);
+ void LRU_Insert(LRUHandle* e);
+
+ // Overflow the last entry in high-pri pool to low-pri pool until size of
+ // high-pri pool is no larger than the size specify by high_pri_pool_pct.
+ void MaintainPoolSize();
+
+ // Just reduce the reference count by 1.
+ // Return true if last reference
+ bool Unref(LRUHandle* e);
+
+ // Free some space following strict LRU policy until enough space
+ // to hold (usage_ + charge) is freed or the lru list is empty
+ // This function is not thread safe - it needs to be executed while
+ // holding the mutex_
+ void EvictFromLRU(size_t charge, autovector<LRUHandle*>* deleted);
+
+ // Initialized before use.
+ size_t capacity_;
+
+ // Memory size for entries in high-pri pool.
+ size_t high_pri_pool_usage_;
+
+ // Whether to reject insertion if cache reaches its full capacity.
+ bool strict_capacity_limit_;
+
+ // Ratio of capacity reserved for high priority cache entries.
+ double high_pri_pool_ratio_;
+
+ // High-pri pool size, equals to capacity * high_pri_pool_ratio.
+ // Remember the value to avoid recomputing each time.
+ double high_pri_pool_capacity_;
+
+ // Dummy head of LRU list.
+ // lru.prev is newest entry, lru.next is oldest entry.
+ // LRU contains items which can be evicted, ie reference only by cache
+ LRUHandle lru_;
+
+ // Pointer to head of low-pri pool in LRU list.
+ LRUHandle* lru_low_pri_;
+
+ // ------------^^^^^^^^^^^^^-----------
+ // Not frequently modified data members
+ // ------------------------------------
+ //
+ // We separate data members that are updated frequently from the ones that
+ // are not frequently updated so that they don't share the same cache line
+ // which will lead into false cache sharing
+ //
+ // ------------------------------------
+ // Frequently modified data members
+ // ------------vvvvvvvvvvvvv-----------
+ LRUHandleTable table_;
+
+ // Memory size for entries residing in the cache
+ size_t usage_;
+
+ // Memory size for entries residing only in the LRU list
+ size_t lru_usage_;
+
+ // mutex_ protects the following state.
+ // We don't count mutex_ as the cache's internal state so semantically we
+ // don't mind mutex_ invoking the non-const actions.
+ mutable port::Mutex mutex_;
+};
+
+class LRUCache : public ShardedCache {
+ public:
+ LRUCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
+ double high_pri_pool_ratio);
+ virtual ~LRUCache();
+ virtual const char* Name() const override { return "LRUCache"; }
+ virtual CacheShard* GetShard(int shard) override;
+ virtual const CacheShard* GetShard(int shard) const override;
+ virtual void* Value(Handle* handle) override;
+ virtual size_t GetCharge(Handle* handle) const override;
+ virtual uint32_t GetHash(Handle* handle) const override;
+ virtual void DisownData() override;
+
+ // Retrieves number of elements in LRU, for unit test purpose only
+ size_t TEST_GetLRUSize();
+
+ private:
+ LRUCacheShard* shards_;
+ int num_shards_ = 0;
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/cache/sharded_cache.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/cache/sharded_cache.cc b/thirdparty/rocksdb/cache/sharded_cache.cc
new file mode 100644
index 0000000..9bdea3a
--- /dev/null
+++ b/thirdparty/rocksdb/cache/sharded_cache.cc
@@ -0,0 +1,161 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include "cache/sharded_cache.h"
+
+#include <string>
+
+#include "util/mutexlock.h"
+
+namespace rocksdb {
+
+ShardedCache::ShardedCache(size_t capacity, int num_shard_bits,
+ bool strict_capacity_limit)
+ : num_shard_bits_(num_shard_bits),
+ capacity_(capacity),
+ strict_capacity_limit_(strict_capacity_limit),
+ last_id_(1) {}
+
+void ShardedCache::SetCapacity(size_t capacity) {
+ int num_shards = 1 << num_shard_bits_;
+ const size_t per_shard = (capacity + (num_shards - 1)) / num_shards;
+ MutexLock l(&capacity_mutex_);
+ for (int s = 0; s < num_shards; s++) {
+ GetShard(s)->SetCapacity(per_shard);
+ }
+ capacity_ = capacity;
+}
+
+void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) {
+ int num_shards = 1 << num_shard_bits_;
+ MutexLock l(&capacity_mutex_);
+ for (int s = 0; s < num_shards; s++) {
+ GetShard(s)->SetStrictCapacityLimit(strict_capacity_limit);
+ }
+ strict_capacity_limit_ = strict_capacity_limit;
+}
+
+Status ShardedCache::Insert(const Slice& key, void* value, size_t charge,
+ void (*deleter)(const Slice& key, void* value),
+ Handle** handle, Priority priority) {
+ uint32_t hash = HashSlice(key);
+ return GetShard(Shard(hash))
+ ->Insert(key, hash, value, charge, deleter, handle, priority);
+}
+
+Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* stats) {
+ uint32_t hash = HashSlice(key);
+ return GetShard(Shard(hash))->Lookup(key, hash);
+}
+
+bool ShardedCache::Ref(Handle* handle) {
+ uint32_t hash = GetHash(handle);
+ return GetShard(Shard(hash))->Ref(handle);
+}
+
+bool ShardedCache::Release(Handle* handle, bool force_erase) {
+ uint32_t hash = GetHash(handle);
+ return GetShard(Shard(hash))->Release(handle, force_erase);
+}
+
+void ShardedCache::Erase(const Slice& key) {
+ uint32_t hash = HashSlice(key);
+ GetShard(Shard(hash))->Erase(key, hash);
+}
+
+uint64_t ShardedCache::NewId() {
+ return last_id_.fetch_add(1, std::memory_order_relaxed);
+}
+
+size_t ShardedCache::GetCapacity() const {
+ MutexLock l(&capacity_mutex_);
+ return capacity_;
+}
+
+bool ShardedCache::HasStrictCapacityLimit() const {
+ MutexLock l(&capacity_mutex_);
+ return strict_capacity_limit_;
+}
+
+size_t ShardedCache::GetUsage() const {
+ // We will not lock the cache when getting the usage from shards.
+ int num_shards = 1 << num_shard_bits_;
+ size_t usage = 0;
+ for (int s = 0; s < num_shards; s++) {
+ usage += GetShard(s)->GetUsage();
+ }
+ return usage;
+}
+
+size_t ShardedCache::GetUsage(Handle* handle) const {
+ return GetCharge(handle);
+}
+
+size_t ShardedCache::GetPinnedUsage() const {
+ // We will not lock the cache when getting the usage from shards.
+ int num_shards = 1 << num_shard_bits_;
+ size_t usage = 0;
+ for (int s = 0; s < num_shards; s++) {
+ usage += GetShard(s)->GetPinnedUsage();
+ }
+ return usage;
+}
+
+void ShardedCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+ bool thread_safe) {
+ int num_shards = 1 << num_shard_bits_;
+ for (int s = 0; s < num_shards; s++) {
+ GetShard(s)->ApplyToAllCacheEntries(callback, thread_safe);
+ }
+}
+
+void ShardedCache::EraseUnRefEntries() {
+ int num_shards = 1 << num_shard_bits_;
+ for (int s = 0; s < num_shards; s++) {
+ GetShard(s)->EraseUnRefEntries();
+ }
+}
+
+std::string ShardedCache::GetPrintableOptions() const {
+ std::string ret;
+ ret.reserve(20000);
+ const int kBufferSize = 200;
+ char buffer[kBufferSize];
+ {
+ MutexLock l(&capacity_mutex_);
+ snprintf(buffer, kBufferSize, " capacity : %" ROCKSDB_PRIszt "\n",
+ capacity_);
+ ret.append(buffer);
+ snprintf(buffer, kBufferSize, " num_shard_bits : %d\n", num_shard_bits_);
+ ret.append(buffer);
+ snprintf(buffer, kBufferSize, " strict_capacity_limit : %d\n",
+ strict_capacity_limit_);
+ ret.append(buffer);
+ }
+ ret.append(GetShard(0)->GetPrintableOptions());
+ return ret;
+}
+int GetDefaultCacheShardBits(size_t capacity) {
+ int num_shard_bits = 0;
+ size_t min_shard_size = 512L * 1024L; // Every shard is at least 512KB.
+ size_t num_shards = capacity / min_shard_size;
+ while (num_shards >>= 1) {
+ if (++num_shard_bits >= 6) {
+ // No more than 6.
+ return num_shard_bits;
+ }
+ }
+ return num_shard_bits;
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/cache/sharded_cache.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/cache/sharded_cache.h b/thirdparty/rocksdb/cache/sharded_cache.h
new file mode 100644
index 0000000..4f9dea2
--- /dev/null
+++ b/thirdparty/rocksdb/cache/sharded_cache.h
@@ -0,0 +1,102 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#include <atomic>
+#include <string>
+
+#include "port/port.h"
+#include "rocksdb/cache.h"
+#include "util/hash.h"
+
+namespace rocksdb {
+
+// Single cache shard interface.
+class CacheShard {
+ public:
+ CacheShard() = default;
+ virtual ~CacheShard() = default;
+
+ virtual Status Insert(const Slice& key, uint32_t hash, void* value,
+ size_t charge,
+ void (*deleter)(const Slice& key, void* value),
+ Cache::Handle** handle, Cache::Priority priority) = 0;
+ virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) = 0;
+ virtual bool Ref(Cache::Handle* handle) = 0;
+ virtual bool Release(Cache::Handle* handle, bool force_erase = false) = 0;
+ virtual void Erase(const Slice& key, uint32_t hash) = 0;
+ virtual void SetCapacity(size_t capacity) = 0;
+ virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
+ virtual size_t GetUsage() const = 0;
+ virtual size_t GetPinnedUsage() const = 0;
+ virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+ bool thread_safe) = 0;
+ virtual void EraseUnRefEntries() = 0;
+ virtual std::string GetPrintableOptions() const { return ""; }
+};
+
+// Generic cache interface which shards cache by hash of keys. 2^num_shard_bits
+// shards will be created, with capacity split evenly to each of the shards.
+// Keys are sharded by the highest num_shard_bits bits of hash value.
+class ShardedCache : public Cache {
+ public:
+ ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit);
+ virtual ~ShardedCache() = default;
+ virtual const char* Name() const override = 0;
+ virtual CacheShard* GetShard(int shard) = 0;
+ virtual const CacheShard* GetShard(int shard) const = 0;
+ virtual void* Value(Handle* handle) override = 0;
+ virtual size_t GetCharge(Handle* handle) const = 0;
+ virtual uint32_t GetHash(Handle* handle) const = 0;
+ virtual void DisownData() override = 0;
+
+ virtual void SetCapacity(size_t capacity) override;
+ virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
+
+ virtual Status Insert(const Slice& key, void* value, size_t charge,
+ void (*deleter)(const Slice& key, void* value),
+ Handle** handle, Priority priority) override;
+ virtual Handle* Lookup(const Slice& key, Statistics* stats) override;
+ virtual bool Ref(Handle* handle) override;
+ virtual bool Release(Handle* handle, bool force_erase = false) override;
+ virtual void Erase(const Slice& key) override;
+ virtual uint64_t NewId() override;
+ virtual size_t GetCapacity() const override;
+ virtual bool HasStrictCapacityLimit() const override;
+ virtual size_t GetUsage() const override;
+ virtual size_t GetUsage(Handle* handle) const override;
+ virtual size_t GetPinnedUsage() const override;
+ virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+ bool thread_safe) override;
+ virtual void EraseUnRefEntries() override;
+ virtual std::string GetPrintableOptions() const override;
+
+ int GetNumShardBits() const { return num_shard_bits_; }
+
+ private:
+ static inline uint32_t HashSlice(const Slice& s) {
+ return Hash(s.data(), s.size(), 0);
+ }
+
+ uint32_t Shard(uint32_t hash) {
+ // Note, hash >> 32 yields hash in gcc, not the zero we expect!
+ return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0;
+ }
+
+ int num_shard_bits_;
+ mutable port::Mutex capacity_mutex_;
+ size_t capacity_;
+ bool strict_capacity_limit_;
+ std::atomic<uint64_t> last_id_;
+};
+
+extern int GetDefaultCacheShardBits(size_t capacity);
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/coverage/parse_gcov_output.py
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/coverage/parse_gcov_output.py b/thirdparty/rocksdb/coverage/parse_gcov_output.py
new file mode 100644
index 0000000..72e8b07
--- /dev/null
+++ b/thirdparty/rocksdb/coverage/parse_gcov_output.py
@@ -0,0 +1,118 @@
+import optparse
+import re
+import sys
+
+from optparse import OptionParser
+
+# the gcov report follows certain pattern. Each file will have two lines
+# of report, from which we can extract the file name, total lines and coverage
+# percentage.
+def parse_gcov_report(gcov_input):
+ per_file_coverage = {}
+ total_coverage = None
+
+ for line in sys.stdin:
+ line = line.strip()
+
+ # --First line of the coverage report (with file name in it)?
+ match_obj = re.match("^File '(.*)'$", line)
+ if match_obj:
+ # fetch the file name from the first line of the report.
+ current_file = match_obj.group(1)
+ continue
+
+ # -- Second line of the file report (with coverage percentage)
+ match_obj = re.match("^Lines executed:(.*)% of (.*)", line)
+
+ if match_obj:
+ coverage = float(match_obj.group(1))
+ lines = int(match_obj.group(2))
+
+ if current_file is not None:
+ per_file_coverage[current_file] = (coverage, lines)
+ current_file = None
+ else:
+ # If current_file is not set, we reach the last line of report,
+ # which contains the summarized coverage percentage.
+ total_coverage = (coverage, lines)
+ continue
+
+ # If the line's pattern doesn't fall into the above categories. We
+ # can simply ignore them since they're either empty line or doesn't
+ # find executable lines of the given file.
+ current_file = None
+
+ return per_file_coverage, total_coverage
+
+def get_option_parser():
+ usage = "Parse the gcov output and generate more human-readable code " +\
+ "coverage report."
+ parser = OptionParser(usage)
+
+ parser.add_option(
+ "--interested-files", "-i",
+ dest="filenames",
+ help="Comma separated files names. if specified, we will display " +
+ "the coverage report only for interested source files. " +
+ "Otherwise we will display the coverage report for all " +
+ "source files."
+ )
+ return parser
+
+def display_file_coverage(per_file_coverage, total_coverage):
+ # To print out auto-adjustable column, we need to know the longest
+ # length of file names.
+ max_file_name_length = max(
+ len(fname) for fname in per_file_coverage.keys()
+ )
+
+ # -- Print header
+ # size of separator is determined by 3 column sizes:
+ # file name, coverage percentage and lines.
+ header_template = \
+ "%" + str(max_file_name_length) + "s\t%s\t%s"
+ separator = "-" * (max_file_name_length + 10 + 20)
+ print header_template % ("Filename", "Coverage", "Lines")
+ print separator
+
+ # -- Print body
+ # template for printing coverage report for each file.
+ record_template = "%" + str(max_file_name_length) + "s\t%5.2f%%\t%10d"
+
+ for fname, coverage_info in per_file_coverage.items():
+ coverage, lines = coverage_info
+ print record_template % (fname, coverage, lines)
+
+ # -- Print footer
+ if total_coverage:
+ print separator
+ print record_template % ("Total", total_coverage[0], total_coverage[1])
+
+def report_coverage():
+ parser = get_option_parser()
+ (options, args) = parser.parse_args()
+
+ interested_files = set()
+ if options.filenames is not None:
+ interested_files = set(f.strip() for f in options.filenames.split(','))
+
+ # To make things simple, right now we only read gcov report from the input
+ per_file_coverage, total_coverage = parse_gcov_report(sys.stdin)
+
+ # Check if we need to display coverage info for interested files.
+ if len(interested_files):
+ per_file_coverage = dict(
+ (fname, per_file_coverage[fname]) for fname in interested_files
+ if fname in per_file_coverage
+ )
+ # If we only interested in several files, it makes no sense to report
+ # the total_coverage
+ total_coverage = None
+
+ if not len(per_file_coverage):
+ print >> sys.stderr, "Cannot find coverage info for the given files."
+ return
+ display_file_coverage(per_file_coverage, total_coverage)
+
+if __name__ == "__main__":
+ report_coverage()
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/builder.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/builder.cc b/thirdparty/rocksdb/db/builder.cc
new file mode 100644
index 0000000..6f973fd
--- /dev/null
+++ b/thirdparty/rocksdb/db/builder.cc
@@ -0,0 +1,228 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/builder.h"
+
+#include <algorithm>
+#include <deque>
+#include <vector>
+
+#include "db/compaction_iterator.h"
+#include "db/dbformat.h"
+#include "db/event_helpers.h"
+#include "db/internal_stats.h"
+#include "db/merge_helper.h"
+#include "db/table_cache.h"
+#include "db/version_edit.h"
+#include "monitoring/iostats_context_imp.h"
+#include "monitoring/thread_status_util.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/options.h"
+#include "rocksdb/table.h"
+#include "table/block_based_table_builder.h"
+#include "table/internal_iterator.h"
+#include "util/file_reader_writer.h"
+#include "util/filename.h"
+#include "util/stop_watch.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+class TableFactory;
+
+TableBuilder* NewTableBuilder(
+ const ImmutableCFOptions& ioptions,
+ const InternalKeyComparator& internal_comparator,
+ const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+ int_tbl_prop_collector_factories,
+ uint32_t column_family_id, const std::string& column_family_name,
+ WritableFileWriter* file, const CompressionType compression_type,
+ const CompressionOptions& compression_opts, int level,
+ const std::string* compression_dict, const bool skip_filters,
+ const uint64_t creation_time) {
+ assert((column_family_id ==
+ TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
+ column_family_name.empty());
+ return ioptions.table_factory->NewTableBuilder(
+ TableBuilderOptions(ioptions, internal_comparator,
+ int_tbl_prop_collector_factories, compression_type,
+ compression_opts, compression_dict, skip_filters,
+ column_family_name, level, creation_time),
+ column_family_id, file);
+}
+
+Status BuildTable(
+ const std::string& dbname, Env* env, const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
+ TableCache* table_cache, InternalIterator* iter,
+ std::unique_ptr<InternalIterator> range_del_iter, FileMetaData* meta,
+ const InternalKeyComparator& internal_comparator,
+ const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+ int_tbl_prop_collector_factories,
+ uint32_t column_family_id, const std::string& column_family_name,
+ std::vector<SequenceNumber> snapshots,
+ SequenceNumber earliest_write_conflict_snapshot,
+ const CompressionType compression,
+ const CompressionOptions& compression_opts, bool paranoid_file_checks,
+ InternalStats* internal_stats, TableFileCreationReason reason,
+ EventLogger* event_logger, int job_id, const Env::IOPriority io_priority,
+ TableProperties* table_properties, int level,
+ const uint64_t creation_time) {
+ assert((column_family_id ==
+ TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
+ column_family_name.empty());
+ // Reports the IOStats for flush for every following bytes.
+ const size_t kReportFlushIOStatsEvery = 1048576;
+ Status s;
+ meta->fd.file_size = 0;
+ iter->SeekToFirst();
+ std::unique_ptr<RangeDelAggregator> range_del_agg(
+ new RangeDelAggregator(internal_comparator, snapshots));
+ s = range_del_agg->AddTombstones(std::move(range_del_iter));
+ if (!s.ok()) {
+ // may be non-ok if a range tombstone key is unparsable
+ return s;
+ }
+
+ std::string fname = TableFileName(ioptions.db_paths, meta->fd.GetNumber(),
+ meta->fd.GetPathId());
+#ifndef ROCKSDB_LITE
+ EventHelpers::NotifyTableFileCreationStarted(
+ ioptions.listeners, dbname, column_family_name, fname, job_id, reason);
+#endif // !ROCKSDB_LITE
+ TableProperties tp;
+
+ if (iter->Valid() || range_del_agg->ShouldAddTombstones()) {
+ TableBuilder* builder;
+ unique_ptr<WritableFileWriter> file_writer;
+ {
+ unique_ptr<WritableFile> file;
+#ifndef NDEBUG
+ bool use_direct_writes = env_options.use_direct_writes;
+ TEST_SYNC_POINT_CALLBACK("BuildTable:create_file", &use_direct_writes);
+#endif // !NDEBUG
+ s = NewWritableFile(env, fname, &file, env_options);
+ if (!s.ok()) {
+ EventHelpers::LogAndNotifyTableFileCreationFinished(
+ event_logger, ioptions.listeners, dbname, column_family_name, fname,
+ job_id, meta->fd, tp, reason, s);
+ return s;
+ }
+ file->SetIOPriority(io_priority);
+
+ file_writer.reset(new WritableFileWriter(std::move(file), env_options,
+ ioptions.statistics));
+
+ builder = NewTableBuilder(
+ ioptions, internal_comparator, int_tbl_prop_collector_factories,
+ column_family_id, column_family_name, file_writer.get(), compression,
+ compression_opts, level, nullptr /* compression_dict */,
+ false /* skip_filters */, creation_time);
+ }
+
+ MergeHelper merge(env, internal_comparator.user_comparator(),
+ ioptions.merge_operator, nullptr, ioptions.info_log,
+ true /* internal key corruption is not ok */,
+ snapshots.empty() ? 0 : snapshots.back());
+
+ CompactionIterator c_iter(
+ iter, internal_comparator.user_comparator(), &merge, kMaxSequenceNumber,
+ &snapshots, earliest_write_conflict_snapshot, env,
+ true /* internal key corruption is not ok */, range_del_agg.get());
+ c_iter.SeekToFirst();
+ for (; c_iter.Valid(); c_iter.Next()) {
+ const Slice& key = c_iter.key();
+ const Slice& value = c_iter.value();
+ builder->Add(key, value);
+ meta->UpdateBoundaries(key, c_iter.ikey().sequence);
+
+ // TODO(noetzli): Update stats after flush, too.
+ if (io_priority == Env::IO_HIGH &&
+ IOSTATS(bytes_written) >= kReportFlushIOStatsEvery) {
+ ThreadStatusUtil::SetThreadOperationProperty(
+ ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written));
+ }
+ }
+ // nullptr for table_{min,max} so all range tombstones will be flushed
+ range_del_agg->AddToBuilder(builder, nullptr /* lower_bound */,
+ nullptr /* upper_bound */, meta);
+
+ // Finish and check for builder errors
+ bool empty = builder->NumEntries() == 0;
+ s = c_iter.status();
+ if (!s.ok() || empty) {
+ builder->Abandon();
+ } else {
+ s = builder->Finish();
+ }
+
+ if (s.ok() && !empty) {
+ uint64_t file_size = builder->FileSize();
+ meta->fd.file_size = file_size;
+ meta->marked_for_compaction = builder->NeedCompact();
+ assert(meta->fd.GetFileSize() > 0);
+ tp = builder->GetTableProperties();
+ if (table_properties) {
+ *table_properties = tp;
+ }
+ }
+ delete builder;
+
+ // Finish and check for file errors
+ if (s.ok() && !empty) {
+ StopWatch sw(env, ioptions.statistics, TABLE_SYNC_MICROS);
+ s = file_writer->Sync(ioptions.use_fsync);
+ }
+ if (s.ok() && !empty) {
+ s = file_writer->Close();
+ }
+
+ if (s.ok() && !empty) {
+ // Verify that the table is usable
+ // We set for_compaction to false and don't OptimizeForCompactionTableRead
+ // here because this is a special case after we finish the table building
+ // No matter whether use_direct_io_for_flush_and_compaction is true,
+ // we will regrad this verification as user reads since the goal is
+ // to cache it here for further user reads
+ std::unique_ptr<InternalIterator> it(table_cache->NewIterator(
+ ReadOptions(), env_options, internal_comparator, meta->fd,
+ nullptr /* range_del_agg */, nullptr,
+ (internal_stats == nullptr) ? nullptr
+ : internal_stats->GetFileReadHist(0),
+ false /* for_compaction */, nullptr /* arena */,
+ false /* skip_filter */, level));
+ s = it->status();
+ if (s.ok() && paranoid_file_checks) {
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+ }
+ s = it->status();
+ }
+ }
+ }
+
+ // Check for input iterator errors
+ if (!iter->status().ok()) {
+ s = iter->status();
+ }
+
+ if (!s.ok() || meta->fd.GetFileSize() == 0) {
+ env->DeleteFile(fname);
+ }
+
+ // Output to event logger and fire events.
+ EventHelpers::LogAndNotifyTableFileCreationFinished(
+ event_logger, ioptions.listeners, dbname, column_family_name, fname,
+ job_id, meta->fd, tp, reason, s);
+
+ return s;
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/builder.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/builder.h b/thirdparty/rocksdb/db/builder.h
new file mode 100644
index 0000000..a432a75
--- /dev/null
+++ b/thirdparty/rocksdb/db/builder.h
@@ -0,0 +1,82 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#pragma once
+#include <string>
+#include <utility>
+#include <vector>
+#include "db/table_properties_collector.h"
+#include "options/cf_options.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/env.h"
+#include "rocksdb/listener.h"
+#include "rocksdb/options.h"
+#include "rocksdb/status.h"
+#include "rocksdb/table_properties.h"
+#include "rocksdb/types.h"
+#include "table/scoped_arena_iterator.h"
+#include "util/event_logger.h"
+
+namespace rocksdb {
+
+struct Options;
+struct FileMetaData;
+
+class Env;
+struct EnvOptions;
+class Iterator;
+class TableCache;
+class VersionEdit;
+class TableBuilder;
+class WritableFileWriter;
+class InternalStats;
+class InternalIterator;
+
+// @param column_family_name Name of the column family that is also identified
+// by column_family_id, or empty string if unknown. It must outlive the
+// TableBuilder returned by this function.
+// @param compression_dict Data for presetting the compression library's
+// dictionary, or nullptr.
+TableBuilder* NewTableBuilder(
+ const ImmutableCFOptions& options,
+ const InternalKeyComparator& internal_comparator,
+ const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+ int_tbl_prop_collector_factories,
+ uint32_t column_family_id, const std::string& column_family_name,
+ WritableFileWriter* file, const CompressionType compression_type,
+ const CompressionOptions& compression_opts, int level,
+ const std::string* compression_dict = nullptr,
+ const bool skip_filters = false, const uint64_t creation_time = 0);
+
+// Build a Table file from the contents of *iter. The generated file
+// will be named according to number specified in meta. On success, the rest of
+// *meta will be filled with metadata about the generated table.
+// If no data is present in *iter, meta->file_size will be set to
+// zero, and no Table file will be produced.
+//
+// @param column_family_name Name of the column family that is also identified
+// by column_family_id, or empty string if unknown.
+extern Status BuildTable(
+ const std::string& dbname, Env* env, const ImmutableCFOptions& options,
+ const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
+ TableCache* table_cache, InternalIterator* iter,
+ std::unique_ptr<InternalIterator> range_del_iter, FileMetaData* meta,
+ const InternalKeyComparator& internal_comparator,
+ const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+ int_tbl_prop_collector_factories,
+ uint32_t column_family_id, const std::string& column_family_name,
+ std::vector<SequenceNumber> snapshots,
+ SequenceNumber earliest_write_conflict_snapshot,
+ const CompressionType compression,
+ const CompressionOptions& compression_opts, bool paranoid_file_checks,
+ InternalStats* internal_stats, TableFileCreationReason reason,
+ EventLogger* event_logger = nullptr, int job_id = 0,
+ const Env::IOPriority io_priority = Env::IO_HIGH,
+ TableProperties* table_properties = nullptr, int level = -1,
+ const uint64_t creation_time = 0);
+
+} // namespace rocksdb
[19/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_impl_write.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_impl_write.cc b/thirdparty/rocksdb/db/db_impl_write.cc
new file mode 100644
index 0000000..b93dd6f
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_impl_write.cc
@@ -0,0 +1,1240 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include "db/db_impl.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+#include <inttypes.h>
+#include "db/event_helpers.h"
+#include "monitoring/perf_context_imp.h"
+#include "options/options_helper.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+// Convenience methods
+Status DBImpl::Put(const WriteOptions& o, ColumnFamilyHandle* column_family,
+ const Slice& key, const Slice& val) {
+ return DB::Put(o, column_family, key, val);
+}
+
+Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family,
+ const Slice& key, const Slice& val) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ if (!cfh->cfd()->ioptions()->merge_operator) {
+ return Status::NotSupported("Provide a merge_operator when opening DB");
+ } else {
+ return DB::Merge(o, column_family, key, val);
+ }
+}
+
+Status DBImpl::Delete(const WriteOptions& write_options,
+ ColumnFamilyHandle* column_family, const Slice& key) {
+ return DB::Delete(write_options, column_family, key);
+}
+
+Status DBImpl::SingleDelete(const WriteOptions& write_options,
+ ColumnFamilyHandle* column_family,
+ const Slice& key) {
+ return DB::SingleDelete(write_options, column_family, key);
+}
+
+Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) {
+ return WriteImpl(write_options, my_batch, nullptr, nullptr);
+}
+
+#ifndef ROCKSDB_LITE
+Status DBImpl::WriteWithCallback(const WriteOptions& write_options,
+ WriteBatch* my_batch,
+ WriteCallback* callback) {
+ return WriteImpl(write_options, my_batch, callback, nullptr);
+}
+#endif // ROCKSDB_LITE
+
+Status DBImpl::WriteImpl(const WriteOptions& write_options,
+ WriteBatch* my_batch, WriteCallback* callback,
+ uint64_t* log_used, uint64_t log_ref,
+ bool disable_memtable) {
+ if (my_batch == nullptr) {
+ return Status::Corruption("Batch is nullptr!");
+ }
+ if (concurrent_prepare_ && immutable_db_options_.enable_pipelined_write) {
+ return Status::NotSupported(
+ "pipelined_writes is not compatible with concurrent prepares");
+ }
+
+ Status status;
+ if (write_options.low_pri) {
+ status = ThrottleLowPriWritesIfNeeded(write_options, my_batch);
+ if (!status.ok()) {
+ return status;
+ }
+ }
+
+ if (concurrent_prepare_ && disable_memtable) {
+ return WriteImplWALOnly(write_options, my_batch, callback, log_used,
+ log_ref);
+ }
+
+ if (immutable_db_options_.enable_pipelined_write) {
+ return PipelinedWriteImpl(write_options, my_batch, callback, log_used,
+ log_ref, disable_memtable);
+ }
+
+ PERF_TIMER_GUARD(write_pre_and_post_process_time);
+ WriteThread::Writer w(write_options, my_batch, callback, log_ref,
+ disable_memtable);
+
+ if (!write_options.disableWAL) {
+ RecordTick(stats_, WRITE_WITH_WAL);
+ }
+
+ StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
+
+ write_thread_.JoinBatchGroup(&w);
+ if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
+ // we are a non-leader in a parallel group
+ PERF_TIMER_GUARD(write_memtable_time);
+
+ if (w.ShouldWriteToMemtable()) {
+ ColumnFamilyMemTablesImpl column_family_memtables(
+ versions_->GetColumnFamilySet());
+ w.status = WriteBatchInternal::InsertInto(
+ &w, w.sequence, &column_family_memtables, &flush_scheduler_,
+ write_options.ignore_missing_column_families, 0 /*log_number*/, this,
+ true /*concurrent_memtable_writes*/);
+ }
+
+ if (write_thread_.CompleteParallelMemTableWriter(&w)) {
+ // we're responsible for exit batch group
+ auto last_sequence = w.write_group->last_sequence;
+ versions_->SetLastSequence(last_sequence);
+ MemTableInsertStatusCheck(w.status);
+ write_thread_.ExitAsBatchGroupFollower(&w);
+ }
+ assert(w.state == WriteThread::STATE_COMPLETED);
+ // STATE_COMPLETED conditional below handles exit
+
+ status = w.FinalStatus();
+ }
+ if (w.state == WriteThread::STATE_COMPLETED) {
+ if (log_used != nullptr) {
+ *log_used = w.log_used;
+ }
+ // write is complete and leader has updated sequence
+ return w.FinalStatus();
+ }
+ // else we are the leader of the write batch group
+ assert(w.state == WriteThread::STATE_GROUP_LEADER);
+
+ // Once reaches this point, the current writer "w" will try to do its write
+ // job. It may also pick up some of the remaining writers in the "writers_"
+ // when it finds suitable, and finish them in the same write batch.
+ // This is how a write job could be done by the other writer.
+ WriteContext write_context;
+ WriteThread::WriteGroup write_group;
+ bool in_parallel_group = false;
+ uint64_t last_sequence = kMaxSequenceNumber;
+ if (!concurrent_prepare_) {
+ last_sequence = versions_->LastSequence();
+ }
+
+ mutex_.Lock();
+
+ bool need_log_sync = !write_options.disableWAL && write_options.sync;
+ bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
+ if (!concurrent_prepare_ || !disable_memtable) {
+ // With concurrent writes we do preprocess only in the write thread that
+ // also does write to memtable to avoid sync issue on shared data structure
+ // with the other thread
+ status = PreprocessWrite(write_options, &need_log_sync, &write_context);
+ }
+ log::Writer* log_writer = logs_.back().writer;
+
+ mutex_.Unlock();
+
+ // Add to log and apply to memtable. We can release the lock
+ // during this phase since &w is currently responsible for logging
+ // and protects against concurrent loggers and concurrent writes
+ // into memtables
+
+ last_batch_group_size_ =
+ write_thread_.EnterAsBatchGroupLeader(&w, &write_group);
+
+ if (status.ok()) {
+ // Rules for when we can update the memtable concurrently
+ // 1. supported by memtable
+ // 2. Puts are not okay if inplace_update_support
+ // 3. Merges are not okay
+ //
+ // Rules 1..2 are enforced by checking the options
+ // during startup (CheckConcurrentWritesSupported), so if
+ // options.allow_concurrent_memtable_write is true then they can be
+ // assumed to be true. Rule 3 is checked for each batch. We could
+ // relax rules 2 if we could prevent write batches from referring
+ // more than once to a particular key.
+ bool parallel = immutable_db_options_.allow_concurrent_memtable_write &&
+ write_group.size > 1;
+ int total_count = 0;
+ uint64_t total_byte_size = 0;
+ for (auto* writer : write_group) {
+ if (writer->CheckCallback(this)) {
+ if (writer->ShouldWriteToMemtable()) {
+ total_count += WriteBatchInternal::Count(writer->batch);
+ parallel = parallel && !writer->batch->HasMerge();
+ }
+
+ total_byte_size = WriteBatchInternal::AppendedByteSize(
+ total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
+ }
+ }
+
+ const bool concurrent_update = concurrent_prepare_;
+ // Update stats while we are an exclusive group leader, so we know
+ // that nobody else can be writing to these particular stats.
+ // We're optimistic, updating the stats before we successfully
+ // commit. That lets us release our leader status early.
+ auto stats = default_cf_internal_stats_;
+ stats->AddDBStats(InternalStats::NUMBER_KEYS_WRITTEN, total_count,
+ concurrent_update);
+ RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count);
+ stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size,
+ concurrent_update);
+ RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
+ stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1, concurrent_update);
+ RecordTick(stats_, WRITE_DONE_BY_SELF);
+ auto write_done_by_other = write_group.size - 1;
+ if (write_done_by_other > 0) {
+ stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER, write_done_by_other,
+ concurrent_update);
+ RecordTick(stats_, WRITE_DONE_BY_OTHER, write_done_by_other);
+ }
+ MeasureTime(stats_, BYTES_PER_WRITE, total_byte_size);
+
+ if (write_options.disableWAL) {
+ has_unpersisted_data_.store(true, std::memory_order_relaxed);
+ }
+
+ PERF_TIMER_STOP(write_pre_and_post_process_time);
+
+ if (!concurrent_prepare_) {
+ if (status.ok() && !write_options.disableWAL) {
+ PERF_TIMER_GUARD(write_wal_time);
+ status = WriteToWAL(write_group, log_writer, log_used, need_log_sync,
+ need_log_dir_sync, last_sequence + 1);
+ }
+ } else {
+ if (status.ok() && !write_options.disableWAL) {
+ PERF_TIMER_GUARD(write_wal_time);
+ // LastToBeWrittenSequence is increased inside WriteToWAL under
+ // wal_write_mutex_ to ensure ordered events in WAL
+ status = ConcurrentWriteToWAL(write_group, log_used, &last_sequence,
+ total_count);
+ } else {
+ // Otherwise we inc seq number for memtable writes
+ last_sequence = versions_->FetchAddLastToBeWrittenSequence(total_count);
+ }
+ }
+ assert(last_sequence != kMaxSequenceNumber);
+ const SequenceNumber current_sequence = last_sequence + 1;
+ last_sequence += total_count;
+
+ if (status.ok()) {
+ PERF_TIMER_GUARD(write_memtable_time);
+
+ if (!parallel) {
+ w.status = WriteBatchInternal::InsertInto(
+ write_group, current_sequence, column_family_memtables_.get(),
+ &flush_scheduler_, write_options.ignore_missing_column_families,
+ 0 /*recovery_log_number*/, this);
+ } else {
+ SequenceNumber next_sequence = current_sequence;
+ for (auto* writer : write_group) {
+ if (writer->ShouldWriteToMemtable()) {
+ writer->sequence = next_sequence;
+ next_sequence += WriteBatchInternal::Count(writer->batch);
+ }
+ }
+ write_group.last_sequence = last_sequence;
+ write_group.running.store(static_cast<uint32_t>(write_group.size),
+ std::memory_order_relaxed);
+ write_thread_.LaunchParallelMemTableWriters(&write_group);
+ in_parallel_group = true;
+
+ // Each parallel follower is doing each own writes. The leader should
+ // also do its own.
+ if (w.ShouldWriteToMemtable()) {
+ ColumnFamilyMemTablesImpl column_family_memtables(
+ versions_->GetColumnFamilySet());
+ assert(w.sequence == current_sequence);
+ w.status = WriteBatchInternal::InsertInto(
+ &w, w.sequence, &column_family_memtables, &flush_scheduler_,
+ write_options.ignore_missing_column_families, 0 /*log_number*/,
+ this, true /*concurrent_memtable_writes*/);
+ }
+ }
+ }
+ }
+ PERF_TIMER_START(write_pre_and_post_process_time);
+
+ if (!w.CallbackFailed()) {
+ WriteCallbackStatusCheck(status);
+ }
+
+ if (need_log_sync) {
+ mutex_.Lock();
+ MarkLogsSynced(logfile_number_, need_log_dir_sync, status);
+ mutex_.Unlock();
+ // Requesting sync with concurrent_prepare_ is expected to be very rare. We
+ // hance provide a simple implementation that is not necessarily efficient.
+ if (concurrent_prepare_) {
+ if (manual_wal_flush_) {
+ status = FlushWAL(true);
+ } else {
+ status = SyncWAL();
+ }
+ }
+ }
+
+ bool should_exit_batch_group = true;
+ if (in_parallel_group) {
+ // CompleteParallelWorker returns true if this thread should
+ // handle exit, false means somebody else did
+ should_exit_batch_group = write_thread_.CompleteParallelMemTableWriter(&w);
+ }
+ if (should_exit_batch_group) {
+ if (status.ok()) {
+ versions_->SetLastSequence(last_sequence);
+ }
+ MemTableInsertStatusCheck(w.status);
+ write_thread_.ExitAsBatchGroupLeader(write_group, w.status);
+ }
+
+ if (status.ok()) {
+ status = w.FinalStatus();
+ }
+ return status;
+}
+
+Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
+ WriteBatch* my_batch, WriteCallback* callback,
+ uint64_t* log_used, uint64_t log_ref,
+ bool disable_memtable) {
+ PERF_TIMER_GUARD(write_pre_and_post_process_time);
+ StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
+
+ WriteContext write_context;
+
+ WriteThread::Writer w(write_options, my_batch, callback, log_ref,
+ disable_memtable);
+ write_thread_.JoinBatchGroup(&w);
+ if (w.state == WriteThread::STATE_GROUP_LEADER) {
+ WriteThread::WriteGroup wal_write_group;
+ if (w.callback && !w.callback->AllowWriteBatching()) {
+ write_thread_.WaitForMemTableWriters();
+ }
+ mutex_.Lock();
+ bool need_log_sync = !write_options.disableWAL && write_options.sync;
+ bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
+ w.status = PreprocessWrite(write_options, &need_log_sync, &write_context);
+ log::Writer* log_writer = logs_.back().writer;
+ mutex_.Unlock();
+
+ // This can set non-OK status if callback fail.
+ last_batch_group_size_ =
+ write_thread_.EnterAsBatchGroupLeader(&w, &wal_write_group);
+ const SequenceNumber current_sequence =
+ write_thread_.UpdateLastSequence(versions_->LastSequence()) + 1;
+ size_t total_count = 0;
+ size_t total_byte_size = 0;
+
+ if (w.status.ok()) {
+ SequenceNumber next_sequence = current_sequence;
+ for (auto writer : wal_write_group) {
+ if (writer->CheckCallback(this)) {
+ if (writer->ShouldWriteToMemtable()) {
+ writer->sequence = next_sequence;
+ size_t count = WriteBatchInternal::Count(writer->batch);
+ next_sequence += count;
+ total_count += count;
+ }
+ total_byte_size = WriteBatchInternal::AppendedByteSize(
+ total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
+ }
+ }
+ if (w.disable_wal) {
+ has_unpersisted_data_.store(true, std::memory_order_relaxed);
+ }
+ write_thread_.UpdateLastSequence(current_sequence + total_count - 1);
+ }
+
+ auto stats = default_cf_internal_stats_;
+ stats->AddDBStats(InternalStats::NUMBER_KEYS_WRITTEN, total_count);
+ RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count);
+ stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size);
+ RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
+
+ PERF_TIMER_STOP(write_pre_and_post_process_time);
+
+ if (w.ShouldWriteToWAL()) {
+ PERF_TIMER_GUARD(write_wal_time);
+ stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1);
+ RecordTick(stats_, WRITE_DONE_BY_SELF, 1);
+ if (wal_write_group.size > 1) {
+ stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER,
+ wal_write_group.size - 1);
+ RecordTick(stats_, WRITE_DONE_BY_OTHER, wal_write_group.size - 1);
+ }
+ w.status = WriteToWAL(wal_write_group, log_writer, log_used,
+ need_log_sync, need_log_dir_sync, current_sequence);
+ }
+
+ if (!w.CallbackFailed()) {
+ WriteCallbackStatusCheck(w.status);
+ }
+
+ if (need_log_sync) {
+ mutex_.Lock();
+ MarkLogsSynced(logfile_number_, need_log_dir_sync, w.status);
+ mutex_.Unlock();
+ }
+
+ write_thread_.ExitAsBatchGroupLeader(wal_write_group, w.status);
+ }
+
+ WriteThread::WriteGroup memtable_write_group;
+ if (w.state == WriteThread::STATE_MEMTABLE_WRITER_LEADER) {
+ PERF_TIMER_GUARD(write_memtable_time);
+ assert(w.status.ok());
+ write_thread_.EnterAsMemTableWriter(&w, &memtable_write_group);
+ if (memtable_write_group.size > 1 &&
+ immutable_db_options_.allow_concurrent_memtable_write) {
+ write_thread_.LaunchParallelMemTableWriters(&memtable_write_group);
+ } else {
+ memtable_write_group.status = WriteBatchInternal::InsertInto(
+ memtable_write_group, w.sequence, column_family_memtables_.get(),
+ &flush_scheduler_, write_options.ignore_missing_column_families,
+ 0 /*log_number*/, this);
+ versions_->SetLastSequence(memtable_write_group.last_sequence);
+ write_thread_.ExitAsMemTableWriter(&w, memtable_write_group);
+ }
+ }
+
+ if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
+ assert(w.ShouldWriteToMemtable());
+ ColumnFamilyMemTablesImpl column_family_memtables(
+ versions_->GetColumnFamilySet());
+ w.status = WriteBatchInternal::InsertInto(
+ &w, w.sequence, &column_family_memtables, &flush_scheduler_,
+ write_options.ignore_missing_column_families, 0 /*log_number*/, this,
+ true /*concurrent_memtable_writes*/);
+ if (write_thread_.CompleteParallelMemTableWriter(&w)) {
+ MemTableInsertStatusCheck(w.status);
+ versions_->SetLastSequence(w.write_group->last_sequence);
+ write_thread_.ExitAsMemTableWriter(&w, *w.write_group);
+ }
+ }
+
+ assert(w.state == WriteThread::STATE_COMPLETED);
+ return w.FinalStatus();
+}
+
+Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
+ WriteBatch* my_batch, WriteCallback* callback,
+ uint64_t* log_used, uint64_t log_ref) {
+ Status status;
+ PERF_TIMER_GUARD(write_pre_and_post_process_time);
+ WriteThread::Writer w(write_options, my_batch, callback, log_ref,
+ true /* disable_memtable */);
+ if (write_options.disableWAL) {
+ return status;
+ }
+ RecordTick(stats_, WRITE_WITH_WAL);
+ StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
+
+ nonmem_write_thread_.JoinBatchGroup(&w);
+ assert(w.state != WriteThread::STATE_PARALLEL_MEMTABLE_WRITER);
+ if (w.state == WriteThread::STATE_COMPLETED) {
+ if (log_used != nullptr) {
+ *log_used = w.log_used;
+ }
+ return w.FinalStatus();
+ }
+ // else we are the leader of the write batch group
+ assert(w.state == WriteThread::STATE_GROUP_LEADER);
+ WriteContext write_context;
+ WriteThread::WriteGroup write_group;
+ uint64_t last_sequence;
+ nonmem_write_thread_.EnterAsBatchGroupLeader(&w, &write_group);
+ // Note: no need to update last_batch_group_size_ here since the batch writes
+ // to WAL only
+
+ uint64_t total_byte_size = 0;
+ for (auto* writer : write_group) {
+ if (writer->CheckCallback(this)) {
+ total_byte_size = WriteBatchInternal::AppendedByteSize(
+ total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
+ }
+ }
+
+ const bool concurrent_update = true;
+ // Update stats while we are an exclusive group leader, so we know
+ // that nobody else can be writing to these particular stats.
+ // We're optimistic, updating the stats before we successfully
+ // commit. That lets us release our leader status early.
+ auto stats = default_cf_internal_stats_;
+ stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size,
+ concurrent_update);
+ RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
+ stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1, concurrent_update);
+ RecordTick(stats_, WRITE_DONE_BY_SELF);
+ auto write_done_by_other = write_group.size - 1;
+ if (write_done_by_other > 0) {
+ stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER, write_done_by_other,
+ concurrent_update);
+ RecordTick(stats_, WRITE_DONE_BY_OTHER, write_done_by_other);
+ }
+ MeasureTime(stats_, BYTES_PER_WRITE, total_byte_size);
+
+ PERF_TIMER_STOP(write_pre_and_post_process_time);
+
+ PERF_TIMER_GUARD(write_wal_time);
+ // LastToBeWrittenSequence is increased inside WriteToWAL under
+ // wal_write_mutex_ to ensure ordered events in WAL
+ status = ConcurrentWriteToWAL(write_group, log_used, &last_sequence,
+ 0 /*total_count*/);
+ if (status.ok() && write_options.sync) {
+ // Requesting sync with concurrent_prepare_ is expected to be very rare. We
+ // hance provide a simple implementation that is not necessarily efficient.
+ if (manual_wal_flush_) {
+ status = FlushWAL(true);
+ } else {
+ status = SyncWAL();
+ }
+ }
+ PERF_TIMER_START(write_pre_and_post_process_time);
+
+ if (!w.CallbackFailed()) {
+ WriteCallbackStatusCheck(status);
+ }
+ nonmem_write_thread_.ExitAsBatchGroupLeader(write_group, w.status);
+ if (status.ok()) {
+ status = w.FinalStatus();
+ }
+ return status;
+}
+
+void DBImpl::WriteCallbackStatusCheck(const Status& status) {
+ // Is setting bg_error_ enough here? This will at least stop
+ // compaction and fail any further writes.
+ if (immutable_db_options_.paranoid_checks && !status.ok() &&
+ !status.IsBusy() && !status.IsIncomplete()) {
+ mutex_.Lock();
+ if (bg_error_.ok()) {
+ Status new_bg_error = status;
+ // may temporarily unlock and lock the mutex.
+ EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
+ BackgroundErrorReason::kWriteCallback,
+ &new_bg_error, &mutex_);
+ if (!new_bg_error.ok()) {
+ bg_error_ = new_bg_error; // stop compaction & fail any further writes
+ }
+ }
+ mutex_.Unlock();
+ }
+}
+
+void DBImpl::MemTableInsertStatusCheck(const Status& status) {
+ // A non-OK status here indicates that the state implied by the
+ // WAL has diverged from the in-memory state. This could be
+ // because of a corrupt write_batch (very bad), or because the
+ // client specified an invalid column family and didn't specify
+ // ignore_missing_column_families.
+ if (!status.ok()) {
+ mutex_.Lock();
+ assert(bg_error_.ok());
+ Status new_bg_error = status;
+ // may temporarily unlock and lock the mutex.
+ EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
+ BackgroundErrorReason::kMemTable,
+ &new_bg_error, &mutex_);
+ if (!new_bg_error.ok()) {
+ bg_error_ = new_bg_error; // stop compaction & fail any further writes
+ }
+ mutex_.Unlock();
+ }
+}
+
+Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
+ bool* need_log_sync,
+ WriteContext* write_context) {
+ mutex_.AssertHeld();
+ assert(write_context != nullptr && need_log_sync != nullptr);
+ Status status;
+
+ assert(!single_column_family_mode_ ||
+ versions_->GetColumnFamilySet()->NumberOfColumnFamilies() == 1);
+ if (UNLIKELY(status.ok() && !single_column_family_mode_ &&
+ total_log_size_ > GetMaxTotalWalSize())) {
+ status = HandleWALFull(write_context);
+ }
+
+ if (UNLIKELY(status.ok() && write_buffer_manager_->ShouldFlush())) {
+ // Before a new memtable is added in SwitchMemtable(),
+ // write_buffer_manager_->ShouldFlush() will keep returning true. If another
+ // thread is writing to another DB with the same write buffer, they may also
+ // be flushed. We may end up with flushing much more DBs than needed. It's
+ // suboptimal but still correct.
+ status = HandleWriteBufferFull(write_context);
+ }
+
+ if (UNLIKELY(status.ok() && !bg_error_.ok())) {
+ return bg_error_;
+ }
+
+ if (UNLIKELY(status.ok() && !flush_scheduler_.Empty())) {
+ status = ScheduleFlushes(write_context);
+ }
+
+ if (UNLIKELY(status.ok() && (write_controller_.IsStopped() ||
+ write_controller_.NeedsDelay()))) {
+ PERF_TIMER_GUARD(write_delay_time);
+ // We don't know size of curent batch so that we always use the size
+ // for previous one. It might create a fairness issue that expiration
+ // might happen for smaller writes but larger writes can go through.
+ // Can optimize it if it is an issue.
+ status = DelayWrite(last_batch_group_size_, write_options);
+ }
+
+ if (status.ok() && *need_log_sync) {
+ // Wait until the parallel syncs are finished. Any sync process has to sync
+ // the front log too so it is enough to check the status of front()
+ // We do a while loop since log_sync_cv_ is signalled when any sync is
+ // finished
+ // Note: there does not seem to be a reason to wait for parallel sync at
+ // this early step but it is not important since parallel sync (SyncWAL) and
+ // need_log_sync are usually not used together.
+ while (logs_.front().getting_synced) {
+ log_sync_cv_.Wait();
+ }
+ for (auto& log : logs_) {
+ assert(!log.getting_synced);
+ // This is just to prevent the logs to be synced by a parallel SyncWAL
+ // call. We will do the actual syncing later after we will write to the
+ // WAL.
+ // Note: there does not seem to be a reason to set this early before we
+ // actually write to the WAL
+ log.getting_synced = true;
+ }
+ } else {
+ *need_log_sync = false;
+ }
+
+ return status;
+}
+
+WriteBatch* DBImpl::MergeBatch(const WriteThread::WriteGroup& write_group,
+ WriteBatch* tmp_batch, size_t* write_with_wal) {
+ assert(write_with_wal != nullptr);
+ assert(tmp_batch != nullptr);
+ WriteBatch* merged_batch = nullptr;
+ *write_with_wal = 0;
+ auto* leader = write_group.leader;
+ if (write_group.size == 1 && leader->ShouldWriteToWAL() &&
+ leader->batch->GetWalTerminationPoint().is_cleared()) {
+ // we simply write the first WriteBatch to WAL if the group only
+ // contains one batch, that batch should be written to the WAL,
+ // and the batch is not wanting to be truncated
+ merged_batch = leader->batch;
+ *write_with_wal = 1;
+ } else {
+ // WAL needs all of the batches flattened into a single batch.
+ // We could avoid copying here with an iov-like AddRecord
+ // interface
+ merged_batch = tmp_batch;
+ for (auto writer : write_group) {
+ if (writer->ShouldWriteToWAL()) {
+ WriteBatchInternal::Append(merged_batch, writer->batch,
+ /*WAL_only*/ true);
+ (*write_with_wal)++;
+ }
+ }
+ }
+ return merged_batch;
+}
+
+// When concurrent_prepare_ is disabled, this function is called from the only
+// write thread. Otherwise this must be called holding log_write_mutex_.
+Status DBImpl::WriteToWAL(const WriteBatch& merged_batch,
+ log::Writer* log_writer, uint64_t* log_used,
+ uint64_t* log_size) {
+ assert(log_size != nullptr);
+ Slice log_entry = WriteBatchInternal::Contents(&merged_batch);
+ *log_size = log_entry.size();
+ Status status = log_writer->AddRecord(log_entry);
+ if (log_used != nullptr) {
+ *log_used = logfile_number_;
+ }
+ total_log_size_ += log_entry.size();
+ // TODO(myabandeh): it might be unsafe to access alive_log_files_.back() here
+ // since alive_log_files_ might be modified concurrently
+ alive_log_files_.back().AddSize(log_entry.size());
+ log_empty_ = false;
+ return status;
+}
+
+Status DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
+ log::Writer* log_writer, uint64_t* log_used,
+ bool need_log_sync, bool need_log_dir_sync,
+ SequenceNumber sequence) {
+ Status status;
+
+ size_t write_with_wal = 0;
+ WriteBatch* merged_batch =
+ MergeBatch(write_group, &tmp_batch_, &write_with_wal);
+ if (merged_batch == write_group.leader->batch) {
+ write_group.leader->log_used = logfile_number_;
+ } else if (write_with_wal > 1) {
+ for (auto writer : write_group) {
+ writer->log_used = logfile_number_;
+ }
+ }
+
+ WriteBatchInternal::SetSequence(merged_batch, sequence);
+
+ uint64_t log_size;
+ status = WriteToWAL(*merged_batch, log_writer, log_used, &log_size);
+
+ if (status.ok() && need_log_sync) {
+ StopWatch sw(env_, stats_, WAL_FILE_SYNC_MICROS);
+ // It's safe to access logs_ with unlocked mutex_ here because:
+ // - we've set getting_synced=true for all logs,
+ // so other threads won't pop from logs_ while we're here,
+ // - only writer thread can push to logs_, and we're in
+ // writer thread, so no one will push to logs_,
+ // - as long as other threads don't modify it, it's safe to read
+ // from std::deque from multiple threads concurrently.
+ for (auto& log : logs_) {
+ status = log.writer->file()->Sync(immutable_db_options_.use_fsync);
+ if (!status.ok()) {
+ break;
+ }
+ }
+ if (status.ok() && need_log_dir_sync) {
+ // We only sync WAL directory the first time WAL syncing is
+ // requested, so that in case users never turn on WAL sync,
+ // we can avoid the disk I/O in the write code path.
+ status = directories_.GetWalDir()->Fsync();
+ }
+ }
+
+ if (merged_batch == &tmp_batch_) {
+ tmp_batch_.Clear();
+ }
+ if (status.ok()) {
+ auto stats = default_cf_internal_stats_;
+ if (need_log_sync) {
+ stats->AddDBStats(InternalStats::WAL_FILE_SYNCED, 1);
+ RecordTick(stats_, WAL_FILE_SYNCED);
+ }
+ stats->AddDBStats(InternalStats::WAL_FILE_BYTES, log_size);
+ RecordTick(stats_, WAL_FILE_BYTES, log_size);
+ stats->AddDBStats(InternalStats::WRITE_WITH_WAL, write_with_wal);
+ RecordTick(stats_, WRITE_WITH_WAL, write_with_wal);
+ }
+ return status;
+}
+
+Status DBImpl::ConcurrentWriteToWAL(const WriteThread::WriteGroup& write_group,
+ uint64_t* log_used,
+ SequenceNumber* last_sequence,
+ int total_count) {
+ Status status;
+
+ WriteBatch tmp_batch;
+ size_t write_with_wal = 0;
+ WriteBatch* merged_batch =
+ MergeBatch(write_group, &tmp_batch, &write_with_wal);
+
+ // We need to lock log_write_mutex_ since logs_ and alive_log_files might be
+ // pushed back concurrently
+ log_write_mutex_.Lock();
+ if (merged_batch == write_group.leader->batch) {
+ write_group.leader->log_used = logfile_number_;
+ } else if (write_with_wal > 1) {
+ for (auto writer : write_group) {
+ writer->log_used = logfile_number_;
+ }
+ }
+ *last_sequence = versions_->FetchAddLastToBeWrittenSequence(total_count);
+ auto sequence = *last_sequence + 1;
+ WriteBatchInternal::SetSequence(merged_batch, sequence);
+
+ log::Writer* log_writer = logs_.back().writer;
+ uint64_t log_size;
+ status = WriteToWAL(*merged_batch, log_writer, log_used, &log_size);
+ log_write_mutex_.Unlock();
+
+ if (status.ok()) {
+ const bool concurrent = true;
+ auto stats = default_cf_internal_stats_;
+ stats->AddDBStats(InternalStats::WAL_FILE_BYTES, log_size, concurrent);
+ RecordTick(stats_, WAL_FILE_BYTES, log_size);
+ stats->AddDBStats(InternalStats::WRITE_WITH_WAL, write_with_wal,
+ concurrent);
+ RecordTick(stats_, WRITE_WITH_WAL, write_with_wal);
+ }
+ return status;
+}
+
+Status DBImpl::HandleWALFull(WriteContext* write_context) {
+ mutex_.AssertHeld();
+ assert(write_context != nullptr);
+ Status status;
+
+ if (alive_log_files_.begin()->getting_flushed) {
+ return status;
+ }
+
+ auto oldest_alive_log = alive_log_files_.begin()->number;
+ auto oldest_log_with_uncommited_prep = FindMinLogContainingOutstandingPrep();
+
+ if (allow_2pc() &&
+ oldest_log_with_uncommited_prep > 0 &&
+ oldest_log_with_uncommited_prep <= oldest_alive_log) {
+ if (unable_to_flush_oldest_log_) {
+ // we already attempted to flush all column families dependent on
+ // the oldest alive log but the log still contained uncommited transactions.
+ // the oldest alive log STILL contains uncommited transaction so there
+ // is still nothing that we can do.
+ return status;
+ } else {
+ ROCKS_LOG_WARN(
+ immutable_db_options_.info_log,
+ "Unable to release oldest log due to uncommited transaction");
+ unable_to_flush_oldest_log_ = true;
+ }
+ } else {
+ // we only mark this log as getting flushed if we have successfully
+ // flushed all data in this log. If this log contains outstanding prepared
+ // transactions then we cannot flush this log until those transactions are commited.
+ unable_to_flush_oldest_log_ = false;
+ alive_log_files_.begin()->getting_flushed = true;
+ }
+
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "Flushing all column families with data in WAL number %" PRIu64
+ ". Total log size is %" PRIu64
+ " while max_total_wal_size is %" PRIu64,
+ oldest_alive_log, total_log_size_.load(), GetMaxTotalWalSize());
+ // no need to refcount because drop is happening in write thread, so can't
+ // happen while we're in the write thread
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ if (cfd->IsDropped()) {
+ continue;
+ }
+ if (cfd->OldestLogToKeep() <= oldest_alive_log) {
+ status = SwitchMemtable(cfd, write_context);
+ if (!status.ok()) {
+ break;
+ }
+ cfd->imm()->FlushRequested();
+ SchedulePendingFlush(cfd);
+ }
+ }
+ MaybeScheduleFlushOrCompaction();
+ return status;
+}
+
+Status DBImpl::HandleWriteBufferFull(WriteContext* write_context) {
+ mutex_.AssertHeld();
+ assert(write_context != nullptr);
+ Status status;
+
+ // Before a new memtable is added in SwitchMemtable(),
+ // write_buffer_manager_->ShouldFlush() will keep returning true. If another
+ // thread is writing to another DB with the same write buffer, they may also
+ // be flushed. We may end up with flushing much more DBs than needed. It's
+ // suboptimal but still correct.
+ ROCKS_LOG_INFO(
+ immutable_db_options_.info_log,
+ "Flushing column family with largest mem table size. Write buffer is "
+ "using %" PRIu64 " bytes out of a total of %" PRIu64 ".",
+ write_buffer_manager_->memory_usage(),
+ write_buffer_manager_->buffer_size());
+ // no need to refcount because drop is happening in write thread, so can't
+ // happen while we're in the write thread
+ ColumnFamilyData* cfd_picked = nullptr;
+ SequenceNumber seq_num_for_cf_picked = kMaxSequenceNumber;
+
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ if (cfd->IsDropped()) {
+ continue;
+ }
+ if (!cfd->mem()->IsEmpty()) {
+ // We only consider active mem table, hoping immutable memtable is
+ // already in the process of flushing.
+ uint64_t seq = cfd->mem()->GetCreationSeq();
+ if (cfd_picked == nullptr || seq < seq_num_for_cf_picked) {
+ cfd_picked = cfd;
+ seq_num_for_cf_picked = seq;
+ }
+ }
+ }
+ if (cfd_picked != nullptr) {
+ status = SwitchMemtable(cfd_picked, write_context);
+ if (status.ok()) {
+ cfd_picked->imm()->FlushRequested();
+ SchedulePendingFlush(cfd_picked);
+ MaybeScheduleFlushOrCompaction();
+ }
+ }
+ return status;
+}
+
+uint64_t DBImpl::GetMaxTotalWalSize() const {
+ mutex_.AssertHeld();
+ return mutable_db_options_.max_total_wal_size == 0
+ ? 4 * max_total_in_memory_state_
+ : mutable_db_options_.max_total_wal_size;
+}
+
+// REQUIRES: mutex_ is held
+// REQUIRES: this thread is currently at the front of the writer queue
+Status DBImpl::DelayWrite(uint64_t num_bytes,
+ const WriteOptions& write_options) {
+ uint64_t time_delayed = 0;
+ bool delayed = false;
+ {
+ StopWatch sw(env_, stats_, WRITE_STALL, &time_delayed);
+ uint64_t delay = write_controller_.GetDelay(env_, num_bytes);
+ if (delay > 0) {
+ if (write_options.no_slowdown) {
+ return Status::Incomplete();
+ }
+ TEST_SYNC_POINT("DBImpl::DelayWrite:Sleep");
+
+ mutex_.Unlock();
+ // We will delay the write until we have slept for delay ms or
+ // we don't need a delay anymore
+ const uint64_t kDelayInterval = 1000;
+ uint64_t stall_end = sw.start_time() + delay;
+ while (write_controller_.NeedsDelay()) {
+ if (env_->NowMicros() >= stall_end) {
+ // We already delayed this write `delay` microseconds
+ break;
+ }
+
+ delayed = true;
+ // Sleep for 0.001 seconds
+ env_->SleepForMicroseconds(kDelayInterval);
+ }
+ mutex_.Lock();
+ }
+
+ while (bg_error_.ok() && write_controller_.IsStopped()) {
+ if (write_options.no_slowdown) {
+ return Status::Incomplete();
+ }
+ delayed = true;
+ TEST_SYNC_POINT("DBImpl::DelayWrite:Wait");
+ bg_cv_.Wait();
+ }
+ }
+ assert(!delayed || !write_options.no_slowdown);
+ if (delayed) {
+ default_cf_internal_stats_->AddDBStats(InternalStats::WRITE_STALL_MICROS,
+ time_delayed);
+ RecordTick(stats_, STALL_MICROS, time_delayed);
+ }
+
+ return bg_error_;
+}
+
+Status DBImpl::ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
+ WriteBatch* my_batch) {
+ assert(write_options.low_pri);
+ // This is called outside the DB mutex. Although it is safe to make the call,
+ // the consistency condition is not guaranteed to hold. It's OK to live with
+ // it in this case.
+ // If we need to speed compaction, it means the compaction is left behind
+ // and we start to limit low pri writes to a limit.
+ if (write_controller_.NeedSpeedupCompaction()) {
+ if (allow_2pc() && (my_batch->HasCommit() || my_batch->HasRollback())) {
+ // For 2PC, we only rate limit prepare, not commit.
+ return Status::OK();
+ }
+ if (write_options.no_slowdown) {
+ return Status::Incomplete();
+ } else {
+ assert(my_batch != nullptr);
+ // Rate limit those writes. The reason that we don't completely wait
+ // is that in case the write is heavy, low pri writes may never have
+ // a chance to run. Now we guarantee we are still slowly making
+ // progress.
+ write_controller_.low_pri_rate_limiter()->Request(
+ my_batch->GetDataSize(), Env::IO_HIGH, nullptr /* stats */,
+ RateLimiter::OpType::kWrite);
+ }
+ }
+ return Status::OK();
+}
+
+Status DBImpl::ScheduleFlushes(WriteContext* context) {
+ ColumnFamilyData* cfd;
+ while ((cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) {
+ auto status = SwitchMemtable(cfd, context);
+ if (cfd->Unref()) {
+ delete cfd;
+ }
+ if (!status.ok()) {
+ return status;
+ }
+ }
+ return Status::OK();
+}
+
+#ifndef ROCKSDB_LITE
+void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* cfd,
+ const MemTableInfo& mem_table_info) {
+ if (immutable_db_options_.listeners.size() == 0U) {
+ return;
+ }
+ if (shutting_down_.load(std::memory_order_acquire)) {
+ return;
+ }
+
+ for (auto listener : immutable_db_options_.listeners) {
+ listener->OnMemTableSealed(mem_table_info);
+ }
+}
+#endif // ROCKSDB_LITE
+
+// REQUIRES: mutex_ is held
+// REQUIRES: this thread is currently at the front of the writer queue
+Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
+ mutex_.AssertHeld();
+ WriteThread::Writer nonmem_w;
+ if (concurrent_prepare_) {
+ // SwitchMemtable is a rare event. To simply the reasoning, we make sure
+ // that there is no concurrent thread writing to WAL.
+ nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
+ }
+
+ unique_ptr<WritableFile> lfile;
+ log::Writer* new_log = nullptr;
+ MemTable* new_mem = nullptr;
+
+ // In case of pipelined write is enabled, wait for all pending memtable
+ // writers.
+ if (immutable_db_options_.enable_pipelined_write) {
+ write_thread_.WaitForMemTableWriters();
+ }
+
+ // Attempt to switch to a new memtable and trigger flush of old.
+ // Do this without holding the dbmutex lock.
+ assert(versions_->prev_log_number() == 0);
+ if (concurrent_prepare_) {
+ log_write_mutex_.Lock();
+ }
+ bool creating_new_log = !log_empty_;
+ if (concurrent_prepare_) {
+ log_write_mutex_.Unlock();
+ }
+ uint64_t recycle_log_number = 0;
+ if (creating_new_log && immutable_db_options_.recycle_log_file_num &&
+ !log_recycle_files.empty()) {
+ recycle_log_number = log_recycle_files.front();
+ log_recycle_files.pop_front();
+ }
+ uint64_t new_log_number =
+ creating_new_log ? versions_->NewFileNumber() : logfile_number_;
+ SuperVersion* new_superversion = nullptr;
+ const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
+
+ // Set current_memtble_info for memtable sealed callback
+#ifndef ROCKSDB_LITE
+ MemTableInfo memtable_info;
+ memtable_info.cf_name = cfd->GetName();
+ memtable_info.first_seqno = cfd->mem()->GetFirstSequenceNumber();
+ memtable_info.earliest_seqno = cfd->mem()->GetEarliestSequenceNumber();
+ memtable_info.num_entries = cfd->mem()->num_entries();
+ memtable_info.num_deletes = cfd->mem()->num_deletes();
+#endif // ROCKSDB_LITE
+ // Log this later after lock release. It may be outdated, e.g., if background
+ // flush happens before logging, but that should be ok.
+ int num_imm_unflushed = cfd->imm()->NumNotFlushed();
+ DBOptions db_options =
+ BuildDBOptions(immutable_db_options_, mutable_db_options_);
+ const auto preallocate_block_size =
+ GetWalPreallocateBlockSize(mutable_cf_options.write_buffer_size);
+ mutex_.Unlock();
+ Status s;
+ {
+ if (creating_new_log) {
+ EnvOptions opt_env_opt =
+ env_->OptimizeForLogWrite(env_options_, db_options);
+ if (recycle_log_number) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "reusing log %" PRIu64 " from recycle list\n",
+ recycle_log_number);
+ s = env_->ReuseWritableFile(
+ LogFileName(immutable_db_options_.wal_dir, new_log_number),
+ LogFileName(immutable_db_options_.wal_dir, recycle_log_number),
+ &lfile, opt_env_opt);
+ } else {
+ s = NewWritableFile(
+ env_, LogFileName(immutable_db_options_.wal_dir, new_log_number),
+ &lfile, opt_env_opt);
+ }
+ if (s.ok()) {
+ // Our final size should be less than write_buffer_size
+ // (compression, etc) but err on the side of caution.
+
+ // use preallocate_block_size instead
+ // of calling GetWalPreallocateBlockSize()
+ lfile->SetPreallocationBlockSize(preallocate_block_size);
+ unique_ptr<WritableFileWriter> file_writer(
+ new WritableFileWriter(std::move(lfile), opt_env_opt));
+ new_log = new log::Writer(
+ std::move(file_writer), new_log_number,
+ immutable_db_options_.recycle_log_file_num > 0, manual_wal_flush_);
+ }
+ }
+
+ if (s.ok()) {
+ SequenceNumber seq = versions_->LastSequence();
+ new_mem = cfd->ConstructNewMemtable(mutable_cf_options, seq);
+ new_superversion = new SuperVersion();
+ }
+
+#ifndef ROCKSDB_LITE
+ // PLEASE NOTE: We assume that there are no failable operations
+ // after lock is acquired below since we are already notifying
+ // client about mem table becoming immutable.
+ NotifyOnMemTableSealed(cfd, memtable_info);
+#endif //ROCKSDB_LITE
+ }
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "[%s] New memtable created with log file: #%" PRIu64
+ ". Immutable memtables: %d.\n",
+ cfd->GetName().c_str(), new_log_number, num_imm_unflushed);
+ mutex_.Lock();
+ if (!s.ok()) {
+ // how do we fail if we're not creating new log?
+ assert(creating_new_log);
+ assert(!new_mem);
+ assert(!new_log);
+ if (concurrent_prepare_) {
+ nonmem_write_thread_.ExitUnbatched(&nonmem_w);
+ }
+ return s;
+ }
+ if (creating_new_log) {
+ log_write_mutex_.Lock();
+ logfile_number_ = new_log_number;
+ assert(new_log != nullptr);
+ log_empty_ = true;
+ log_dir_synced_ = false;
+ if (!logs_.empty()) {
+ // Alway flush the buffer of the last log before switching to a new one
+ log::Writer* cur_log_writer = logs_.back().writer;
+ cur_log_writer->WriteBuffer();
+ }
+ logs_.emplace_back(logfile_number_, new_log);
+ alive_log_files_.push_back(LogFileNumberSize(logfile_number_));
+ log_write_mutex_.Unlock();
+ }
+ for (auto loop_cfd : *versions_->GetColumnFamilySet()) {
+ // all this is just optimization to delete logs that
+ // are no longer needed -- if CF is empty, that means it
+ // doesn't need that particular log to stay alive, so we just
+ // advance the log number. no need to persist this in the manifest
+ if (loop_cfd->mem()->GetFirstSequenceNumber() == 0 &&
+ loop_cfd->imm()->NumNotFlushed() == 0) {
+ if (creating_new_log) {
+ loop_cfd->SetLogNumber(logfile_number_);
+ }
+ loop_cfd->mem()->SetCreationSeq(versions_->LastSequence());
+ }
+ }
+
+ cfd->mem()->SetNextLogNumber(logfile_number_);
+ cfd->imm()->Add(cfd->mem(), &context->memtables_to_free_);
+ new_mem->Ref();
+ cfd->SetMemtable(new_mem);
+ context->superversions_to_free_.push_back(InstallSuperVersionAndScheduleWork(
+ cfd, new_superversion, mutable_cf_options));
+ if (concurrent_prepare_) {
+ nonmem_write_thread_.ExitUnbatched(&nonmem_w);
+ }
+ return s;
+}
+
+size_t DBImpl::GetWalPreallocateBlockSize(uint64_t write_buffer_size) const {
+ mutex_.AssertHeld();
+ size_t bsize = write_buffer_size / 10 + write_buffer_size;
+ // Some users might set very high write_buffer_size and rely on
+ // max_total_wal_size or other parameters to control the WAL size.
+ if (mutable_db_options_.max_total_wal_size > 0) {
+ bsize = std::min<size_t>(bsize, mutable_db_options_.max_total_wal_size);
+ }
+ if (immutable_db_options_.db_write_buffer_size > 0) {
+ bsize = std::min<size_t>(bsize, immutable_db_options_.db_write_buffer_size);
+ }
+ if (immutable_db_options_.write_buffer_manager &&
+ immutable_db_options_.write_buffer_manager->enabled()) {
+ bsize = std::min<size_t>(
+ bsize, immutable_db_options_.write_buffer_manager->buffer_size());
+ }
+
+ return bsize;
+}
+
+// Default implementations of convenience methods that subclasses of DB
+// can call if they wish
+Status DB::Put(const WriteOptions& opt, ColumnFamilyHandle* column_family,
+ const Slice& key, const Slice& value) {
+ // Pre-allocate size of write batch conservatively.
+ // 8 bytes are taken by header, 4 bytes for count, 1 byte for type,
+ // and we allocate 11 extra bytes for key length, as well as value length.
+ WriteBatch batch(key.size() + value.size() + 24);
+ batch.Put(column_family, key, value);
+ return Write(opt, &batch);
+}
+
+Status DB::Delete(const WriteOptions& opt, ColumnFamilyHandle* column_family,
+ const Slice& key) {
+ WriteBatch batch;
+ batch.Delete(column_family, key);
+ return Write(opt, &batch);
+}
+
+Status DB::SingleDelete(const WriteOptions& opt,
+ ColumnFamilyHandle* column_family, const Slice& key) {
+ WriteBatch batch;
+ batch.SingleDelete(column_family, key);
+ return Write(opt, &batch);
+}
+
+Status DB::DeleteRange(const WriteOptions& opt,
+ ColumnFamilyHandle* column_family,
+ const Slice& begin_key, const Slice& end_key) {
+ WriteBatch batch;
+ batch.DeleteRange(column_family, begin_key, end_key);
+ return Write(opt, &batch);
+}
+
+Status DB::Merge(const WriteOptions& opt, ColumnFamilyHandle* column_family,
+ const Slice& key, const Slice& value) {
+ WriteBatch batch;
+ batch.Merge(column_family, key, value);
+ return Write(opt, &batch);
+}
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_info_dumper.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_info_dumper.cc b/thirdparty/rocksdb/db/db_info_dumper.cc
new file mode 100644
index 0000000..1668a16
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_info_dumper.cc
@@ -0,0 +1,127 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include "db/db_info_dumper.h"
+
+#include <inttypes.h>
+#include <stdio.h>
+#include <string>
+#include <algorithm>
+#include <vector>
+
+#include "rocksdb/env.h"
+#include "util/filename.h"
+
+namespace rocksdb {
+
+void DumpDBFileSummary(const ImmutableDBOptions& options,
+ const std::string& dbname) {
+ if (options.info_log == nullptr) {
+ return;
+ }
+
+ auto* env = options.env;
+ uint64_t number = 0;
+ FileType type = kInfoLogFile;
+
+ std::vector<std::string> files;
+ uint64_t file_num = 0;
+ uint64_t file_size;
+ std::string file_info, wal_info;
+
+ Header(options.info_log, "DB SUMMARY\n");
+ // Get files in dbname dir
+ if (!env->GetChildren(dbname, &files).ok()) {
+ Error(options.info_log,
+ "Error when reading %s dir\n", dbname.c_str());
+ }
+ std::sort(files.begin(), files.end());
+ for (std::string file : files) {
+ if (!ParseFileName(file, &number, &type)) {
+ continue;
+ }
+ switch (type) {
+ case kCurrentFile:
+ Header(options.info_log, "CURRENT file: %s\n", file.c_str());
+ break;
+ case kIdentityFile:
+ Header(options.info_log, "IDENTITY file: %s\n", file.c_str());
+ break;
+ case kDescriptorFile:
+ env->GetFileSize(dbname + "/" + file, &file_size);
+ Header(options.info_log, "MANIFEST file: %s size: %" PRIu64 " Bytes\n",
+ file.c_str(), file_size);
+ break;
+ case kLogFile:
+ env->GetFileSize(dbname + "/" + file, &file_size);
+ char str[16];
+ snprintf(str, sizeof(str), "%" PRIu64, file_size);
+ wal_info.append(file).append(" size: ").
+ append(str).append(" ; ");
+ break;
+ case kTableFile:
+ if (++file_num < 10) {
+ file_info.append(file).append(" ");
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ // Get sst files in db_path dir
+ for (auto& db_path : options.db_paths) {
+ if (dbname.compare(db_path.path) != 0) {
+ if (!env->GetChildren(db_path.path, &files).ok()) {
+ Error(options.info_log,
+ "Error when reading %s dir\n",
+ db_path.path.c_str());
+ continue;
+ }
+ std::sort(files.begin(), files.end());
+ for (std::string file : files) {
+ if (ParseFileName(file, &number, &type)) {
+ if (type == kTableFile && ++file_num < 10) {
+ file_info.append(file).append(" ");
+ }
+ }
+ }
+ }
+ Header(options.info_log,
+ "SST files in %s dir, Total Num: %" PRIu64 ", files: %s\n",
+ db_path.path.c_str(), file_num, file_info.c_str());
+ file_num = 0;
+ file_info.clear();
+ }
+
+ // Get wal file in wal_dir
+ if (dbname.compare(options.wal_dir) != 0) {
+ if (!env->GetChildren(options.wal_dir, &files).ok()) {
+ Error(options.info_log,
+ "Error when reading %s dir\n",
+ options.wal_dir.c_str());
+ return;
+ }
+ wal_info.clear();
+ for (std::string file : files) {
+ if (ParseFileName(file, &number, &type)) {
+ if (type == kLogFile) {
+ env->GetFileSize(options.wal_dir + "/" + file, &file_size);
+ char str[16];
+ snprintf(str, sizeof(str), "%" PRIu64, file_size);
+ wal_info.append(file).append(" size: ").
+ append(str).append(" ; ");
+ }
+ }
+ }
+ }
+ Header(options.info_log, "Write Ahead Log file in %s: %s\n",
+ options.wal_dir.c_str(), wal_info.c_str());
+}
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_info_dumper.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_info_dumper.h b/thirdparty/rocksdb/db/db_info_dumper.h
new file mode 100644
index 0000000..acff8f1
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_info_dumper.h
@@ -0,0 +1,14 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+#pragma once
+
+#include <string>
+
+#include "options/db_options.h"
+
+namespace rocksdb {
+void DumpDBFileSummary(const ImmutableDBOptions& options,
+ const std::string& dbname);
+} // namespace rocksdb
[12/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/memtable.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/memtable.h b/thirdparty/rocksdb/db/memtable.h
new file mode 100644
index 0000000..fe9feaf
--- /dev/null
+++ b/thirdparty/rocksdb/db/memtable.h
@@ -0,0 +1,427 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <atomic>
+#include <deque>
+#include <functional>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include "db/dbformat.h"
+#include "db/range_del_aggregator.h"
+#include "db/version_edit.h"
+#include "monitoring/instrumented_mutex.h"
+#include "options/cf_options.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/memtablerep.h"
+#include "util/allocator.h"
+#include "util/concurrent_arena.h"
+#include "util/dynamic_bloom.h"
+#include "util/hash.h"
+
+namespace rocksdb {
+
+class Mutex;
+class MemTableIterator;
+class MergeContext;
+class InternalIterator;
+
+struct MemTableOptions {
+ explicit MemTableOptions(
+ const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& mutable_cf_options);
+ size_t write_buffer_size;
+ size_t arena_block_size;
+ uint32_t memtable_prefix_bloom_bits;
+ size_t memtable_huge_page_size;
+ bool inplace_update_support;
+ size_t inplace_update_num_locks;
+ UpdateStatus (*inplace_callback)(char* existing_value,
+ uint32_t* existing_value_size,
+ Slice delta_value,
+ std::string* merged_value);
+ size_t max_successive_merges;
+ Statistics* statistics;
+ MergeOperator* merge_operator;
+ Logger* info_log;
+};
+
+// Batched counters to updated when inserting keys in one write batch.
+// In post process of the write batch, these can be updated together.
+// Only used in concurrent memtable insert case.
+struct MemTablePostProcessInfo {
+ uint64_t data_size = 0;
+ uint64_t num_entries = 0;
+ uint64_t num_deletes = 0;
+};
+
+// Note: Many of the methods in this class have comments indicating that
+// external synchromization is required as these methods are not thread-safe.
+// It is up to higher layers of code to decide how to prevent concurrent
+// invokation of these methods. This is usually done by acquiring either
+// the db mutex or the single writer thread.
+//
+// Some of these methods are documented to only require external
+// synchronization if this memtable is immutable. Calling MarkImmutable() is
+// not sufficient to guarantee immutability. It is up to higher layers of
+// code to determine if this MemTable can still be modified by other threads.
+// Eg: The Superversion stores a pointer to the current MemTable (that can
+// be modified) and a separate list of the MemTables that can no longer be
+// written to (aka the 'immutable memtables').
+class MemTable {
+ public:
+ struct KeyComparator : public MemTableRep::KeyComparator {
+ const InternalKeyComparator comparator;
+ explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { }
+ virtual int operator()(const char* prefix_len_key1,
+ const char* prefix_len_key2) const override;
+ virtual int operator()(const char* prefix_len_key,
+ const Slice& key) const override;
+ };
+
+ // MemTables are reference counted. The initial reference count
+ // is zero and the caller must call Ref() at least once.
+ //
+ // earliest_seq should be the current SequenceNumber in the db such that any
+ // key inserted into this memtable will have an equal or larger seq number.
+ // (When a db is first created, the earliest sequence number will be 0).
+ // If the earliest sequence number is not known, kMaxSequenceNumber may be
+ // used, but this may prevent some transactions from succeeding until the
+ // first key is inserted into the memtable.
+ explicit MemTable(const InternalKeyComparator& comparator,
+ const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& mutable_cf_options,
+ WriteBufferManager* write_buffer_manager,
+ SequenceNumber earliest_seq, uint32_t column_family_id);
+
+ // Do not delete this MemTable unless Unref() indicates it not in use.
+ ~MemTable();
+
+ // Increase reference count.
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable.
+ void Ref() { ++refs_; }
+
+ // Drop reference count.
+ // If the refcount goes to zero return this memtable, otherwise return null.
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable.
+ MemTable* Unref() {
+ --refs_;
+ assert(refs_ >= 0);
+ if (refs_ <= 0) {
+ return this;
+ }
+ return nullptr;
+ }
+
+ // Returns an estimate of the number of bytes of data in use by this
+ // data structure.
+ //
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable (unless this Memtable is immutable).
+ size_t ApproximateMemoryUsage();
+
+ // This method heuristically determines if the memtable should continue to
+ // host more data.
+ bool ShouldScheduleFlush() const {
+ return flush_state_.load(std::memory_order_relaxed) == FLUSH_REQUESTED;
+ }
+
+ // Returns true if a flush should be scheduled and the caller should
+ // be the one to schedule it
+ bool MarkFlushScheduled() {
+ auto before = FLUSH_REQUESTED;
+ return flush_state_.compare_exchange_strong(before, FLUSH_SCHEDULED,
+ std::memory_order_relaxed,
+ std::memory_order_relaxed);
+ }
+
+ // Return an iterator that yields the contents of the memtable.
+ //
+ // The caller must ensure that the underlying MemTable remains live
+ // while the returned iterator is live. The keys returned by this
+ // iterator are internal keys encoded by AppendInternalKey in the
+ // db/dbformat.{h,cc} module.
+ //
+ // By default, it returns an iterator for prefix seek if prefix_extractor
+ // is configured in Options.
+ // arena: If not null, the arena needs to be used to allocate the Iterator.
+ // Calling ~Iterator of the iterator will destroy all the states but
+ // those allocated in arena.
+ InternalIterator* NewIterator(const ReadOptions& read_options, Arena* arena);
+
+ InternalIterator* NewRangeTombstoneIterator(const ReadOptions& read_options);
+
+ // Add an entry into memtable that maps key to value at the
+ // specified sequence number and with the specified type.
+ // Typically value will be empty if type==kTypeDeletion.
+ //
+ // REQUIRES: if allow_concurrent = false, external synchronization to prevent
+ // simultaneous operations on the same MemTable.
+ void Add(SequenceNumber seq, ValueType type, const Slice& key,
+ const Slice& value, bool allow_concurrent = false,
+ MemTablePostProcessInfo* post_process_info = nullptr);
+
+ // If memtable contains a value for key, store it in *value and return true.
+ // If memtable contains a deletion for key, store a NotFound() error
+ // in *status and return true.
+ // If memtable contains Merge operation as the most recent entry for a key,
+ // and the merge process does not stop (not reaching a value or delete),
+ // prepend the current merge operand to *operands.
+ // store MergeInProgress in s, and return false.
+ // Else, return false.
+ // If any operation was found, its most recent sequence number
+ // will be stored in *seq on success (regardless of whether true/false is
+ // returned). Otherwise, *seq will be set to kMaxSequenceNumber.
+ // On success, *s may be set to OK, NotFound, or MergeInProgress. Any other
+ // status returned indicates a corruption or other unexpected error.
+ bool Get(const LookupKey& key, std::string* value, Status* s,
+ MergeContext* merge_context, RangeDelAggregator* range_del_agg,
+ SequenceNumber* seq, const ReadOptions& read_opts);
+
+ bool Get(const LookupKey& key, std::string* value, Status* s,
+ MergeContext* merge_context, RangeDelAggregator* range_del_agg,
+ const ReadOptions& read_opts) {
+ SequenceNumber seq;
+ return Get(key, value, s, merge_context, range_del_agg, &seq, read_opts);
+ }
+
+ // Attempts to update the new_value inplace, else does normal Add
+ // Pseudocode
+ // if key exists in current memtable && prev_value is of type kTypeValue
+ // if new sizeof(new_value) <= sizeof(prev_value)
+ // update inplace
+ // else add(key, new_value)
+ // else add(key, new_value)
+ //
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable.
+ void Update(SequenceNumber seq,
+ const Slice& key,
+ const Slice& value);
+
+ // If prev_value for key exists, attempts to update it inplace.
+ // else returns false
+ // Pseudocode
+ // if key exists in current memtable && prev_value is of type kTypeValue
+ // new_value = delta(prev_value)
+ // if sizeof(new_value) <= sizeof(prev_value)
+ // update inplace
+ // else add(key, new_value)
+ // else return false
+ //
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable.
+ bool UpdateCallback(SequenceNumber seq,
+ const Slice& key,
+ const Slice& delta);
+
+ // Returns the number of successive merge entries starting from the newest
+ // entry for the key up to the last non-merge entry or last entry for the
+ // key in the memtable.
+ size_t CountSuccessiveMergeEntries(const LookupKey& key);
+
+ // Update counters and flush status after inserting a whole write batch
+ // Used in concurrent memtable inserts.
+ void BatchPostProcess(const MemTablePostProcessInfo& update_counters) {
+ num_entries_.fetch_add(update_counters.num_entries,
+ std::memory_order_relaxed);
+ data_size_.fetch_add(update_counters.data_size, std::memory_order_relaxed);
+ if (update_counters.num_deletes != 0) {
+ num_deletes_.fetch_add(update_counters.num_deletes,
+ std::memory_order_relaxed);
+ }
+ UpdateFlushState();
+ }
+
+ // Get total number of entries in the mem table.
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable (unless this Memtable is immutable).
+ uint64_t num_entries() const {
+ return num_entries_.load(std::memory_order_relaxed);
+ }
+
+ // Get total number of deletes in the mem table.
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable (unless this Memtable is immutable).
+ uint64_t num_deletes() const {
+ return num_deletes_.load(std::memory_order_relaxed);
+ }
+
+ // Returns the edits area that is needed for flushing the memtable
+ VersionEdit* GetEdits() { return &edit_; }
+
+ // Returns if there is no entry inserted to the mem table.
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable (unless this Memtable is immutable).
+ bool IsEmpty() const { return first_seqno_ == 0; }
+
+ // Returns the sequence number of the first element that was inserted
+ // into the memtable.
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable (unless this Memtable is immutable).
+ SequenceNumber GetFirstSequenceNumber() {
+ return first_seqno_.load(std::memory_order_relaxed);
+ }
+
+ // Returns the sequence number that is guaranteed to be smaller than or equal
+ // to the sequence number of any key that could be inserted into this
+ // memtable. It can then be assumed that any write with a larger(or equal)
+ // sequence number will be present in this memtable or a later memtable.
+ //
+ // If the earliest sequence number could not be determined,
+ // kMaxSequenceNumber will be returned.
+ SequenceNumber GetEarliestSequenceNumber() {
+ return earliest_seqno_.load(std::memory_order_relaxed);
+ }
+
+ // DB's latest sequence ID when the memtable is created. This number
+ // may be updated to a more recent one before any key is inserted.
+ SequenceNumber GetCreationSeq() const { return creation_seq_; }
+
+ void SetCreationSeq(SequenceNumber sn) { creation_seq_ = sn; }
+
+ // Returns the next active logfile number when this memtable is about to
+ // be flushed to storage
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable.
+ uint64_t GetNextLogNumber() { return mem_next_logfile_number_; }
+
+ // Sets the next active logfile number when this memtable is about to
+ // be flushed to storage
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable.
+ void SetNextLogNumber(uint64_t num) { mem_next_logfile_number_ = num; }
+
+ // if this memtable contains data from a committed
+ // two phase transaction we must take note of the
+ // log which contains that data so we can know
+ // when to relese that log
+ void RefLogContainingPrepSection(uint64_t log);
+ uint64_t GetMinLogContainingPrepSection();
+
+ // Notify the underlying storage that no more items will be added.
+ // REQUIRES: external synchronization to prevent simultaneous
+ // operations on the same MemTable.
+ // After MarkImmutable() is called, you should not attempt to
+ // write anything to this MemTable(). (Ie. do not call Add() or Update()).
+ void MarkImmutable() {
+ table_->MarkReadOnly();
+ mem_tracker_.DoneAllocating();
+ }
+
+ // return true if the current MemTableRep supports merge operator.
+ bool IsMergeOperatorSupported() const {
+ return table_->IsMergeOperatorSupported();
+ }
+
+ // return true if the current MemTableRep supports snapshots.
+ // inplace update prevents snapshots,
+ bool IsSnapshotSupported() const {
+ return table_->IsSnapshotSupported() && !moptions_.inplace_update_support;
+ }
+
+ struct MemTableStats {
+ uint64_t size;
+ uint64_t count;
+ };
+
+ MemTableStats ApproximateStats(const Slice& start_ikey,
+ const Slice& end_ikey);
+
+ // Get the lock associated for the key
+ port::RWMutex* GetLock(const Slice& key);
+
+ const InternalKeyComparator& GetInternalKeyComparator() const {
+ return comparator_.comparator;
+ }
+
+ const MemTableOptions* GetMemTableOptions() const { return &moptions_; }
+
+ private:
+ enum FlushStateEnum { FLUSH_NOT_REQUESTED, FLUSH_REQUESTED, FLUSH_SCHEDULED };
+
+ friend class MemTableIterator;
+ friend class MemTableBackwardIterator;
+ friend class MemTableList;
+
+ KeyComparator comparator_;
+ const MemTableOptions moptions_;
+ int refs_;
+ const size_t kArenaBlockSize;
+ AllocTracker mem_tracker_;
+ ConcurrentArena arena_;
+ unique_ptr<MemTableRep> table_;
+ unique_ptr<MemTableRep> range_del_table_;
+ bool is_range_del_table_empty_;
+
+ // Total data size of all data inserted
+ std::atomic<uint64_t> data_size_;
+ std::atomic<uint64_t> num_entries_;
+ std::atomic<uint64_t> num_deletes_;
+
+ // These are used to manage memtable flushes to storage
+ bool flush_in_progress_; // started the flush
+ bool flush_completed_; // finished the flush
+ uint64_t file_number_; // filled up after flush is complete
+
+ // The updates to be applied to the transaction log when this
+ // memtable is flushed to storage.
+ VersionEdit edit_;
+
+ // The sequence number of the kv that was inserted first
+ std::atomic<SequenceNumber> first_seqno_;
+
+ // The db sequence number at the time of creation or kMaxSequenceNumber
+ // if not set.
+ std::atomic<SequenceNumber> earliest_seqno_;
+
+ SequenceNumber creation_seq_;
+
+ // The log files earlier than this number can be deleted.
+ uint64_t mem_next_logfile_number_;
+
+ // the earliest log containing a prepared section
+ // which has been inserted into this memtable.
+ std::atomic<uint64_t> min_prep_log_referenced_;
+
+ // rw locks for inplace updates
+ std::vector<port::RWMutex> locks_;
+
+ const SliceTransform* const prefix_extractor_;
+ std::unique_ptr<DynamicBloom> prefix_bloom_;
+
+ std::atomic<FlushStateEnum> flush_state_;
+
+ Env* env_;
+
+ // Extract sequential insert prefixes.
+ const SliceTransform* insert_with_hint_prefix_extractor_;
+
+ // Insert hints for each prefix.
+ std::unordered_map<Slice, void*, SliceHasher> insert_hints_;
+
+ // Returns a heuristic flush decision
+ bool ShouldFlushNow() const;
+
+ // Updates flush_state_ using ShouldFlushNow()
+ void UpdateFlushState();
+
+ // No copying allowed
+ MemTable(const MemTable&);
+ MemTable& operator=(const MemTable&);
+};
+
+extern const char* EncodeKey(std::string* scratch, const Slice& target);
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/memtable_list.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/memtable_list.cc b/thirdparty/rocksdb/db/memtable_list.cc
new file mode 100644
index 0000000..8f710c2
--- /dev/null
+++ b/thirdparty/rocksdb/db/memtable_list.cc
@@ -0,0 +1,482 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+#include "db/memtable_list.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <string>
+#include "db/memtable.h"
+#include "db/version_set.h"
+#include "monitoring/thread_status_util.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/iterator.h"
+#include "table/merging_iterator.h"
+#include "util/coding.h"
+#include "util/log_buffer.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+class InternalKeyComparator;
+class Mutex;
+class VersionSet;
+
+void MemTableListVersion::AddMemTable(MemTable* m) {
+ memlist_.push_front(m);
+ *parent_memtable_list_memory_usage_ += m->ApproximateMemoryUsage();
+}
+
+void MemTableListVersion::UnrefMemTable(autovector<MemTable*>* to_delete,
+ MemTable* m) {
+ if (m->Unref()) {
+ to_delete->push_back(m);
+ assert(*parent_memtable_list_memory_usage_ >= m->ApproximateMemoryUsage());
+ *parent_memtable_list_memory_usage_ -= m->ApproximateMemoryUsage();
+ } else {
+ }
+}
+
+MemTableListVersion::MemTableListVersion(
+ size_t* parent_memtable_list_memory_usage, MemTableListVersion* old)
+ : max_write_buffer_number_to_maintain_(
+ old->max_write_buffer_number_to_maintain_),
+ parent_memtable_list_memory_usage_(parent_memtable_list_memory_usage) {
+ if (old != nullptr) {
+ memlist_ = old->memlist_;
+ for (auto& m : memlist_) {
+ m->Ref();
+ }
+
+ memlist_history_ = old->memlist_history_;
+ for (auto& m : memlist_history_) {
+ m->Ref();
+ }
+ }
+}
+
+MemTableListVersion::MemTableListVersion(
+ size_t* parent_memtable_list_memory_usage,
+ int max_write_buffer_number_to_maintain)
+ : max_write_buffer_number_to_maintain_(max_write_buffer_number_to_maintain),
+ parent_memtable_list_memory_usage_(parent_memtable_list_memory_usage) {}
+
+void MemTableListVersion::Ref() { ++refs_; }
+
+// called by superversion::clean()
+void MemTableListVersion::Unref(autovector<MemTable*>* to_delete) {
+ assert(refs_ >= 1);
+ --refs_;
+ if (refs_ == 0) {
+ // if to_delete is equal to nullptr it means we're confident
+ // that refs_ will not be zero
+ assert(to_delete != nullptr);
+ for (const auto& m : memlist_) {
+ UnrefMemTable(to_delete, m);
+ }
+ for (const auto& m : memlist_history_) {
+ UnrefMemTable(to_delete, m);
+ }
+ delete this;
+ }
+}
+
+int MemTableList::NumNotFlushed() const {
+ int size = static_cast<int>(current_->memlist_.size());
+ assert(num_flush_not_started_ <= size);
+ return size;
+}
+
+int MemTableList::NumFlushed() const {
+ return static_cast<int>(current_->memlist_history_.size());
+}
+
+// Search all the memtables starting from the most recent one.
+// Return the most recent value found, if any.
+// Operands stores the list of merge operations to apply, so far.
+bool MemTableListVersion::Get(const LookupKey& key, std::string* value,
+ Status* s, MergeContext* merge_context,
+ RangeDelAggregator* range_del_agg,
+ SequenceNumber* seq,
+ const ReadOptions& read_opts) {
+ return GetFromList(&memlist_, key, value, s, merge_context, range_del_agg,
+ seq, read_opts);
+}
+
+bool MemTableListVersion::GetFromHistory(const LookupKey& key,
+ std::string* value, Status* s,
+ MergeContext* merge_context,
+ RangeDelAggregator* range_del_agg,
+ SequenceNumber* seq,
+ const ReadOptions& read_opts) {
+ return GetFromList(&memlist_history_, key, value, s, merge_context,
+ range_del_agg, seq, read_opts);
+}
+
+bool MemTableListVersion::GetFromList(std::list<MemTable*>* list,
+ const LookupKey& key, std::string* value,
+ Status* s, MergeContext* merge_context,
+ RangeDelAggregator* range_del_agg,
+ SequenceNumber* seq,
+ const ReadOptions& read_opts) {
+ *seq = kMaxSequenceNumber;
+
+ for (auto& memtable : *list) {
+ SequenceNumber current_seq = kMaxSequenceNumber;
+
+ bool done = memtable->Get(key, value, s, merge_context, range_del_agg,
+ ¤t_seq, read_opts);
+ if (*seq == kMaxSequenceNumber) {
+ // Store the most recent sequence number of any operation on this key.
+ // Since we only care about the most recent change, we only need to
+ // return the first operation found when searching memtables in
+ // reverse-chronological order.
+ *seq = current_seq;
+ }
+
+ if (done) {
+ assert(*seq != kMaxSequenceNumber);
+ return true;
+ }
+ if (!done && !s->ok() && !s->IsMergeInProgress() && !s->IsNotFound()) {
+ return false;
+ }
+ }
+ return false;
+}
+
+Status MemTableListVersion::AddRangeTombstoneIterators(
+ const ReadOptions& read_opts, Arena* arena,
+ RangeDelAggregator* range_del_agg) {
+ assert(range_del_agg != nullptr);
+ for (auto& m : memlist_) {
+ std::unique_ptr<InternalIterator> range_del_iter(
+ m->NewRangeTombstoneIterator(read_opts));
+ Status s = range_del_agg->AddTombstones(std::move(range_del_iter));
+ if (!s.ok()) {
+ return s;
+ }
+ }
+ return Status::OK();
+}
+
+Status MemTableListVersion::AddRangeTombstoneIterators(
+ const ReadOptions& read_opts,
+ std::vector<InternalIterator*>* range_del_iters) {
+ for (auto& m : memlist_) {
+ auto* range_del_iter = m->NewRangeTombstoneIterator(read_opts);
+ if (range_del_iter != nullptr) {
+ range_del_iters->push_back(range_del_iter);
+ }
+ }
+ return Status::OK();
+}
+
+void MemTableListVersion::AddIterators(
+ const ReadOptions& options, std::vector<InternalIterator*>* iterator_list,
+ Arena* arena) {
+ for (auto& m : memlist_) {
+ iterator_list->push_back(m->NewIterator(options, arena));
+ }
+}
+
+void MemTableListVersion::AddIterators(
+ const ReadOptions& options, MergeIteratorBuilder* merge_iter_builder) {
+ for (auto& m : memlist_) {
+ merge_iter_builder->AddIterator(
+ m->NewIterator(options, merge_iter_builder->GetArena()));
+ }
+}
+
+uint64_t MemTableListVersion::GetTotalNumEntries() const {
+ uint64_t total_num = 0;
+ for (auto& m : memlist_) {
+ total_num += m->num_entries();
+ }
+ return total_num;
+}
+
+MemTable::MemTableStats MemTableListVersion::ApproximateStats(
+ const Slice& start_ikey, const Slice& end_ikey) {
+ MemTable::MemTableStats total_stats = {0, 0};
+ for (auto& m : memlist_) {
+ auto mStats = m->ApproximateStats(start_ikey, end_ikey);
+ total_stats.size += mStats.size;
+ total_stats.count += mStats.count;
+ }
+ return total_stats;
+}
+
+uint64_t MemTableListVersion::GetTotalNumDeletes() const {
+ uint64_t total_num = 0;
+ for (auto& m : memlist_) {
+ total_num += m->num_deletes();
+ }
+ return total_num;
+}
+
+SequenceNumber MemTableListVersion::GetEarliestSequenceNumber(
+ bool include_history) const {
+ if (include_history && !memlist_history_.empty()) {
+ return memlist_history_.back()->GetEarliestSequenceNumber();
+ } else if (!memlist_.empty()) {
+ return memlist_.back()->GetEarliestSequenceNumber();
+ } else {
+ return kMaxSequenceNumber;
+ }
+}
+
+// caller is responsible for referencing m
+void MemTableListVersion::Add(MemTable* m, autovector<MemTable*>* to_delete) {
+ assert(refs_ == 1); // only when refs_ == 1 is MemTableListVersion mutable
+ AddMemTable(m);
+
+ TrimHistory(to_delete);
+}
+
+// Removes m from list of memtables not flushed. Caller should NOT Unref m.
+void MemTableListVersion::Remove(MemTable* m,
+ autovector<MemTable*>* to_delete) {
+ assert(refs_ == 1); // only when refs_ == 1 is MemTableListVersion mutable
+ memlist_.remove(m);
+
+ if (max_write_buffer_number_to_maintain_ > 0) {
+ memlist_history_.push_front(m);
+ TrimHistory(to_delete);
+ } else {
+ UnrefMemTable(to_delete, m);
+ }
+}
+
+// Make sure we don't use up too much space in history
+void MemTableListVersion::TrimHistory(autovector<MemTable*>* to_delete) {
+ while (memlist_.size() + memlist_history_.size() >
+ static_cast<size_t>(max_write_buffer_number_to_maintain_) &&
+ !memlist_history_.empty()) {
+ MemTable* x = memlist_history_.back();
+ memlist_history_.pop_back();
+
+ UnrefMemTable(to_delete, x);
+ }
+}
+
+// Returns true if there is at least one memtable on which flush has
+// not yet started.
+bool MemTableList::IsFlushPending() const {
+ if ((flush_requested_ && num_flush_not_started_ >= 1) ||
+ (num_flush_not_started_ >= min_write_buffer_number_to_merge_)) {
+ assert(imm_flush_needed.load(std::memory_order_relaxed));
+ return true;
+ }
+ return false;
+}
+
+// Returns the memtables that need to be flushed.
+void MemTableList::PickMemtablesToFlush(autovector<MemTable*>* ret) {
+ AutoThreadOperationStageUpdater stage_updater(
+ ThreadStatus::STAGE_PICK_MEMTABLES_TO_FLUSH);
+ const auto& memlist = current_->memlist_;
+ for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) {
+ MemTable* m = *it;
+ if (!m->flush_in_progress_) {
+ assert(!m->flush_completed_);
+ num_flush_not_started_--;
+ if (num_flush_not_started_ == 0) {
+ imm_flush_needed.store(false, std::memory_order_release);
+ }
+ m->flush_in_progress_ = true; // flushing will start very soon
+ ret->push_back(m);
+ }
+ }
+ flush_requested_ = false; // start-flush request is complete
+}
+
+void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
+ uint64_t file_number) {
+ AutoThreadOperationStageUpdater stage_updater(
+ ThreadStatus::STAGE_MEMTABLE_ROLLBACK);
+ assert(!mems.empty());
+
+ // If the flush was not successful, then just reset state.
+ // Maybe a succeeding attempt to flush will be successful.
+ for (MemTable* m : mems) {
+ assert(m->flush_in_progress_);
+ assert(m->file_number_ == 0);
+
+ m->flush_in_progress_ = false;
+ m->flush_completed_ = false;
+ m->edit_.Clear();
+ num_flush_not_started_++;
+ }
+ imm_flush_needed.store(true, std::memory_order_release);
+}
+
+// Record a successful flush in the manifest file
+Status MemTableList::InstallMemtableFlushResults(
+ ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
+ const autovector<MemTable*>& mems, VersionSet* vset, InstrumentedMutex* mu,
+ uint64_t file_number, autovector<MemTable*>* to_delete,
+ Directory* db_directory, LogBuffer* log_buffer) {
+ AutoThreadOperationStageUpdater stage_updater(
+ ThreadStatus::STAGE_MEMTABLE_INSTALL_FLUSH_RESULTS);
+ mu->AssertHeld();
+
+ // flush was successful
+ for (size_t i = 0; i < mems.size(); ++i) {
+ // All the edits are associated with the first memtable of this batch.
+ assert(i == 0 || mems[i]->GetEdits()->NumEntries() == 0);
+
+ mems[i]->flush_completed_ = true;
+ mems[i]->file_number_ = file_number;
+ }
+
+ // if some other thread is already committing, then return
+ Status s;
+ if (commit_in_progress_) {
+ TEST_SYNC_POINT("MemTableList::InstallMemtableFlushResults:InProgress");
+ return s;
+ }
+
+ // Only a single thread can be executing this piece of code
+ commit_in_progress_ = true;
+
+ // Retry until all completed flushes are committed. New flushes can finish
+ // while the current thread is writing manifest where mutex is released.
+ while (s.ok()) {
+ auto& memlist = current_->memlist_;
+ if (memlist.empty() || !memlist.back()->flush_completed_) {
+ break;
+ }
+ // scan all memtables from the earliest, and commit those
+ // (in that order) that have finished flushing. Memetables
+ // are always committed in the order that they were created.
+ uint64_t batch_file_number = 0;
+ size_t batch_count = 0;
+ autovector<VersionEdit*> edit_list;
+ // enumerate from the last (earliest) element to see how many batch finished
+ for (auto it = memlist.rbegin(); it != memlist.rend(); ++it) {
+ MemTable* m = *it;
+ if (!m->flush_completed_) {
+ break;
+ }
+ if (it == memlist.rbegin() || batch_file_number != m->file_number_) {
+ batch_file_number = m->file_number_;
+ ROCKS_LOG_BUFFER(log_buffer,
+ "[%s] Level-0 commit table #%" PRIu64 " started",
+ cfd->GetName().c_str(), m->file_number_);
+ edit_list.push_back(&m->edit_);
+ }
+ batch_count++;
+ }
+
+ if (batch_count > 0) {
+ // this can release and reacquire the mutex.
+ s = vset->LogAndApply(cfd, mutable_cf_options, edit_list, mu,
+ db_directory);
+
+ // we will be changing the version in the next code path,
+ // so we better create a new one, since versions are immutable
+ InstallNewVersion();
+
+ // All the later memtables that have the same filenum
+ // are part of the same batch. They can be committed now.
+ uint64_t mem_id = 1; // how many memtables have been flushed.
+ if (s.ok()) { // commit new state
+ while (batch_count-- > 0) {
+ MemTable* m = current_->memlist_.back();
+ ROCKS_LOG_BUFFER(log_buffer, "[%s] Level-0 commit table #%" PRIu64
+ ": memtable #%" PRIu64 " done",
+ cfd->GetName().c_str(), m->file_number_, mem_id);
+ assert(m->file_number_ > 0);
+ current_->Remove(m, to_delete);
+ ++mem_id;
+ }
+ } else {
+ for (auto it = current_->memlist_.rbegin(); batch_count-- > 0; it++) {
+ MemTable* m = *it;
+ // commit failed. setup state so that we can flush again.
+ ROCKS_LOG_BUFFER(log_buffer, "Level-0 commit table #%" PRIu64
+ ": memtable #%" PRIu64 " failed",
+ m->file_number_, mem_id);
+ m->flush_completed_ = false;
+ m->flush_in_progress_ = false;
+ m->edit_.Clear();
+ num_flush_not_started_++;
+ m->file_number_ = 0;
+ imm_flush_needed.store(true, std::memory_order_release);
+ ++mem_id;
+ }
+ }
+ }
+ }
+ commit_in_progress_ = false;
+ return s;
+}
+
+// New memtables are inserted at the front of the list.
+void MemTableList::Add(MemTable* m, autovector<MemTable*>* to_delete) {
+ assert(static_cast<int>(current_->memlist_.size()) >= num_flush_not_started_);
+ InstallNewVersion();
+ // this method is used to move mutable memtable into an immutable list.
+ // since mutable memtable is already refcounted by the DBImpl,
+ // and when moving to the imutable list we don't unref it,
+ // we don't have to ref the memtable here. we just take over the
+ // reference from the DBImpl.
+ current_->Add(m, to_delete);
+ m->MarkImmutable();
+ num_flush_not_started_++;
+ if (num_flush_not_started_ == 1) {
+ imm_flush_needed.store(true, std::memory_order_release);
+ }
+}
+
+// Returns an estimate of the number of bytes of data in use.
+size_t MemTableList::ApproximateUnflushedMemTablesMemoryUsage() {
+ size_t total_size = 0;
+ for (auto& memtable : current_->memlist_) {
+ total_size += memtable->ApproximateMemoryUsage();
+ }
+ return total_size;
+}
+
+size_t MemTableList::ApproximateMemoryUsage() { return current_memory_usage_; }
+
+void MemTableList::InstallNewVersion() {
+ if (current_->refs_ == 1) {
+ // we're the only one using the version, just keep using it
+ } else {
+ // somebody else holds the current version, we need to create new one
+ MemTableListVersion* version = current_;
+ current_ = new MemTableListVersion(¤t_memory_usage_, current_);
+ current_->Ref();
+ version->Unref();
+ }
+}
+
+uint64_t MemTableList::GetMinLogContainingPrepSection() {
+ uint64_t min_log = 0;
+
+ for (auto& m : current_->memlist_) {
+ // this mem has been flushed it no longer
+ // needs to hold on the its prep section
+ if (m->flush_completed_) {
+ continue;
+ }
+
+ auto log = m->GetMinLogContainingPrepSection();
+
+ if (log > 0 && (min_log == 0 || log < min_log)) {
+ min_log = log;
+ }
+ }
+
+ return min_log;
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/memtable_list.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/memtable_list.h b/thirdparty/rocksdb/db/memtable_list.h
new file mode 100644
index 0000000..ed475b8
--- /dev/null
+++ b/thirdparty/rocksdb/db/memtable_list.h
@@ -0,0 +1,258 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+#pragma once
+
+#include <string>
+#include <list>
+#include <vector>
+#include <set>
+#include <deque>
+
+#include "db/dbformat.h"
+#include "db/memtable.h"
+#include "db/range_del_aggregator.h"
+#include "monitoring/instrumented_mutex.h"
+#include "rocksdb/db.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/options.h"
+#include "rocksdb/types.h"
+#include "util/autovector.h"
+#include "util/filename.h"
+#include "util/log_buffer.h"
+
+namespace rocksdb {
+
+class ColumnFamilyData;
+class InternalKeyComparator;
+class InstrumentedMutex;
+class MergeIteratorBuilder;
+
+// keeps a list of immutable memtables in a vector. the list is immutable
+// if refcount is bigger than one. It is used as a state for Get() and
+// Iterator code paths
+//
+// This class is not thread-safe. External synchronization is required
+// (such as holding the db mutex or being on the write thread).
+class MemTableListVersion {
+ public:
+ explicit MemTableListVersion(size_t* parent_memtable_list_memory_usage,
+ MemTableListVersion* old = nullptr);
+ explicit MemTableListVersion(size_t* parent_memtable_list_memory_usage,
+ int max_write_buffer_number_to_maintain);
+
+ void Ref();
+ void Unref(autovector<MemTable*>* to_delete = nullptr);
+
+ // Search all the memtables starting from the most recent one.
+ // Return the most recent value found, if any.
+ //
+ // If any operation was found for this key, its most recent sequence number
+ // will be stored in *seq on success (regardless of whether true/false is
+ // returned). Otherwise, *seq will be set to kMaxSequenceNumber.
+ bool Get(const LookupKey& key, std::string* value, Status* s,
+ MergeContext* merge_context, RangeDelAggregator* range_del_agg,
+ SequenceNumber* seq, const ReadOptions& read_opts);
+
+ bool Get(const LookupKey& key, std::string* value, Status* s,
+ MergeContext* merge_context, RangeDelAggregator* range_del_agg,
+ const ReadOptions& read_opts) {
+ SequenceNumber seq;
+ return Get(key, value, s, merge_context, range_del_agg, &seq, read_opts);
+ }
+
+ // Similar to Get(), but searches the Memtable history of memtables that
+ // have already been flushed. Should only be used from in-memory only
+ // queries (such as Transaction validation) as the history may contain
+ // writes that are also present in the SST files.
+ bool GetFromHistory(const LookupKey& key, std::string* value, Status* s,
+ MergeContext* merge_context,
+ RangeDelAggregator* range_del_agg, SequenceNumber* seq,
+ const ReadOptions& read_opts);
+ bool GetFromHistory(const LookupKey& key, std::string* value, Status* s,
+ MergeContext* merge_context,
+ RangeDelAggregator* range_del_agg,
+ const ReadOptions& read_opts) {
+ SequenceNumber seq;
+ return GetFromHistory(key, value, s, merge_context, range_del_agg, &seq,
+ read_opts);
+ }
+
+ Status AddRangeTombstoneIterators(const ReadOptions& read_opts, Arena* arena,
+ RangeDelAggregator* range_del_agg);
+ Status AddRangeTombstoneIterators(
+ const ReadOptions& read_opts,
+ std::vector<InternalIterator*>* range_del_iters);
+
+ void AddIterators(const ReadOptions& options,
+ std::vector<InternalIterator*>* iterator_list,
+ Arena* arena);
+
+ void AddIterators(const ReadOptions& options,
+ MergeIteratorBuilder* merge_iter_builder);
+
+ uint64_t GetTotalNumEntries() const;
+
+ uint64_t GetTotalNumDeletes() const;
+
+ MemTable::MemTableStats ApproximateStats(const Slice& start_ikey,
+ const Slice& end_ikey);
+
+ // Returns the value of MemTable::GetEarliestSequenceNumber() on the most
+ // recent MemTable in this list or kMaxSequenceNumber if the list is empty.
+ // If include_history=true, will also search Memtables in MemTableList
+ // History.
+ SequenceNumber GetEarliestSequenceNumber(bool include_history = false) const;
+
+ private:
+ // REQUIRE: m is an immutable memtable
+ void Add(MemTable* m, autovector<MemTable*>* to_delete);
+ // REQUIRE: m is an immutable memtable
+ void Remove(MemTable* m, autovector<MemTable*>* to_delete);
+
+ void TrimHistory(autovector<MemTable*>* to_delete);
+
+ bool GetFromList(std::list<MemTable*>* list, const LookupKey& key,
+ std::string* value, Status* s, MergeContext* merge_context,
+ RangeDelAggregator* range_del_agg, SequenceNumber* seq,
+ const ReadOptions& read_opts);
+
+ void AddMemTable(MemTable* m);
+
+ void UnrefMemTable(autovector<MemTable*>* to_delete, MemTable* m);
+
+ friend class MemTableList;
+
+ // Immutable MemTables that have not yet been flushed.
+ std::list<MemTable*> memlist_;
+
+ // MemTables that have already been flushed
+ // (used during Transaction validation)
+ std::list<MemTable*> memlist_history_;
+
+ // Maximum number of MemTables to keep in memory (including both flushed
+ // and not-yet-flushed tables).
+ const int max_write_buffer_number_to_maintain_;
+
+ int refs_ = 0;
+
+ size_t* parent_memtable_list_memory_usage_;
+};
+
+// This class stores references to all the immutable memtables.
+// The memtables are flushed to L0 as soon as possible and in
+// any order. If there are more than one immutable memtable, their
+// flushes can occur concurrently. However, they are 'committed'
+// to the manifest in FIFO order to maintain correctness and
+// recoverability from a crash.
+//
+//
+// Other than imm_flush_needed, this class is not thread-safe and requires
+// external synchronization (such as holding the db mutex or being on the
+// write thread.)
+class MemTableList {
+ public:
+ // A list of memtables.
+ explicit MemTableList(int min_write_buffer_number_to_merge,
+ int max_write_buffer_number_to_maintain)
+ : imm_flush_needed(false),
+ min_write_buffer_number_to_merge_(min_write_buffer_number_to_merge),
+ current_(new MemTableListVersion(¤t_memory_usage_,
+ max_write_buffer_number_to_maintain)),
+ num_flush_not_started_(0),
+ commit_in_progress_(false),
+ flush_requested_(false) {
+ current_->Ref();
+ current_memory_usage_ = 0;
+ }
+
+ // Should not delete MemTableList without making sure MemTableList::current()
+ // is Unref()'d.
+ ~MemTableList() {}
+
+ MemTableListVersion* current() { return current_; }
+
+ // so that background threads can detect non-nullptr pointer to
+ // determine whether there is anything more to start flushing.
+ std::atomic<bool> imm_flush_needed;
+
+ // Returns the total number of memtables in the list that haven't yet
+ // been flushed and logged.
+ int NumNotFlushed() const;
+
+ // Returns total number of memtables in the list that have been
+ // completely flushed and logged.
+ int NumFlushed() const;
+
+ // Returns true if there is at least one memtable on which flush has
+ // not yet started.
+ bool IsFlushPending() const;
+
+ // Returns the earliest memtables that needs to be flushed. The returned
+ // memtables are guaranteed to be in the ascending order of created time.
+ void PickMemtablesToFlush(autovector<MemTable*>* mems);
+
+ // Reset status of the given memtable list back to pending state so that
+ // they can get picked up again on the next round of flush.
+ void RollbackMemtableFlush(const autovector<MemTable*>& mems,
+ uint64_t file_number);
+
+ // Commit a successful flush in the manifest file
+ Status InstallMemtableFlushResults(
+ ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
+ const autovector<MemTable*>& m, VersionSet* vset, InstrumentedMutex* mu,
+ uint64_t file_number, autovector<MemTable*>* to_delete,
+ Directory* db_directory, LogBuffer* log_buffer);
+
+ // New memtables are inserted at the front of the list.
+ // Takes ownership of the referenced held on *m by the caller of Add().
+ void Add(MemTable* m, autovector<MemTable*>* to_delete);
+
+ // Returns an estimate of the number of bytes of data in use.
+ size_t ApproximateMemoryUsage();
+
+ // Returns an estimate of the number of bytes of data used by
+ // the unflushed mem-tables.
+ size_t ApproximateUnflushedMemTablesMemoryUsage();
+
+ // Request a flush of all existing memtables to storage. This will
+ // cause future calls to IsFlushPending() to return true if this list is
+ // non-empty (regardless of the min_write_buffer_number_to_merge
+ // parameter). This flush request will persist until the next time
+ // PickMemtablesToFlush() is called.
+ void FlushRequested() { flush_requested_ = true; }
+
+ bool HasFlushRequested() { return flush_requested_; }
+
+ // Copying allowed
+ // MemTableList(const MemTableList&);
+ // void operator=(const MemTableList&);
+
+ size_t* current_memory_usage() { return ¤t_memory_usage_; }
+
+ uint64_t GetMinLogContainingPrepSection();
+
+ private:
+ // DB mutex held
+ void InstallNewVersion();
+
+ const int min_write_buffer_number_to_merge_;
+
+ MemTableListVersion* current_;
+
+ // the number of elements that still need flushing
+ int num_flush_not_started_;
+
+ // committing in progress
+ bool commit_in_progress_;
+
+ // Requested a flush of all memtables to storage
+ bool flush_requested_;
+
+ // The current memory usage.
+ size_t current_memory_usage_;
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/merge_context.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/merge_context.h b/thirdparty/rocksdb/db/merge_context.h
new file mode 100644
index 0000000..5e75e09
--- /dev/null
+++ b/thirdparty/rocksdb/db/merge_context.h
@@ -0,0 +1,116 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+#pragma once
+#include <string>
+#include <vector>
+#include "db/dbformat.h"
+#include "rocksdb/slice.h"
+
+namespace rocksdb {
+
+const std::vector<Slice> empty_operand_list;
+
+// The merge context for merging a user key.
+// When doing a Get(), DB will create such a class and pass it when
+// issuing Get() operation to memtables and version_set. The operands
+// will be fetched from the context when issuing partial of full merge.
+class MergeContext {
+ public:
+ // Clear all the operands
+ void Clear() {
+ if (operand_list_) {
+ operand_list_->clear();
+ copied_operands_->clear();
+ }
+ }
+
+ // Push a merge operand
+ void PushOperand(const Slice& operand_slice, bool operand_pinned = false) {
+ Initialize();
+ SetDirectionBackward();
+
+ if (operand_pinned) {
+ operand_list_->push_back(operand_slice);
+ } else {
+ // We need to have our own copy of the operand since it's not pinned
+ copied_operands_->emplace_back(
+ new std::string(operand_slice.data(), operand_slice.size()));
+ operand_list_->push_back(*copied_operands_->back());
+ }
+ }
+
+ // Push back a merge operand
+ void PushOperandBack(const Slice& operand_slice,
+ bool operand_pinned = false) {
+ Initialize();
+ SetDirectionForward();
+
+ if (operand_pinned) {
+ operand_list_->push_back(operand_slice);
+ } else {
+ // We need to have our own copy of the operand since it's not pinned
+ copied_operands_->emplace_back(
+ new std::string(operand_slice.data(), operand_slice.size()));
+ operand_list_->push_back(*copied_operands_->back());
+ }
+ }
+
+ // return total number of operands in the list
+ size_t GetNumOperands() const {
+ if (!operand_list_) {
+ return 0;
+ }
+ return operand_list_->size();
+ }
+
+ // Get the operand at the index.
+ Slice GetOperand(int index) {
+ assert(operand_list_);
+
+ SetDirectionForward();
+ return (*operand_list_)[index];
+ }
+
+ // Return all the operands.
+ const std::vector<Slice>& GetOperands() {
+ if (!operand_list_) {
+ return empty_operand_list;
+ }
+
+ SetDirectionForward();
+ return *operand_list_;
+ }
+
+ private:
+ void Initialize() {
+ if (!operand_list_) {
+ operand_list_.reset(new std::vector<Slice>());
+ copied_operands_.reset(new std::vector<std::unique_ptr<std::string>>());
+ }
+ }
+
+ void SetDirectionForward() {
+ if (operands_reversed_ == true) {
+ std::reverse(operand_list_->begin(), operand_list_->end());
+ operands_reversed_ = false;
+ }
+ }
+
+ void SetDirectionBackward() {
+ if (operands_reversed_ == false) {
+ std::reverse(operand_list_->begin(), operand_list_->end());
+ operands_reversed_ = true;
+ }
+ }
+
+ // List of operands
+ std::unique_ptr<std::vector<Slice>> operand_list_;
+ // Copy of operands that are not pinned.
+ std::unique_ptr<std::vector<std::unique_ptr<std::string>>> copied_operands_;
+ bool operands_reversed_ = true;
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/merge_helper.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/merge_helper.cc b/thirdparty/rocksdb/db/merge_helper.cc
new file mode 100644
index 0000000..625de27
--- /dev/null
+++ b/thirdparty/rocksdb/db/merge_helper.cc
@@ -0,0 +1,365 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "db/merge_helper.h"
+
+#include <stdio.h>
+#include <string>
+
+#include "db/dbformat.h"
+#include "monitoring/perf_context_imp.h"
+#include "monitoring/statistics.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/db.h"
+#include "rocksdb/merge_operator.h"
+#include "table/internal_iterator.h"
+
+namespace rocksdb {
+
+Status MergeHelper::TimedFullMerge(const MergeOperator* merge_operator,
+ const Slice& key, const Slice* value,
+ const std::vector<Slice>& operands,
+ std::string* result, Logger* logger,
+ Statistics* statistics, Env* env,
+ Slice* result_operand,
+ bool update_num_ops_stats) {
+ assert(merge_operator != nullptr);
+
+ if (operands.size() == 0) {
+ assert(value != nullptr && result != nullptr);
+ result->assign(value->data(), value->size());
+ return Status::OK();
+ }
+
+ if (update_num_ops_stats) {
+ MeasureTime(statistics, READ_NUM_MERGE_OPERANDS,
+ static_cast<uint64_t>(operands.size()));
+ }
+
+ bool success;
+ Slice tmp_result_operand(nullptr, 0);
+ const MergeOperator::MergeOperationInput merge_in(key, value, operands,
+ logger);
+ MergeOperator::MergeOperationOutput merge_out(*result, tmp_result_operand);
+ {
+ // Setup to time the merge
+ StopWatchNano timer(env, statistics != nullptr);
+ PERF_TIMER_GUARD(merge_operator_time_nanos);
+
+ // Do the merge
+ success = merge_operator->FullMergeV2(merge_in, &merge_out);
+
+ if (tmp_result_operand.data()) {
+ // FullMergeV2 result is an existing operand
+ if (result_operand != nullptr) {
+ *result_operand = tmp_result_operand;
+ } else {
+ result->assign(tmp_result_operand.data(), tmp_result_operand.size());
+ }
+ } else if (result_operand) {
+ *result_operand = Slice(nullptr, 0);
+ }
+
+ RecordTick(statistics, MERGE_OPERATION_TOTAL_TIME,
+ statistics ? timer.ElapsedNanos() : 0);
+ }
+
+ if (!success) {
+ RecordTick(statistics, NUMBER_MERGE_FAILURES);
+ return Status::Corruption("Error: Could not perform merge.");
+ }
+
+ return Status::OK();
+}
+
+// PRE: iter points to the first merge type entry
+// POST: iter points to the first entry beyond the merge process (or the end)
+// keys_, operands_ are updated to reflect the merge result.
+// keys_ stores the list of keys encountered while merging.
+// operands_ stores the list of merge operands encountered while merging.
+// keys_[i] corresponds to operands_[i] for each i.
+Status MergeHelper::MergeUntil(InternalIterator* iter,
+ RangeDelAggregator* range_del_agg,
+ const SequenceNumber stop_before,
+ const bool at_bottom) {
+ // Get a copy of the internal key, before it's invalidated by iter->Next()
+ // Also maintain the list of merge operands seen.
+ assert(HasOperator());
+ keys_.clear();
+ merge_context_.Clear();
+ has_compaction_filter_skip_until_ = false;
+ assert(user_merge_operator_);
+ bool first_key = true;
+
+ // We need to parse the internal key again as the parsed key is
+ // backed by the internal key!
+ // Assume no internal key corruption as it has been successfully parsed
+ // by the caller.
+ // original_key_is_iter variable is just caching the information:
+ // original_key_is_iter == (iter->key().ToString() == original_key)
+ bool original_key_is_iter = true;
+ std::string original_key = iter->key().ToString();
+ // Important:
+ // orig_ikey is backed by original_key if keys_.empty()
+ // orig_ikey is backed by keys_.back() if !keys_.empty()
+ ParsedInternalKey orig_ikey;
+ ParseInternalKey(original_key, &orig_ikey);
+
+ Status s;
+ bool hit_the_next_user_key = false;
+ for (; iter->Valid(); iter->Next(), original_key_is_iter = false) {
+ if (IsShuttingDown()) {
+ return Status::ShutdownInProgress();
+ }
+
+ ParsedInternalKey ikey;
+ assert(keys_.size() == merge_context_.GetNumOperands());
+
+ if (!ParseInternalKey(iter->key(), &ikey)) {
+ // stop at corrupted key
+ if (assert_valid_internal_key_) {
+ assert(!"Corrupted internal key not expected.");
+ return Status::Corruption("Corrupted internal key not expected.");
+ }
+ break;
+ } else if (first_key) {
+ assert(user_comparator_->Equal(ikey.user_key, orig_ikey.user_key));
+ first_key = false;
+ } else if (!user_comparator_->Equal(ikey.user_key, orig_ikey.user_key)) {
+ // hit a different user key, stop right here
+ hit_the_next_user_key = true;
+ break;
+ } else if (stop_before && ikey.sequence <= stop_before) {
+ // hit an entry that's visible by the previous snapshot, can't touch that
+ break;
+ }
+
+ // At this point we are guaranteed that we need to process this key.
+
+ assert(IsValueType(ikey.type));
+ if (ikey.type != kTypeMerge) {
+
+ // hit a put/delete/single delete
+ // => merge the put value or a nullptr with operands_
+ // => store result in operands_.back() (and update keys_.back())
+ // => change the entry type to kTypeValue for keys_.back()
+ // We are done! Success!
+
+ // If there are no operands, just return the Status::OK(). That will cause
+ // the compaction iterator to write out the key we're currently at, which
+ // is the put/delete we just encountered.
+ if (keys_.empty()) {
+ return Status::OK();
+ }
+
+ // TODO(noetzli) If the merge operator returns false, we are currently
+ // (almost) silently dropping the put/delete. That's probably not what we
+ // want. Also if we're in compaction and it's a put, it would be nice to
+ // run compaction filter on it.
+ const Slice val = iter->value();
+ const Slice* val_ptr = (kTypeValue == ikey.type) ? &val : nullptr;
+ std::string merge_result;
+ s = TimedFullMerge(user_merge_operator_, ikey.user_key, val_ptr,
+ merge_context_.GetOperands(), &merge_result, logger_,
+ stats_, env_);
+
+ // We store the result in keys_.back() and operands_.back()
+ // if nothing went wrong (i.e.: no operand corruption on disk)
+ if (s.ok()) {
+ // The original key encountered
+ original_key = std::move(keys_.back());
+ orig_ikey.type = kTypeValue;
+ UpdateInternalKey(&original_key, orig_ikey.sequence, orig_ikey.type);
+ keys_.clear();
+ merge_context_.Clear();
+ keys_.emplace_front(std::move(original_key));
+ merge_context_.PushOperand(merge_result);
+ }
+
+ // move iter to the next entry
+ iter->Next();
+ return s;
+ } else {
+ // hit a merge
+ // => if there is a compaction filter, apply it.
+ // => check for range tombstones covering the operand
+ // => merge the operand into the front of the operands_ list
+ // if not filtered
+ // => then continue because we haven't yet seen a Put/Delete.
+ //
+ // Keep queuing keys and operands until we either meet a put / delete
+ // request or later did a partial merge.
+
+ Slice value_slice = iter->value();
+ // add an operand to the list if:
+ // 1) it's included in one of the snapshots. in that case we *must* write
+ // it out, no matter what compaction filter says
+ // 2) it's not filtered by a compaction filter
+ CompactionFilter::Decision filter =
+ ikey.sequence <= latest_snapshot_
+ ? CompactionFilter::Decision::kKeep
+ : FilterMerge(orig_ikey.user_key, value_slice);
+ if (filter != CompactionFilter::Decision::kRemoveAndSkipUntil &&
+ range_del_agg != nullptr &&
+ range_del_agg->ShouldDelete(
+ iter->key(),
+ RangeDelAggregator::RangePositioningMode::kForwardTraversal)) {
+ filter = CompactionFilter::Decision::kRemove;
+ }
+ if (filter == CompactionFilter::Decision::kKeep ||
+ filter == CompactionFilter::Decision::kChangeValue) {
+ if (original_key_is_iter) {
+ // this is just an optimization that saves us one memcpy
+ keys_.push_front(std::move(original_key));
+ } else {
+ keys_.push_front(iter->key().ToString());
+ }
+ if (keys_.size() == 1) {
+ // we need to re-anchor the orig_ikey because it was anchored by
+ // original_key before
+ ParseInternalKey(keys_.back(), &orig_ikey);
+ }
+ if (filter == CompactionFilter::Decision::kKeep) {
+ merge_context_.PushOperand(
+ value_slice, iter->IsValuePinned() /* operand_pinned */);
+ } else { // kChangeValue
+ // Compaction filter asked us to change the operand from value_slice
+ // to compaction_filter_value_.
+ merge_context_.PushOperand(compaction_filter_value_, false);
+ }
+ } else if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil) {
+ // Compaction filter asked us to remove this key altogether
+ // (not just this operand), along with some keys following it.
+ keys_.clear();
+ merge_context_.Clear();
+ has_compaction_filter_skip_until_ = true;
+ return Status::OK();
+ }
+ }
+ }
+
+ if (merge_context_.GetNumOperands() == 0) {
+ // we filtered out all the merge operands
+ return Status::OK();
+ }
+
+ // We are sure we have seen this key's entire history if we are at the
+ // last level and exhausted all internal keys of this user key.
+ // NOTE: !iter->Valid() does not necessarily mean we hit the
+ // beginning of a user key, as versions of a user key might be
+ // split into multiple files (even files on the same level)
+ // and some files might not be included in the compaction/merge.
+ //
+ // There are also cases where we have seen the root of history of this
+ // key without being sure of it. Then, we simply miss the opportunity
+ // to combine the keys. Since VersionSet::SetupOtherInputs() always makes
+ // sure that all merge-operands on the same level get compacted together,
+ // this will simply lead to these merge operands moving to the next level.
+ //
+ // So, we only perform the following logic (to merge all operands together
+ // without a Put/Delete) if we are certain that we have seen the end of key.
+ bool surely_seen_the_beginning = hit_the_next_user_key && at_bottom;
+ if (surely_seen_the_beginning) {
+ // do a final merge with nullptr as the existing value and say
+ // bye to the merge type (it's now converted to a Put)
+ assert(kTypeMerge == orig_ikey.type);
+ assert(merge_context_.GetNumOperands() >= 1);
+ assert(merge_context_.GetNumOperands() == keys_.size());
+ std::string merge_result;
+ s = TimedFullMerge(user_merge_operator_, orig_ikey.user_key, nullptr,
+ merge_context_.GetOperands(), &merge_result, logger_,
+ stats_, env_);
+ if (s.ok()) {
+ // The original key encountered
+ // We are certain that keys_ is not empty here (see assertions couple of
+ // lines before).
+ original_key = std::move(keys_.back());
+ orig_ikey.type = kTypeValue;
+ UpdateInternalKey(&original_key, orig_ikey.sequence, orig_ikey.type);
+ keys_.clear();
+ merge_context_.Clear();
+ keys_.emplace_front(std::move(original_key));
+ merge_context_.PushOperand(merge_result);
+ }
+ } else {
+ // We haven't seen the beginning of the key nor a Put/Delete.
+ // Attempt to use the user's associative merge function to
+ // merge the stacked merge operands into a single operand.
+ s = Status::MergeInProgress();
+ if (merge_context_.GetNumOperands() >= 2) {
+ bool merge_success = false;
+ std::string merge_result;
+ {
+ StopWatchNano timer(env_, stats_ != nullptr);
+ PERF_TIMER_GUARD(merge_operator_time_nanos);
+ merge_success = user_merge_operator_->PartialMergeMulti(
+ orig_ikey.user_key,
+ std::deque<Slice>(merge_context_.GetOperands().begin(),
+ merge_context_.GetOperands().end()),
+ &merge_result, logger_);
+ RecordTick(stats_, MERGE_OPERATION_TOTAL_TIME,
+ stats_ ? timer.ElapsedNanosSafe() : 0);
+ }
+ if (merge_success) {
+ // Merging of operands (associative merge) was successful.
+ // Replace operands with the merge result
+ merge_context_.Clear();
+ merge_context_.PushOperand(merge_result);
+ keys_.erase(keys_.begin(), keys_.end() - 1);
+ }
+ }
+ }
+
+ return s;
+}
+
+MergeOutputIterator::MergeOutputIterator(const MergeHelper* merge_helper)
+ : merge_helper_(merge_helper) {
+ it_keys_ = merge_helper_->keys().rend();
+ it_values_ = merge_helper_->values().rend();
+}
+
+void MergeOutputIterator::SeekToFirst() {
+ const auto& keys = merge_helper_->keys();
+ const auto& values = merge_helper_->values();
+ assert(keys.size() == values.size());
+ it_keys_ = keys.rbegin();
+ it_values_ = values.rbegin();
+}
+
+void MergeOutputIterator::Next() {
+ ++it_keys_;
+ ++it_values_;
+}
+
+CompactionFilter::Decision MergeHelper::FilterMerge(const Slice& user_key,
+ const Slice& value_slice) {
+ if (compaction_filter_ == nullptr) {
+ return CompactionFilter::Decision::kKeep;
+ }
+ if (stats_ != nullptr) {
+ filter_timer_.Start();
+ }
+ compaction_filter_value_.clear();
+ compaction_filter_skip_until_.Clear();
+ auto ret = compaction_filter_->FilterV2(
+ level_, user_key, CompactionFilter::ValueType::kMergeOperand, value_slice,
+ &compaction_filter_value_, compaction_filter_skip_until_.rep());
+ if (ret == CompactionFilter::Decision::kRemoveAndSkipUntil) {
+ if (user_comparator_->Compare(*compaction_filter_skip_until_.rep(),
+ user_key) <= 0) {
+ // Invalid skip_until returned from compaction filter.
+ // Keep the key as per FilterV2 documentation.
+ ret = CompactionFilter::Decision::kKeep;
+ } else {
+ compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber,
+ kValueTypeForSeek);
+ }
+ }
+ total_filter_time_ += filter_timer_.ElapsedNanosSafe();
+ return ret;
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/merge_helper.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/merge_helper.h b/thirdparty/rocksdb/db/merge_helper.h
new file mode 100644
index 0000000..59da47a
--- /dev/null
+++ b/thirdparty/rocksdb/db/merge_helper.h
@@ -0,0 +1,209 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+#ifndef MERGE_HELPER_H
+#define MERGE_HELPER_H
+
+#include <deque>
+#include <string>
+#include <vector>
+
+#include "db/dbformat.h"
+#include "db/merge_context.h"
+#include "db/range_del_aggregator.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/env.h"
+#include "rocksdb/slice.h"
+#include "util/stop_watch.h"
+
+namespace rocksdb {
+
+class Comparator;
+class Iterator;
+class Logger;
+class MergeOperator;
+class Statistics;
+class InternalIterator;
+
+class MergeHelper {
+ public:
+ MergeHelper(Env* env, const Comparator* user_comparator,
+ const MergeOperator* user_merge_operator,
+ const CompactionFilter* compaction_filter, Logger* logger,
+ bool assert_valid_internal_key, SequenceNumber latest_snapshot,
+ int level = 0, Statistics* stats = nullptr,
+ const std::atomic<bool>* shutting_down = nullptr)
+ : env_(env),
+ user_comparator_(user_comparator),
+ user_merge_operator_(user_merge_operator),
+ compaction_filter_(compaction_filter),
+ shutting_down_(shutting_down),
+ logger_(logger),
+ assert_valid_internal_key_(assert_valid_internal_key),
+ latest_snapshot_(latest_snapshot),
+ level_(level),
+ keys_(),
+ filter_timer_(env_),
+ total_filter_time_(0U),
+ stats_(stats) {
+ assert(user_comparator_ != nullptr);
+ }
+
+ // Wrapper around MergeOperator::FullMergeV2() that records perf statistics.
+ // Result of merge will be written to result if status returned is OK.
+ // If operands is empty, the value will simply be copied to result.
+ // Set `update_num_ops_stats` to true if it is from a user read, so that
+ // the latency is sensitive.
+ // Returns one of the following statuses:
+ // - OK: Entries were successfully merged.
+ // - Corruption: Merge operator reported unsuccessful merge.
+ static Status TimedFullMerge(const MergeOperator* merge_operator,
+ const Slice& key, const Slice* value,
+ const std::vector<Slice>& operands,
+ std::string* result, Logger* logger,
+ Statistics* statistics, Env* env,
+ Slice* result_operand = nullptr,
+ bool update_num_ops_stats = false);
+
+ // Merge entries until we hit
+ // - a corrupted key
+ // - a Put/Delete,
+ // - a different user key,
+ // - a specific sequence number (snapshot boundary),
+ // - REMOVE_AND_SKIP_UNTIL returned from compaction filter,
+ // or - the end of iteration
+ // iter: (IN) points to the first merge type entry
+ // (OUT) points to the first entry not included in the merge process
+ // range_del_agg: (IN) filters merge operands covered by range tombstones.
+ // stop_before: (IN) a sequence number that merge should not cross.
+ // 0 means no restriction
+ // at_bottom: (IN) true if the iterator covers the bottem level, which means
+ // we could reach the start of the history of this user key.
+ //
+ // Returns one of the following statuses:
+ // - OK: Entries were successfully merged.
+ // - MergeInProgress: Put/Delete not encountered, and didn't reach the start
+ // of key's history. Output consists of merge operands only.
+ // - Corruption: Merge operator reported unsuccessful merge or a corrupted
+ // key has been encountered and not expected (applies only when compiling
+ // with asserts removed).
+ // - ShutdownInProgress: interrupted by shutdown (*shutting_down == true).
+ //
+ // REQUIRED: The first key in the input is not corrupted.
+ Status MergeUntil(InternalIterator* iter,
+ RangeDelAggregator* range_del_agg = nullptr,
+ const SequenceNumber stop_before = 0,
+ const bool at_bottom = false);
+
+ // Filters a merge operand using the compaction filter specified
+ // in the constructor. Returns the decision that the filter made.
+ // Uses compaction_filter_value_ and compaction_filter_skip_until_ for the
+ // optional outputs of compaction filter.
+ CompactionFilter::Decision FilterMerge(const Slice& user_key,
+ const Slice& value_slice);
+
+ // Query the merge result
+ // These are valid until the next MergeUntil call
+ // If the merging was successful:
+ // - keys() contains a single element with the latest sequence number of
+ // the merges. The type will be Put or Merge. See IMPORTANT 1 note, below.
+ // - values() contains a single element with the result of merging all the
+ // operands together
+ //
+ // IMPORTANT 1: the key type could change after the MergeUntil call.
+ // Put/Delete + Merge + ... + Merge => Put
+ // Merge + ... + Merge => Merge
+ //
+ // If the merge operator is not associative, and if a Put/Delete is not found
+ // then the merging will be unsuccessful. In this case:
+ // - keys() contains the list of internal keys seen in order of iteration.
+ // - values() contains the list of values (merges) seen in the same order.
+ // values() is parallel to keys() so that the first entry in
+ // keys() is the key associated with the first entry in values()
+ // and so on. These lists will be the same length.
+ // All of these pairs will be merges over the same user key.
+ // See IMPORTANT 2 note below.
+ //
+ // IMPORTANT 2: The entries were traversed in order from BACK to FRONT.
+ // So keys().back() was the first key seen by iterator.
+ // TODO: Re-style this comment to be like the first one
+ const std::deque<std::string>& keys() const { return keys_; }
+ const std::vector<Slice>& values() const {
+ return merge_context_.GetOperands();
+ }
+ uint64_t TotalFilterTime() const { return total_filter_time_; }
+ bool HasOperator() const { return user_merge_operator_ != nullptr; }
+
+ // If compaction filter returned REMOVE_AND_SKIP_UNTIL, this method will
+ // return true and fill *until with the key to which we should skip.
+ // If true, keys() and values() are empty.
+ bool FilteredUntil(Slice* skip_until) const {
+ if (!has_compaction_filter_skip_until_) {
+ return false;
+ }
+ assert(compaction_filter_ != nullptr);
+ assert(skip_until != nullptr);
+ assert(compaction_filter_skip_until_.Valid());
+ *skip_until = compaction_filter_skip_until_.Encode();
+ return true;
+ }
+
+ private:
+ Env* env_;
+ const Comparator* user_comparator_;
+ const MergeOperator* user_merge_operator_;
+ const CompactionFilter* compaction_filter_;
+ const std::atomic<bool>* shutting_down_;
+ Logger* logger_;
+ bool assert_valid_internal_key_; // enforce no internal key corruption?
+ SequenceNumber latest_snapshot_;
+ int level_;
+
+ // the scratch area that holds the result of MergeUntil
+ // valid up to the next MergeUntil call
+
+ // Keeps track of the sequence of keys seen
+ std::deque<std::string> keys_;
+ // Parallel with keys_; stores the operands
+ mutable MergeContext merge_context_;
+
+ StopWatchNano filter_timer_;
+ uint64_t total_filter_time_;
+ Statistics* stats_;
+
+ bool has_compaction_filter_skip_until_ = false;
+ std::string compaction_filter_value_;
+ InternalKey compaction_filter_skip_until_;
+
+ bool IsShuttingDown() {
+ // This is a best-effort facility, so memory_order_relaxed is sufficient.
+ return shutting_down_ && shutting_down_->load(std::memory_order_relaxed);
+ }
+};
+
+// MergeOutputIterator can be used to iterate over the result of a merge.
+class MergeOutputIterator {
+ public:
+ // The MergeOutputIterator is bound to a MergeHelper instance.
+ explicit MergeOutputIterator(const MergeHelper* merge_helper);
+
+ // Seeks to the first record in the output.
+ void SeekToFirst();
+ // Advances to the next record in the output.
+ void Next();
+
+ Slice key() { return Slice(*it_keys_); }
+ Slice value() { return Slice(*it_values_); }
+ bool Valid() { return it_keys_ != merge_helper_->keys().rend(); }
+
+ private:
+ const MergeHelper* merge_helper_;
+ std::deque<std::string>::const_reverse_iterator it_keys_;
+ std::vector<Slice>::const_reverse_iterator it_values_;
+};
+
+} // namespace rocksdb
+
+#endif
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/merge_operator.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/merge_operator.cc b/thirdparty/rocksdb/db/merge_operator.cc
new file mode 100644
index 0000000..1981e65
--- /dev/null
+++ b/thirdparty/rocksdb/db/merge_operator.cc
@@ -0,0 +1,86 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+/**
+ * Back-end implementation details specific to the Merge Operator.
+ */
+
+#include "rocksdb/merge_operator.h"
+
+namespace rocksdb {
+
+bool MergeOperator::FullMergeV2(const MergeOperationInput& merge_in,
+ MergeOperationOutput* merge_out) const {
+ // If FullMergeV2 is not implemented, we convert the operand_list to
+ // std::deque<std::string> and pass it to FullMerge
+ std::deque<std::string> operand_list_str;
+ for (auto& op : merge_in.operand_list) {
+ operand_list_str.emplace_back(op.data(), op.size());
+ }
+ return FullMerge(merge_in.key, merge_in.existing_value, operand_list_str,
+ &merge_out->new_value, merge_in.logger);
+}
+
+// The default implementation of PartialMergeMulti, which invokes
+// PartialMerge multiple times internally and merges two operands at
+// a time.
+bool MergeOperator::PartialMergeMulti(const Slice& key,
+ const std::deque<Slice>& operand_list,
+ std::string* new_value,
+ Logger* logger) const {
+ assert(operand_list.size() >= 2);
+ // Simply loop through the operands
+ Slice temp_slice(operand_list[0]);
+
+ for (size_t i = 1; i < operand_list.size(); ++i) {
+ auto& operand = operand_list[i];
+ std::string temp_value;
+ if (!PartialMerge(key, temp_slice, operand, &temp_value, logger)) {
+ return false;
+ }
+ swap(temp_value, *new_value);
+ temp_slice = Slice(*new_value);
+ }
+
+ // The result will be in *new_value. All merges succeeded.
+ return true;
+}
+
+// Given a "real" merge from the library, call the user's
+// associative merge function one-by-one on each of the operands.
+// NOTE: It is assumed that the client's merge-operator will handle any errors.
+bool AssociativeMergeOperator::FullMergeV2(
+ const MergeOperationInput& merge_in,
+ MergeOperationOutput* merge_out) const {
+ // Simply loop through the operands
+ Slice temp_existing;
+ const Slice* existing_value = merge_in.existing_value;
+ for (const auto& operand : merge_in.operand_list) {
+ std::string temp_value;
+ if (!Merge(merge_in.key, existing_value, operand, &temp_value,
+ merge_in.logger)) {
+ return false;
+ }
+ swap(temp_value, merge_out->new_value);
+ temp_existing = Slice(merge_out->new_value);
+ existing_value = &temp_existing;
+ }
+
+ // The result will be in *new_value. All merges succeeded.
+ return true;
+}
+
+// Call the user defined simple merge on the operands;
+// NOTE: It is assumed that the client's merge-operator will handle any errors.
+bool AssociativeMergeOperator::PartialMerge(
+ const Slice& key,
+ const Slice& left_operand,
+ const Slice& right_operand,
+ std::string* new_value,
+ Logger* logger) const {
+ return Merge(key, &left_operand, right_operand, new_value, logger);
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/pinned_iterators_manager.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/pinned_iterators_manager.h b/thirdparty/rocksdb/db/pinned_iterators_manager.h
new file mode 100644
index 0000000..7874eef
--- /dev/null
+++ b/thirdparty/rocksdb/db/pinned_iterators_manager.h
@@ -0,0 +1,87 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+#pragma once
+#include <algorithm>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "table/internal_iterator.h"
+
+namespace rocksdb {
+
+// PinnedIteratorsManager will be notified whenever we need to pin an Iterator
+// and it will be responsible for deleting pinned Iterators when they are
+// not needed anymore.
+class PinnedIteratorsManager : public Cleanable {
+ public:
+ PinnedIteratorsManager() : pinning_enabled(false) {}
+ ~PinnedIteratorsManager() {
+ if (pinning_enabled) {
+ ReleasePinnedData();
+ }
+ }
+
+ // Enable Iterators pinning
+ void StartPinning() {
+ assert(pinning_enabled == false);
+ pinning_enabled = true;
+ }
+
+ // Is pinning enabled ?
+ bool PinningEnabled() { return pinning_enabled; }
+
+ // Take ownership of iter and delete it when ReleasePinnedData() is called
+ void PinIterator(InternalIterator* iter, bool arena = false) {
+ if (arena) {
+ PinPtr(iter, &PinnedIteratorsManager::ReleaseArenaInternalIterator);
+ } else {
+ PinPtr(iter, &PinnedIteratorsManager::ReleaseInternalIterator);
+ }
+ }
+
+ typedef void (*ReleaseFunction)(void* arg1);
+ void PinPtr(void* ptr, ReleaseFunction release_func) {
+ assert(pinning_enabled);
+ if (ptr == nullptr) {
+ return;
+ }
+ pinned_ptrs_.emplace_back(ptr, release_func);
+ }
+
+ // Release pinned Iterators
+ inline void ReleasePinnedData() {
+ assert(pinning_enabled == true);
+ pinning_enabled = false;
+
+ // Remove duplicate pointers
+ std::sort(pinned_ptrs_.begin(), pinned_ptrs_.end());
+ auto unique_end = std::unique(pinned_ptrs_.begin(), pinned_ptrs_.end());
+
+ for (auto i = pinned_ptrs_.begin(); i != unique_end; ++i) {
+ void* ptr = i->first;
+ ReleaseFunction release_func = i->second;
+ release_func(ptr);
+ }
+ pinned_ptrs_.clear();
+ // Also do cleanups from the base Cleanable
+ Cleanable::Reset();
+ }
+
+ private:
+ static void ReleaseInternalIterator(void* ptr) {
+ delete reinterpret_cast<InternalIterator*>(ptr);
+ }
+
+ static void ReleaseArenaInternalIterator(void* ptr) {
+ reinterpret_cast<InternalIterator*>(ptr)->~InternalIterator();
+ }
+
+ bool pinning_enabled;
+ std::vector<std::pair<void*, ReleaseFunction>> pinned_ptrs_;
+};
+
+} // namespace rocksdb
[22/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_impl.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_impl.h b/thirdparty/rocksdb/db/db_impl.h
new file mode 100644
index 0000000..31d69a9
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_impl.h
@@ -0,0 +1,1284 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#pragma once
+
+#include <atomic>
+#include <deque>
+#include <functional>
+#include <limits>
+#include <list>
+#include <map>
+#include <queue>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "db/column_family.h"
+#include "db/compaction_job.h"
+#include "db/dbformat.h"
+#include "db/external_sst_file_ingestion_job.h"
+#include "db/flush_job.h"
+#include "db/flush_scheduler.h"
+#include "db/internal_stats.h"
+#include "db/log_writer.h"
+#include "db/snapshot_impl.h"
+#include "db/version_edit.h"
+#include "db/wal_manager.h"
+#include "db/write_controller.h"
+#include "db/write_thread.h"
+#include "memtable_list.h"
+#include "monitoring/instrumented_mutex.h"
+#include "options/db_options.h"
+#include "port/port.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/status.h"
+#include "rocksdb/transaction_log.h"
+#include "rocksdb/write_buffer_manager.h"
+#include "table/scoped_arena_iterator.h"
+#include "util/autovector.h"
+#include "util/event_logger.h"
+#include "util/hash.h"
+#include "util/stop_watch.h"
+#include "util/thread_local.h"
+
+namespace rocksdb {
+
+class MemTable;
+class TableCache;
+class Version;
+class VersionEdit;
+class VersionSet;
+class Arena;
+class WriteCallback;
+struct JobContext;
+struct ExternalSstFileInfo;
+struct MemTableInfo;
+
+class DBImpl : public DB {
+ public:
+ DBImpl(const DBOptions& options, const std::string& dbname);
+ virtual ~DBImpl();
+
+ // Implementations of the DB interface
+ using DB::Put;
+ virtual Status Put(const WriteOptions& options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ const Slice& value) override;
+ using DB::Merge;
+ virtual Status Merge(const WriteOptions& options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ const Slice& value) override;
+ using DB::Delete;
+ virtual Status Delete(const WriteOptions& options,
+ ColumnFamilyHandle* column_family,
+ const Slice& key) override;
+ using DB::SingleDelete;
+ virtual Status SingleDelete(const WriteOptions& options,
+ ColumnFamilyHandle* column_family,
+ const Slice& key) override;
+ using DB::Write;
+ virtual Status Write(const WriteOptions& options,
+ WriteBatch* updates) override;
+
+ using DB::Get;
+ virtual Status Get(const ReadOptions& options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ PinnableSlice* value) override;
+ using DB::MultiGet;
+ virtual std::vector<Status> MultiGet(
+ const ReadOptions& options,
+ const std::vector<ColumnFamilyHandle*>& column_family,
+ const std::vector<Slice>& keys,
+ std::vector<std::string>* values) override;
+
+ virtual Status CreateColumnFamily(const ColumnFamilyOptions& cf_options,
+ const std::string& column_family,
+ ColumnFamilyHandle** handle) override;
+ virtual Status CreateColumnFamilies(
+ const ColumnFamilyOptions& cf_options,
+ const std::vector<std::string>& column_family_names,
+ std::vector<ColumnFamilyHandle*>* handles) override;
+ virtual Status CreateColumnFamilies(
+ const std::vector<ColumnFamilyDescriptor>& column_families,
+ std::vector<ColumnFamilyHandle*>* handles) override;
+ virtual Status DropColumnFamily(ColumnFamilyHandle* column_family) override;
+ virtual Status DropColumnFamilies(
+ const std::vector<ColumnFamilyHandle*>& column_families) override;
+
+ // Returns false if key doesn't exist in the database and true if it may.
+ // If value_found is not passed in as null, then return the value if found in
+ // memory. On return, if value was found, then value_found will be set to true
+ // , otherwise false.
+ using DB::KeyMayExist;
+ virtual bool KeyMayExist(const ReadOptions& options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ std::string* value,
+ bool* value_found = nullptr) override;
+ using DB::NewIterator;
+ virtual Iterator* NewIterator(const ReadOptions& options,
+ ColumnFamilyHandle* column_family) override;
+ virtual Status NewIterators(
+ const ReadOptions& options,
+ const std::vector<ColumnFamilyHandle*>& column_families,
+ std::vector<Iterator*>* iterators) override;
+ virtual const Snapshot* GetSnapshot() override;
+ virtual void ReleaseSnapshot(const Snapshot* snapshot) override;
+ using DB::GetProperty;
+ virtual bool GetProperty(ColumnFamilyHandle* column_family,
+ const Slice& property, std::string* value) override;
+ using DB::GetMapProperty;
+ virtual bool GetMapProperty(ColumnFamilyHandle* column_family,
+ const Slice& property,
+ std::map<std::string, double>* value) override;
+ using DB::GetIntProperty;
+ virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
+ const Slice& property, uint64_t* value) override;
+ using DB::GetAggregatedIntProperty;
+ virtual bool GetAggregatedIntProperty(const Slice& property,
+ uint64_t* aggregated_value) override;
+ using DB::GetApproximateSizes;
+ virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
+ const Range* range, int n, uint64_t* sizes,
+ uint8_t include_flags
+ = INCLUDE_FILES) override;
+ using DB::GetApproximateMemTableStats;
+ virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
+ const Range& range,
+ uint64_t* const count,
+ uint64_t* const size) override;
+ using DB::CompactRange;
+ virtual Status CompactRange(const CompactRangeOptions& options,
+ ColumnFamilyHandle* column_family,
+ const Slice* begin, const Slice* end) override;
+
+ using DB::CompactFiles;
+ virtual Status CompactFiles(const CompactionOptions& compact_options,
+ ColumnFamilyHandle* column_family,
+ const std::vector<std::string>& input_file_names,
+ const int output_level,
+ const int output_path_id = -1) override;
+
+ virtual Status PauseBackgroundWork() override;
+ virtual Status ContinueBackgroundWork() override;
+
+ virtual Status EnableAutoCompaction(
+ const std::vector<ColumnFamilyHandle*>& column_family_handles) override;
+
+ using DB::SetOptions;
+ Status SetOptions(
+ ColumnFamilyHandle* column_family,
+ const std::unordered_map<std::string, std::string>& options_map) override;
+
+ virtual Status SetDBOptions(
+ const std::unordered_map<std::string, std::string>& options_map) override;
+
+ using DB::NumberLevels;
+ virtual int NumberLevels(ColumnFamilyHandle* column_family) override;
+ using DB::MaxMemCompactionLevel;
+ virtual int MaxMemCompactionLevel(ColumnFamilyHandle* column_family) override;
+ using DB::Level0StopWriteTrigger;
+ virtual int Level0StopWriteTrigger(
+ ColumnFamilyHandle* column_family) override;
+ virtual const std::string& GetName() const override;
+ virtual Env* GetEnv() const override;
+ using DB::GetOptions;
+ virtual Options GetOptions(ColumnFamilyHandle* column_family) const override;
+ using DB::GetDBOptions;
+ virtual DBOptions GetDBOptions() const override;
+ using DB::Flush;
+ virtual Status Flush(const FlushOptions& options,
+ ColumnFamilyHandle* column_family) override;
+ virtual Status FlushWAL(bool sync) override;
+ virtual Status SyncWAL() override;
+
+ virtual SequenceNumber GetLatestSequenceNumber() const override;
+
+ bool HasActiveSnapshotLaterThanSN(SequenceNumber sn);
+
+#ifndef ROCKSDB_LITE
+ using DB::ResetStats;
+ virtual Status ResetStats() override;
+ virtual Status DisableFileDeletions() override;
+ virtual Status EnableFileDeletions(bool force) override;
+ virtual int IsFileDeletionsEnabled() const;
+ // All the returned filenames start with "/"
+ virtual Status GetLiveFiles(std::vector<std::string>&,
+ uint64_t* manifest_file_size,
+ bool flush_memtable = true) override;
+ virtual Status GetSortedWalFiles(VectorLogPtr& files) override;
+
+ virtual Status GetUpdatesSince(
+ SequenceNumber seq_number, unique_ptr<TransactionLogIterator>* iter,
+ const TransactionLogIterator::ReadOptions&
+ read_options = TransactionLogIterator::ReadOptions()) override;
+ virtual Status DeleteFile(std::string name) override;
+ Status DeleteFilesInRange(ColumnFamilyHandle* column_family,
+ const Slice* begin, const Slice* end);
+
+ virtual void GetLiveFilesMetaData(
+ std::vector<LiveFileMetaData>* metadata) override;
+
+ // Obtains the meta data of the specified column family of the DB.
+ // Status::NotFound() will be returned if the current DB does not have
+ // any column family match the specified name.
+ // TODO(yhchiang): output parameter is placed in the end in this codebase.
+ virtual void GetColumnFamilyMetaData(
+ ColumnFamilyHandle* column_family,
+ ColumnFamilyMetaData* metadata) override;
+
+ Status SuggestCompactRange(ColumnFamilyHandle* column_family,
+ const Slice* begin, const Slice* end) override;
+
+ Status PromoteL0(ColumnFamilyHandle* column_family,
+ int target_level) override;
+
+ // Similar to Write() but will call the callback once on the single write
+ // thread to determine whether it is safe to perform the write.
+ virtual Status WriteWithCallback(const WriteOptions& write_options,
+ WriteBatch* my_batch,
+ WriteCallback* callback);
+
+ // Returns the sequence number that is guaranteed to be smaller than or equal
+ // to the sequence number of any key that could be inserted into the current
+ // memtables. It can then be assumed that any write with a larger(or equal)
+ // sequence number will be present in this memtable or a later memtable.
+ //
+ // If the earliest sequence number could not be determined,
+ // kMaxSequenceNumber will be returned.
+ //
+ // If include_history=true, will also search Memtables in MemTableList
+ // History.
+ SequenceNumber GetEarliestMemTableSequenceNumber(SuperVersion* sv,
+ bool include_history);
+
+ // For a given key, check to see if there are any records for this key
+ // in the memtables, including memtable history. If cache_only is false,
+ // SST files will also be checked.
+ //
+ // If a key is found, *found_record_for_key will be set to true and
+ // *seq will be set to the stored sequence number for the latest
+ // operation on this key or kMaxSequenceNumber if unknown.
+ // If no key is found, *found_record_for_key will be set to false.
+ //
+ // Note: If cache_only=false, it is possible for *seq to be set to 0 if
+ // the sequence number has been cleared from the record. If the caller is
+ // holding an active db snapshot, we know the missing sequence must be less
+ // than the snapshot's sequence number (sequence numbers are only cleared
+ // when there are no earlier active snapshots).
+ //
+ // If NotFound is returned and found_record_for_key is set to false, then no
+ // record for this key was found. If the caller is holding an active db
+ // snapshot, we know that no key could have existing after this snapshot
+ // (since we do not compact keys that have an earlier snapshot).
+ //
+ // Returns OK or NotFound on success,
+ // other status on unexpected error.
+ // TODO(andrewkr): this API need to be aware of range deletion operations
+ Status GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
+ bool cache_only, SequenceNumber* seq,
+ bool* found_record_for_key);
+
+ using DB::IngestExternalFile;
+ virtual Status IngestExternalFile(
+ ColumnFamilyHandle* column_family,
+ const std::vector<std::string>& external_files,
+ const IngestExternalFileOptions& ingestion_options) override;
+
+ virtual Status VerifyChecksum() override;
+
+#endif // ROCKSDB_LITE
+
+ // Similar to GetSnapshot(), but also lets the db know that this snapshot
+ // will be used for transaction write-conflict checking. The DB can then
+ // make sure not to compact any keys that would prevent a write-conflict from
+ // being detected.
+ const Snapshot* GetSnapshotForWriteConflictBoundary();
+
+ // checks if all live files exist on file system and that their file sizes
+ // match to our in-memory records
+ virtual Status CheckConsistency();
+
+ virtual Status GetDbIdentity(std::string& identity) const override;
+
+ Status RunManualCompaction(ColumnFamilyData* cfd, int input_level,
+ int output_level, uint32_t output_path_id,
+ const Slice* begin, const Slice* end,
+ bool exclusive,
+ bool disallow_trivial_move = false);
+
+ // Return an internal iterator over the current state of the database.
+ // The keys of this iterator are internal keys (see format.h).
+ // The returned iterator should be deleted when no longer needed.
+ InternalIterator* NewInternalIterator(
+ Arena* arena, RangeDelAggregator* range_del_agg,
+ ColumnFamilyHandle* column_family = nullptr);
+
+#ifndef NDEBUG
+ // Extra methods (for testing) that are not in the public DB interface
+ // Implemented in db_impl_debug.cc
+
+ // Compact any files in the named level that overlap [*begin, *end]
+ Status TEST_CompactRange(int level, const Slice* begin, const Slice* end,
+ ColumnFamilyHandle* column_family = nullptr,
+ bool disallow_trivial_move = false);
+
+ void TEST_HandleWALFull();
+
+ bool TEST_UnableToFlushOldestLog() {
+ return unable_to_flush_oldest_log_;
+ }
+
+ bool TEST_IsLogGettingFlushed() {
+ return alive_log_files_.begin()->getting_flushed;
+ }
+
+ // Force current memtable contents to be flushed.
+ Status TEST_FlushMemTable(bool wait = true,
+ ColumnFamilyHandle* cfh = nullptr);
+
+ // Wait for memtable compaction
+ Status TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family = nullptr);
+
+ // Wait for any compaction
+ Status TEST_WaitForCompact();
+
+ // Return the maximum overlapping data (in bytes) at next level for any
+ // file at a level >= 1.
+ int64_t TEST_MaxNextLevelOverlappingBytes(ColumnFamilyHandle* column_family =
+ nullptr);
+
+ // Return the current manifest file no.
+ uint64_t TEST_Current_Manifest_FileNo();
+
+ // get total level0 file size. Only for testing.
+ uint64_t TEST_GetLevel0TotalSize();
+
+ void TEST_GetFilesMetaData(ColumnFamilyHandle* column_family,
+ std::vector<std::vector<FileMetaData>>* metadata);
+
+ void TEST_LockMutex();
+
+ void TEST_UnlockMutex();
+
+ // REQUIRES: mutex locked
+ void* TEST_BeginWrite();
+
+ // REQUIRES: mutex locked
+ // pass the pointer that you got from TEST_BeginWrite()
+ void TEST_EndWrite(void* w);
+
+ uint64_t TEST_MaxTotalInMemoryState() const {
+ return max_total_in_memory_state_;
+ }
+
+ size_t TEST_LogsToFreeSize();
+
+ uint64_t TEST_LogfileNumber();
+
+ uint64_t TEST_total_log_size() const { return total_log_size_; }
+
+ // Returns column family name to ImmutableCFOptions map.
+ Status TEST_GetAllImmutableCFOptions(
+ std::unordered_map<std::string, const ImmutableCFOptions*>* iopts_map);
+
+ // Return the lastest MutableCFOptions of a column family
+ Status TEST_GetLatestMutableCFOptions(ColumnFamilyHandle* column_family,
+ MutableCFOptions* mutable_cf_options);
+
+ Cache* TEST_table_cache() { return table_cache_.get(); }
+
+ WriteController& TEST_write_controler() { return write_controller_; }
+
+ uint64_t TEST_FindMinLogContainingOutstandingPrep();
+ uint64_t TEST_FindMinPrepLogReferencedByMemTable();
+
+ int TEST_BGCompactionsAllowed() const;
+ int TEST_BGFlushesAllowed() const;
+
+#endif // NDEBUG
+
+ struct BGJobLimits {
+ int max_flushes;
+ int max_compactions;
+ };
+ // Returns maximum background flushes and compactions allowed to be scheduled
+ BGJobLimits GetBGJobLimits() const;
+ // Need a static version that can be called during SanitizeOptions().
+ static BGJobLimits GetBGJobLimits(int max_background_flushes,
+ int max_background_compactions,
+ int max_background_jobs,
+ bool parallelize_compactions);
+
+ // move logs pending closing from job_context to the DB queue and
+ // schedule a purge
+ void ScheduleBgLogWriterClose(JobContext* job_context);
+
+ uint64_t MinLogNumberToKeep();
+
+ // Returns the list of live files in 'live' and the list
+ // of all files in the filesystem in 'candidate_files'.
+ // If force == false and the last call was less than
+ // db_options_.delete_obsolete_files_period_micros microseconds ago,
+ // it will not fill up the job_context
+ void FindObsoleteFiles(JobContext* job_context, bool force,
+ bool no_full_scan = false);
+
+ // Diffs the files listed in filenames and those that do not
+ // belong to live files are posibly removed. Also, removes all the
+ // files in sst_delete_files and log_delete_files.
+ // It is not necessary to hold the mutex when invoking this method.
+ void PurgeObsoleteFiles(const JobContext& background_contet,
+ bool schedule_only = false);
+
+ void SchedulePurge();
+
+ ColumnFamilyHandle* DefaultColumnFamily() const override;
+
+ const SnapshotList& snapshots() const { return snapshots_; }
+
+ const ImmutableDBOptions& immutable_db_options() const {
+ return immutable_db_options_;
+ }
+
+ void CancelAllBackgroundWork(bool wait);
+
+ // Find Super version and reference it. Based on options, it might return
+ // the thread local cached one.
+ // Call ReturnAndCleanupSuperVersion() when it is no longer needed.
+ SuperVersion* GetAndRefSuperVersion(ColumnFamilyData* cfd);
+
+ // Similar to the previous function but looks up based on a column family id.
+ // nullptr will be returned if this column family no longer exists.
+ // REQUIRED: this function should only be called on the write thread or if the
+ // mutex is held.
+ SuperVersion* GetAndRefSuperVersion(uint32_t column_family_id);
+
+ // Un-reference the super version and return it to thread local cache if
+ // needed. If it is the last reference of the super version. Clean it up
+ // after un-referencing it.
+ void ReturnAndCleanupSuperVersion(ColumnFamilyData* cfd, SuperVersion* sv);
+
+ // Similar to the previous function but looks up based on a column family id.
+ // nullptr will be returned if this column family no longer exists.
+ // REQUIRED: this function should only be called on the write thread.
+ void ReturnAndCleanupSuperVersion(uint32_t colun_family_id, SuperVersion* sv);
+
+ // REQUIRED: this function should only be called on the write thread or if the
+ // mutex is held. Return value only valid until next call to this function or
+ // mutex is released.
+ ColumnFamilyHandle* GetColumnFamilyHandle(uint32_t column_family_id);
+
+ // Same as above, should called without mutex held and not on write thread.
+ ColumnFamilyHandle* GetColumnFamilyHandleUnlocked(uint32_t column_family_id);
+
+ // Returns the number of currently running flushes.
+ // REQUIREMENT: mutex_ must be held when calling this function.
+ int num_running_flushes() {
+ mutex_.AssertHeld();
+ return num_running_flushes_;
+ }
+
+ // Returns the number of currently running compactions.
+ // REQUIREMENT: mutex_ must be held when calling this function.
+ int num_running_compactions() {
+ mutex_.AssertHeld();
+ return num_running_compactions_;
+ }
+
+ const WriteController& write_controller() { return write_controller_; }
+
+ InternalIterator* NewInternalIterator(const ReadOptions&,
+ ColumnFamilyData* cfd,
+ SuperVersion* super_version,
+ Arena* arena,
+ RangeDelAggregator* range_del_agg);
+
+ // hollow transactions shell used for recovery.
+ // these will then be passed to TransactionDB so that
+ // locks can be reacquired before writing can resume.
+ struct RecoveredTransaction {
+ uint64_t log_number_;
+ std::string name_;
+ WriteBatch* batch_;
+ explicit RecoveredTransaction(const uint64_t log, const std::string& name,
+ WriteBatch* batch)
+ : log_number_(log), name_(name), batch_(batch) {}
+
+ ~RecoveredTransaction() { delete batch_; }
+ };
+
+ bool allow_2pc() const { return immutable_db_options_.allow_2pc; }
+
+ std::unordered_map<std::string, RecoveredTransaction*>
+ recovered_transactions() {
+ return recovered_transactions_;
+ }
+
+ RecoveredTransaction* GetRecoveredTransaction(const std::string& name) {
+ auto it = recovered_transactions_.find(name);
+ if (it == recovered_transactions_.end()) {
+ return nullptr;
+ } else {
+ return it->second;
+ }
+ }
+
+ void InsertRecoveredTransaction(const uint64_t log, const std::string& name,
+ WriteBatch* batch) {
+ recovered_transactions_[name] = new RecoveredTransaction(log, name, batch);
+ MarkLogAsContainingPrepSection(log);
+ }
+
+ void DeleteRecoveredTransaction(const std::string& name) {
+ auto it = recovered_transactions_.find(name);
+ assert(it != recovered_transactions_.end());
+ auto* trx = it->second;
+ recovered_transactions_.erase(it);
+ MarkLogAsHavingPrepSectionFlushed(trx->log_number_);
+ delete trx;
+ }
+
+ void DeleteAllRecoveredTransactions() {
+ for (auto it = recovered_transactions_.begin();
+ it != recovered_transactions_.end(); it++) {
+ delete it->second;
+ }
+ recovered_transactions_.clear();
+ }
+
+ void MarkLogAsHavingPrepSectionFlushed(uint64_t log);
+ void MarkLogAsContainingPrepSection(uint64_t log);
+ void AddToLogsToFreeQueue(log::Writer* log_writer) {
+ logs_to_free_queue_.push_back(log_writer);
+ }
+ InstrumentedMutex* mutex() { return &mutex_; }
+
+ Status NewDB();
+
+ protected:
+ Env* const env_;
+ const std::string dbname_;
+ unique_ptr<VersionSet> versions_;
+ const DBOptions initial_db_options_;
+ const ImmutableDBOptions immutable_db_options_;
+ MutableDBOptions mutable_db_options_;
+ Statistics* stats_;
+ std::unordered_map<std::string, RecoveredTransaction*>
+ recovered_transactions_;
+
+ // Except in DB::Open(), WriteOptionsFile can only be called when:
+ // Persist options to options file.
+ // If need_mutex_lock = false, the method will lock DB mutex.
+ // If need_enter_write_thread = false, the method will enter write thread.
+ Status WriteOptionsFile(bool need_mutex_lock, bool need_enter_write_thread);
+
+ // The following two functions can only be called when:
+ // 1. WriteThread::Writer::EnterUnbatched() is used.
+ // 2. db_mutex is NOT held
+ Status RenameTempFileToOptionsFile(const std::string& file_name);
+ Status DeleteObsoleteOptionsFiles();
+
+ void NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
+ const MutableCFOptions& mutable_cf_options,
+ int job_id, TableProperties prop);
+
+ void NotifyOnFlushCompleted(ColumnFamilyData* cfd, FileMetaData* file_meta,
+ const MutableCFOptions& mutable_cf_options,
+ int job_id, TableProperties prop);
+
+ void NotifyOnCompactionCompleted(ColumnFamilyData* cfd,
+ Compaction *c, const Status &st,
+ const CompactionJobStats& job_stats,
+ int job_id);
+ void NotifyOnMemTableSealed(ColumnFamilyData* cfd,
+ const MemTableInfo& mem_table_info);
+
+#ifndef ROCKSDB_LITE
+ void NotifyOnExternalFileIngested(
+ ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job);
+#endif // !ROCKSDB_LITE
+
+ void NewThreadStatusCfInfo(ColumnFamilyData* cfd) const;
+
+ void EraseThreadStatusCfInfo(ColumnFamilyData* cfd) const;
+
+ void EraseThreadStatusDbInfo() const;
+
+ Status WriteImpl(const WriteOptions& options, WriteBatch* updates,
+ WriteCallback* callback = nullptr,
+ uint64_t* log_used = nullptr, uint64_t log_ref = 0,
+ bool disable_memtable = false);
+
+ Status PipelinedWriteImpl(const WriteOptions& options, WriteBatch* updates,
+ WriteCallback* callback = nullptr,
+ uint64_t* log_used = nullptr, uint64_t log_ref = 0,
+ bool disable_memtable = false);
+
+ Status WriteImplWALOnly(const WriteOptions& options, WriteBatch* updates,
+ WriteCallback* callback = nullptr,
+ uint64_t* log_used = nullptr, uint64_t log_ref = 0);
+
+ uint64_t FindMinLogContainingOutstandingPrep();
+ uint64_t FindMinPrepLogReferencedByMemTable();
+
+ private:
+ friend class DB;
+ friend class InternalStats;
+ friend class PessimisticTransaction;
+ friend class WriteCommittedTxn;
+ friend class WritePreparedTxn;
+#ifndef ROCKSDB_LITE
+ friend class ForwardIterator;
+#endif
+ friend struct SuperVersion;
+ friend class CompactedDBImpl;
+#ifndef NDEBUG
+ friend class XFTransactionWriteHandler;
+#endif
+ struct CompactionState;
+
+ struct WriteContext {
+ autovector<SuperVersion*> superversions_to_free_;
+ autovector<MemTable*> memtables_to_free_;
+
+ ~WriteContext() {
+ for (auto& sv : superversions_to_free_) {
+ delete sv;
+ }
+ for (auto& m : memtables_to_free_) {
+ delete m;
+ }
+ }
+ };
+
+ struct PrepickedCompaction;
+ struct PurgeFileInfo;
+
+ // Recover the descriptor from persistent storage. May do a significant
+ // amount of work to recover recently logged updates. Any changes to
+ // be made to the descriptor are added to *edit.
+ Status Recover(const std::vector<ColumnFamilyDescriptor>& column_families,
+ bool read_only = false, bool error_if_log_file_exist = false,
+ bool error_if_data_exists_in_logs = false);
+
+ void MaybeIgnoreError(Status* s) const;
+
+ const Status CreateArchivalDirectory();
+
+ Status CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options,
+ const std::string& cf_name,
+ ColumnFamilyHandle** handle);
+
+ Status DropColumnFamilyImpl(ColumnFamilyHandle* column_family);
+
+ // Delete any unneeded files and stale in-memory entries.
+ void DeleteObsoleteFiles();
+ // Delete obsolete files and log status and information of file deletion
+ void DeleteObsoleteFileImpl(Status file_deletion_status, int job_id,
+ const std::string& fname, FileType type,
+ uint64_t number, uint32_t path_id);
+
+ // Background process needs to call
+ // auto x = CaptureCurrentFileNumberInPendingOutputs()
+ // auto file_num = versions_->NewFileNumber();
+ // <do something>
+ // ReleaseFileNumberFromPendingOutputs(x)
+ // This will protect any file with number `file_num` or greater from being
+ // deleted while <do something> is running.
+ // -----------
+ // This function will capture current file number and append it to
+ // pending_outputs_. This will prevent any background process to delete any
+ // file created after this point.
+ std::list<uint64_t>::iterator CaptureCurrentFileNumberInPendingOutputs();
+ // This function should be called with the result of
+ // CaptureCurrentFileNumberInPendingOutputs(). It then marks that any file
+ // created between the calls CaptureCurrentFileNumberInPendingOutputs() and
+ // ReleaseFileNumberFromPendingOutputs() can now be deleted (if it's not live
+ // and blocked by any other pending_outputs_ calls)
+ void ReleaseFileNumberFromPendingOutputs(std::list<uint64_t>::iterator v);
+
+ Status SyncClosedLogs(JobContext* job_context);
+
+ // Flush the in-memory write buffer to storage. Switches to a new
+ // log-file/memtable and writes a new descriptor iff successful.
+ Status FlushMemTableToOutputFile(ColumnFamilyData* cfd,
+ const MutableCFOptions& mutable_cf_options,
+ bool* madeProgress, JobContext* job_context,
+ LogBuffer* log_buffer);
+
+ // REQUIRES: log_numbers are sorted in ascending order
+ Status RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
+ SequenceNumber* next_sequence, bool read_only);
+
+ // The following two methods are used to flush a memtable to
+ // storage. The first one is used at database RecoveryTime (when the
+ // database is opened) and is heavyweight because it holds the mutex
+ // for the entire period. The second method WriteLevel0Table supports
+ // concurrent flush memtables to storage.
+ Status WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
+ MemTable* mem, VersionEdit* edit);
+
+ // num_bytes: for slowdown case, delay time is calculated based on
+ // `num_bytes` going through.
+ Status DelayWrite(uint64_t num_bytes, const WriteOptions& write_options);
+
+ Status ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
+ WriteBatch* my_batch);
+
+ Status ScheduleFlushes(WriteContext* context);
+
+ Status SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context);
+
+ // Force current memtable contents to be flushed.
+ Status FlushMemTable(ColumnFamilyData* cfd, const FlushOptions& options,
+ bool writes_stopped = false);
+
+ // Wait for memtable flushed
+ Status WaitForFlushMemTable(ColumnFamilyData* cfd);
+
+ // REQUIRES: mutex locked
+ Status HandleWALFull(WriteContext* write_context);
+
+ // REQUIRES: mutex locked
+ Status HandleWriteBufferFull(WriteContext* write_context);
+
+ // REQUIRES: mutex locked
+ Status PreprocessWrite(const WriteOptions& write_options, bool* need_log_sync,
+ WriteContext* write_context);
+
+ WriteBatch* MergeBatch(const WriteThread::WriteGroup& write_group,
+ WriteBatch* tmp_batch, size_t* write_with_wal);
+
+ Status WriteToWAL(const WriteBatch& merged_batch, log::Writer* log_writer,
+ uint64_t* log_used, uint64_t* log_size);
+
+ Status WriteToWAL(const WriteThread::WriteGroup& write_group,
+ log::Writer* log_writer, uint64_t* log_used,
+ bool need_log_sync, bool need_log_dir_sync,
+ SequenceNumber sequence);
+
+ Status ConcurrentWriteToWAL(const WriteThread::WriteGroup& write_group,
+ uint64_t* log_used, SequenceNumber* last_sequence,
+ int total_count);
+
+ // Used by WriteImpl to update bg_error_ if paranoid check is enabled.
+ void WriteCallbackStatusCheck(const Status& status);
+
+ // Used by WriteImpl to update bg_error_ in case of memtable insert error.
+ void MemTableInsertStatusCheck(const Status& memtable_insert_status);
+
+#ifndef ROCKSDB_LITE
+
+ Status CompactFilesImpl(const CompactionOptions& compact_options,
+ ColumnFamilyData* cfd, Version* version,
+ const std::vector<std::string>& input_file_names,
+ const int output_level, int output_path_id,
+ JobContext* job_context, LogBuffer* log_buffer);
+
+ // Wait for current IngestExternalFile() calls to finish.
+ // REQUIRES: mutex_ held
+ void WaitForIngestFile();
+
+#else
+ // IngestExternalFile is not supported in ROCKSDB_LITE so this function
+ // will be no-op
+ void WaitForIngestFile() {}
+#endif // ROCKSDB_LITE
+
+ ColumnFamilyData* GetColumnFamilyDataByName(const std::string& cf_name);
+
+ void MaybeScheduleFlushOrCompaction();
+ void SchedulePendingFlush(ColumnFamilyData* cfd);
+ void SchedulePendingCompaction(ColumnFamilyData* cfd);
+ void SchedulePendingPurge(std::string fname, FileType type, uint64_t number,
+ uint32_t path_id, int job_id);
+ static void BGWorkCompaction(void* arg);
+ // Runs a pre-chosen universal compaction involving bottom level in a
+ // separate, bottom-pri thread pool.
+ static void BGWorkBottomCompaction(void* arg);
+ static void BGWorkFlush(void* db);
+ static void BGWorkPurge(void* arg);
+ static void UnscheduleCallback(void* arg);
+ void BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
+ Env::Priority bg_thread_pri);
+ void BackgroundCallFlush();
+ void BackgroundCallPurge();
+ Status BackgroundCompaction(bool* madeProgress, JobContext* job_context,
+ LogBuffer* log_buffer,
+ PrepickedCompaction* prepicked_compaction);
+ Status BackgroundFlush(bool* madeProgress, JobContext* job_context,
+ LogBuffer* log_buffer);
+
+ void PrintStatistics();
+
+ // dump rocksdb.stats to LOG
+ void MaybeDumpStats();
+
+ // Return the minimum empty level that could hold the total data in the
+ // input level. Return the input level, if such level could not be found.
+ int FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd,
+ const MutableCFOptions& mutable_cf_options, int level);
+
+ // Move the files in the input level to the target level.
+ // If target_level < 0, automatically calculate the minimum level that could
+ // hold the data set.
+ Status ReFitLevel(ColumnFamilyData* cfd, int level, int target_level = -1);
+
+ // helper functions for adding and removing from flush & compaction queues
+ void AddToCompactionQueue(ColumnFamilyData* cfd);
+ ColumnFamilyData* PopFirstFromCompactionQueue();
+ void AddToFlushQueue(ColumnFamilyData* cfd);
+ ColumnFamilyData* PopFirstFromFlushQueue();
+
+ // helper function to call after some of the logs_ were synced
+ void MarkLogsSynced(uint64_t up_to, bool synced_dir, const Status& status);
+
+ const Snapshot* GetSnapshotImpl(bool is_write_conflict_boundary);
+
+ uint64_t GetMaxTotalWalSize() const;
+
+ // table_cache_ provides its own synchronization
+ std::shared_ptr<Cache> table_cache_;
+
+ // Lock over the persistent DB state. Non-nullptr iff successfully acquired.
+ FileLock* db_lock_;
+
+ // In addition to mutex_, log_write_mutex_ protected writes to logs_ and
+ // logfile_number_. With concurrent_prepare it also protects alive_log_files_,
+ // and log_empty_. Refer to the definition of each variable below for more
+ // details.
+ InstrumentedMutex log_write_mutex_;
+ // State below is protected by mutex_
+ // With concurrent_prepare enabled, some of the variables that accessed during
+ // WriteToWAL need different synchronization: log_empty_, alive_log_files_,
+ // logs_, logfile_number_. Refer to the definition of each variable below for
+ // more description.
+ mutable InstrumentedMutex mutex_;
+
+ std::atomic<bool> shutting_down_;
+ // This condition variable is signaled on these conditions:
+ // * whenever bg_compaction_scheduled_ goes down to 0
+ // * if AnyManualCompaction, whenever a compaction finishes, even if it hasn't
+ // made any progress
+ // * whenever a compaction made any progress
+ // * whenever bg_flush_scheduled_ or bg_purge_scheduled_ value decreases
+ // (i.e. whenever a flush is done, even if it didn't make any progress)
+ // * whenever there is an error in background purge, flush or compaction
+ // * whenever num_running_ingest_file_ goes to 0.
+ InstrumentedCondVar bg_cv_;
+ // Writes are protected by locking both mutex_ and log_write_mutex_, and reads
+ // must be under either mutex_ or log_write_mutex_. Since after ::Open,
+ // logfile_number_ is currently updated only in write_thread_, it can be read
+ // from the same write_thread_ without any locks.
+ uint64_t logfile_number_;
+ std::deque<uint64_t>
+ log_recycle_files; // a list of log files that we can recycle
+ bool log_dir_synced_;
+ // Without concurrent_prepare, read and writes to log_empty_ are protected by
+ // mutex_. Since it is currently updated/read only in write_thread_, it can be
+ // accessed from the same write_thread_ without any locks. With
+ // concurrent_prepare writes, where it can be updated in different threads,
+ // read and writes are protected by log_write_mutex_ instead. This is to avoid
+ // expesnive mutex_ lock during WAL write, which update log_empty_.
+ bool log_empty_;
+ ColumnFamilyHandleImpl* default_cf_handle_;
+ InternalStats* default_cf_internal_stats_;
+ unique_ptr<ColumnFamilyMemTablesImpl> column_family_memtables_;
+ struct LogFileNumberSize {
+ explicit LogFileNumberSize(uint64_t _number)
+ : number(_number) {}
+ void AddSize(uint64_t new_size) { size += new_size; }
+ uint64_t number;
+ uint64_t size = 0;
+ bool getting_flushed = false;
+ };
+ struct LogWriterNumber {
+ // pass ownership of _writer
+ LogWriterNumber(uint64_t _number, log::Writer* _writer)
+ : number(_number), writer(_writer) {}
+
+ log::Writer* ReleaseWriter() {
+ auto* w = writer;
+ writer = nullptr;
+ return w;
+ }
+ void ClearWriter() {
+ delete writer;
+ writer = nullptr;
+ }
+
+ uint64_t number;
+ // Visual Studio doesn't support deque's member to be noncopyable because
+ // of a unique_ptr as a member.
+ log::Writer* writer; // own
+ // true for some prefix of logs_
+ bool getting_synced = false;
+ };
+ // Without concurrent_prepare, read and writes to alive_log_files_ are
+ // protected by mutex_. However since back() is never popped, and push_back()
+ // is done only from write_thread_, the same thread can access the item
+ // reffered by back() without mutex_. With concurrent_prepare_, writes
+ // are protected by locking both mutex_ and log_write_mutex_, and reads must
+ // be under either mutex_ or log_write_mutex_.
+ std::deque<LogFileNumberSize> alive_log_files_;
+ // Log files that aren't fully synced, and the current log file.
+ // Synchronization:
+ // - push_back() is done from write_thread_ with locked mutex_ and
+ // log_write_mutex_
+ // - pop_front() is done from any thread with locked mutex_ and
+ // log_write_mutex_
+ // - reads are done with either locked mutex_ or log_write_mutex_
+ // - back() and items with getting_synced=true are not popped,
+ // - The same thread that sets getting_synced=true will reset it.
+ // - it follows that the object referred by back() can be safely read from
+ // the write_thread_ without using mutex
+ // - it follows that the items with getting_synced=true can be safely read
+ // from the same thread that has set getting_synced=true
+ std::deque<LogWriterNumber> logs_;
+ // Signaled when getting_synced becomes false for some of the logs_.
+ InstrumentedCondVar log_sync_cv_;
+ std::atomic<uint64_t> total_log_size_;
+ // only used for dynamically adjusting max_total_wal_size. it is a sum of
+ // [write_buffer_size * max_write_buffer_number] over all column families
+ uint64_t max_total_in_memory_state_;
+ // If true, we have only one (default) column family. We use this to optimize
+ // some code-paths
+ bool single_column_family_mode_;
+ // If this is non-empty, we need to delete these log files in background
+ // threads. Protected by db mutex.
+ autovector<log::Writer*> logs_to_free_;
+
+ bool is_snapshot_supported_;
+
+ // Class to maintain directories for all database paths other than main one.
+ class Directories {
+ public:
+ Status SetDirectories(Env* env, const std::string& dbname,
+ const std::string& wal_dir,
+ const std::vector<DbPath>& data_paths);
+
+ Directory* GetDataDir(size_t path_id);
+
+ Directory* GetWalDir() {
+ if (wal_dir_) {
+ return wal_dir_.get();
+ }
+ return db_dir_.get();
+ }
+
+ Directory* GetDbDir() { return db_dir_.get(); }
+
+ private:
+ std::unique_ptr<Directory> db_dir_;
+ std::vector<std::unique_ptr<Directory>> data_dirs_;
+ std::unique_ptr<Directory> wal_dir_;
+
+ Status CreateAndNewDirectory(Env* env, const std::string& dirname,
+ std::unique_ptr<Directory>* directory) const;
+ };
+
+ Directories directories_;
+
+ WriteBufferManager* write_buffer_manager_;
+
+ WriteThread write_thread_;
+ WriteBatch tmp_batch_;
+ // The write thread when the writers have no memtable write. This will be used
+ // in 2PC to batch the prepares separately from the serial commit.
+ WriteThread nonmem_write_thread_;
+
+ WriteController write_controller_;
+
+ unique_ptr<RateLimiter> low_pri_write_rate_limiter_;
+
+ // Size of the last batch group. In slowdown mode, next write needs to
+ // sleep if it uses up the quota.
+ // Note: This is to protect memtable and compaction. If the batch only writes
+ // to the WAL its size need not to be included in this.
+ uint64_t last_batch_group_size_;
+
+ FlushScheduler flush_scheduler_;
+
+ SnapshotList snapshots_;
+
+ // For each background job, pending_outputs_ keeps the current file number at
+ // the time that background job started.
+ // FindObsoleteFiles()/PurgeObsoleteFiles() never deletes any file that has
+ // number bigger than any of the file number in pending_outputs_. Since file
+ // numbers grow monotonically, this also means that pending_outputs_ is always
+ // sorted. After a background job is done executing, its file number is
+ // deleted from pending_outputs_, which allows PurgeObsoleteFiles() to clean
+ // it up.
+ // State is protected with db mutex.
+ std::list<uint64_t> pending_outputs_;
+
+ // PurgeFileInfo is a structure to hold information of files to be deleted in
+ // purge_queue_
+ struct PurgeFileInfo {
+ std::string fname;
+ FileType type;
+ uint64_t number;
+ uint32_t path_id;
+ int job_id;
+ PurgeFileInfo(std::string fn, FileType t, uint64_t num, uint32_t pid,
+ int jid)
+ : fname(fn), type(t), number(num), path_id(pid), job_id(jid) {}
+ };
+
+ // flush_queue_ and compaction_queue_ hold column families that we need to
+ // flush and compact, respectively.
+ // A column family is inserted into flush_queue_ when it satisfies condition
+ // cfd->imm()->IsFlushPending()
+ // A column family is inserted into compaction_queue_ when it satisfied
+ // condition cfd->NeedsCompaction()
+ // Column families in this list are all Ref()-erenced
+ // TODO(icanadi) Provide some kind of ReferencedColumnFamily class that will
+ // do RAII on ColumnFamilyData
+ // Column families are in this queue when they need to be flushed or
+ // compacted. Consumers of these queues are flush and compaction threads. When
+ // column family is put on this queue, we increase unscheduled_flushes_ and
+ // unscheduled_compactions_. When these variables are bigger than zero, that
+ // means we need to schedule background threads for compaction and thread.
+ // Once the background threads are scheduled, we decrease unscheduled_flushes_
+ // and unscheduled_compactions_. That way we keep track of number of
+ // compaction and flush threads we need to schedule. This scheduling is done
+ // in MaybeScheduleFlushOrCompaction()
+ // invariant(column family present in flush_queue_ <==>
+ // ColumnFamilyData::pending_flush_ == true)
+ std::deque<ColumnFamilyData*> flush_queue_;
+ // invariant(column family present in compaction_queue_ <==>
+ // ColumnFamilyData::pending_compaction_ == true)
+ std::deque<ColumnFamilyData*> compaction_queue_;
+
+ // A queue to store filenames of the files to be purged
+ std::deque<PurgeFileInfo> purge_queue_;
+
+ // A queue to store log writers to close
+ std::deque<log::Writer*> logs_to_free_queue_;
+ int unscheduled_flushes_;
+ int unscheduled_compactions_;
+
+ // count how many background compactions are running or have been scheduled in
+ // the BOTTOM pool
+ int bg_bottom_compaction_scheduled_;
+
+ // count how many background compactions are running or have been scheduled
+ int bg_compaction_scheduled_;
+
+ // stores the number of compactions are currently running
+ int num_running_compactions_;
+
+ // number of background memtable flush jobs, submitted to the HIGH pool
+ int bg_flush_scheduled_;
+
+ // stores the number of flushes are currently running
+ int num_running_flushes_;
+
+ // number of background obsolete file purge jobs, submitted to the HIGH pool
+ int bg_purge_scheduled_;
+
+ // Information for a manual compaction
+ struct ManualCompactionState {
+ ColumnFamilyData* cfd;
+ int input_level;
+ int output_level;
+ uint32_t output_path_id;
+ Status status;
+ bool done;
+ bool in_progress; // compaction request being processed?
+ bool incomplete; // only part of requested range compacted
+ bool exclusive; // current behavior of only one manual
+ bool disallow_trivial_move; // Force actual compaction to run
+ const InternalKey* begin; // nullptr means beginning of key range
+ const InternalKey* end; // nullptr means end of key range
+ InternalKey* manual_end; // how far we are compacting
+ InternalKey tmp_storage; // Used to keep track of compaction progress
+ InternalKey tmp_storage1; // Used to keep track of compaction progress
+ };
+ struct PrepickedCompaction {
+ // background compaction takes ownership of `compaction`.
+ Compaction* compaction;
+ // caller retains ownership of `manual_compaction_state` as it is reused
+ // across background compactions.
+ ManualCompactionState* manual_compaction_state; // nullptr if non-manual
+ };
+ std::deque<ManualCompactionState*> manual_compaction_dequeue_;
+
+ struct CompactionArg {
+ // caller retains ownership of `db`.
+ DBImpl* db;
+ // background compaction takes ownership of `prepicked_compaction`.
+ PrepickedCompaction* prepicked_compaction;
+ };
+
+ // Have we encountered a background error in paranoid mode?
+ Status bg_error_;
+
+ // shall we disable deletion of obsolete files
+ // if 0 the deletion is enabled.
+ // if non-zero, files will not be getting deleted
+ // This enables two different threads to call
+ // EnableFileDeletions() and DisableFileDeletions()
+ // without any synchronization
+ int disable_delete_obsolete_files_;
+
+ // last time when DeleteObsoleteFiles with full scan was executed. Originaly
+ // initialized with startup time.
+ uint64_t delete_obsolete_files_last_run_;
+
+ // last time stats were dumped to LOG
+ std::atomic<uint64_t> last_stats_dump_time_microsec_;
+
+ // Each flush or compaction gets its own job id. this counter makes sure
+ // they're unique
+ std::atomic<int> next_job_id_;
+
+ // A flag indicating whether the current rocksdb database has any
+ // data that is not yet persisted into either WAL or SST file.
+ // Used when disableWAL is true.
+ std::atomic<bool> has_unpersisted_data_;
+
+ // if an attempt was made to flush all column families that
+ // the oldest log depends on but uncommited data in the oldest
+ // log prevents the log from being released.
+ // We must attempt to free the dependent memtables again
+ // at a later time after the transaction in the oldest
+ // log is fully commited.
+ bool unable_to_flush_oldest_log_;
+
+ static const int KEEP_LOG_FILE_NUM = 1000;
+ // MSVC version 1800 still does not have constexpr for ::max()
+ static const uint64_t kNoTimeOut = port::kMaxUint64;
+
+ std::string db_absolute_path_;
+
+ // The options to access storage files
+ const EnvOptions env_options_;
+
+ // Number of running IngestExternalFile() calls.
+ // REQUIRES: mutex held
+ int num_running_ingest_file_;
+
+#ifndef ROCKSDB_LITE
+ WalManager wal_manager_;
+#endif // ROCKSDB_LITE
+
+ // Unified interface for logging events
+ EventLogger event_logger_;
+
+ // A value of > 0 temporarily disables scheduling of background work
+ int bg_work_paused_;
+
+ // A value of > 0 temporarily disables scheduling of background compaction
+ int bg_compaction_paused_;
+
+ // Guard against multiple concurrent refitting
+ bool refitting_level_;
+
+ // Indicate DB was opened successfully
+ bool opened_successfully_;
+
+ // minimum log number still containing prepared data.
+ // this is used by FindObsoleteFiles to determine which
+ // flushed logs we must keep around because they still
+ // contain prepared data which has not been flushed or rolled back
+ std::priority_queue<uint64_t, std::vector<uint64_t>, std::greater<uint64_t>>
+ min_log_with_prep_;
+
+ // to be used in conjunction with min_log_with_prep_.
+ // once a transaction with data in log L is committed or rolled back
+ // rather than removing the value from the heap we add that value
+ // to prepared_section_completed_ which maps LOG -> instance_count
+ // since a log could contain multiple prepared sections
+ //
+ // when trying to determine the minimum log still active we first
+ // consult min_log_with_prep_. while that root value maps to
+ // a value > 0 in prepared_section_completed_ we decrement the
+ // instance_count for that log and pop the root value in
+ // min_log_with_prep_. This will work the same as a min_heap
+ // where we are deleteing arbitrary elements and the up heaping.
+ std::unordered_map<uint64_t, uint64_t> prepared_section_completed_;
+ std::mutex prep_heap_mutex_;
+
+ // No copying allowed
+ DBImpl(const DBImpl&);
+ void operator=(const DBImpl&);
+
+ // Background threads call this function, which is just a wrapper around
+ // the InstallSuperVersion() function. Background threads carry
+ // job_context which can have new_superversion already
+ // allocated.
+ void InstallSuperVersionAndScheduleWorkWrapper(
+ ColumnFamilyData* cfd, JobContext* job_context,
+ const MutableCFOptions& mutable_cf_options);
+
+ // All ColumnFamily state changes go through this function. Here we analyze
+ // the new state and we schedule background work if we detect that the new
+ // state needs flush or compaction.
+ SuperVersion* InstallSuperVersionAndScheduleWork(
+ ColumnFamilyData* cfd, SuperVersion* new_sv,
+ const MutableCFOptions& mutable_cf_options);
+
+#ifndef ROCKSDB_LITE
+ using DB::GetPropertiesOfAllTables;
+ virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
+ TablePropertiesCollection* props)
+ override;
+ virtual Status GetPropertiesOfTablesInRange(
+ ColumnFamilyHandle* column_family, const Range* range, std::size_t n,
+ TablePropertiesCollection* props) override;
+
+#endif // ROCKSDB_LITE
+
+ // Function that Get and KeyMayExist call with no_io true or false
+ // Note: 'value_found' from KeyMayExist propagates here
+ Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family,
+ const Slice& key, PinnableSlice* value,
+ bool* value_found = nullptr);
+
+ bool GetIntPropertyInternal(ColumnFamilyData* cfd,
+ const DBPropertyInfo& property_info,
+ bool is_locked, uint64_t* value);
+
+ bool HasPendingManualCompaction();
+ bool HasExclusiveManualCompaction();
+ void AddManualCompaction(ManualCompactionState* m);
+ void RemoveManualCompaction(ManualCompactionState* m);
+ bool ShouldntRunManualCompaction(ManualCompactionState* m);
+ bool HaveManualCompaction(ColumnFamilyData* cfd);
+ bool MCOverlap(ManualCompactionState* m, ManualCompactionState* m1);
+
+ size_t GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;
+
+ // When set, we use a seprate queue for writes that dont write to memtable. In
+ // 2PC these are the writes at Prepare phase.
+ const bool concurrent_prepare_;
+ const bool manual_wal_flush_;
+};
+
+extern Options SanitizeOptions(const std::string& db,
+ const Options& src);
+
+extern DBOptions SanitizeOptions(const std::string& db, const DBOptions& src);
+
+extern CompressionType GetCompressionFlush(
+ const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& mutable_cf_options);
+
+// Fix user-supplied options to be reasonable
+template <class T, class V>
+static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
+ if (static_cast<V>(*ptr) > maxvalue) *ptr = maxvalue;
+ if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
+}
+
+} // namespace rocksdb
[49/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/repository/FlowFileRepository.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/repository/FlowFileRepository.h b/libminifi/include/core/repository/FlowFileRepository.h
deleted file mode 100644
index f2691ac..0000000
--- a/libminifi/include/core/repository/FlowFileRepository.h
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_
-#define LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_
-
-#include "leveldb/db.h"
-#include "leveldb/options.h"
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-#include "core/Repository.h"
-#include "core/Core.h"
-#include "Connection.h"
-#include "core/logging/LoggerConfiguration.h"
-#include "concurrentqueue.h"
-
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace core {
-namespace repository {
-
-#define FLOWFILE_REPOSITORY_DIRECTORY "./flowfile_repository"
-#define MAX_FLOWFILE_REPOSITORY_STORAGE_SIZE (10*1024*1024) // 10M
-#define MAX_FLOWFILE_REPOSITORY_ENTRY_LIFE_TIME (600000) // 10 minute
-#define FLOWFILE_REPOSITORY_PURGE_PERIOD (2000) // 2000 msec
-
-/**
- * Flow File repository
- * Design: Extends Repository and implements the run function, using LevelDB as the primary substrate.
- */
-class FlowFileRepository : public core::Repository, public std::enable_shared_from_this<FlowFileRepository> {
- public:
- // Constructor
-
- FlowFileRepository(const std::string repo_name = "", std::string directory = FLOWFILE_REPOSITORY_DIRECTORY, int64_t maxPartitionMillis = MAX_FLOWFILE_REPOSITORY_ENTRY_LIFE_TIME,
- int64_t maxPartitionBytes = MAX_FLOWFILE_REPOSITORY_STORAGE_SIZE, uint64_t purgePeriod = FLOWFILE_REPOSITORY_PURGE_PERIOD)
- : Repository(repo_name.length() > 0 ? repo_name : core::getClassName<FlowFileRepository>(), directory, maxPartitionMillis, maxPartitionBytes, purgePeriod),
- logger_(logging::LoggerFactory<FlowFileRepository>::getLogger()),
- content_repo_(nullptr) {
- db_ = NULL;
- }
-
- // Destructor
- ~FlowFileRepository() {
- if (db_)
- delete db_;
- }
-
- virtual void flush();
-
- // initialize
- virtual bool initialize(const std::shared_ptr<Configure> &configure) {
- std::string value;
-
- if (configure->get(Configure::nifi_flowfile_repository_directory_default, value)) {
- directory_ = value;
- }
- logger_->log_info("NiFi FlowFile Repository Directory %s", directory_.c_str());
- if (configure->get(Configure::nifi_flowfile_repository_max_storage_size, value)) {
- Property::StringToInt(value, max_partition_bytes_);
- }
- logger_->log_info("NiFi FlowFile Max Partition Bytes %d", max_partition_bytes_);
- if (configure->get(Configure::nifi_flowfile_repository_max_storage_time, value)) {
- TimeUnit unit;
- if (Property::StringToTime(value, max_partition_millis_, unit) && Property::ConvertTimeUnitToMS(max_partition_millis_, unit, max_partition_millis_)) {
- }
- }
- logger_->log_info("NiFi FlowFile Max Storage Time: [%d] ms", max_partition_millis_);
- leveldb::Options options;
- options.create_if_missing = true;
- leveldb::Status status = leveldb::DB::Open(options, directory_.c_str(), &db_);
- if (status.ok()) {
- logger_->log_info("NiFi FlowFile Repository database open %s success", directory_.c_str());
- } else {
- logger_->log_error("NiFi FlowFile Repository database open %s fail", directory_.c_str());
- return false;
- }
- return true;
- }
-
- virtual void run();
-
- virtual bool Put(std::string key, const uint8_t *buf, size_t bufLen) {
-
- // persistent to the DB
- leveldb::Slice value((const char *) buf, bufLen);
- leveldb::Status status;
- repo_size_ += bufLen;
- status = db_->Put(leveldb::WriteOptions(), key, value);
- if (status.ok())
- return true;
- else
- return false;
- }
- /**
- *
- * Deletes the key
- * @return status of the delete operation
- */
- virtual bool Delete(std::string key) {
- keys_to_delete.enqueue(key);
- return true;
- }
- /**
- * Sets the value from the provided key
- * @return status of the get operation.
- */
- virtual bool Get(const std::string &key, std::string &value) {
- if (db_ == nullptr)
- return false;
- leveldb::Status status;
- status = db_->Get(leveldb::ReadOptions(), key, &value);
- if (status.ok())
- return true;
- else
- return false;
- }
-
- void setConnectionMap(std::map<std::string, std::shared_ptr<minifi::Connection>> &connectionMap) {
- this->connectionMap = connectionMap;
- }
- virtual void loadComponent(const std::shared_ptr<core::ContentRepository> &content_repo);
-
- void start() {
- if (this->purge_period_ <= 0) {
- return;
- }
- if (running_) {
- return;
- }
- running_ = true;
- thread_ = std::thread(&FlowFileRepository::run, shared_from_this());
- logger_->log_info("%s Repository Monitor Thread Start", name_.c_str());
- }
-
- private:
- moodycamel::ConcurrentQueue<std::string> keys_to_delete;
- std::map<std::string, std::shared_ptr<minifi::Connection>> connectionMap;
- std::shared_ptr<core::ContentRepository> content_repo_;
- leveldb::DB* db_;
- std::shared_ptr<logging::Logger> logger_;
-};
-
-} /* namespace repository */
-} /* namespace core */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
-
-#endif /* LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_ */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/repository/VolatileContentRepository.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/repository/VolatileContentRepository.h b/libminifi/include/core/repository/VolatileContentRepository.h
index 8507216..623f7be 100644
--- a/libminifi/include/core/repository/VolatileContentRepository.h
+++ b/libminifi/include/core/repository/VolatileContentRepository.h
@@ -24,6 +24,7 @@
#include "../ContentRepository.h"
#include "core/repository/VolatileRepository.h"
#include "properties/Configure.h"
+#include "core/Connectable.h"
#include "core/logging/LoggerConfiguration.h"
namespace org {
namespace apache {
@@ -36,13 +37,14 @@ namespace repository {
* Purpose: Stages content into a volatile area of memory. Note that when the maximum number
* of entries is consumed we will rollback a session to wait for others to be freed.
*/
-class VolatileContentRepository : public core::ContentRepository, public core::repository::VolatileRepository<std::shared_ptr<minifi::ResourceClaim>> {
+class VolatileContentRepository : public core::ContentRepository, public virtual core::repository::VolatileRepository<std::shared_ptr<minifi::ResourceClaim>> {
public:
static const char *minimal_locking;
explicit VolatileContentRepository(std::string name = getClassName<VolatileContentRepository>())
: core::repository::VolatileRepository<std::shared_ptr<minifi::ResourceClaim>>(name),
+ core::SerializableComponent(name,0),
logger_(logging::LoggerFactory<VolatileContentRepository>::getLogger()),
minimize_locking_(true) {
max_count_ = 15000;
@@ -100,13 +102,14 @@ class VolatileContentRepository : public core::ContentRepository, public core::r
virtual bool remove(const std::shared_ptr<minifi::ResourceClaim> &claim);
protected:
+
virtual void start();
virtual void run();
template<typename T2>
std::shared_ptr<T2> shared_from_parent() {
- return std::static_pointer_cast<T2>(shared_from_this());
+ return std::dynamic_pointer_cast<T2>(shared_from_this());
}
private:
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/repository/VolatileFlowFileRepository.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/repository/VolatileFlowFileRepository.h b/libminifi/include/core/repository/VolatileFlowFileRepository.h
index 0e75580..1a6be6b 100644
--- a/libminifi/include/core/repository/VolatileFlowFileRepository.h
+++ b/libminifi/include/core/repository/VolatileFlowFileRepository.h
@@ -19,6 +19,7 @@
#define LIBMINIFI_INCLUDE_CORE_REPOSITORY_VOLATILEFLOWFILEREPOSITORY_H_
#include "VolatileRepository.h"
+#include "FlowFileRecord.h"
namespace org {
namespace apache {
@@ -36,7 +37,8 @@ class VolatileFlowFileRepository : public VolatileRepository<std::string> {
explicit VolatileFlowFileRepository(std::string repo_name = "", std::string dir = REPOSITORY_DIRECTORY, int64_t maxPartitionMillis = MAX_REPOSITORY_ENTRY_LIFE_TIME, int64_t maxPartitionBytes =
MAX_REPOSITORY_STORAGE_SIZE,
uint64_t purgePeriod = REPOSITORY_PURGE_PERIOD)
- : VolatileRepository(repo_name.length() > 0 ? repo_name : core::getClassName<VolatileRepository>(), "", maxPartitionMillis, maxPartitionBytes, purgePeriod)
+ : core::SerializableComponent(repo_name, 0),
+ VolatileRepository(repo_name.length() > 0 ? repo_name : core::getClassName<VolatileRepository>(), "", maxPartitionMillis, maxPartitionBytes, purgePeriod)
{
purge_required_ = true;
@@ -50,8 +52,11 @@ class VolatileFlowFileRepository : public VolatileRepository<std::string> {
if (purge_required_ && nullptr != content_repo_) {
std::lock_guard<std::mutex> lock(purge_mutex_);
for (auto purgeItem : purge_list_) {
- std::shared_ptr<minifi::ResourceClaim> newClaim = std::make_shared<minifi::ResourceClaim>(purgeItem, content_repo_, true);
- content_repo_->remove(newClaim);
+ std::shared_ptr<FlowFileRecord> eventRead = std::make_shared<FlowFileRecord>(shared_from_this(), content_repo_);
+ if (eventRead->DeSerialize(reinterpret_cast<const uint8_t *>(purgeItem.data()), purgeItem.size())) {
+ std::shared_ptr<minifi::ResourceClaim> newClaim = eventRead->getResourceClaim();
+ content_repo_->remove(newClaim);
+ }
}
purge_list_.resize(0);
purge_list_.clear();
@@ -66,6 +71,13 @@ class VolatileFlowFileRepository : public VolatileRepository<std::string> {
protected:
+ virtual void emplace(RepoValue<std::string> &old_value) {
+ std::string buffer;
+ old_value.emplace(buffer);
+ std::lock_guard<std::mutex> lock(purge_mutex_);
+ purge_list_.push_back(buffer);
+ }
+
std::shared_ptr<core::ContentRepository> content_repo_;
};
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/repository/VolatileProvenanceRepository.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/repository/VolatileProvenanceRepository.h b/libminifi/include/core/repository/VolatileProvenanceRepository.h
index 2717510..cabb76f 100644
--- a/libminifi/include/core/repository/VolatileProvenanceRepository.h
+++ b/libminifi/include/core/repository/VolatileProvenanceRepository.h
@@ -36,7 +36,7 @@ class VolatileProvenanceRepository : public VolatileRepository<std::string> {
explicit VolatileProvenanceRepository(std::string repo_name = "", std::string dir = REPOSITORY_DIRECTORY, int64_t maxPartitionMillis = MAX_REPOSITORY_ENTRY_LIFE_TIME, int64_t maxPartitionBytes =
MAX_REPOSITORY_STORAGE_SIZE,
uint64_t purgePeriod = REPOSITORY_PURGE_PERIOD)
- : VolatileRepository(repo_name.length() > 0 ? repo_name : core::getClassName<VolatileRepository>(), "", maxPartitionMillis, maxPartitionBytes, purgePeriod)
+ : core::SerializableComponent(repo_name, 0),VolatileRepository(repo_name.length() > 0 ? repo_name : core::getClassName<VolatileRepository>(), "", maxPartitionMillis, maxPartitionBytes, purgePeriod)
{
purge_required_ = false;
@@ -45,6 +45,10 @@ class VolatileProvenanceRepository : public VolatileRepository<std::string> {
virtual void run() {
repo_full_ = false;
}
+ protected:
+ virtual void emplace(RepoValue<std::string> &old_value) {
+ purge_list_.push_back(old_value.getKey());
+ }
private:
};
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/repository/VolatileRepository.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/repository/VolatileRepository.h b/libminifi/include/core/repository/VolatileRepository.h
index da6608c..3b5d5e1 100644
--- a/libminifi/include/core/repository/VolatileRepository.h
+++ b/libminifi/include/core/repository/VolatileRepository.h
@@ -37,7 +37,7 @@ namespace repository {
/**
* Flow File repository
- * Design: Extends Repository and implements the run function, using LevelDB as the primary substrate.
+ * Design: Extends Repository and implements the run function, using RocksDB as the primary substrate.
*/
template<typename T>
class VolatileRepository : public core::Repository, public std::enable_shared_from_this<VolatileRepository<T>> {
@@ -50,7 +50,8 @@ class VolatileRepository : public core::Repository, public std::enable_shared_fr
explicit VolatileRepository(std::string repo_name = "", std::string dir = REPOSITORY_DIRECTORY, int64_t maxPartitionMillis = MAX_REPOSITORY_ENTRY_LIFE_TIME, int64_t maxPartitionBytes =
MAX_REPOSITORY_STORAGE_SIZE,
uint64_t purgePeriod = REPOSITORY_PURGE_PERIOD)
- : Repository(repo_name.length() > 0 ? repo_name : core::getClassName<VolatileRepository>(), "", maxPartitionMillis, maxPartitionBytes, purgePeriod),
+ : core::SerializableComponent(repo_name, 0),
+ Repository(repo_name.length() > 0 ? repo_name : core::getClassName<VolatileRepository>(), "", maxPartitionMillis, maxPartitionBytes, purgePeriod),
max_size_(maxPartitionBytes * 0.75),
current_index_(0),
max_count_(10000),
@@ -121,6 +122,11 @@ class VolatileRepository : public core::Repository, public std::enable_shared_fr
protected:
+ virtual void emplace(RepoValue<T> &old_value) {
+ std::lock_guard<std::mutex> lock(purge_mutex_);
+ purge_list_.push_back(old_value.getKey());
+ }
+
/**
* Tests whether or not the current size exceeds the capacity
* if the new prospectiveSize is inserted.
@@ -208,7 +214,7 @@ bool VolatileRepository<T>::initialize(const std::shared_ptr<Configure> &configu
}
logger_->log_info("Resizing value_vector_ for %s count is %d", getName(), max_count_);
- logger_->log_info("Using a maximum size of %u", max_size_);
+ logger_->log_info("Using a maximum size for %s of %u", getName(), max_size_);
value_vector_.reserve(max_count_);
for (int i = 0; i < max_count_; i++) {
value_vector_.emplace_back(new AtomicEntry<T>(¤t_size_, &max_size_));
@@ -245,7 +251,7 @@ bool VolatileRepository<T>::Put(T key, const uint8_t *buf, size_t bufLen) {
logger_->log_debug("Set repo value at %d out of %d updated %d current_size %d, adding %d to %d", private_index, max_count_, updated == true, reclaimed_size, size, current_size_.load());
if (updated && reclaimed_size > 0) {
std::lock_guard<std::mutex> lock(mutex_);
- purge_list_.push_back(old_value.getKey());
+ emplace(old_value);
}
if (reclaimed_size > 0) {
/**
@@ -270,11 +276,14 @@ bool VolatileRepository<T>::Put(T key, const uint8_t *buf, size_t bufLen) {
*/
template<typename T>
bool VolatileRepository<T>::Delete(T key) {
+ logger_->log_debug("Delete from volatile");
for (auto ent : value_vector_) {
// let the destructor do the cleanup
RepoValue<T> value;
if (ent->getValue(key, value)) {
current_size_ -= value.size();
+ logger_->log_debug("Delete and pushed into purge_list from volatile");
+ emplace(value);
return true;
}
}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/properties/Configure.h
----------------------------------------------------------------------
diff --git a/libminifi/include/properties/Configure.h b/libminifi/include/properties/Configure.h
index f71ce9c..0d7bb55 100644
--- a/libminifi/include/properties/Configure.h
+++ b/libminifi/include/properties/Configure.h
@@ -51,6 +51,7 @@ class Configure : public Properties {
static const char *nifi_provenance_repository_directory_default;
static const char *nifi_provenance_repository_enable;
static const char *nifi_flowfile_repository_max_storage_time;
+ static const char *nifi_dbcontent_repository_directory_default;
static const char *nifi_flowfile_repository_max_storage_size;
static const char *nifi_flowfile_repository_directory_default;
static const char *nifi_flowfile_repository_enable;
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/provenance/ProvenanceRepository.h
----------------------------------------------------------------------
diff --git a/libminifi/include/provenance/ProvenanceRepository.h b/libminifi/include/provenance/ProvenanceRepository.h
deleted file mode 100644
index 53f489f..0000000
--- a/libminifi/include/provenance/ProvenanceRepository.h
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_
-#define LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_
-
-#include "leveldb/db.h"
-#include "leveldb/options.h"
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-#include "core/Repository.h"
-#include "core/Core.h"
-#include "provenance/Provenance.h"
-#include "core/logging/LoggerConfiguration.h"
-#include "concurrentqueue.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace provenance {
-
-#define PROVENANCE_DIRECTORY "./provenance_repository"
-#define MAX_PROVENANCE_STORAGE_SIZE (10*1024*1024) // 10M
-#define MAX_PROVENANCE_ENTRY_LIFE_TIME (60000) // 1 minute
-#define PROVENANCE_PURGE_PERIOD (2500) // 2500 msec
-
-class ProvenanceRepository : public core::Repository, public std::enable_shared_from_this<ProvenanceRepository> {
- public:
- // Constructor
- /*!
- * Create a new provenance repository
- */
- ProvenanceRepository(const std::string repo_name = "", std::string directory = PROVENANCE_DIRECTORY, int64_t maxPartitionMillis = MAX_PROVENANCE_ENTRY_LIFE_TIME, int64_t maxPartitionBytes =
- MAX_PROVENANCE_STORAGE_SIZE,
- uint64_t purgePeriod = PROVENANCE_PURGE_PERIOD)
- : Repository(repo_name.length() > 0 ? repo_name : core::getClassName<ProvenanceRepository>(), directory, maxPartitionMillis, maxPartitionBytes, purgePeriod),
- logger_(logging::LoggerFactory<ProvenanceRepository>::getLogger()) {
- db_ = NULL;
- }
-
- // Destructor
- virtual ~ProvenanceRepository() {
- if (db_)
- delete db_;
- }
-
- virtual void flush();
-
- void start() {
- if (this->purge_period_ <= 0)
- return;
- if (running_)
- return;
- running_ = true;
- thread_ = std::thread(&ProvenanceRepository::run, shared_from_this());
- logger_->log_info("%s Repository Monitor Thread Start", name_.c_str());
- }
-
- // initialize
- virtual bool initialize(const std::shared_ptr<org::apache::nifi::minifi::Configure> &config) {
- std::string value;
- if (config->get(Configure::nifi_provenance_repository_directory_default, value)) {
- directory_ = value;
- }
- logger_->log_info("NiFi Provenance Repository Directory %s", directory_.c_str());
- if (config->get(Configure::nifi_provenance_repository_max_storage_size, value)) {
- core::Property::StringToInt(value, max_partition_bytes_);
- }
- logger_->log_info("NiFi Provenance Max Partition Bytes %d", max_partition_bytes_);
- if (config->get(Configure::nifi_provenance_repository_max_storage_time, value)) {
- core::TimeUnit unit;
- if (core::Property::StringToTime(value, max_partition_millis_, unit) && core::Property::ConvertTimeUnitToMS(max_partition_millis_, unit, max_partition_millis_)) {
- }
- }
- logger_->log_info("NiFi Provenance Max Storage Time: [%d] ms", max_partition_millis_);
- leveldb::Options options;
- options.create_if_missing = true;
- leveldb::Status status = leveldb::DB::Open(options, directory_.c_str(), &db_);
- if (status.ok()) {
- logger_->log_info("NiFi Provenance Repository database open %s success", directory_.c_str());
- } else {
- logger_->log_error("NiFi Provenance Repository database open %s fail", directory_.c_str());
- return false;
- }
-
- return true;
- }
- // Put
- virtual bool Put(std::string key, const uint8_t *buf, size_t bufLen) {
-
- if (repo_full_) {
- return false;
- }
-
- // persistent to the DB
- leveldb::Slice value((const char *) buf, bufLen);
- leveldb::Status status;
- repo_size_ += bufLen;
- status = db_->Put(leveldb::WriteOptions(), key, value);
- if (status.ok())
- return true;
- else
- return false;
- }
- // Delete
- virtual bool Delete(std::string key) {
- keys_to_delete.enqueue(key);
- return true;
- }
- // Get
- virtual bool Get(const std::string &key, std::string &value) {
- leveldb::Status status;
- status = db_->Get(leveldb::ReadOptions(), key, &value);
- if (status.ok())
- return true;
- else
- return false;
- }
-
- // Remove event
- void removeEvent(ProvenanceEventRecord *event) {
- Delete(event->getEventId());
- }
-
- virtual bool Serialize(const std::string &key, const uint8_t *buffer, const size_t bufferSize) {
- return Put(key, buffer, bufferSize);
- }
-
- virtual bool get(std::vector<std::shared_ptr<core::CoreComponent>> &store, size_t max_size) {
- leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions());
- for (it->SeekToFirst(); it->Valid(); it->Next()) {
- std::shared_ptr<ProvenanceEventRecord> eventRead = std::make_shared<ProvenanceEventRecord>();
- std::string key = it->key().ToString();
- if (store.size() >= max_size)
- break;
- if (eventRead->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) {
- store.push_back(std::dynamic_pointer_cast<core::CoreComponent>(eventRead));
- }
- }
- delete it;
- return true;
- }
-
- virtual bool DeSerialize(std::vector<std::shared_ptr<core::SerializableComponent>> &records, size_t &max_size, std::function<std::shared_ptr<core::SerializableComponent>()> lambda) {
- leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions());
- size_t requested_batch = max_size;
- max_size = 0;
- for (it->SeekToFirst(); it->Valid(); it->Next()) {
-
- if (max_size >= requested_batch)
- break;
- std::shared_ptr<core::SerializableComponent> eventRead = lambda();
- std::string key = it->key().ToString();
- if (eventRead->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) {
- max_size++;
- records.push_back(eventRead);
- }
-
- }
- delete it;
-
- if (max_size > 0) {
- return true;
- } else {
- return false;
- }
- }
- //! get record
- void getProvenanceRecord(std::vector<std::shared_ptr<ProvenanceEventRecord>> &records, int maxSize) {
- leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions());
- for (it->SeekToFirst(); it->Valid(); it->Next()) {
- std::shared_ptr<ProvenanceEventRecord> eventRead = std::make_shared<ProvenanceEventRecord>();
- std::string key = it->key().ToString();
- if (records.size() >= maxSize)
- break;
- if (eventRead->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) {
- records.push_back(eventRead);
- }
- }
- delete it;
- }
-
- virtual bool DeSerialize(std::vector<std::shared_ptr<core::SerializableComponent>> &store, size_t &max_size) {
- leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions());
- max_size = 0;
- for (it->SeekToFirst(); it->Valid(); it->Next()) {
- std::shared_ptr<ProvenanceEventRecord> eventRead = std::make_shared<ProvenanceEventRecord>();
- std::string key = it->key().ToString();
-
- if (store.at(max_size)->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) {
- max_size++;
- }
- if (store.size() >= max_size)
- break;
- }
- delete it;
- if (max_size > 0) {
- return true;
- } else {
- return false;
- }
- }
- //! purge record
- void purgeProvenanceRecord(std::vector<std::shared_ptr<ProvenanceEventRecord>> &records) {
- for (auto record : records) {
- Delete(record->getEventId());
- }
- flush();
- }
- // destroy
- void destroy() {
- if (db_) {
- delete db_;
- db_ = NULL;
- }
- }
- // Run function for the thread
- void run();
-
- // Prevent default copy constructor and assignment operation
- // Only support pass by reference or pointer
- ProvenanceRepository(const ProvenanceRepository &parent) = delete;
- ProvenanceRepository &operator=(const ProvenanceRepository &parent) = delete;
-
- private:
- moodycamel::ConcurrentQueue<std::string> keys_to_delete;
- leveldb::DB* db_;
- std::shared_ptr<logging::Logger> logger_;
-};
-
-} /* namespace provenance */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
-#endif /* LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_ */
-
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/src/Configure.cpp
----------------------------------------------------------------------
diff --git a/libminifi/src/Configure.cpp b/libminifi/src/Configure.cpp
index 9dcd6c6..db9840e 100644
--- a/libminifi/src/Configure.cpp
+++ b/libminifi/src/Configure.cpp
@@ -44,6 +44,7 @@ const char *Configure::nifi_provenance_repository_directory_default = "nifi.prov
const char *Configure::nifi_flowfile_repository_max_storage_size = "nifi.flowfile.repository.max.storage.size";
const char *Configure::nifi_flowfile_repository_max_storage_time = "nifi.flowfile.repository.max.storage.time";
const char *Configure::nifi_flowfile_repository_directory_default = "nifi.flowfile.repository.directory.default";
+const char *Configure::nifi_dbcontent_repository_directory_default = "nifi.database.content.repository.directory.default";
const char *Configure::nifi_remote_input_secure = "nifi.remote.input.secure";
const char *Configure::nifi_security_need_ClientAuth = "nifi.security.need.ClientAuth";
const char *Configure::nifi_security_client_certificate = "nifi.security.client.certificate";
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/src/FlowController.cpp
----------------------------------------------------------------------
diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp
index 6485e6c..28294c9 100644
--- a/libminifi/src/FlowController.cpp
+++ b/libminifi/src/FlowController.cpp
@@ -49,7 +49,7 @@
#include "SchedulingAgent.h"
#include "core/controller/ControllerServiceProvider.h"
#include "core/logging/LoggerConfiguration.h"
-#include "core/repository/FlowFileRepository.h"
+#include "core/Connectable.h"
namespace org {
namespace apache {
@@ -306,15 +306,12 @@ void FlowController::reload(std::string yamlFile) {
void FlowController::loadFlowRepo() {
if (this->flow_file_repo_ != nullptr) {
logger_->log_debug("Getting connection map");
- std::map<std::string, std::shared_ptr<Connection>> connectionMap;
+ std::map<std::string, std::shared_ptr<core::Connectable>> connectionMap;
if (this->root_ != nullptr) {
this->root_->getConnections(connectionMap);
}
logger_->log_debug("Number of connections from connectionMap %d", connectionMap.size());
- auto rep = std::dynamic_pointer_cast<core::repository::FlowFileRepository>(flow_file_repo_);
- if (nullptr != rep) {
- rep->setConnectionMap(connectionMap);
- }
+ flow_file_repo_->setConnectionMap(connectionMap);
flow_file_repo_->loadComponent(content_repo_);
} else {
logger_->log_debug("Flow file repository is not set");
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/src/FlowFileRecord.cpp
----------------------------------------------------------------------
diff --git a/libminifi/src/FlowFileRecord.cpp b/libminifi/src/FlowFileRecord.cpp
index b97f290..c89a00b 100644
--- a/libminifi/src/FlowFileRecord.cpp
+++ b/libminifi/src/FlowFileRecord.cpp
@@ -77,7 +77,7 @@ FlowFileRecord::FlowFileRecord(std::shared_ptr<core::Repository> flow_repository
entry_date_ = event->getEntryDate();
lineage_start_date_ = event->getlineageStartDate();
lineage_Identifiers_ = event->getlineageIdentifiers();
- uuid_str_ = event->getUUIDStr();
+ uuidStr_ = event->getUUIDStr();
attributes_ = event->getAttributes();
size_ = event->getSize();
offset_ = event->getOffset();
@@ -99,16 +99,16 @@ FlowFileRecord::FlowFileRecord(std::shared_ptr<core::Repository> flow_repository
FlowFileRecord::~FlowFileRecord() {
if (!snapshot_)
- logger_->log_debug("Delete FlowFile UUID %s", uuid_str_.c_str());
+ logger_->log_debug("Delete FlowFile UUID %s", uuidStr_.c_str());
else
- logger_->log_debug("Delete SnapShot FlowFile UUID %s", uuid_str_.c_str());
+ logger_->log_debug("Delete SnapShot FlowFile UUID %s", uuidStr_.c_str());
if (claim_) {
// Decrease the flow file record owned count for the resource claim
claim_->decreaseFlowFileRecordOwnedCount();
std::string value;
if (claim_->getFlowFileRecordOwnedCount() <= 0) {
// we cannot rely on the stored variable here since we
- if (flow_repository_ != nullptr && !flow_repository_->Get(uuid_str_, value)) {
+ if (flow_repository_ != nullptr && !flow_repository_->Get(uuidStr_, value)) {
logger_->log_debug("Delete Resource Claim %s", claim_->getContentFullPath().c_str());
content_repo_->remove(claim_);
}
@@ -181,9 +181,9 @@ bool FlowFileRecord::DeSerialize(std::string key) {
ret = DeSerialize(stream);
if (ret) {
- logger_->log_debug("NiFi FlowFile retrieve uuid %s size %d connection %s success", uuid_str_.c_str(), stream.getSize(), uuid_connection_.c_str());
+ logger_->log_debug("NiFi FlowFile retrieve uuid %s size %d connection %s success", uuidStr_.c_str(), stream.getSize(), uuid_connection_.c_str());
} else {
- logger_->log_debug("NiFi FlowFile retrieve uuid %s size %d connection %d fail", uuid_str_.c_str(), stream.getSize(), uuid_connection_.c_str());
+ logger_->log_debug("NiFi FlowFile retrieve uuid %s size %d connection %d fail", uuidStr_.c_str(), stream.getSize(), uuid_connection_.c_str());
}
return ret;
@@ -209,7 +209,7 @@ bool FlowFileRecord::Serialize() {
return false;
}
- ret = writeUTF(this->uuid_str_, &outStream);
+ ret = writeUTF(this->uuidStr_, &outStream);
if (ret <= 0) {
return false;
}
@@ -251,11 +251,11 @@ bool FlowFileRecord::Serialize() {
return false;
}
- if (flow_repository_->Put(uuid_str_, const_cast<uint8_t*>(outStream.getBuffer()), outStream.getSize())) {
- logger_->log_debug("NiFi FlowFile Store event %s size %d success", uuid_str_.c_str(), outStream.getSize());
+ if (flow_repository_->Put(uuidStr_, const_cast<uint8_t*>(outStream.getBuffer()), outStream.getSize())) {
+ logger_->log_debug("NiFi FlowFile Store event %s size %d success", uuidStr_.c_str(), outStream.getSize());
return true;
} else {
- logger_->log_error("NiFi FlowFile Store event %s size %d fail", uuid_str_.c_str(), outStream.getSize());
+ logger_->log_error("NiFi FlowFile Store event %s size %d fail", uuidStr_.c_str(), outStream.getSize());
return false;
}
@@ -282,7 +282,7 @@ bool FlowFileRecord::DeSerialize(const uint8_t *buffer, const int bufferSize) {
return false;
}
- ret = readUTF(this->uuid_str_, &outStream);
+ ret = readUTF(this->uuidStr_, &outStream);
if (ret <= 0) {
return false;
}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/src/core/FlowConfiguration.cpp
----------------------------------------------------------------------
diff --git a/libminifi/src/core/FlowConfiguration.cpp b/libminifi/src/core/FlowConfiguration.cpp
index 16a9778..68b7520 100644
--- a/libminifi/src/core/FlowConfiguration.cpp
+++ b/libminifi/src/core/FlowConfiguration.cpp
@@ -29,12 +29,11 @@ namespace minifi {
namespace core {
std::vector<std::string> FlowConfiguration::statics_sl_funcs_;
+std::mutex FlowConfiguration::atomic_initialization_;
FlowConfiguration::~FlowConfiguration() {
}
-
-
std::shared_ptr<core::Processor> FlowConfiguration::createProcessor(std::string name, uuid_t uuid) {
auto ptr = core::ClassLoader::getDefaultClassLoader().instantiate(name, uuid);
if (nullptr == ptr) {
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/src/core/FlowFile.cpp
----------------------------------------------------------------------
diff --git a/libminifi/src/core/FlowFile.cpp b/libminifi/src/core/FlowFile.cpp
index d08ea4b..f5d8754 100644
--- a/libminifi/src/core/FlowFile.cpp
+++ b/libminifi/src/core/FlowFile.cpp
@@ -33,7 +33,8 @@ std::shared_ptr<utils::IdGenerator> FlowFile::id_generator_ = utils::IdGenerator
std::shared_ptr<logging::Logger> FlowFile::logger_ = logging::LoggerFactory<FlowFile>::getLogger();
FlowFile::FlowFile()
- : size_(0),
+ : Connectable("FlowFile", 0),
+ size_(0),
id_(0),
stored(false),
offset_(0),
@@ -47,14 +48,6 @@ FlowFile::FlowFile()
entry_date_ = getTimeMillis();
event_time_ = entry_date_;
lineage_start_date_ = entry_date_;
-
- char uuidStr[37] = { 0 };
-
- // Generate the global UUID for the flow record
- id_generator_->generate(uuid_);
-
- uuid_unparse_lower(uuid_, uuidStr);
- uuid_str_ = uuidStr;
}
FlowFile::~FlowFile() {
@@ -74,7 +67,7 @@ FlowFile& FlowFile::operator=(const FlowFile& other) {
claim_ = other.claim_;
if (claim_ != nullptr)
this->claim_->increaseFlowFileRecordOwnedCount();
- uuid_str_ = other.uuid_str_;
+ uuidStr_ = other.uuidStr_;
connection_ = other.connection_;
original_connection_ = other.original_connection_;
return *this;
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/src/core/ProcessGroup.cpp
----------------------------------------------------------------------
diff --git a/libminifi/src/core/ProcessGroup.cpp b/libminifi/src/core/ProcessGroup.cpp
index a537f1a..5915089 100644
--- a/libminifi/src/core/ProcessGroup.cpp
+++ b/libminifi/src/core/ProcessGroup.cpp
@@ -259,6 +259,16 @@ void ProcessGroup::getConnections(std::map<std::string, std::shared_ptr<Connecti
}
}
+void ProcessGroup::getConnections(std::map<std::string, std::shared_ptr<Connectable>> &connectionMap) {
+ for (auto connection : connections_) {
+ connectionMap[connection->getUUIDStr()] = connection;
+ connectionMap[connection->getName()] = connection;
+ }
+ for (auto processGroup : child_process_groups_) {
+ processGroup->getConnections(connectionMap);
+ }
+}
+
void ProcessGroup::addConnection(std::shared_ptr<Connection> connection) {
std::lock_guard<std::recursive_mutex> lock(mutex_);
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/src/core/Repository.cpp
----------------------------------------------------------------------
diff --git a/libminifi/src/core/Repository.cpp b/libminifi/src/core/Repository.cpp
index be21b16..575f694 100644
--- a/libminifi/src/core/Repository.cpp
+++ b/libminifi/src/core/Repository.cpp
@@ -20,7 +20,6 @@
#include <cstdint>
#include <vector>
-#include "../../include/core/repository/FlowFileRepository.h"
#include "io/DataStream.h"
#include "io/Serializable.h"
#include "core/Relationship.h"
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/src/core/RepositoryFactory.cpp
----------------------------------------------------------------------
diff --git a/libminifi/src/core/RepositoryFactory.cpp b/libminifi/src/core/RepositoryFactory.cpp
index b25e87c..d753a90 100644
--- a/libminifi/src/core/RepositoryFactory.cpp
+++ b/libminifi/src/core/RepositoryFactory.cpp
@@ -17,52 +17,45 @@
#include "core/RepositoryFactory.h"
#include <memory>
#include <string>
+#include <utility>
#include <algorithm>
#include "core/ContentRepository.h"
-#include "core/repository/FileSystemRepository.h"
#include "core/repository/VolatileContentRepository.h"
#include "core/Repository.h"
-#ifdef LEVELDB_SUPPORT
-#include "core/repository/FlowFileRepository.h"
-#include "provenance/ProvenanceRepository.h"
-#endif
-
-#include "core/repository/VolatileProvenanceRepository.h"
+#include "core/ClassLoader.h"
+#include "core/repository/FileSystemRepository.h"
#include "core/repository/VolatileFlowFileRepository.h"
+#include "core/repository/VolatileProvenanceRepository.h"
namespace org {
namespace apache {
namespace nifi {
namespace minifi {
-#ifndef LEVELDB_SUPPORT
-namespace provenance {
-class ProvenanceRepository;
-}
-#endif
namespace core {
-#ifndef LEVELDB_SUPPORT
-class FlowFileRepository;
-#endif
-
std::shared_ptr<core::Repository> createRepository(const std::string configuration_class_name, bool fail_safe, const std::string repo_name) {
std::shared_ptr<core::Repository> return_obj = nullptr;
std::string class_name_lc = configuration_class_name;
std::transform(class_name_lc.begin(), class_name_lc.end(), class_name_lc.begin(), ::tolower);
try {
std::shared_ptr<core::Repository> return_obj = nullptr;
- if (class_name_lc == "flowfilerepository") {
- return_obj = instantiate<core::repository::FlowFileRepository>(repo_name);
- } else if (class_name_lc == "provenancerepository") {
- return_obj = instantiate<provenance::ProvenanceRepository>(repo_name);
- } else if (class_name_lc == "volatileflowfilerepository") {
+
+ auto ptr = core::ClassLoader::getDefaultClassLoader().instantiate<core::Repository>(class_name_lc, class_name_lc);
+ if (nullptr != ptr) {
+ return_obj = ptr;
+ }
+
+ if (return_obj) {
+ return return_obj;
+ }
+ // if the desired repos don't exist, we can try doing string matches and reoly on volatile repositories
+ if (class_name_lc == "flowfilerepository" || class_name_lc == "volatileflowfilerepository") {
return_obj = instantiate<repository::VolatileFlowFileRepository>(repo_name);
- } else if (class_name_lc == "volatileprovenancefilerepository") {
+ } else if (class_name_lc == "provenancerepository" || class_name_lc == "volatileprovenancefilerepository") {
return_obj = instantiate<repository::VolatileProvenanceRepository>(repo_name);
} else if (class_name_lc == "nooprepository") {
return_obj = instantiate<core::Repository>(repo_name);
}
-
if (return_obj) {
return return_obj;
}
@@ -86,23 +79,27 @@ std::shared_ptr<core::ContentRepository> createContentRepository(const std::stri
std::transform(class_name_lc.begin(), class_name_lc.end(), class_name_lc.begin(), ::tolower);
try {
std::shared_ptr<core::ContentRepository> return_obj = nullptr;
- if (class_name_lc == "volatilecontentrepository") {
- return_obj = instantiate<core::repository::VolatileContentRepository>(repo_name);
- } else {
- return_obj = instantiate<core::repository::FileSystemRepository>(repo_name);
- }
+ auto ptr = core::ClassLoader::getDefaultClassLoader().instantiate<core::ContentRepository>(class_name_lc, class_name_lc);
+ if (nullptr != ptr) {
+ return_obj = ptr;
+ }
if (return_obj) {
return return_obj;
}
+ if (class_name_lc == "volatilecontentrepository") {
+ return std::make_shared<core::repository::VolatileContentRepository>(repo_name);
+ } else if (class_name_lc == "filesystemrepository") {
+ return std::make_shared<core::repository::FileSystemRepository>(repo_name);
+ }
if (fail_safe) {
- return std::make_shared<core::repository::FileSystemRepository>("fail_safe");
+ return std::make_shared<core::repository::VolatileContentRepository>("fail_safe");
} else {
throw std::runtime_error("Support for the provided configuration class could not be found");
}
} catch (const std::runtime_error &r) {
if (fail_safe) {
- return std::make_shared<core::repository::FileSystemRepository>("fail_safe");
+ return std::make_shared<core::repository::VolatileContentRepository>("fail_safe");
}
}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/src/core/repository/FlowFileRepository.cpp
----------------------------------------------------------------------
diff --git a/libminifi/src/core/repository/FlowFileRepository.cpp b/libminifi/src/core/repository/FlowFileRepository.cpp
deleted file mode 100644
index 3ed7fbf..0000000
--- a/libminifi/src/core/repository/FlowFileRepository.cpp
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "core/repository/FlowFileRepository.h"
-#include "leveldb/write_batch.h"
-#include <memory>
-#include <string>
-#include <utility>
-#include <vector>
-#include "FlowFileRecord.h"
-
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace core {
-namespace repository {
-
-void FlowFileRepository::flush() {
- leveldb::WriteBatch batch;
- std::string key;
- std::string value;
- leveldb::ReadOptions options;
-
- std::vector<std::shared_ptr<FlowFileRecord>> purgeList;
-
- uint64_t decrement_total = 0;
- while (keys_to_delete.size_approx() > 0) {
- if (keys_to_delete.try_dequeue(key)) {
- db_->Get(options, key, &value);
- decrement_total += value.size();
- std::shared_ptr<FlowFileRecord> eventRead = std::make_shared<FlowFileRecord>(shared_from_this(), content_repo_);
- if (eventRead->DeSerialize(reinterpret_cast<const uint8_t *>(value.data()), value.size())) {
- purgeList.push_back(eventRead);
- }
- logger_->log_info("Issuing batch delete, including %s, Content path %s", eventRead->getUUIDStr(), eventRead->getContentFullPath());
- batch.Delete(key);
- }
- }
- if (db_->Write(leveldb::WriteOptions(), &batch).ok()) {
- logger_->log_info("Decrementing %u from a repo size of %u", decrement_total, repo_size_.load());
- if (decrement_total > repo_size_.load()) {
- repo_size_ = 0;
- } else {
- repo_size_ -= decrement_total;
- }
- }
-
- if (nullptr != content_repo_) {
- for (const auto &ffr : purgeList) {
- auto claim = ffr->getResourceClaim();
- if (claim != nullptr) {
- content_repo_->removeIfOrphaned(claim);
- }
- }
- }
-}
-
-void FlowFileRepository::run() {
- // threshold for purge
- uint64_t purgeThreshold = max_partition_bytes_ * 3 / 4;
-
- while (running_) {
- std::this_thread::sleep_for(std::chrono::milliseconds(purge_period_));
- uint64_t curTime = getTimeMillis();
-
- flush();
-
- uint64_t size = getRepoSize();
-
- if (size > max_partition_bytes_)
- repo_full_ = true;
- else
- repo_full_ = false;
- }
-}
-
-void FlowFileRepository::loadComponent(const std::shared_ptr<core::ContentRepository> &content_repo) {
- content_repo_ = content_repo;
- std::vector<std::pair<std::string, uint64_t>> purgeList;
- leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions());
-
- repo_size_ = 0;
- for (it->SeekToFirst(); it->Valid(); it->Next()) {
- std::shared_ptr<FlowFileRecord> eventRead = std::make_shared<FlowFileRecord>(shared_from_this(), content_repo_);
- std::string key = it->key().ToString();
- repo_size_ += it->value().size();
- if (eventRead->DeSerialize(reinterpret_cast<const uint8_t *>(it->value().data()), it->value().size())) {
- logger_->log_info("Found connection for %s, path %s ", eventRead->getConnectionUuid(), eventRead->getContentFullPath());
- auto search = connectionMap.find(eventRead->getConnectionUuid());
- if (search != connectionMap.end()) {
- // we find the connection for the persistent flowfile, create the flowfile and enqueue that
- std::shared_ptr<core::FlowFile> flow_file_ref = std::static_pointer_cast<core::FlowFile>(eventRead);
- eventRead->setStoredToRepository(true);
- search->second->put(eventRead);
- } else {
- logger_->log_info("Could not find connectinon for %s, path %s ", eventRead->getConnectionUuid(), eventRead->getContentFullPath());
- if (eventRead->getContentFullPath().length() > 0) {
- if (nullptr != eventRead->getResourceClaim()) {
- content_repo_->remove(eventRead->getResourceClaim());
- }
- }
- purgeList.push_back(std::make_pair(key, it->value().size()));
- }
- } else {
- purgeList.push_back(std::make_pair(key, it->value().size()));
- }
- }
-
- delete it;
- for (auto eventId : purgeList) {
- logger_->log_info("Repository Repo %s Purge %s", name_.c_str(), eventId.first.c_str());
- if (Delete(eventId.first)) {
- repo_size_ -= eventId.second;
- }
- }
-
- return;
-}
-
-} /* namespace repository */
-} /* namespace core */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/src/core/repository/VolatileContentRepository.cpp
----------------------------------------------------------------------
diff --git a/libminifi/src/core/repository/VolatileContentRepository.cpp b/libminifi/src/core/repository/VolatileContentRepository.cpp
index 65f1cf9..d3e696b 100644
--- a/libminifi/src/core/repository/VolatileContentRepository.cpp
+++ b/libminifi/src/core/repository/VolatileContentRepository.cpp
@@ -83,11 +83,11 @@ void VolatileContentRepository::start() {
thread_ = std::thread(&VolatileContentRepository::run, shared_from_parent<VolatileContentRepository>());
thread_.detach();
running_ = true;
- logger_->log_info("%s Repository Monitor Thread Start", name_);
+ logger_->log_info("%s Repository Monitor Thread Start", getName());
}
std::shared_ptr<io::BaseStream> VolatileContentRepository::write(const std::shared_ptr<minifi::ResourceClaim> &claim) {
- logger_->log_debug("enter write");
+ logger_->log_debug("enter write for %s", claim->getContentFullPath());
{
std::lock_guard<std::mutex> lock(map_mutex_);
auto claim_check = master_list_.find(claim->getContentFullPath());
@@ -133,7 +133,7 @@ std::shared_ptr<io::BaseStream> VolatileContentRepository::write(const std::shar
}
bool VolatileContentRepository::exists(const std::shared_ptr<minifi::ResourceClaim> &claim) {
- logger_->log_debug("enter exists");
+ logger_->log_debug("enter exists for %s", claim->getContentFullPath());
int size = 0;
{
std::lock_guard<std::mutex> lock(map_mutex_);
@@ -150,7 +150,7 @@ bool VolatileContentRepository::exists(const std::shared_ptr<minifi::ResourceCla
}
std::shared_ptr<io::BaseStream> VolatileContentRepository::read(const std::shared_ptr<minifi::ResourceClaim> &claim) {
- logger_->log_debug("enter read");
+ logger_->log_debug("enter read for %s", claim->getContentFullPath());
int size = 0;
{
std::lock_guard<std::mutex> lock(map_mutex_);
@@ -175,19 +175,28 @@ bool VolatileContentRepository::remove(const std::shared_ptr<minifi::ResourceCla
if (ent != master_list_.end()) {
// if we cannot remove the entry we will let the owner's destructor
// decrement the reference count and free it
+ master_list_.erase(claim->getContentFullPath());
if (ent->second->freeValue(claim)) {
logger_->log_debug("removed %s", claim->getContentFullPath());
+ logger_->log_debug("Remove for %s, reduced to %d", claim->getContentFullPath(), current_size_.load());
return true;
+ } else {
+ logger_->log_debug("free failed for %s", claim->getContentFullPath());
}
- master_list_.erase(claim->getContentFullPath());
+ } else {
+ logger_->log_debug("Could not remove for %s, size is %d", claim->getContentFullPath(), current_size_.load());
}
} else {
std::lock_guard<std::mutex> lock(map_mutex_);
+ auto size = master_list_[claim->getContentFullPath()]->getLength();
delete master_list_[claim->getContentFullPath()];
master_list_.erase(claim->getContentFullPath());
+ current_size_ -= size;
+ logger_->log_debug("Remove for %s, reduced to %d", claim->getContentFullPath(), current_size_.load());
return true;
}
+ logger_->log_debug("Remove for %s, reduced to %d", claim->getContentFullPath(), current_size_.load());
logger_->log_debug("could not remove %s", claim->getContentFullPath());
return false;
}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/src/provenance/ProvenanceRepository.cpp
----------------------------------------------------------------------
diff --git a/libminifi/src/provenance/ProvenanceRepository.cpp b/libminifi/src/provenance/ProvenanceRepository.cpp
deleted file mode 100644
index 665837c..0000000
--- a/libminifi/src/provenance/ProvenanceRepository.cpp
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "provenance/ProvenanceRepository.h"
-#include "leveldb/write_batch.h"
-#include <string>
-#include <vector>
-#include "provenance/Provenance.h"
-namespace org {
-namespace apache {
-namespace nifi {
-namespace minifi {
-namespace provenance {
-
-void ProvenanceRepository::flush() {
- leveldb::WriteBatch batch;
- std::string key;
- std::string value;
- leveldb::ReadOptions options;
- uint64_t decrement_total = 0;
- while (keys_to_delete.size_approx() > 0) {
- if (keys_to_delete.try_dequeue(key)) {
- db_->Get(options, key, &value);
- decrement_total += value.size();
- batch.Delete(key);
- logger_->log_info("Removing %s", key);
- }
- }
- if (db_->Write(leveldb::WriteOptions(), &batch).ok()) {
- logger_->log_info("Decrementing %u from a repo size of %u", decrement_total, repo_size_.load());
- if (decrement_total > repo_size_.load()) {
- repo_size_ = 0;
- } else {
- repo_size_ -= decrement_total;
- }
- }
-}
-
-void ProvenanceRepository::run() {
- while (running_) {
- std::this_thread::sleep_for(std::chrono::milliseconds(purge_period_));
- uint64_t curTime = getTimeMillis();
- // threshold for purge
- uint64_t purgeThreshold = max_partition_bytes_ * 3 / 4;
-
- uint64_t size = getRepoSize();
-
- if (size >= purgeThreshold) {
- leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions());
- for (it->SeekToFirst(); it->Valid(); it->Next()) {
- ProvenanceEventRecord eventRead;
- std::string key = it->key().ToString();
- uint64_t eventTime = eventRead.getEventTime(reinterpret_cast<uint8_t*>(const_cast<char*>(it->value().data())), it->value().size());
- if (eventTime > 0) {
- if ((curTime - eventTime) > max_partition_millis_)
- Delete(key);
- } else {
- logger_->log_debug("NiFi Provenance retrieve event %s fail", key.c_str());
- Delete(key);
- }
- }
- delete it;
- }
- flush();
- size = getRepoSize();
- if (size > max_partition_bytes_)
- repo_full_ = true;
- else
- repo_full_ = false;
- }
-}
-} /* namespace provenance */
-} /* namespace minifi */
-} /* namespace nifi */
-} /* namespace apache */
-} /* namespace org */
-
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/test/curl-tests/C2NullConfiguration.cpp
----------------------------------------------------------------------
diff --git a/libminifi/test/curl-tests/C2NullConfiguration.cpp b/libminifi/test/curl-tests/C2NullConfiguration.cpp
index a2a3906..de8d3b8 100644
--- a/libminifi/test/curl-tests/C2NullConfiguration.cpp
+++ b/libminifi/test/curl-tests/C2NullConfiguration.cpp
@@ -98,8 +98,8 @@ class VerifyC2Server : public IntegrationBase {
parse_http_components(url, port, scheme, path);
std::cout << "path is " << path << std::endl;
configuration->set("c2.agent.protocol.class", "null");
- configuration->set("c2.rest.url", "null");
- configuration->set("c2.rest.url.ack", "null");
+ configuration->set("c2.rest.url", "");
+ configuration->set("c2.rest.url.ack", "");
configuration->set("c2.agent.heartbeat.reporter.classes", "null");
configuration->set("c2.rest.listener.port", "null");
configuration->set("c2.agent.heartbeat.period", "null");
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/test/nodefs/NoLevelDB.cpp
----------------------------------------------------------------------
diff --git a/libminifi/test/nodefs/NoLevelDB.cpp b/libminifi/test/nodefs/NoLevelDB.cpp
deleted file mode 100644
index 6856287..0000000
--- a/libminifi/test/nodefs/NoLevelDB.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "../TestBase.h"
-#include <memory>
-#include "core/Core.h"
-#include "core/RepositoryFactory.h"
-
-TEST_CASE("NoLevelDBTest1", "[NoLevelDBTest]") {
- std::shared_ptr<core::Repository> prov_repo = core::createRepository("provenancerepository", true);
- REQUIRE(nullptr != prov_repo);
-}
-
-TEST_CASE("NoLevelDBTest2", "[NoLevelDBTest]") {
- std::shared_ptr<core::Repository> prov_repo = core::createRepository("flowfilerepository", true);
- REQUIRE(nullptr != prov_repo);
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/test/nodefs/NoRocksDB.cpp
----------------------------------------------------------------------
diff --git a/libminifi/test/nodefs/NoRocksDB.cpp b/libminifi/test/nodefs/NoRocksDB.cpp
new file mode 100644
index 0000000..472334f
--- /dev/null
+++ b/libminifi/test/nodefs/NoRocksDB.cpp
@@ -0,0 +1,32 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "../TestBase.h"
+#include <memory>
+#include "core/Core.h"
+#include "core/RepositoryFactory.h"
+
+TEST_CASE("NoRocksDBTest1", "[NoRocksDBTest]") {
+ std::shared_ptr<core::Repository> prov_repo = core::createRepository("provenancerepository", true);
+ REQUIRE(nullptr != prov_repo);
+}
+
+TEST_CASE("NoRocksDBTest2", "[NoRocksDBTest]") {
+ std::shared_ptr<core::Repository> prov_repo = core::createRepository("flowfilerepository", true);
+ REQUIRE(nullptr != prov_repo);
+}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/test/rocksdb-tests/DBContentRepositoryTests.cpp
----------------------------------------------------------------------
diff --git a/libminifi/test/rocksdb-tests/DBContentRepositoryTests.cpp b/libminifi/test/rocksdb-tests/DBContentRepositoryTests.cpp
new file mode 100644
index 0000000..d92c038
--- /dev/null
+++ b/libminifi/test/rocksdb-tests/DBContentRepositoryTests.cpp
@@ -0,0 +1,246 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#define CATCH_CONFIG_MAIN // This tells Catch to provide a main() - only do this in one cpp file
+#include "../TestBase.h"
+#include <memory>
+#include <string>
+#include "../unit/ProvenanceTestHelper.h"
+#include "provenance/Provenance.h"
+#include "FlowFileRecord.h"
+#include "core/Core.h"
+#include "DatabaseContentRepository.h"
+#include "properties/Configure.h"
+
+TEST_CASE("Write Claim", "[TestDBCR1]") {
+ TestController testController;
+ char format[] = "/tmp/testRepo.XXXXXX";
+ char *dir = testController.createTempDirectory(format);
+ auto content_repo = std::make_shared<core::repository::DatabaseContentRepository>();
+
+ auto configuration = std::make_shared<org::apache::nifi::minifi::Configure>();
+ configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, dir);
+ REQUIRE(true == content_repo->initialize(configuration));
+
+
+ auto claim = std::make_shared<minifi::ResourceClaim>(content_repo, dir);
+ auto stream = content_repo->write(claim);
+
+ stream->writeUTF("well hello there");
+
+ stream->closeStream();
+
+ content_repo->stop();
+ // reclaim the memory
+ content_repo = nullptr;
+
+ content_repo = std::make_shared<core::repository::DatabaseContentRepository>();
+
+ configuration = std::make_shared<org::apache::nifi::minifi::Configure>();
+ configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, dir);
+ REQUIRE(true == content_repo->initialize(configuration));
+
+ auto read_stream = content_repo->read(claim);
+
+ std::string readstr;
+ read_stream->readUTF(readstr);
+
+ REQUIRE(readstr == "well hello there");
+
+ // should not be able to write to the read stream
+ // -1 will indicate that we were not able to write any data
+
+ REQUIRE(read_stream->writeUTF("other value") == -1);
+}
+
+TEST_CASE("Delete Claim", "[TestDBCR2]") {
+ TestController testController;
+ char format[] = "/tmp/testRepo.XXXXXX";
+ char *dir = testController.createTempDirectory(format);
+ auto content_repo = std::make_shared<core::repository::DatabaseContentRepository>();
+
+ auto configuration = std::make_shared<org::apache::nifi::minifi::Configure>();
+ configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, dir);
+ REQUIRE(true == content_repo->initialize(configuration));
+
+
+ auto claim = std::make_shared<minifi::ResourceClaim>(content_repo, dir);
+ auto stream = content_repo->write(claim);
+
+ stream->writeUTF("well hello there");
+
+ stream->closeStream();
+
+ content_repo->stop();
+
+ // reclaim the memory
+ content_repo = nullptr;
+
+ content_repo = std::make_shared<core::repository::DatabaseContentRepository>();
+
+ configuration = std::make_shared<org::apache::nifi::minifi::Configure>();
+ configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, dir);
+ REQUIRE(true == content_repo->initialize(configuration));
+
+ content_repo->remove(claim);
+
+ auto read_stream = content_repo->read(claim);
+
+ std::string readstr;
+
+ // -1 tell us we have an invalid stream
+ REQUIRE(read_stream->readUTF(readstr) == -1);
+}
+
+TEST_CASE("Test Empty Claim", "[TestDBCR3]") {
+ TestController testController;
+ char format[] = "/tmp/testRepo.XXXXXX";
+ char *dir = testController.createTempDirectory(format);
+ auto content_repo = std::make_shared<core::repository::DatabaseContentRepository>();
+
+ auto configuration = std::make_shared<org::apache::nifi::minifi::Configure>();
+ configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, dir);
+ REQUIRE(true == content_repo->initialize(configuration));
+
+ auto claim = std::make_shared<minifi::ResourceClaim>(content_repo, dir);
+ auto stream = content_repo->write(claim);
+
+ // we're writing nothing to the stream.
+
+ stream->closeStream();
+
+ content_repo->stop();
+
+ // reclaim the memory
+ content_repo = nullptr;
+
+ content_repo = std::make_shared<core::repository::DatabaseContentRepository>();
+
+ configuration = std::make_shared<org::apache::nifi::minifi::Configure>();
+ configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, dir);
+ REQUIRE(true == content_repo->initialize(configuration));
+
+ auto read_stream = content_repo->read(claim);
+
+ std::string readstr;
+
+ // -1 tell us we have an invalid stream
+ REQUIRE(read_stream->readUTF(readstr) == -1);
+}
+
+TEST_CASE("Test Null Claim", "[TestDBCR4]") {
+ TestController testController;
+ char format[] = "/tmp/testRepo.XXXXXX";
+ char *dir = testController.createTempDirectory(format);
+ auto content_repo = std::make_shared<core::repository::DatabaseContentRepository>();
+
+ auto configuration = std::make_shared<org::apache::nifi::minifi::Configure>();
+ configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, dir);
+ REQUIRE(true == content_repo->initialize(configuration));
+
+
+ auto claim = std::make_shared<minifi::ResourceClaim>(content_repo, dir);
+ auto stream = content_repo->write(nullptr);
+
+ REQUIRE(stream == nullptr);
+
+ auto read_stream = content_repo->write(nullptr);
+
+ REQUIRE(read_stream == nullptr);
+}
+
+TEST_CASE("Delete Null Claim", "[TestDBCR5]") {
+ TestController testController;
+ char format[] = "/tmp/testRepo.XXXXXX";
+ char *dir = testController.createTempDirectory(format);
+ auto content_repo = std::make_shared<core::repository::DatabaseContentRepository>();
+
+ auto configuration = std::make_shared<org::apache::nifi::minifi::Configure>();
+ configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, dir);
+ REQUIRE(true == content_repo->initialize(configuration));
+
+ auto claim = std::make_shared<minifi::ResourceClaim>(content_repo, dir);
+ auto stream = content_repo->write(claim);
+
+ stream->writeUTF("well hello there");
+
+ stream->closeStream();
+
+ content_repo->stop();
+
+ // reclaim the memory
+ content_repo = nullptr;
+
+ content_repo = std::make_shared<core::repository::DatabaseContentRepository>();
+
+ configuration = std::make_shared<org::apache::nifi::minifi::Configure>();
+ configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, dir);
+ REQUIRE(true == content_repo->initialize(configuration));
+
+ REQUIRE(false == content_repo->remove(nullptr));
+
+ auto read_stream = content_repo->read(claim);
+
+ std::string readstr;
+
+ // -1 tell us we have an invalid stream
+ read_stream->readUTF(readstr);
+
+ REQUIRE(readstr == "well hello there");
+}
+
+TEST_CASE("Delete NonExistent Claim", "[TestDBCR5]") {
+ TestController testController;
+ char format[] = "/tmp/testRepo.XXXXXX";
+ char *dir = testController.createTempDirectory(format);
+ auto content_repo = std::make_shared<core::repository::DatabaseContentRepository>();
+
+ auto configuration = std::make_shared<org::apache::nifi::minifi::Configure>();
+ configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, dir);
+ REQUIRE(true == content_repo->initialize(configuration));
+
+ auto claim = std::make_shared<minifi::ResourceClaim>(content_repo, dir);
+ auto claim2 = std::make_shared<minifi::ResourceClaim>(content_repo, dir);
+ auto stream = content_repo->write(claim);
+
+ stream->writeUTF("well hello there");
+
+ stream->closeStream();
+
+ content_repo->stop();
+
+ // reclaim the memory
+ content_repo = nullptr;
+
+ content_repo = std::make_shared<core::repository::DatabaseContentRepository>();
+
+ configuration = std::make_shared<org::apache::nifi::minifi::Configure>();
+ configuration->set(minifi::Configure::nifi_dbcontent_repository_directory_default, dir);
+ REQUIRE(true == content_repo->initialize(configuration));
+
+ // we won't complain if it does not exist
+ REQUIRE(true == content_repo->remove(claim2));
+
+ auto read_stream = content_repo->read(claim);
+
+ std::string readstr;
+
+ // -1 tell us we have an invalid stream
+ read_stream->readUTF(readstr);
+
+ REQUIRE(readstr == "well hello there");
+}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/test/rocksdb-tests/ProvenanceTests.cpp
----------------------------------------------------------------------
diff --git a/libminifi/test/rocksdb-tests/ProvenanceTests.cpp b/libminifi/test/rocksdb-tests/ProvenanceTests.cpp
new file mode 100644
index 0000000..ffc5d98
--- /dev/null
+++ b/libminifi/test/rocksdb-tests/ProvenanceTests.cpp
@@ -0,0 +1,165 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CATCH_CONFIG_MAIN // This tells Catch to provide a main() - only do this in one cpp file
+#include "../TestBase.h"
+#include <utility>
+#include <memory>
+#include <string>
+#include <map>
+#include "../unit/ProvenanceTestHelper.h"
+#include "provenance/Provenance.h"
+#include "FlowFileRecord.h"
+#include "core/Core.h"
+#include "core/repository/AtomicRepoEntries.h"
+#include "FlowFileRepository.h"
+#include "core/repository/VolatileProvenanceRepository.h"
+
+TEST_CASE("Test Provenance record create", "[Testprovenance::ProvenanceEventRecord]") {
+ provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CREATE, "blah", "blahblah");
+ REQUIRE(record1.getAttributes().size() == 0);
+ REQUIRE(record1.getAlternateIdentifierUri().length() == 0);
+}
+
+TEST_CASE("Test Provenance record serialization", "[Testprovenance::ProvenanceEventRecordSerializeDeser]") {
+ provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CREATE, "componentid", "componenttype");
+
+ std::string eventId = record1.getEventId();
+
+ std::string smileyface = ":)";
+ record1.setDetails(smileyface);
+
+ uint64_t sample = 65555;
+ std::shared_ptr<core::Repository> testRepository = std::make_shared<TestRepository>();
+ record1.setEventDuration(sample);
+
+ record1.Serialize(testRepository);
+ provenance::ProvenanceEventRecord record2;
+ record2.setEventId(eventId);
+ REQUIRE(record2.DeSerialize(testRepository) == true);
+ REQUIRE(record2.getEventId() == record1.getEventId());
+ REQUIRE(record2.getComponentId() == record1.getComponentId());
+ REQUIRE(record2.getComponentType() == record1.getComponentType());
+ REQUIRE(record2.getDetails() == record1.getDetails());
+ REQUIRE(record2.getDetails() == smileyface);
+ REQUIRE(record2.getEventDuration() == sample);
+}
+
+TEST_CASE("Test Flowfile record added to provenance", "[TestFlowAndProv1]") {
+ provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CLONE, "componentid", "componenttype");
+ std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>();
+ std::string eventId = record1.getEventId();
+ std::map<std::string, std::string> attributes;
+ attributes.insert(std::pair<std::string, std::string>("potato", "potatoe"));
+ attributes.insert(std::pair<std::string, std::string>("tomato", "tomatoe"));
+ std::shared_ptr<core::repository::FlowFileRepository> frepo = std::make_shared<core::repository::FlowFileRepository>("ff", "./content_repository", 0, 0, 0);
+ std::shared_ptr<minifi::FlowFileRecord> ffr1 = std::make_shared<minifi::FlowFileRecord>(frepo, content_repo, attributes);
+
+ record1.addChildFlowFile(ffr1);
+
+ uint64_t sample = 65555;
+ std::shared_ptr<core::Repository> testRepository = std::make_shared<TestRepository>();
+ record1.setEventDuration(sample);
+
+ record1.Serialize(testRepository);
+ provenance::ProvenanceEventRecord record2;
+ record2.setEventId(eventId);
+ REQUIRE(record2.DeSerialize(testRepository) == true);
+ REQUIRE(record1.getChildrenUuids().size() == 1);
+ REQUIRE(record2.getChildrenUuids().size() == 1);
+ std::string childId = record2.getChildrenUuids().at(0);
+ REQUIRE(childId == ffr1->getUUIDStr());
+ record2.removeChildUuid(childId);
+ REQUIRE(record2.getChildrenUuids().size() == 0);
+}
+
+TEST_CASE("Test Provenance record serialization Volatile", "[Testprovenance::ProvenanceEventRecordSerializeDeser]") {
+ provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CREATE, "componentid", "componenttype");
+
+ std::string eventId = record1.getEventId();
+
+ std::string smileyface = ":)";
+ record1.setDetails(smileyface);
+
+ uint64_t sample = 65555;
+
+ std::shared_ptr<core::Repository> testRepository = std::make_shared<core::repository::VolatileProvenanceRepository>();
+ testRepository->initialize(0);
+ record1.setEventDuration(sample);
+
+ record1.Serialize(testRepository);
+ provenance::ProvenanceEventRecord record2;
+ record2.setEventId(eventId);
+ REQUIRE(record2.DeSerialize(testRepository) == true);
+ REQUIRE(record2.getEventId() == record1.getEventId());
+ REQUIRE(record2.getComponentId() == record1.getComponentId());
+ REQUIRE(record2.getComponentType() == record1.getComponentType());
+ REQUIRE(record2.getDetails() == record1.getDetails());
+ REQUIRE(record2.getDetails() == smileyface);
+ REQUIRE(record2.getEventDuration() == sample);
+}
+
+TEST_CASE("Test Flowfile record added to provenance using Volatile Repo", "[TestFlowAndProv1]") {
+ provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CLONE, "componentid", "componenttype");
+ std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>();
+ std::string eventId = record1.getEventId();
+ std::map<std::string, std::string> attributes;
+ attributes.insert(std::pair<std::string, std::string>("potato", "potatoe"));
+ attributes.insert(std::pair<std::string, std::string>("tomato", "tomatoe"));
+ std::shared_ptr<core::Repository> frepo = std::make_shared<core::repository::VolatileProvenanceRepository>();
+ frepo->initialize(0);
+ std::shared_ptr<minifi::FlowFileRecord> ffr1 = std::make_shared<minifi::FlowFileRecord>(frepo, content_repo, attributes);
+
+ record1.addChildFlowFile(ffr1);
+
+ uint64_t sample = 65555;
+ std::shared_ptr<core::Repository> testRepository = std::make_shared<core::repository::VolatileProvenanceRepository>();
+ testRepository->initialize(0);
+ record1.setEventDuration(sample);
+
+ record1.Serialize(testRepository);
+ provenance::ProvenanceEventRecord record2;
+ record2.setEventId(eventId);
+ REQUIRE(record2.DeSerialize(testRepository) == true);
+ REQUIRE(record1.getChildrenUuids().size() == 1);
+ REQUIRE(record2.getChildrenUuids().size() == 1);
+ std::string childId = record2.getChildrenUuids().at(0);
+ REQUIRE(childId == ffr1->getUUIDStr());
+ record2.removeChildUuid(childId);
+ REQUIRE(record2.getChildrenUuids().size() == 0);
+}
+
+TEST_CASE("Test Provenance record serialization NoOp", "[Testprovenance::ProvenanceEventRecordSerializeDeser]") {
+ provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CREATE, "componentid", "componenttype");
+
+ std::string eventId = record1.getEventId();
+
+ std::string smileyface = ":)";
+ record1.setDetails(smileyface);
+
+ uint64_t sample = 65555;
+
+ std::shared_ptr<core::Repository> testRepository = std::make_shared<core::Repository>();
+ testRepository->initialize(0);
+ record1.setEventDuration(sample);
+
+ REQUIRE(record1.Serialize(testRepository) == true);
+ provenance::ProvenanceEventRecord record2;
+ record2.setEventId(eventId);
+ REQUIRE(record2.DeSerialize(testRepository) == false);
+}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/test/rocksdb-tests/RepoTests.cpp
----------------------------------------------------------------------
diff --git a/libminifi/test/rocksdb-tests/RepoTests.cpp b/libminifi/test/rocksdb-tests/RepoTests.cpp
new file mode 100644
index 0000000..e966aba
--- /dev/null
+++ b/libminifi/test/rocksdb-tests/RepoTests.cpp
@@ -0,0 +1,162 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#define CATCH_CONFIG_MAIN // This tells Catch to provide a main() - only do this in one cpp file
+#include "../TestBase.h"
+#include <memory>
+#include <string>
+#include <map>
+#include "../unit/ProvenanceTestHelper.h"
+#include "provenance/Provenance.h"
+#include "FlowFileRecord.h"
+#include "core/Core.h"
+#include "FlowFileRepository.h"
+#include "core/repository/AtomicRepoEntries.h"
+#include "properties/Configure.h"
+
+TEST_CASE("Test Repo Empty Value Attribute", "[TestFFR1]") {
+ TestController testController;
+ char format[] = "/tmp/testRepo.XXXXXX";
+ char *dir = testController.createTempDirectory(format);
+ std::shared_ptr<core::repository::FlowFileRepository> repository = std::make_shared<core::repository::FlowFileRepository>("ff", dir, 0, 0, 1);
+
+ repository->initialize(std::make_shared<minifi::Configure>());
+
+ std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>();
+ minifi::FlowFileRecord record(repository, content_repo);
+
+ record.addAttribute("keyA", "");
+
+ REQUIRE(true == record.Serialize());
+
+ repository->stop();
+}
+
+TEST_CASE("Test Repo Empty Key Attribute ", "[TestFFR2]") {
+ TestController testController;
+ char format[] = "/tmp/testRepo.XXXXXX";
+ char *dir = testController.createTempDirectory(format);
+ std::shared_ptr<core::repository::FlowFileRepository> repository = std::make_shared<core::repository::FlowFileRepository>("ff", dir, 0, 0, 1);
+
+ repository->initialize(std::make_shared<minifi::Configure>());
+ std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>();
+ minifi::FlowFileRecord record(repository, content_repo);
+
+ record.addAttribute("keyA", "hasdgasdgjsdgasgdsgsadaskgasd");
+
+ record.addAttribute("", "hasdgasdgjsdgasgdsgsadaskgasd");
+
+ REQUIRE(true == record.Serialize());
+
+ repository->stop();
+}
+
+TEST_CASE("Test Repo Key Attribute Verify ", "[TestFFR3]") {
+ TestController testController;
+ char format[] = "/tmp/testRepo.XXXXXX";
+ char *dir = testController.createTempDirectory(format);
+ std::shared_ptr<core::repository::FlowFileRepository> repository = std::make_shared<core::repository::FlowFileRepository>("ff", dir, 0, 0, 1);
+
+ repository->initialize(std::make_shared<org::apache::nifi::minifi::Configure>());
+
+ std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>();
+ minifi::FlowFileRecord record(repository, content_repo);
+
+ minifi::FlowFileRecord record2(repository, content_repo);
+
+ std::string uuid = record.getUUIDStr();
+
+ record.addAttribute("keyA", "hasdgasdgjsdgasgdsgsadaskgasd");
+
+ record.addAttribute("keyB", "");
+
+ record.addAttribute("", "");
+
+ record.updateAttribute("", "hasdgasdgjsdgasgdsgsadaskgasd2");
+
+ record.addAttribute("", "sdgsdg");
+
+ REQUIRE(true == record.Serialize());
+
+ repository->stop();
+
+ record2.DeSerialize(uuid);
+
+ std::string value;
+ REQUIRE(true == record2.getAttribute("", value));
+
+ REQUIRE("hasdgasdgjsdgasgdsgsadaskgasd2" == value);
+
+ REQUIRE(false == record2.getAttribute("key", value));
+ REQUIRE(true == record2.getAttribute("keyA", value));
+ REQUIRE("hasdgasdgjsdgasgdsgsadaskgasd" == value);
+
+ REQUIRE(true == record2.getAttribute("keyB", value));
+ REQUIRE("" == value);
+}
+
+TEST_CASE("Test Delete Content ", "[TestFFR4]") {
+ TestController testController;
+ char format[] = "/tmp/testRepo.XXXXXX";
+ LogTestController::getInstance().setDebug<core::ContentRepository>();
+ LogTestController::getInstance().setDebug<core::repository::FileSystemRepository>();
+ LogTestController::getInstance().setDebug<core::repository::FlowFileRepository>();
+
+ char *dir = testController.createTempDirectory(format);
+
+ std::shared_ptr<core::repository::FlowFileRepository> repository = std::make_shared<core::repository::FlowFileRepository>("ff", dir, 0, 0, 1);
+
+ std::map<std::string, std::string> attributes;
+
+ std::fstream file;
+ std::stringstream ss;
+ ss << dir << "/" << "tstFile.ext";
+ file.open(ss.str(), std::ios::out);
+ file << "tempFile";
+ file.close();
+
+ std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::FileSystemRepository>();
+
+ repository->initialize(std::make_shared<minifi::Configure>());
+
+ repository->loadComponent(content_repo);
+
+ std::shared_ptr<minifi::ResourceClaim> claim = std::make_shared<minifi::ResourceClaim>(ss.str(), content_repo);
+
+ minifi::FlowFileRecord record(repository, content_repo, attributes, claim);
+
+ record.addAttribute("keyA", "hasdgasdgjsdgasgdsgsadaskgasd");
+
+ record.addAttribute("", "hasdgasdgjsdgasgdsgsadaskgasd");
+
+ REQUIRE(true == record.Serialize());
+
+ claim->decreaseFlowFileRecordOwnedCount();
+
+ claim->decreaseFlowFileRecordOwnedCount();
+
+ repository->Delete(record.getUUIDStr());
+
+ repository->flush();
+
+ repository->stop();
+
+ std::ifstream fileopen(ss.str());
+ REQUIRE(false == fileopen.good());
+
+ LogTestController::getInstance().reset();
+}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/test/unit/ProvenanceTestHelper.h
----------------------------------------------------------------------
diff --git a/libminifi/test/unit/ProvenanceTestHelper.h b/libminifi/test/unit/ProvenanceTestHelper.h
index 2583a09..d3bccd0 100644
--- a/libminifi/test/unit/ProvenanceTestHelper.h
+++ b/libminifi/test/unit/ProvenanceTestHelper.h
@@ -28,7 +28,6 @@
#include <vector>
#include "core/repository/VolatileContentRepository.h"
#include "../../include/core/Processor.h"
-#include "../../include/core/repository/FlowFileRepository.h"
#include "../../include/Connection.h"
#include "../../include/FlowController.h"
#include "../../include/properties/Configure.h"
@@ -40,7 +39,7 @@
class TestRepository : public core::Repository {
public:
TestRepository()
- : Repository("repo_name", "./dir", 1000, 100, 0) {
+ : Repository("repo_name", "./dir", 1000, 100, 0), core::SerializableComponent("repo_name",0) {
}
// initialize
bool initialize() {
@@ -145,10 +144,10 @@ class TestRepository : public core::Repository {
std::map<std::string, std::string> repositoryResults;
};
-class TestFlowRepository : public core::repository::FlowFileRepository {
+class TestFlowRepository : public core::Repository {
public:
TestFlowRepository()
- : core::repository::FlowFileRepository("ff", "./dir", 1000, 100, 0) {
+ : core::Repository("ff", "./dir", 1000, 100, 0), core::SerializableComponent("ff",0) {
}
// initialize
bool initialize() {
[04/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-05-14-lock.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-05-14-lock.markdown b/thirdparty/rocksdb/docs/_posts/2014-05-14-lock.markdown
new file mode 100644
index 0000000..12009cc
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-05-14-lock.markdown
@@ -0,0 +1,88 @@
+---
+title: Reducing Lock Contention in RocksDB
+layout: post
+author: sdong
+category: blog
+redirect_from:
+ - /blog/521/lock/
+---
+
+In this post, we briefly introduce the recent improvements we did to RocksDB to improve the issue of lock contention costs.
+
+RocksDB has a simple thread synchronization mechanism (See [RocksDB Architecture Guide](https://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide) to understand terms used below, like SST tables or mem tables). SST tables are immutable after being written and mem tables are lock-free data structures supporting single writer and multiple readers. There is only one single major lock, the DB mutex (DBImpl.mutex_) protecting all the meta operations, including:
+
+<!--truncate-->
+
+ * Increase or decrease reference counters of mem tables and SST tables
+
+
+ * Change and check meta data structures, before and after finishing compactions, flushes and new mem table creations
+
+
+ * Coordinating writers
+
+
+This DB mutex used to be scalability bottleneck preventing us from scaling to more than 16 threads. To address the issue, we improved RocksDB in several ways.
+
+1. Consolidate reference counters and introduce "super version". For every read operation, mutex was acquired, and reference counters for each mem table and each SST table were increased. One such operation is not expensive but if you are building a high throughput server with lots of reads, the lock contention will become the bottleneck. This is especially true if you store all your data in RAM.
+
+To solve this problem, we created a meta-meta data structure called “[super version](https://reviews.facebook.net/rROCKSDB1fdb3f7dc60e96394e3e5b69a46ede5d67fb976c)”, which holds reference counters to all those mem table and SST tables, so that readers only need to increase the reference counters for this single data structure. In RocksDB, list of live mem tables and SST tables only changes infrequently, which would happen when new mem tables are created or flush/compaction happens. Now, at those times, a new super version is created with their reference counters increased. A super version lists live mem tables and SST tables so a reader only needs acquire the lock in order to find the latest super version and increase its reference counter. From the super version, the reader can find all the mem and SST tables which are safety accessible as long as the reader holds the reference count for the super version.
+
+2. We replace some reference counters to stc::atomic objects, so that decreasing reference count of an object usually doesn’t need to be inside the mutex any more.
+
+3. Make fetching super version and reference counting lock-free in read queries. After consolidating reference counting to one single super version and removing the locking for decreasing reference counts, in read case, we only acquire mutex for one thing: fetch the latest super version and increase the reference count for that (dereference the counter is done in an atomic decrease). We designed and implemented a (mostly) lock-free approach to do it. See [details](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Lei-Lockless-Get.pdf). We will write a separate blog post for that.
+
+4. Avoid disk I/O inside the mutex. As we know, each disk I/O to hard drives takes several milliseconds. It can be even longer if file system journal is involved or I/Os are queued. Even occasional disk I/O within mutex can cause huge performance outliers.
+We identified in two situations, we might do disk I/O inside mutex and we removed them:
+(1) Opening and closing transactional log files. We moved those operations out of the mutex.
+(2) Information logging. In multiple places we write to logs within mutex. There is a chance that file write will wait for disk I/O to finish before finishing, even if fsync() is not issued, especially in EXT systems. We occasionally see 100+ milliseconds write() latency on EXT. Instead of removing those logging, we came up with a solution of delay logging. When inside mutex, instead of directly writing to the log file, we write to a log buffer, with the timing information. As soon as mutex is released, we flush the log buffer to log files.
+
+5. Reduce object creation inside the mutex.
+Object creation can be slow because it involves malloc (in our case). Malloc sometimes is slow because it needs to lock some shared data structures. Allocating can also be slow because we sometimes do expensive operations in some of our classes' constructors. For these reasons, we try to reduce object creations inside the mutex. Here are two examples:
+
+(1) std::vector uses malloc inside. We introduced “[autovector](https://reviews.facebook.net/rROCKSDBc01676e46d3be08c3c140361ef1f5884f47d3b3c)” data structure, in which memory for first a few elements are pre-allocated as members of the autovector class. When an autovector is used as a stack variable, no malloc will be needed unless the pre-allocated buffer is used up. This autovector is quite useful for manipulating those meta data structures. Those meta operations are often locked inside DB mutex.
+
+(2) When building an iterator, we used to creating iterator of every live men table and SST table within the mutex and a merging iterator on top of them. Besides malloc, some of those iterators can be quite expensive to create, like sorting. Now, instead of doing that, we simply increase the reference counters of them, and release the mutex before creating any iterator.
+
+6. Deal with mutexes in LRU caches.
+When I said there was only one single major lock, I was lying. In RocksDB, all LRU caches had exclusive mutexes within to protect writes to the LRU lists, which are done in both of read and write operations. LRU caches are used in block cache and table cache. Both of them are accessed more frequently than DB data structures. Lock contention of these two locks are as intense as the DB mutex. Even if LRU cache is sharded into ShardedLRUCache, we can still see lock contentions, especially table caches. We further address this issue in two way:
+(1) Bypassing table caches. A table cache maintains list of SST table’s read handlers. Those handlers contain SST files’ descriptors, table metadata, and possibly data indexes, as well as bloom filters. When the table handler needs to be evicted based on LRU, those information is cleared. When the SST table needs to be read and its table handler is not in LRU cache, the table is opened and those metadata is loaded. In some cases, users want to tune the system in a way that table handler evictions should never happen. It is common for high-throughput, low-latency servers. We introduce a mode where table cache is bypassed in read queries. In this mode, all table handlers are cached and accessed directly, so there is no need to query and adjust table caches for reading the database. It is the users’ responsibility to reserve enough resource for it. This mode can be turned on by setting options.max_open_files=-1.
+
+(2) [New PlainTable format](//github.com/facebook/rocksdb/wiki/PlainTable-Format) (optimized for SST in ramfs/tmpfs) does not organize data by blocks. Data are located by memory addresses so no block cache is needed.
+
+With all of those improvements, lock contention is not a bottleneck anymore, which is shown in our [memory-only benchmark](https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks) . Furthermore, lock contentions are not causing some huge (50 milliseconds+) latency outliers they used to cause.
+
+### Comments
+
+**[Lee Hounshell](lee@apsalar.com)**
+
+Please post an example of reading the same rocksdb concurrently.
+
+We are using the latest 3.0 rocksdb; however, when two separate processes
+try and open the same rocksdb for reading, only one of the open requests
+succeed. The other open always fails with “db/LOCK: Resource temporarily unavailable” So far we have not found an option that allows sharing the rocksdb for reads. An example would be most appreciated.
+
+**[Siying Dong](siying.d@fb.com)**
+
+Sorry for the delay. We don’t have feature support for this scenario yet. Here is an example you can work around this problem. You can build a snapshot of the DB by doing this:
+
+1. create a separate directory on the same host for a snapshot of the DB.
+1. call `DB::DisableFileDeletions()`
+1. call `DB::GetLiveFiles()` to get a full list of the files.
+1. for all the files except manifest, add a hardlink file in your new directory pointing to the original file
+1. copy the manifest file and truncate the size (you can read the comments of `DB::GetLiveFiles()` for more information)
+1. call `DB::EnableFileDeletions()`
+1. now you can open the snapshot directory in another process to access those files. Please remember to delete the directory after reading the data to allow those files to be recycled.
+
+By the way, the best way to ask those questions is in our [facebook group](https://www.facebook.com/groups/rocksdb.dev/). Let us know if you need any further help.
+
+**[Darshan](darshan.ghumare@gmail.com)**
+
+Will this consistency problem of RocksDB all occurs in case of single put/write?
+What all ACID properties is supported by RocksDB, only durability irrespective of single or batch write?
+
+**[Siying Dong](siying.d@fb.com)**
+
+We recently [introduced optimistic transaction](https://reviews.facebook.net/D33435) which can help you ensure all of ACID.
+
+This blog post is mainly about optimizations in implementation. The RocksDB consistency semantic is not changed.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-05-19-rocksdb-3-0-release.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-05-19-rocksdb-3-0-release.markdown b/thirdparty/rocksdb/docs/_posts/2014-05-19-rocksdb-3-0-release.markdown
new file mode 100644
index 0000000..61c90dc
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-05-19-rocksdb-3-0-release.markdown
@@ -0,0 +1,24 @@
+---
+title: RocksDB 3.0 release
+layout: post
+author: icanadi
+category: blog
+redirect_from:
+ - /blog/557/rocksdb-3-0-release/
+---
+
+Check out new RocksDB release on [Github](https://github.com/facebook/rocksdb/releases/tag/3.0.fb)!
+
+New features in RocksDB 3.0:
+
+ * [Column Family support](https://github.com/facebook/rocksdb/wiki/Column-Families)
+
+
+ * [Ability to chose different checksum function](https://github.com/facebook/rocksdb/commit/0afc8bc29a5800e3212388c327c750d32e31f3d6)
+
+
+ * Deprecated ReadOptions::prefix_seek and ReadOptions::prefix
+
+<!--truncate-->
+
+Check out the full [change log](https://github.com/facebook/rocksdb/blob/3.0.fb/HISTORY.md).
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-05-22-rocksdb-3-1-release.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-05-22-rocksdb-3-1-release.markdown b/thirdparty/rocksdb/docs/_posts/2014-05-22-rocksdb-3-1-release.markdown
new file mode 100644
index 0000000..3015674
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-05-22-rocksdb-3-1-release.markdown
@@ -0,0 +1,20 @@
+---
+title: RocksDB 3.1 release
+layout: post
+author: icanadi
+category: blog
+redirect_from:
+ - /blog/575/rocksdb-3-1-release/
+---
+
+Check out the new release on [Github](https://github.com/facebook/rocksdb/releases/tag/rocksdb-3.1)!
+
+New features in RocksDB 3.1:
+
+ * [Materialized hash index](https://github.com/facebook/rocksdb/commit/0b3d03d026a7248e438341264b4c6df339edc1d7)
+
+
+ * [FIFO compaction style](https://github.com/facebook/rocksdb/wiki/FIFO-compaction-style)
+
+
+We released 3.1 so fast after 3.0 because one of our internal customers needed materialized hash index.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-06-23-plaintable-a-new-file-format.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-06-23-plaintable-a-new-file-format.markdown b/thirdparty/rocksdb/docs/_posts/2014-06-23-plaintable-a-new-file-format.markdown
new file mode 100644
index 0000000..6a641f2
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-06-23-plaintable-a-new-file-format.markdown
@@ -0,0 +1,47 @@
+---
+title: PlainTable — A New File Format
+layout: post
+author: sdong
+category: blog
+redirect_from:
+ - /blog/599/plaintable-a-new-file-format/
+---
+
+In this post, we are introducing "PlainTable" -- a file format we designed for RocksDB, initially to satisfy a production use case at Facebook.
+
+Design goals:
+
+1. All data stored in memory, in files stored in tmpfs/ramfs. Support DBs larger than 100GB (may be sharded across multiple RocksDB instance).
+1. Optimize for [prefix hashing](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Siying-Prefix-Hash.pdf)
+1. Less than or around 1 micro-second average latency for single Get() or Seek().
+1. Minimize memory consumption.
+1. Queries efficiently return empty results
+
+<!--truncate-->
+
+Notice that our priority was not to maximize query performance, but to strike a balance between query performance and memory consumption. PlainTable query performance is not as good as you would see with a nicely-designed hash table, but they are of the same order of magnitude, while keeping memory overhead to a minimum.
+
+Since we are targeting micro-second latency, it is on the level of the number of CPU cache misses (if they cannot be parallellized, which are usually the case for index look-ups). On our target hardware with Intel CPUs of multiple sockets with NUMA, we can only allow 4-5 CPU cache misses (including costs of data TLB).
+
+To meet our requirements, given that only hash prefix iterating is needed, we made two decisions:
+
+1. to use a hash index, which is
+1. directly addressed to rows, with no block structure.
+
+Having addressed our latency goal, the next task was to design a very compact hash index to minimize memory consumption. Some tricks we used to meet this goal:
+
+1. We only use 32-bit integers for data and index offsets.The first bit serves as a flag, so we can avoid using 8-byte pointers.
+1. We never copy keys or parts of keys to index search structures. We store only offsets from which keys can be retrieved, to make comparisons with search keys.
+1. Since our file is immutable, we can accurately estimate the number of hash buckets needed.
+
+To make sure the format works efficiently with empty queries, we added a bloom filter check before the query. This adds only one cache miss for non-empty cases [1], but avoids multiple cache misses for most empty results queries. This is a good trade-off for use cases with a large percentage of empty results.
+
+These are the design goals and basic ideas of PlainTable file format. For detailed information, see [this wiki page](https://github.com/facebook/rocksdb/wiki/PlainTable-Format).
+
+[1] Bloom filter checks typically require multiple memory access. However, because they are independent, they usually do not make the CPU pipeline stale. In any case, we improved the bloom filter to improve data locality - we may cover this further in a future blog post.
+
+### Comments
+
+**[Siying Dong](siying.d@fb.com)**
+
+Does [http://rocksdb.org/feed/](http://rocksdb.org/feed/) work?
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-06-27-avoid-expensive-locks-in-get.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-06-27-avoid-expensive-locks-in-get.markdown b/thirdparty/rocksdb/docs/_posts/2014-06-27-avoid-expensive-locks-in-get.markdown
new file mode 100644
index 0000000..4411c7a
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-06-27-avoid-expensive-locks-in-get.markdown
@@ -0,0 +1,89 @@
+---
+title: Avoid Expensive Locks in Get()
+layout: post
+author: leijin
+category: blog
+redirect_from:
+ - /blog/677/avoid-expensive-locks-in-get/
+---
+
+As promised in the previous [blog post](blog/2014/05/14/lock.html)!
+
+RocksDB employs a multiversion concurrency control strategy. Before reading data, it needs to grab the current version, which is encapsulated in a data structure called [SuperVersion](https://reviews.facebook.net/rROCKSDB1fdb3f7dc60e96394e3e5b69a46ede5d67fb976c).
+
+<!--truncate-->
+
+At the beginning of `GetImpl()`, it used to do this:
+
+
+ <span class="zw-portion">mutex_.Lock();
+ </span>auto* s = super_version_->Ref();
+ mutex_.Unlock();
+
+
+The lock is necessary because pointer super_version_ may be updated, the corresponding SuperVersion may be deleted while Ref() is in progress.
+
+
+`Ref()` simply increases the reference counter and returns “this” pointer. However, this simple operation posed big challenges for in-memory workload and stopped RocksDB from scaling read throughput beyond 8 cores. Running 32 read threads on a 32-core CPU leads to [70% system CPU usage](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Lei-Lockless-Get.pdf). This is outrageous!
+
+
+
+
+Luckily, we found a way to circumvent this problem by using [thread local storage](http://en.wikipedia.org/wiki/Thread-local_storage). Version change is a rare event comparable to millions of read requests. On the very first Get() request, each thread pays the mutex cost to acquire a reference to the new super version. Instead of releasing the reference after use, the reference is cached in thread’s local storage. An atomic variable is used to track global super version number. Subsequent reads simply compare the local super version number against the global super version number. If they are the same, the cached super version reference may be used directly, at no cost. If a version change is detected, mutex must be acquired to update the reference. The cost of mutex lock is amortized among millions of reads and becomes negligible.
+
+
+
+
+The code looks something like this:
+
+
+
+
+
+ SuperVersion* s = thread_local_->Get();
+ if (s->version_number != super_version_number_.load()) {
+ // slow path, cleanup of current super version is omitted
+ mutex_.Lock();
+ s = super_version_->Ref();
+ mutex_.Unlock();
+ }
+
+
+
+
+The result is quite amazing. RocksDB can nicely [scale to 32 cores](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Lei-Lockless-Get.pdf) and most CPU time is spent in user land.
+
+
+
+
+Daryl Grove gives a pretty good [comparison between mutex and atomic](https://blogs.oracle.com/d/entry/the_cost_of_mutexes). However, the real cost difference lies beyond what is shown in the assembly code. Mutex can keep threads spinning on CPU or even trigger thread context switches in which all readers compete to access the critical area. Our approach prevents mutual competition by directing threads to check against a global version which does not change at high frequency, and is therefore much more cache-friendly.
+
+
+
+
+The new approach entails one issue: a thread can visit GetImpl() once but can never come back again. SuperVersion is referenced and cached in its thread local storage. All resources (e.g., memtables, files) which belong to that version are frozen. A “supervisor” is required to visit each thread’s local storage and free its resources without incurring a lock. We designed a lockless sweep using CAS (compare and switch instruction). Here is how it works:
+
+
+
+
+(1) A reader thread uses CAS to acquire SuperVersion from its local storage and to put in a special flag (SuperVersion::kSVInUse).
+
+
+
+
+(2) Upon completion of GetImpl(), the reader thread tries to return SuperVersion to local storage by CAS, expecting the special flag (SuperVersion::kSVInUse) in its local storage. If it does not see SuperVersion::kSVInUse, that means a “sweep” was done and the reader thread is responsible for cleanup (this is expensive, but does not happen often on the hot path).
+
+
+
+
+(3) After any flush/compaction, the background thread performs a sweep (CAS) across all threads’ local storage and frees encountered SuperVersion. A reader thread must re-acquire a new SuperVersion reference on its next visit.
+
+### Comments
+
+**[David Barbour](dmbarbour@gmail.com)**
+
+Please post an example of reading the same rocksdb concurrently.
+
+We are using the latest 3.0 rocksdb; however, when two separate processes
+try and open the same rocksdb for reading, only one of the open requests
+succeed. The other open always fails with “db/LOCK: Resource temporarily unavailable” So far we have not found an option that allows sharing the rocksdb for reads. An example would be most appreciated.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-06-27-rocksdb-3-2-release.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-06-27-rocksdb-3-2-release.markdown b/thirdparty/rocksdb/docs/_posts/2014-06-27-rocksdb-3-2-release.markdown
new file mode 100644
index 0000000..e4eba6a
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-06-27-rocksdb-3-2-release.markdown
@@ -0,0 +1,30 @@
+---
+title: RocksDB 3.2 release
+layout: post
+author: leijin
+category: blog
+redirect_from:
+ - /blog/647/rocksdb-3-2-release/
+---
+
+Check out new RocksDB release on [GitHub](https://github.com/facebook/rocksdb/releases/tag/rocksdb-3.2)!
+
+New Features in RocksDB 3.2:
+
+ * PlainTable now supports a new key encoding: for keys of the same prefix, the prefix is only written once. It can be enabled through encoding_type paramter of NewPlainTableFactory()
+
+
+ * Add AdaptiveTableFactory, which is used to convert from a DB of PlainTable to BlockBasedTabe, or vise versa. It can be created using NewAdaptiveTableFactory()
+
+<!--truncate-->
+
+Public API changes:
+
+
+ * We removed seek compaction as a concept from RocksDB
+
+
+ * Add two paramters to NewHashLinkListRepFactory() for logging on too many entries in a hash bucket when flushing
+
+
+ * Added new option BlockBasedTableOptions::hash_index_allow_collision. When enabled, prefix hash index for block-based table will not store prefix and allow hash collision, reducing memory consumption
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-07-29-rocksdb-3-3-release.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-07-29-rocksdb-3-3-release.markdown b/thirdparty/rocksdb/docs/_posts/2014-07-29-rocksdb-3-3-release.markdown
new file mode 100644
index 0000000..d858e4f
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-07-29-rocksdb-3-3-release.markdown
@@ -0,0 +1,34 @@
+---
+title: RocksDB 3.3 Release
+layout: post
+author: yhciang
+category: blog
+redirect_from:
+ - /blog/1301/rocksdb-3-3-release/
+---
+
+Check out new RocksDB release on [GitHub](https://github.com/facebook/rocksdb/releases/tag/rocksdb-3.3)!
+
+New Features in RocksDB 3.3:
+
+ * **JSON API prototype**.
+
+
+ * **Performance improvement on HashLinkList**: We addressed performance outlier of HashLinkList caused by skewed bucket by switching data in the bucket from linked list to skip list. Add parameter threshold_use_skiplist in NewHashLinkListRepFactory().
+
+<!--truncate-->
+
+ * **More effective on storage space reclaim**: RocksDB is now able to reclaim storage space more effectively during the compaction process. This is done by compensating the size of each deletion entry by the 2X average value size, which makes compaction to be triggerred by deletion entries more easily.
+
+
+ * **TimeOut API to write**: Now WriteOptions have a variable called timeout_hint_us. With timeout_hint_us set to non-zero, any write associated with this timeout_hint_us may be aborted when it runs longer than the specified timeout_hint_us, and it is guaranteed that any write completes earlier than the specified time-out will not be aborted due to the time-out condition.
+
+
+ * **rate_limiter option**: We added an option that controls total throughput of flush and compaction. The throughput is specified in bytes/sec. Flush always has precedence over compaction when available bandwidth is constrained.
+
+
+
+Public API changes:
+
+
+ * Removed NewTotalOrderPlainTableFactory because it is not used and implemented semantically incorrect.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-09-12-cuckoo.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-09-12-cuckoo.markdown b/thirdparty/rocksdb/docs/_posts/2014-09-12-cuckoo.markdown
new file mode 100644
index 0000000..22178f7
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-09-12-cuckoo.markdown
@@ -0,0 +1,74 @@
+---
+title: Cuckoo Hashing Table Format
+layout: post
+author: radheshyam
+category: blog
+redirect_from:
+ - /blog/1427/new-bloom-filter-format/
+---
+
+## Introduction
+
+We recently introduced a new [Cuckoo Hashing](http://en.wikipedia.org/wiki/Cuckoo_hashing) based SST file format which is optimized for fast point lookups. The new format was built for applications which require very high point lookup rates (~4Mqps) in read only mode but do not use operations like range scan, merge operator, etc. But, the existing RocksDB file formats were built to support range scan and other operations and the current best point lookup in RocksDB is 1.2 Mqps given by [PlainTable](https://github.com/facebook/rocksdb/wiki/PlainTable-Format)[ format](https://github.com/facebook/rocksdb/wiki/PlainTable-Format). This prompted a hashing based file format, which we present here. The new table format uses a cache friendly version of Cuckoo Hashing algorithm with only 1 or 2 memory accesses per lookup.
+
+<!--truncate-->
+
+Goals:
+
+ * Reduce memory accesses per lookup to 1 or 2
+
+
+ * Get an end to end point lookup rate of at least 4 Mqps
+
+
+ * Minimize database size
+
+
+Assumptions:
+
+ * Key length and value length are fixed
+
+
+ * The database is operated in read only mode
+
+
+Non-goals:
+
+
+ * While optimizing the performance of Get() operation was our primary goal, compaction and build times were secondary. We may work on improving them in future.
+
+
+Details for setting up the table format can be found in [GitHub](https://github.com/facebook/rocksdb/wiki/CuckooTable-Format).
+
+
+## Cuckoo Hashing Algorithm
+
+In order to achieve high lookup speeds, we did multiple optimizations, including a cache friendly cuckoo hash algorithm. Cuckoo Hashing uses multiple hash functions, _h1, ..., __hn._
+
+### Original Cuckoo Hashing
+
+To insert any new key _k_, we compute hashes of the key _h1(k), ..., __hn__(k)_. We insert the key in the first hash location that is free. If all the locations are blocked, we try to move one of the colliding keys to a different location by trying to re-insert it.
+
+Finding smallest set of keys to displace in order to accommodate the new key is naturally a shortest path problem in a directed graph where nodes are buckets of hash table and there is an edge from bucket _A_ to bucket _B_ if the element stored in bucket _A_ can be accommodated in bucket _B_ using one of the hash functions. The source nodes are the possible hash locations for the given key _k_ and destination is any one of the empty buckets. We use this algorithm to handle collision.
+
+To retrieve a key _k_, we compute hashes, _h1(k), ..., __hn__(k)_ and the key must be present in one of these locations.
+
+Our goal is to minimize average (and maximum) number of hash functions required and hence the number of memory accesses. In our experiments, with a hash utilization of 90%, we found that the average number of lookups is 1.8 and maximum is 3. Around 44% of keys are accommodated in first hash location and 33% in second location.
+
+
+### Cache Friendly Cuckoo Hashing
+
+We noticed the following two sub-optimal properties in original Cuckoo implementation:
+
+
+ * If the key is not present in first hash location, we jump to second hash location which may not be in cache. This results in many cache misses.
+
+
+ * Because only 44% of keys are located in first cuckoo block, we couldn't have an optimal prefetching strategy - prefetching all hash locations for a key is wasteful. But prefetching only the first hash location helps only 44% of cases.
+
+
+
+The solution is to insert more keys near first location. In case of collision in the first hash location - _h1(k)_, we try to insert it in next few buckets, _h1(k)+1, _h1(k)+2, _..., h1(k)+t-1_. If all of these _t_ locations are occupied, we skip over to next hash function _h2_ and repeat the process. We call the set of _t_ buckets as a _Cuckoo Block_. We chose _t_ such that size of a block is not bigger than a cache line and we prefetch the first cuckoo block.
+
+
+With the new algorithm, for 90% hash utilization, we found that 85% of keys are accommodated in first Cuckoo Block. Prefetching the first cuckoo block yields best results. For a database of 100 million keys with key length 8 and value length 4, the hash algorithm alone can achieve 9.6 Mqps and we are working on improving it further. End to end RocksDB performance results can be found [here](https://github.com/facebook/rocksdb/wiki/CuckooTable-Format).
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-09-12-new-bloom-filter-format.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-09-12-new-bloom-filter-format.markdown b/thirdparty/rocksdb/docs/_posts/2014-09-12-new-bloom-filter-format.markdown
new file mode 100644
index 0000000..96fa50a
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-09-12-new-bloom-filter-format.markdown
@@ -0,0 +1,52 @@
+---
+title: New Bloom Filter Format
+layout: post
+author: zagfox
+category: blog
+redirect_from:
+ - /blog/1367/cuckoo/
+---
+
+## Introduction
+
+In this post, we are introducing "full filter block" --- a new bloom filter format for [block based table](https://github.com/facebook/rocksdb/wiki/Rocksdb-BlockBasedTable-Format). This could bring about 40% of improvement for key query under in-memory (all data stored in memory, files stored in tmpfs/ramfs, an [example](https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks) workload. The main idea behind is to generate a big filter that covers all the keys in SST file to avoid lots of unnecessary memory look ups.
+
+
+<!--truncate-->
+
+## What is Bloom Filter
+
+In brief, [bloom filter](https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter) is a bits array generated for a set of keys that could tell if an arbitrary key may exist in that set.
+
+In RocksDB, we generate such a bloom filter for each SST file. When we conduct a query for a key, we first goes to the bloom filter block of SST file. If key may exist in filter, we goes into data block in SST file to search for the key. If not, we would return directly. So it could help speed up point look up operation a lot.
+
+## Original Bloom Filter Format
+
+Original bloom filter creates filters for each individual data block in SST file. It has complex structure (ref [here](https://github.com/facebook/rocksdb/wiki/Rocksdb-BlockBasedTable-Format#filter-meta-block)) which results in a lot of non-adjacent memory look ups.
+
+Here's the work flow for checking original bloom filter in block based table:
+
+1. Given the target key, we goes to the index block to get the "data block ID" where this key may reside.
+1. Using the "data block ID", we goes to the filter block and get the correct "offset of filter".
+1. Using the "offset of filter", we goes to the actual filter and do the checking.
+
+## New Bloom Filter Format
+
+New bloom filter creates filter for all keys in SST file and we name it "full filter". The data structure of full filter is very simple, there is just one big filter:
+
+ [ full filter ]
+
+In this way, the work flow of bloom filter checking is much simplified.
+
+(1) Given the target key, we goes directly to the filter block and conduct the filter checking.
+
+To be specific, there would be no checking for index block and no address jumping inside of filter block.
+
+Though it is a big filter, the total filter size would be the same as the original filter.
+
+One little draw back is that the new bloom filter introduces more memory consumption when building SST file because we need to buffer keys (or their hashes) before generating filter. Original filter just creates a bunch of small filters so it just buffer a small amount of keys. For full filter, we buffer hashes of all keys, which would take more memory when SST file size increases.
+
+
+## Usage & Customization
+
+You can refer to the document here for [usage](https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#usage-of-new-bloom-filter) and [customization](https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#customize-your-own-filterpolicy).
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-09-15-rocksdb-3-5-release.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-09-15-rocksdb-3-5-release.markdown b/thirdparty/rocksdb/docs/_posts/2014-09-15-rocksdb-3-5-release.markdown
new file mode 100644
index 0000000..1878a5a
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-09-15-rocksdb-3-5-release.markdown
@@ -0,0 +1,38 @@
+---
+title: RocksDB 3.5 Release!
+layout: post
+author: leijin
+category: blog
+redirect_from:
+ - /blog/1547/rocksdb-3-5-release/
+---
+
+New RocksDB release - 3.5!
+
+
+**New Features**
+
+
+ 1. Add include/utilities/write_batch_with_index.h, providing a utility class to query data out of WriteBatch when building it.
+
+
+ 2. new ReadOptions.total_order_seek to force total order seek when block-based table is built with hash index.
+
+<!--truncate-->
+
+**Public API changes**
+
+
+ 1. The Prefix Extractor used with V2 compaction filters is now passed user key to SliceTransform::Transform instead of unparsed RocksDB key.
+
+
+ 2. Move BlockBasedTable related options to BlockBasedTableOptions from Options. Change corresponding JNI interface. Options affected include: no_block_cache, block_cache, block_cache_compressed, block_size, block_size_deviation, block_restart_interval, filter_policy, whole_key_filtering. filter_policy is changed to shared_ptr from a raw pointer.
+
+
+ 3. Remove deprecated options: disable_seek_compaction and db_stats_log_interval
+
+
+ 4. OptimizeForPointLookup() takes one parameter for block cache size. It now builds hash index, bloom filter, and block cache.
+
+
+[https://github.com/facebook/rocksdb/releases/tag/v3.5](https://github.com/facebook/rocksdb/releases/tag/rocksdb-3.5)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-01-16-migrating-from-leveldb-to-rocksdb-2.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-01-16-migrating-from-leveldb-to-rocksdb-2.markdown b/thirdparty/rocksdb/docs/_posts/2015-01-16-migrating-from-leveldb-to-rocksdb-2.markdown
new file mode 100644
index 0000000..f18de0b
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-01-16-migrating-from-leveldb-to-rocksdb-2.markdown
@@ -0,0 +1,112 @@
+---
+title: Migrating from LevelDB to RocksDB
+layout: post
+author: lgalanis
+category: blog
+redirect_from:
+ - /blog/1811/migrating-from-leveldb-to-rocksdb-2/
+---
+
+If you have an existing application that uses LevelDB and would like to migrate to using RocksDB, one problem you need to overcome is to map the options for LevelDB to proper options for RocksDB. As of release 3.9 this can be automatically done by using our option conversion utility found in rocksdb/utilities/leveldb_options.h. What is needed, is to first replace `leveldb::Options` with `rocksdb::LevelDBOptions`. Then, use `rocksdb::ConvertOptions( )` to convert the `LevelDBOptions` struct into appropriate RocksDB options. Here is an example:
+
+<!--truncate-->
+
+LevelDB code:
+
+```c++
+#include <string>
+#include "leveldb/db.h"
+
+using namespace leveldb;
+
+int main(int argc, char** argv) {
+ DB *db;
+
+ Options opt;
+ opt.create_if_missing = true;
+ opt.max_open_files = 1000;
+ opt.block_size = 4096;
+
+ Status s = DB::Open(opt, "/tmp/mydb", &db);
+
+ delete db;
+}
+```
+
+RocksDB code:
+
+```c++
+#include <string>
+#include "rocksdb/db.h"
+#include "rocksdb/utilities/leveldb_options.h"
+
+using namespace rocksdb;
+
+int main(int argc, char** argv) {
+ DB *db;
+
+ LevelDBOptions opt;
+ opt.create_if_missing = true;
+ opt.max_open_files = 1000;
+ opt.block_size = 4096;
+
+ Options rocksdb_options = ConvertOptions(opt);
+ // add rocksdb specific options here
+
+ Status s = DB::Open(rocksdb_options, "/tmp/mydb_rocks", &db);
+
+ delete db;
+}
+```
+
+The difference is:
+
+```diff
+-#include "leveldb/db.h"
++#include "rocksdb/db.h"
++#include "rocksdb/utilities/leveldb_options.h"
+
+-using namespace leveldb;
++using namespace rocksdb;
+
+- Options opt;
++ LevelDBOptions opt;
+
+- Status s = DB::Open(opt, "/tmp/mydb", &db);
++ Options rocksdb_options = ConvertOptions(opt);
++ // add rockdb specific options here
++
++ Status s = DB::Open(rocksdb_options, "/tmp/mydb_rocks", &db);
+```
+
+Once you get up and running with RocksDB you can then focus on tuning RocksDB further by modifying the converted options struct.
+
+The reason why ConvertOptions is handy is because a lot of individual options in RocksDB have moved to other structures in different components. For example, block_size is not available in struct rocksdb::Options. It resides in struct rocksdb::BlockBasedTableOptions, which is used to create a TableFactory object that RocksDB uses internally to create the proper TableBuilder objects. If you were to write your application from scratch it would look like this:
+
+RocksDB code from scratch:
+
+```c++
+#include <string>
+#include "rocksdb/db.h"
+#include "rocksdb/table.h"
+
+using namespace rocksdb;
+
+int main(int argc, char** argv) {
+ DB *db;
+
+ Options opt;
+ opt.create_if_missing = true;
+ opt.max_open_files = 1000;
+
+ BlockBasedTableOptions topt;
+ topt.block_size = 4096;
+ opt.table_factory.reset(NewBlockBasedTableFactory(topt));
+
+ Status s = DB::Open(opt, "/tmp/mydb_rocks", &db);
+
+ delete db;
+}
+```
+
+The LevelDBOptions utility can ease migration to RocksDB from LevelDB and allows us to break down the various options across classes as it is needed.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-02-24-reading-rocksdb-options-from-a-file.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-02-24-reading-rocksdb-options-from-a-file.markdown b/thirdparty/rocksdb/docs/_posts/2015-02-24-reading-rocksdb-options-from-a-file.markdown
new file mode 100644
index 0000000..cddc0dd
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-02-24-reading-rocksdb-options-from-a-file.markdown
@@ -0,0 +1,41 @@
+---
+title: Reading RocksDB options from a file
+layout: post
+author: lgalanis
+category: blog
+redirect_from:
+ - /blog/1883/reading-rocksdb-options-from-a-file/
+---
+
+RocksDB options can be provided using a file or any string to RocksDB. The format is straightforward: `write_buffer_size=1024;max_write_buffer_number=2`. Any whitespace around `=` and `;` is OK. Moreover, options can be nested as necessary. For example `BlockBasedTableOptions` can be nested as follows: `write_buffer_size=1024; max_write_buffer_number=2; block_based_table_factory={block_size=4k};`. Similarly any white space around `{` or `}` is ok. Here is what it looks like in code:
+
+<!--truncate-->
+
+```c++
+#include <string>
+#include "rocksdb/db.h"
+#include "rocksdb/table.h"
+#include "rocksdb/utilities/convenience.h"
+
+using namespace rocksdb;
+
+int main(int argc, char** argv) {
+ DB *db;
+
+ Options opt;
+
+ std::string options_string =
+ "create_if_missing=true;max_open_files=1000;"
+ "block_based_table_factory={block_size=4096}";
+
+ Status s = GetDBOptionsFromString(opt, options_string, &opt);
+
+ s = DB::Open(opt, "/tmp/mydb_rocks", &db);
+
+ // use db
+
+ delete db;
+}
+```
+
+Using `GetDBOptionsFromString` is a convenient way of changing options for your RocksDB application without needing to resort to recompilation or tedious command line parsing.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-02-27-write-batch-with-index.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-02-27-write-batch-with-index.markdown b/thirdparty/rocksdb/docs/_posts/2015-02-27-write-batch-with-index.markdown
new file mode 100644
index 0000000..7f9f776
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-02-27-write-batch-with-index.markdown
@@ -0,0 +1,20 @@
+---
+title: 'WriteBatchWithIndex: Utility for Implementing Read-Your-Own-Writes'
+layout: post
+author: sdong
+category: blog
+redirect_from:
+ - /blog/1901/write-batch-with-index/
+---
+
+RocksDB can be used as a storage engine of a higher level database. In fact, we are currently plugging RocksDB into MySQL and MongoDB as one of their storage engines. RocksDB can help with guaranteeing some of the ACID properties: durability is guaranteed by RocksDB by design; while consistency and isolation need to be enforced by concurrency controls on top of RocksDB; Atomicity can be implemented by committing a transaction's writes with one write batch to RocksDB in the end.
+
+<!--truncate-->
+
+However, if we enforce atomicity by only committing all writes in the end of the transaction in one batch, you cannot get the updated value from RocksDB previously written by the same transaction (read-your-own-write). To read the updated value, the databases on top of RocksDB need to maintain an internal buffer for all the written keys, and when a read happens they need to merge the result from RocksDB and from this buffer. This is a problem we faced when building the RocksDB storage engine in MongoDB. We solved it by creating a utility class, WriteBatchWithIndex (a write batch with a searchable index) and made it part of public API so that the community can also benefit from it.
+
+Before talking about the index part, let me introduce write batch first. The write batch class, `WriteBatch`, is a RocksDB data structure for atomic writes of multiple keys. Users can buffer their updates to a `WriteBatch` by calling `write_batch.Put("key1", "value1")` or `write_batch.Delete("key2")`, similar as calling RocksDB's functions of the same names. In the end, they call `db->Write(write_batch)` to atomically update all those batched operations to the DB. It is how a database can guarantee atomicity, as shown above. Adding a searchable index to `WriteBatch`, we now have `WriteBatchWithIndex`. Users can put updates to WriteBatchIndex in the same way as to `WriteBatch`. In the end, users can get a `WriteBatch` object from it and issue `db->Write()`. Additionally, users can create an iterator of a WriteBatchWithIndex, seek to any key location and iterate from there.
+
+To implement read-your-own-write using `WriteBatchWithIndex`, every time the user creates a transaction, we create a `WriteBatchWithIndex` attached to it. All the writes of the transaction go to the `WriteBatchWithIndex` first. When we commit the transaction, we atomically write the batch to RocksDB. When the user wants to call `Get()`, we first check if the value exists in the `WriteBatchWithIndex` and return the value if existing, by seeking and reading from an iterator of the write batch, before checking data in RocksDB. For example, here is the we implement it in MongoDB's RocksDB storage engine: [link](https://github.com/mongodb/mongo/blob/a31cc114a89a3645e97645805ba77db32c433dce/src/mongo/db/storage/rocks/rocks_recovery_unit.cpp#L245-L260). If a range query comes, we pass a DB's iterator to `WriteBatchWithIndex`, which creates a super iterator which combines the results from the DB iterator with the batch's iterator. Using this super iterator, we can iterate the DB with the t
ransaction's own writes. Here is the iterator creation codes in MongoDB's RocksDB storage engine: [link](https://github.com/mongodb/mongo/blob/a31cc114a89a3645e97645805ba77db32c433dce/src/mongo/db/storage/rocks/rocks_recovery_unit.cpp#L266-L269). In this way, the database can solve the read-your-own-write problem by using RocksDB to handle a transaction's uncommitted writes.
+
+Using `WriteBatchWithIndex`, we successfully implemented read-your-own-writes in the RocksDB storage engine of MongoDB. If you also have a read-your-own-write problem, `WriteBatchWithIndex` can help you implement it quickly and correctly.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-04-22-integrating-rocksdb-with-mongodb-2.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-04-22-integrating-rocksdb-with-mongodb-2.markdown b/thirdparty/rocksdb/docs/_posts/2015-04-22-integrating-rocksdb-with-mongodb-2.markdown
new file mode 100644
index 0000000..1ffe2c5
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-04-22-integrating-rocksdb-with-mongodb-2.markdown
@@ -0,0 +1,16 @@
+---
+title: Integrating RocksDB with MongoDB
+layout: post
+author: icanadi
+category: blog
+redirect_from:
+ - /blog/1967/integrating-rocksdb-with-mongodb-2/
+---
+
+Over the last couple of years, we have been busy integrating RocksDB with various services here at Facebook that needed to store key-value pairs locally. We have also seen other companies using RocksDB as local storage components of their distributed systems.
+
+<!--truncate-->
+
+The next big challenge for us is to bring RocksDB storage engine to general purpose databases. Today we have an exciting milestone to share with our community! We're running MongoDB with RocksDB in production and seeing great results! You can read more about it here: [http://blog.parse.com/announcements/mongodb-rocksdb-parse/](http://blog.parse.com/announcements/mongodb-rocksdb-parse/)
+
+Keep tuned for benchmarks and more stability and performance improvements.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-06-12-rocksdb-in-osquery.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-06-12-rocksdb-in-osquery.markdown b/thirdparty/rocksdb/docs/_posts/2015-06-12-rocksdb-in-osquery.markdown
new file mode 100644
index 0000000..f3a55fa
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-06-12-rocksdb-in-osquery.markdown
@@ -0,0 +1,10 @@
+---
+title: RocksDB in osquery
+layout: post
+author: icanadi
+category: lgalanis
+redirect_from:
+ - /blog/1997/rocksdb-in-osquery/
+---
+
+Check out [this](https://code.facebook.com/posts/1411870269134471/how-rocksdb-is-used-in-osquery/) blog post by [Mike Arpaia](https://www.facebook.com/mike.arpaia) and [Ted Reed](https://www.facebook.com/treeded) about how osquery leverages RocksDB to build an embedded pub-sub system. This article is a great read and contains insights on how to properly use RocksDB.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-07-15-rocksdb-2015-h2-roadmap.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-07-15-rocksdb-2015-h2-roadmap.markdown b/thirdparty/rocksdb/docs/_posts/2015-07-15-rocksdb-2015-h2-roadmap.markdown
new file mode 100644
index 0000000..b3e2703
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-07-15-rocksdb-2015-h2-roadmap.markdown
@@ -0,0 +1,92 @@
+---
+title: RocksDB 2015 H2 roadmap
+layout: post
+author: icanadi
+category: blog
+redirect_from:
+ - /blog/2015/rocksdb-2015-h2-roadmap/
+---
+
+Every 6 months, RocksDB team gets together to prioritize the work ahead of us. We just went through this exercise and we wanted to share the results with the community. Here's what RocksDB team will be focusing on for the next 6 months:
+
+<!--truncate-->
+
+**MyRocks**
+
+As you might know, we're working hard to integrate RocksDB as a storage engine for MySQL. This project is pretty important for us because we're heavy users of MySQL. We're already getting pretty good performance results, but there is more work to be done. We need to focus on both performance and stability. The most high priority items on are list are:
+
+
+
+
+ 1. Reduce CPU costs of RocksDB as a MySQL storage engine
+
+
+ 2. Implement pessimistic concurrency control to support repeatable read isolation level in MyRocks
+
+
+ 3. Reduce P99 read latency, which is high mostly because of lingering tombstones
+
+
+ 4. Port ZSTD compression
+
+
+**MongoRocks**
+
+Another database that we're working on is MongoDB. The project of integrating MongoDB with RocksDB storage engine is called MongoRocks. It's already running in production at Parse [1] and we're seeing surprisingly few issues. Our plans for the next half:
+
+
+
+
+ 1. Keep improving performance and stability, possibly reuse work done on MyRocks (workloads are pretty similar).
+
+
+ 2. Increase internal and external adoption.
+
+
+ 3. Support new MongoDB 3.2.
+
+
+**RocksDB on cheaper storage media**
+
+Up to now, our mission was to build the best key-value store “for fast storage” (flash and in-memory). However, there are some use-cases at Facebook that don't need expensive high-end storage. In the next six months, we plan to deploy RocksDB on cheaper storage media. We will optimize performance to RocksDB on either or both:
+
+
+
+
+ 1. Hard drive storage array.
+
+
+ 2. Tiered Storage.
+
+
+**Quality of Service**
+
+When talking to our customers, there are couple of issues that keep reoccurring. We need to fix them to make our customers happy. We will improve RocksDB to provide better assurance of performance and resource usage. Non-exhaustive list includes:
+
+
+
+
+ 1. Iterate P99 can be high due to the presence of tombstones.
+
+
+ 2. Write stalls can happen during high write loads.
+
+
+ 3. Better control of memory and disk usage.
+
+
+ 4. Service quality and performance of backup engine.
+
+
+**Operation's user experience**
+
+As we increase deployment of RocksDB, engineers are spending more time on debugging RocksDB issues. We plan to improve user experience when running RocksDB. The goal is to reduce TTD (time-to-debug). The work includes monitoring, visualizations and documentations.
+
+[1]( http://blog.parse.com/announcements/mongodb-rocksdb-parse/](http://blog.parse.com/announcements/mongodb-rocksdb-parse/)
+
+
+### Comments
+
+**[Mike](allspace2012@outlook.com)**
+
+What’s the status of this roadmap? “RocksDB on cheaper storage media”, has this been implemented?
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-07-17-spatial-indexing-in-rocksdb.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-07-17-spatial-indexing-in-rocksdb.markdown b/thirdparty/rocksdb/docs/_posts/2015-07-17-spatial-indexing-in-rocksdb.markdown
new file mode 100644
index 0000000..fe7b7b2
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-07-17-spatial-indexing-in-rocksdb.markdown
@@ -0,0 +1,78 @@
+---
+title: Spatial indexing in RocksDB
+layout: post
+author: icanadi
+category: blog
+redirect_from:
+ - /blog/2039/spatial-indexing-in-rocksdb/
+---
+
+About a year ago, there was a need to develop a spatial database at Facebook. We needed to store and index Earth's map data. Before building our own, we looked at the existing spatial databases. They were all very good technology, but also general purpose. We could sacrifice a general-purpose API, so we thought we could build a more performant database, since it would be specifically designed for our use-case. Furthermore, we decided to build the spatial database on top of RocksDB, because we have a lot of operational experience with running and tuning RocksDB at a large scale.
+
+<!--truncate-->
+
+When we started looking at this project, the first thing that surprised us was that our planet is not that big. Earth's entire map data can fit in memory on a reasonably high-end machine. Thus, we also decided to build a spatial database optimized for memory-resident dataset.
+
+The first use-case of our spatial database was an experimental map renderer. As part of our project, we successfully loaded [Open Street Maps](https://www.openstreetmap.org/) dataset and hooked it up with [Mapnik](http://mapnik.org/), a map rendering engine.
+
+The usual Mapnik workflow is to load the map data into a SQL-based database and then define map layers with SQL statements. To render a tile, Mapnik needs to execute a couple of SQL queries. The benefit of this approach is that you don't need to reload your database when you change your map style. You can just change your SQL query and Mapnik picks it up. In our model, we decided to precompute the features we need for each tile. We need to know the map style before we create the database. However, when rendering the map tile, we only fetch the features that we need to render.
+
+We haven't open sourced the RocksDB Mapnik plugin or the database loading pipeline. However, the spatial indexing is available in RocksDB under a name [SpatialDB](https://github.com/facebook/rocksdb/blob/master/include/rocksdb/utilities/spatial_db.h). The API is focused on map rendering use-case, but we hope that it can also be used for other spatial-based applications.
+
+Let's take a tour of the API. When you create a spatial database, you specify the spatial indexes that need to be built. Each spatial index is defined by a bounding box and granularity. For map rendering, we create a spatial index for each zoom levels. Higher zoom levels have more granularity.
+
+
+
+ SpatialDB::Create(
+ SpatialDBOptions(),
+ "/data/map", {
+ SpatialIndexOptions("zoom10", BoundingBox(0, 0, 100, 100), 10),
+ SpatialIndexOptions("zoom16", BoundingBox(0, 0, 100, 100), 16)
+ }
+ );
+
+
+
+
+When you insert a feature (building, street, country border) into SpatialDB, you need to specify the list of spatial indexes that will index the feature. In the loading phase we process the map style to determine the list of zoom levels on which we'll render the feature. For example, we will not render the building on zoom level that shows an entire country. Building will only be indexed on higher zoom level's index. Country borders will be indexes on all zoom levels.
+
+
+
+ FeatureSet feature;
+ feature.Set("type", "building");
+ feature.Set("height", 6);
+ db->Insert(WriteOptions(), BoundingBox<double>(5, 5, 10, 10),
+ well_known_binary_blob, feature, {"zoom16"});
+
+
+
+
+The indexing part is pretty simple. For each feature, we first find a list of index tiles that it intersects. Then, we add a link from the tile's [quad key](https://msdn.microsoft.com/en-us/library/bb259689.aspx) to the feature's primary key. Using quad keys improves data locality, i.e. features closer together geographically will have similar quad keys. Even though we're optimizing for a memory-resident dataset, data locality is still very important due to different caching effects.
+
+After you're done inserting all the features, you can call an API Compact() that will compact the dataset and speed up read queries.
+
+
+
+ db->Compact();
+
+
+
+
+SpatialDB's query specifies: 1) bounding box we're interested in, and 2) a zoom level. We find all tiles that intersect with the query's bounding box and return all features in those tiles.
+
+
+
+
+ Cursor* c = db_->Query(ReadOptions(), BoundingBox<double>(1, 1, 7, 7), "zoom16");
+ for (c->Valid(); c->Next()) {
+ Render(c->blob(), c->feature_set());
+ }
+
+
+
+
+Note: `Render()` function is not part of RocksDB. You will need to use one of many open source map renderers, for example check out [Mapnik](http://mapnik.org/).
+
+TL;DR If you need an embedded spatial database, check out RocksDB's SpatialDB. [Let us know](https://www.facebook.com/groups/rocksdb.dev/) how we can make it better.
+
+If you're interested in learning more, check out this [talk](https://www.youtube.com/watch?v=T1jWsDMONM8).
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-07-22-rocksdb-is-now-available-in-windows-platform.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-07-22-rocksdb-is-now-available-in-windows-platform.markdown b/thirdparty/rocksdb/docs/_posts/2015-07-22-rocksdb-is-now-available-in-windows-platform.markdown
new file mode 100644
index 0000000..b6bb47d
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-07-22-rocksdb-is-now-available-in-windows-platform.markdown
@@ -0,0 +1,30 @@
+---
+title: RocksDB is now available in Windows Platform
+layout: post
+author: dmitrism
+category: blog
+redirect_from:
+ - /blog/2033/rocksdb-is-now-available-in-windows-platform/
+---
+
+Over the past 6 months we have seen a number of use cases where RocksDB is successfully used by the community and various companies to achieve high throughput and volume in a modern server environment.
+
+We at Microsoft Bing could not be left behind. As a result we are happy to [announce](http://bit.ly/1OmWBT9) the availability of the Windows Port created here at Microsoft which we intend to use as a storage option for one of our key/value data stores.
+
+<!--truncate-->
+
+We are happy to make this available for the community. Keep tuned for more announcements to come.
+
+### Comments
+
+**[Siying Dong](siying.d@fb.com)**
+
+Appreciate your contributions to RocksDB project! I believe it will benefits many users!
+
+**[empresas sevilla](oxofkx@gmail.com)**
+
+Magnifico artículo|, un placer leer el blog
+
+**[jak usunac](tomogedac@o2.pl)**
+
+I believe it will benefits too
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-07-23-dynamic-level.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-07-23-dynamic-level.markdown b/thirdparty/rocksdb/docs/_posts/2015-07-23-dynamic-level.markdown
new file mode 100644
index 0000000..0ff3a05
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-07-23-dynamic-level.markdown
@@ -0,0 +1,29 @@
+---
+title: Dynamic Level Size for Level-Based Compaction
+layout: post
+author: sdong
+category: blog
+redirect_from:
+ - /blog/2207/dynamic-level/
+---
+
+In this article, we follow up on the first part of an answer to one of the questions in our [AMA](https://www.reddit.com/r/IAmA/comments/3de3cv/we_are_rocksdb_engineering_team_ask_us_anything/ct4a8tb), the dynamic level size in level-based compaction.
+
+<!--truncate-->
+
+Level-based compaction is the original LevelDB compaction style and one of the two major compaction styles in RocksDB (See [our wiki](https://github.com/facebook/rocksdb/wiki/RocksDB-Basics#multi-threaded-compactions)). In RocksDB we introduced parallelism and more configurable options to it but the main algorithm stayed the same, until we recently introduced the dynamic level size mode.
+
+
+In level-based compaction, we organize data to different sorted runs, called levels. Each level has a target size. Usually target size of levels increases by the same size multiplier. For example, you can set target size of level 1 to be 1GB, and size multiplier to be 10, and the target size of level 1, 2, 3, 4 will be 1GB, 10GB, 100GB and 1000GB. Before level 1, there will be some staging file flushed from mem tables, called Level 0 files, which will later be merged to level 1. Compactions will be triggered as soon as actual size of a level exceeds its target size. We will merge a subset of data of that level to next level, to reduce size of the level. More compactions will be triggered until sizes of all the levels are lower than their target sizes. In a steady state, the size of each level will be around the same size of the size of level targets.
+
+
+Level-based compaction’s advantage is its good space efficiency. We usually use the metric space amplification to measure the space efficiency. In this article ignore the effects of data compression so space amplification= size_on_file_system / size_of_user_data.
+
+
+How do we estimate space amplification of level-based compaction? We focus specifically on the databases in steady state, which means database size is stable or grows slowly over time. This means updates will add roughly the same or little more data than what is removed by deletes. Given that, if we compact all the data all to the last level, the size of level will be equal as the size of last level before the compaction. On the other hand, the size of user data will be approximately the size of DB if we compact all the levels down to the last level. So the size of the last level will be a good estimation of user data size. So total size of the DB divided by the size of the last level will be a good estimation of space amplification.
+
+
+Applying the equation, if we have four non-zero levels, their sizes are 1GB, 10GB, 100GB, 1000GB, the size amplification will be approximately (1000GB + 100GB + 10GB + 1GB) / 1000GB = 1.111, which is a very good number. However, there is a catch here: how to make sure the last level’s size is 1000GB, the same as the level’s size target? A user has to fine tune level sizes to achieve this number and will need to re-tune if DB size changes. The theoretic number 1.11 is hard to achieve in practice. In a worse case, if you have the target size of last level to be 1000GB but the user data is only 200GB, then the actual space amplification will be (200GB + 100GB + 10GB + 1GB) / 200GB = 1.555, a much worse number.
+
+
+To solve this problem, my colleague Igor Kabiljo came up with a solution of dynamic level size target mode. You can enable it by setting options.level_compaction_dynamic_level_bytes=true. In this mode, size target of levels are changed dynamically based on size of the last level. Suppose the level size multiplier to be 10, and the DB size is 200GB. The target size of the last level is automatically set to be the actual size of the level, which is 200GB, the second to last level’s size target will be automatically set to be size_last_level / 10 = 20GB, the third last level’s will be size_last_level/100 = 2GB, and next level to be size_last_level/1000 = 200MB. We stop here because 200MB is within the range of the first level. In this way, we can achieve the 1.111 space amplification, without fine tuning of the level size targets. More details can be found in [code comments of the option](https://github.com/facebook/rocksdb/blob/v3.11/include/rocksdb/options.h#L366-L423) in the h
eader file.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-10-27-getthreadlist.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-10-27-getthreadlist.markdown b/thirdparty/rocksdb/docs/_posts/2015-10-27-getthreadlist.markdown
new file mode 100644
index 0000000..332a29f
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-10-27-getthreadlist.markdown
@@ -0,0 +1,193 @@
+---
+title: GetThreadList
+layout: post
+author: yhciang
+category: blog
+redirect_from:
+ - /blog/2261/getthreadlist/
+---
+
+We recently added a new API, called `GetThreadList()`, that exposes the RocksDB background thread activity. With this feature, developers will be able to obtain the real-time information about the currently running compactions and flushes such as the input / output size, elapsed time, the number of bytes it has written. Below is an example output of `GetThreadList`. To better illustrate the example, we have put a sample output of `GetThreadList` into a table where each column represents a thread status:
+
+<!--truncate-->
+
+<table width="637" >
+<tbody >
+<tr style="border:2px solid #000000" >
+
+<td style="padding:3px" >ThreadID
+</td>
+
+<td >140716395198208
+</td>
+
+<td >140716416169728
+</td>
+</tr>
+<tr >
+
+<td style="padding:3px" >DB
+</td>
+
+<td >db1
+</td>
+
+<td >db2
+</td>
+</tr>
+<tr >
+
+<td style="padding:3px" >CF
+</td>
+
+<td >default
+</td>
+
+<td >picachu
+</td>
+</tr>
+<tr >
+
+<td style="padding:3px" >ThreadType
+</td>
+
+<td >High Pri
+</td>
+
+<td >Low Pri
+</td>
+</tr>
+<tr >
+
+<td style="padding:3px" >Operation
+</td>
+
+<td >Flush
+</td>
+
+<td >Compaction
+</td>
+</tr>
+<tr >
+
+<td style="padding:3px" >ElapsedTime
+</td>
+
+<td >143.459 ms
+</td>
+
+<td >607.538 ms
+</td>
+</tr>
+<tr >
+
+<td style="padding:3px" >Stage
+</td>
+
+<td >FlushJob::WriteLevel0Table
+</td>
+
+<td >CompactionJob::Install
+</td>
+</tr>
+<tr >
+
+<td style="vertical-align:top;padding:3px" >OperationProperties
+</td>
+
+<td style="vertical-align:top;padding:3px" >
+BytesMemtables 4092938
+BytesWritten 1050701
+</td>
+
+<td style="vertical-align:top" >
+BaseInputLevel 1
+BytesRead 4876417
+BytesWritten 4140109
+IsDeletion 0
+IsManual 0
+IsTrivialMove 0
+JobID 146
+OutputLevel 2
+TotalInputBytes 4883044
+</td>
+</tr>
+</tbody>
+</table>
+
+In the above output, we can see `GetThreadList()` reports the activity of two threads: one thread running flush job (middle column) and the other thread running a compaction job (right-most column). In each thread status, it shows basic information about the thread such as thread id, it's target db / column family, and the job it is currently doing and the current status of the job. For instance, we can see thread 140716416169728 is doing compaction on the `picachu` column family in database `db2`. In addition, we can see the compaction has been running for 600 ms, and it has read 4876417 bytes out of 4883044 bytes. This indicates the compaction is about to complete. The stage property indicates which code block the thread is currently executing. For instance, thread 140716416169728 is currently running `CompactionJob::Install`, which further indicates the compaction job is almost done.
+
+Below we briefly describe its API.
+
+
+## How to Enable it?
+
+
+To enable thread-tracking of a rocksdb instance, simply set `enable_thread_tracking` to true in its DBOptions:
+
+```c++
+// If true, then the status of the threads involved in this DB will
+// be tracked and available via GetThreadList() API.
+//
+// Default: false
+bool enable_thread_tracking;
+```
+
+
+
+## The API
+
+
+The GetThreadList API is defined in [include/rocksdb/env.h](https://github.com/facebook/rocksdb/blob/master/include/rocksdb/env.h#L317-L318), which is an Env
+function:
+
+```c++
+virtual Status GetThreadList(std::vector* thread_list)
+```
+
+Since an Env can be shared across multiple rocksdb instances, the output of
+`GetThreadList()` include the background activity of all the rocksdb instances
+that using the same Env.
+
+The `GetThreadList()` API simply returns a vector of `ThreadStatus`, each describes
+the current status of a thread. The `ThreadStatus` structure, defined in
+[include/rocksdb/thread_status.h](https://github.com/facebook/rocksdb/blob/master/include/rocksdb/thread_status.h), contains the following information:
+
+```c++
+// An unique ID for the thread.
+const uint64_t thread_id;
+
+// The type of the thread, it could be HIGH_PRIORITY,
+// LOW_PRIORITY, and USER
+const ThreadType thread_type;
+
+// The name of the DB instance where the thread is currently
+// involved with. It would be set to empty string if the thread
+// does not involve in any DB operation.
+const std::string db_name;
+
+// The name of the column family where the thread is currently
+// It would be set to empty string if the thread does not involve
+// in any column family.
+const std::string cf_name;
+
+// The operation (high-level action) that the current thread is involved.
+const OperationType operation_type;
+
+// The elapsed time in micros of the current thread operation.
+const uint64_t op_elapsed_micros;
+
+// An integer showing the current stage where the thread is involved
+// in the current operation.
+const OperationStage operation_stage;
+
+// A list of properties that describe some details about the current
+// operation. Same field in op_properties[] might have different
+// meanings for different operations.
+uint64_t op_properties[kNumOperationProperties];
+
+// The state (lower-level action) that the current thread is involved.
+const StateType state_type;
+```
+
+If you are interested in the background thread activity of your RocksDB application, please feel free to give `GetThreadList()` a try :)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-11-10-use-checkpoints-for-efficient-snapshots.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-11-10-use-checkpoints-for-efficient-snapshots.markdown b/thirdparty/rocksdb/docs/_posts/2015-11-10-use-checkpoints-for-efficient-snapshots.markdown
new file mode 100644
index 0000000..6852b8f
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-11-10-use-checkpoints-for-efficient-snapshots.markdown
@@ -0,0 +1,45 @@
+---
+title: Use Checkpoints for Efficient Snapshots
+layout: post
+author: rven2
+category: blog
+redirect_from:
+ - /blog/2609/use-checkpoints-for-efficient-snapshots/
+---
+
+**Checkpoint** is a feature in RocksDB which provides the ability to take a snapshot of a running RocksDB database in a separate directory. Checkpoints can be used as a point in time snapshot, which can be opened Read-only to query rows as of the point in time or as a Writeable snapshot by opening it Read-Write. Checkpoints can be used for both full and incremental backups.
+
+<!--truncate-->
+
+
+The Checkpoint feature enables RocksDB to create a consistent snapshot of a given RocksDB database in the specified directory. If the snapshot is on the same filesystem as the original database, the SST files will be hard-linked, otherwise SST files will be copied. The manifest and CURRENT files will be copied. In addition, if there are multiple column families, log files will be copied for the period covering the start and end of the checkpoint, in order to provide a consistent snapshot across column families.
+
+
+
+
+A Checkpoint object needs to be created for a database before checkpoints are created. The API is as follows:
+
+
+
+
+`Status Create(DB* db, Checkpoint** checkpoint_ptr);`
+
+
+
+
+Given a checkpoint object and a directory, the CreateCheckpoint function creates a consistent snapshot of the database in the given directory.
+
+
+
+
+`Status CreateCheckpoint(const std::string& checkpoint_dir);`
+
+
+
+
+The directory should not already exist and will be created by this API. The directory will be an absolute path. The checkpoint can be used as a read-only copy of the DB or can be opened as a standalone DB. When opened read/write, the SST files continue to be hard links and these links are removed when the files are obsoleted. When the user is done with the snapshot, the user can delete the directory to remove the snapshot.
+
+
+
+
+Checkpoints are used for online backup in MyRocks. which is MySQL using RocksDB as the storage engine . ([MySQL on RocksDB](https://github.com/facebook/mysql-5.6))
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2015-11-16-analysis-file-read-latency-by-level.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2015-11-16-analysis-file-read-latency-by-level.markdown b/thirdparty/rocksdb/docs/_posts/2015-11-16-analysis-file-read-latency-by-level.markdown
new file mode 100644
index 0000000..b21b04f
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2015-11-16-analysis-file-read-latency-by-level.markdown
@@ -0,0 +1,244 @@
+---
+title: Analysis File Read Latency by Level
+layout: post
+author: sdong
+category: blog
+redirect_from:
+ - /blog/2537/analysis-file-read-latency-by-level/
+---
+
+In many use cases of RocksDB, people rely on OS page cache for caching compressed data. With this approach, verifying effective of the OS page caching is challenging, because file system is a black box to users.
+
+As an example, a user can tune the DB as following: use level-based compaction, with L1 - L4 sizes to be 1GB, 10GB, 100GB and 1TB. And they reserve about 20GB memory as OS page cache, expecting level 0, 1 and 2 are mostly cached in memory, leaving only reads from level 3 and 4 requiring disk I/Os. However, in practice, it's not easy to verify whether OS page cache does exactly what we expect. For example, if we end up with doing 4 instead of 2 I/Os per query, it's not easy for users to figure out whether the it's because of efficiency of OS page cache or reading multiple blocks for a level. Analysis like it is especially important if users run RocksDB on hard drive disks, for the gap of latency between hard drives and memory is much higher than flash-based SSDs.
+
+<!--truncate-->
+
+In order to make tuning easier, we added new instrumentation to help users analysis latency distribution of file reads in different levels. If users turn DB statistics on, we always keep track of distribution of file read latency for each level. Users can retrieve the information by querying DB property “rocksdb.stats” ( [https://github.com/facebook/rocksdb/blob/v3.13.1/include/rocksdb/db.h#L315-L316](https://github.com/facebook/rocksdb/blob/v3.13.1/include/rocksdb/db.h#L315-L316) ). It will also printed out as a part of compaction summary in info logs periodically.
+
+The output looks like this:
+
+
+```
+** Level 0 read latency histogram (micros):
+Count: 696 Average: 489.8118 StdDev: 222.40
+Min: 3.0000 Median: 452.3077 Max: 1896.0000
+Percentiles: P50: 452.31 P75: 641.30 P99: 1068.00 P99.9: 1860.80 P99.99: 1896.00
+------------------------------------------------------
+[ 2, 3 ) 1 0.144% 0.144%
+[ 18, 20 ) 1 0.144% 0.287%
+[ 45, 50 ) 5 0.718% 1.006%
+[ 50, 60 ) 26 3.736% 4.741% #
+[ 60, 70 ) 6 0.862% 5.603%
+[ 90, 100 ) 1 0.144% 5.747%
+[ 120, 140 ) 2 0.287% 6.034%
+[ 140, 160 ) 1 0.144% 6.178%
+[ 160, 180 ) 1 0.144% 6.322%
+[ 200, 250 ) 9 1.293% 7.615%
+[ 250, 300 ) 45 6.466% 14.080% #
+[ 300, 350 ) 88 12.644% 26.724% ###
+[ 350, 400 ) 88 12.644% 39.368% ###
+[ 400, 450 ) 71 10.201% 49.569% ##
+[ 450, 500 ) 65 9.339% 58.908% ##
+[ 500, 600 ) 74 10.632% 69.540% ##
+[ 600, 700 ) 92 13.218% 82.759% ###
+[ 700, 800 ) 64 9.195% 91.954% ##
+[ 800, 900 ) 35 5.029% 96.983% #
+[ 900, 1000 ) 12 1.724% 98.707%
+[ 1000, 1200 ) 6 0.862% 99.569%
+[ 1200, 1400 ) 2 0.287% 99.856%
+[ 1800, 2000 ) 1 0.144% 100.000%
+
+** Level 1 read latency histogram (micros):
+(......not pasted.....)
+
+** Level 2 read latency histogram (micros):
+(......not pasted.....)
+
+** Level 3 read latency histogram (micros):
+(......not pasted.....)
+
+** Level 4 read latency histogram (micros):
+(......not pasted.....)
+
+** Level 5 read latency histogram (micros):
+Count: 25583746 Average: 421.1326 StdDev: 385.11
+Min: 1.0000 Median: 376.0011 Max: 202444.0000
+Percentiles: P50: 376.00 P75: 438.00 P99: 1421.68 P99.9: 4164.43 P99.99: 9056.52
+------------------------------------------------------
+[ 0, 1 ) 2351 0.009% 0.009%
+[ 1, 2 ) 6077 0.024% 0.033%
+[ 2, 3 ) 8471 0.033% 0.066%
+[ 3, 4 ) 788 0.003% 0.069%
+[ 4, 5 ) 393 0.002% 0.071%
+[ 5, 6 ) 786 0.003% 0.074%
+[ 6, 7 ) 1709 0.007% 0.080%
+[ 7, 8 ) 1769 0.007% 0.087%
+[ 8, 9 ) 1573 0.006% 0.093%
+[ 9, 10 ) 1495 0.006% 0.099%
+[ 10, 12 ) 3043 0.012% 0.111%
+[ 12, 14 ) 2259 0.009% 0.120%
+[ 14, 16 ) 1233 0.005% 0.125%
+[ 16, 18 ) 762 0.003% 0.128%
+[ 18, 20 ) 451 0.002% 0.130%
+[ 20, 25 ) 794 0.003% 0.133%
+[ 25, 30 ) 1279 0.005% 0.138%
+[ 30, 35 ) 1172 0.005% 0.142%
+[ 35, 40 ) 1363 0.005% 0.148%
+[ 40, 45 ) 409 0.002% 0.149%
+[ 45, 50 ) 105 0.000% 0.150%
+[ 50, 60 ) 80 0.000% 0.150%
+[ 60, 70 ) 280 0.001% 0.151%
+[ 70, 80 ) 1583 0.006% 0.157%
+[ 80, 90 ) 4245 0.017% 0.174%
+[ 90, 100 ) 6572 0.026% 0.200%
+[ 100, 120 ) 9724 0.038% 0.238%
+[ 120, 140 ) 3713 0.015% 0.252%
+[ 140, 160 ) 2383 0.009% 0.261%
+[ 160, 180 ) 18344 0.072% 0.333%
+[ 180, 200 ) 51873 0.203% 0.536%
+[ 200, 250 ) 631722 2.469% 3.005%
+[ 250, 300 ) 2721970 10.639% 13.644% ##
+[ 300, 350 ) 5909249 23.098% 36.742% #####
+[ 350, 400 ) 6522507 25.495% 62.237% #####
+[ 400, 450 ) 4296332 16.793% 79.030% ###
+[ 450, 500 ) 2130323 8.327% 87.357% ##
+[ 500, 600 ) 1553208 6.071% 93.428% #
+[ 600, 700 ) 642129 2.510% 95.938% #
+[ 700, 800 ) 372428 1.456% 97.394%
+[ 800, 900 ) 187561 0.733% 98.127%
+[ 900, 1000 ) 85858 0.336% 98.462%
+[ 1000, 1200 ) 82730 0.323% 98.786%
+[ 1200, 1400 ) 50691 0.198% 98.984%
+[ 1400, 1600 ) 38026 0.149% 99.133%
+[ 1600, 1800 ) 32991 0.129% 99.261%
+[ 1800, 2000 ) 30200 0.118% 99.380%
+[ 2000, 2500 ) 62195 0.243% 99.623%
+[ 2500, 3000 ) 36684 0.143% 99.766%
+[ 3000, 3500 ) 21317 0.083% 99.849%
+[ 3500, 4000 ) 10216 0.040% 99.889%
+[ 4000, 4500 ) 8351 0.033% 99.922%
+[ 4500, 5000 ) 4152 0.016% 99.938%
+[ 5000, 6000 ) 6328 0.025% 99.963%
+[ 6000, 7000 ) 3253 0.013% 99.976%
+[ 7000, 8000 ) 2082 0.008% 99.984%
+[ 8000, 9000 ) 1546 0.006% 99.990%
+[ 9000, 10000 ) 1055 0.004% 99.994%
+[ 10000, 12000 ) 1566 0.006% 100.000%
+[ 12000, 14000 ) 761 0.003% 100.003%
+[ 14000, 16000 ) 462 0.002% 100.005%
+[ 16000, 18000 ) 226 0.001% 100.006%
+[ 18000, 20000 ) 126 0.000% 100.006%
+[ 20000, 25000 ) 107 0.000% 100.007%
+[ 25000, 30000 ) 43 0.000% 100.007%
+[ 30000, 35000 ) 15 0.000% 100.007%
+[ 35000, 40000 ) 14 0.000% 100.007%
+[ 40000, 45000 ) 16 0.000% 100.007%
+[ 45000, 50000 ) 1 0.000% 100.007%
+[ 50000, 60000 ) 22 0.000% 100.007%
+[ 60000, 70000 ) 10 0.000% 100.007%
+[ 70000, 80000 ) 5 0.000% 100.007%
+[ 80000, 90000 ) 14 0.000% 100.007%
+[ 90000, 100000 ) 11 0.000% 100.007%
+[ 100000, 120000 ) 33 0.000% 100.007%
+[ 120000, 140000 ) 6 0.000% 100.007%
+[ 140000, 160000 ) 3 0.000% 100.007%
+[ 160000, 180000 ) 7 0.000% 100.007%
+[ 200000, 250000 ) 2 0.000% 100.007%
+```
+
+
+In this example, you can see we only issued 696 reads from level 0 while issued 25 million reads from level 5. The latency distribution is also clearly shown among those reads. This will be helpful for users to analysis OS page cache efficiency.
+
+Currently the read latency per level includes reads from data blocks, index blocks, as well as bloom filter blocks. We are also working on a feature to break down those three type of blocks.
+
+### Comments
+
+**[Tao Feng](fengtao04@gmail.com)**
+
+Is this feature also included in RocksJava?
+
+**[Siying Dong](siying.d@fb.com)**
+
+Should be. As long as you enable statistics, you should be able to get the value from `RocksDB.getProperty()` with property `rocksdb.dbstats`. Let me know if you can’t find it.
+
+**[chiddu](cnbscience@gmail.com)**
+
+> In this example, you can see we only issued 696 reads from level 0 while issued 256K reads from level 5.
+
+Isn’t it 2.5 M of reads instead of 256K ? .
+
+Also could anyone please provide more description on the histogram ? especially
+
+> Count: 25583746 Average: 421.1326 StdDev: 385.11
+> Min: 1.0000 Median: 376.0011 Max: 202444.0000
+> Percentiles: P50: 376.00 P75: 438.00 P99: 1421.68 P99.9: 4164.43 P99.99: 9056.52
+
+and
+
+> [ 0, 1 ) 2351 0.009% 0.009%
+> [ 1, 2 ) 6077 0.024% 0.033%
+> [ 2, 3 ) 8471 0.033% 0.066%
+> [ 3, 4 ) 788 0.003% 0.069%”
+
+thanks in advance
+
+**[Siying Dong](siying.d@fb.com)**
+
+Thank you for pointing out the mistake. I fixed it now.
+
+In this output, there are 2.5 million samples, average latency is 421 micro seconds, with standard deviation 385. Median is 376, max value is 202 milliseconds. 0.009% has value of 1, 0.024% has value of 1, 0.033% has value of 2. Accumulated value from 0 to 2 is 0.066%.
+
+Hope it helps.
+
+**[chiddu](cnbscience@gmail.com)**
+
+Thank you Siying for the quick reply, I was running couple of benchmark testing to check the performance of rocksdb on SSD. One of the test is similar to what is mentioned in the wiki, TEST 4 : Random read , except the key_size is 10 and value_size is 20. I am inserting 1 billion hashes and reading 1 billion hashes with 32 threads. The histogram shows something like this
+
+```
+Level 5 read latency histogram (micros):
+Count: 7133903059 Average: 480.4357 StdDev: 309.18
+Min: 0.0000 Median: 551.1491 Max: 224142.0000
+Percentiles: P50: 551.15 P75: 651.44 P99: 996.52 P99.9: 2073.07 P99.99: 3196.32
+——————————————————
+[ 0, 1 ) 28587385 0.401% 0.401%
+[ 1, 2 ) 686572516 9.624% 10.025% ##
+[ 2, 3 ) 567317522 7.952% 17.977% ##
+[ 3, 4 ) 44979472 0.631% 18.608%
+[ 4, 5 ) 50379685 0.706% 19.314%
+[ 5, 6 ) 64930061 0.910% 20.224%
+[ 6, 7 ) 22613561 0.317% 20.541%
+…………more………….
+```
+
+If I understand your previous comment correctly,
+
+1. How is it that the count is around 7 billion when I have only inserted 1 billion hashes ? is the stat broken ?
+1. What does the percentiles and the numbers signify ?
+1. 0, 1 ) 28587385 0.401% 0.401% what does this “28587385” stand for in the histogram row ?
+
+**[Siying Dong](siying.d@fb.com)**
+
+If I remember correctly, with db_bench, if you specify –num=1000000000 –threads=32, it is every thread reading one billion keys, total of 32 billions. Is it the case you ran into?
+
+28,587,385 means that number of data points take the value [0,1)
+28,587,385 / 7,133,903,058 = 0.401% provides percentage.
+
+**[chiddu](cnbscience@gmail.com)**
+
+I do have `num=1000000000` and `t=32`. The script says reading 1 billion hashes and not 32 billion hashes.
+
+this is the script on which I have used
+
+```
+echo “Load 1B keys sequentially into database…..”
+bpl=10485760;overlap=10;mcz=2;del=300000000;levels=6;ctrig=4; delay=8; stop=12; wbn=3; mbc=20; mb=67108864;wbs=134217728; dds=1; sync=0; r=1000000000; t=1; vs=20; bs=4096; cs=1048576; of=500000; si=1000000; ./db_bench –benchmarks=fillseq –disable_seek_compaction=1 –mmap_read=0 –statistics=1 –histogram=1 –num=$r –threads=$t –value_size=$vs –block_size=$bs –cache_size=$cs –bloom_bits=10 –cache_numshardbits=6 –open_files=$of –verify_checksum=1 –db=/data/mysql/leveldb/test –sync=$sync –disable_wal=1 –compression_type=none –stats_interval=$si –compression_ratio=0.5 –disable_data_sync=$dds –write_buffer_size=$wbs –target_file_size_base=$mb –max_write_buffer_number=$wbn –max_background_compactions=$mbc –level0_file_num_compaction_trigger=$ctrig –level0_slowdown_writes_trigger=$delay –level0_stop_writes_trigger=$stop –num_levels=$levels –delete_obsolete_files_period_micros=$del –min_level_to_compress=$mcz –max_grandparent_overl
ap_factor=$overlap –stats_per_interval=1 –max_bytes_for_level_base=$bpl –use_existing_db=0 –key_size=10
+
+echo “Reading 1B keys in database in random order….”
+bpl=10485760;overlap=10;mcz=2;del=300000000;levels=6;ctrig=4; delay=8; stop=12; wbn=3; mbc=20; mb=67108864;wbs=134217728; dds=0; sync=0; r=1000000000; t=32; vs=20; bs=4096; cs=1048576; of=500000; si=1000000; ./db_bench –benchmarks=readrandom –disable_seek_compaction=1 –mmap_read=0 –statistics=1 –histogram=1 –num=$r –threads=$t –value_size=$vs –block_size=$bs –cache_size=$cs –bloom_bits=10 –cache_numshardbits=6 –open_files=$of –verify_checksum=1 –db=/some_data_base –sync=$sync –disable_wal=1 –compression_type=none –stats_interval=$si –compression_ratio=0.5 –disable_data_sync=$dds –write_buffer_size=$wbs –target_file_size_base=$mb –max_write_buffer_number=$wbn –max_background_compactions=$mbc –level0_file_num_compaction_trigger=$ctrig –level0_slowdown_writes_trigger=$delay –level0_stop_writes_trigger=$stop –num_levels=$levels –delete_obsolete_files_period_micros=$del –min_level_to_compress=$mcz –max_grandparent_overlap_fa
ctor=$overlap –stats_per_interval=1 –max_bytes_for_level_base=$bpl –use_existing_db=1 –key_size=10
+```
+
+After running this script, there were no issues wrt to loading billion hashes , but when it came to reading part, its been almost 4 days and still I have only read 7 billion hashes and have read 200 million hashes in 2 and half days. Is there something which is missing in db_bench or something which I am missing ?
+
+**[Siying Dong](siying.d@fb.com)**
+
+It’s a printing error then. If you have `num=1000000000` and `t=32`, it will be 32 threads, and each reads 1 billion keys.
[26/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compaction_job.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compaction_job.cc b/thirdparty/rocksdb/db/compaction_job.cc
new file mode 100644
index 0000000..75f5ab6
--- /dev/null
+++ b/thirdparty/rocksdb/db/compaction_job.cc
@@ -0,0 +1,1467 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/compaction_job.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <algorithm>
+#include <functional>
+#include <list>
+#include <memory>
+#include <random>
+#include <set>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#include "db/builder.h"
+#include "db/db_iter.h"
+#include "db/dbformat.h"
+#include "db/event_helpers.h"
+#include "db/log_reader.h"
+#include "db/log_writer.h"
+#include "db/memtable.h"
+#include "db/memtable_list.h"
+#include "db/merge_context.h"
+#include "db/merge_helper.h"
+#include "db/version_set.h"
+#include "monitoring/iostats_context_imp.h"
+#include "monitoring/perf_context_imp.h"
+#include "monitoring/thread_status_util.h"
+#include "port/likely.h"
+#include "port/port.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/status.h"
+#include "rocksdb/table.h"
+#include "table/block.h"
+#include "table/block_based_table_factory.h"
+#include "table/merging_iterator.h"
+#include "table/table_builder.h"
+#include "util/coding.h"
+#include "util/file_reader_writer.h"
+#include "util/filename.h"
+#include "util/log_buffer.h"
+#include "util/logging.h"
+#include "util/mutexlock.h"
+#include "util/random.h"
+#include "util/sst_file_manager_impl.h"
+#include "util/stop_watch.h"
+#include "util/string_util.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+// Maintains state for each sub-compaction
+struct CompactionJob::SubcompactionState {
+ const Compaction* compaction;
+ std::unique_ptr<CompactionIterator> c_iter;
+
+ // The boundaries of the key-range this compaction is interested in. No two
+ // subcompactions may have overlapping key-ranges.
+ // 'start' is inclusive, 'end' is exclusive, and nullptr means unbounded
+ Slice *start, *end;
+
+ // The return status of this subcompaction
+ Status status;
+
+ // Files produced by this subcompaction
+ struct Output {
+ FileMetaData meta;
+ bool finished;
+ std::shared_ptr<const TableProperties> table_properties;
+ };
+
+ // State kept for output being generated
+ std::vector<Output> outputs;
+ std::unique_ptr<WritableFileWriter> outfile;
+ std::unique_ptr<TableBuilder> builder;
+ Output* current_output() {
+ if (outputs.empty()) {
+ // This subcompaction's outptut could be empty if compaction was aborted
+ // before this subcompaction had a chance to generate any output files.
+ // When subcompactions are executed sequentially this is more likely and
+ // will be particulalry likely for the later subcompactions to be empty.
+ // Once they are run in parallel however it should be much rarer.
+ return nullptr;
+ } else {
+ return &outputs.back();
+ }
+ }
+
+ uint64_t current_output_file_size;
+
+ // State during the subcompaction
+ uint64_t total_bytes;
+ uint64_t num_input_records;
+ uint64_t num_output_records;
+ CompactionJobStats compaction_job_stats;
+ uint64_t approx_size;
+ // An index that used to speed up ShouldStopBefore().
+ size_t grandparent_index = 0;
+ // The number of bytes overlapping between the current output and
+ // grandparent files used in ShouldStopBefore().
+ uint64_t overlapped_bytes = 0;
+ // A flag determine whether the key has been seen in ShouldStopBefore()
+ bool seen_key = false;
+ std::string compression_dict;
+
+ SubcompactionState(Compaction* c, Slice* _start, Slice* _end,
+ uint64_t size = 0)
+ : compaction(c),
+ start(_start),
+ end(_end),
+ outfile(nullptr),
+ builder(nullptr),
+ current_output_file_size(0),
+ total_bytes(0),
+ num_input_records(0),
+ num_output_records(0),
+ approx_size(size),
+ grandparent_index(0),
+ overlapped_bytes(0),
+ seen_key(false),
+ compression_dict() {
+ assert(compaction != nullptr);
+ }
+
+ SubcompactionState(SubcompactionState&& o) { *this = std::move(o); }
+
+ SubcompactionState& operator=(SubcompactionState&& o) {
+ compaction = std::move(o.compaction);
+ start = std::move(o.start);
+ end = std::move(o.end);
+ status = std::move(o.status);
+ outputs = std::move(o.outputs);
+ outfile = std::move(o.outfile);
+ builder = std::move(o.builder);
+ current_output_file_size = std::move(o.current_output_file_size);
+ total_bytes = std::move(o.total_bytes);
+ num_input_records = std::move(o.num_input_records);
+ num_output_records = std::move(o.num_output_records);
+ compaction_job_stats = std::move(o.compaction_job_stats);
+ approx_size = std::move(o.approx_size);
+ grandparent_index = std::move(o.grandparent_index);
+ overlapped_bytes = std::move(o.overlapped_bytes);
+ seen_key = std::move(o.seen_key);
+ compression_dict = std::move(o.compression_dict);
+ return *this;
+ }
+
+ // Because member unique_ptrs do not have these.
+ SubcompactionState(const SubcompactionState&) = delete;
+
+ SubcompactionState& operator=(const SubcompactionState&) = delete;
+
+ // Returns true iff we should stop building the current output
+ // before processing "internal_key".
+ bool ShouldStopBefore(const Slice& internal_key, uint64_t curr_file_size) {
+ const InternalKeyComparator* icmp =
+ &compaction->column_family_data()->internal_comparator();
+ const std::vector<FileMetaData*>& grandparents = compaction->grandparents();
+
+ // Scan to find earliest grandparent file that contains key.
+ while (grandparent_index < grandparents.size() &&
+ icmp->Compare(internal_key,
+ grandparents[grandparent_index]->largest.Encode()) >
+ 0) {
+ if (seen_key) {
+ overlapped_bytes += grandparents[grandparent_index]->fd.GetFileSize();
+ }
+ assert(grandparent_index + 1 >= grandparents.size() ||
+ icmp->Compare(
+ grandparents[grandparent_index]->largest.Encode(),
+ grandparents[grandparent_index + 1]->smallest.Encode()) <= 0);
+ grandparent_index++;
+ }
+ seen_key = true;
+
+ if (overlapped_bytes + curr_file_size >
+ compaction->max_compaction_bytes()) {
+ // Too much overlap for current output; start new output
+ overlapped_bytes = 0;
+ return true;
+ }
+
+ return false;
+ }
+};
+
+// Maintains state for the entire compaction
+struct CompactionJob::CompactionState {
+ Compaction* const compaction;
+
+ // REQUIRED: subcompaction states are stored in order of increasing
+ // key-range
+ std::vector<CompactionJob::SubcompactionState> sub_compact_states;
+ Status status;
+
+ uint64_t total_bytes;
+ uint64_t num_input_records;
+ uint64_t num_output_records;
+
+ explicit CompactionState(Compaction* c)
+ : compaction(c),
+ total_bytes(0),
+ num_input_records(0),
+ num_output_records(0) {}
+
+ size_t NumOutputFiles() {
+ size_t total = 0;
+ for (auto& s : sub_compact_states) {
+ total += s.outputs.size();
+ }
+ return total;
+ }
+
+ Slice SmallestUserKey() {
+ for (const auto& sub_compact_state : sub_compact_states) {
+ if (!sub_compact_state.outputs.empty() &&
+ sub_compact_state.outputs[0].finished) {
+ return sub_compact_state.outputs[0].meta.smallest.user_key();
+ }
+ }
+ // If there is no finished output, return an empty slice.
+ return Slice(nullptr, 0);
+ }
+
+ Slice LargestUserKey() {
+ for (auto it = sub_compact_states.rbegin(); it < sub_compact_states.rend();
+ ++it) {
+ if (!it->outputs.empty() && it->current_output()->finished) {
+ assert(it->current_output() != nullptr);
+ return it->current_output()->meta.largest.user_key();
+ }
+ }
+ // If there is no finished output, return an empty slice.
+ return Slice(nullptr, 0);
+ }
+};
+
+void CompactionJob::AggregateStatistics() {
+ for (SubcompactionState& sc : compact_->sub_compact_states) {
+ compact_->total_bytes += sc.total_bytes;
+ compact_->num_input_records += sc.num_input_records;
+ compact_->num_output_records += sc.num_output_records;
+ }
+ if (compaction_job_stats_) {
+ for (SubcompactionState& sc : compact_->sub_compact_states) {
+ compaction_job_stats_->Add(sc.compaction_job_stats);
+ }
+ }
+}
+
+CompactionJob::CompactionJob(
+ int job_id, Compaction* compaction, const ImmutableDBOptions& db_options,
+ const EnvOptions& env_options, VersionSet* versions,
+ const std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
+ Directory* db_directory, Directory* output_directory, Statistics* stats,
+ InstrumentedMutex* db_mutex, Status* db_bg_error,
+ std::vector<SequenceNumber> existing_snapshots,
+ SequenceNumber earliest_write_conflict_snapshot,
+ std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
+ bool paranoid_file_checks, bool measure_io_stats, const std::string& dbname,
+ CompactionJobStats* compaction_job_stats)
+ : job_id_(job_id),
+ compact_(new CompactionState(compaction)),
+ compaction_job_stats_(compaction_job_stats),
+ compaction_stats_(1),
+ dbname_(dbname),
+ db_options_(db_options),
+ env_options_(env_options),
+ env_(db_options.env),
+ versions_(versions),
+ shutting_down_(shutting_down),
+ log_buffer_(log_buffer),
+ db_directory_(db_directory),
+ output_directory_(output_directory),
+ stats_(stats),
+ db_mutex_(db_mutex),
+ db_bg_error_(db_bg_error),
+ existing_snapshots_(std::move(existing_snapshots)),
+ earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
+ table_cache_(std::move(table_cache)),
+ event_logger_(event_logger),
+ paranoid_file_checks_(paranoid_file_checks),
+ measure_io_stats_(measure_io_stats) {
+ assert(log_buffer_ != nullptr);
+ const auto* cfd = compact_->compaction->column_family_data();
+ ThreadStatusUtil::SetColumnFamily(cfd, cfd->ioptions()->env,
+ db_options_.enable_thread_tracking);
+ ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION);
+ ReportStartedCompaction(compaction);
+}
+
+CompactionJob::~CompactionJob() {
+ assert(compact_ == nullptr);
+ ThreadStatusUtil::ResetThreadStatus();
+}
+
+void CompactionJob::ReportStartedCompaction(
+ Compaction* compaction) {
+ const auto* cfd = compact_->compaction->column_family_data();
+ ThreadStatusUtil::SetColumnFamily(cfd, cfd->ioptions()->env,
+ db_options_.enable_thread_tracking);
+
+ ThreadStatusUtil::SetThreadOperationProperty(
+ ThreadStatus::COMPACTION_JOB_ID,
+ job_id_);
+
+ ThreadStatusUtil::SetThreadOperationProperty(
+ ThreadStatus::COMPACTION_INPUT_OUTPUT_LEVEL,
+ (static_cast<uint64_t>(compact_->compaction->start_level()) << 32) +
+ compact_->compaction->output_level());
+
+ // In the current design, a CompactionJob is always created
+ // for non-trivial compaction.
+ assert(compaction->IsTrivialMove() == false ||
+ compaction->is_manual_compaction() == true);
+
+ ThreadStatusUtil::SetThreadOperationProperty(
+ ThreadStatus::COMPACTION_PROP_FLAGS,
+ compaction->is_manual_compaction() +
+ (compaction->deletion_compaction() << 1));
+
+ ThreadStatusUtil::SetThreadOperationProperty(
+ ThreadStatus::COMPACTION_TOTAL_INPUT_BYTES,
+ compaction->CalculateTotalInputSize());
+
+ IOSTATS_RESET(bytes_written);
+ IOSTATS_RESET(bytes_read);
+ ThreadStatusUtil::SetThreadOperationProperty(
+ ThreadStatus::COMPACTION_BYTES_WRITTEN, 0);
+ ThreadStatusUtil::SetThreadOperationProperty(
+ ThreadStatus::COMPACTION_BYTES_READ, 0);
+
+ // Set the thread operation after operation properties
+ // to ensure GetThreadList() can always show them all together.
+ ThreadStatusUtil::SetThreadOperation(
+ ThreadStatus::OP_COMPACTION);
+
+ if (compaction_job_stats_) {
+ compaction_job_stats_->is_manual_compaction =
+ compaction->is_manual_compaction();
+ }
+}
+
+void CompactionJob::Prepare() {
+ AutoThreadOperationStageUpdater stage_updater(
+ ThreadStatus::STAGE_COMPACTION_PREPARE);
+
+ // Generate file_levels_ for compaction berfore making Iterator
+ auto* c = compact_->compaction;
+ assert(c->column_family_data() != nullptr);
+ assert(c->column_family_data()->current()->storage_info()
+ ->NumLevelFiles(compact_->compaction->level()) > 0);
+
+ // Is this compaction producing files at the bottommost level?
+ bottommost_level_ = c->bottommost_level();
+
+ if (c->ShouldFormSubcompactions()) {
+ const uint64_t start_micros = env_->NowMicros();
+ GenSubcompactionBoundaries();
+ MeasureTime(stats_, SUBCOMPACTION_SETUP_TIME,
+ env_->NowMicros() - start_micros);
+
+ assert(sizes_.size() == boundaries_.size() + 1);
+
+ for (size_t i = 0; i <= boundaries_.size(); i++) {
+ Slice* start = i == 0 ? nullptr : &boundaries_[i - 1];
+ Slice* end = i == boundaries_.size() ? nullptr : &boundaries_[i];
+ compact_->sub_compact_states.emplace_back(c, start, end, sizes_[i]);
+ }
+ MeasureTime(stats_, NUM_SUBCOMPACTIONS_SCHEDULED,
+ compact_->sub_compact_states.size());
+ } else {
+ compact_->sub_compact_states.emplace_back(c, nullptr, nullptr);
+ }
+}
+
+struct RangeWithSize {
+ Range range;
+ uint64_t size;
+
+ RangeWithSize(const Slice& a, const Slice& b, uint64_t s = 0)
+ : range(a, b), size(s) {}
+};
+
+// Generates a histogram representing potential divisions of key ranges from
+// the input. It adds the starting and/or ending keys of certain input files
+// to the working set and then finds the approximate size of data in between
+// each consecutive pair of slices. Then it divides these ranges into
+// consecutive groups such that each group has a similar size.
+void CompactionJob::GenSubcompactionBoundaries() {
+ auto* c = compact_->compaction;
+ auto* cfd = c->column_family_data();
+ const Comparator* cfd_comparator = cfd->user_comparator();
+ std::vector<Slice> bounds;
+ int start_lvl = c->start_level();
+ int out_lvl = c->output_level();
+
+ // Add the starting and/or ending key of certain input files as a potential
+ // boundary
+ for (size_t lvl_idx = 0; lvl_idx < c->num_input_levels(); lvl_idx++) {
+ int lvl = c->level(lvl_idx);
+ if (lvl >= start_lvl && lvl <= out_lvl) {
+ const LevelFilesBrief* flevel = c->input_levels(lvl_idx);
+ size_t num_files = flevel->num_files;
+
+ if (num_files == 0) {
+ continue;
+ }
+
+ if (lvl == 0) {
+ // For level 0 add the starting and ending key of each file since the
+ // files may have greatly differing key ranges (not range-partitioned)
+ for (size_t i = 0; i < num_files; i++) {
+ bounds.emplace_back(flevel->files[i].smallest_key);
+ bounds.emplace_back(flevel->files[i].largest_key);
+ }
+ } else {
+ // For all other levels add the smallest/largest key in the level to
+ // encompass the range covered by that level
+ bounds.emplace_back(flevel->files[0].smallest_key);
+ bounds.emplace_back(flevel->files[num_files - 1].largest_key);
+ if (lvl == out_lvl) {
+ // For the last level include the starting keys of all files since
+ // the last level is the largest and probably has the widest key
+ // range. Since it's range partitioned, the ending key of one file
+ // and the starting key of the next are very close (or identical).
+ for (size_t i = 1; i < num_files; i++) {
+ bounds.emplace_back(flevel->files[i].smallest_key);
+ }
+ }
+ }
+ }
+ }
+
+ std::sort(bounds.begin(), bounds.end(),
+ [cfd_comparator] (const Slice& a, const Slice& b) -> bool {
+ return cfd_comparator->Compare(ExtractUserKey(a), ExtractUserKey(b)) < 0;
+ });
+ // Remove duplicated entries from bounds
+ bounds.erase(std::unique(bounds.begin(), bounds.end(),
+ [cfd_comparator] (const Slice& a, const Slice& b) -> bool {
+ return cfd_comparator->Compare(ExtractUserKey(a), ExtractUserKey(b)) == 0;
+ }), bounds.end());
+
+ // Combine consecutive pairs of boundaries into ranges with an approximate
+ // size of data covered by keys in that range
+ uint64_t sum = 0;
+ std::vector<RangeWithSize> ranges;
+ auto* v = cfd->current();
+ for (auto it = bounds.begin();;) {
+ const Slice a = *it;
+ it++;
+
+ if (it == bounds.end()) {
+ break;
+ }
+
+ const Slice b = *it;
+ uint64_t size = versions_->ApproximateSize(v, a, b, start_lvl, out_lvl + 1);
+ ranges.emplace_back(a, b, size);
+ sum += size;
+ }
+
+ // Group the ranges into subcompactions
+ const double min_file_fill_percent = 4.0 / 5;
+ uint64_t max_output_files = static_cast<uint64_t>(
+ std::ceil(sum / min_file_fill_percent /
+ c->mutable_cf_options()->MaxFileSizeForLevel(out_lvl)));
+ uint64_t subcompactions =
+ std::min({static_cast<uint64_t>(ranges.size()),
+ static_cast<uint64_t>(db_options_.max_subcompactions),
+ max_output_files});
+
+ if (subcompactions > 1) {
+ double mean = sum * 1.0 / subcompactions;
+ // Greedily add ranges to the subcompaction until the sum of the ranges'
+ // sizes becomes >= the expected mean size of a subcompaction
+ sum = 0;
+ for (size_t i = 0; i < ranges.size() - 1; i++) {
+ sum += ranges[i].size;
+ if (subcompactions == 1) {
+ // If there's only one left to schedule then it goes to the end so no
+ // need to put an end boundary
+ continue;
+ }
+ if (sum >= mean) {
+ boundaries_.emplace_back(ExtractUserKey(ranges[i].range.limit));
+ sizes_.emplace_back(sum);
+ subcompactions--;
+ sum = 0;
+ }
+ }
+ sizes_.emplace_back(sum + ranges.back().size);
+ } else {
+ // Only one range so its size is the total sum of sizes computed above
+ sizes_.emplace_back(sum);
+ }
+}
+
+Status CompactionJob::Run() {
+ AutoThreadOperationStageUpdater stage_updater(
+ ThreadStatus::STAGE_COMPACTION_RUN);
+ TEST_SYNC_POINT("CompactionJob::Run():Start");
+ log_buffer_->FlushBufferToLog();
+ LogCompaction();
+
+ const size_t num_threads = compact_->sub_compact_states.size();
+ assert(num_threads > 0);
+ const uint64_t start_micros = env_->NowMicros();
+
+ // Launch a thread for each of subcompactions 1...num_threads-1
+ std::vector<port::Thread> thread_pool;
+ thread_pool.reserve(num_threads - 1);
+ for (size_t i = 1; i < compact_->sub_compact_states.size(); i++) {
+ thread_pool.emplace_back(&CompactionJob::ProcessKeyValueCompaction, this,
+ &compact_->sub_compact_states[i]);
+ }
+
+ // Always schedule the first subcompaction (whether or not there are also
+ // others) in the current thread to be efficient with resources
+ ProcessKeyValueCompaction(&compact_->sub_compact_states[0]);
+
+ // Wait for all other threads (if there are any) to finish execution
+ for (auto& thread : thread_pool) {
+ thread.join();
+ }
+
+ if (output_directory_) {
+ output_directory_->Fsync();
+ }
+
+ compaction_stats_.micros = env_->NowMicros() - start_micros;
+ MeasureTime(stats_, COMPACTION_TIME, compaction_stats_.micros);
+
+ // Check if any thread encountered an error during execution
+ Status status;
+ for (const auto& state : compact_->sub_compact_states) {
+ if (!state.status.ok()) {
+ status = state.status;
+ break;
+ }
+ }
+
+ TablePropertiesCollection tp;
+ for (const auto& state : compact_->sub_compact_states) {
+ for (const auto& output : state.outputs) {
+ auto fn = TableFileName(db_options_.db_paths, output.meta.fd.GetNumber(),
+ output.meta.fd.GetPathId());
+ tp[fn] = output.table_properties;
+ }
+ }
+ compact_->compaction->SetOutputTableProperties(std::move(tp));
+
+ // Finish up all book-keeping to unify the subcompaction results
+ AggregateStatistics();
+ UpdateCompactionStats();
+ RecordCompactionIOStats();
+ LogFlush(db_options_.info_log);
+ TEST_SYNC_POINT("CompactionJob::Run():End");
+
+ compact_->status = status;
+ return status;
+}
+
+Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
+ AutoThreadOperationStageUpdater stage_updater(
+ ThreadStatus::STAGE_COMPACTION_INSTALL);
+ db_mutex_->AssertHeld();
+ Status status = compact_->status;
+ ColumnFamilyData* cfd = compact_->compaction->column_family_data();
+ cfd->internal_stats()->AddCompactionStats(
+ compact_->compaction->output_level(), compaction_stats_);
+
+ if (status.ok()) {
+ status = InstallCompactionResults(mutable_cf_options);
+ }
+ VersionStorageInfo::LevelSummaryStorage tmp;
+ auto vstorage = cfd->current()->storage_info();
+ const auto& stats = compaction_stats_;
+
+ double read_write_amp = 0.0;
+ double write_amp = 0.0;
+ double bytes_read_per_sec = 0;
+ double bytes_written_per_sec = 0;
+
+ if (stats.bytes_read_non_output_levels > 0) {
+ read_write_amp = (stats.bytes_written + stats.bytes_read_output_level +
+ stats.bytes_read_non_output_levels) /
+ static_cast<double>(stats.bytes_read_non_output_levels);
+ write_amp = stats.bytes_written /
+ static_cast<double>(stats.bytes_read_non_output_levels);
+ }
+ if (stats.micros > 0) {
+ bytes_read_per_sec =
+ (stats.bytes_read_non_output_levels + stats.bytes_read_output_level) /
+ static_cast<double>(stats.micros);
+ bytes_written_per_sec =
+ stats.bytes_written / static_cast<double>(stats.micros);
+ }
+
+ ROCKS_LOG_BUFFER(
+ log_buffer_,
+ "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
+ "files in(%d, %d) out(%d) "
+ "MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
+ "write-amplify(%.1f) %s, records in: %d, records dropped: %d\n",
+ cfd->GetName().c_str(), vstorage->LevelSummary(&tmp), bytes_read_per_sec,
+ bytes_written_per_sec, compact_->compaction->output_level(),
+ stats.num_input_files_in_non_output_levels,
+ stats.num_input_files_in_output_level, stats.num_output_files,
+ stats.bytes_read_non_output_levels / 1048576.0,
+ stats.bytes_read_output_level / 1048576.0,
+ stats.bytes_written / 1048576.0, read_write_amp, write_amp,
+ status.ToString().c_str(), stats.num_input_records,
+ stats.num_dropped_records);
+
+ UpdateCompactionJobStats(stats);
+
+ auto stream = event_logger_->LogToBuffer(log_buffer_);
+ stream << "job" << job_id_
+ << "event" << "compaction_finished"
+ << "compaction_time_micros" << compaction_stats_.micros
+ << "output_level" << compact_->compaction->output_level()
+ << "num_output_files" << compact_->NumOutputFiles()
+ << "total_output_size" << compact_->total_bytes
+ << "num_input_records" << compact_->num_input_records
+ << "num_output_records" << compact_->num_output_records
+ << "num_subcompactions" << compact_->sub_compact_states.size();
+
+ if (compaction_job_stats_ != nullptr) {
+ stream << "num_single_delete_mismatches"
+ << compaction_job_stats_->num_single_del_mismatch;
+ stream << "num_single_delete_fallthrough"
+ << compaction_job_stats_->num_single_del_fallthru;
+ }
+
+ if (measure_io_stats_ && compaction_job_stats_ != nullptr) {
+ stream << "file_write_nanos" << compaction_job_stats_->file_write_nanos;
+ stream << "file_range_sync_nanos"
+ << compaction_job_stats_->file_range_sync_nanos;
+ stream << "file_fsync_nanos" << compaction_job_stats_->file_fsync_nanos;
+ stream << "file_prepare_write_nanos"
+ << compaction_job_stats_->file_prepare_write_nanos;
+ }
+
+ stream << "lsm_state";
+ stream.StartArray();
+ for (int level = 0; level < vstorage->num_levels(); ++level) {
+ stream << vstorage->NumLevelFiles(level);
+ }
+ stream.EndArray();
+
+ CleanupCompaction();
+ return status;
+}
+
+void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
+ assert(sub_compact != nullptr);
+ ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
+ std::unique_ptr<RangeDelAggregator> range_del_agg(
+ new RangeDelAggregator(cfd->internal_comparator(), existing_snapshots_));
+ std::unique_ptr<InternalIterator> input(versions_->MakeInputIterator(
+ sub_compact->compaction, range_del_agg.get()));
+
+ AutoThreadOperationStageUpdater stage_updater(
+ ThreadStatus::STAGE_COMPACTION_PROCESS_KV);
+
+ // I/O measurement variables
+ PerfLevel prev_perf_level = PerfLevel::kEnableTime;
+ const uint64_t kRecordStatsEvery = 1000;
+ uint64_t prev_write_nanos = 0;
+ uint64_t prev_fsync_nanos = 0;
+ uint64_t prev_range_sync_nanos = 0;
+ uint64_t prev_prepare_write_nanos = 0;
+ if (measure_io_stats_) {
+ prev_perf_level = GetPerfLevel();
+ SetPerfLevel(PerfLevel::kEnableTime);
+ prev_write_nanos = IOSTATS(write_nanos);
+ prev_fsync_nanos = IOSTATS(fsync_nanos);
+ prev_range_sync_nanos = IOSTATS(range_sync_nanos);
+ prev_prepare_write_nanos = IOSTATS(prepare_write_nanos);
+ }
+
+ const MutableCFOptions* mutable_cf_options =
+ sub_compact->compaction->mutable_cf_options();
+
+ // To build compression dictionary, we sample the first output file, assuming
+ // it'll reach the maximum length, and then use the dictionary for compressing
+ // subsequent output files. The dictionary may be less than max_dict_bytes if
+ // the first output file's length is less than the maximum.
+ const int kSampleLenShift = 6; // 2^6 = 64-byte samples
+ std::set<size_t> sample_begin_offsets;
+ if (bottommost_level_ &&
+ cfd->ioptions()->compression_opts.max_dict_bytes > 0) {
+ const size_t kMaxSamples =
+ cfd->ioptions()->compression_opts.max_dict_bytes >> kSampleLenShift;
+ const size_t kOutFileLen = mutable_cf_options->MaxFileSizeForLevel(
+ compact_->compaction->output_level());
+ if (kOutFileLen != port::kMaxSizet) {
+ const size_t kOutFileNumSamples = kOutFileLen >> kSampleLenShift;
+ Random64 generator{versions_->NewFileNumber()};
+ for (size_t i = 0; i < kMaxSamples; ++i) {
+ sample_begin_offsets.insert(generator.Uniform(kOutFileNumSamples)
+ << kSampleLenShift);
+ }
+ }
+ }
+
+ auto compaction_filter = cfd->ioptions()->compaction_filter;
+ std::unique_ptr<CompactionFilter> compaction_filter_from_factory = nullptr;
+ if (compaction_filter == nullptr) {
+ compaction_filter_from_factory =
+ sub_compact->compaction->CreateCompactionFilter();
+ compaction_filter = compaction_filter_from_factory.get();
+ }
+ MergeHelper merge(
+ env_, cfd->user_comparator(), cfd->ioptions()->merge_operator,
+ compaction_filter, db_options_.info_log.get(),
+ false /* internal key corruption is expected */,
+ existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
+ compact_->compaction->level(), db_options_.statistics.get(),
+ shutting_down_);
+
+ TEST_SYNC_POINT("CompactionJob::Run():Inprogress");
+
+ Slice* start = sub_compact->start;
+ Slice* end = sub_compact->end;
+ if (start != nullptr) {
+ IterKey start_iter;
+ start_iter.SetInternalKey(*start, kMaxSequenceNumber, kValueTypeForSeek);
+ input->Seek(start_iter.GetInternalKey());
+ } else {
+ input->SeekToFirst();
+ }
+
+ // we allow only 1 compaction event listener. Used by blob storage
+ CompactionEventListener* comp_event_listener = nullptr;
+#ifndef ROCKSDB_LITE
+ for (auto& celitr : cfd->ioptions()->listeners) {
+ comp_event_listener = celitr->GetCompactionEventListener();
+ if (comp_event_listener != nullptr) {
+ break;
+ }
+ }
+#endif // ROCKSDB_LITE
+
+ Status status;
+ sub_compact->c_iter.reset(new CompactionIterator(
+ input.get(), cfd->user_comparator(), &merge, versions_->LastSequence(),
+ &existing_snapshots_, earliest_write_conflict_snapshot_, env_, false,
+ range_del_agg.get(), sub_compact->compaction, compaction_filter,
+ comp_event_listener, shutting_down_));
+ auto c_iter = sub_compact->c_iter.get();
+ c_iter->SeekToFirst();
+ if (c_iter->Valid() &&
+ sub_compact->compaction->output_level() != 0) {
+ // ShouldStopBefore() maintains state based on keys processed so far. The
+ // compaction loop always calls it on the "next" key, thus won't tell it the
+ // first key. So we do that here.
+ sub_compact->ShouldStopBefore(
+ c_iter->key(), sub_compact->current_output_file_size);
+ }
+ const auto& c_iter_stats = c_iter->iter_stats();
+ auto sample_begin_offset_iter = sample_begin_offsets.cbegin();
+ // data_begin_offset and compression_dict are only valid while generating
+ // dictionary from the first output file.
+ size_t data_begin_offset = 0;
+ std::string compression_dict;
+ compression_dict.reserve(cfd->ioptions()->compression_opts.max_dict_bytes);
+
+ while (status.ok() && !cfd->IsDropped() && c_iter->Valid()) {
+ // Invariant: c_iter.status() is guaranteed to be OK if c_iter->Valid()
+ // returns true.
+ const Slice& key = c_iter->key();
+ const Slice& value = c_iter->value();
+
+ // If an end key (exclusive) is specified, check if the current key is
+ // >= than it and exit if it is because the iterator is out of its range
+ if (end != nullptr &&
+ cfd->user_comparator()->Compare(c_iter->user_key(), *end) >= 0) {
+ break;
+ }
+ if (c_iter_stats.num_input_records % kRecordStatsEvery ==
+ kRecordStatsEvery - 1) {
+ RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
+ c_iter->ResetRecordCounts();
+ RecordCompactionIOStats();
+ }
+
+ // Open output file if necessary
+ if (sub_compact->builder == nullptr) {
+ status = OpenCompactionOutputFile(sub_compact);
+ if (!status.ok()) {
+ break;
+ }
+ }
+ assert(sub_compact->builder != nullptr);
+ assert(sub_compact->current_output() != nullptr);
+ sub_compact->builder->Add(key, value);
+ sub_compact->current_output_file_size = sub_compact->builder->FileSize();
+ sub_compact->current_output()->meta.UpdateBoundaries(
+ key, c_iter->ikey().sequence);
+ sub_compact->num_output_records++;
+
+ if (sub_compact->outputs.size() == 1) { // first output file
+ // Check if this key/value overlaps any sample intervals; if so, appends
+ // overlapping portions to the dictionary.
+ for (const auto& data_elmt : {key, value}) {
+ size_t data_end_offset = data_begin_offset + data_elmt.size();
+ while (sample_begin_offset_iter != sample_begin_offsets.cend() &&
+ *sample_begin_offset_iter < data_end_offset) {
+ size_t sample_end_offset =
+ *sample_begin_offset_iter + (1 << kSampleLenShift);
+ // Invariant: Because we advance sample iterator while processing the
+ // data_elmt containing the sample's last byte, the current sample
+ // cannot end before the current data_elmt.
+ assert(data_begin_offset < sample_end_offset);
+
+ size_t data_elmt_copy_offset, data_elmt_copy_len;
+ if (*sample_begin_offset_iter <= data_begin_offset) {
+ // The sample starts before data_elmt starts, so take bytes starting
+ // at the beginning of data_elmt.
+ data_elmt_copy_offset = 0;
+ } else {
+ // data_elmt starts before the sample starts, so take bytes starting
+ // at the below offset into data_elmt.
+ data_elmt_copy_offset =
+ *sample_begin_offset_iter - data_begin_offset;
+ }
+ if (sample_end_offset <= data_end_offset) {
+ // The sample ends before data_elmt ends, so take as many bytes as
+ // needed.
+ data_elmt_copy_len =
+ sample_end_offset - (data_begin_offset + data_elmt_copy_offset);
+ } else {
+ // data_elmt ends before the sample ends, so take all remaining
+ // bytes in data_elmt.
+ data_elmt_copy_len =
+ data_end_offset - (data_begin_offset + data_elmt_copy_offset);
+ }
+ compression_dict.append(&data_elmt.data()[data_elmt_copy_offset],
+ data_elmt_copy_len);
+ if (sample_end_offset > data_end_offset) {
+ // Didn't finish sample. Try to finish it with the next data_elmt.
+ break;
+ }
+ // Next sample may require bytes from same data_elmt.
+ sample_begin_offset_iter++;
+ }
+ data_begin_offset = data_end_offset;
+ }
+ }
+
+ // Close output file if it is big enough. Two possibilities determine it's
+ // time to close it: (1) the current key should be this file's last key, (2)
+ // the next key should not be in this file.
+ //
+ // TODO(aekmekji): determine if file should be closed earlier than this
+ // during subcompactions (i.e. if output size, estimated by input size, is
+ // going to be 1.2MB and max_output_file_size = 1MB, prefer to have 0.6MB
+ // and 0.6MB instead of 1MB and 0.2MB)
+ bool output_file_ended = false;
+ Status input_status;
+ if (sub_compact->compaction->output_level() != 0 &&
+ sub_compact->current_output_file_size >=
+ sub_compact->compaction->max_output_file_size()) {
+ // (1) this key terminates the file. For historical reasons, the iterator
+ // status before advancing will be given to FinishCompactionOutputFile().
+ input_status = input->status();
+ output_file_ended = true;
+ }
+ c_iter->Next();
+ if (!output_file_ended && c_iter->Valid() &&
+ sub_compact->compaction->output_level() != 0 &&
+ sub_compact->ShouldStopBefore(
+ c_iter->key(), sub_compact->current_output_file_size) &&
+ sub_compact->builder != nullptr) {
+ // (2) this key belongs to the next file. For historical reasons, the
+ // iterator status after advancing will be given to
+ // FinishCompactionOutputFile().
+ input_status = input->status();
+ output_file_ended = true;
+ }
+ if (output_file_ended) {
+ const Slice* next_key = nullptr;
+ if (c_iter->Valid()) {
+ next_key = &c_iter->key();
+ }
+ CompactionIterationStats range_del_out_stats;
+ status = FinishCompactionOutputFile(input_status, sub_compact,
+ range_del_agg.get(),
+ &range_del_out_stats, next_key);
+ RecordDroppedKeys(range_del_out_stats,
+ &sub_compact->compaction_job_stats);
+ if (sub_compact->outputs.size() == 1) {
+ // Use dictionary from first output file for compression of subsequent
+ // files.
+ sub_compact->compression_dict = std::move(compression_dict);
+ }
+ }
+ }
+
+ sub_compact->num_input_records = c_iter_stats.num_input_records;
+ sub_compact->compaction_job_stats.num_input_deletion_records =
+ c_iter_stats.num_input_deletion_records;
+ sub_compact->compaction_job_stats.num_corrupt_keys =
+ c_iter_stats.num_input_corrupt_records;
+ sub_compact->compaction_job_stats.num_single_del_fallthru =
+ c_iter_stats.num_single_del_fallthru;
+ sub_compact->compaction_job_stats.num_single_del_mismatch =
+ c_iter_stats.num_single_del_mismatch;
+ sub_compact->compaction_job_stats.total_input_raw_key_bytes +=
+ c_iter_stats.total_input_raw_key_bytes;
+ sub_compact->compaction_job_stats.total_input_raw_value_bytes +=
+ c_iter_stats.total_input_raw_value_bytes;
+
+ RecordTick(stats_, FILTER_OPERATION_TOTAL_TIME,
+ c_iter_stats.total_filter_time);
+ RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
+ RecordCompactionIOStats();
+
+ if (status.ok() && (shutting_down_->load(std::memory_order_relaxed) ||
+ cfd->IsDropped())) {
+ status = Status::ShutdownInProgress(
+ "Database shutdown or Column family drop during compaction");
+ }
+ if (status.ok()) {
+ status = input->status();
+ }
+ if (status.ok()) {
+ status = c_iter->status();
+ }
+
+ if (status.ok() && sub_compact->builder == nullptr &&
+ sub_compact->outputs.size() == 0 &&
+ range_del_agg->ShouldAddTombstones(bottommost_level_)) {
+ // handle subcompaction containing only range deletions
+ status = OpenCompactionOutputFile(sub_compact);
+ }
+
+ // Call FinishCompactionOutputFile() even if status is not ok: it needs to
+ // close the output file.
+ if (sub_compact->builder != nullptr) {
+ CompactionIterationStats range_del_out_stats;
+ Status s = FinishCompactionOutputFile(
+ status, sub_compact, range_del_agg.get(), &range_del_out_stats);
+ if (status.ok()) {
+ status = s;
+ }
+ RecordDroppedKeys(range_del_out_stats, &sub_compact->compaction_job_stats);
+ }
+
+ if (measure_io_stats_) {
+ sub_compact->compaction_job_stats.file_write_nanos +=
+ IOSTATS(write_nanos) - prev_write_nanos;
+ sub_compact->compaction_job_stats.file_fsync_nanos +=
+ IOSTATS(fsync_nanos) - prev_fsync_nanos;
+ sub_compact->compaction_job_stats.file_range_sync_nanos +=
+ IOSTATS(range_sync_nanos) - prev_range_sync_nanos;
+ sub_compact->compaction_job_stats.file_prepare_write_nanos +=
+ IOSTATS(prepare_write_nanos) - prev_prepare_write_nanos;
+ if (prev_perf_level != PerfLevel::kEnableTime) {
+ SetPerfLevel(prev_perf_level);
+ }
+ }
+
+ sub_compact->c_iter.reset();
+ input.reset();
+ sub_compact->status = status;
+}
+
+void CompactionJob::RecordDroppedKeys(
+ const CompactionIterationStats& c_iter_stats,
+ CompactionJobStats* compaction_job_stats) {
+ if (c_iter_stats.num_record_drop_user > 0) {
+ RecordTick(stats_, COMPACTION_KEY_DROP_USER,
+ c_iter_stats.num_record_drop_user);
+ }
+ if (c_iter_stats.num_record_drop_hidden > 0) {
+ RecordTick(stats_, COMPACTION_KEY_DROP_NEWER_ENTRY,
+ c_iter_stats.num_record_drop_hidden);
+ if (compaction_job_stats) {
+ compaction_job_stats->num_records_replaced +=
+ c_iter_stats.num_record_drop_hidden;
+ }
+ }
+ if (c_iter_stats.num_record_drop_obsolete > 0) {
+ RecordTick(stats_, COMPACTION_KEY_DROP_OBSOLETE,
+ c_iter_stats.num_record_drop_obsolete);
+ if (compaction_job_stats) {
+ compaction_job_stats->num_expired_deletion_records +=
+ c_iter_stats.num_record_drop_obsolete;
+ }
+ }
+ if (c_iter_stats.num_record_drop_range_del > 0) {
+ RecordTick(stats_, COMPACTION_KEY_DROP_RANGE_DEL,
+ c_iter_stats.num_record_drop_range_del);
+ }
+ if (c_iter_stats.num_range_del_drop_obsolete > 0) {
+ RecordTick(stats_, COMPACTION_RANGE_DEL_DROP_OBSOLETE,
+ c_iter_stats.num_range_del_drop_obsolete);
+ }
+}
+
+Status CompactionJob::FinishCompactionOutputFile(
+ const Status& input_status, SubcompactionState* sub_compact,
+ RangeDelAggregator* range_del_agg,
+ CompactionIterationStats* range_del_out_stats,
+ const Slice* next_table_min_key /* = nullptr */) {
+ AutoThreadOperationStageUpdater stage_updater(
+ ThreadStatus::STAGE_COMPACTION_SYNC_FILE);
+ assert(sub_compact != nullptr);
+ assert(sub_compact->outfile);
+ assert(sub_compact->builder != nullptr);
+ assert(sub_compact->current_output() != nullptr);
+
+ uint64_t output_number = sub_compact->current_output()->meta.fd.GetNumber();
+ assert(output_number != 0);
+
+ // Check for iterator errors
+ Status s = input_status;
+ auto meta = &sub_compact->current_output()->meta;
+ if (s.ok()) {
+ Slice lower_bound_guard, upper_bound_guard;
+ const Slice *lower_bound, *upper_bound;
+ if (sub_compact->outputs.size() == 1) {
+ // For the first output table, include range tombstones before the min key
+ // but after the subcompaction boundary.
+ lower_bound = sub_compact->start;
+ } else if (meta->smallest.size() > 0) {
+ // For subsequent output tables, only include range tombstones from min
+ // key onwards since the previous file was extended to contain range
+ // tombstones falling before min key.
+ lower_bound_guard = meta->smallest.user_key();
+ lower_bound = &lower_bound_guard;
+ } else {
+ lower_bound = nullptr;
+ }
+ if (next_table_min_key != nullptr) {
+ // This isn't the last file in the subcompaction, so extend until the next
+ // file starts.
+ upper_bound_guard = ExtractUserKey(*next_table_min_key);
+ upper_bound = &upper_bound_guard;
+ } else {
+ // This is the last file in the subcompaction, so extend until the
+ // subcompaction ends.
+ upper_bound = sub_compact->end;
+ }
+ range_del_agg->AddToBuilder(sub_compact->builder.get(), lower_bound,
+ upper_bound, meta, range_del_out_stats,
+ bottommost_level_);
+ }
+ const uint64_t current_entries = sub_compact->builder->NumEntries();
+ meta->marked_for_compaction = sub_compact->builder->NeedCompact();
+ if (s.ok()) {
+ s = sub_compact->builder->Finish();
+ } else {
+ sub_compact->builder->Abandon();
+ }
+ const uint64_t current_bytes = sub_compact->builder->FileSize();
+ meta->fd.file_size = current_bytes;
+ sub_compact->current_output()->finished = true;
+ sub_compact->total_bytes += current_bytes;
+
+ // Finish and check for file errors
+ if (s.ok()) {
+ StopWatch sw(env_, stats_, COMPACTION_OUTFILE_SYNC_MICROS);
+ s = sub_compact->outfile->Sync(db_options_.use_fsync);
+ }
+ if (s.ok()) {
+ s = sub_compact->outfile->Close();
+ }
+ sub_compact->outfile.reset();
+
+ if (s.ok() && current_entries == 0) {
+ // If there is nothing to output, no necessary to generate a sst file.
+ // This happens when the output level is bottom level, at the same time
+ // the sub_compact output nothing.
+ std::string fname = TableFileName(
+ db_options_.db_paths, meta->fd.GetNumber(), meta->fd.GetPathId());
+ env_->DeleteFile(fname);
+
+ // Also need to remove the file from outputs, or it will be added to the
+ // VersionEdit.
+ assert(!sub_compact->outputs.empty());
+ sub_compact->outputs.pop_back();
+ sub_compact->builder.reset();
+ sub_compact->current_output_file_size = 0;
+ return s;
+ }
+
+ ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
+ TableProperties tp;
+ if (s.ok() && current_entries > 0) {
+ // Verify that the table is usable
+ // We set for_compaction to false and don't OptimizeForCompactionTableRead
+ // here because this is a special case after we finish the table building
+ // No matter whether use_direct_io_for_flush_and_compaction is true,
+ // we will regrad this verification as user reads since the goal is
+ // to cache it here for further user reads
+ InternalIterator* iter = cfd->table_cache()->NewIterator(
+ ReadOptions(), env_options_, cfd->internal_comparator(), meta->fd,
+ nullptr /* range_del_agg */, nullptr,
+ cfd->internal_stats()->GetFileReadHist(
+ compact_->compaction->output_level()),
+ false);
+ s = iter->status();
+
+ if (s.ok() && paranoid_file_checks_) {
+ for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {}
+ s = iter->status();
+ }
+
+ delete iter;
+
+ // Output to event logger and fire events.
+ if (s.ok()) {
+ tp = sub_compact->builder->GetTableProperties();
+ sub_compact->current_output()->table_properties =
+ std::make_shared<TableProperties>(tp);
+ ROCKS_LOG_INFO(db_options_.info_log,
+ "[%s] [JOB %d] Generated table #%" PRIu64 ": %" PRIu64
+ " keys, %" PRIu64 " bytes%s",
+ cfd->GetName().c_str(), job_id_, output_number,
+ current_entries, current_bytes,
+ meta->marked_for_compaction ? " (need compaction)" : "");
+ }
+ }
+ std::string fname = TableFileName(db_options_.db_paths, meta->fd.GetNumber(),
+ meta->fd.GetPathId());
+ EventHelpers::LogAndNotifyTableFileCreationFinished(
+ event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname,
+ job_id_, meta->fd, tp, TableFileCreationReason::kCompaction, s);
+
+#ifndef ROCKSDB_LITE
+ // Report new file to SstFileManagerImpl
+ auto sfm =
+ static_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
+ if (sfm && meta->fd.GetPathId() == 0) {
+ auto fn = TableFileName(cfd->ioptions()->db_paths, meta->fd.GetNumber(),
+ meta->fd.GetPathId());
+ sfm->OnAddFile(fn);
+ if (sfm->IsMaxAllowedSpaceReached()) {
+ // TODO(ajkr): should we return OK() if max space was reached by the final
+ // compaction output file (similarly to how flush works when full)?
+ s = Status::IOError("Max allowed space was reached");
+ TEST_SYNC_POINT(
+ "CompactionJob::FinishCompactionOutputFile:"
+ "MaxAllowedSpaceReached");
+ InstrumentedMutexLock l(db_mutex_);
+ if (db_bg_error_->ok()) {
+ Status new_bg_error = s;
+ // may temporarily unlock and lock the mutex.
+ EventHelpers::NotifyOnBackgroundError(
+ cfd->ioptions()->listeners, BackgroundErrorReason::kCompaction,
+ &new_bg_error, db_mutex_);
+ if (!new_bg_error.ok()) {
+ *db_bg_error_ = new_bg_error;
+ }
+ }
+ }
+ }
+#endif
+
+ sub_compact->builder.reset();
+ sub_compact->current_output_file_size = 0;
+ return s;
+}
+
+Status CompactionJob::InstallCompactionResults(
+ const MutableCFOptions& mutable_cf_options) {
+ db_mutex_->AssertHeld();
+
+ auto* compaction = compact_->compaction;
+ // paranoia: verify that the files that we started with
+ // still exist in the current version and in the same original level.
+ // This ensures that a concurrent compaction did not erroneously
+ // pick the same files to compact_.
+ if (!versions_->VerifyCompactionFileConsistency(compaction)) {
+ Compaction::InputLevelSummaryBuffer inputs_summary;
+
+ ROCKS_LOG_ERROR(db_options_.info_log, "[%s] [JOB %d] Compaction %s aborted",
+ compaction->column_family_data()->GetName().c_str(),
+ job_id_, compaction->InputLevelSummary(&inputs_summary));
+ return Status::Corruption("Compaction input files inconsistent");
+ }
+
+ {
+ Compaction::InputLevelSummaryBuffer inputs_summary;
+ ROCKS_LOG_INFO(
+ db_options_.info_log, "[%s] [JOB %d] Compacted %s => %" PRIu64 " bytes",
+ compaction->column_family_data()->GetName().c_str(), job_id_,
+ compaction->InputLevelSummary(&inputs_summary), compact_->total_bytes);
+ }
+
+ // Add compaction outputs
+ compaction->AddInputDeletions(compact_->compaction->edit());
+
+ for (const auto& sub_compact : compact_->sub_compact_states) {
+ for (const auto& out : sub_compact.outputs) {
+ compaction->edit()->AddFile(compaction->output_level(), out.meta);
+ }
+ }
+ return versions_->LogAndApply(compaction->column_family_data(),
+ mutable_cf_options, compaction->edit(),
+ db_mutex_, db_directory_);
+}
+
+void CompactionJob::RecordCompactionIOStats() {
+ RecordTick(stats_, COMPACT_READ_BYTES, IOSTATS(bytes_read));
+ ThreadStatusUtil::IncreaseThreadOperationProperty(
+ ThreadStatus::COMPACTION_BYTES_READ, IOSTATS(bytes_read));
+ IOSTATS_RESET(bytes_read);
+ RecordTick(stats_, COMPACT_WRITE_BYTES, IOSTATS(bytes_written));
+ ThreadStatusUtil::IncreaseThreadOperationProperty(
+ ThreadStatus::COMPACTION_BYTES_WRITTEN, IOSTATS(bytes_written));
+ IOSTATS_RESET(bytes_written);
+}
+
+Status CompactionJob::OpenCompactionOutputFile(
+ SubcompactionState* sub_compact) {
+ assert(sub_compact != nullptr);
+ assert(sub_compact->builder == nullptr);
+ // no need to lock because VersionSet::next_file_number_ is atomic
+ uint64_t file_number = versions_->NewFileNumber();
+ std::string fname = TableFileName(db_options_.db_paths, file_number,
+ sub_compact->compaction->output_path_id());
+ // Fire events.
+ ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
+#ifndef ROCKSDB_LITE
+ EventHelpers::NotifyTableFileCreationStarted(
+ cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname, job_id_,
+ TableFileCreationReason::kCompaction);
+#endif // !ROCKSDB_LITE
+ // Make the output file
+ unique_ptr<WritableFile> writable_file;
+ EnvOptions opt_env_opts =
+ env_->OptimizeForCompactionTableWrite(env_options_, db_options_);
+ TEST_SYNC_POINT_CALLBACK("CompactionJob::OpenCompactionOutputFile",
+ &opt_env_opts.use_direct_writes);
+ Status s = NewWritableFile(env_, fname, &writable_file, opt_env_opts);
+ if (!s.ok()) {
+ ROCKS_LOG_ERROR(
+ db_options_.info_log,
+ "[%s] [JOB %d] OpenCompactionOutputFiles for table #%" PRIu64
+ " fails at NewWritableFile with status %s",
+ sub_compact->compaction->column_family_data()->GetName().c_str(),
+ job_id_, file_number, s.ToString().c_str());
+ LogFlush(db_options_.info_log);
+ EventHelpers::LogAndNotifyTableFileCreationFinished(
+ event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(),
+ fname, job_id_, FileDescriptor(), TableProperties(),
+ TableFileCreationReason::kCompaction, s);
+ return s;
+ }
+
+ SubcompactionState::Output out;
+ out.meta.fd =
+ FileDescriptor(file_number, sub_compact->compaction->output_path_id(), 0);
+ out.finished = false;
+
+ sub_compact->outputs.push_back(out);
+ writable_file->SetIOPriority(Env::IO_LOW);
+ writable_file->SetPreallocationBlockSize(static_cast<size_t>(
+ sub_compact->compaction->OutputFilePreallocationSize()));
+ sub_compact->outfile.reset(new WritableFileWriter(
+ std::move(writable_file), env_options_, db_options_.statistics.get()));
+
+ // If the Column family flag is to only optimize filters for hits,
+ // we can skip creating filters if this is the bottommost_level where
+ // data is going to be found
+ bool skip_filters =
+ cfd->ioptions()->optimize_filters_for_hits && bottommost_level_;
+
+ uint64_t output_file_creation_time =
+ sub_compact->compaction->MaxInputFileCreationTime();
+ if (output_file_creation_time == 0) {
+ int64_t _current_time = 0;
+ db_options_.env->GetCurrentTime(&_current_time); // ignore error
+ output_file_creation_time = static_cast<uint64_t>(_current_time);
+ }
+
+ sub_compact->builder.reset(NewTableBuilder(
+ *cfd->ioptions(), cfd->internal_comparator(),
+ cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
+ sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
+ cfd->ioptions()->compression_opts,
+ sub_compact->compaction->output_level(), &sub_compact->compression_dict,
+ skip_filters, output_file_creation_time));
+ LogFlush(db_options_.info_log);
+ return s;
+}
+
+void CompactionJob::CleanupCompaction() {
+ for (SubcompactionState& sub_compact : compact_->sub_compact_states) {
+ const auto& sub_status = sub_compact.status;
+
+ if (sub_compact.builder != nullptr) {
+ // May happen if we get a shutdown call in the middle of compaction
+ sub_compact.builder->Abandon();
+ sub_compact.builder.reset();
+ } else {
+ assert(!sub_status.ok() || sub_compact.outfile == nullptr);
+ }
+ for (const auto& out : sub_compact.outputs) {
+ // If this file was inserted into the table cache then remove
+ // them here because this compaction was not committed.
+ if (!sub_status.ok()) {
+ TableCache::Evict(table_cache_.get(), out.meta.fd.GetNumber());
+ }
+ }
+ }
+ delete compact_;
+ compact_ = nullptr;
+}
+
+#ifndef ROCKSDB_LITE
+namespace {
+void CopyPrefix(
+ const Slice& src, size_t prefix_length, std::string* dst) {
+ assert(prefix_length > 0);
+ size_t length = src.size() > prefix_length ? prefix_length : src.size();
+ dst->assign(src.data(), length);
+}
+} // namespace
+
+#endif // !ROCKSDB_LITE
+
+void CompactionJob::UpdateCompactionStats() {
+ Compaction* compaction = compact_->compaction;
+ compaction_stats_.num_input_files_in_non_output_levels = 0;
+ compaction_stats_.num_input_files_in_output_level = 0;
+ for (int input_level = 0;
+ input_level < static_cast<int>(compaction->num_input_levels());
+ ++input_level) {
+ if (compaction->level(input_level) != compaction->output_level()) {
+ UpdateCompactionInputStatsHelper(
+ &compaction_stats_.num_input_files_in_non_output_levels,
+ &compaction_stats_.bytes_read_non_output_levels,
+ input_level);
+ } else {
+ UpdateCompactionInputStatsHelper(
+ &compaction_stats_.num_input_files_in_output_level,
+ &compaction_stats_.bytes_read_output_level,
+ input_level);
+ }
+ }
+
+ for (const auto& sub_compact : compact_->sub_compact_states) {
+ size_t num_output_files = sub_compact.outputs.size();
+ if (sub_compact.builder != nullptr) {
+ // An error occurred so ignore the last output.
+ assert(num_output_files > 0);
+ --num_output_files;
+ }
+ compaction_stats_.num_output_files += static_cast<int>(num_output_files);
+
+ for (const auto& out : sub_compact.outputs) {
+ compaction_stats_.bytes_written += out.meta.fd.file_size;
+ }
+ if (sub_compact.num_input_records > sub_compact.num_output_records) {
+ compaction_stats_.num_dropped_records +=
+ sub_compact.num_input_records - sub_compact.num_output_records;
+ }
+ }
+}
+
+void CompactionJob::UpdateCompactionInputStatsHelper(
+ int* num_files, uint64_t* bytes_read, int input_level) {
+ const Compaction* compaction = compact_->compaction;
+ auto num_input_files = compaction->num_input_files(input_level);
+ *num_files += static_cast<int>(num_input_files);
+
+ for (size_t i = 0; i < num_input_files; ++i) {
+ const auto* file_meta = compaction->input(input_level, i);
+ *bytes_read += file_meta->fd.GetFileSize();
+ compaction_stats_.num_input_records +=
+ static_cast<uint64_t>(file_meta->num_entries);
+ }
+}
+
+void CompactionJob::UpdateCompactionJobStats(
+ const InternalStats::CompactionStats& stats) const {
+#ifndef ROCKSDB_LITE
+ if (compaction_job_stats_) {
+ compaction_job_stats_->elapsed_micros = stats.micros;
+
+ // input information
+ compaction_job_stats_->total_input_bytes =
+ stats.bytes_read_non_output_levels +
+ stats.bytes_read_output_level;
+ compaction_job_stats_->num_input_records =
+ compact_->num_input_records;
+ compaction_job_stats_->num_input_files =
+ stats.num_input_files_in_non_output_levels +
+ stats.num_input_files_in_output_level;
+ compaction_job_stats_->num_input_files_at_output_level =
+ stats.num_input_files_in_output_level;
+
+ // output information
+ compaction_job_stats_->total_output_bytes = stats.bytes_written;
+ compaction_job_stats_->num_output_records =
+ compact_->num_output_records;
+ compaction_job_stats_->num_output_files = stats.num_output_files;
+
+ if (compact_->NumOutputFiles() > 0U) {
+ CopyPrefix(
+ compact_->SmallestUserKey(),
+ CompactionJobStats::kMaxPrefixLength,
+ &compaction_job_stats_->smallest_output_key_prefix);
+ CopyPrefix(
+ compact_->LargestUserKey(),
+ CompactionJobStats::kMaxPrefixLength,
+ &compaction_job_stats_->largest_output_key_prefix);
+ }
+ }
+#endif // !ROCKSDB_LITE
+}
+
+void CompactionJob::LogCompaction() {
+ Compaction* compaction = compact_->compaction;
+ ColumnFamilyData* cfd = compaction->column_family_data();
+
+ // Let's check if anything will get logged. Don't prepare all the info if
+ // we're not logging
+ if (db_options_.info_log_level <= InfoLogLevel::INFO_LEVEL) {
+ Compaction::InputLevelSummaryBuffer inputs_summary;
+ ROCKS_LOG_INFO(
+ db_options_.info_log, "[%s] [JOB %d] Compacting %s, score %.2f",
+ cfd->GetName().c_str(), job_id_,
+ compaction->InputLevelSummary(&inputs_summary), compaction->score());
+ char scratch[2345];
+ compaction->Summary(scratch, sizeof(scratch));
+ ROCKS_LOG_INFO(db_options_.info_log, "[%s] Compaction start summary: %s\n",
+ cfd->GetName().c_str(), scratch);
+ // build event logger report
+ auto stream = event_logger_->Log();
+ stream << "job" << job_id_ << "event"
+ << "compaction_started";
+ for (size_t i = 0; i < compaction->num_input_levels(); ++i) {
+ stream << ("files_L" + ToString(compaction->level(i)));
+ stream.StartArray();
+ for (auto f : *compaction->inputs(i)) {
+ stream << f->fd.GetNumber();
+ }
+ stream.EndArray();
+ }
+ stream << "score" << compaction->score() << "input_data_size"
+ << compaction->CalculateTotalInputSize();
+ }
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compaction_job.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compaction_job.h b/thirdparty/rocksdb/db/compaction_job.h
new file mode 100644
index 0000000..6ca5d62
--- /dev/null
+++ b/thirdparty/rocksdb/db/compaction_job.h
@@ -0,0 +1,165 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#pragma once
+
+#include <atomic>
+#include <deque>
+#include <functional>
+#include <limits>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "db/column_family.h"
+#include "db/compaction_iterator.h"
+#include "db/dbformat.h"
+#include "db/flush_scheduler.h"
+#include "db/internal_stats.h"
+#include "db/job_context.h"
+#include "db/log_writer.h"
+#include "db/memtable_list.h"
+#include "db/range_del_aggregator.h"
+#include "db/version_edit.h"
+#include "db/write_controller.h"
+#include "db/write_thread.h"
+#include "options/db_options.h"
+#include "port/port.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/compaction_job_stats.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/transaction_log.h"
+#include "table/scoped_arena_iterator.h"
+#include "util/autovector.h"
+#include "util/event_logger.h"
+#include "util/stop_watch.h"
+#include "util/thread_local.h"
+
+namespace rocksdb {
+
+class MemTable;
+class TableCache;
+class Version;
+class VersionEdit;
+class VersionSet;
+class Arena;
+
+class CompactionJob {
+ public:
+ CompactionJob(int job_id, Compaction* compaction,
+ const ImmutableDBOptions& db_options,
+ const EnvOptions& env_options, VersionSet* versions,
+ const std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
+ Directory* db_directory, Directory* output_directory,
+ Statistics* stats, InstrumentedMutex* db_mutex,
+ Status* db_bg_error,
+ std::vector<SequenceNumber> existing_snapshots,
+ SequenceNumber earliest_write_conflict_snapshot,
+ std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
+ bool paranoid_file_checks, bool measure_io_stats,
+ const std::string& dbname,
+ CompactionJobStats* compaction_job_stats);
+
+ ~CompactionJob();
+
+ // no copy/move
+ CompactionJob(CompactionJob&& job) = delete;
+ CompactionJob(const CompactionJob& job) = delete;
+ CompactionJob& operator=(const CompactionJob& job) = delete;
+
+ // REQUIRED: mutex held
+ void Prepare();
+ // REQUIRED mutex not held
+ Status Run();
+
+ // REQUIRED: mutex held
+ Status Install(const MutableCFOptions& mutable_cf_options);
+
+ private:
+ struct SubcompactionState;
+
+ void AggregateStatistics();
+ void GenSubcompactionBoundaries();
+
+ // update the thread status for starting a compaction.
+ void ReportStartedCompaction(Compaction* compaction);
+ void AllocateCompactionOutputFileNumbers();
+ // Call compaction filter. Then iterate through input and compact the
+ // kv-pairs
+ void ProcessKeyValueCompaction(SubcompactionState* sub_compact);
+
+ Status FinishCompactionOutputFile(
+ const Status& input_status, SubcompactionState* sub_compact,
+ RangeDelAggregator* range_del_agg,
+ CompactionIterationStats* range_del_out_stats,
+ const Slice* next_table_min_key = nullptr);
+ Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options);
+ void RecordCompactionIOStats();
+ Status OpenCompactionOutputFile(SubcompactionState* sub_compact);
+ void CleanupCompaction();
+ void UpdateCompactionJobStats(
+ const InternalStats::CompactionStats& stats) const;
+ void RecordDroppedKeys(const CompactionIterationStats& c_iter_stats,
+ CompactionJobStats* compaction_job_stats = nullptr);
+
+ void UpdateCompactionStats();
+ void UpdateCompactionInputStatsHelper(
+ int* num_files, uint64_t* bytes_read, int input_level);
+
+ void LogCompaction();
+
+ int job_id_;
+
+ // CompactionJob state
+ struct CompactionState;
+ CompactionState* compact_;
+ CompactionJobStats* compaction_job_stats_;
+ InternalStats::CompactionStats compaction_stats_;
+
+ // DBImpl state
+ const std::string& dbname_;
+ const ImmutableDBOptions& db_options_;
+ const EnvOptions& env_options_;
+
+ Env* env_;
+ VersionSet* versions_;
+ const std::atomic<bool>* shutting_down_;
+ LogBuffer* log_buffer_;
+ Directory* db_directory_;
+ Directory* output_directory_;
+ Statistics* stats_;
+ InstrumentedMutex* db_mutex_;
+ Status* db_bg_error_;
+ // If there were two snapshots with seq numbers s1 and
+ // s2 and s1 < s2, and if we find two instances of a key k1 then lies
+ // entirely within s1 and s2, then the earlier version of k1 can be safely
+ // deleted because that version is not visible in any snapshot.
+ std::vector<SequenceNumber> existing_snapshots_;
+
+ // This is the earliest snapshot that could be used for write-conflict
+ // checking by a transaction. For any user-key newer than this snapshot, we
+ // should make sure not to remove evidence that a write occurred.
+ SequenceNumber earliest_write_conflict_snapshot_;
+
+ std::shared_ptr<Cache> table_cache_;
+
+ EventLogger* event_logger_;
+
+ bool bottommost_level_;
+ bool paranoid_file_checks_;
+ bool measure_io_stats_;
+ // Stores the Slices that designate the boundaries for each subcompaction
+ std::vector<Slice> boundaries_;
+ // Stores the approx size of keys covered in the range of each subcompaction
+ std::vector<uint64_t> sizes_;
+};
+
+} // namespace rocksdb
[03/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2016-01-29-compaction_pri.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2016-01-29-compaction_pri.markdown b/thirdparty/rocksdb/docs/_posts/2016-01-29-compaction_pri.markdown
new file mode 100644
index 0000000..ba9ee62
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2016-01-29-compaction_pri.markdown
@@ -0,0 +1,51 @@
+---
+title: Option of Compaction Priority
+layout: post
+author: sdong
+category: blog
+redirect_from:
+ - /blog/2921/compaction_pri/
+---
+
+The most popular compaction style of RocksDB is level-based compaction, which is an improved version of LevelDB's compaction algorithm. Page 9- 16 of this [slides](https://github.com/facebook/rocksdb/blob/gh-pages/talks/2015-09-29-HPTS-Siying-RocksDB.pdf) gives an illustrated introduction of this compaction style. The basic idea that: data is organized by multiple levels with exponential increasing target size. Except a special level 0, every level is key-range partitioned into many files. When size of a level exceeds its target size, we pick one or more of its files, and merge the file into the next level.
+
+<!--truncate-->
+
+Which file to pick to compact is an interesting question. LevelDB only uses one thread for compaction and it always picks files in round robin manner. We implemented multi-thread compaction in RocksDB by picking multiple files from the same level and compact them in parallel. We had to move away from LevelDB's file picking approach. Recently, we created an option [options.compaction_pri](https://github.com/facebook/rocksdb/blob/d6c838f1e130d8860407bc771fa6d4ac238859ba/include/rocksdb/options.h#L83-L93), which indicated three different algorithms to pick files to compact.
+
+Why do we need to multiple algorithms to choose from? Because there are different factors to consider when picking the files, and we now don't yet know how to balance them automatically, so we expose it to users to choose. Here are factors to consider:
+
+**Write amplification**
+
+When we estimate write amplification, we usually simplify the problem by assuming keys are uniformly distributed inside each level. In reality, it is not the case, even if user updates are uniformly distributed across the whole key range. For instance, when we compact one file of a level to the next level, it creates a hole. Over time, incoming compaction will fill data to the hole, but the density will still be lower for a while. Picking a file with keys least densely populated is more expensive to get the file to the next level, because there will be more overlapping files in the next level so we need to rewrite more data. For example, assume a file is 100MB, if an L2 file overlaps with 8 L3 files, we need to rewrite about 800MB of data to get the file to L3. If the file overlaps with 12 L3 files, we'll need to rewrite about 1200MB to get a file of the same size out of L2. It uses 50% more writes. (This analysis ignores the key density of the next level, because the range covers N
times of files in that level so one hole only impacts write amplification by 1/N)
+
+If all the updates are uniformly distributed, LevelDB's approach optimizes write amplification, because a file being picked covers a range whose last compaction time to the next level is the oldest, so the range will accumulated keys from incoming compactions for the longest and the density is the highest.
+
+We created a compaction priority **kOldestSmallestSeqFirst** for the same effect. With this mode, we always pick the file covers the oldest updates in the level, which usually is contains the densest key range. If you have a use case where writes are uniformly distributed across the key space and you want to reduce write amplification, you should set options.compaction_pri=kOldestSmallestSeqFirst.
+
+**Optimize for small working set**
+
+We are assuming updates are uniformly distributed across the whole key space in previous analysis. However, in many use cases, there are subset of keys that are frequently updated while other key ranges are very cold. In this case, keeping hot key ranges from compacting to deeper levels will benefit write amplification, as well as space amplification. For example, if in a DB only key 150-160 are updated and other keys are seldom updated. If level 1 contains 20 keys, we want to keep 150-160 all stay in level 1. Because when next level 0 -> 1 compaction comes, it will simply overwrite existing keys so size level 1 doesn't increase, so no need to schedule further compaction for level 1->2. On the other hand, if we compact key 150-155 to level2, when a new Level 1->2 compaction comes, it increases the size of level 1, making size of level 1 exceed target size and more compactions will be needed, which generates more writes.
+
+The compaction priority **kOldestLargestSeqFirst** optimizes this use case. In this mode, we will pick a file whose latest update is the oldest. It means there is no incoming data for the range for the longest. Usually it is the coldest range. By compacting coldest range first, we leave the hot ranges in the level. If your use case is to overwrite existing keys in a small range, try options.compaction_pri=kOldestLargestSeqFirst**.**
+
+**Drop delete marker sooner**
+
+If one file contains a lot of delete markers, it may slow down iterating over this area, because we still need to iterate those deleted keys just to ignore them. Furthermore, the sooner we compact delete keys into the last level, the sooner the disk space is reclaimed, so it is good for space efficiency.
+
+Our default compaction priority **kByCompensatedSize** considers the case. If number of deletes in a file exceeds number of inserts, it is more likely to be picked for compaction. The more number of deletes exceed inserts, the more likely it is being compacted. The optimization is added to avoid the worst performance of space efficiency and query performance when a large percentage of the DB is deleted.
+
+**Efficiency of compaction filter**
+
+Usually people use [compaction filters](https://github.com/facebook/rocksdb/blob/v4.1/include/rocksdb/options.h#L201-L226) to clean up old data to free up space. Picking files to compact may impact space efficiency. We don't yet have a a compaction priority to optimize this case. In some of our use cases, we solved the problem in a different way: we have an external service checking modify time of all SST files. If any of the files is too old, we force the single file to compaction by calling DB::CompactFiles() using the single file. In this way, we can provide a time bound of data passing through compaction filters.
+
+
+In all, there three choices of compaction priority modes optimizing different scenarios. if you have a new use case, we suggest you start with `options.compaction_pri=kOldestSmallestSeqFirst` (note it is not the default one for backward compatible reason). If you want to further optimize your use case, you can try other two use cases if your use cases apply.
+
+If you have good ideas about better compaction picker approach, you are welcome to implement and benchmark it. We'll be glad to review and merge your a pull requests.
+
+### Comments
+
+**[Mark Callaghan](mdcallag@gmail.com)**
+
+Performance results for compaction_pri values and linkbench are explained at [http://smalldatum.blogspot.com/2016/02/compaction-priority-in-rocksdb.html](http://smalldatum.blogspot.com/2016/02/compaction-priority-in-rocksdb.html)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2016-02-24-rocksdb-4-2-release.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2016-02-24-rocksdb-4-2-release.markdown b/thirdparty/rocksdb/docs/_posts/2016-02-24-rocksdb-4-2-release.markdown
new file mode 100644
index 0000000..409015c
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2016-02-24-rocksdb-4-2-release.markdown
@@ -0,0 +1,41 @@
+---
+title: RocksDB 4.2 Release!
+layout: post
+author: sdong
+category: blog
+redirect_from:
+ - /blog/3017/rocksdb-4-2-release/
+---
+
+New RocksDB release - 4.2!
+
+
+**New Features**
+
+ 1. Introduce CreateLoggerFromOptions(), this function create a Logger for provided DBOptions.
+
+
+ 2. Add GetAggregatedIntProperty(), which returns the sum of the GetIntProperty of all the column families.
+
+
+ 3. Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances.
+
+
+<!--truncate-->
+
+
+**Public API changes**
+
+ 1. CompactionFilter::Context includes information of Column Family ID
+
+
+ 2. The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower.
+
+
+ 3. TablePropertiesCollectorFactory::CreateTablePropertiesCollector() now takes an option Context, containing the information of column family ID for the file being written.
+
+
+ 4. Remove DefaultCompactionFilterFactory.
+
+
+[https://github.com/facebook/rocksdb/releases/tag/v4.2](https://github.com/facebook/rocksdb/releases/tag/v4.2)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2016-02-25-rocksdb-ama.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2016-02-25-rocksdb-ama.markdown b/thirdparty/rocksdb/docs/_posts/2016-02-25-rocksdb-ama.markdown
new file mode 100644
index 0000000..2ba04f3
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2016-02-25-rocksdb-ama.markdown
@@ -0,0 +1,20 @@
+---
+title: RocksDB AMA
+layout: post
+author: yhchiang
+category: blog
+redirect_from:
+ - /blog/3065/rocksdb-ama/
+---
+
+RocksDB developers are doing a Reddit Ask-Me-Anything now at 10AM – 11AM PDT! We welcome you to stop by and ask any RocksDB related questions, including existing / upcoming features, tuning tips, or database design.
+
+Here are some enhancements that we'd like to focus on over the next six months:
+
+* 2-Phase Commit
+* Lua support in some custom functions
+* Backup and repair tools
+* Direct I/O to bypass OS cache
+* RocksDB Java API
+
+[https://www.reddit.com/r/IAmA/comments/47k1si/we_are_rocksdb_developers_ask_us_anything/](https://www.reddit.com/r/IAmA/comments/47k1si/we_are_rocksdb_developers_ask_us_anything/)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2016-03-07-rocksdb-options-file.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2016-03-07-rocksdb-options-file.markdown b/thirdparty/rocksdb/docs/_posts/2016-03-07-rocksdb-options-file.markdown
new file mode 100644
index 0000000..703449b
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2016-03-07-rocksdb-options-file.markdown
@@ -0,0 +1,24 @@
+---
+title: RocksDB Options File
+layout: post
+author: yhciang
+category: blog
+redirect_from:
+ - /blog/3089/rocksdb-options-file/
+---
+
+In RocksDB 4.3, we added a new set of features that makes managing RocksDB options easier. Specifically:
+
+ * **Persisting Options Automatically**: Each RocksDB database will now automatically persist its current set of options into an INI file on every successful call of DB::Open(), SetOptions(), and CreateColumnFamily() / DropColumnFamily().
+
+
+
+ * **Load Options from File**: We added [LoadLatestOptions() / LoadOptionsFromFile()](https://github.com/facebook/rocksdb/blob/4.3.fb/include/rocksdb/utilities/options_util.h#L48-L58) that enables developers to construct RocksDB options object from an options file.
+
+
+
+ * **Sanity Check Options**: We added [CheckOptionsCompatibility](https://github.com/facebook/rocksdb/blob/4.3.fb/include/rocksdb/utilities/options_util.h#L64-L77) that performs compatibility check on two sets of RocksDB options.
+
+<!--truncate-->
+
+Want to know more about how to use this new features? Check out the [RocksDB Options File wiki page](https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File) and start using this new feature today!
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2016-04-26-rocksdb-4-5-1-released.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2016-04-26-rocksdb-4-5-1-released.markdown b/thirdparty/rocksdb/docs/_posts/2016-04-26-rocksdb-4-5-1-released.markdown
new file mode 100644
index 0000000..247768d
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2016-04-26-rocksdb-4-5-1-released.markdown
@@ -0,0 +1,60 @@
+---
+title: RocksDB 4.5.1 Released!
+layout: post
+author: sdong
+category: blog
+redirect_from:
+ - /blog/3179/rocksdb-4-5-1-released/
+---
+
+## 4.5.1 (3/25/2016)
+
+### Bug Fixes
+
+ * Fix failures caused by the destorying order of singleton objects.
+
+<br/>
+
+## 4.5.0 (2/5/2016)
+
+### Public API Changes
+
+ * Add a new perf context level between kEnableCount and kEnableTime. Level 2 now does not include timers for mutexes.
+ * Statistics of mutex operation durations will not be measured by default. If you want to have them enabled, you need to set Statistics::stats_level_ to kAll.
+ * DBOptions::delete_scheduler and NewDeleteScheduler() are removed, please use DBOptions::sst_file_manager and NewSstFileManager() instead
+
+### New Features
+ * ldb tool now supports operations to non-default column families.
+ * Add kPersistedTier to ReadTier. This option allows Get and MultiGet to read only the persited data and skip mem-tables if writes were done with disableWAL = true.
+ * Add DBOptions::sst_file_manager. Use NewSstFileManager() in include/rocksdb/sst_file_manager.h to create a SstFileManager that can be used to track the total size of SST files and control the SST files deletion rate.
+
+<br/>
+
+<!--truncate-->
+
+## 4.4.0 (1/14/2016)
+
+### Public API Changes
+
+ * Change names in CompactionPri and add a new one.
+ * Deprecate options.soft_rate_limit and add options.soft_pending_compaction_bytes_limit.
+ * If options.max_write_buffer_number > 3, writes will be slowed down when writing to the last write buffer to delay a full stop.
+ * Introduce CompactionJobInfo::compaction_reason, this field include the reason to trigger the compaction.
+ * After slow down is triggered, if estimated pending compaction bytes keep increasing, slowdown more.
+ * Increase default options.delayed_write_rate to 2MB/s.
+ * Added a new parameter --path to ldb tool. --path accepts the name of either MANIFEST, SST or a WAL file. Either --db or --path can be used when calling ldb.
+
+<br/>
+
+## 4.3.0 (12/8/2015)
+
+### New Features
+
+ * CompactionFilter has new member function called IgnoreSnapshots which allows CompactionFilter to be called even if there are snapshots later than the key.
+ * RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions.
+ * Introduce LoadLatestOptions() in rocksdb/utilities/options_util.h. This function can construct the latest DBOptions / ColumnFamilyOptions used by the specified RocksDB intance.
+ * Introduce CheckOptionsCompatibility() in rocksdb/utilities/options_util.h. This function checks whether the input set of options is able to open the specified DB successfully.
+
+### Public API Changes
+
+ * When options.db_write_buffer_size triggers, only the column family with the largest column family size will be flushed, not all the column families.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2016-07-26-rocksdb-4-8-released.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2016-07-26-rocksdb-4-8-released.markdown b/thirdparty/rocksdb/docs/_posts/2016-07-26-rocksdb-4-8-released.markdown
new file mode 100644
index 0000000..b42a66e
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2016-07-26-rocksdb-4-8-released.markdown
@@ -0,0 +1,48 @@
+---
+title: RocksDB 4.8 Released!
+layout: post
+author: yiwu
+category: blog
+redirect_from:
+ - /blog/3239/rocksdb-4-8-released/
+---
+
+## 4.8.0 (5/2/2016)
+
+### [](https://github.com/facebook/rocksdb/blob/master/HISTORY.md#public-api-change-1)Public API Change
+
+ * Allow preset compression dictionary for improved compression of block-based tables. This is supported for zlib, zstd, and lz4. The compression dictionary's size is configurable via CompressionOptions::max_dict_bytes.
+ * Delete deprecated classes for creating backups (BackupableDB) and restoring from backups (RestoreBackupableDB). Now, BackupEngine should be used for creating backups, and BackupEngineReadOnly should be used for restorations. For more details, see [https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F](https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F)
+ * Expose estimate of per-level compression ratio via DB property: "rocksdb.compression-ratio-at-levelN".
+ * Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
+
+### [](https://github.com/facebook/rocksdb/blob/master/HISTORY.md#new-features-2)New Features
+
+ * Add ReadOptions::readahead_size. If non-zero, NewIterator will create a new table reader which performs reads of the given size.
+
+<br/>
+
+<!--truncate-->
+
+## [](https://github.com/facebook/rocksdb/blob/master/HISTORY.md#470-482016)4.7.0 (4/8/2016)
+
+### [](https://github.com/facebook/rocksdb/blob/master/HISTORY.md#public-api-change-2)Public API Change
+
+ * rename options compaction_measure_io_stats to report_bg_io_stats and include flush too.
+ * Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See [https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File](https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File) for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.
+
+<br/>
+
+## [](https://github.com/facebook/rocksdb/blob/master/HISTORY.md#460-3102016)4.6.0 (3/10/2016)
+
+### [](https://github.com/facebook/rocksdb/blob/master/HISTORY.md#public-api-changes-1)Public API Changes
+
+ * Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier
+ * Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signature of Cache::Insert() is updated accordingly.
+ * Tickers [NUMBER_DB_NEXT, NUMBER_DB_PREV, NUMBER_DB_NEXT_FOUND, NUMBER_DB_PREV_FOUND, ITER_BYTES_READ] are not updated immediately. The are updated when the Iterator is deleted.
+ * Add monotonically increasing counter (DB property "rocksdb.current-super-version-number") that increments upon any change to the LSM tree.
+
+### [](https://github.com/facebook/rocksdb/blob/master/HISTORY.md#new-features-3)New Features
+
+ * Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification.
+ * Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned"
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2016-09-28-rocksdb-4-11-2-released.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2016-09-28-rocksdb-4-11-2-released.markdown b/thirdparty/rocksdb/docs/_posts/2016-09-28-rocksdb-4-11-2-released.markdown
new file mode 100644
index 0000000..87c20eb
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2016-09-28-rocksdb-4-11-2-released.markdown
@@ -0,0 +1,49 @@
+---
+title: RocksDB 4.11.2 Released!
+layout: post
+author: sdong
+category: blog
+---
+We abandoned release candidates 4.10.x and directly go to 4.11.2 from 4.9, to make sure the latest release is stable. In 4.11.2, we fixed several data corruption related bugs introduced in 4.9.0.
+
+## 4.11.2 (9/15/2016)
+
+### Bug fixes
+
+ * Segfault when failing to open an SST file for read-ahead iterators.
+ * WAL without data for all CFs is not deleted after recovery.
+
+<!--truncate-->
+
+## 4.11.1 (8/30/2016)
+
+### Bug Fixes
+
+ * Mitigate the regression bug of deadlock condition during recovery when options.max_successive_merges hits.
+ * Fix data race condition related to hash index in block based table when putting indexes in the block cache.
+
+## 4.11.0 (8/1/2016)
+
+### Public API Change
+
+ * options.memtable_prefix_bloom_huge_page_tlb_size => memtable_huge_page_size. When it is set, RocksDB will try to allocate memory from huge page for memtable too, rather than just memtable bloom filter.
+
+### New Features
+
+ * A tool to migrate DB after options change. See include/rocksdb/utilities/option_change_migration.h.
+ * Add ReadOptions.background_purge_on_iterator_cleanup. If true, we avoid file deletion when destorying iterators.
+
+## 4.10.0 (7/5/2016)
+
+### Public API Change
+
+ * options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes
+ * enum type CompressionType and PerfLevel changes from char to unsigned char. Value of all PerfLevel shift by one.
+ * Deprecate options.filter_deletes.
+
+### New Features
+
+ * Add avoid_flush_during_recovery option.
+ * Add a read option background_purge_on_iterator_cleanup to avoid deleting files in foreground when destroying iterators. Instead, a job is scheduled in high priority queue and would be executed in a separate background thread.
+ * RepairDB support for column families. RepairDB now associates data with non-default column families using information embedded in the SST/WAL files (4.7 or later). For data written by 4.6 or earlier, RepairDB associates it with the default column family.
+ * Add options.write_buffer_manager which allows users to control total memtable sizes across multiple DB instances.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2017-01-06-rocksdb-5-0-1-released.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2017-01-06-rocksdb-5-0-1-released.markdown b/thirdparty/rocksdb/docs/_posts/2017-01-06-rocksdb-5-0-1-released.markdown
new file mode 100644
index 0000000..fb04130
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2017-01-06-rocksdb-5-0-1-released.markdown
@@ -0,0 +1,26 @@
+---
+title: RocksDB 5.0.1 Released!
+layout: post
+author: yiwu
+category: blog
+---
+
+### Public API Change
+
+ * Options::max_bytes_for_level_multiplier is now a double along with all getters and setters.
+ * Support dynamically change `delayed_write_rate` and `max_total_wal_size` options via SetDBOptions().
+ * Introduce DB::DeleteRange for optimized deletion of large ranges of contiguous keys.
+ * Support dynamically change `delayed_write_rate` option via SetDBOptions().
+ * Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default.
+ * Remove Tickers::SEQUENCE_NUMBER to avoid confusion if statistics object is shared among RocksDB instance. Alternatively DB::GetLatestSequenceNumber() can be used to get the same value.
+ * Options.level0_stop_writes_trigger default value changes from 24 to 32.
+ * New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys.
+ * Removed flashcache support.
+ * DB::AddFile() is deprecated and is replaced with DB::IngestExternalFile(). DB::IngestExternalFile() remove all the restrictions that existed for DB::AddFile.
+
+### New Features
+
+ * Add avoid_flush_during_shutdown option, which speeds up DB shutdown by not flushing unpersisted data (i.e. with disableWAL = true). Unpersisted data will be lost. The options is dynamically changeable via SetDBOptions().
+ * Add memtable_insert_with_hint_prefix_extractor option. The option is mean to reduce CPU usage for inserting keys into memtable, if keys can be group by prefix and insert for each prefix are sequential or almost sequential. See include/rocksdb/options.h for more details.
+ * Add LuaCompactionFilter in utilities. This allows developers to write compaction filters in Lua. To use this feature, LUA_PATH needs to be set to the root directory of Lua.
+ * No longer populate "LATEST_BACKUP" file in backup directory, which formerly contained the number of the latest backup. The latest backup can be determined by finding the highest numbered file in the "meta/" subdirectory.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2017-02-07-rocksdb-5-1-2-released.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2017-02-07-rocksdb-5-1-2-released.markdown b/thirdparty/rocksdb/docs/_posts/2017-02-07-rocksdb-5-1-2-released.markdown
new file mode 100644
index 0000000..35bafb2
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2017-02-07-rocksdb-5-1-2-released.markdown
@@ -0,0 +1,15 @@
+---
+title: RocksDB 5.1.2 Released!
+layout: post
+author: maysamyabandeh
+category: blog
+---
+
+### Public API Change
+* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
+* Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully.
+* BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env.
+
+### Bug Fixes
+* Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions.
+* When file copying is needed when creating checkpoints or bulk loading files, fsync the file after the file copying.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2017-02-17-bulkoad-ingest-sst-file.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2017-02-17-bulkoad-ingest-sst-file.markdown b/thirdparty/rocksdb/docs/_posts/2017-02-17-bulkoad-ingest-sst-file.markdown
new file mode 100644
index 0000000..9a43a84
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2017-02-17-bulkoad-ingest-sst-file.markdown
@@ -0,0 +1,50 @@
+---
+title: Bulkloading by ingesting external SST files
+layout: post
+author: IslamAbdelRahman
+category: blog
+---
+
+## Introduction
+
+One of the basic operations of RocksDB is writing to RocksDB, Writes happen when user call (DB::Put, DB::Write, DB::Delete ... ), but what happens when you write to RocksDB ? .. this is a brief description of what happens.
+- User insert a new key/value by calling DB::Put() (or DB::Write())
+- We create a new entry for the new key/value in our in-memory structure (memtable / SkipList by default) and we assign it a new sequence number.
+- When the memtable exceeds a specific size (64 MB for example), we convert this memtable to a SST file, and put this file in level 0 of our LSM-Tree
+- Later, compaction will kick in and move data from level 0 to level 1, and then from level 1 to level 2 .. and so on
+
+But what if we can skip these steps and add data to the lowest possible level directly ? This is what bulk-loading does
+
+## Bulkloading
+
+- Write all of our keys and values into SST file outside of the DB
+- Add the SST file into the LSM directly
+
+This is bulk-loading, and in specific use-cases it allow users to achieve faster data loading and better write-amplification.
+
+and doing it is as simple as
+```cpp
+Options options;
+SstFileWriter sst_file_writer(EnvOptions(), options, options.comparator);
+Status s = sst_file_writer.Open(file_path);
+assert(s.ok());
+
+// Insert rows into the SST file, note that inserted keys must be
+// strictly increasing (based on options.comparator)
+for (...) {
+ s = sst_file_writer.Add(key, value);
+ assert(s.ok());
+}
+
+// Ingest the external SST file into the DB
+s = db_->IngestExternalFile({"/home/usr/file1.sst"}, IngestExternalFileOptions());
+assert(s.ok());
+```
+
+You can find more details about how to generate SST files and ingesting them into RocksDB in this [wiki page](https://github.com/facebook/rocksdb/wiki/Creating-and-Ingesting-SST-files)
+
+## Use cases
+There are multiple use cases where bulkloading could be useful, for example
+- Generating SST files in offline jobs in Hadoop, then downloading and ingesting the SST files into RocksDB
+- Migrating shards between machines by dumping key-range in SST File and loading the file in a different machine
+- Migrating from a different storage (InnoDB to RocksDB migration in MyRocks)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2017-03-02-rocksdb-5-2-1-released.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2017-03-02-rocksdb-5-2-1-released.markdown b/thirdparty/rocksdb/docs/_posts/2017-03-02-rocksdb-5-2-1-released.markdown
new file mode 100644
index 0000000..c6ce27d
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2017-03-02-rocksdb-5-2-1-released.markdown
@@ -0,0 +1,22 @@
+---
+title: RocksDB 5.2.1 Released!
+layout: post
+author: sdong
+category: blog
+---
+
+### Public API Change
+* NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explict provide one.
+* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
+
+### New Features
+* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations.
+* Added new function GetApproximateMemTableStats that approximates both number of records and size of memtables.
+* (Experimental) Two-level indexing that partition the index and creates a 2nd level index on the partitions. The feature can be enabled by setting kTwoLevelIndexSearch as IndexType and configuring index_per_partition.
+
+### Bug Fixes
+* RangeSync() should work if ROCKSDB_FALLOCATE_PRESENT is not set
+* Fix wrong results in a data race case in Get()
+* Some fixes related to 2PC.
+* Fix several bugs in Direct I/O supports.
+* Fix a regression bug which can cause Seek() to miss some keys if the return key has been updated many times after the snapshot which is used by the iterator.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2017-05-12-partitioned-index-filter.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2017-05-12-partitioned-index-filter.markdown b/thirdparty/rocksdb/docs/_posts/2017-05-12-partitioned-index-filter.markdown
new file mode 100644
index 0000000..fb4f62c
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2017-05-12-partitioned-index-filter.markdown
@@ -0,0 +1,34 @@
+---
+title: Partitioned Index/Filters
+layout: post
+author: maysamyabandeh
+category: blog
+---
+
+As DB/mem ratio gets larger, the memory footprint of filter/index blocks becomes non-trivial. Although `cache_index_and_filter_blocks` allows storing only a subset of them in block cache, their relatively large size negatively affects the performance by i) occupying the block cache space that could otherwise be used for caching data, ii) increasing the load on the disk storage by loading them into the cache after a miss. Here we illustrate these problems in more detail and explain how partitioning index/filters alleviates the overhead.
+
+### How large are the index/filter blocks?
+
+RocksDB has by default one index/filter block per SST file. The size of the index/filter varies based on the configuration but for a SST of size 256MB the index/filter block of size 0.5/5MB is typical, which is much larger than the typical data block size of 4-32KB. That is fine when all index/filters fit perfectly into memory and hence are read once per SST lifetime, not so much when they compete with data blocks for the block cache space and are also likely to be re-read many times from the disk.
+
+### What is the big deal with large index/filter blocks?
+
+When index/filter blocks are stored in block cache they are effectively competing with data blocks (as well as with each other) on this scarce resource. A filter of size 5MB is occupying the space that could otherwise be used to cache 1000s of data blocks (of size 4KB). This would result in more cache misses for data blocks. The large index/filters also kick each other out of the block cache more often and exacerbate their own cache miss rate too. This is while only a small part of the index/filter block might have been actually used during its lifetime in the cache.
+
+After the cache miss of an index/filter, it has to be reloaded from the disk, and its large size is not helping in reducing the IO cost. While a simple point lookup might need at most a couple of data block reads (of size 4KB) one from each layer of LSM, it might end up also loading multiple megabytes of index/filter blocks. If that happens often then the disk is spending more time serving index/filters rather than the actual data blocks.
+
+## What is partitioned index/filters?
+
+With partitioning, the index/filter of a SST file is partitioned into smaller blocks with an additional top-level index on them. When reading an index/filter, only top-level index is loaded into memory. The partitioned index/filter then uses the top-level index to load on demand into the block cache the partitions that are required to perform the index/filter query. The top-level index, which has much smaller memory footprint, can be stored in heap or block cache depending on the `cache_index_and_filter_blocks` setting.
+
+### Success stories
+
+#### HDD, 100TB DB
+
+In this example we have a DB of size 86G on HDD and emulate the small memory that is present to a node with 100TB of data by using direct IO (skipping OS file cache) and a very small block cache of size 60MB. Partitioning improves throughput by 11x from 5 op/s to 55 op/s.
+
+#### SSD, Linkbench
+
+In this example we have a DB of size 300G on SSD and emulate the small memory that would be available in presence of other DBs on the same node by by using direct IO (skipping OS file cache) and block cache of size 6G and 2G. Without partitioning the linkbench throughput drops from 38k tps to 23k when reducing block cache size from 6G to 2G. With partitioning the throughput drops from 38k to only 30k.
+
+Learn more (here)[https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters].
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2017-05-14-core-local-stats.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2017-05-14-core-local-stats.markdown b/thirdparty/rocksdb/docs/_posts/2017-05-14-core-local-stats.markdown
new file mode 100644
index 0000000..a806541
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2017-05-14-core-local-stats.markdown
@@ -0,0 +1,106 @@
+---
+title: Core-local Statistics
+layout: post
+author: ajkr
+category: blog
+---
+
+## Origins: Global Atomics
+
+Until RocksDB 4.12, ticker/histogram statistics were implemented with std::atomic values shared across the entire program. A ticker consists of a single atomic, while a histogram consists of several atomics to represent things like min/max/per-bucket counters. These statistics could be updated by all user/background threads.
+
+For concurrent/high-throughput workloads, cache line bouncing of atomics caused high CPU utilization. For example, we have tickers that count block cache hits and misses. Almost every user read increments these tickers a few times. Many concurrent user reads would cause the cache lines containing these atomics to bounce between cores.
+
+### Performance
+
+Here are perf results for 32 reader threads where most reads (99%+) are served by uncompressed block cache. Such a scenario stresses the statistics code heavily.
+
+Benchmark command: `TEST_TMPDIR=/dev/shm/ perf record -g ./db_bench -statistics -use_existing_db=true -benchmarks=readrandom -threads=32 -cache_size=1048576000 -num=1000000 -reads=1000000 && perf report -g --children`
+
+Perf snippet for "cycles" event:
+
+```
+ Children Self Command Shared Object Symbol
++ 30.33% 30.17% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick
++ 3.65% 0.98% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime
+```
+
+Perf snippet for "cache-misses" event:
+
+```
+ Children Self Command Shared Object Symbol
++ 19.54% 19.50% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick
++ 3.44% 0.57% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime
+```
+
+The high CPU overhead for updating tickers and histograms corresponds well to the high cache misses.
+
+## Thread-locals: Faster Updates
+
+Since RocksDB 4.12, ticker/histogram statistics use thread-local storage. Each thread has a local set of atomic values that no other thread can update. This prevents the cache line bouncing problem described above. Even though updates to a given value are always made by the same thread, atomics are still useful to synchronize with aggregations for querying statistics.
+
+Implementing this approach involved a couple challenges. First, each query for a statistic's global value must aggregate all threads' local values. This adds some overhead, which may pass unnoticed if statistics are queried infrequently. Second, exited threads' local values are still needed to provide accurate statistics. We handle this by merging a thread's local values into process-wide variables upon thread exit.
+
+### Performance
+
+Update benchmark setup is same as before. CPU overhead improved 7.8x compared to global atomics, corresponding to a 17.8x reduction in cache-misses overhead.
+
+Perf snippet for "cycles" event:
+
+```
+ Children Self Command Shared Object Symbol
++ 2.96% 0.87% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick
++ 1.37% 0.10% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime
+```
+
+Perf snippet for "cache-misses" event:
+
+```
+ Children Self Command Shared Object Symbol
++ 1.21% 0.65% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick
+ 0.08% 0.00% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime
+```
+
+To measure statistics query latency, we ran sysbench with 4K OLTP clients concurrently with one client that queries statistics repeatedly. Times shown are in milliseconds.
+
+```
+ min: 18.45
+ avg: 27.91
+ max: 231.65
+ 95th percentile: 55.82
+```
+
+## Core-locals: Faster Querying
+
+The thread-local approach is working well for applications calling RocksDB from only a few threads, or polling statistics infrequently. Eventually, though, we found use cases where those assumptions do not hold. For example, one application has per-connection threads and typically runs into performance issues when connection count grows very high. For debugging such issues, they want high-frequency statistics polling to correlate issues in their application with changes in RocksDB's state.
+
+Once [PR #2258](https://github.com/facebook/rocksdb/pull/2258) lands, ticker/histogram statistics will be local to each CPU core. Similarly to thread-local, each core updates only its local values, thus avoiding cache line bouncing. Local values are still atomics to make aggregation possible. With this change, query work depends only on number of cores, not the number of threads. So, applications with many more threads than cores can no longer impact statistics query latency.
+
+### Performance
+
+Update benchmark setup is same as before. CPU overhead worsened ~23% compared to thread-local, while cache performance was unchanged.
+
+Perf snippet for "cycles" event:
+
+```
+ Children Self Command Shared Object Symbol
++ 2.96% 0.87% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick
++ 1.37% 0.10% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime
+```
+
+Perf snippet for "cache-misses" event:
+
+```
+ Children Self Command Shared Object Symbol
++ 1.21% 0.65% db_bench db_bench [.] rocksdb::StatisticsImpl::recordTick
+ 0.08% 0.00% db_bench db_bench [.] rocksdb::StatisticsImpl::measureTime
+```
+
+Query latency is measured same as before with times in milliseconds. Average latency improved by 6.3x compared to thread-local.
+
+```
+ min: 2.47
+ avg: 4.45
+ max: 91.13
+ 95th percentile: 7.56
+```
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2017-05-26-rocksdb-5-4-5-released.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2017-05-26-rocksdb-5-4-5-released.markdown b/thirdparty/rocksdb/docs/_posts/2017-05-26-rocksdb-5-4-5-released.markdown
new file mode 100644
index 0000000..561dab4
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2017-05-26-rocksdb-5-4-5-released.markdown
@@ -0,0 +1,39 @@
+---
+title: RocksDB 5.4.5 Released!
+layout: post
+author: sagar0
+category: blog
+---
+
+### Public API Change
+* Support dynamically changing `stats_dump_period_sec` option via SetDBOptions().
+* Added ReadOptions::max_skippable_internal_keys to set a threshold to fail a request as incomplete when too many keys are being skipped while using iterators.
+* DB::Get in place of std::string accepts PinnableSlice, which avoids the extra memcpy of value to std::string in most of cases.
+ * PinnableSlice releases the pinned resources that contain the value when it is destructed or when ::Reset() is called on it.
+ * The old API that accepts std::string, although discouraged, is still supported.
+* Replace Options::use_direct_writes with Options::use_direct_io_for_flush_and_compaction. See Direct IO wiki for details.
+
+### New Features
+* Memtable flush can be avoided during checkpoint creation if total log file size is smaller than a threshold specified by the user.
+* Introduce level-based L0->L0 compactions to reduce file count, so write delays are incurred less often.
+* (Experimental) Partitioning filters which creates an index on the partitions. The feature can be enabled by setting partition_filters when using kFullFilter. Currently the feature also requires two-level indexing to be enabled. Number of partitions is the same as the number of partitions for indexes, which is controlled by metadata_block_size.
+* DB::ResetStats() to reset internal stats.
+* Added CompactionEventListener and EventListener::OnFlushBegin interfaces.
+* Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families.
+* Facility for cross-building RocksJava using Docker.
+
+### Bug Fixes
+* Fix WriteBatchWithIndex address use after scope error.
+* Fix WritableFile buffer size in direct IO.
+* Add prefetch to PosixRandomAccessFile in buffered io.
+* Fix PinnableSlice access invalid address when row cache is enabled.
+* Fix huge fallocate calls fail and make XFS unhappy.
+* Fix memory alignment with logical sector size.
+* Fix alignment in ReadaheadRandomAccessFile.
+* Fix bias with read amplification stats (READ_AMP_ESTIMATE_USEFUL_BYTES and READ_AMP_TOTAL_READ_BYTES).
+* Fix a manual / auto compaction data race.
+* Fix CentOS 5 cross-building of RocksJava.
+* Build and link with ZStd when creating the static RocksJava build.
+* Fix snprintf's usage to be cross-platform.
+* Fix build errors with blob DB.
+* Fix readamp test type inconsistency.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2017-06-26-17-level-based-changes.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2017-06-26-17-level-based-changes.markdown b/thirdparty/rocksdb/docs/_posts/2017-06-26-17-level-based-changes.markdown
new file mode 100644
index 0000000..9e838eb
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2017-06-26-17-level-based-changes.markdown
@@ -0,0 +1,60 @@
+---
+title: Level-based Compaction Changes
+layout: post
+author: ajkr
+category: blog
+---
+
+### Introduction
+
+RocksDB provides an option to limit the number of L0 files, which bounds read-amplification. Since L0 files (unlike files at lower levels) can span the entire key-range, a key might be in any file, thus reads need to check them one-by-one. Users often wish to configure a low limit to improve their read latency.
+
+Although, the mechanism with which we enforce L0's file count limit may be unappealing. When the limit is reached, RocksDB intentionally delays user writes. This slows down accumulation of files in L0, and frees up resources for compacting files down to lower levels. But adding delays will significantly increase user-visible write latency jitter.
+
+Also, due to how L0 files can span the entire key-range, compaction parallelization is limited. Files at L0 or L1 may be locked due to involvement in pending L0->L1 or L1->L2 compactions. We can only schedule a parallel L0->L1 compaction if it does not require any of the locked files, which is typically not the case.
+
+To handle these constraints better, we added a new type of compaction, L0->L0. It quickly reduces file count in L0 and can be scheduled even when L1 files are locked, unlike L0->L1. We also changed the L0->L1 picking algorithm to increase opportunities for parallelism.
+
+### Old L0->L1 Picking Logic
+
+Previously, our logic for picking which L0 file to compact was the same as every other level: pick the largest file in the level. One special property of L0->L1 compaction is that files can overlap in the input level, so those overlapping files must be pulled in as well. For example, a compaction may look like this:
+
+
+
+This compaction pulls in every L0 and L1 file. This happens regardless of which L0 file is initially chosen as each file overlaps with every other file.
+
+Users may insert their data less uniformly in the key-range. For example, a database may look like this during L0->L1 compaction:
+
+
+
+Let's say the third file from the top is the largest, and let's say the top two files are created after the compaction started. When the compaction is picked, the fourth L0 file and six rightmost L1 files are pulled in due to overlap. Notice this leaves the database in a state where we might not be able to schedule parallel compactions. For example, if the sixth file from the top is the next largest, we can't compact it because it overlaps with the top two files, which overlap with the locked L0 files.
+
+We can now see the high-level problems with this approach more clearly. First, locked files in L0 or L1 prevent us from parallelizing compactions. When locked files block L0->L1 compaction, there is nothing we can do to eliminate L0 files. Second, L0->L1 compactions are relatively slow. As we saw, when keys are uniformly distributed, L0->L1 compacts two entire levels. While this is happening, new files are being flushed to L0, advancing towards the file count limit.
+
+### New L0->L0 Algorithm
+
+We introduced compaction within L0 to improve both parallelization and speed of reducing L0 file count. An L0->L0 compaction may look like this:
+
+
+
+Say the L1->L2 compaction started first. Now L0->L1 is prevented by the locked L1 file. In this case, we compact files within L0. This allows us to start the work for eliminating L0 files earlier. It also lets us do less work since we don't pull in any L1 files, whereas L0->L1 compaction would've pulled in all of them. This lets us quickly reduce L0 file count to keep read-amp low while sustaining large bursts of writes (i.e., fast accumulation of L0 files).
+
+The tradeoff is this increases total compaction work, as we're now compacting files without contributing towards our eventual goal of moving them towards lower levels. Our benchmarks, though, consistently show less compaction stalls and improved write throughput. One justification is that L0 file data is highly likely in page cache and/or block cache due to it being recently written and frequently accessed. So, this type of compaction is relatively cheap compared to compactions at lower levels.
+
+This feature is available since RocksDB 5.4.
+
+### New L0->L1 Picking Logic
+
+Recall how the old L0->L1 picking algorithm chose the largest L0 file for compaction. This didn't fit well with L0->L0 compaction, which operates on a span of files. That span begins at the newest L0 file, and expands towards older files as long as they're not being compacted. Since the largest file may be anywhere, the old L0->L1 picking logic could arbitrarily prevent us from getting a long span of files. See the second illustration in this post for a scenario where this would happen.
+
+So, we changed the L0->L1 picking algorithm to start from the oldest file and expand towards newer files as long as they're not being compacted. For example:
+
+
+
+Now, there can never be L0 files unreachable for L0->L0 due to L0->L1 selecting files in the middle. When longer spans of files are available for L0->L0, we perform less compaction work per deleted L0 file, thus improving efficiency.
+
+This feature will be available in RocksDB 5.7.
+
+### Performance Changes
+
+Mark Callaghan did the most extensive benchmarking of this feature's impact on MyRocks. See his results [here](http://smalldatum.blogspot.com/2017/05/innodb-myrocks-and-tokudb-on-insert.html). Note the primary change between his March 17 and April 14 builds is the latter performs L0->L0 compaction.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2017-06-29-rocksdb-5-5-1-released.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2017-06-29-rocksdb-5-5-1-released.markdown b/thirdparty/rocksdb/docs/_posts/2017-06-29-rocksdb-5-5-1-released.markdown
new file mode 100644
index 0000000..d785608
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2017-06-29-rocksdb-5-5-1-released.markdown
@@ -0,0 +1,22 @@
+---
+title: RocksDB 5.5.1 Released!
+layout: post
+author: lightmark
+category: blog
+---
+
+### New Features
+* FIFO compaction to support Intra L0 compaction too with CompactionOptionsFIFO.allow_compaction=true.
+* Statistics::Reset() to reset user stats.
+* ldb add option --try_load_options, which will open DB with its own option file.
+* Introduce WriteBatch::PopSavePoint to pop the most recent save point explicitly.
+* Support dynamically change `max_open_files` option via SetDBOptions()
+* Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families.
+* Add debugging function `GetAllKeyVersions` to see internal versions of a range of keys.
+* Support file ingestion with universal compaction style
+* Support file ingestion behind with option `allow_ingest_behind`
+* New option enable_pipelined_write which may improve write throughput in case writing from multiple threads and WAL enabled.
+
+### Bug Fixes
+* Fix the bug that Direct I/O uses direct reads for non-SST file
+* Fix the bug that flush doesn't respond to fsync result
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2017-07-25-rocksdb-5-6-1-released.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2017-07-25-rocksdb-5-6-1-released.markdown b/thirdparty/rocksdb/docs/_posts/2017-07-25-rocksdb-5-6-1-released.markdown
new file mode 100644
index 0000000..3b54ffd
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2017-07-25-rocksdb-5-6-1-released.markdown
@@ -0,0 +1,22 @@
+---
+title: RocksDB 5.6.1 Released!
+layout: post
+author: yiwu
+category: blog
+---
+
+### Public API Change
+* Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads. See https://github.com/facebook/rocksdb/wiki/Thread-Pool for more details.
+* Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction.
+* options.delayed_write_rate by default take the value of options.rate_limiter rate.
+* Replace global variable `IOStatsContext iostats_context` with `IOStatsContext* get_iostats_context()`; replace global variable `PerfContext perf_context` with `PerfContext* get_perf_context()`.
+
+### New Features
+* Change ticker/histogram statistics implementations to use core-local storage. This improves aggregation speed compared to our previous thread-local approach, particularly for applications with many threads. See http://rocksdb.org/blog/2017/05/14/core-local-stats.html for more details.
+* Users can pass a cache object to write buffer manager, so that they can cap memory usage for memtable and block cache using one single limit.
+* Flush will be triggered when 7/8 of the limit introduced by write_buffer_manager or db_write_buffer_size is triggered, so that the hard threshold is hard to hit. See https://github.com/facebook/rocksdb/wiki/Write-Buffer-Manager for more details.
+* Introduce WriteOptions.low_pri. If it is true, low priority writes will be throttled if the compaction is behind. See https://github.com/facebook/rocksdb/wiki/Low-Priority-Write for more details.
+* `DB::IngestExternalFile()` now supports ingesting files into a database containing range deletions.
+
+### Bug Fixes
+* Shouldn't ignore return value of fsync() in flush.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_base.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_base.scss b/thirdparty/rocksdb/docs/_sass/_base.scss
new file mode 100644
index 0000000..6d26d9f
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_base.scss
@@ -0,0 +1,492 @@
+body {
+ background: $secondary-bg;
+ color: $text;
+ font: normal #{$base-font-size}/#{$base-line-height} $base-font-family;
+ height: 100vh;
+ text-align: left;
+ text-rendering: optimizeLegibility;
+}
+
+img {
+ max-width: 100%;
+}
+
+article {
+ p {
+ img {
+ max-width: 100%;
+ display:block;
+ margin-left: auto;
+ margin-right: auto;
+ }
+ }
+}
+
+a {
+ border-bottom: 1px dotted $primary-bg;
+ color: $text;
+ text-decoration: none;
+ -webkit-transition: background 0.3s, color 0.3s;
+ transition: background 0.3s, color 0.3s;
+}
+
+blockquote {
+ padding: 15px 30px 15px 15px;
+ margin: 20px 0 0 10px;
+ background-color: rgba(204, 122, 111, 0.1);
+ border-left: 10px solid rgba(191, 87, 73, 0.2);
+}
+
+#fb_oss a {
+ border: 0;
+}
+
+h1, h2, h3, h4 {
+ font-family: $header-font-family;
+ font-weight: 900;
+}
+
+.navPusher {
+ border-top: $header-height + $header-ptop + $header-pbot solid $primary-bg;
+ height: 100%;
+ left: 0;
+ position: relative;
+ z-index: 99;
+}
+
+.homeContainer {
+ background: $primary-bg;
+ color: $primary-overlay;
+
+ a {
+ color: $primary-overlay;
+ }
+
+ .homeSplashFade {
+ color: white;
+ }
+
+ .homeWrapper {
+ padding: 2em 10px;
+ text-align: left;
+
+ .wrapper {
+ margin: 0px auto;
+ max-width: $content-width;
+ padding: 0 20px;
+ }
+
+ .projectLogo {
+ img {
+ height: 100px;
+ margin-bottom: 0px;
+ }
+ }
+
+ h1#project_title {
+ font-family: $header-font-family;
+ font-size: 300%;
+ letter-spacing: -0.08em;
+ line-height: 1em;
+ margin-bottom: 80px;
+ }
+
+ h2#project_tagline {
+ font-family: $header-font-family;
+ font-size: 200%;
+ letter-spacing: -0.04em;
+ line-height: 1em;
+ }
+ }
+}
+
+.wrapper {
+ margin: 0px auto;
+ max-width: $content-width;
+ padding: 0 10px;
+}
+
+.projectLogo {
+ display: none;
+
+ img {
+ height: 100px;
+ margin-bottom: 0px;
+ }
+}
+
+section#intro {
+ margin: 40px 0;
+}
+
+.fbossFontLight {
+ font-family: $base-font-family;
+ font-weight: 300;
+ font-style: normal;
+}
+
+.fb-like {
+ display: block;
+ margin-bottom: 20px;
+ width: 100%;
+}
+
+.center {
+ display: block;
+ text-align: center;
+}
+
+.mainContainer {
+ background: $secondary-bg;
+ overflow: auto;
+
+ .mainWrapper {
+ padding: 4vh 10px;
+ text-align: left;
+
+ .allShareBlock {
+ padding: 10px 0;
+
+ .pluginBlock {
+ margin: 12px 0;
+ padding: 0;
+ }
+ }
+
+ a {
+ &:hover,
+ &:focus {
+ background: $primary-bg;
+ color: $primary-overlay;
+ }
+ }
+
+ em, i {
+ font-style: italic;
+ }
+
+ strong, b {
+ font-weight: bold;
+ }
+
+ h1 {
+ font-size: 300%;
+ line-height: 1em;
+ padding: 1.4em 0 1em;
+ text-align: left;
+ }
+
+ h2 {
+ font-size: 250%;
+ line-height: 1em;
+ margin-bottom: 20px;
+ padding: 1.4em 0 20px;
+ text-align: left;
+
+ & {
+ border-bottom: 1px solid darken($primary-bg, 10%);
+ color: darken($primary-bg, 10%);
+ font-size: 22px;
+ padding: 10px 0;
+ }
+
+ &.blockHeader {
+ border-bottom: 1px solid white;
+ color: white;
+ font-size: 22px;
+ margin-bottom: 20px;
+ padding: 10px 0;
+ }
+ }
+
+ h3 {
+ font-size: 150%;
+ line-height: 1.2em;
+ padding: 1em 0 0.8em;
+ }
+
+ h4 {
+ font-size: 130%;
+ line-height: 1.2em;
+ padding: 1em 0 0.8em;
+ }
+
+ p {
+ padding: 0.8em 0;
+ }
+
+ ul {
+ list-style: disc;
+ }
+
+ ol, ul {
+ padding-left: 24px;
+ li {
+ padding-bottom: 4px;
+ padding-left: 6px;
+ }
+ }
+
+ strong {
+ font-weight: bold;
+ }
+
+ .post {
+ position: relative;
+
+ .katex {
+ font-weight: 700;
+ }
+
+ &.basicPost {
+ margin-top: 30px;
+ }
+
+ a {
+ color: $primary-bg;
+
+ &:hover,
+ &:focus {
+ color: #fff;
+ }
+ }
+
+ h2 {
+ border-bottom: 4px solid $primary-bg;
+ font-size: 130%;
+ }
+
+ h3 {
+ border-bottom: 1px solid $primary-bg;
+ font-size: 110%;
+ }
+
+ ol {
+ list-style: decimal outside none;
+ }
+
+ .post-header {
+ padding: 1em 0;
+
+ h1 {
+ font-size: 150%;
+ line-height: 1em;
+ padding: 0.4em 0 0;
+
+ a {
+ border: none;
+ }
+ }
+
+ .post-meta {
+ color: $primary-bg;
+ font-family: $header-font-family;
+ text-align: center;
+ }
+ }
+
+ .postSocialPlugins {
+ padding-top: 1em;
+ }
+
+ .docPagination {
+ background: $primary-bg;
+ bottom: 0px;
+ left: 0px;
+ position: absolute;
+ right: 0px;
+
+ .pager {
+ display: inline-block;
+ width: 50%;
+ }
+
+ .pagingNext {
+ float: right;
+ text-align: right;
+ }
+
+ a {
+ border: none;
+ color: $primary-overlay;
+ display: block;
+ padding: 4px 12px;
+
+ &:hover {
+ background-color: $secondary-bg;
+ color: $text;
+ }
+
+ .pagerLabel {
+ display: inline;
+ }
+
+ .pagerTitle {
+ display: none;
+ }
+ }
+ }
+ }
+
+ .posts {
+ .post {
+ margin-bottom: 6vh;
+ }
+ }
+ }
+}
+
+#integrations_title {
+ font-size: 250%;
+ margin: 80px 0;
+}
+
+.ytVideo {
+ height: 0;
+ overflow: hidden;
+ padding-bottom: 53.4%; /* 16:9 */
+ padding-top: 25px;
+ position: relative;
+}
+
+.ytVideo iframe,
+.ytVideo object,
+.ytVideo embed {
+ height: 100%;
+ left: 0;
+ position: absolute;
+ top: 0;
+ width: 100%;
+}
+
+@media only screen and (min-width: 480px) {
+ h1#project_title {
+ font-size: 500%;
+ }
+
+ h2#project_tagline {
+ font-size: 250%;
+ }
+
+ .projectLogo {
+ img {
+ margin-bottom: 10px;
+ height: 200px;
+ }
+ }
+
+ .homeContainer .homeWrapper {
+ padding-left: 10px;
+ padding-right: 10px;
+ }
+
+ .mainContainer {
+ .mainWrapper {
+ .post {
+ h2 {
+ font-size: 180%;
+ }
+
+ h3 {
+ font-size: 120%;
+ }
+
+ .docPagination {
+ a {
+ .pagerLabel {
+ display: none;
+ }
+ .pagerTitle {
+ display: inline;
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+@media only screen and (min-width: 900px) {
+ .homeContainer {
+ .homeWrapper {
+ position: relative;
+
+ #inner {
+ box-sizing: border-box;
+ max-width: 600px;
+ padding-right: 40px;
+ }
+
+ .projectLogo {
+ align-items: center;
+ bottom: 0;
+ display: flex;
+ justify-content: flex-end;
+ left: 0;
+ padding: 2em 20px 4em;
+ position: absolute;
+ right: 20px;
+ top: 0;
+
+ img {
+ height: 100%;
+ max-height: 250px;
+ }
+ }
+ }
+ }
+}
+
+@media only screen and (min-width: 1024px) {
+ .mainContainer {
+ .mainWrapper {
+ .post {
+ box-sizing: border-box;
+ display: block;
+
+ .post-header {
+ h1 {
+ font-size: 250%;
+ }
+ }
+ }
+
+ .posts {
+ .post {
+ margin-bottom: 4vh;
+ width: 100%;
+ }
+ }
+ }
+ }
+}
+
+@media only screen and (min-width: 1200px) {
+ .homeContainer {
+ .homeWrapper {
+ #inner {
+ max-width: 750px;
+ }
+ }
+ }
+
+ .wrapper {
+ max-width: 1100px;
+ }
+}
+
+@media only screen and (min-width: 1500px) {
+ .homeContainer {
+ .homeWrapper {
+ #inner {
+ max-width: 1100px;
+ padding-bottom: 40px;
+ padding-top: 40px;
+ }
+ }
+ }
+
+ .wrapper {
+ max-width: 1400px;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_blog.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_blog.scss b/thirdparty/rocksdb/docs/_sass/_blog.scss
new file mode 100644
index 0000000..74335d1
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_blog.scss
@@ -0,0 +1,45 @@
+.blogContainer {
+ .posts {
+ margin-top: 60px;
+
+ .post {
+ border: 1px solid $primary-bg;
+ border-radius: 3px;
+ padding: 10px 20px 20px;
+ }
+ }
+
+ .lonePost {
+ margin-top: 60px;
+
+ .post {
+ padding: 10px 0px 0px;
+ }
+ }
+
+ .post-header {
+ h1 {
+ text-align: center;
+ }
+
+ .post-authorName {
+ color: rgba($text, 0.7);
+ font-size: 14px;
+ font-weight: 900;
+ margin-top: 0;
+ padding: 0;
+ text-align: center;
+ }
+
+ .authorPhoto {
+ border-radius: 50%;
+ height: 50px;
+ left: 50%;
+ margin-left: -25px;
+ overflow: hidden;
+ position: absolute;
+ top: -25px;
+ width: 50px;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_buttons.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_buttons.scss b/thirdparty/rocksdb/docs/_sass/_buttons.scss
new file mode 100644
index 0000000..a037161
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_buttons.scss
@@ -0,0 +1,47 @@
+.button {
+ border: 1px solid $primary-bg;
+ border-radius: 3px;
+ color: $primary-bg;
+ display: inline-block;
+ font-size: 14px;
+ font-weight: 900;
+ line-height: 1.2em;
+ padding: 10px;
+ text-transform: uppercase;
+ transition: background 0.3s, color 0.3s;
+
+ &:hover {
+ background: $primary-bg;
+ color: $primary-overlay;
+ }
+}
+
+.homeContainer {
+ .button {
+ border-color: $primary-overlay;
+ border-width: 1px;
+ color: $primary-overlay;
+
+ &:hover {
+ background: $primary-overlay;
+ color: $primary-bg;
+ }
+ }
+}
+
+.blockButton {
+ display: block;
+}
+
+.edit-page-link {
+ float: right;
+ font-size: 14px;
+ font-weight: normal;
+ line-height: 20px;
+ opacity: 0.6;
+ transition: opacity 0.5s;
+}
+
+.edit-page-link:hover {
+ opacity: 1;
+}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_footer.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_footer.scss b/thirdparty/rocksdb/docs/_sass/_footer.scss
new file mode 100644
index 0000000..5b74395
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_footer.scss
@@ -0,0 +1,82 @@
+.footerContainer {
+ background: $secondary-bg;
+ color: $primary-bg;
+ overflow: hidden;
+ padding: 0 10px;
+ text-align: left;
+
+ .footerWrapper {
+ border-top: 1px solid $primary-bg;
+ padding: 0;
+
+ .footerBlocks {
+ align-items: center;
+ align-content: center;
+ display: flex;
+ flex-flow: row wrap;
+ margin: 0 -20px;
+ padding: 10px 0;
+ }
+
+ .footerSection {
+ box-sizing: border-box;
+ flex: 1 1 25%;
+ font-size: 14px;
+ min-width: 275px;
+ padding: 0px 20px;
+
+ a {
+ border: 0;
+ color: inherit;
+ display: inline-block;
+ line-height: 1.2em;
+ }
+
+ .footerLink {
+ padding-right: 20px;
+ }
+ }
+
+ .fbOpenSourceFooter {
+ align-items: center;
+ display: flex;
+ flex-flow: row nowrap;
+ max-width: 25%;
+
+ .facebookOSSLogoSvg {
+ flex: 0 0 31px;
+ height: 30px;
+ margin-right: 10px;
+ width: 31px;
+
+ path {
+ fill: $primary-bg;
+ }
+
+ .middleRing {
+ opacity: 0.7;
+ }
+
+ .innerRing {
+ opacity: 0.45;
+ }
+ }
+
+ h2 {
+ display: block;
+ font-weight: 900;
+ line-height: 1em;
+ }
+ }
+ }
+}
+
+@media only screen and (min-width: 900px) {
+ .footerSection {
+ &.rightAlign {
+ margin-left: auto;
+ max-width: 25%;
+ text-align: right;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_gridBlock.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_gridBlock.scss b/thirdparty/rocksdb/docs/_sass/_gridBlock.scss
new file mode 100644
index 0000000..679b31c
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_gridBlock.scss
@@ -0,0 +1,115 @@
+.gridBlock {
+ margin: -5px 0;
+ padding: 0;
+ padding-bottom: 20px;
+
+ .blockElement {
+ padding: 5px 0;
+
+ img {
+ max-width: 100%;
+ }
+
+ h3 {
+ border-bottom: 1px solid rgba($primary-bg, 0.5);
+ color: $primary-bg;
+ font-size: 18px;
+ margin: 0;
+ padding: 10px 0;
+ }
+ }
+
+ .gridClear {
+ clear: both;
+ }
+
+}
+
+.gridBlock .alignCenter {
+ text-align: center;
+}
+.gridBlock .alignRight {
+ text-align: right;
+}
+.gridBlock .imageAlignSide {
+ align-items: center;
+ display: flex;
+ flex-flow: row wrap;
+}
+.blockImage {
+ max-width: 150px;
+ width: 50%;
+}
+.imageAlignTop .blockImage {
+ margin-bottom: 20px;
+}
+.imageAlignTop.alignCenter .blockImage {
+ margin-left: auto;
+ margin-right: auto;
+}
+.imageAlignSide .blockImage {
+ flex: 0 1 100px;
+ margin-right: 20px;
+}
+.imageAlignSide .blockContent {
+ flex: 1 1;
+}
+
+@media only screen and (max-width: 1023px) {
+ .responsiveList .blockContent {
+ position: relative;
+ }
+ .responsiveList .blockContent > div {
+ padding-left: 20px;
+ }
+ .responsiveList .blockContent::before {
+ content: "\2022";
+ position: absolute;
+ }
+}
+
+@media only screen and (min-width: 1024px) {
+ .gridBlock {
+ display: flex;
+ flex-direction: row;
+ flex-wrap: wrap;
+ margin: -10px -10px 10px -10px;
+
+ .twoByGridBlock {
+ box-sizing: border-box;
+ flex: 1 0 50%;
+ padding: 10px;
+ }
+
+ .fourByGridBlock {
+ box-sizing: border-box;
+ flex: 1 0 25%;
+ padding: 10px;
+ }
+ }
+
+ h2 + .gridBlock {
+ padding-top: 20px;
+ }
+}
+
+@media only screen and (min-width: 1400px) {
+ .gridBlock {
+ display: flex;
+ flex-direction: row;
+ flex-wrap: wrap;
+ margin: -10px -20px 10px -20px;
+
+ .twoByGridBlock {
+ box-sizing: border-box;
+ flex: 1 0 50%;
+ padding: 10px 20px;
+ }
+
+ .fourByGridBlock {
+ box-sizing: border-box;
+ flex: 1 0 25%;
+ padding: 10px 20px;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_header.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_header.scss b/thirdparty/rocksdb/docs/_sass/_header.scss
new file mode 100644
index 0000000..b4cd071
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_header.scss
@@ -0,0 +1,138 @@
+.fixedHeaderContainer {
+ background: $primary-bg;
+ color: $primary-overlay;
+ height: $header-height;
+ padding: $header-ptop 0 $header-pbot;
+ position: fixed;
+ width: 100%;
+ z-index: 9999;
+
+ a {
+ align-items: center;
+ border: 0;
+ color: $primary-overlay;
+ display: flex;
+ flex-flow: row nowrap;
+ height: $header-height;
+ }
+
+ header {
+ display: flex;
+ flex-flow: row nowrap;
+ position: relative;
+ text-align: left;
+
+ img {
+ height: 24px;
+ margin-right: 10px;
+ }
+
+ h2 {
+ display: block;
+ font-family: $header-font-family;
+ font-weight: 900;
+ line-height: 18px;
+ position: relative;
+ }
+ }
+}
+
+.navigationFull {
+ height: 34px;
+ margin-left: auto;
+
+ nav {
+ position: relative;
+
+ ul {
+ display: flex;
+ flex-flow: row nowrap;
+ margin: 0 -10px;
+
+ li {
+ padding: 0 10px;
+ display: block;
+
+ a {
+ border: 0;
+ color: $primary-overlay-special;
+ font-size: 16px;
+ font-weight: 400;
+ line-height: 1.2em;
+
+ &:hover {
+ border-bottom: 2px solid $primary-overlay;
+ color: $primary-overlay;
+ }
+ }
+
+ &.navItemActive {
+ a {
+ color: $primary-overlay;
+ }
+ }
+ }
+ }
+ }
+}
+
+/* 900px
+
+
+ .fixedHeaderContainer {
+ .navigationWrapper {
+ nav {
+ padding: 0 1em;
+ position: relative;
+ top: -9px;
+
+ ul {
+ margin: 0 -0.4em;
+ li {
+ display: inline-block;
+
+ a {
+ padding: 14px 0.4em;
+ border: 0;
+ color: $primary-overlay-special;
+ display: inline-block;
+
+ &:hover {
+ color: $primary-overlay;
+ }
+ }
+
+ &.navItemActive {
+ a {
+ color: $primary-overlay;
+ }
+ }
+ }
+ }
+ }
+
+ &.navigationFull {
+ display: inline-block;
+ }
+
+ &.navigationSlider {
+ display: none;
+ }
+ }
+ }
+
+ 1200px
+
+ .fixedHeaderContainer {
+ header {
+ max-width: 1100px;
+ }
+ }
+
+ 1500px
+ .fixedHeaderContainer {
+ header {
+ max-width: 1400px;
+ }
+ }
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_poweredby.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_poweredby.scss b/thirdparty/rocksdb/docs/_sass/_poweredby.scss
new file mode 100644
index 0000000..4155b60
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_poweredby.scss
@@ -0,0 +1,69 @@
+.poweredByContainer {
+ background: $primary-bg;
+ color: $primary-overlay;
+ margin-bottom: 20px;
+
+ a {
+ color: $primary-overlay;
+ }
+
+ .poweredByWrapper {
+ h2 {
+ border-color: $primary-overlay-special;
+ color: $primary-overlay-special;
+ }
+ }
+
+ .poweredByMessage {
+ color: $primary-overlay-special;
+ font-size: 14px;
+ padding-top: 20px;
+ }
+}
+
+.poweredByItems {
+ display: flex;
+ flex-flow: row wrap;
+ margin: 0 -10px;
+}
+
+.poweredByItem {
+ box-sizing: border-box;
+ flex: 1 0 50%;
+ line-height: 1.1em;
+ padding: 5px 10px;
+
+ &.itemLarge {
+ flex-basis: 100%;
+ padding: 10px;
+ text-align: center;
+
+ &:nth-child(4) {
+ padding-bottom: 20px;
+ }
+
+ img {
+ max-height: 30px;
+ }
+ }
+}
+
+@media only screen and (min-width: 480px) {
+ .itemLarge {
+ flex-basis: 50%;
+ max-width: 50%;
+ }
+}
+
+@media only screen and (min-width: 1024px) {
+ .poweredByItem {
+ flex-basis: 25%;
+ max-width: 25%;
+
+ &.itemLarge {
+ padding-bottom: 20px;
+ text-align: left;
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_promo.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_promo.scss b/thirdparty/rocksdb/docs/_sass/_promo.scss
new file mode 100644
index 0000000..8c9a809
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_promo.scss
@@ -0,0 +1,55 @@
+.promoSection {
+ display: flex;
+ flex-flow: column wrap;
+ font-size: 125%;
+ line-height: 1.6em;
+ margin: -10px 0;
+ position: relative;
+ z-index: 99;
+
+ .promoRow {
+ padding: 10px 0;
+
+ .pluginWrapper {
+ display: block;
+
+ &.ghWatchWrapper, &.ghStarWrapper {
+ height: 28px;
+ }
+ }
+
+ .pluginRowBlock {
+ display: flex;
+ flex-flow: row wrap;
+ margin: 0 -2px;
+
+ .pluginWrapper {
+ padding: 0 2px;
+ }
+ }
+ }
+}
+
+iframe.pluginIframe {
+ height: 500px;
+ margin-top: 20px;
+ width: 100%;
+}
+
+.iframeContent {
+ display: none;
+}
+
+.iframePreview {
+ display: inline-block;
+ margin-top: 20px;
+}
+
+@media only screen and (min-width: 1024px) {
+ .iframeContent {
+ display: block;
+ }
+ .iframePreview {
+ display: none;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_react_docs_nav.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_react_docs_nav.scss b/thirdparty/rocksdb/docs/_sass/_react_docs_nav.scss
new file mode 100644
index 0000000..f0a651e
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_react_docs_nav.scss
@@ -0,0 +1,332 @@
+.docsNavContainer {
+ background: $sidenav;
+ height: 35px;
+ left: 0;
+ position: fixed;
+ width: 100%;
+ z-index: 100;
+}
+
+.docMainWrapper {
+ .wrapper {
+ &.mainWrapper {
+ padding-left: 0;
+ padding-right: 0;
+ padding-top: 10px;
+ }
+ }
+}
+
+.docsSliderActive {
+ .docsNavContainer {
+ box-sizing: border-box;
+ height: 100%;
+ overflow-y: auto;
+ -webkit-overflow-scrolling: touch;
+ padding-bottom: 50px;
+ }
+
+ .mainContainer {
+ display: none;
+ }
+}
+
+.navBreadcrumb {
+ box-sizing: border-box;
+ display: flex;
+ flex-flow: row nowrap;
+ font-size: 12px;
+ height: 35px;
+ overflow: hidden;
+ padding: 5px 10px;
+
+ a, span {
+ border: 0;
+ color: $sidenav-text;
+ }
+
+ i {
+ padding: 0 3px;
+ }
+}
+
+nav.toc {
+ position: relative;
+
+ section {
+ padding: 0px;
+ position: relative;
+
+ .navGroups {
+ display: none;
+ padding: 40px 10px 10px;
+ }
+ }
+
+ .toggleNav {
+ background: $sidenav;
+ color: $sidenav-text;
+ position: relative;
+ transition: background-color 0.3s, color 0.3s;
+
+ .navToggle {
+ cursor: pointer;
+ height: 24px;
+ margin-right: 10px;
+ position: relative;
+ text-align: left;
+ width: 18px;
+
+ &::before, &::after {
+ content: "";
+ position: absolute;
+ top: 50%;
+ left: 0;
+ left: 8px;
+ width: 3px;
+ height: 6px;
+ border: 5px solid $sidenav-text;
+ border-width: 5px 0;
+ margin-top: -8px;
+ transform: rotate(45deg);
+ z-index: 1;
+ }
+
+ &::after {
+ transform: rotate(-45deg);
+ }
+
+ i {
+ &::before, &::after {
+ content: "";
+ position: absolute;
+ top: 50%;
+ left: 2px;
+ background: transparent;
+ border-width: 0 5px 5px;
+ border-style: solid;
+ border-color: transparent $sidenav-text;
+ height: 0;
+ margin-top: -7px;
+ opacity: 1;
+ width: 5px;
+ z-index: 10;
+ }
+
+ &::after {
+ border-width: 5px 5px 0;
+ margin-top: 2px;
+ }
+ }
+ }
+
+ .navGroup {
+ background: $sidenav-overlay;
+ margin: 1px 0;
+
+ ul {
+ display: none;
+ }
+
+ h3 {
+ background: $sidenav-overlay;
+ color: $sidenav-text;
+ cursor: pointer;
+ font-size: 14px;
+ font-weight: 400;
+ line-height: 1.2em;
+ padding: 10px;
+ transition: color 0.2s;
+
+ i:not(:empty) {
+ width: 16px;
+ height: 16px;
+ display: inline-block;
+ box-sizing: border-box;
+ text-align: center;
+ color: rgba($sidenav-text, 0.5);
+ margin-right: 10px;
+ transition: color 0.2s;
+ }
+
+ &:hover {
+ color: $primary-bg;
+
+ i:not(:empty) {
+ color: $primary-bg;
+ }
+ }
+ }
+
+ &.navGroupActive {
+ background: $sidenav-active;
+ color: $sidenav-text;
+
+ ul {
+ display: block;
+ padding-bottom: 10px;
+ padding-top: 10px;
+ }
+
+ h3 {
+ background: $primary-bg;
+ color: $primary-overlay;
+
+ i {
+ display: none;
+ }
+ }
+ }
+ }
+
+ ul {
+ padding-left: 0;
+ padding-right: 24px;
+
+ li {
+ list-style-type: none;
+ padding-bottom: 0;
+ padding-left: 0;
+
+ a {
+ border: none;
+ color: $sidenav-text;
+ display: inline-block;
+ font-size: 14px;
+ line-height: 1.1em;
+ margin: 2px 10px 5px;
+ padding: 5px 0 2px;
+ transition: color 0.3s;
+
+ &:hover,
+ &:focus {
+ color: $primary-bg;
+ }
+
+ &.navItemActive {
+ color: $primary-bg;
+ font-weight: 900;
+ }
+ }
+ }
+ }
+ }
+
+ .toggleNavActive {
+ .navBreadcrumb {
+ background: $sidenav;
+ margin-bottom: 20px;
+ position: fixed;
+ width: 100%;
+ }
+
+ section {
+ .navGroups {
+ display: block;
+ }
+ }
+
+
+ .navToggle {
+ &::before, &::after {
+ border-width: 6px 0;
+ height: 0px;
+ margin-top: -6px;
+ }
+
+ i {
+ opacity: 0;
+ }
+ }
+ }
+}
+
+.docsNavVisible {
+ .navPusher {
+ .mainContainer {
+ padding-top: 35px;
+ }
+ }
+}
+
+@media only screen and (min-width: 900px) {
+ .navBreadcrumb {
+ padding: 5px 0;
+ }
+
+ nav.toc {
+ section {
+ .navGroups {
+ padding: 40px 0 0;
+ }
+ }
+ }
+}
+
+@media only screen and (min-width: 1024px) {
+ .navToggle {
+ display: none;
+ }
+
+ .docsSliderActive {
+ .mainContainer {
+ display: block;
+ }
+ }
+
+ .docsNavVisible {
+ .navPusher {
+ .mainContainer {
+ padding-top: 0;
+ }
+ }
+ }
+
+ .docsNavContainer {
+ background: none;
+ box-sizing: border-box;
+ height: auto;
+ margin: 40px 40px 0 0;
+ overflow-y: auto;
+ position: relative;
+ width: 300px;
+ }
+
+ nav.toc {
+ section {
+ .navGroups {
+ display: block;
+ padding-top: 0px;
+ }
+ }
+
+ .toggleNavActive {
+ .navBreadcrumb {
+ margin-bottom: 0;
+ position: relative;
+ }
+ }
+ }
+
+ .docMainWrapper {
+ display: flex;
+ flex-flow: row nowrap;
+ margin-bottom: 40px;
+
+ .wrapper {
+ padding-left: 0;
+ padding-right: 0;
+
+ &.mainWrapper {
+ padding-top: 0;
+ }
+ }
+ }
+
+ .navBreadcrumb {
+ display: none;
+ h2 {
+ padding: 0 10px;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_react_header_nav.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_react_header_nav.scss b/thirdparty/rocksdb/docs/_sass/_react_header_nav.scss
new file mode 100644
index 0000000..13c0e56
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_react_header_nav.scss
@@ -0,0 +1,141 @@
+.navigationFull {
+ display: none;
+}
+
+.navigationSlider {
+ position: absolute;
+ right: 0px;
+
+ .navSlideout {
+ cursor: pointer;
+ padding-top: 4px;
+ position: absolute;
+ right: 10px;
+ top: 0;
+ transition: top 0.3s;
+ z-index: 101;
+ }
+
+ .slidingNav {
+ background: $secondary-bg;
+ box-sizing: border-box;
+ height: 0px;
+ overflow-x: hidden;
+ padding: 0;
+ position: absolute;
+ right: 0px;
+ top: 0;
+ transition: height 0.3s cubic-bezier(0.68, -0.55, 0.265, 1.55), width 0.3s cubic-bezier(0.68, -0.55, 0.265, 1.55);
+ width: 0;
+
+ ul {
+ flex-flow: column nowrap;
+ list-style: none;
+ padding: 10px;
+
+ li {
+ margin: 0;
+ padding: 2px 0;
+
+ a {
+ color: $primary-bg;
+ display: inline;
+ margin: 3px 5px;
+ padding: 2px 0px;
+ transition: background-color 0.3s;
+
+ &:focus,
+ &:hover {
+ border-bottom: 2px solid $primary-bg;
+ }
+ }
+ }
+ }
+ }
+
+ .navSlideoutActive {
+ .slidingNav {
+ height: auto;
+ padding-top: $header-height + $header-pbot;
+ width: 300px;
+ }
+
+ .navSlideout {
+ top: -2px;
+ .menuExpand {
+ span:nth-child(1) {
+ background-color: $text;
+ top: 16px;
+ transform: rotate(45deg);
+ }
+ span:nth-child(2) {
+ opacity: 0;
+ }
+ span:nth-child(3) {
+ background-color: $text;
+ transform: rotate(-45deg);
+ }
+ }
+ }
+ }
+}
+
+.menuExpand {
+ display: flex;
+ flex-flow: column nowrap;
+ height: 20px;
+ justify-content: space-between;
+
+ span {
+ background: $primary-overlay;
+ border-radius: 3px;
+ display: block;
+ flex: 0 0 4px;
+ height: 4px;
+ position: relative;
+ top: 0;
+ transition: background-color 0.3s, top 0.3s, opacity 0.3s, transform 0.3s;
+ width: 20px;
+ }
+}
+
+.navPusher {
+ border-top: $header-height + $header-ptop + $header-pbot solid $primary-bg;
+ position: relative;
+ left: 0;
+ z-index: 99;
+ height: 100%;
+
+ &::after {
+ position: absolute;
+ top: 0;
+ right: 0;
+ width: 0;
+ height: 0;
+ background: rgba(0,0,0,0.4);
+ content: '';
+ opacity: 0;
+ -webkit-transition: opacity 0.5s, width 0.1s 0.5s, height 0.1s 0.5s;
+ transition: opacity 0.5s, width 0.1s 0.5s, height 0.1s 0.5s;
+ }
+
+ .sliderActive &::after {
+ width: 100%;
+ height: 100%;
+ opacity: 1;
+ -webkit-transition: opacity 0.5s;
+ transition: opacity 0.5s;
+ z-index: 100;
+ }
+}
+
+
+@media only screen and (min-width: 1024px) {
+ .navigationFull {
+ display: block;
+ }
+
+ .navigationSlider {
+ display: none;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_reset.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_reset.scss b/thirdparty/rocksdb/docs/_sass/_reset.scss
new file mode 100644
index 0000000..0e5f2e0
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_reset.scss
@@ -0,0 +1,43 @@
+html, body, div, span, applet, object, iframe,
+h1, h2, h3, h4, h5, h6, p, blockquote, pre,
+a, abbr, acronym, address, big, cite, code,
+del, dfn, em, img, ins, kbd, q, s, samp,
+small, strike, strong, sub, sup, tt, var,
+b, u, i, center,
+dl, dt, dd, ol, ul, li,
+fieldset, form, label, legend,
+table, caption, tbody, tfoot, thead, tr, th, td,
+article, aside, canvas, details, embed,
+figure, figcaption, footer, header, hgroup,
+menu, nav, output, ruby, section, summary,
+time, mark, audio, video {
+ margin: 0;
+ padding: 0;
+ border: 0;
+ font-size: 100%;
+ font: inherit;
+ vertical-align: baseline;
+}
+/* HTML5 display-role reset for older browsers */
+article, aside, details, figcaption, figure,
+footer, header, hgroup, menu, nav, section {
+ display: block;
+}
+body {
+ line-height: 1;
+}
+ol, ul {
+ list-style: none;
+}
+blockquote, q {
+ quotes: none;
+}
+blockquote:before, blockquote:after,
+q:before, q:after {
+ content: '';
+ content: none;
+}
+table {
+ border-collapse: collapse;
+ border-spacing: 0;
+}
[28/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/column_family.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/column_family.cc b/thirdparty/rocksdb/db/column_family.cc
new file mode 100644
index 0000000..b00eda0
--- /dev/null
+++ b/thirdparty/rocksdb/db/column_family.cc
@@ -0,0 +1,1137 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/column_family.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <vector>
+#include <string>
+#include <algorithm>
+#include <limits>
+
+#include "db/compaction_picker.h"
+#include "db/compaction_picker_universal.h"
+#include "db/db_impl.h"
+#include "db/internal_stats.h"
+#include "db/job_context.h"
+#include "db/table_properties_collector.h"
+#include "db/version_set.h"
+#include "db/write_controller.h"
+#include "memtable/hash_skiplist_rep.h"
+#include "monitoring/thread_status_util.h"
+#include "options/options_helper.h"
+#include "table/block_based_table_factory.h"
+#include "util/autovector.h"
+#include "util/compression.h"
+
+namespace rocksdb {
+
+ColumnFamilyHandleImpl::ColumnFamilyHandleImpl(
+ ColumnFamilyData* column_family_data, DBImpl* db, InstrumentedMutex* mutex)
+ : cfd_(column_family_data), db_(db), mutex_(mutex) {
+ if (cfd_ != nullptr) {
+ cfd_->Ref();
+ }
+}
+
+ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
+ if (cfd_ != nullptr) {
+#ifndef ROCKSDB_LITE
+ for (auto& listener : cfd_->ioptions()->listeners) {
+ listener->OnColumnFamilyHandleDeletionStarted(this);
+ }
+#endif // ROCKSDB_LITE
+ // Job id == 0 means that this is not our background process, but rather
+ // user thread
+ JobContext job_context(0);
+ mutex_->Lock();
+ if (cfd_->Unref()) {
+ delete cfd_;
+ }
+ db_->FindObsoleteFiles(&job_context, false, true);
+ mutex_->Unlock();
+ if (job_context.HaveSomethingToDelete()) {
+ db_->PurgeObsoleteFiles(job_context);
+ }
+ job_context.Clean();
+ }
+}
+
+uint32_t ColumnFamilyHandleImpl::GetID() const { return cfd()->GetID(); }
+
+const std::string& ColumnFamilyHandleImpl::GetName() const {
+ return cfd()->GetName();
+}
+
+Status ColumnFamilyHandleImpl::GetDescriptor(ColumnFamilyDescriptor* desc) {
+#ifndef ROCKSDB_LITE
+ // accessing mutable cf-options requires db mutex.
+ InstrumentedMutexLock l(mutex_);
+ *desc = ColumnFamilyDescriptor(cfd()->GetName(), cfd()->GetLatestCFOptions());
+ return Status::OK();
+#else
+ return Status::NotSupported();
+#endif // !ROCKSDB_LITE
+}
+
+const Comparator* ColumnFamilyHandleImpl::GetComparator() const {
+ return cfd()->user_comparator();
+}
+
+void GetIntTblPropCollectorFactory(
+ const ImmutableCFOptions& ioptions,
+ std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+ int_tbl_prop_collector_factories) {
+ auto& collector_factories = ioptions.table_properties_collector_factories;
+ for (size_t i = 0; i < ioptions.table_properties_collector_factories.size();
+ ++i) {
+ assert(collector_factories[i]);
+ int_tbl_prop_collector_factories->emplace_back(
+ new UserKeyTablePropertiesCollectorFactory(collector_factories[i]));
+ }
+ // Add collector to collect internal key statistics
+ int_tbl_prop_collector_factories->emplace_back(
+ new InternalKeyPropertiesCollectorFactory);
+}
+
+Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) {
+ if (!cf_options.compression_per_level.empty()) {
+ for (size_t level = 0; level < cf_options.compression_per_level.size();
+ ++level) {
+ if (!CompressionTypeSupported(cf_options.compression_per_level[level])) {
+ return Status::InvalidArgument(
+ "Compression type " +
+ CompressionTypeToString(cf_options.compression_per_level[level]) +
+ " is not linked with the binary.");
+ }
+ }
+ } else {
+ if (!CompressionTypeSupported(cf_options.compression)) {
+ return Status::InvalidArgument(
+ "Compression type " +
+ CompressionTypeToString(cf_options.compression) +
+ " is not linked with the binary.");
+ }
+ }
+ return Status::OK();
+}
+
+Status CheckConcurrentWritesSupported(const ColumnFamilyOptions& cf_options) {
+ if (cf_options.inplace_update_support) {
+ return Status::InvalidArgument(
+ "In-place memtable updates (inplace_update_support) is not compatible "
+ "with concurrent writes (allow_concurrent_memtable_write)");
+ }
+ if (!cf_options.memtable_factory->IsInsertConcurrentlySupported()) {
+ return Status::InvalidArgument(
+ "Memtable doesn't concurrent writes (allow_concurrent_memtable_write)");
+ }
+ return Status::OK();
+}
+
+ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
+ const ColumnFamilyOptions& src) {
+ ColumnFamilyOptions result = src;
+ size_t clamp_max = std::conditional<
+ sizeof(size_t) == 4, std::integral_constant<size_t, 0xffffffff>,
+ std::integral_constant<uint64_t, 64ull << 30>>::type::value;
+ ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, clamp_max);
+ // if user sets arena_block_size, we trust user to use this value. Otherwise,
+ // calculate a proper value from writer_buffer_size;
+ if (result.arena_block_size <= 0) {
+ result.arena_block_size = result.write_buffer_size / 8;
+
+ // Align up to 4k
+ const size_t align = 4 * 1024;
+ result.arena_block_size =
+ ((result.arena_block_size + align - 1) / align) * align;
+ }
+ result.min_write_buffer_number_to_merge =
+ std::min(result.min_write_buffer_number_to_merge,
+ result.max_write_buffer_number - 1);
+ if (result.min_write_buffer_number_to_merge < 1) {
+ result.min_write_buffer_number_to_merge = 1;
+ }
+
+ if (result.num_levels < 1) {
+ result.num_levels = 1;
+ }
+ if (result.compaction_style == kCompactionStyleLevel &&
+ result.num_levels < 2) {
+ result.num_levels = 2;
+ }
+
+ if (result.compaction_style == kCompactionStyleUniversal &&
+ db_options.allow_ingest_behind && result.num_levels < 3) {
+ result.num_levels = 3;
+ }
+
+ if (result.max_write_buffer_number < 2) {
+ result.max_write_buffer_number = 2;
+ }
+ if (result.max_write_buffer_number_to_maintain < 0) {
+ result.max_write_buffer_number_to_maintain = result.max_write_buffer_number;
+ }
+ // bloom filter size shouldn't exceed 1/4 of memtable size.
+ if (result.memtable_prefix_bloom_size_ratio > 0.25) {
+ result.memtable_prefix_bloom_size_ratio = 0.25;
+ } else if (result.memtable_prefix_bloom_size_ratio < 0) {
+ result.memtable_prefix_bloom_size_ratio = 0;
+ }
+
+ if (!result.prefix_extractor) {
+ assert(result.memtable_factory);
+ Slice name = result.memtable_factory->Name();
+ if (name.compare("HashSkipListRepFactory") == 0 ||
+ name.compare("HashLinkListRepFactory") == 0) {
+ result.memtable_factory = std::make_shared<SkipListFactory>();
+ }
+ }
+
+ if (result.compaction_style == kCompactionStyleFIFO) {
+ result.num_levels = 1;
+ // since we delete level0 files in FIFO compaction when there are too many
+ // of them, these options don't really mean anything
+ result.level0_slowdown_writes_trigger = std::numeric_limits<int>::max();
+ result.level0_stop_writes_trigger = std::numeric_limits<int>::max();
+ }
+
+ if (result.max_bytes_for_level_multiplier <= 0) {
+ result.max_bytes_for_level_multiplier = 1;
+ }
+
+ if (result.level0_file_num_compaction_trigger == 0) {
+ ROCKS_LOG_WARN(db_options.info_log.get(),
+ "level0_file_num_compaction_trigger cannot be 0");
+ result.level0_file_num_compaction_trigger = 1;
+ }
+
+ if (result.level0_stop_writes_trigger <
+ result.level0_slowdown_writes_trigger ||
+ result.level0_slowdown_writes_trigger <
+ result.level0_file_num_compaction_trigger) {
+ ROCKS_LOG_WARN(db_options.info_log.get(),
+ "This condition must be satisfied: "
+ "level0_stop_writes_trigger(%d) >= "
+ "level0_slowdown_writes_trigger(%d) >= "
+ "level0_file_num_compaction_trigger(%d)",
+ result.level0_stop_writes_trigger,
+ result.level0_slowdown_writes_trigger,
+ result.level0_file_num_compaction_trigger);
+ if (result.level0_slowdown_writes_trigger <
+ result.level0_file_num_compaction_trigger) {
+ result.level0_slowdown_writes_trigger =
+ result.level0_file_num_compaction_trigger;
+ }
+ if (result.level0_stop_writes_trigger <
+ result.level0_slowdown_writes_trigger) {
+ result.level0_stop_writes_trigger = result.level0_slowdown_writes_trigger;
+ }
+ ROCKS_LOG_WARN(db_options.info_log.get(),
+ "Adjust the value to "
+ "level0_stop_writes_trigger(%d)"
+ "level0_slowdown_writes_trigger(%d)"
+ "level0_file_num_compaction_trigger(%d)",
+ result.level0_stop_writes_trigger,
+ result.level0_slowdown_writes_trigger,
+ result.level0_file_num_compaction_trigger);
+ }
+
+ if (result.soft_pending_compaction_bytes_limit == 0) {
+ result.soft_pending_compaction_bytes_limit =
+ result.hard_pending_compaction_bytes_limit;
+ } else if (result.hard_pending_compaction_bytes_limit > 0 &&
+ result.soft_pending_compaction_bytes_limit >
+ result.hard_pending_compaction_bytes_limit) {
+ result.soft_pending_compaction_bytes_limit =
+ result.hard_pending_compaction_bytes_limit;
+ }
+
+ if (result.level_compaction_dynamic_level_bytes) {
+ if (result.compaction_style != kCompactionStyleLevel ||
+ db_options.db_paths.size() > 1U) {
+ // 1. level_compaction_dynamic_level_bytes only makes sense for
+ // level-based compaction.
+ // 2. we don't yet know how to make both of this feature and multiple
+ // DB path work.
+ result.level_compaction_dynamic_level_bytes = false;
+ }
+ }
+
+ if (result.max_compaction_bytes == 0) {
+ result.max_compaction_bytes = result.target_file_size_base * 25;
+ }
+
+ return result;
+}
+
+int SuperVersion::dummy = 0;
+void* const SuperVersion::kSVInUse = &SuperVersion::dummy;
+void* const SuperVersion::kSVObsolete = nullptr;
+
+SuperVersion::~SuperVersion() {
+ for (auto td : to_delete) {
+ delete td;
+ }
+}
+
+SuperVersion* SuperVersion::Ref() {
+ refs.fetch_add(1, std::memory_order_relaxed);
+ return this;
+}
+
+bool SuperVersion::Unref() {
+ // fetch_sub returns the previous value of ref
+ uint32_t previous_refs = refs.fetch_sub(1);
+ assert(previous_refs > 0);
+ return previous_refs == 1;
+}
+
+void SuperVersion::Cleanup() {
+ assert(refs.load(std::memory_order_relaxed) == 0);
+ imm->Unref(&to_delete);
+ MemTable* m = mem->Unref();
+ if (m != nullptr) {
+ auto* memory_usage = current->cfd()->imm()->current_memory_usage();
+ assert(*memory_usage >= m->ApproximateMemoryUsage());
+ *memory_usage -= m->ApproximateMemoryUsage();
+ to_delete.push_back(m);
+ }
+ current->Unref();
+}
+
+void SuperVersion::Init(MemTable* new_mem, MemTableListVersion* new_imm,
+ Version* new_current) {
+ mem = new_mem;
+ imm = new_imm;
+ current = new_current;
+ mem->Ref();
+ imm->Ref();
+ current->Ref();
+ refs.store(1, std::memory_order_relaxed);
+}
+
+namespace {
+void SuperVersionUnrefHandle(void* ptr) {
+ // UnrefHandle is called when a thread exists or a ThreadLocalPtr gets
+ // destroyed. When former happens, the thread shouldn't see kSVInUse.
+ // When latter happens, we are in ~ColumnFamilyData(), no get should happen as
+ // well.
+ SuperVersion* sv = static_cast<SuperVersion*>(ptr);
+ if (sv->Unref()) {
+ sv->db_mutex->Lock();
+ sv->Cleanup();
+ sv->db_mutex->Unlock();
+ delete sv;
+ }
+}
+} // anonymous namespace
+
+ColumnFamilyData::ColumnFamilyData(
+ uint32_t id, const std::string& name, Version* _dummy_versions,
+ Cache* _table_cache, WriteBufferManager* write_buffer_manager,
+ const ColumnFamilyOptions& cf_options, const ImmutableDBOptions& db_options,
+ const EnvOptions& env_options, ColumnFamilySet* column_family_set)
+ : id_(id),
+ name_(name),
+ dummy_versions_(_dummy_versions),
+ current_(nullptr),
+ refs_(0),
+ initialized_(false),
+ dropped_(false),
+ internal_comparator_(cf_options.comparator),
+ initial_cf_options_(SanitizeOptions(db_options, cf_options)),
+ ioptions_(db_options, initial_cf_options_),
+ mutable_cf_options_(initial_cf_options_),
+ is_delete_range_supported_(
+ cf_options.table_factory->IsDeleteRangeSupported()),
+ write_buffer_manager_(write_buffer_manager),
+ mem_(nullptr),
+ imm_(ioptions_.min_write_buffer_number_to_merge,
+ ioptions_.max_write_buffer_number_to_maintain),
+ super_version_(nullptr),
+ super_version_number_(0),
+ local_sv_(new ThreadLocalPtr(&SuperVersionUnrefHandle)),
+ next_(nullptr),
+ prev_(nullptr),
+ log_number_(0),
+ column_family_set_(column_family_set),
+ pending_flush_(false),
+ pending_compaction_(false),
+ prev_compaction_needed_bytes_(0),
+ allow_2pc_(db_options.allow_2pc) {
+ Ref();
+
+ // Convert user defined table properties collector factories to internal ones.
+ GetIntTblPropCollectorFactory(ioptions_, &int_tbl_prop_collector_factories_);
+
+ // if _dummy_versions is nullptr, then this is a dummy column family.
+ if (_dummy_versions != nullptr) {
+ internal_stats_.reset(
+ new InternalStats(ioptions_.num_levels, db_options.env, this));
+ table_cache_.reset(new TableCache(ioptions_, env_options, _table_cache));
+ if (ioptions_.compaction_style == kCompactionStyleLevel) {
+ compaction_picker_.reset(
+ new LevelCompactionPicker(ioptions_, &internal_comparator_));
+#ifndef ROCKSDB_LITE
+ } else if (ioptions_.compaction_style == kCompactionStyleUniversal) {
+ compaction_picker_.reset(
+ new UniversalCompactionPicker(ioptions_, &internal_comparator_));
+ } else if (ioptions_.compaction_style == kCompactionStyleFIFO) {
+ compaction_picker_.reset(
+ new FIFOCompactionPicker(ioptions_, &internal_comparator_));
+ } else if (ioptions_.compaction_style == kCompactionStyleNone) {
+ compaction_picker_.reset(new NullCompactionPicker(
+ ioptions_, &internal_comparator_));
+ ROCKS_LOG_WARN(ioptions_.info_log,
+ "Column family %s does not use any background compaction. "
+ "Compactions can only be done via CompactFiles\n",
+ GetName().c_str());
+#endif // !ROCKSDB_LITE
+ } else {
+ ROCKS_LOG_ERROR(ioptions_.info_log,
+ "Unable to recognize the specified compaction style %d. "
+ "Column family %s will use kCompactionStyleLevel.\n",
+ ioptions_.compaction_style, GetName().c_str());
+ compaction_picker_.reset(
+ new LevelCompactionPicker(ioptions_, &internal_comparator_));
+ }
+
+ if (column_family_set_->NumberOfColumnFamilies() < 10) {
+ ROCKS_LOG_INFO(ioptions_.info_log,
+ "--------------- Options for column family [%s]:\n",
+ name.c_str());
+ initial_cf_options_.Dump(ioptions_.info_log);
+ } else {
+ ROCKS_LOG_INFO(ioptions_.info_log, "\t(skipping printing options)\n");
+ }
+ }
+
+ RecalculateWriteStallConditions(mutable_cf_options_);
+}
+
+// DB mutex held
+ColumnFamilyData::~ColumnFamilyData() {
+ assert(refs_.load(std::memory_order_relaxed) == 0);
+ // remove from linked list
+ auto prev = prev_;
+ auto next = next_;
+ prev->next_ = next;
+ next->prev_ = prev;
+
+ if (!dropped_ && column_family_set_ != nullptr) {
+ // If it's dropped, it's already removed from column family set
+ // If column_family_set_ == nullptr, this is dummy CFD and not in
+ // ColumnFamilySet
+ column_family_set_->RemoveColumnFamily(this);
+ }
+
+ if (current_ != nullptr) {
+ current_->Unref();
+ }
+
+ // It would be wrong if this ColumnFamilyData is in flush_queue_ or
+ // compaction_queue_ and we destroyed it
+ assert(!pending_flush_);
+ assert(!pending_compaction_);
+
+ if (super_version_ != nullptr) {
+ // Release SuperVersion reference kept in ThreadLocalPtr.
+ // This must be done outside of mutex_ since unref handler can lock mutex.
+ super_version_->db_mutex->Unlock();
+ local_sv_.reset();
+ super_version_->db_mutex->Lock();
+
+ bool is_last_reference __attribute__((unused));
+ is_last_reference = super_version_->Unref();
+ assert(is_last_reference);
+ super_version_->Cleanup();
+ delete super_version_;
+ super_version_ = nullptr;
+ }
+
+ if (dummy_versions_ != nullptr) {
+ // List must be empty
+ assert(dummy_versions_->TEST_Next() == dummy_versions_);
+ bool deleted __attribute__((unused)) = dummy_versions_->Unref();
+ assert(deleted);
+ }
+
+ if (mem_ != nullptr) {
+ delete mem_->Unref();
+ }
+ autovector<MemTable*> to_delete;
+ imm_.current()->Unref(&to_delete);
+ for (MemTable* m : to_delete) {
+ delete m;
+ }
+}
+
+void ColumnFamilyData::SetDropped() {
+ // can't drop default CF
+ assert(id_ != 0);
+ dropped_ = true;
+ write_controller_token_.reset();
+
+ // remove from column_family_set
+ column_family_set_->RemoveColumnFamily(this);
+}
+
+ColumnFamilyOptions ColumnFamilyData::GetLatestCFOptions() const {
+ return BuildColumnFamilyOptions(initial_cf_options_, mutable_cf_options_);
+}
+
+uint64_t ColumnFamilyData::OldestLogToKeep() {
+ auto current_log = GetLogNumber();
+
+ if (allow_2pc_) {
+ auto imm_prep_log = imm()->GetMinLogContainingPrepSection();
+ auto mem_prep_log = mem()->GetMinLogContainingPrepSection();
+
+ if (imm_prep_log > 0 && imm_prep_log < current_log) {
+ current_log = imm_prep_log;
+ }
+
+ if (mem_prep_log > 0 && mem_prep_log < current_log) {
+ current_log = mem_prep_log;
+ }
+ }
+
+ return current_log;
+}
+
+const double kIncSlowdownRatio = 0.8;
+const double kDecSlowdownRatio = 1 / kIncSlowdownRatio;
+const double kNearStopSlowdownRatio = 0.6;
+const double kDelayRecoverSlowdownRatio = 1.4;
+
+namespace {
+// If penalize_stop is true, we further reduce slowdown rate.
+std::unique_ptr<WriteControllerToken> SetupDelay(
+ WriteController* write_controller, uint64_t compaction_needed_bytes,
+ uint64_t prev_compaction_need_bytes, bool penalize_stop,
+ bool auto_comapctions_disabled) {
+ const uint64_t kMinWriteRate = 16 * 1024u; // Minimum write rate 16KB/s.
+
+ uint64_t max_write_rate = write_controller->max_delayed_write_rate();
+ uint64_t write_rate = write_controller->delayed_write_rate();
+
+ if (auto_comapctions_disabled) {
+ // When auto compaction is disabled, always use the value user gave.
+ write_rate = max_write_rate;
+ } else if (write_controller->NeedsDelay() && max_write_rate > kMinWriteRate) {
+ // If user gives rate less than kMinWriteRate, don't adjust it.
+ //
+ // If already delayed, need to adjust based on previous compaction debt.
+ // When there are two or more column families require delay, we always
+ // increase or reduce write rate based on information for one single
+ // column family. It is likely to be OK but we can improve if there is a
+ // problem.
+ // Ignore compaction_needed_bytes = 0 case because compaction_needed_bytes
+ // is only available in level-based compaction
+ //
+ // If the compaction debt stays the same as previously, we also further slow
+ // down. It usually means a mem table is full. It's mainly for the case
+ // where both of flush and compaction are much slower than the speed we
+ // insert to mem tables, so we need to actively slow down before we get
+ // feedback signal from compaction and flushes to avoid the full stop
+ // because of hitting the max write buffer number.
+ //
+ // If DB just falled into the stop condition, we need to further reduce
+ // the write rate to avoid the stop condition.
+ if (penalize_stop) {
+ // Penalize the near stop or stop condition by more aggressive slowdown.
+ // This is to provide the long term slowdown increase signal.
+ // The penalty is more than the reward of recovering to the normal
+ // condition.
+ write_rate = static_cast<uint64_t>(static_cast<double>(write_rate) *
+ kNearStopSlowdownRatio);
+ if (write_rate < kMinWriteRate) {
+ write_rate = kMinWriteRate;
+ }
+ } else if (prev_compaction_need_bytes > 0 &&
+ prev_compaction_need_bytes <= compaction_needed_bytes) {
+ write_rate = static_cast<uint64_t>(static_cast<double>(write_rate) *
+ kIncSlowdownRatio);
+ if (write_rate < kMinWriteRate) {
+ write_rate = kMinWriteRate;
+ }
+ } else if (prev_compaction_need_bytes > compaction_needed_bytes) {
+ // We are speeding up by ratio of kSlowdownRatio when we have paid
+ // compaction debt. But we'll never speed up to faster than the write rate
+ // given by users.
+ write_rate = static_cast<uint64_t>(static_cast<double>(write_rate) *
+ kDecSlowdownRatio);
+ if (write_rate > max_write_rate) {
+ write_rate = max_write_rate;
+ }
+ }
+ }
+ return write_controller->GetDelayToken(write_rate);
+}
+
+int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
+ int level0_slowdown_writes_trigger) {
+ // SanitizeOptions() ensures it.
+ assert(level0_file_num_compaction_trigger <= level0_slowdown_writes_trigger);
+
+ if (level0_file_num_compaction_trigger < 0) {
+ return std::numeric_limits<int>::max();
+ }
+
+ const int64_t twice_level0_trigger =
+ static_cast<int64_t>(level0_file_num_compaction_trigger) * 2;
+
+ const int64_t one_fourth_trigger_slowdown =
+ static_cast<int64_t>(level0_file_num_compaction_trigger) +
+ ((level0_slowdown_writes_trigger - level0_file_num_compaction_trigger) /
+ 4);
+
+ assert(twice_level0_trigger >= 0);
+ assert(one_fourth_trigger_slowdown >= 0);
+
+ // 1/4 of the way between L0 compaction trigger threshold and slowdown
+ // condition.
+ // Or twice as compaction trigger, if it is smaller.
+ int64_t res = std::min(twice_level0_trigger, one_fourth_trigger_slowdown);
+ if (res >= port::kMaxInt32) {
+ return port::kMaxInt32;
+ } else {
+ // res fits in int
+ return static_cast<int>(res);
+ }
+}
+} // namespace
+
+void ColumnFamilyData::RecalculateWriteStallConditions(
+ const MutableCFOptions& mutable_cf_options) {
+ if (current_ != nullptr) {
+ auto* vstorage = current_->storage_info();
+ auto write_controller = column_family_set_->write_controller_;
+ uint64_t compaction_needed_bytes =
+ vstorage->estimated_compaction_needed_bytes();
+
+ bool was_stopped = write_controller->IsStopped();
+ bool needed_delay = write_controller->NeedsDelay();
+
+ if (imm()->NumNotFlushed() >= mutable_cf_options.max_write_buffer_number) {
+ write_controller_token_ = write_controller->GetStopToken();
+ internal_stats_->AddCFStats(InternalStats::MEMTABLE_COMPACTION, 1);
+ ROCKS_LOG_WARN(
+ ioptions_.info_log,
+ "[%s] Stopping writes because we have %d immutable memtables "
+ "(waiting for flush), max_write_buffer_number is set to %d",
+ name_.c_str(), imm()->NumNotFlushed(),
+ mutable_cf_options.max_write_buffer_number);
+ } else if (!mutable_cf_options.disable_auto_compactions &&
+ vstorage->l0_delay_trigger_count() >=
+ mutable_cf_options.level0_stop_writes_trigger) {
+ write_controller_token_ = write_controller->GetStopToken();
+ internal_stats_->AddCFStats(InternalStats::LEVEL0_NUM_FILES_TOTAL, 1);
+ if (compaction_picker_->IsLevel0CompactionInProgress()) {
+ internal_stats_->AddCFStats(
+ InternalStats::LEVEL0_NUM_FILES_WITH_COMPACTION, 1);
+ }
+ ROCKS_LOG_WARN(ioptions_.info_log,
+ "[%s] Stopping writes because we have %d level-0 files",
+ name_.c_str(), vstorage->l0_delay_trigger_count());
+ } else if (!mutable_cf_options.disable_auto_compactions &&
+ mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
+ compaction_needed_bytes >=
+ mutable_cf_options.hard_pending_compaction_bytes_limit) {
+ write_controller_token_ = write_controller->GetStopToken();
+ internal_stats_->AddCFStats(
+ InternalStats::HARD_PENDING_COMPACTION_BYTES_LIMIT, 1);
+ ROCKS_LOG_WARN(
+ ioptions_.info_log,
+ "[%s] Stopping writes because of estimated pending compaction "
+ "bytes %" PRIu64,
+ name_.c_str(), compaction_needed_bytes);
+ } else if (mutable_cf_options.max_write_buffer_number > 3 &&
+ imm()->NumNotFlushed() >=
+ mutable_cf_options.max_write_buffer_number - 1) {
+ write_controller_token_ =
+ SetupDelay(write_controller, compaction_needed_bytes,
+ prev_compaction_needed_bytes_, was_stopped,
+ mutable_cf_options.disable_auto_compactions);
+ internal_stats_->AddCFStats(InternalStats::MEMTABLE_SLOWDOWN, 1);
+ ROCKS_LOG_WARN(
+ ioptions_.info_log,
+ "[%s] Stalling writes because we have %d immutable memtables "
+ "(waiting for flush), max_write_buffer_number is set to %d "
+ "rate %" PRIu64,
+ name_.c_str(), imm()->NumNotFlushed(),
+ mutable_cf_options.max_write_buffer_number,
+ write_controller->delayed_write_rate());
+ } else if (!mutable_cf_options.disable_auto_compactions &&
+ mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
+ vstorage->l0_delay_trigger_count() >=
+ mutable_cf_options.level0_slowdown_writes_trigger) {
+ // L0 is the last two files from stopping.
+ bool near_stop = vstorage->l0_delay_trigger_count() >=
+ mutable_cf_options.level0_stop_writes_trigger - 2;
+ write_controller_token_ =
+ SetupDelay(write_controller, compaction_needed_bytes,
+ prev_compaction_needed_bytes_, was_stopped || near_stop,
+ mutable_cf_options.disable_auto_compactions);
+ internal_stats_->AddCFStats(InternalStats::LEVEL0_SLOWDOWN_TOTAL, 1);
+ if (compaction_picker_->IsLevel0CompactionInProgress()) {
+ internal_stats_->AddCFStats(
+ InternalStats::LEVEL0_SLOWDOWN_WITH_COMPACTION, 1);
+ }
+ ROCKS_LOG_WARN(ioptions_.info_log,
+ "[%s] Stalling writes because we have %d level-0 files "
+ "rate %" PRIu64,
+ name_.c_str(), vstorage->l0_delay_trigger_count(),
+ write_controller->delayed_write_rate());
+ } else if (!mutable_cf_options.disable_auto_compactions &&
+ mutable_cf_options.soft_pending_compaction_bytes_limit > 0 &&
+ vstorage->estimated_compaction_needed_bytes() >=
+ mutable_cf_options.soft_pending_compaction_bytes_limit) {
+ // If the distance to hard limit is less than 1/4 of the gap between soft
+ // and
+ // hard bytes limit, we think it is near stop and speed up the slowdown.
+ bool near_stop =
+ mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
+ (compaction_needed_bytes -
+ mutable_cf_options.soft_pending_compaction_bytes_limit) >
+ 3 * (mutable_cf_options.hard_pending_compaction_bytes_limit -
+ mutable_cf_options.soft_pending_compaction_bytes_limit) /
+ 4;
+
+ write_controller_token_ =
+ SetupDelay(write_controller, compaction_needed_bytes,
+ prev_compaction_needed_bytes_, was_stopped || near_stop,
+ mutable_cf_options.disable_auto_compactions);
+ internal_stats_->AddCFStats(
+ InternalStats::SOFT_PENDING_COMPACTION_BYTES_LIMIT, 1);
+ ROCKS_LOG_WARN(
+ ioptions_.info_log,
+ "[%s] Stalling writes because of estimated pending compaction "
+ "bytes %" PRIu64 " rate %" PRIu64,
+ name_.c_str(), vstorage->estimated_compaction_needed_bytes(),
+ write_controller->delayed_write_rate());
+ } else {
+ if (vstorage->l0_delay_trigger_count() >=
+ GetL0ThresholdSpeedupCompaction(
+ mutable_cf_options.level0_file_num_compaction_trigger,
+ mutable_cf_options.level0_slowdown_writes_trigger)) {
+ write_controller_token_ =
+ write_controller->GetCompactionPressureToken();
+ ROCKS_LOG_INFO(
+ ioptions_.info_log,
+ "[%s] Increasing compaction threads because we have %d level-0 "
+ "files ",
+ name_.c_str(), vstorage->l0_delay_trigger_count());
+ } else if (vstorage->estimated_compaction_needed_bytes() >=
+ mutable_cf_options.soft_pending_compaction_bytes_limit / 4) {
+ // Increase compaction threads if bytes needed for compaction exceeds
+ // 1/4 of threshold for slowing down.
+ // If soft pending compaction byte limit is not set, always speed up
+ // compaction.
+ write_controller_token_ =
+ write_controller->GetCompactionPressureToken();
+ if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) {
+ ROCKS_LOG_INFO(
+ ioptions_.info_log,
+ "[%s] Increasing compaction threads because of estimated pending "
+ "compaction "
+ "bytes %" PRIu64,
+ name_.c_str(), vstorage->estimated_compaction_needed_bytes());
+ }
+ } else {
+ write_controller_token_.reset();
+ }
+ // If the DB recovers from delay conditions, we reward with reducing
+ // double the slowdown ratio. This is to balance the long term slowdown
+ // increase signal.
+ if (needed_delay) {
+ uint64_t write_rate = write_controller->delayed_write_rate();
+ write_controller->set_delayed_write_rate(static_cast<uint64_t>(
+ static_cast<double>(write_rate) * kDelayRecoverSlowdownRatio));
+ // Set the low pri limit to be 1/4 the delayed write rate.
+ // Note we don't reset this value even after delay condition is relased.
+ // Low-pri rate will continue to apply if there is a compaction
+ // pressure.
+ write_controller->low_pri_rate_limiter()->SetBytesPerSecond(write_rate /
+ 4);
+ }
+ }
+ prev_compaction_needed_bytes_ = compaction_needed_bytes;
+ }
+}
+
+const EnvOptions* ColumnFamilyData::soptions() const {
+ return &(column_family_set_->env_options_);
+}
+
+void ColumnFamilyData::SetCurrent(Version* current_version) {
+ current_ = current_version;
+}
+
+uint64_t ColumnFamilyData::GetNumLiveVersions() const {
+ return VersionSet::GetNumLiveVersions(dummy_versions_);
+}
+
+uint64_t ColumnFamilyData::GetTotalSstFilesSize() const {
+ return VersionSet::GetTotalSstFilesSize(dummy_versions_);
+}
+
+MemTable* ColumnFamilyData::ConstructNewMemtable(
+ const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
+ return new MemTable(internal_comparator_, ioptions_, mutable_cf_options,
+ write_buffer_manager_, earliest_seq, id_);
+}
+
+void ColumnFamilyData::CreateNewMemtable(
+ const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
+ if (mem_ != nullptr) {
+ delete mem_->Unref();
+ }
+ SetMemtable(ConstructNewMemtable(mutable_cf_options, earliest_seq));
+ mem_->Ref();
+}
+
+bool ColumnFamilyData::NeedsCompaction() const {
+ return compaction_picker_->NeedsCompaction(current_->storage_info());
+}
+
+Compaction* ColumnFamilyData::PickCompaction(
+ const MutableCFOptions& mutable_options, LogBuffer* log_buffer) {
+ auto* result = compaction_picker_->PickCompaction(
+ GetName(), mutable_options, current_->storage_info(), log_buffer);
+ if (result != nullptr) {
+ result->SetInputVersion(current_);
+ }
+ return result;
+}
+
+bool ColumnFamilyData::RangeOverlapWithCompaction(
+ const Slice& smallest_user_key, const Slice& largest_user_key,
+ int level) const {
+ return compaction_picker_->RangeOverlapWithCompaction(
+ smallest_user_key, largest_user_key, level);
+}
+
+const int ColumnFamilyData::kCompactAllLevels = -1;
+const int ColumnFamilyData::kCompactToBaseLevel = -2;
+
+Compaction* ColumnFamilyData::CompactRange(
+ const MutableCFOptions& mutable_cf_options, int input_level,
+ int output_level, uint32_t output_path_id, const InternalKey* begin,
+ const InternalKey* end, InternalKey** compaction_end, bool* conflict) {
+ auto* result = compaction_picker_->CompactRange(
+ GetName(), mutable_cf_options, current_->storage_info(), input_level,
+ output_level, output_path_id, begin, end, compaction_end, conflict);
+ if (result != nullptr) {
+ result->SetInputVersion(current_);
+ }
+ return result;
+}
+
+SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(
+ InstrumentedMutex* db_mutex) {
+ SuperVersion* sv = nullptr;
+ sv = GetThreadLocalSuperVersion(db_mutex);
+ sv->Ref();
+ if (!ReturnThreadLocalSuperVersion(sv)) {
+ sv->Unref();
+ }
+ return sv;
+}
+
+SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(
+ InstrumentedMutex* db_mutex) {
+ SuperVersion* sv = nullptr;
+ // The SuperVersion is cached in thread local storage to avoid acquiring
+ // mutex when SuperVersion does not change since the last use. When a new
+ // SuperVersion is installed, the compaction or flush thread cleans up
+ // cached SuperVersion in all existing thread local storage. To avoid
+ // acquiring mutex for this operation, we use atomic Swap() on the thread
+ // local pointer to guarantee exclusive access. If the thread local pointer
+ // is being used while a new SuperVersion is installed, the cached
+ // SuperVersion can become stale. In that case, the background thread would
+ // have swapped in kSVObsolete. We re-check the value at when returning
+ // SuperVersion back to thread local, with an atomic compare and swap.
+ // The superversion will need to be released if detected to be stale.
+ void* ptr = local_sv_->Swap(SuperVersion::kSVInUse);
+ // Invariant:
+ // (1) Scrape (always) installs kSVObsolete in ThreadLocal storage
+ // (2) the Swap above (always) installs kSVInUse, ThreadLocal storage
+ // should only keep kSVInUse before ReturnThreadLocalSuperVersion call
+ // (if no Scrape happens).
+ assert(ptr != SuperVersion::kSVInUse);
+ sv = static_cast<SuperVersion*>(ptr);
+ if (sv == SuperVersion::kSVObsolete ||
+ sv->version_number != super_version_number_.load()) {
+ RecordTick(ioptions_.statistics, NUMBER_SUPERVERSION_ACQUIRES);
+ SuperVersion* sv_to_delete = nullptr;
+
+ if (sv && sv->Unref()) {
+ RecordTick(ioptions_.statistics, NUMBER_SUPERVERSION_CLEANUPS);
+ db_mutex->Lock();
+ // NOTE: underlying resources held by superversion (sst files) might
+ // not be released until the next background job.
+ sv->Cleanup();
+ sv_to_delete = sv;
+ } else {
+ db_mutex->Lock();
+ }
+ sv = super_version_->Ref();
+ db_mutex->Unlock();
+
+ delete sv_to_delete;
+ }
+ assert(sv != nullptr);
+ return sv;
+}
+
+bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) {
+ assert(sv != nullptr);
+ // Put the SuperVersion back
+ void* expected = SuperVersion::kSVInUse;
+ if (local_sv_->CompareAndSwap(static_cast<void*>(sv), expected)) {
+ // When we see kSVInUse in the ThreadLocal, we are sure ThreadLocal
+ // storage has not been altered and no Scrape has happened. The
+ // SuperVersion is still current.
+ return true;
+ } else {
+ // ThreadLocal scrape happened in the process of this GetImpl call (after
+ // thread local Swap() at the beginning and before CompareAndSwap()).
+ // This means the SuperVersion it holds is obsolete.
+ assert(expected == SuperVersion::kSVObsolete);
+ }
+ return false;
+}
+
+SuperVersion* ColumnFamilyData::InstallSuperVersion(
+ SuperVersion* new_superversion, InstrumentedMutex* db_mutex) {
+ db_mutex->AssertHeld();
+ return InstallSuperVersion(new_superversion, db_mutex, mutable_cf_options_);
+}
+
+SuperVersion* ColumnFamilyData::InstallSuperVersion(
+ SuperVersion* new_superversion, InstrumentedMutex* db_mutex,
+ const MutableCFOptions& mutable_cf_options) {
+ new_superversion->db_mutex = db_mutex;
+ new_superversion->mutable_cf_options = mutable_cf_options;
+ new_superversion->Init(mem_, imm_.current(), current_);
+ SuperVersion* old_superversion = super_version_;
+ super_version_ = new_superversion;
+ ++super_version_number_;
+ super_version_->version_number = super_version_number_;
+ // Reset SuperVersions cached in thread local storage
+ ResetThreadLocalSuperVersions();
+
+ RecalculateWriteStallConditions(mutable_cf_options);
+
+ if (old_superversion != nullptr && old_superversion->Unref()) {
+ old_superversion->Cleanup();
+ return old_superversion; // will let caller delete outside of mutex
+ }
+ return nullptr;
+}
+
+void ColumnFamilyData::ResetThreadLocalSuperVersions() {
+ autovector<void*> sv_ptrs;
+ local_sv_->Scrape(&sv_ptrs, SuperVersion::kSVObsolete);
+ for (auto ptr : sv_ptrs) {
+ assert(ptr);
+ if (ptr == SuperVersion::kSVInUse) {
+ continue;
+ }
+ auto sv = static_cast<SuperVersion*>(ptr);
+ if (sv->Unref()) {
+ sv->Cleanup();
+ delete sv;
+ }
+ }
+}
+
+#ifndef ROCKSDB_LITE
+Status ColumnFamilyData::SetOptions(
+ const std::unordered_map<std::string, std::string>& options_map) {
+ MutableCFOptions new_mutable_cf_options;
+ Status s = GetMutableOptionsFromStrings(mutable_cf_options_, options_map,
+ &new_mutable_cf_options);
+ if (s.ok()) {
+ mutable_cf_options_ = new_mutable_cf_options;
+ mutable_cf_options_.RefreshDerivedOptions(ioptions_);
+ }
+ return s;
+}
+#endif // ROCKSDB_LITE
+
+ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
+ const ImmutableDBOptions* db_options,
+ const EnvOptions& env_options,
+ Cache* table_cache,
+ WriteBufferManager* write_buffer_manager,
+ WriteController* write_controller)
+ : max_column_family_(0),
+ dummy_cfd_(new ColumnFamilyData(0, "", nullptr, nullptr, nullptr,
+ ColumnFamilyOptions(), *db_options,
+ env_options, nullptr)),
+ default_cfd_cache_(nullptr),
+ db_name_(dbname),
+ db_options_(db_options),
+ env_options_(env_options),
+ table_cache_(table_cache),
+ write_buffer_manager_(write_buffer_manager),
+ write_controller_(write_controller) {
+ // initialize linked list
+ dummy_cfd_->prev_ = dummy_cfd_;
+ dummy_cfd_->next_ = dummy_cfd_;
+}
+
+ColumnFamilySet::~ColumnFamilySet() {
+ while (column_family_data_.size() > 0) {
+ // cfd destructor will delete itself from column_family_data_
+ auto cfd = column_family_data_.begin()->second;
+ cfd->Unref();
+ delete cfd;
+ }
+ dummy_cfd_->Unref();
+ delete dummy_cfd_;
+}
+
+ColumnFamilyData* ColumnFamilySet::GetDefault() const {
+ assert(default_cfd_cache_ != nullptr);
+ return default_cfd_cache_;
+}
+
+ColumnFamilyData* ColumnFamilySet::GetColumnFamily(uint32_t id) const {
+ auto cfd_iter = column_family_data_.find(id);
+ if (cfd_iter != column_family_data_.end()) {
+ return cfd_iter->second;
+ } else {
+ return nullptr;
+ }
+}
+
+ColumnFamilyData* ColumnFamilySet::GetColumnFamily(const std::string& name)
+ const {
+ auto cfd_iter = column_families_.find(name);
+ if (cfd_iter != column_families_.end()) {
+ auto cfd = GetColumnFamily(cfd_iter->second);
+ assert(cfd != nullptr);
+ return cfd;
+ } else {
+ return nullptr;
+ }
+}
+
+uint32_t ColumnFamilySet::GetNextColumnFamilyID() {
+ return ++max_column_family_;
+}
+
+uint32_t ColumnFamilySet::GetMaxColumnFamily() { return max_column_family_; }
+
+void ColumnFamilySet::UpdateMaxColumnFamily(uint32_t new_max_column_family) {
+ max_column_family_ = std::max(new_max_column_family, max_column_family_);
+}
+
+size_t ColumnFamilySet::NumberOfColumnFamilies() const {
+ return column_families_.size();
+}
+
+// under a DB mutex AND write thread
+ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
+ const std::string& name, uint32_t id, Version* dummy_versions,
+ const ColumnFamilyOptions& options) {
+ assert(column_families_.find(name) == column_families_.end());
+ ColumnFamilyData* new_cfd = new ColumnFamilyData(
+ id, name, dummy_versions, table_cache_, write_buffer_manager_, options,
+ *db_options_, env_options_, this);
+ column_families_.insert({name, id});
+ column_family_data_.insert({id, new_cfd});
+ max_column_family_ = std::max(max_column_family_, id);
+ // add to linked list
+ new_cfd->next_ = dummy_cfd_;
+ auto prev = dummy_cfd_->prev_;
+ new_cfd->prev_ = prev;
+ prev->next_ = new_cfd;
+ dummy_cfd_->prev_ = new_cfd;
+ if (id == 0) {
+ default_cfd_cache_ = new_cfd;
+ }
+ return new_cfd;
+}
+
+// REQUIRES: DB mutex held
+void ColumnFamilySet::FreeDeadColumnFamilies() {
+ autovector<ColumnFamilyData*> to_delete;
+ for (auto cfd = dummy_cfd_->next_; cfd != dummy_cfd_; cfd = cfd->next_) {
+ if (cfd->refs_.load(std::memory_order_relaxed) == 0) {
+ to_delete.push_back(cfd);
+ }
+ }
+ for (auto cfd : to_delete) {
+ // this is very rare, so it's not a problem that we do it under a mutex
+ delete cfd;
+ }
+}
+
+// under a DB mutex AND from a write thread
+void ColumnFamilySet::RemoveColumnFamily(ColumnFamilyData* cfd) {
+ auto cfd_iter = column_family_data_.find(cfd->GetID());
+ assert(cfd_iter != column_family_data_.end());
+ column_family_data_.erase(cfd_iter);
+ column_families_.erase(cfd->GetName());
+}
+
+// under a DB mutex OR from a write thread
+bool ColumnFamilyMemTablesImpl::Seek(uint32_t column_family_id) {
+ if (column_family_id == 0) {
+ // optimization for common case
+ current_ = column_family_set_->GetDefault();
+ } else {
+ current_ = column_family_set_->GetColumnFamily(column_family_id);
+ }
+ handle_.SetCFD(current_);
+ return current_ != nullptr;
+}
+
+uint64_t ColumnFamilyMemTablesImpl::GetLogNumber() const {
+ assert(current_ != nullptr);
+ return current_->GetLogNumber();
+}
+
+MemTable* ColumnFamilyMemTablesImpl::GetMemTable() const {
+ assert(current_ != nullptr);
+ return current_->mem();
+}
+
+ColumnFamilyHandle* ColumnFamilyMemTablesImpl::GetColumnFamilyHandle() {
+ assert(current_ != nullptr);
+ return &handle_;
+}
+
+uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family) {
+ uint32_t column_family_id = 0;
+ if (column_family != nullptr) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ column_family_id = cfh->GetID();
+ }
+ return column_family_id;
+}
+
+const Comparator* GetColumnFamilyUserComparator(
+ ColumnFamilyHandle* column_family) {
+ if (column_family != nullptr) {
+ return column_family->GetComparator();
+ }
+ return nullptr;
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/column_family.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/column_family.h b/thirdparty/rocksdb/db/column_family.h
new file mode 100644
index 0000000..3a807d2
--- /dev/null
+++ b/thirdparty/rocksdb/db/column_family.h
@@ -0,0 +1,577 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#include <unordered_map>
+#include <string>
+#include <vector>
+#include <atomic>
+
+#include "db/memtable_list.h"
+#include "db/table_cache.h"
+#include "db/table_properties_collector.h"
+#include "db/write_batch_internal.h"
+#include "db/write_controller.h"
+#include "options/cf_options.h"
+#include "rocksdb/compaction_job_stats.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+#include "util/thread_local.h"
+
+namespace rocksdb {
+
+class Version;
+class VersionSet;
+class MemTable;
+class MemTableListVersion;
+class CompactionPicker;
+class Compaction;
+class InternalKey;
+class InternalStats;
+class ColumnFamilyData;
+class DBImpl;
+class LogBuffer;
+class InstrumentedMutex;
+class InstrumentedMutexLock;
+
+extern const double kIncSlowdownRatio;
+
+// ColumnFamilyHandleImpl is the class that clients use to access different
+// column families. It has non-trivial destructor, which gets called when client
+// is done using the column family
+class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
+ public:
+ // create while holding the mutex
+ ColumnFamilyHandleImpl(
+ ColumnFamilyData* cfd, DBImpl* db, InstrumentedMutex* mutex);
+ // destroy without mutex
+ virtual ~ColumnFamilyHandleImpl();
+ virtual ColumnFamilyData* cfd() const { return cfd_; }
+
+ virtual uint32_t GetID() const override;
+ virtual const std::string& GetName() const override;
+ virtual Status GetDescriptor(ColumnFamilyDescriptor* desc) override;
+ virtual const Comparator* GetComparator() const override;
+
+ private:
+ ColumnFamilyData* cfd_;
+ DBImpl* db_;
+ InstrumentedMutex* mutex_;
+};
+
+// Does not ref-count ColumnFamilyData
+// We use this dummy ColumnFamilyHandleImpl because sometimes MemTableInserter
+// calls DBImpl methods. When this happens, MemTableInserter need access to
+// ColumnFamilyHandle (same as the client would need). In that case, we feed
+// MemTableInserter dummy ColumnFamilyHandle and enable it to call DBImpl
+// methods
+class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
+ public:
+ ColumnFamilyHandleInternal()
+ : ColumnFamilyHandleImpl(nullptr, nullptr, nullptr) {}
+
+ void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; }
+ virtual ColumnFamilyData* cfd() const override { return internal_cfd_; }
+
+ private:
+ ColumnFamilyData* internal_cfd_;
+};
+
+// holds references to memtable, all immutable memtables and version
+struct SuperVersion {
+ // Accessing members of this class is not thread-safe and requires external
+ // synchronization (ie db mutex held or on write thread).
+ MemTable* mem;
+ MemTableListVersion* imm;
+ Version* current;
+ MutableCFOptions mutable_cf_options;
+ // Version number of the current SuperVersion
+ uint64_t version_number;
+
+ InstrumentedMutex* db_mutex;
+
+ // should be called outside the mutex
+ SuperVersion() = default;
+ ~SuperVersion();
+ SuperVersion* Ref();
+ // If Unref() returns true, Cleanup() should be called with mutex held
+ // before deleting this SuperVersion.
+ bool Unref();
+
+ // call these two methods with db mutex held
+ // Cleanup unrefs mem, imm and current. Also, it stores all memtables
+ // that needs to be deleted in to_delete vector. Unrefing those
+ // objects needs to be done in the mutex
+ void Cleanup();
+ void Init(MemTable* new_mem, MemTableListVersion* new_imm,
+ Version* new_current);
+
+ // The value of dummy is not actually used. kSVInUse takes its address as a
+ // mark in the thread local storage to indicate the SuperVersion is in use
+ // by thread. This way, the value of kSVInUse is guaranteed to have no
+ // conflict with SuperVersion object address and portable on different
+ // platform.
+ static int dummy;
+ static void* const kSVInUse;
+ static void* const kSVObsolete;
+
+ private:
+ std::atomic<uint32_t> refs;
+ // We need to_delete because during Cleanup(), imm->Unref() returns
+ // all memtables that we need to free through this vector. We then
+ // delete all those memtables outside of mutex, during destruction
+ autovector<MemTable*> to_delete;
+};
+
+extern Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options);
+
+extern Status CheckConcurrentWritesSupported(
+ const ColumnFamilyOptions& cf_options);
+
+extern ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
+ const ColumnFamilyOptions& src);
+// Wrap user defined table proproties collector factories `from cf_options`
+// into internal ones in int_tbl_prop_collector_factories. Add a system internal
+// one too.
+extern void GetIntTblPropCollectorFactory(
+ const ImmutableCFOptions& ioptions,
+ std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+ int_tbl_prop_collector_factories);
+
+class ColumnFamilySet;
+
+// This class keeps all the data that a column family needs.
+// Most methods require DB mutex held, unless otherwise noted
+class ColumnFamilyData {
+ public:
+ ~ColumnFamilyData();
+
+ // thread-safe
+ uint32_t GetID() const { return id_; }
+ // thread-safe
+ const std::string& GetName() const { return name_; }
+
+ // Ref() can only be called from a context where the caller can guarantee
+ // that ColumnFamilyData is alive (while holding a non-zero ref already,
+ // holding a DB mutex, or as the leader in a write batch group).
+ void Ref() { refs_.fetch_add(1, std::memory_order_relaxed); }
+
+ // Unref decreases the reference count, but does not handle deletion
+ // when the count goes to 0. If this method returns true then the
+ // caller should delete the instance immediately, or later, by calling
+ // FreeDeadColumnFamilies(). Unref() can only be called while holding
+ // a DB mutex, or during single-threaded recovery.
+ bool Unref() {
+ int old_refs = refs_.fetch_sub(1, std::memory_order_relaxed);
+ assert(old_refs > 0);
+ return old_refs == 1;
+ }
+
+ // SetDropped() can only be called under following conditions:
+ // 1) Holding a DB mutex,
+ // 2) from single-threaded write thread, AND
+ // 3) from single-threaded VersionSet::LogAndApply()
+ // After dropping column family no other operation on that column family
+ // will be executed. All the files and memory will be, however, kept around
+ // until client drops the column family handle. That way, client can still
+ // access data from dropped column family.
+ // Column family can be dropped and still alive. In that state:
+ // *) Compaction and flush is not executed on the dropped column family.
+ // *) Client can continue reading from column family. Writes will fail unless
+ // WriteOptions::ignore_missing_column_families is true
+ // When the dropped column family is unreferenced, then we:
+ // *) Remove column family from the linked list maintained by ColumnFamilySet
+ // *) delete all memory associated with that column family
+ // *) delete all the files associated with that column family
+ void SetDropped();
+ bool IsDropped() const { return dropped_; }
+
+ // thread-safe
+ int NumberLevels() const { return ioptions_.num_levels; }
+
+ void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
+ uint64_t GetLogNumber() const { return log_number_; }
+
+ // thread-safe
+ const EnvOptions* soptions() const;
+ const ImmutableCFOptions* ioptions() const { return &ioptions_; }
+ // REQUIRES: DB mutex held
+ // This returns the MutableCFOptions used by current SuperVersion
+ // You should use this API to reference MutableCFOptions most of the time.
+ const MutableCFOptions* GetCurrentMutableCFOptions() const {
+ return &(super_version_->mutable_cf_options);
+ }
+ // REQUIRES: DB mutex held
+ // This returns the latest MutableCFOptions, which may be not in effect yet.
+ const MutableCFOptions* GetLatestMutableCFOptions() const {
+ return &mutable_cf_options_;
+ }
+
+ // REQUIRES: DB mutex held
+ // Build ColumnFamiliesOptions with immutable options and latest mutable
+ // options.
+ ColumnFamilyOptions GetLatestCFOptions() const;
+
+ bool is_delete_range_supported() { return is_delete_range_supported_; }
+
+#ifndef ROCKSDB_LITE
+ // REQUIRES: DB mutex held
+ Status SetOptions(
+ const std::unordered_map<std::string, std::string>& options_map);
+#endif // ROCKSDB_LITE
+
+ InternalStats* internal_stats() { return internal_stats_.get(); }
+
+ MemTableList* imm() { return &imm_; }
+ MemTable* mem() { return mem_; }
+ Version* current() { return current_; }
+ Version* dummy_versions() { return dummy_versions_; }
+ void SetCurrent(Version* _current);
+ uint64_t GetNumLiveVersions() const; // REQUIRE: DB mutex held
+ uint64_t GetTotalSstFilesSize() const; // REQUIRE: DB mutex held
+ void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }
+
+ // calculate the oldest log needed for the durability of this column family
+ uint64_t OldestLogToKeep();
+
+ // See Memtable constructor for explanation of earliest_seq param.
+ MemTable* ConstructNewMemtable(const MutableCFOptions& mutable_cf_options,
+ SequenceNumber earliest_seq);
+ void CreateNewMemtable(const MutableCFOptions& mutable_cf_options,
+ SequenceNumber earliest_seq);
+
+ TableCache* table_cache() const { return table_cache_.get(); }
+
+ // See documentation in compaction_picker.h
+ // REQUIRES: DB mutex held
+ bool NeedsCompaction() const;
+ // REQUIRES: DB mutex held
+ Compaction* PickCompaction(const MutableCFOptions& mutable_options,
+ LogBuffer* log_buffer);
+
+ // Check if the passed range overlap with any running compactions.
+ // REQUIRES: DB mutex held
+ bool RangeOverlapWithCompaction(const Slice& smallest_user_key,
+ const Slice& largest_user_key,
+ int level) const;
+
+ // A flag to tell a manual compaction is to compact all levels together
+ // instad of for specific level.
+ static const int kCompactAllLevels;
+ // A flag to tell a manual compaction's output is base level.
+ static const int kCompactToBaseLevel;
+ // REQUIRES: DB mutex held
+ Compaction* CompactRange(const MutableCFOptions& mutable_cf_options,
+ int input_level, int output_level,
+ uint32_t output_path_id, const InternalKey* begin,
+ const InternalKey* end, InternalKey** compaction_end,
+ bool* manual_conflict);
+
+ CompactionPicker* compaction_picker() { return compaction_picker_.get(); }
+ // thread-safe
+ const Comparator* user_comparator() const {
+ return internal_comparator_.user_comparator();
+ }
+ // thread-safe
+ const InternalKeyComparator& internal_comparator() const {
+ return internal_comparator_;
+ }
+
+ const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+ int_tbl_prop_collector_factories() const {
+ return &int_tbl_prop_collector_factories_;
+ }
+
+ SuperVersion* GetSuperVersion() { return super_version_; }
+ // thread-safe
+ // Return a already referenced SuperVersion to be used safely.
+ SuperVersion* GetReferencedSuperVersion(InstrumentedMutex* db_mutex);
+ // thread-safe
+ // Get SuperVersion stored in thread local storage. If it does not exist,
+ // get a reference from a current SuperVersion.
+ SuperVersion* GetThreadLocalSuperVersion(InstrumentedMutex* db_mutex);
+ // Try to return SuperVersion back to thread local storage. Retrun true on
+ // success and false on failure. It fails when the thread local storage
+ // contains anything other than SuperVersion::kSVInUse flag.
+ bool ReturnThreadLocalSuperVersion(SuperVersion* sv);
+ // thread-safe
+ uint64_t GetSuperVersionNumber() const {
+ return super_version_number_.load();
+ }
+ // will return a pointer to SuperVersion* if previous SuperVersion
+ // if its reference count is zero and needs deletion or nullptr if not
+ // As argument takes a pointer to allocated SuperVersion to enable
+ // the clients to allocate SuperVersion outside of mutex.
+ // IMPORTANT: Only call this from DBImpl::InstallSuperVersion()
+ SuperVersion* InstallSuperVersion(SuperVersion* new_superversion,
+ InstrumentedMutex* db_mutex,
+ const MutableCFOptions& mutable_cf_options);
+ SuperVersion* InstallSuperVersion(SuperVersion* new_superversion,
+ InstrumentedMutex* db_mutex);
+
+ void ResetThreadLocalSuperVersions();
+
+ // Protected by DB mutex
+ void set_pending_flush(bool value) { pending_flush_ = value; }
+ void set_pending_compaction(bool value) { pending_compaction_ = value; }
+ bool pending_flush() { return pending_flush_; }
+ bool pending_compaction() { return pending_compaction_; }
+
+ // Recalculate some small conditions, which are changed only during
+ // compaction, adding new memtable and/or
+ // recalculation of compaction score. These values are used in
+ // DBImpl::MakeRoomForWrite function to decide, if it need to make
+ // a write stall
+ void RecalculateWriteStallConditions(
+ const MutableCFOptions& mutable_cf_options);
+
+ void set_initialized() { initialized_.store(true); }
+
+ bool initialized() const { return initialized_.load(); }
+
+ private:
+ friend class ColumnFamilySet;
+ ColumnFamilyData(uint32_t id, const std::string& name,
+ Version* dummy_versions, Cache* table_cache,
+ WriteBufferManager* write_buffer_manager,
+ const ColumnFamilyOptions& options,
+ const ImmutableDBOptions& db_options,
+ const EnvOptions& env_options,
+ ColumnFamilySet* column_family_set);
+
+ uint32_t id_;
+ const std::string name_;
+ Version* dummy_versions_; // Head of circular doubly-linked list of versions.
+ Version* current_; // == dummy_versions->prev_
+
+ std::atomic<int> refs_; // outstanding references to ColumnFamilyData
+ std::atomic<bool> initialized_;
+ bool dropped_; // true if client dropped it
+
+ const InternalKeyComparator internal_comparator_;
+ std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
+ int_tbl_prop_collector_factories_;
+
+ const ColumnFamilyOptions initial_cf_options_;
+ const ImmutableCFOptions ioptions_;
+ MutableCFOptions mutable_cf_options_;
+
+ const bool is_delete_range_supported_;
+
+ std::unique_ptr<TableCache> table_cache_;
+
+ std::unique_ptr<InternalStats> internal_stats_;
+
+ WriteBufferManager* write_buffer_manager_;
+
+ MemTable* mem_;
+ MemTableList imm_;
+ SuperVersion* super_version_;
+
+ // An ordinal representing the current SuperVersion. Updated by
+ // InstallSuperVersion(), i.e. incremented every time super_version_
+ // changes.
+ std::atomic<uint64_t> super_version_number_;
+
+ // Thread's local copy of SuperVersion pointer
+ // This needs to be destructed before mutex_
+ std::unique_ptr<ThreadLocalPtr> local_sv_;
+
+ // pointers for a circular linked list. we use it to support iterations over
+ // all column families that are alive (note: dropped column families can also
+ // be alive as long as client holds a reference)
+ ColumnFamilyData* next_;
+ ColumnFamilyData* prev_;
+
+ // This is the earliest log file number that contains data from this
+ // Column Family. All earlier log files must be ignored and not
+ // recovered from
+ uint64_t log_number_;
+
+ // An object that keeps all the compaction stats
+ // and picks the next compaction
+ std::unique_ptr<CompactionPicker> compaction_picker_;
+
+ ColumnFamilySet* column_family_set_;
+
+ std::unique_ptr<WriteControllerToken> write_controller_token_;
+
+ // If true --> this ColumnFamily is currently present in DBImpl::flush_queue_
+ bool pending_flush_;
+
+ // If true --> this ColumnFamily is currently present in
+ // DBImpl::compaction_queue_
+ bool pending_compaction_;
+
+ uint64_t prev_compaction_needed_bytes_;
+
+ // if the database was opened with 2pc enabled
+ bool allow_2pc_;
+};
+
+// ColumnFamilySet has interesting thread-safety requirements
+// * CreateColumnFamily() or RemoveColumnFamily() -- need to be protected by DB
+// mutex AND executed in the write thread.
+// CreateColumnFamily() should ONLY be called from VersionSet::LogAndApply() AND
+// single-threaded write thread. It is also called during Recovery and in
+// DumpManifest().
+// RemoveColumnFamily() is only called from SetDropped(). DB mutex needs to be
+// held and it needs to be executed from the write thread. SetDropped() also
+// guarantees that it will be called only from single-threaded LogAndApply(),
+// but this condition is not that important.
+// * Iteration -- hold DB mutex, but you can release it in the body of
+// iteration. If you release DB mutex in body, reference the column
+// family before the mutex and unreference after you unlock, since the column
+// family might get dropped when the DB mutex is released
+// * GetDefault() -- thread safe
+// * GetColumnFamily() -- either inside of DB mutex or from a write thread
+// * GetNextColumnFamilyID(), GetMaxColumnFamily(), UpdateMaxColumnFamily(),
+// NumberOfColumnFamilies -- inside of DB mutex
+class ColumnFamilySet {
+ public:
+ // ColumnFamilySet supports iteration
+ class iterator {
+ public:
+ explicit iterator(ColumnFamilyData* cfd)
+ : current_(cfd) {}
+ iterator& operator++() {
+ // dropped column families might still be included in this iteration
+ // (we're only removing them when client drops the last reference to the
+ // column family).
+ // dummy is never dead, so this will never be infinite
+ do {
+ current_ = current_->next_;
+ } while (current_->refs_.load(std::memory_order_relaxed) == 0);
+ return *this;
+ }
+ bool operator!=(const iterator& other) {
+ return this->current_ != other.current_;
+ }
+ ColumnFamilyData* operator*() { return current_; }
+
+ private:
+ ColumnFamilyData* current_;
+ };
+
+ ColumnFamilySet(const std::string& dbname,
+ const ImmutableDBOptions* db_options,
+ const EnvOptions& env_options, Cache* table_cache,
+ WriteBufferManager* write_buffer_manager,
+ WriteController* write_controller);
+ ~ColumnFamilySet();
+
+ ColumnFamilyData* GetDefault() const;
+ // GetColumnFamily() calls return nullptr if column family is not found
+ ColumnFamilyData* GetColumnFamily(uint32_t id) const;
+ ColumnFamilyData* GetColumnFamily(const std::string& name) const;
+ // this call will return the next available column family ID. it guarantees
+ // that there is no column family with id greater than or equal to the
+ // returned value in the current running instance or anytime in RocksDB
+ // instance history.
+ uint32_t GetNextColumnFamilyID();
+ uint32_t GetMaxColumnFamily();
+ void UpdateMaxColumnFamily(uint32_t new_max_column_family);
+ size_t NumberOfColumnFamilies() const;
+
+ ColumnFamilyData* CreateColumnFamily(const std::string& name, uint32_t id,
+ Version* dummy_version,
+ const ColumnFamilyOptions& options);
+
+ iterator begin() { return iterator(dummy_cfd_->next_); }
+ iterator end() { return iterator(dummy_cfd_); }
+
+ // REQUIRES: DB mutex held
+ // Don't call while iterating over ColumnFamilySet
+ void FreeDeadColumnFamilies();
+
+ Cache* get_table_cache() { return table_cache_; }
+
+ private:
+ friend class ColumnFamilyData;
+ // helper function that gets called from cfd destructor
+ // REQUIRES: DB mutex held
+ void RemoveColumnFamily(ColumnFamilyData* cfd);
+
+ // column_families_ and column_family_data_ need to be protected:
+ // * when mutating both conditions have to be satisfied:
+ // 1. DB mutex locked
+ // 2. thread currently in single-threaded write thread
+ // * when reading, at least one condition needs to be satisfied:
+ // 1. DB mutex locked
+ // 2. accessed from a single-threaded write thread
+ std::unordered_map<std::string, uint32_t> column_families_;
+ std::unordered_map<uint32_t, ColumnFamilyData*> column_family_data_;
+
+ uint32_t max_column_family_;
+ ColumnFamilyData* dummy_cfd_;
+ // We don't hold the refcount here, since default column family always exists
+ // We are also not responsible for cleaning up default_cfd_cache_. This is
+ // just a cache that makes common case (accessing default column family)
+ // faster
+ ColumnFamilyData* default_cfd_cache_;
+
+ const std::string db_name_;
+ const ImmutableDBOptions* const db_options_;
+ const EnvOptions env_options_;
+ Cache* table_cache_;
+ WriteBufferManager* write_buffer_manager_;
+ WriteController* write_controller_;
+};
+
+// We use ColumnFamilyMemTablesImpl to provide WriteBatch a way to access
+// memtables of different column families (specified by ID in the write batch)
+class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables {
+ public:
+ explicit ColumnFamilyMemTablesImpl(ColumnFamilySet* column_family_set)
+ : column_family_set_(column_family_set), current_(nullptr) {}
+
+ // Constructs a ColumnFamilyMemTablesImpl equivalent to one constructed
+ // with the arguments used to construct *orig.
+ explicit ColumnFamilyMemTablesImpl(ColumnFamilyMemTablesImpl* orig)
+ : column_family_set_(orig->column_family_set_), current_(nullptr) {}
+
+ // sets current_ to ColumnFamilyData with column_family_id
+ // returns false if column family doesn't exist
+ // REQUIRES: use this function of DBImpl::column_family_memtables_ should be
+ // under a DB mutex OR from a write thread
+ bool Seek(uint32_t column_family_id) override;
+
+ // Returns log number of the selected column family
+ // REQUIRES: under a DB mutex OR from a write thread
+ uint64_t GetLogNumber() const override;
+
+ // REQUIRES: Seek() called first
+ // REQUIRES: use this function of DBImpl::column_family_memtables_ should be
+ // under a DB mutex OR from a write thread
+ virtual MemTable* GetMemTable() const override;
+
+ // Returns column family handle for the selected column family
+ // REQUIRES: use this function of DBImpl::column_family_memtables_ should be
+ // under a DB mutex OR from a write thread
+ virtual ColumnFamilyHandle* GetColumnFamilyHandle() override;
+
+ // Cannot be called while another thread is calling Seek().
+ // REQUIRES: use this function of DBImpl::column_family_memtables_ should be
+ // under a DB mutex OR from a write thread
+ virtual ColumnFamilyData* current() override { return current_; }
+
+ private:
+ ColumnFamilySet* column_family_set_;
+ ColumnFamilyData* current_;
+ ColumnFamilyHandleInternal handle_;
+};
+
+extern uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family);
+
+extern const Comparator* GetColumnFamilyUserComparator(
+ ColumnFamilyHandle* column_family);
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compacted_db_impl.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compacted_db_impl.cc b/thirdparty/rocksdb/db/compacted_db_impl.cc
new file mode 100644
index 0000000..d1007d9
--- /dev/null
+++ b/thirdparty/rocksdb/db/compacted_db_impl.cc
@@ -0,0 +1,166 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+#include "db/compacted_db_impl.h"
+#include "db/db_impl.h"
+#include "db/version_set.h"
+#include "table/get_context.h"
+
+namespace rocksdb {
+
+extern void MarkKeyMayExist(void* arg);
+extern bool SaveValue(void* arg, const ParsedInternalKey& parsed_key,
+ const Slice& v, bool hit_and_return);
+
+CompactedDBImpl::CompactedDBImpl(
+ const DBOptions& options, const std::string& dbname)
+ : DBImpl(options, dbname) {
+}
+
+CompactedDBImpl::~CompactedDBImpl() {
+}
+
+size_t CompactedDBImpl::FindFile(const Slice& key) {
+ size_t left = 0;
+ size_t right = files_.num_files - 1;
+ while (left < right) {
+ size_t mid = (left + right) >> 1;
+ const FdWithKeyRange& f = files_.files[mid];
+ if (user_comparator_->Compare(ExtractUserKey(f.largest_key), key) < 0) {
+ // Key at "mid.largest" is < "target". Therefore all
+ // files at or before "mid" are uninteresting.
+ left = mid + 1;
+ } else {
+ // Key at "mid.largest" is >= "target". Therefore all files
+ // after "mid" are uninteresting.
+ right = mid;
+ }
+ }
+ return right;
+}
+
+Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
+ const Slice& key, PinnableSlice* value) {
+ GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
+ GetContext::kNotFound, key, value, nullptr, nullptr,
+ nullptr, nullptr);
+ LookupKey lkey(key, kMaxSequenceNumber);
+ files_.files[FindFile(key)].fd.table_reader->Get(
+ options, lkey.internal_key(), &get_context);
+ if (get_context.State() == GetContext::kFound) {
+ return Status::OK();
+ }
+ return Status::NotFound();
+}
+
+std::vector<Status> CompactedDBImpl::MultiGet(const ReadOptions& options,
+ const std::vector<ColumnFamilyHandle*>&,
+ const std::vector<Slice>& keys, std::vector<std::string>* values) {
+ autovector<TableReader*, 16> reader_list;
+ for (const auto& key : keys) {
+ const FdWithKeyRange& f = files_.files[FindFile(key)];
+ if (user_comparator_->Compare(key, ExtractUserKey(f.smallest_key)) < 0) {
+ reader_list.push_back(nullptr);
+ } else {
+ LookupKey lkey(key, kMaxSequenceNumber);
+ f.fd.table_reader->Prepare(lkey.internal_key());
+ reader_list.push_back(f.fd.table_reader);
+ }
+ }
+ std::vector<Status> statuses(keys.size(), Status::NotFound());
+ values->resize(keys.size());
+ int idx = 0;
+ for (auto* r : reader_list) {
+ if (r != nullptr) {
+ PinnableSlice pinnable_val;
+ std::string& value = (*values)[idx];
+ GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
+ GetContext::kNotFound, keys[idx], &pinnable_val,
+ nullptr, nullptr, nullptr, nullptr);
+ LookupKey lkey(keys[idx], kMaxSequenceNumber);
+ r->Get(options, lkey.internal_key(), &get_context);
+ value.assign(pinnable_val.data(), pinnable_val.size());
+ if (get_context.State() == GetContext::kFound) {
+ statuses[idx] = Status::OK();
+ }
+ }
+ ++idx;
+ }
+ return statuses;
+}
+
+Status CompactedDBImpl::Init(const Options& options) {
+ mutex_.Lock();
+ ColumnFamilyDescriptor cf(kDefaultColumnFamilyName,
+ ColumnFamilyOptions(options));
+ Status s = Recover({cf}, true /* read only */, false, true);
+ if (s.ok()) {
+ cfd_ = reinterpret_cast<ColumnFamilyHandleImpl*>(
+ DefaultColumnFamily())->cfd();
+ delete cfd_->InstallSuperVersion(new SuperVersion(), &mutex_);
+ }
+ mutex_.Unlock();
+ if (!s.ok()) {
+ return s;
+ }
+ NewThreadStatusCfInfo(cfd_);
+ version_ = cfd_->GetSuperVersion()->current;
+ user_comparator_ = cfd_->user_comparator();
+ auto* vstorage = version_->storage_info();
+ if (vstorage->num_non_empty_levels() == 0) {
+ return Status::NotSupported("no file exists");
+ }
+ const LevelFilesBrief& l0 = vstorage->LevelFilesBrief(0);
+ // L0 should not have files
+ if (l0.num_files > 1) {
+ return Status::NotSupported("L0 contain more than 1 file");
+ }
+ if (l0.num_files == 1) {
+ if (vstorage->num_non_empty_levels() > 1) {
+ return Status::NotSupported("Both L0 and other level contain files");
+ }
+ files_ = l0;
+ return Status::OK();
+ }
+
+ for (int i = 1; i < vstorage->num_non_empty_levels() - 1; ++i) {
+ if (vstorage->LevelFilesBrief(i).num_files > 0) {
+ return Status::NotSupported("Other levels also contain files");
+ }
+ }
+
+ int level = vstorage->num_non_empty_levels() - 1;
+ if (vstorage->LevelFilesBrief(level).num_files > 0) {
+ files_ = vstorage->LevelFilesBrief(level);
+ return Status::OK();
+ }
+ return Status::NotSupported("no file exists");
+}
+
+Status CompactedDBImpl::Open(const Options& options,
+ const std::string& dbname, DB** dbptr) {
+ *dbptr = nullptr;
+
+ if (options.max_open_files != -1) {
+ return Status::InvalidArgument("require max_open_files = -1");
+ }
+ if (options.merge_operator.get() != nullptr) {
+ return Status::InvalidArgument("merge operator is not supported");
+ }
+ DBOptions db_options(options);
+ std::unique_ptr<CompactedDBImpl> db(new CompactedDBImpl(db_options, dbname));
+ Status s = db->Init(options);
+ if (s.ok()) {
+ ROCKS_LOG_INFO(db->immutable_db_options_.info_log,
+ "Opened the db as fully compacted mode");
+ LogFlush(db->immutable_db_options_.info_log);
+ *dbptr = db.release();
+ }
+ return s;
+}
+
+} // namespace rocksdb
+#endif // ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compacted_db_impl.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compacted_db_impl.h b/thirdparty/rocksdb/db/compacted_db_impl.h
new file mode 100644
index 0000000..de32f21
--- /dev/null
+++ b/thirdparty/rocksdb/db/compacted_db_impl.h
@@ -0,0 +1,102 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+#ifndef ROCKSDB_LITE
+#include "db/db_impl.h"
+#include <vector>
+#include <string>
+
+namespace rocksdb {
+
+class CompactedDBImpl : public DBImpl {
+ public:
+ CompactedDBImpl(const DBOptions& options, const std::string& dbname);
+ virtual ~CompactedDBImpl();
+
+ static Status Open(const Options& options, const std::string& dbname,
+ DB** dbptr);
+
+ // Implementations of the DB interface
+ using DB::Get;
+ virtual Status Get(const ReadOptions& options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ PinnableSlice* value) override;
+ using DB::MultiGet;
+ virtual std::vector<Status> MultiGet(
+ const ReadOptions& options,
+ const std::vector<ColumnFamilyHandle*>&,
+ const std::vector<Slice>& keys, std::vector<std::string>* values)
+ override;
+
+ using DBImpl::Put;
+ virtual Status Put(const WriteOptions& options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ const Slice& value) override {
+ return Status::NotSupported("Not supported in compacted db mode.");
+ }
+ using DBImpl::Merge;
+ virtual Status Merge(const WriteOptions& options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ const Slice& value) override {
+ return Status::NotSupported("Not supported in compacted db mode.");
+ }
+ using DBImpl::Delete;
+ virtual Status Delete(const WriteOptions& options,
+ ColumnFamilyHandle* column_family,
+ const Slice& key) override {
+ return Status::NotSupported("Not supported in compacted db mode.");
+ }
+ virtual Status Write(const WriteOptions& options,
+ WriteBatch* updates) override {
+ return Status::NotSupported("Not supported in compacted db mode.");
+ }
+ using DBImpl::CompactRange;
+ virtual Status CompactRange(const CompactRangeOptions& options,
+ ColumnFamilyHandle* column_family,
+ const Slice* begin, const Slice* end) override {
+ return Status::NotSupported("Not supported in compacted db mode.");
+ }
+
+ virtual Status DisableFileDeletions() override {
+ return Status::NotSupported("Not supported in compacted db mode.");
+ }
+ virtual Status EnableFileDeletions(bool force) override {
+ return Status::NotSupported("Not supported in compacted db mode.");
+ }
+ virtual Status GetLiveFiles(std::vector<std::string>&,
+ uint64_t* manifest_file_size,
+ bool flush_memtable = true) override {
+ return Status::NotSupported("Not supported in compacted db mode.");
+ }
+ using DBImpl::Flush;
+ virtual Status Flush(const FlushOptions& options,
+ ColumnFamilyHandle* column_family) override {
+ return Status::NotSupported("Not supported in compacted db mode.");
+ }
+ using DB::IngestExternalFile;
+ virtual Status IngestExternalFile(
+ ColumnFamilyHandle* column_family,
+ const std::vector<std::string>& external_files,
+ const IngestExternalFileOptions& ingestion_options) override {
+ return Status::NotSupported("Not supported in compacted db mode.");
+ }
+
+ private:
+ friend class DB;
+ inline size_t FindFile(const Slice& key);
+ Status Init(const Options& options);
+
+ ColumnFamilyData* cfd_;
+ Version* version_;
+ const Comparator* user_comparator_;
+ LevelFilesBrief files_;
+
+ // No copying allowed
+ CompactedDBImpl(const CompactedDBImpl&);
+ void operator=(const CompactedDBImpl&);
+};
+}
+#endif // ROCKSDB_LITE
[10/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/table_properties_collector.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/table_properties_collector.cc b/thirdparty/rocksdb/db/table_properties_collector.cc
new file mode 100644
index 0000000..a1f4dba
--- /dev/null
+++ b/thirdparty/rocksdb/db/table_properties_collector.cc
@@ -0,0 +1,134 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "db/table_properties_collector.h"
+
+#include "db/dbformat.h"
+#include "util/coding.h"
+#include "util/string_util.h"
+
+namespace rocksdb {
+
+Status InternalKeyPropertiesCollector::InternalAdd(const Slice& key,
+ const Slice& value,
+ uint64_t file_size) {
+ ParsedInternalKey ikey;
+ if (!ParseInternalKey(key, &ikey)) {
+ return Status::InvalidArgument("Invalid internal key");
+ }
+
+ // Note: We count both, deletions and single deletions here.
+ if (ikey.type == ValueType::kTypeDeletion ||
+ ikey.type == ValueType::kTypeSingleDeletion) {
+ ++deleted_keys_;
+ } else if (ikey.type == ValueType::kTypeMerge) {
+ ++merge_operands_;
+ }
+
+ return Status::OK();
+}
+
+Status InternalKeyPropertiesCollector::Finish(
+ UserCollectedProperties* properties) {
+ assert(properties);
+ assert(properties->find(
+ InternalKeyTablePropertiesNames::kDeletedKeys) == properties->end());
+ assert(properties->find(InternalKeyTablePropertiesNames::kMergeOperands) ==
+ properties->end());
+
+ std::string val_deleted_keys;
+ PutVarint64(&val_deleted_keys, deleted_keys_);
+ properties->insert(
+ {InternalKeyTablePropertiesNames::kDeletedKeys, val_deleted_keys});
+
+ std::string val_merge_operands;
+ PutVarint64(&val_merge_operands, merge_operands_);
+ properties->insert(
+ {InternalKeyTablePropertiesNames::kMergeOperands, val_merge_operands});
+
+ return Status::OK();
+}
+
+UserCollectedProperties
+InternalKeyPropertiesCollector::GetReadableProperties() const {
+ return {{"kDeletedKeys", ToString(deleted_keys_)},
+ {"kMergeOperands", ToString(merge_operands_)}};
+}
+
+namespace {
+
+EntryType GetEntryType(ValueType value_type) {
+ switch (value_type) {
+ case kTypeValue:
+ return kEntryPut;
+ case kTypeDeletion:
+ return kEntryDelete;
+ case kTypeSingleDeletion:
+ return kEntrySingleDelete;
+ case kTypeMerge:
+ return kEntryMerge;
+ default:
+ return kEntryOther;
+ }
+}
+
+uint64_t GetUint64Property(const UserCollectedProperties& props,
+ const std::string property_name,
+ bool* property_present) {
+ auto pos = props.find(property_name);
+ if (pos == props.end()) {
+ *property_present = false;
+ return 0;
+ }
+ Slice raw = pos->second;
+ uint64_t val = 0;
+ *property_present = true;
+ return GetVarint64(&raw, &val) ? val : 0;
+}
+
+} // namespace
+
+Status UserKeyTablePropertiesCollector::InternalAdd(const Slice& key,
+ const Slice& value,
+ uint64_t file_size) {
+ ParsedInternalKey ikey;
+ if (!ParseInternalKey(key, &ikey)) {
+ return Status::InvalidArgument("Invalid internal key");
+ }
+
+ return collector_->AddUserKey(ikey.user_key, value, GetEntryType(ikey.type),
+ ikey.sequence, file_size);
+}
+
+Status UserKeyTablePropertiesCollector::Finish(
+ UserCollectedProperties* properties) {
+ return collector_->Finish(properties);
+}
+
+UserCollectedProperties
+UserKeyTablePropertiesCollector::GetReadableProperties() const {
+ return collector_->GetReadableProperties();
+}
+
+
+const std::string InternalKeyTablePropertiesNames::kDeletedKeys
+ = "rocksdb.deleted.keys";
+const std::string InternalKeyTablePropertiesNames::kMergeOperands =
+ "rocksdb.merge.operands";
+
+uint64_t GetDeletedKeys(
+ const UserCollectedProperties& props) {
+ bool property_present_ignored;
+ return GetUint64Property(props, InternalKeyTablePropertiesNames::kDeletedKeys,
+ &property_present_ignored);
+}
+
+uint64_t GetMergeOperands(const UserCollectedProperties& props,
+ bool* property_present) {
+ return GetUint64Property(
+ props, InternalKeyTablePropertiesNames::kMergeOperands, property_present);
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/table_properties_collector.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/table_properties_collector.h b/thirdparty/rocksdb/db/table_properties_collector.h
new file mode 100644
index 0000000..d8cd756
--- /dev/null
+++ b/thirdparty/rocksdb/db/table_properties_collector.h
@@ -0,0 +1,137 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// This file defines a collection of statistics collectors.
+#pragma once
+
+#include "rocksdb/table_properties.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace rocksdb {
+
+struct InternalKeyTablePropertiesNames {
+ static const std::string kDeletedKeys;
+ static const std::string kMergeOperands;
+};
+
+// Base class for internal table properties collector.
+class IntTblPropCollector {
+ public:
+ virtual ~IntTblPropCollector() {}
+ virtual Status Finish(UserCollectedProperties* properties) = 0;
+
+ virtual const char* Name() const = 0;
+
+ // @params key the user key that is inserted into the table.
+ // @params value the value that is inserted into the table.
+ virtual Status InternalAdd(const Slice& key, const Slice& value,
+ uint64_t file_size) = 0;
+
+ virtual UserCollectedProperties GetReadableProperties() const = 0;
+
+ virtual bool NeedCompact() const { return false; }
+};
+
+// Factory for internal table properties collector.
+class IntTblPropCollectorFactory {
+ public:
+ virtual ~IntTblPropCollectorFactory() {}
+ // has to be thread-safe
+ virtual IntTblPropCollector* CreateIntTblPropCollector(
+ uint32_t column_family_id) = 0;
+
+ // The name of the properties collector can be used for debugging purpose.
+ virtual const char* Name() const = 0;
+};
+
+// Collecting the statistics for internal keys. Visible only by internal
+// rocksdb modules.
+class InternalKeyPropertiesCollector : public IntTblPropCollector {
+ public:
+ virtual Status InternalAdd(const Slice& key, const Slice& value,
+ uint64_t file_size) override;
+
+ virtual Status Finish(UserCollectedProperties* properties) override;
+
+ virtual const char* Name() const override {
+ return "InternalKeyPropertiesCollector";
+ }
+
+ UserCollectedProperties GetReadableProperties() const override;
+
+ private:
+ uint64_t deleted_keys_ = 0;
+ uint64_t merge_operands_ = 0;
+};
+
+class InternalKeyPropertiesCollectorFactory
+ : public IntTblPropCollectorFactory {
+ public:
+ virtual IntTblPropCollector* CreateIntTblPropCollector(
+ uint32_t column_family_id) override {
+ return new InternalKeyPropertiesCollector();
+ }
+
+ virtual const char* Name() const override {
+ return "InternalKeyPropertiesCollectorFactory";
+ }
+};
+
+// When rocksdb creates a new table, it will encode all "user keys" into
+// "internal keys", which contains meta information of a given entry.
+//
+// This class extracts user key from the encoded internal key when Add() is
+// invoked.
+class UserKeyTablePropertiesCollector : public IntTblPropCollector {
+ public:
+ // transfer of ownership
+ explicit UserKeyTablePropertiesCollector(TablePropertiesCollector* collector)
+ : collector_(collector) {}
+
+ virtual ~UserKeyTablePropertiesCollector() {}
+
+ virtual Status InternalAdd(const Slice& key, const Slice& value,
+ uint64_t file_size) override;
+
+ virtual Status Finish(UserCollectedProperties* properties) override;
+
+ virtual const char* Name() const override { return collector_->Name(); }
+
+ UserCollectedProperties GetReadableProperties() const override;
+
+ virtual bool NeedCompact() const override {
+ return collector_->NeedCompact();
+ }
+
+ protected:
+ std::unique_ptr<TablePropertiesCollector> collector_;
+};
+
+class UserKeyTablePropertiesCollectorFactory
+ : public IntTblPropCollectorFactory {
+ public:
+ explicit UserKeyTablePropertiesCollectorFactory(
+ std::shared_ptr<TablePropertiesCollectorFactory> user_collector_factory)
+ : user_collector_factory_(user_collector_factory) {}
+ virtual IntTblPropCollector* CreateIntTblPropCollector(
+ uint32_t column_family_id) override {
+ TablePropertiesCollectorFactory::Context context;
+ context.column_family_id = column_family_id;
+ return new UserKeyTablePropertiesCollector(
+ user_collector_factory_->CreateTablePropertiesCollector(context));
+ }
+
+ virtual const char* Name() const override {
+ return user_collector_factory_->Name();
+ }
+
+ private:
+ std::shared_ptr<TablePropertiesCollectorFactory> user_collector_factory_;
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/transaction_log_impl.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/transaction_log_impl.cc b/thirdparty/rocksdb/db/transaction_log_impl.cc
new file mode 100644
index 0000000..e22c0c4
--- /dev/null
+++ b/thirdparty/rocksdb/db/transaction_log_impl.cc
@@ -0,0 +1,272 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include "db/transaction_log_impl.h"
+#include <inttypes.h>
+#include "db/write_batch_internal.h"
+#include "util/file_reader_writer.h"
+
+namespace rocksdb {
+
+TransactionLogIteratorImpl::TransactionLogIteratorImpl(
+ const std::string& dir, const ImmutableDBOptions* options,
+ const TransactionLogIterator::ReadOptions& read_options,
+ const EnvOptions& soptions, const SequenceNumber seq,
+ std::unique_ptr<VectorLogPtr> files, VersionSet const* const versions)
+ : dir_(dir),
+ options_(options),
+ read_options_(read_options),
+ soptions_(soptions),
+ startingSequenceNumber_(seq),
+ files_(std::move(files)),
+ started_(false),
+ isValid_(false),
+ currentFileIndex_(0),
+ currentBatchSeq_(0),
+ currentLastSeq_(0),
+ versions_(versions) {
+ assert(files_ != nullptr);
+ assert(versions_ != nullptr);
+
+ reporter_.env = options_->env;
+ reporter_.info_log = options_->info_log.get();
+ SeekToStartSequence(); // Seek till starting sequence
+}
+
+Status TransactionLogIteratorImpl::OpenLogFile(
+ const LogFile* logFile, unique_ptr<SequentialFileReader>* file_reader) {
+ Env* env = options_->env;
+ unique_ptr<SequentialFile> file;
+ Status s;
+ EnvOptions optimized_env_options = env->OptimizeForLogRead(soptions_);
+ if (logFile->Type() == kArchivedLogFile) {
+ std::string fname = ArchivedLogFileName(dir_, logFile->LogNumber());
+ s = env->NewSequentialFile(fname, &file, optimized_env_options);
+ } else {
+ std::string fname = LogFileName(dir_, logFile->LogNumber());
+ s = env->NewSequentialFile(fname, &file, optimized_env_options);
+ if (!s.ok()) {
+ // If cannot open file in DB directory.
+ // Try the archive dir, as it could have moved in the meanwhile.
+ fname = ArchivedLogFileName(dir_, logFile->LogNumber());
+ s = env->NewSequentialFile(fname, &file, optimized_env_options);
+ }
+ }
+ if (s.ok()) {
+ file_reader->reset(new SequentialFileReader(std::move(file)));
+ }
+ return s;
+}
+
+BatchResult TransactionLogIteratorImpl::GetBatch() {
+ assert(isValid_); // cannot call in a non valid state.
+ BatchResult result;
+ result.sequence = currentBatchSeq_;
+ result.writeBatchPtr = std::move(currentBatch_);
+ return result;
+}
+
+Status TransactionLogIteratorImpl::status() {
+ return currentStatus_;
+}
+
+bool TransactionLogIteratorImpl::Valid() {
+ return started_ && isValid_;
+}
+
+bool TransactionLogIteratorImpl::RestrictedRead(
+ Slice* record,
+ std::string* scratch) {
+ // Don't read if no more complete entries to read from logs
+ if (currentLastSeq_ >= versions_->LastSequence()) {
+ return false;
+ }
+ return currentLogReader_->ReadRecord(record, scratch);
+}
+
+void TransactionLogIteratorImpl::SeekToStartSequence(
+ uint64_t startFileIndex,
+ bool strict) {
+ std::string scratch;
+ Slice record;
+ started_ = false;
+ isValid_ = false;
+ if (files_->size() <= startFileIndex) {
+ return;
+ }
+ Status s = OpenLogReader(files_->at(startFileIndex).get());
+ if (!s.ok()) {
+ currentStatus_ = s;
+ reporter_.Info(currentStatus_.ToString().c_str());
+ return;
+ }
+ while (RestrictedRead(&record, &scratch)) {
+ if (record.size() < WriteBatchInternal::kHeader) {
+ reporter_.Corruption(
+ record.size(), Status::Corruption("very small log record"));
+ continue;
+ }
+ UpdateCurrentWriteBatch(record);
+ if (currentLastSeq_ >= startingSequenceNumber_) {
+ if (strict && currentBatchSeq_ != startingSequenceNumber_) {
+ currentStatus_ = Status::Corruption("Gap in sequence number. Could not "
+ "seek to required sequence number");
+ reporter_.Info(currentStatus_.ToString().c_str());
+ return;
+ } else if (strict) {
+ reporter_.Info("Could seek required sequence number. Iterator will "
+ "continue.");
+ }
+ isValid_ = true;
+ started_ = true; // set started_ as we could seek till starting sequence
+ return;
+ } else {
+ isValid_ = false;
+ }
+ }
+
+ // Could not find start sequence in first file. Normally this must be the
+ // only file. Otherwise log the error and let the iterator return next entry
+ // If strict is set, we want to seek exactly till the start sequence and it
+ // should have been present in the file we scanned above
+ if (strict) {
+ currentStatus_ = Status::Corruption("Gap in sequence number. Could not "
+ "seek to required sequence number");
+ reporter_.Info(currentStatus_.ToString().c_str());
+ } else if (files_->size() != 1) {
+ currentStatus_ = Status::Corruption("Start sequence was not found, "
+ "skipping to the next available");
+ reporter_.Info(currentStatus_.ToString().c_str());
+ // Let NextImpl find the next available entry. started_ remains false
+ // because we don't want to check for gaps while moving to start sequence
+ NextImpl(true);
+ }
+}
+
+void TransactionLogIteratorImpl::Next() {
+ return NextImpl(false);
+}
+
+void TransactionLogIteratorImpl::NextImpl(bool internal) {
+ std::string scratch;
+ Slice record;
+ isValid_ = false;
+ if (!internal && !started_) {
+ // Runs every time until we can seek to the start sequence
+ return SeekToStartSequence();
+ }
+ while(true) {
+ assert(currentLogReader_);
+ if (currentLogReader_->IsEOF()) {
+ currentLogReader_->UnmarkEOF();
+ }
+ while (RestrictedRead(&record, &scratch)) {
+ if (record.size() < WriteBatchInternal::kHeader) {
+ reporter_.Corruption(
+ record.size(), Status::Corruption("very small log record"));
+ continue;
+ } else {
+ // started_ should be true if called by application
+ assert(internal || started_);
+ // started_ should be false if called internally
+ assert(!internal || !started_);
+ UpdateCurrentWriteBatch(record);
+ if (internal && !started_) {
+ started_ = true;
+ }
+ return;
+ }
+ }
+
+ // Open the next file
+ if (currentFileIndex_ < files_->size() - 1) {
+ ++currentFileIndex_;
+ Status s = OpenLogReader(files_->at(currentFileIndex_).get());
+ if (!s.ok()) {
+ isValid_ = false;
+ currentStatus_ = s;
+ return;
+ }
+ } else {
+ isValid_ = false;
+ if (currentLastSeq_ == versions_->LastSequence()) {
+ currentStatus_ = Status::OK();
+ } else {
+ currentStatus_ = Status::Corruption("NO MORE DATA LEFT");
+ }
+ return;
+ }
+ }
+}
+
+bool TransactionLogIteratorImpl::IsBatchExpected(
+ const WriteBatch* batch,
+ const SequenceNumber expectedSeq) {
+ assert(batch);
+ SequenceNumber batchSeq = WriteBatchInternal::Sequence(batch);
+ if (batchSeq != expectedSeq) {
+ char buf[200];
+ snprintf(buf, sizeof(buf),
+ "Discontinuity in log records. Got seq=%" PRIu64
+ ", Expected seq=%" PRIu64 ", Last flushed seq=%" PRIu64
+ ".Log iterator will reseek the correct batch.",
+ batchSeq, expectedSeq, versions_->LastSequence());
+ reporter_.Info(buf);
+ return false;
+ }
+ return true;
+}
+
+void TransactionLogIteratorImpl::UpdateCurrentWriteBatch(const Slice& record) {
+ std::unique_ptr<WriteBatch> batch(new WriteBatch());
+ WriteBatchInternal::SetContents(batch.get(), record);
+
+ SequenceNumber expectedSeq = currentLastSeq_ + 1;
+ // If the iterator has started, then confirm that we get continuous batches
+ if (started_ && !IsBatchExpected(batch.get(), expectedSeq)) {
+ // Seek to the batch having expected sequence number
+ if (expectedSeq < files_->at(currentFileIndex_)->StartSequence()) {
+ // Expected batch must lie in the previous log file
+ // Avoid underflow.
+ if (currentFileIndex_ != 0) {
+ currentFileIndex_--;
+ }
+ }
+ startingSequenceNumber_ = expectedSeq;
+ // currentStatus_ will be set to Ok if reseek succeeds
+ currentStatus_ = Status::NotFound("Gap in sequence numbers");
+ return SeekToStartSequence(currentFileIndex_, true);
+ }
+
+ currentBatchSeq_ = WriteBatchInternal::Sequence(batch.get());
+ currentLastSeq_ = currentBatchSeq_ +
+ WriteBatchInternal::Count(batch.get()) - 1;
+ // currentBatchSeq_ can only change here
+ assert(currentLastSeq_ <= versions_->LastSequence());
+
+ currentBatch_ = std::move(batch);
+ isValid_ = true;
+ currentStatus_ = Status::OK();
+}
+
+Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* logFile) {
+ unique_ptr<SequentialFileReader> file;
+ Status s = OpenLogFile(logFile, &file);
+ if (!s.ok()) {
+ return s;
+ }
+ assert(file);
+ currentLogReader_.reset(new log::Reader(
+ options_->info_log, std::move(file), &reporter_,
+ read_options_.verify_checksums_, 0, logFile->LogNumber()));
+ return Status::OK();
+}
+} // namespace rocksdb
+#endif // ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/transaction_log_impl.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/transaction_log_impl.h b/thirdparty/rocksdb/db/transaction_log_impl.h
new file mode 100644
index 0000000..769d833
--- /dev/null
+++ b/thirdparty/rocksdb/db/transaction_log_impl.h
@@ -0,0 +1,124 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+#pragma once
+
+#ifndef ROCKSDB_LITE
+#include <vector>
+
+#include "db/log_reader.h"
+#include "db/version_set.h"
+#include "options/db_options.h"
+#include "port/port.h"
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+#include "rocksdb/transaction_log.h"
+#include "rocksdb/types.h"
+#include "util/filename.h"
+
+namespace rocksdb {
+
+class LogFileImpl : public LogFile {
+ public:
+ LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
+ uint64_t sizeBytes) :
+ logNumber_(logNum),
+ type_(logType),
+ startSequence_(startSeq),
+ sizeFileBytes_(sizeBytes) {
+ }
+
+ std::string PathName() const override {
+ if (type_ == kArchivedLogFile) {
+ return ArchivedLogFileName("", logNumber_);
+ }
+ return LogFileName("", logNumber_);
+ }
+
+ uint64_t LogNumber() const override { return logNumber_; }
+
+ WalFileType Type() const override { return type_; }
+
+ SequenceNumber StartSequence() const override { return startSequence_; }
+
+ uint64_t SizeFileBytes() const override { return sizeFileBytes_; }
+
+ bool operator < (const LogFile& that) const {
+ return LogNumber() < that.LogNumber();
+ }
+
+ private:
+ uint64_t logNumber_;
+ WalFileType type_;
+ SequenceNumber startSequence_;
+ uint64_t sizeFileBytes_;
+
+};
+
+class TransactionLogIteratorImpl : public TransactionLogIterator {
+ public:
+ TransactionLogIteratorImpl(
+ const std::string& dir, const ImmutableDBOptions* options,
+ const TransactionLogIterator::ReadOptions& read_options,
+ const EnvOptions& soptions, const SequenceNumber seqNum,
+ std::unique_ptr<VectorLogPtr> files, VersionSet const* const versions);
+
+ virtual bool Valid() override;
+
+ virtual void Next() override;
+
+ virtual Status status() override;
+
+ virtual BatchResult GetBatch() override;
+
+ private:
+ const std::string& dir_;
+ const ImmutableDBOptions* options_;
+ const TransactionLogIterator::ReadOptions read_options_;
+ const EnvOptions& soptions_;
+ SequenceNumber startingSequenceNumber_;
+ std::unique_ptr<VectorLogPtr> files_;
+ bool started_;
+ bool isValid_; // not valid when it starts of.
+ Status currentStatus_;
+ size_t currentFileIndex_;
+ std::unique_ptr<WriteBatch> currentBatch_;
+ unique_ptr<log::Reader> currentLogReader_;
+ Status OpenLogFile(const LogFile* logFile,
+ unique_ptr<SequentialFileReader>* file);
+
+ struct LogReporter : public log::Reader::Reporter {
+ Env* env;
+ Logger* info_log;
+ virtual void Corruption(size_t bytes, const Status& s) override {
+ ROCKS_LOG_ERROR(info_log, "dropping %" ROCKSDB_PRIszt " bytes; %s", bytes,
+ s.ToString().c_str());
+ }
+ virtual void Info(const char* s) { ROCKS_LOG_INFO(info_log, "%s", s); }
+ } reporter_;
+
+ SequenceNumber currentBatchSeq_; // sequence number at start of current batch
+ SequenceNumber currentLastSeq_; // last sequence in the current batch
+ // Used only to get latest seq. num
+ // TODO(icanadi) can this be just a callback?
+ VersionSet const* const versions_;
+
+ // Reads from transaction log only if the writebatch record has been written
+ bool RestrictedRead(Slice* record, std::string* scratch);
+ // Seeks to startingSequenceNumber reading from startFileIndex in files_.
+ // If strict is set,then must get a batch starting with startingSequenceNumber
+ void SeekToStartSequence(uint64_t startFileIndex = 0, bool strict = false);
+ // Implementation of Next. SeekToStartSequence calls it internally with
+ // internal=true to let it find next entry even if it has to jump gaps because
+ // the iterator may start off from the first available entry but promises to
+ // be continuous after that
+ void NextImpl(bool internal = false);
+ // Check if batch is expected, else return false
+ bool IsBatchExpected(const WriteBatch* batch, SequenceNumber expectedSeq);
+ // Update current batch if a continuous batch is found, else return false
+ void UpdateCurrentWriteBatch(const Slice& record);
+ Status OpenLogReader(const LogFile* file);
+};
+} // namespace rocksdb
+#endif // ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/version_builder.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/version_builder.cc b/thirdparty/rocksdb/db/version_builder.cc
new file mode 100644
index 0000000..bab8d11
--- /dev/null
+++ b/thirdparty/rocksdb/db/version_builder.cc
@@ -0,0 +1,412 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/version_builder.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <algorithm>
+#include <atomic>
+#include <functional>
+#include <set>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "db/dbformat.h"
+#include "db/internal_stats.h"
+#include "db/table_cache.h"
+#include "db/version_set.h"
+#include "port/port.h"
+#include "table/table_reader.h"
+
+namespace rocksdb {
+
+bool NewestFirstBySeqNo(FileMetaData* a, FileMetaData* b) {
+ if (a->largest_seqno != b->largest_seqno) {
+ return a->largest_seqno > b->largest_seqno;
+ }
+ if (a->smallest_seqno != b->smallest_seqno) {
+ return a->smallest_seqno > b->smallest_seqno;
+ }
+ // Break ties by file number
+ return a->fd.GetNumber() > b->fd.GetNumber();
+}
+
+namespace {
+bool BySmallestKey(FileMetaData* a, FileMetaData* b,
+ const InternalKeyComparator* cmp) {
+ int r = cmp->Compare(a->smallest, b->smallest);
+ if (r != 0) {
+ return (r < 0);
+ }
+ // Break ties by file number
+ return (a->fd.GetNumber() < b->fd.GetNumber());
+}
+} // namespace
+
+class VersionBuilder::Rep {
+ private:
+ // Helper to sort files_ in v
+ // kLevel0 -- NewestFirstBySeqNo
+ // kLevelNon0 -- BySmallestKey
+ struct FileComparator {
+ enum SortMethod { kLevel0 = 0, kLevelNon0 = 1, } sort_method;
+ const InternalKeyComparator* internal_comparator;
+
+ bool operator()(FileMetaData* f1, FileMetaData* f2) const {
+ switch (sort_method) {
+ case kLevel0:
+ return NewestFirstBySeqNo(f1, f2);
+ case kLevelNon0:
+ return BySmallestKey(f1, f2, internal_comparator);
+ }
+ assert(false);
+ return false;
+ }
+ };
+
+ struct LevelState {
+ std::unordered_set<uint64_t> deleted_files;
+ // Map from file number to file meta data.
+ std::unordered_map<uint64_t, FileMetaData*> added_files;
+ };
+
+ const EnvOptions& env_options_;
+ Logger* info_log_;
+ TableCache* table_cache_;
+ VersionStorageInfo* base_vstorage_;
+ LevelState* levels_;
+ FileComparator level_zero_cmp_;
+ FileComparator level_nonzero_cmp_;
+
+ public:
+ Rep(const EnvOptions& env_options, Logger* info_log, TableCache* table_cache,
+ VersionStorageInfo* base_vstorage)
+ : env_options_(env_options),
+ info_log_(info_log),
+ table_cache_(table_cache),
+ base_vstorage_(base_vstorage) {
+ levels_ = new LevelState[base_vstorage_->num_levels()];
+ level_zero_cmp_.sort_method = FileComparator::kLevel0;
+ level_nonzero_cmp_.sort_method = FileComparator::kLevelNon0;
+ level_nonzero_cmp_.internal_comparator =
+ base_vstorage_->InternalComparator();
+ }
+
+ ~Rep() {
+ for (int level = 0; level < base_vstorage_->num_levels(); level++) {
+ const auto& added = levels_[level].added_files;
+ for (auto& pair : added) {
+ UnrefFile(pair.second);
+ }
+ }
+
+ delete[] levels_;
+ }
+
+ void UnrefFile(FileMetaData* f) {
+ f->refs--;
+ if (f->refs <= 0) {
+ if (f->table_reader_handle) {
+ assert(table_cache_ != nullptr);
+ table_cache_->ReleaseHandle(f->table_reader_handle);
+ f->table_reader_handle = nullptr;
+ }
+ delete f;
+ }
+ }
+
+ void CheckConsistency(VersionStorageInfo* vstorage) {
+#ifdef NDEBUG
+ if (!vstorage->force_consistency_checks()) {
+ // Dont run consistency checks in release mode except if
+ // explicitly asked to
+ return;
+ }
+#endif
+ // make sure the files are sorted correctly
+ for (int level = 0; level < vstorage->num_levels(); level++) {
+ auto& level_files = vstorage->LevelFiles(level);
+ for (size_t i = 1; i < level_files.size(); i++) {
+ auto f1 = level_files[i - 1];
+ auto f2 = level_files[i];
+ if (level == 0) {
+ if (!level_zero_cmp_(f1, f2)) {
+ fprintf(stderr, "L0 files are not sorted properly");
+ abort();
+ }
+
+ if (f2->smallest_seqno == f2->largest_seqno) {
+ // This is an external file that we ingested
+ SequenceNumber external_file_seqno = f2->smallest_seqno;
+ if (!(external_file_seqno < f1->largest_seqno ||
+ external_file_seqno == 0)) {
+ fprintf(stderr, "L0 file with seqno %" PRIu64 " %" PRIu64
+ " vs. file with global_seqno %" PRIu64 "\n",
+ f1->smallest_seqno, f1->largest_seqno,
+ external_file_seqno);
+ abort();
+ }
+ } else if (f1->smallest_seqno <= f2->smallest_seqno) {
+ fprintf(stderr, "L0 files seqno %" PRIu64 " %" PRIu64
+ " vs. %" PRIu64 " %" PRIu64 "\n",
+ f1->smallest_seqno, f1->largest_seqno, f2->smallest_seqno,
+ f2->largest_seqno);
+ abort();
+ }
+ } else {
+ if (!level_nonzero_cmp_(f1, f2)) {
+ fprintf(stderr, "L%d files are not sorted properly", level);
+ abort();
+ }
+
+ // Make sure there is no overlap in levels > 0
+ if (vstorage->InternalComparator()->Compare(f1->largest,
+ f2->smallest) >= 0) {
+ fprintf(stderr, "L%d have overlapping ranges %s vs. %s\n", level,
+ (f1->largest).DebugString(true).c_str(),
+ (f2->smallest).DebugString(true).c_str());
+ abort();
+ }
+ }
+ }
+ }
+ }
+
+ void CheckConsistencyForDeletes(VersionEdit* edit, uint64_t number,
+ int level) {
+#ifdef NDEBUG
+ if (!base_vstorage_->force_consistency_checks()) {
+ // Dont run consistency checks in release mode except if
+ // explicitly asked to
+ return;
+ }
+#endif
+ // a file to be deleted better exist in the previous version
+ bool found = false;
+ for (int l = 0; !found && l < base_vstorage_->num_levels(); l++) {
+ const std::vector<FileMetaData*>& base_files =
+ base_vstorage_->LevelFiles(l);
+ for (size_t i = 0; i < base_files.size(); i++) {
+ FileMetaData* f = base_files[i];
+ if (f->fd.GetNumber() == number) {
+ found = true;
+ break;
+ }
+ }
+ }
+ // if the file did not exist in the previous version, then it
+ // is possibly moved from lower level to higher level in current
+ // version
+ for (int l = level + 1; !found && l < base_vstorage_->num_levels(); l++) {
+ auto& level_added = levels_[l].added_files;
+ auto got = level_added.find(number);
+ if (got != level_added.end()) {
+ found = true;
+ break;
+ }
+ }
+
+ // maybe this file was added in a previous edit that was Applied
+ if (!found) {
+ auto& level_added = levels_[level].added_files;
+ auto got = level_added.find(number);
+ if (got != level_added.end()) {
+ found = true;
+ }
+ }
+ if (!found) {
+ fprintf(stderr, "not found %" PRIu64 "\n", number);
+ abort();
+ }
+ }
+
+ // Apply all of the edits in *edit to the current state.
+ void Apply(VersionEdit* edit) {
+ CheckConsistency(base_vstorage_);
+
+ // Delete files
+ const VersionEdit::DeletedFileSet& del = edit->GetDeletedFiles();
+ for (const auto& del_file : del) {
+ const auto level = del_file.first;
+ const auto number = del_file.second;
+ levels_[level].deleted_files.insert(number);
+ CheckConsistencyForDeletes(edit, number, level);
+
+ auto exising = levels_[level].added_files.find(number);
+ if (exising != levels_[level].added_files.end()) {
+ UnrefFile(exising->second);
+ levels_[level].added_files.erase(number);
+ }
+ }
+
+ // Add new files
+ for (const auto& new_file : edit->GetNewFiles()) {
+ const int level = new_file.first;
+ FileMetaData* f = new FileMetaData(new_file.second);
+ f->refs = 1;
+
+ assert(levels_[level].added_files.find(f->fd.GetNumber()) ==
+ levels_[level].added_files.end());
+ levels_[level].deleted_files.erase(f->fd.GetNumber());
+ levels_[level].added_files[f->fd.GetNumber()] = f;
+ }
+ }
+
+ // Save the current state in *v.
+ void SaveTo(VersionStorageInfo* vstorage) {
+ CheckConsistency(base_vstorage_);
+ CheckConsistency(vstorage);
+
+ for (int level = 0; level < base_vstorage_->num_levels(); level++) {
+ const auto& cmp = (level == 0) ? level_zero_cmp_ : level_nonzero_cmp_;
+ // Merge the set of added files with the set of pre-existing files.
+ // Drop any deleted files. Store the result in *v.
+ const auto& base_files = base_vstorage_->LevelFiles(level);
+ auto base_iter = base_files.begin();
+ auto base_end = base_files.end();
+ const auto& unordered_added_files = levels_[level].added_files;
+ vstorage->Reserve(level,
+ base_files.size() + unordered_added_files.size());
+
+ // Sort added files for the level.
+ std::vector<FileMetaData*> added_files;
+ added_files.reserve(unordered_added_files.size());
+ for (const auto& pair : unordered_added_files) {
+ added_files.push_back(pair.second);
+ }
+ std::sort(added_files.begin(), added_files.end(), cmp);
+
+#ifndef NDEBUG
+ FileMetaData* prev_file = nullptr;
+#endif
+
+ for (const auto& added : added_files) {
+#ifndef NDEBUG
+ if (level > 0 && prev_file != nullptr) {
+ assert(base_vstorage_->InternalComparator()->Compare(
+ prev_file->smallest, added->smallest) <= 0);
+ }
+ prev_file = added;
+#endif
+
+ // Add all smaller files listed in base_
+ for (auto bpos = std::upper_bound(base_iter, base_end, added, cmp);
+ base_iter != bpos; ++base_iter) {
+ MaybeAddFile(vstorage, level, *base_iter);
+ }
+
+ MaybeAddFile(vstorage, level, added);
+ }
+
+ // Add remaining base files
+ for (; base_iter != base_end; ++base_iter) {
+ MaybeAddFile(vstorage, level, *base_iter);
+ }
+ }
+
+ CheckConsistency(vstorage);
+ }
+
+ void LoadTableHandlers(InternalStats* internal_stats, int max_threads,
+ bool prefetch_index_and_filter_in_cache) {
+ assert(table_cache_ != nullptr);
+ // <file metadata, level>
+ std::vector<std::pair<FileMetaData*, int>> files_meta;
+ for (int level = 0; level < base_vstorage_->num_levels(); level++) {
+ for (auto& file_meta_pair : levels_[level].added_files) {
+ auto* file_meta = file_meta_pair.second;
+ assert(!file_meta->table_reader_handle);
+ files_meta.emplace_back(file_meta, level);
+ }
+ }
+
+ std::atomic<size_t> next_file_meta_idx(0);
+ std::function<void()> load_handlers_func = [&]() {
+ while (true) {
+ size_t file_idx = next_file_meta_idx.fetch_add(1);
+ if (file_idx >= files_meta.size()) {
+ break;
+ }
+
+ auto* file_meta = files_meta[file_idx].first;
+ int level = files_meta[file_idx].second;
+ table_cache_->FindTable(env_options_,
+ *(base_vstorage_->InternalComparator()),
+ file_meta->fd, &file_meta->table_reader_handle,
+ false /*no_io */, true /* record_read_stats */,
+ internal_stats->GetFileReadHist(level), false,
+ level, prefetch_index_and_filter_in_cache);
+ if (file_meta->table_reader_handle != nullptr) {
+ // Load table_reader
+ file_meta->fd.table_reader = table_cache_->GetTableReaderFromHandle(
+ file_meta->table_reader_handle);
+ }
+ }
+ };
+
+ if (max_threads <= 1) {
+ load_handlers_func();
+ } else {
+ std::vector<port::Thread> threads;
+ for (int i = 0; i < max_threads; i++) {
+ threads.emplace_back(load_handlers_func);
+ }
+
+ for (auto& t : threads) {
+ t.join();
+ }
+ }
+ }
+
+ void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f) {
+ if (levels_[level].deleted_files.count(f->fd.GetNumber()) > 0) {
+ // f is to-be-delected table file
+ vstorage->RemoveCurrentStats(f);
+ } else {
+ vstorage->AddFile(level, f, info_log_);
+ }
+ }
+};
+
+VersionBuilder::VersionBuilder(const EnvOptions& env_options,
+ TableCache* table_cache,
+ VersionStorageInfo* base_vstorage,
+ Logger* info_log)
+ : rep_(new Rep(env_options, info_log, table_cache, base_vstorage)) {}
+VersionBuilder::~VersionBuilder() { delete rep_; }
+void VersionBuilder::CheckConsistency(VersionStorageInfo* vstorage) {
+ rep_->CheckConsistency(vstorage);
+}
+void VersionBuilder::CheckConsistencyForDeletes(VersionEdit* edit,
+ uint64_t number, int level) {
+ rep_->CheckConsistencyForDeletes(edit, number, level);
+}
+void VersionBuilder::Apply(VersionEdit* edit) { rep_->Apply(edit); }
+void VersionBuilder::SaveTo(VersionStorageInfo* vstorage) {
+ rep_->SaveTo(vstorage);
+}
+void VersionBuilder::LoadTableHandlers(
+ InternalStats* internal_stats, int max_threads,
+ bool prefetch_index_and_filter_in_cache) {
+ rep_->LoadTableHandlers(internal_stats, max_threads,
+ prefetch_index_and_filter_in_cache);
+}
+void VersionBuilder::MaybeAddFile(VersionStorageInfo* vstorage, int level,
+ FileMetaData* f) {
+ rep_->MaybeAddFile(vstorage, level, f);
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/version_builder.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/version_builder.h b/thirdparty/rocksdb/db/version_builder.h
new file mode 100644
index 0000000..235f79d
--- /dev/null
+++ b/thirdparty/rocksdb/db/version_builder.h
@@ -0,0 +1,44 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+#pragma once
+#include "rocksdb/env.h"
+
+namespace rocksdb {
+
+class TableCache;
+class VersionStorageInfo;
+class VersionEdit;
+struct FileMetaData;
+class InternalStats;
+
+// A helper class so we can efficiently apply a whole sequence
+// of edits to a particular state without creating intermediate
+// Versions that contain full copies of the intermediate state.
+class VersionBuilder {
+ public:
+ VersionBuilder(const EnvOptions& env_options, TableCache* table_cache,
+ VersionStorageInfo* base_vstorage, Logger* info_log = nullptr);
+ ~VersionBuilder();
+ void CheckConsistency(VersionStorageInfo* vstorage);
+ void CheckConsistencyForDeletes(VersionEdit* edit, uint64_t number,
+ int level);
+ void Apply(VersionEdit* edit);
+ void SaveTo(VersionStorageInfo* vstorage);
+ void LoadTableHandlers(InternalStats* internal_stats, int max_threads,
+ bool prefetch_index_and_filter_in_cache);
+ void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f);
+
+ private:
+ class Rep;
+ Rep* rep_;
+};
+
+extern bool NewestFirstBySeqNo(FileMetaData* a, FileMetaData* b);
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/version_edit.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/version_edit.cc b/thirdparty/rocksdb/db/version_edit.cc
new file mode 100644
index 0000000..b01f7bb
--- /dev/null
+++ b/thirdparty/rocksdb/db/version_edit.cc
@@ -0,0 +1,591 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/version_edit.h"
+
+#include "db/version_set.h"
+#include "rocksdb/slice.h"
+#include "util/coding.h"
+#include "util/event_logger.h"
+#include "util/string_util.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+// Tag numbers for serialized VersionEdit. These numbers are written to
+// disk and should not be changed.
+enum Tag {
+ kComparator = 1,
+ kLogNumber = 2,
+ kNextFileNumber = 3,
+ kLastSequence = 4,
+ kCompactPointer = 5,
+ kDeletedFile = 6,
+ kNewFile = 7,
+ // 8 was used for large value refs
+ kPrevLogNumber = 9,
+
+ // these are new formats divergent from open source leveldb
+ kNewFile2 = 100,
+ kNewFile3 = 102,
+ kNewFile4 = 103, // 4th (the latest) format version of adding files
+ kColumnFamily = 200, // specify column family for version edit
+ kColumnFamilyAdd = 201,
+ kColumnFamilyDrop = 202,
+ kMaxColumnFamily = 203,
+};
+
+enum CustomTag {
+ kTerminate = 1, // The end of customized fields
+ kNeedCompaction = 2,
+ kPathId = 65,
+};
+// If this bit for the custom tag is set, opening DB should fail if
+// we don't know this field.
+uint32_t kCustomTagNonSafeIgnoreMask = 1 << 6;
+
+uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) {
+ assert(number <= kFileNumberMask);
+ return number | (path_id * (kFileNumberMask + 1));
+}
+
+void VersionEdit::Clear() {
+ comparator_.clear();
+ max_level_ = 0;
+ log_number_ = 0;
+ prev_log_number_ = 0;
+ last_sequence_ = 0;
+ next_file_number_ = 0;
+ max_column_family_ = 0;
+ has_comparator_ = false;
+ has_log_number_ = false;
+ has_prev_log_number_ = false;
+ has_next_file_number_ = false;
+ has_last_sequence_ = false;
+ has_max_column_family_ = false;
+ deleted_files_.clear();
+ new_files_.clear();
+ column_family_ = 0;
+ is_column_family_add_ = 0;
+ is_column_family_drop_ = 0;
+ column_family_name_.clear();
+}
+
+bool VersionEdit::EncodeTo(std::string* dst) const {
+ if (has_comparator_) {
+ PutVarint32(dst, kComparator);
+ PutLengthPrefixedSlice(dst, comparator_);
+ }
+ if (has_log_number_) {
+ PutVarint32Varint64(dst, kLogNumber, log_number_);
+ }
+ if (has_prev_log_number_) {
+ PutVarint32Varint64(dst, kPrevLogNumber, prev_log_number_);
+ }
+ if (has_next_file_number_) {
+ PutVarint32Varint64(dst, kNextFileNumber, next_file_number_);
+ }
+ if (has_last_sequence_) {
+ PutVarint32Varint64(dst, kLastSequence, last_sequence_);
+ }
+ if (has_max_column_family_) {
+ PutVarint32Varint32(dst, kMaxColumnFamily, max_column_family_);
+ }
+
+ for (const auto& deleted : deleted_files_) {
+ PutVarint32Varint32Varint64(dst, kDeletedFile, deleted.first /* level */,
+ deleted.second /* file number */);
+ }
+
+ for (size_t i = 0; i < new_files_.size(); i++) {
+ const FileMetaData& f = new_files_[i].second;
+ if (!f.smallest.Valid() || !f.largest.Valid()) {
+ return false;
+ }
+ bool has_customized_fields = false;
+ if (f.marked_for_compaction) {
+ PutVarint32(dst, kNewFile4);
+ has_customized_fields = true;
+ } else if (f.fd.GetPathId() == 0) {
+ // Use older format to make sure user can roll back the build if they
+ // don't config multiple DB paths.
+ PutVarint32(dst, kNewFile2);
+ } else {
+ PutVarint32(dst, kNewFile3);
+ }
+ PutVarint32Varint64(dst, new_files_[i].first /* level */, f.fd.GetNumber());
+ if (f.fd.GetPathId() != 0 && !has_customized_fields) {
+ // kNewFile3
+ PutVarint32(dst, f.fd.GetPathId());
+ }
+ PutVarint64(dst, f.fd.GetFileSize());
+ PutLengthPrefixedSlice(dst, f.smallest.Encode());
+ PutLengthPrefixedSlice(dst, f.largest.Encode());
+ PutVarint64Varint64(dst, f.smallest_seqno, f.largest_seqno);
+ if (has_customized_fields) {
+ // Customized fields' format:
+ // +-----------------------------+
+ // | 1st field's tag (varint32) |
+ // +-----------------------------+
+ // | 1st field's size (varint32) |
+ // +-----------------------------+
+ // | bytes for 1st field |
+ // | (based on size decoded) |
+ // +-----------------------------+
+ // | |
+ // | ...... |
+ // | |
+ // +-----------------------------+
+ // | last field's size (varint32)|
+ // +-----------------------------+
+ // | bytes for last field |
+ // | (based on size decoded) |
+ // +-----------------------------+
+ // | terminating tag (varint32) |
+ // +-----------------------------+
+ //
+ // Customized encoding for fields:
+ // tag kPathId: 1 byte as path_id
+ // tag kNeedCompaction:
+ // now only can take one char value 1 indicating need-compaction
+ //
+ if (f.fd.GetPathId() != 0) {
+ PutVarint32(dst, CustomTag::kPathId);
+ char p = static_cast<char>(f.fd.GetPathId());
+ PutLengthPrefixedSlice(dst, Slice(&p, 1));
+ }
+ if (f.marked_for_compaction) {
+ PutVarint32(dst, CustomTag::kNeedCompaction);
+ char p = static_cast<char>(1);
+ PutLengthPrefixedSlice(dst, Slice(&p, 1));
+ }
+ TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:NewFile4:CustomizeFields",
+ dst);
+
+ PutVarint32(dst, CustomTag::kTerminate);
+ }
+ }
+
+ // 0 is default and does not need to be explicitly written
+ if (column_family_ != 0) {
+ PutVarint32Varint32(dst, kColumnFamily, column_family_);
+ }
+
+ if (is_column_family_add_) {
+ PutVarint32(dst, kColumnFamilyAdd);
+ PutLengthPrefixedSlice(dst, Slice(column_family_name_));
+ }
+
+ if (is_column_family_drop_) {
+ PutVarint32(dst, kColumnFamilyDrop);
+ }
+ return true;
+}
+
+static bool GetInternalKey(Slice* input, InternalKey* dst) {
+ Slice str;
+ if (GetLengthPrefixedSlice(input, &str)) {
+ dst->DecodeFrom(str);
+ return dst->Valid();
+ } else {
+ return false;
+ }
+}
+
+bool VersionEdit::GetLevel(Slice* input, int* level, const char** msg) {
+ uint32_t v;
+ if (GetVarint32(input, &v)) {
+ *level = v;
+ if (max_level_ < *level) {
+ max_level_ = *level;
+ }
+ return true;
+ } else {
+ return false;
+ }
+}
+
+const char* VersionEdit::DecodeNewFile4From(Slice* input) {
+ const char* msg = nullptr;
+ int level;
+ FileMetaData f;
+ uint64_t number;
+ uint32_t path_id = 0;
+ uint64_t file_size;
+ if (GetLevel(input, &level, &msg) && GetVarint64(input, &number) &&
+ GetVarint64(input, &file_size) && GetInternalKey(input, &f.smallest) &&
+ GetInternalKey(input, &f.largest) &&
+ GetVarint64(input, &f.smallest_seqno) &&
+ GetVarint64(input, &f.largest_seqno)) {
+ // See comments in VersionEdit::EncodeTo() for format of customized fields
+ while (true) {
+ uint32_t custom_tag;
+ Slice field;
+ if (!GetVarint32(input, &custom_tag)) {
+ return "new-file4 custom field";
+ }
+ if (custom_tag == kTerminate) {
+ break;
+ }
+ if (!GetLengthPrefixedSlice(input, &field)) {
+ return "new-file4 custom field lenth prefixed slice error";
+ }
+ switch (custom_tag) {
+ case kPathId:
+ if (field.size() != 1) {
+ return "path_id field wrong size";
+ }
+ path_id = field[0];
+ if (path_id > 3) {
+ return "path_id wrong vaue";
+ }
+ break;
+ case kNeedCompaction:
+ if (field.size() != 1) {
+ return "need_compaction field wrong size";
+ }
+ f.marked_for_compaction = (field[0] == 1);
+ break;
+ default:
+ if ((custom_tag & kCustomTagNonSafeIgnoreMask) != 0) {
+ // Should not proceed if cannot understand it
+ return "new-file4 custom field not supported";
+ }
+ break;
+ }
+ }
+ } else {
+ return "new-file4 entry";
+ }
+ f.fd = FileDescriptor(number, path_id, file_size);
+ new_files_.push_back(std::make_pair(level, f));
+ return nullptr;
+}
+
+Status VersionEdit::DecodeFrom(const Slice& src) {
+ Clear();
+ Slice input = src;
+ const char* msg = nullptr;
+ uint32_t tag;
+
+ // Temporary storage for parsing
+ int level;
+ FileMetaData f;
+ Slice str;
+ InternalKey key;
+
+ while (msg == nullptr && GetVarint32(&input, &tag)) {
+ switch (tag) {
+ case kComparator:
+ if (GetLengthPrefixedSlice(&input, &str)) {
+ comparator_ = str.ToString();
+ has_comparator_ = true;
+ } else {
+ msg = "comparator name";
+ }
+ break;
+
+ case kLogNumber:
+ if (GetVarint64(&input, &log_number_)) {
+ has_log_number_ = true;
+ } else {
+ msg = "log number";
+ }
+ break;
+
+ case kPrevLogNumber:
+ if (GetVarint64(&input, &prev_log_number_)) {
+ has_prev_log_number_ = true;
+ } else {
+ msg = "previous log number";
+ }
+ break;
+
+ case kNextFileNumber:
+ if (GetVarint64(&input, &next_file_number_)) {
+ has_next_file_number_ = true;
+ } else {
+ msg = "next file number";
+ }
+ break;
+
+ case kLastSequence:
+ if (GetVarint64(&input, &last_sequence_)) {
+ has_last_sequence_ = true;
+ } else {
+ msg = "last sequence number";
+ }
+ break;
+
+ case kMaxColumnFamily:
+ if (GetVarint32(&input, &max_column_family_)) {
+ has_max_column_family_ = true;
+ } else {
+ msg = "max column family";
+ }
+ break;
+
+ case kCompactPointer:
+ if (GetLevel(&input, &level, &msg) &&
+ GetInternalKey(&input, &key)) {
+ // we don't use compact pointers anymore,
+ // but we should not fail if they are still
+ // in manifest
+ } else {
+ if (!msg) {
+ msg = "compaction pointer";
+ }
+ }
+ break;
+
+ case kDeletedFile: {
+ uint64_t number;
+ if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number)) {
+ deleted_files_.insert(std::make_pair(level, number));
+ } else {
+ if (!msg) {
+ msg = "deleted file";
+ }
+ }
+ break;
+ }
+
+ case kNewFile: {
+ uint64_t number;
+ uint64_t file_size;
+ if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) &&
+ GetVarint64(&input, &file_size) &&
+ GetInternalKey(&input, &f.smallest) &&
+ GetInternalKey(&input, &f.largest)) {
+ f.fd = FileDescriptor(number, 0, file_size);
+ new_files_.push_back(std::make_pair(level, f));
+ } else {
+ if (!msg) {
+ msg = "new-file entry";
+ }
+ }
+ break;
+ }
+ case kNewFile2: {
+ uint64_t number;
+ uint64_t file_size;
+ if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) &&
+ GetVarint64(&input, &file_size) &&
+ GetInternalKey(&input, &f.smallest) &&
+ GetInternalKey(&input, &f.largest) &&
+ GetVarint64(&input, &f.smallest_seqno) &&
+ GetVarint64(&input, &f.largest_seqno)) {
+ f.fd = FileDescriptor(number, 0, file_size);
+ new_files_.push_back(std::make_pair(level, f));
+ } else {
+ if (!msg) {
+ msg = "new-file2 entry";
+ }
+ }
+ break;
+ }
+
+ case kNewFile3: {
+ uint64_t number;
+ uint32_t path_id;
+ uint64_t file_size;
+ if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number) &&
+ GetVarint32(&input, &path_id) && GetVarint64(&input, &file_size) &&
+ GetInternalKey(&input, &f.smallest) &&
+ GetInternalKey(&input, &f.largest) &&
+ GetVarint64(&input, &f.smallest_seqno) &&
+ GetVarint64(&input, &f.largest_seqno)) {
+ f.fd = FileDescriptor(number, path_id, file_size);
+ new_files_.push_back(std::make_pair(level, f));
+ } else {
+ if (!msg) {
+ msg = "new-file3 entry";
+ }
+ }
+ break;
+ }
+
+ case kNewFile4: {
+ msg = DecodeNewFile4From(&input);
+ break;
+ }
+
+ case kColumnFamily:
+ if (!GetVarint32(&input, &column_family_)) {
+ if (!msg) {
+ msg = "set column family id";
+ }
+ }
+ break;
+
+ case kColumnFamilyAdd:
+ if (GetLengthPrefixedSlice(&input, &str)) {
+ is_column_family_add_ = true;
+ column_family_name_ = str.ToString();
+ } else {
+ if (!msg) {
+ msg = "column family add";
+ }
+ }
+ break;
+
+ case kColumnFamilyDrop:
+ is_column_family_drop_ = true;
+ break;
+
+ default:
+ msg = "unknown tag";
+ break;
+ }
+ }
+
+ if (msg == nullptr && !input.empty()) {
+ msg = "invalid tag";
+ }
+
+ Status result;
+ if (msg != nullptr) {
+ result = Status::Corruption("VersionEdit", msg);
+ }
+ return result;
+}
+
+std::string VersionEdit::DebugString(bool hex_key) const {
+ std::string r;
+ r.append("VersionEdit {");
+ if (has_comparator_) {
+ r.append("\n Comparator: ");
+ r.append(comparator_);
+ }
+ if (has_log_number_) {
+ r.append("\n LogNumber: ");
+ AppendNumberTo(&r, log_number_);
+ }
+ if (has_prev_log_number_) {
+ r.append("\n PrevLogNumber: ");
+ AppendNumberTo(&r, prev_log_number_);
+ }
+ if (has_next_file_number_) {
+ r.append("\n NextFileNumber: ");
+ AppendNumberTo(&r, next_file_number_);
+ }
+ if (has_last_sequence_) {
+ r.append("\n LastSeq: ");
+ AppendNumberTo(&r, last_sequence_);
+ }
+ for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
+ iter != deleted_files_.end();
+ ++iter) {
+ r.append("\n DeleteFile: ");
+ AppendNumberTo(&r, iter->first);
+ r.append(" ");
+ AppendNumberTo(&r, iter->second);
+ }
+ for (size_t i = 0; i < new_files_.size(); i++) {
+ const FileMetaData& f = new_files_[i].second;
+ r.append("\n AddFile: ");
+ AppendNumberTo(&r, new_files_[i].first);
+ r.append(" ");
+ AppendNumberTo(&r, f.fd.GetNumber());
+ r.append(" ");
+ AppendNumberTo(&r, f.fd.GetFileSize());
+ r.append(" ");
+ r.append(f.smallest.DebugString(hex_key));
+ r.append(" .. ");
+ r.append(f.largest.DebugString(hex_key));
+ }
+ r.append("\n ColumnFamily: ");
+ AppendNumberTo(&r, column_family_);
+ if (is_column_family_add_) {
+ r.append("\n ColumnFamilyAdd: ");
+ r.append(column_family_name_);
+ }
+ if (is_column_family_drop_) {
+ r.append("\n ColumnFamilyDrop");
+ }
+ if (has_max_column_family_) {
+ r.append("\n MaxColumnFamily: ");
+ AppendNumberTo(&r, max_column_family_);
+ }
+ r.append("\n}\n");
+ return r;
+}
+
+std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
+ JSONWriter jw;
+ jw << "EditNumber" << edit_num;
+
+ if (has_comparator_) {
+ jw << "Comparator" << comparator_;
+ }
+ if (has_log_number_) {
+ jw << "LogNumber" << log_number_;
+ }
+ if (has_prev_log_number_) {
+ jw << "PrevLogNumber" << prev_log_number_;
+ }
+ if (has_next_file_number_) {
+ jw << "NextFileNumber" << next_file_number_;
+ }
+ if (has_last_sequence_) {
+ jw << "LastSeq" << last_sequence_;
+ }
+
+ if (!deleted_files_.empty()) {
+ jw << "DeletedFiles";
+ jw.StartArray();
+
+ for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
+ iter != deleted_files_.end();
+ ++iter) {
+ jw.StartArrayedObject();
+ jw << "Level" << iter->first;
+ jw << "FileNumber" << iter->second;
+ jw.EndArrayedObject();
+ }
+
+ jw.EndArray();
+ }
+
+ if (!new_files_.empty()) {
+ jw << "AddedFiles";
+ jw.StartArray();
+
+ for (size_t i = 0; i < new_files_.size(); i++) {
+ jw.StartArrayedObject();
+ jw << "Level" << new_files_[i].first;
+ const FileMetaData& f = new_files_[i].second;
+ jw << "FileNumber" << f.fd.GetNumber();
+ jw << "FileSize" << f.fd.GetFileSize();
+ jw << "SmallestIKey" << f.smallest.DebugString(hex_key);
+ jw << "LargestIKey" << f.largest.DebugString(hex_key);
+ jw.EndArrayedObject();
+ }
+
+ jw.EndArray();
+ }
+
+ jw << "ColumnFamily" << column_family_;
+
+ if (is_column_family_add_) {
+ jw << "ColumnFamilyAdd" << column_family_name_;
+ }
+ if (is_column_family_drop_) {
+ jw << "ColumnFamilyDrop" << column_family_name_;
+ }
+ if (has_max_column_family_) {
+ jw << "MaxColumnFamily" << max_column_family_;
+ }
+
+ jw.EndObject();
+
+ return jw.Get();
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/version_edit.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/version_edit.h b/thirdparty/rocksdb/db/version_edit.h
new file mode 100644
index 0000000..47ebf5b
--- /dev/null
+++ b/thirdparty/rocksdb/db/version_edit.h
@@ -0,0 +1,309 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <algorithm>
+#include <set>
+#include <utility>
+#include <vector>
+#include <string>
+#include "rocksdb/cache.h"
+#include "db/dbformat.h"
+#include "util/arena.h"
+#include "util/autovector.h"
+
+namespace rocksdb {
+
+class VersionSet;
+
+const uint64_t kFileNumberMask = 0x3FFFFFFFFFFFFFFF;
+
+extern uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id);
+
+// A copyable structure contains information needed to read data from an SST
+// file. It can contains a pointer to a table reader opened for the file, or
+// file number and size, which can be used to create a new table reader for it.
+// The behavior is undefined when a copied of the structure is used when the
+// file is not in any live version any more.
+struct FileDescriptor {
+ // Table reader in table_reader_handle
+ TableReader* table_reader;
+ uint64_t packed_number_and_path_id;
+ uint64_t file_size; // File size in bytes
+
+ FileDescriptor() : FileDescriptor(0, 0, 0) {}
+
+ FileDescriptor(uint64_t number, uint32_t path_id, uint64_t _file_size)
+ : table_reader(nullptr),
+ packed_number_and_path_id(PackFileNumberAndPathId(number, path_id)),
+ file_size(_file_size) {}
+
+ FileDescriptor& operator=(const FileDescriptor& fd) {
+ table_reader = fd.table_reader;
+ packed_number_and_path_id = fd.packed_number_and_path_id;
+ file_size = fd.file_size;
+ return *this;
+ }
+
+ uint64_t GetNumber() const {
+ return packed_number_and_path_id & kFileNumberMask;
+ }
+ uint32_t GetPathId() const {
+ return static_cast<uint32_t>(
+ packed_number_and_path_id / (kFileNumberMask + 1));
+ }
+ uint64_t GetFileSize() const { return file_size; }
+};
+
+struct FileSampledStats {
+ FileSampledStats() : num_reads_sampled(0) {}
+ FileSampledStats(const FileSampledStats& other) { *this = other; }
+ FileSampledStats& operator=(const FileSampledStats& other) {
+ num_reads_sampled = other.num_reads_sampled.load();
+ return *this;
+ }
+
+ // number of user reads to this file.
+ mutable std::atomic<uint64_t> num_reads_sampled;
+};
+
+struct FileMetaData {
+ FileDescriptor fd;
+ InternalKey smallest; // Smallest internal key served by table
+ InternalKey largest; // Largest internal key served by table
+ SequenceNumber smallest_seqno; // The smallest seqno in this file
+ SequenceNumber largest_seqno; // The largest seqno in this file
+
+ // Needs to be disposed when refs becomes 0.
+ Cache::Handle* table_reader_handle;
+
+ FileSampledStats stats;
+
+ // Stats for compensating deletion entries during compaction
+
+ // File size compensated by deletion entry.
+ // This is updated in Version::UpdateAccumulatedStats() first time when the
+ // file is created or loaded. After it is updated (!= 0), it is immutable.
+ uint64_t compensated_file_size;
+ // These values can mutate, but they can only be read or written from
+ // single-threaded LogAndApply thread
+ uint64_t num_entries; // the number of entries.
+ uint64_t num_deletions; // the number of deletion entries.
+ uint64_t raw_key_size; // total uncompressed key size.
+ uint64_t raw_value_size; // total uncompressed value size.
+
+ int refs; // Reference count
+
+ bool being_compacted; // Is this file undergoing compaction?
+ bool init_stats_from_file; // true if the data-entry stats of this file
+ // has initialized from file.
+
+ bool marked_for_compaction; // True if client asked us nicely to compact this
+ // file.
+
+ FileMetaData()
+ : smallest_seqno(kMaxSequenceNumber),
+ largest_seqno(0),
+ table_reader_handle(nullptr),
+ compensated_file_size(0),
+ num_entries(0),
+ num_deletions(0),
+ raw_key_size(0),
+ raw_value_size(0),
+ refs(0),
+ being_compacted(false),
+ init_stats_from_file(false),
+ marked_for_compaction(false) {}
+
+ // REQUIRED: Keys must be given to the function in sorted order (it expects
+ // the last key to be the largest).
+ void UpdateBoundaries(const Slice& key, SequenceNumber seqno) {
+ if (smallest.size() == 0) {
+ smallest.DecodeFrom(key);
+ }
+ largest.DecodeFrom(key);
+ smallest_seqno = std::min(smallest_seqno, seqno);
+ largest_seqno = std::max(largest_seqno, seqno);
+ }
+};
+
+// A compressed copy of file meta data that just contain minimum data needed
+// to server read operations, while still keeping the pointer to full metadata
+// of the file in case it is needed.
+struct FdWithKeyRange {
+ FileDescriptor fd;
+ FileMetaData* file_metadata; // Point to all metadata
+ Slice smallest_key; // slice that contain smallest key
+ Slice largest_key; // slice that contain largest key
+
+ FdWithKeyRange()
+ : fd(),
+ smallest_key(),
+ largest_key() {
+ }
+
+ FdWithKeyRange(FileDescriptor _fd, Slice _smallest_key, Slice _largest_key,
+ FileMetaData* _file_metadata)
+ : fd(_fd),
+ file_metadata(_file_metadata),
+ smallest_key(_smallest_key),
+ largest_key(_largest_key) {}
+};
+
+// Data structure to store an array of FdWithKeyRange in one level
+// Actual data is guaranteed to be stored closely
+struct LevelFilesBrief {
+ size_t num_files;
+ FdWithKeyRange* files;
+ LevelFilesBrief() {
+ num_files = 0;
+ files = nullptr;
+ }
+};
+
+class VersionEdit {
+ public:
+ VersionEdit() { Clear(); }
+ ~VersionEdit() { }
+
+ void Clear();
+
+ void SetComparatorName(const Slice& name) {
+ has_comparator_ = true;
+ comparator_ = name.ToString();
+ }
+ void SetLogNumber(uint64_t num) {
+ has_log_number_ = true;
+ log_number_ = num;
+ }
+ void SetPrevLogNumber(uint64_t num) {
+ has_prev_log_number_ = true;
+ prev_log_number_ = num;
+ }
+ void SetNextFile(uint64_t num) {
+ has_next_file_number_ = true;
+ next_file_number_ = num;
+ }
+ void SetLastSequence(SequenceNumber seq) {
+ has_last_sequence_ = true;
+ last_sequence_ = seq;
+ }
+ void SetMaxColumnFamily(uint32_t max_column_family) {
+ has_max_column_family_ = true;
+ max_column_family_ = max_column_family;
+ }
+
+ // Add the specified file at the specified number.
+ // REQUIRES: This version has not been saved (see VersionSet::SaveTo)
+ // REQUIRES: "smallest" and "largest" are smallest and largest keys in file
+ void AddFile(int level, uint64_t file, uint32_t file_path_id,
+ uint64_t file_size, const InternalKey& smallest,
+ const InternalKey& largest, const SequenceNumber& smallest_seqno,
+ const SequenceNumber& largest_seqno,
+ bool marked_for_compaction) {
+ assert(smallest_seqno <= largest_seqno);
+ FileMetaData f;
+ f.fd = FileDescriptor(file, file_path_id, file_size);
+ f.smallest = smallest;
+ f.largest = largest;
+ f.smallest_seqno = smallest_seqno;
+ f.largest_seqno = largest_seqno;
+ f.marked_for_compaction = marked_for_compaction;
+ new_files_.emplace_back(level, std::move(f));
+ }
+
+ void AddFile(int level, const FileMetaData& f) {
+ assert(f.smallest_seqno <= f.largest_seqno);
+ new_files_.emplace_back(level, f);
+ }
+
+ // Delete the specified "file" from the specified "level".
+ void DeleteFile(int level, uint64_t file) {
+ deleted_files_.insert({level, file});
+ }
+
+ // Number of edits
+ size_t NumEntries() { return new_files_.size() + deleted_files_.size(); }
+
+ bool IsColumnFamilyManipulation() {
+ return is_column_family_add_ || is_column_family_drop_;
+ }
+
+ void SetColumnFamily(uint32_t column_family_id) {
+ column_family_ = column_family_id;
+ }
+
+ // set column family ID by calling SetColumnFamily()
+ void AddColumnFamily(const std::string& name) {
+ assert(!is_column_family_drop_);
+ assert(!is_column_family_add_);
+ assert(NumEntries() == 0);
+ is_column_family_add_ = true;
+ column_family_name_ = name;
+ }
+
+ // set column family ID by calling SetColumnFamily()
+ void DropColumnFamily() {
+ assert(!is_column_family_drop_);
+ assert(!is_column_family_add_);
+ assert(NumEntries() == 0);
+ is_column_family_drop_ = true;
+ }
+
+ // return true on success.
+ bool EncodeTo(std::string* dst) const;
+ Status DecodeFrom(const Slice& src);
+
+ const char* DecodeNewFile4From(Slice* input);
+
+ typedef std::set<std::pair<int, uint64_t>> DeletedFileSet;
+
+ const DeletedFileSet& GetDeletedFiles() { return deleted_files_; }
+ const std::vector<std::pair<int, FileMetaData>>& GetNewFiles() {
+ return new_files_;
+ }
+
+ std::string DebugString(bool hex_key = false) const;
+ std::string DebugJSON(int edit_num, bool hex_key = false) const;
+
+ private:
+ friend class VersionSet;
+ friend class Version;
+
+ bool GetLevel(Slice* input, int* level, const char** msg);
+
+ int max_level_;
+ std::string comparator_;
+ uint64_t log_number_;
+ uint64_t prev_log_number_;
+ uint64_t next_file_number_;
+ uint32_t max_column_family_;
+ SequenceNumber last_sequence_;
+ bool has_comparator_;
+ bool has_log_number_;
+ bool has_prev_log_number_;
+ bool has_next_file_number_;
+ bool has_last_sequence_;
+ bool has_max_column_family_;
+
+ DeletedFileSet deleted_files_;
+ std::vector<std::pair<int, FileMetaData>> new_files_;
+
+ // Each version edit record should have column_family_id set
+ // If it's not set, it is default (0)
+ uint32_t column_family_;
+ // a version edit can be either column_family add or
+ // column_family drop. If it's column family add,
+ // it also includes column family name.
+ bool is_column_family_drop_;
+ bool is_column_family_add_;
+ std::string column_family_name_;
+};
+
+} // namespace rocksdb
[23/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_impl.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_impl.cc b/thirdparty/rocksdb/db/db_impl.cc
new file mode 100644
index 0000000..cdba039
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_impl.cc
@@ -0,0 +1,2824 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include "db/db_impl.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+#include <inttypes.h>
+#include <stdint.h>
+#ifdef OS_SOLARIS
+#include <alloca.h>
+#endif
+
+#include <algorithm>
+#include <climits>
+#include <cstdio>
+#include <map>
+#include <set>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "db/builder.h"
+#include "db/compaction_job.h"
+#include "db/db_info_dumper.h"
+#include "db/db_iter.h"
+#include "db/dbformat.h"
+#include "db/event_helpers.h"
+#include "db/external_sst_file_ingestion_job.h"
+#include "db/flush_job.h"
+#include "db/forward_iterator.h"
+#include "db/job_context.h"
+#include "db/log_reader.h"
+#include "db/log_writer.h"
+#include "db/malloc_stats.h"
+#include "db/managed_iterator.h"
+#include "db/memtable.h"
+#include "db/memtable_list.h"
+#include "db/merge_context.h"
+#include "db/merge_helper.h"
+#include "db/range_del_aggregator.h"
+#include "db/table_cache.h"
+#include "db/table_properties_collector.h"
+#include "db/transaction_log_impl.h"
+#include "db/version_set.h"
+#include "db/write_batch_internal.h"
+#include "db/write_callback.h"
+#include "memtable/hash_linklist_rep.h"
+#include "memtable/hash_skiplist_rep.h"
+#include "monitoring/iostats_context_imp.h"
+#include "monitoring/perf_context_imp.h"
+#include "monitoring/thread_status_updater.h"
+#include "monitoring/thread_status_util.h"
+#include "options/cf_options.h"
+#include "options/options_helper.h"
+#include "options/options_parser.h"
+#include "port/likely.h"
+#include "port/port.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/status.h"
+#include "rocksdb/table.h"
+#include "rocksdb/version.h"
+#include "rocksdb/write_buffer_manager.h"
+#include "table/block.h"
+#include "table/block_based_table_factory.h"
+#include "table/merging_iterator.h"
+#include "table/table_builder.h"
+#include "table/two_level_iterator.h"
+#include "tools/sst_dump_tool_imp.h"
+#include "util/auto_roll_logger.h"
+#include "util/autovector.h"
+#include "util/build_version.h"
+#include "util/coding.h"
+#include "util/compression.h"
+#include "util/crc32c.h"
+#include "util/file_reader_writer.h"
+#include "util/file_util.h"
+#include "util/filename.h"
+#include "util/log_buffer.h"
+#include "util/logging.h"
+#include "util/mutexlock.h"
+#include "util/sst_file_manager_impl.h"
+#include "util/stop_watch.h"
+#include "util/string_util.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+const std::string kDefaultColumnFamilyName("default");
+void DumpRocksDBBuildVersion(Logger * log);
+
+CompressionType GetCompressionFlush(
+ const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& mutable_cf_options) {
+ // Compressing memtable flushes might not help unless the sequential load
+ // optimization is used for leveled compaction. Otherwise the CPU and
+ // latency overhead is not offset by saving much space.
+ if (ioptions.compaction_style == kCompactionStyleUniversal) {
+ if (ioptions.compaction_options_universal.compression_size_percent < 0) {
+ return mutable_cf_options.compression;
+ } else {
+ return kNoCompression;
+ }
+ } else if (!ioptions.compression_per_level.empty()) {
+ // For leveled compress when min_level_to_compress != 0.
+ return ioptions.compression_per_level[0];
+ } else {
+ return mutable_cf_options.compression;
+ }
+}
+
+namespace {
+void DumpSupportInfo(Logger* logger) {
+ ROCKS_LOG_HEADER(logger, "Compression algorithms supported:");
+ ROCKS_LOG_HEADER(logger, "\tSnappy supported: %d", Snappy_Supported());
+ ROCKS_LOG_HEADER(logger, "\tZlib supported: %d", Zlib_Supported());
+ ROCKS_LOG_HEADER(logger, "\tBzip supported: %d", BZip2_Supported());
+ ROCKS_LOG_HEADER(logger, "\tLZ4 supported: %d", LZ4_Supported());
+ ROCKS_LOG_HEADER(logger, "\tZSTD supported: %d", ZSTD_Supported());
+ ROCKS_LOG_HEADER(logger, "Fast CRC32 supported: %d",
+ crc32c::IsFastCrc32Supported());
+}
+
+int64_t kDefaultLowPriThrottledRate = 2 * 1024 * 1024;
+} // namespace
+
+DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
+ : env_(options.env),
+ dbname_(dbname),
+ initial_db_options_(SanitizeOptions(dbname, options)),
+ immutable_db_options_(initial_db_options_),
+ mutable_db_options_(initial_db_options_),
+ stats_(immutable_db_options_.statistics.get()),
+ db_lock_(nullptr),
+ mutex_(stats_, env_, DB_MUTEX_WAIT_MICROS,
+ immutable_db_options_.use_adaptive_mutex),
+ shutting_down_(false),
+ bg_cv_(&mutex_),
+ logfile_number_(0),
+ log_dir_synced_(false),
+ log_empty_(true),
+ default_cf_handle_(nullptr),
+ log_sync_cv_(&mutex_),
+ total_log_size_(0),
+ max_total_in_memory_state_(0),
+ is_snapshot_supported_(true),
+ write_buffer_manager_(immutable_db_options_.write_buffer_manager.get()),
+ write_thread_(immutable_db_options_),
+ nonmem_write_thread_(immutable_db_options_),
+ write_controller_(mutable_db_options_.delayed_write_rate),
+ // Use delayed_write_rate as a base line to determine the initial
+ // low pri write rate limit. It may be adjusted later.
+ low_pri_write_rate_limiter_(NewGenericRateLimiter(std::min(
+ static_cast<int64_t>(mutable_db_options_.delayed_write_rate / 8),
+ kDefaultLowPriThrottledRate))),
+ last_batch_group_size_(0),
+ unscheduled_flushes_(0),
+ unscheduled_compactions_(0),
+ bg_bottom_compaction_scheduled_(0),
+ bg_compaction_scheduled_(0),
+ num_running_compactions_(0),
+ bg_flush_scheduled_(0),
+ num_running_flushes_(0),
+ bg_purge_scheduled_(0),
+ disable_delete_obsolete_files_(0),
+ delete_obsolete_files_last_run_(env_->NowMicros()),
+ last_stats_dump_time_microsec_(0),
+ next_job_id_(1),
+ has_unpersisted_data_(false),
+ unable_to_flush_oldest_log_(false),
+ env_options_(BuildDBOptions(immutable_db_options_, mutable_db_options_)),
+ num_running_ingest_file_(0),
+#ifndef ROCKSDB_LITE
+ wal_manager_(immutable_db_options_, env_options_),
+#endif // ROCKSDB_LITE
+ event_logger_(immutable_db_options_.info_log.get()),
+ bg_work_paused_(0),
+ bg_compaction_paused_(0),
+ refitting_level_(false),
+ opened_successfully_(false),
+ concurrent_prepare_(options.concurrent_prepare),
+ manual_wal_flush_(options.manual_wal_flush) {
+ env_->GetAbsolutePath(dbname, &db_absolute_path_);
+
+ // Reserve ten files or so for other uses and give the rest to TableCache.
+ // Give a large number for setting of "infinite" open files.
+ const int table_cache_size = (mutable_db_options_.max_open_files == -1)
+ ? TableCache::kInfiniteCapacity
+ : mutable_db_options_.max_open_files - 10;
+ table_cache_ = NewLRUCache(table_cache_size,
+ immutable_db_options_.table_cache_numshardbits);
+
+ versions_.reset(new VersionSet(dbname_, &immutable_db_options_, env_options_,
+ table_cache_.get(), write_buffer_manager_,
+ &write_controller_));
+ column_family_memtables_.reset(
+ new ColumnFamilyMemTablesImpl(versions_->GetColumnFamilySet()));
+
+ DumpRocksDBBuildVersion(immutable_db_options_.info_log.get());
+ DumpDBFileSummary(immutable_db_options_, dbname_);
+ immutable_db_options_.Dump(immutable_db_options_.info_log.get());
+ mutable_db_options_.Dump(immutable_db_options_.info_log.get());
+ DumpSupportInfo(immutable_db_options_.info_log.get());
+}
+
+// Will lock the mutex_, will wait for completion if wait is true
+void DBImpl::CancelAllBackgroundWork(bool wait) {
+ InstrumentedMutexLock l(&mutex_);
+
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "Shutdown: canceling all background work");
+
+ if (!shutting_down_.load(std::memory_order_acquire) &&
+ has_unpersisted_data_.load(std::memory_order_relaxed) &&
+ !mutable_db_options_.avoid_flush_during_shutdown) {
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ if (!cfd->IsDropped() && cfd->initialized() && !cfd->mem()->IsEmpty()) {
+ cfd->Ref();
+ mutex_.Unlock();
+ FlushMemTable(cfd, FlushOptions());
+ mutex_.Lock();
+ cfd->Unref();
+ }
+ }
+ versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();
+ }
+
+ shutting_down_.store(true, std::memory_order_release);
+ bg_cv_.SignalAll();
+ if (!wait) {
+ return;
+ }
+ // Wait for background work to finish
+ while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
+ bg_flush_scheduled_) {
+ bg_cv_.Wait();
+ }
+}
+
+DBImpl::~DBImpl() {
+ // CancelAllBackgroundWork called with false means we just set the shutdown
+ // marker. After this we do a variant of the waiting and unschedule work
+ // (to consider: moving all the waiting into CancelAllBackgroundWork(true))
+ CancelAllBackgroundWork(false);
+ int bottom_compactions_unscheduled =
+ env_->UnSchedule(this, Env::Priority::BOTTOM);
+ int compactions_unscheduled = env_->UnSchedule(this, Env::Priority::LOW);
+ int flushes_unscheduled = env_->UnSchedule(this, Env::Priority::HIGH);
+ mutex_.Lock();
+ bg_bottom_compaction_scheduled_ -= bottom_compactions_unscheduled;
+ bg_compaction_scheduled_ -= compactions_unscheduled;
+ bg_flush_scheduled_ -= flushes_unscheduled;
+
+ // Wait for background work to finish
+ while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
+ bg_flush_scheduled_ || bg_purge_scheduled_) {
+ TEST_SYNC_POINT("DBImpl::~DBImpl:WaitJob");
+ bg_cv_.Wait();
+ }
+ EraseThreadStatusDbInfo();
+ flush_scheduler_.Clear();
+
+ while (!flush_queue_.empty()) {
+ auto cfd = PopFirstFromFlushQueue();
+ if (cfd->Unref()) {
+ delete cfd;
+ }
+ }
+ while (!compaction_queue_.empty()) {
+ auto cfd = PopFirstFromCompactionQueue();
+ if (cfd->Unref()) {
+ delete cfd;
+ }
+ }
+
+ if (default_cf_handle_ != nullptr) {
+ // we need to delete handle outside of lock because it does its own locking
+ mutex_.Unlock();
+ delete default_cf_handle_;
+ mutex_.Lock();
+ }
+
+ // Clean up obsolete files due to SuperVersion release.
+ // (1) Need to delete to obsolete files before closing because RepairDB()
+ // scans all existing files in the file system and builds manifest file.
+ // Keeping obsolete files confuses the repair process.
+ // (2) Need to check if we Open()/Recover() the DB successfully before
+ // deleting because if VersionSet recover fails (may be due to corrupted
+ // manifest file), it is not able to identify live files correctly. As a
+ // result, all "live" files can get deleted by accident. However, corrupted
+ // manifest is recoverable by RepairDB().
+ if (opened_successfully_) {
+ JobContext job_context(next_job_id_.fetch_add(1));
+ FindObsoleteFiles(&job_context, true);
+
+ mutex_.Unlock();
+ // manifest number starting from 2
+ job_context.manifest_file_number = 1;
+ if (job_context.HaveSomethingToDelete()) {
+ PurgeObsoleteFiles(job_context);
+ }
+ job_context.Clean();
+ mutex_.Lock();
+ }
+
+ for (auto l : logs_to_free_) {
+ delete l;
+ }
+ for (auto& log : logs_) {
+ log.ClearWriter();
+ }
+ logs_.clear();
+
+ // Table cache may have table handles holding blocks from the block cache.
+ // We need to release them before the block cache is destroyed. The block
+ // cache may be destroyed inside versions_.reset(), when column family data
+ // list is destroyed, so leaving handles in table cache after
+ // versions_.reset() may cause issues.
+ // Here we clean all unreferenced handles in table cache.
+ // Now we assume all user queries have finished, so only version set itself
+ // can possibly hold the blocks from block cache. After releasing unreferenced
+ // handles here, only handles held by version set left and inside
+ // versions_.reset(), we will release them. There, we need to make sure every
+ // time a handle is released, we erase it from the cache too. By doing that,
+ // we can guarantee that after versions_.reset(), table cache is empty
+ // so the cache can be safely destroyed.
+ table_cache_->EraseUnRefEntries();
+
+ for (auto& txn_entry : recovered_transactions_) {
+ delete txn_entry.second;
+ }
+
+ // versions need to be destroyed before table_cache since it can hold
+ // references to table_cache.
+ versions_.reset();
+ mutex_.Unlock();
+ if (db_lock_ != nullptr) {
+ env_->UnlockFile(db_lock_);
+ }
+
+ ROCKS_LOG_INFO(immutable_db_options_.info_log, "Shutdown complete");
+ LogFlush(immutable_db_options_.info_log);
+}
+
+void DBImpl::MaybeIgnoreError(Status* s) const {
+ if (s->ok() || immutable_db_options_.paranoid_checks) {
+ // No change needed
+ } else {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log, "Ignoring error %s",
+ s->ToString().c_str());
+ *s = Status::OK();
+ }
+}
+
+const Status DBImpl::CreateArchivalDirectory() {
+ if (immutable_db_options_.wal_ttl_seconds > 0 ||
+ immutable_db_options_.wal_size_limit_mb > 0) {
+ std::string archivalPath = ArchivalDirectory(immutable_db_options_.wal_dir);
+ return env_->CreateDirIfMissing(archivalPath);
+ }
+ return Status::OK();
+}
+
+void DBImpl::PrintStatistics() {
+ auto dbstats = immutable_db_options_.statistics.get();
+ if (dbstats) {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log, "STATISTICS:\n %s",
+ dbstats->ToString().c_str());
+ }
+}
+
+void DBImpl::MaybeDumpStats() {
+ mutex_.Lock();
+ unsigned int stats_dump_period_sec =
+ mutable_db_options_.stats_dump_period_sec;
+ mutex_.Unlock();
+ if (stats_dump_period_sec == 0) return;
+
+ const uint64_t now_micros = env_->NowMicros();
+
+ if (last_stats_dump_time_microsec_ + stats_dump_period_sec * 1000000 <=
+ now_micros) {
+ // Multiple threads could race in here simultaneously.
+ // However, the last one will update last_stats_dump_time_microsec_
+ // atomically. We could see more than one dump during one dump
+ // period in rare cases.
+ last_stats_dump_time_microsec_ = now_micros;
+
+#ifndef ROCKSDB_LITE
+ const DBPropertyInfo* cf_property_info =
+ GetPropertyInfo(DB::Properties::kCFStats);
+ assert(cf_property_info != nullptr);
+ const DBPropertyInfo* db_property_info =
+ GetPropertyInfo(DB::Properties::kDBStats);
+ assert(db_property_info != nullptr);
+
+ std::string stats;
+ {
+ InstrumentedMutexLock l(&mutex_);
+ default_cf_internal_stats_->GetStringProperty(
+ *db_property_info, DB::Properties::kDBStats, &stats);
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ if (cfd->initialized()) {
+ cfd->internal_stats()->GetStringProperty(
+ *cf_property_info, DB::Properties::kCFStatsNoFileHistogram,
+ &stats);
+ }
+ }
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ if (cfd->initialized()) {
+ cfd->internal_stats()->GetStringProperty(
+ *cf_property_info, DB::Properties::kCFFileHistogram, &stats);
+ }
+ }
+ }
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "------- DUMPING STATS -------");
+ ROCKS_LOG_WARN(immutable_db_options_.info_log, "%s", stats.c_str());
+ if (immutable_db_options_.dump_malloc_stats) {
+ stats.clear();
+ DumpMallocStats(&stats);
+ if (!stats.empty()) {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "------- Malloc STATS -------");
+ ROCKS_LOG_WARN(immutable_db_options_.info_log, "%s", stats.c_str());
+ }
+ }
+#endif // !ROCKSDB_LITE
+
+ PrintStatistics();
+ }
+}
+
+void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) {
+ if (!job_context->logs_to_free.empty()) {
+ for (auto l : job_context->logs_to_free) {
+ AddToLogsToFreeQueue(l);
+ }
+ job_context->logs_to_free.clear();
+ SchedulePurge();
+ }
+}
+
+Directory* DBImpl::Directories::GetDataDir(size_t path_id) {
+ assert(path_id < data_dirs_.size());
+ Directory* ret_dir = data_dirs_[path_id].get();
+ if (ret_dir == nullptr) {
+ // Should use db_dir_
+ return db_dir_.get();
+ }
+ return ret_dir;
+}
+
+Status DBImpl::SetOptions(ColumnFamilyHandle* column_family,
+ const std::unordered_map<std::string, std::string>& options_map) {
+#ifdef ROCKSDB_LITE
+ return Status::NotSupported("Not supported in ROCKSDB LITE");
+#else
+ auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
+ if (options_map.empty()) {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "SetOptions() on column family [%s], empty input",
+ cfd->GetName().c_str());
+ return Status::InvalidArgument("empty input");
+ }
+
+ MutableCFOptions new_options;
+ Status s;
+ Status persist_options_status;
+ WriteThread::Writer w;
+ {
+ InstrumentedMutexLock l(&mutex_);
+ s = cfd->SetOptions(options_map);
+ if (s.ok()) {
+ new_options = *cfd->GetLatestMutableCFOptions();
+ // Append new version to recompute compaction score.
+ VersionEdit dummy_edit;
+ versions_->LogAndApply(cfd, new_options, &dummy_edit, &mutex_,
+ directories_.GetDbDir());
+ // Trigger possible flush/compactions. This has to be before we persist
+ // options to file, otherwise there will be a deadlock with writer
+ // thread.
+ auto* old_sv =
+ InstallSuperVersionAndScheduleWork(cfd, nullptr, new_options);
+ delete old_sv;
+
+ persist_options_status = WriteOptionsFile(
+ false /*need_mutex_lock*/, true /*need_enter_write_thread*/);
+ }
+ }
+
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "SetOptions() on column family [%s], inputs:",
+ cfd->GetName().c_str());
+ for (const auto& o : options_map) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log, "%s: %s\n", o.first.c_str(),
+ o.second.c_str());
+ }
+ if (s.ok()) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "[%s] SetOptions() succeeded", cfd->GetName().c_str());
+ new_options.Dump(immutable_db_options_.info_log.get());
+ if (!persist_options_status.ok()) {
+ s = persist_options_status;
+ }
+ } else {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log, "[%s] SetOptions() failed",
+ cfd->GetName().c_str());
+ }
+ LogFlush(immutable_db_options_.info_log);
+ return s;
+#endif // ROCKSDB_LITE
+}
+
+Status DBImpl::SetDBOptions(
+ const std::unordered_map<std::string, std::string>& options_map) {
+#ifdef ROCKSDB_LITE
+ return Status::NotSupported("Not supported in ROCKSDB LITE");
+#else
+ if (options_map.empty()) {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "SetDBOptions(), empty input.");
+ return Status::InvalidArgument("empty input");
+ }
+
+ MutableDBOptions new_options;
+ Status s;
+ Status persist_options_status;
+ WriteThread::Writer w;
+ WriteContext write_context;
+ {
+ InstrumentedMutexLock l(&mutex_);
+ s = GetMutableDBOptionsFromStrings(mutable_db_options_, options_map,
+ &new_options);
+ if (s.ok()) {
+ if (new_options.max_background_compactions >
+ mutable_db_options_.max_background_compactions) {
+ env_->IncBackgroundThreadsIfNeeded(
+ new_options.max_background_compactions, Env::Priority::LOW);
+ MaybeScheduleFlushOrCompaction();
+ }
+
+ write_controller_.set_max_delayed_write_rate(new_options.delayed_write_rate);
+ table_cache_.get()->SetCapacity(new_options.max_open_files == -1
+ ? TableCache::kInfiniteCapacity
+ : new_options.max_open_files - 10);
+
+ mutable_db_options_ = new_options;
+
+ write_thread_.EnterUnbatched(&w, &mutex_);
+ if (total_log_size_ > GetMaxTotalWalSize()) {
+ Status purge_wal_status = HandleWALFull(&write_context);
+ if (!purge_wal_status.ok()) {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "Unable to purge WAL files in SetDBOptions() -- %s",
+ purge_wal_status.ToString().c_str());
+ }
+ }
+ persist_options_status = WriteOptionsFile(
+ false /*need_mutex_lock*/, false /*need_enter_write_thread*/);
+ write_thread_.ExitUnbatched(&w);
+ }
+ }
+ ROCKS_LOG_INFO(immutable_db_options_.info_log, "SetDBOptions(), inputs:");
+ for (const auto& o : options_map) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log, "%s: %s\n", o.first.c_str(),
+ o.second.c_str());
+ }
+ if (s.ok()) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log, "SetDBOptions() succeeded");
+ new_options.Dump(immutable_db_options_.info_log.get());
+ if (!persist_options_status.ok()) {
+ if (immutable_db_options_.fail_if_options_file_error) {
+ s = Status::IOError(
+ "SetDBOptions() succeeded, but unable to persist options",
+ persist_options_status.ToString());
+ }
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "Unable to persist options in SetDBOptions() -- %s",
+ persist_options_status.ToString().c_str());
+ }
+ } else {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log, "SetDBOptions failed");
+ }
+ LogFlush(immutable_db_options_.info_log);
+ return s;
+#endif // ROCKSDB_LITE
+}
+
+// return the same level if it cannot be moved
+int DBImpl::FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd,
+ const MutableCFOptions& mutable_cf_options, int level) {
+ mutex_.AssertHeld();
+ const auto* vstorage = cfd->current()->storage_info();
+ int minimum_level = level;
+ for (int i = level - 1; i > 0; --i) {
+ // stop if level i is not empty
+ if (vstorage->NumLevelFiles(i) > 0) break;
+ // stop if level i is too small (cannot fit the level files)
+ if (vstorage->MaxBytesForLevel(i) < vstorage->NumLevelBytes(level)) {
+ break;
+ }
+
+ minimum_level = i;
+ }
+ return minimum_level;
+}
+
+Status DBImpl::FlushWAL(bool sync) {
+ {
+ // We need to lock log_write_mutex_ since logs_ might change concurrently
+ InstrumentedMutexLock wl(&log_write_mutex_);
+ log::Writer* cur_log_writer = logs_.back().writer;
+ auto s = cur_log_writer->WriteBuffer();
+ if (!s.ok()) {
+ ROCKS_LOG_ERROR(immutable_db_options_.info_log, "WAL flush error %s",
+ s.ToString().c_str());
+ }
+ if (!sync) {
+ ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "FlushWAL sync=false");
+ return s;
+ }
+ }
+ // sync = true
+ ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "FlushWAL sync=true");
+ return SyncWAL();
+}
+
+Status DBImpl::SyncWAL() {
+ autovector<log::Writer*, 1> logs_to_sync;
+ bool need_log_dir_sync;
+ uint64_t current_log_number;
+
+ {
+ InstrumentedMutexLock l(&mutex_);
+ assert(!logs_.empty());
+
+ // This SyncWAL() call only cares about logs up to this number.
+ current_log_number = logfile_number_;
+
+ while (logs_.front().number <= current_log_number &&
+ logs_.front().getting_synced) {
+ log_sync_cv_.Wait();
+ }
+ // First check that logs are safe to sync in background.
+ for (auto it = logs_.begin();
+ it != logs_.end() && it->number <= current_log_number; ++it) {
+ if (!it->writer->file()->writable_file()->IsSyncThreadSafe()) {
+ return Status::NotSupported(
+ "SyncWAL() is not supported for this implementation of WAL file",
+ immutable_db_options_.allow_mmap_writes
+ ? "try setting Options::allow_mmap_writes to false"
+ : Slice());
+ }
+ }
+ for (auto it = logs_.begin();
+ it != logs_.end() && it->number <= current_log_number; ++it) {
+ auto& log = *it;
+ assert(!log.getting_synced);
+ log.getting_synced = true;
+ logs_to_sync.push_back(log.writer);
+ }
+
+ need_log_dir_sync = !log_dir_synced_;
+ }
+
+ TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:1");
+ RecordTick(stats_, WAL_FILE_SYNCED);
+ Status status;
+ for (log::Writer* log : logs_to_sync) {
+ status = log->file()->SyncWithoutFlush(immutable_db_options_.use_fsync);
+ if (!status.ok()) {
+ break;
+ }
+ }
+ if (status.ok() && need_log_dir_sync) {
+ status = directories_.GetWalDir()->Fsync();
+ }
+ TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2");
+
+ TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:1");
+ {
+ InstrumentedMutexLock l(&mutex_);
+ MarkLogsSynced(current_log_number, need_log_dir_sync, status);
+ }
+ TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:2");
+
+ return status;
+}
+
+void DBImpl::MarkLogsSynced(
+ uint64_t up_to, bool synced_dir, const Status& status) {
+ mutex_.AssertHeld();
+ if (synced_dir &&
+ logfile_number_ == up_to &&
+ status.ok()) {
+ log_dir_synced_ = true;
+ }
+ for (auto it = logs_.begin(); it != logs_.end() && it->number <= up_to;) {
+ auto& log = *it;
+ assert(log.getting_synced);
+ if (status.ok() && logs_.size() > 1) {
+ logs_to_free_.push_back(log.ReleaseWriter());
+ it = logs_.erase(it);
+ } else {
+ log.getting_synced = false;
+ ++it;
+ }
+ }
+ assert(!status.ok() || logs_.empty() || logs_[0].number > up_to ||
+ (logs_.size() == 1 && !logs_[0].getting_synced));
+ log_sync_cv_.SignalAll();
+}
+
+SequenceNumber DBImpl::GetLatestSequenceNumber() const {
+ return versions_->LastSequence();
+}
+
+InternalIterator* DBImpl::NewInternalIterator(
+ Arena* arena, RangeDelAggregator* range_del_agg,
+ ColumnFamilyHandle* column_family) {
+ ColumnFamilyData* cfd;
+ if (column_family == nullptr) {
+ cfd = default_cf_handle_->cfd();
+ } else {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ cfd = cfh->cfd();
+ }
+
+ mutex_.Lock();
+ SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
+ mutex_.Unlock();
+ ReadOptions roptions;
+ return NewInternalIterator(roptions, cfd, super_version, arena,
+ range_del_agg);
+}
+
+void DBImpl::SchedulePurge() {
+ mutex_.AssertHeld();
+ assert(opened_successfully_);
+
+ // Purge operations are put into High priority queue
+ bg_purge_scheduled_++;
+ env_->Schedule(&DBImpl::BGWorkPurge, this, Env::Priority::HIGH, nullptr);
+}
+
+void DBImpl::BackgroundCallPurge() {
+ mutex_.Lock();
+
+ // We use one single loop to clear both queues so that after existing the loop
+ // both queues are empty. This is stricter than what is needed, but can make
+ // it easier for us to reason the correctness.
+ while (!purge_queue_.empty() || !logs_to_free_queue_.empty()) {
+ if (!purge_queue_.empty()) {
+ auto purge_file = purge_queue_.begin();
+ auto fname = purge_file->fname;
+ auto type = purge_file->type;
+ auto number = purge_file->number;
+ auto path_id = purge_file->path_id;
+ auto job_id = purge_file->job_id;
+ purge_queue_.pop_front();
+
+ mutex_.Unlock();
+ Status file_deletion_status;
+ DeleteObsoleteFileImpl(file_deletion_status, job_id, fname, type, number,
+ path_id);
+ mutex_.Lock();
+ } else {
+ assert(!logs_to_free_queue_.empty());
+ log::Writer* log_writer = *(logs_to_free_queue_.begin());
+ logs_to_free_queue_.pop_front();
+ mutex_.Unlock();
+ delete log_writer;
+ mutex_.Lock();
+ }
+ }
+ bg_purge_scheduled_--;
+
+ bg_cv_.SignalAll();
+ // IMPORTANT:there should be no code after calling SignalAll. This call may
+ // signal the DB destructor that it's OK to proceed with destruction. In
+ // that case, all DB variables will be dealloacated and referencing them
+ // will cause trouble.
+ mutex_.Unlock();
+}
+
+namespace {
+struct IterState {
+ IterState(DBImpl* _db, InstrumentedMutex* _mu, SuperVersion* _super_version,
+ bool _background_purge)
+ : db(_db),
+ mu(_mu),
+ super_version(_super_version),
+ background_purge(_background_purge) {}
+
+ DBImpl* db;
+ InstrumentedMutex* mu;
+ SuperVersion* super_version;
+ bool background_purge;
+};
+
+static void CleanupIteratorState(void* arg1, void* arg2) {
+ IterState* state = reinterpret_cast<IterState*>(arg1);
+
+ if (state->super_version->Unref()) {
+ // Job id == 0 means that this is not our background process, but rather
+ // user thread
+ JobContext job_context(0);
+
+ state->mu->Lock();
+ state->super_version->Cleanup();
+ state->db->FindObsoleteFiles(&job_context, false, true);
+ if (state->background_purge) {
+ state->db->ScheduleBgLogWriterClose(&job_context);
+ }
+ state->mu->Unlock();
+
+ delete state->super_version;
+ if (job_context.HaveSomethingToDelete()) {
+ if (state->background_purge) {
+ // PurgeObsoleteFiles here does not delete files. Instead, it adds the
+ // files to be deleted to a job queue, and deletes it in a separate
+ // background thread.
+ state->db->PurgeObsoleteFiles(job_context, true /* schedule only */);
+ state->mu->Lock();
+ state->db->SchedulePurge();
+ state->mu->Unlock();
+ } else {
+ state->db->PurgeObsoleteFiles(job_context);
+ }
+ }
+ job_context.Clean();
+ }
+
+ delete state;
+}
+} // namespace
+
+InternalIterator* DBImpl::NewInternalIterator(
+ const ReadOptions& read_options, ColumnFamilyData* cfd,
+ SuperVersion* super_version, Arena* arena,
+ RangeDelAggregator* range_del_agg) {
+ InternalIterator* internal_iter;
+ assert(arena != nullptr);
+ assert(range_del_agg != nullptr);
+ // Need to create internal iterator from the arena.
+ MergeIteratorBuilder merge_iter_builder(
+ &cfd->internal_comparator(), arena,
+ !read_options.total_order_seek &&
+ cfd->ioptions()->prefix_extractor != nullptr);
+ // Collect iterator for mutable mem
+ merge_iter_builder.AddIterator(
+ super_version->mem->NewIterator(read_options, arena));
+ std::unique_ptr<InternalIterator> range_del_iter;
+ Status s;
+ if (!read_options.ignore_range_deletions) {
+ range_del_iter.reset(
+ super_version->mem->NewRangeTombstoneIterator(read_options));
+ s = range_del_agg->AddTombstones(std::move(range_del_iter));
+ }
+ // Collect all needed child iterators for immutable memtables
+ if (s.ok()) {
+ super_version->imm->AddIterators(read_options, &merge_iter_builder);
+ if (!read_options.ignore_range_deletions) {
+ s = super_version->imm->AddRangeTombstoneIterators(read_options, arena,
+ range_del_agg);
+ }
+ }
+ if (s.ok()) {
+ // Collect iterators for files in L0 - Ln
+ if (read_options.read_tier != kMemtableTier) {
+ super_version->current->AddIterators(read_options, env_options_,
+ &merge_iter_builder, range_del_agg);
+ }
+ internal_iter = merge_iter_builder.Finish();
+ IterState* cleanup =
+ new IterState(this, &mutex_, super_version,
+ read_options.background_purge_on_iterator_cleanup);
+ internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, nullptr);
+
+ return internal_iter;
+ }
+ return NewErrorInternalIterator(s);
+}
+
+ColumnFamilyHandle* DBImpl::DefaultColumnFamily() const {
+ return default_cf_handle_;
+}
+
+Status DBImpl::Get(const ReadOptions& read_options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ PinnableSlice* value) {
+ return GetImpl(read_options, column_family, key, value);
+}
+
+Status DBImpl::GetImpl(const ReadOptions& read_options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ PinnableSlice* pinnable_val, bool* value_found) {
+ assert(pinnable_val != nullptr);
+ StopWatch sw(env_, stats_, DB_GET);
+ PERF_TIMER_GUARD(get_snapshot_time);
+
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ auto cfd = cfh->cfd();
+
+ // Acquire SuperVersion
+ SuperVersion* sv = GetAndRefSuperVersion(cfd);
+
+ TEST_SYNC_POINT("DBImpl::GetImpl:1");
+ TEST_SYNC_POINT("DBImpl::GetImpl:2");
+
+ SequenceNumber snapshot;
+ if (read_options.snapshot != nullptr) {
+ snapshot = reinterpret_cast<const SnapshotImpl*>(
+ read_options.snapshot)->number_;
+ } else {
+ // Since we get and reference the super version before getting
+ // the snapshot number, without a mutex protection, it is possible
+ // that a memtable switch happened in the middle and not all the
+ // data for this snapshot is available. But it will contain all
+ // the data available in the super version we have, which is also
+ // a valid snapshot to read from.
+ // We shouldn't get snapshot before finding and referencing the
+ // super versipon because a flush happening in between may compact
+ // away data for the snapshot, but the snapshot is earlier than the
+ // data overwriting it, so users may see wrong results.
+ snapshot = versions_->LastSequence();
+ }
+ TEST_SYNC_POINT("DBImpl::GetImpl:3");
+ TEST_SYNC_POINT("DBImpl::GetImpl:4");
+
+ // Prepare to store a list of merge operations if merge occurs.
+ MergeContext merge_context;
+ RangeDelAggregator range_del_agg(cfd->internal_comparator(), snapshot);
+
+ Status s;
+ // First look in the memtable, then in the immutable memtable (if any).
+ // s is both in/out. When in, s could either be OK or MergeInProgress.
+ // merge_operands will contain the sequence of merges in the latter case.
+ LookupKey lkey(key, snapshot);
+ PERF_TIMER_STOP(get_snapshot_time);
+
+ bool skip_memtable = (read_options.read_tier == kPersistedTier &&
+ has_unpersisted_data_.load(std::memory_order_relaxed));
+ bool done = false;
+ if (!skip_memtable) {
+ if (sv->mem->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
+ &range_del_agg, read_options)) {
+ done = true;
+ pinnable_val->PinSelf();
+ RecordTick(stats_, MEMTABLE_HIT);
+ } else if ((s.ok() || s.IsMergeInProgress()) &&
+ sv->imm->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
+ &range_del_agg, read_options)) {
+ done = true;
+ pinnable_val->PinSelf();
+ RecordTick(stats_, MEMTABLE_HIT);
+ }
+ if (!done && !s.ok() && !s.IsMergeInProgress()) {
+ return s;
+ }
+ }
+ if (!done) {
+ PERF_TIMER_GUARD(get_from_output_files_time);
+ sv->current->Get(read_options, lkey, pinnable_val, &s, &merge_context,
+ &range_del_agg, value_found);
+ RecordTick(stats_, MEMTABLE_MISS);
+ }
+
+ {
+ PERF_TIMER_GUARD(get_post_process_time);
+
+ ReturnAndCleanupSuperVersion(cfd, sv);
+
+ RecordTick(stats_, NUMBER_KEYS_READ);
+ size_t size = pinnable_val->size();
+ RecordTick(stats_, BYTES_READ, size);
+ MeasureTime(stats_, BYTES_PER_READ, size);
+ }
+ return s;
+}
+
+std::vector<Status> DBImpl::MultiGet(
+ const ReadOptions& read_options,
+ const std::vector<ColumnFamilyHandle*>& column_family,
+ const std::vector<Slice>& keys, std::vector<std::string>* values) {
+
+ StopWatch sw(env_, stats_, DB_MULTIGET);
+ PERF_TIMER_GUARD(get_snapshot_time);
+
+ SequenceNumber snapshot;
+
+ struct MultiGetColumnFamilyData {
+ ColumnFamilyData* cfd;
+ SuperVersion* super_version;
+ };
+ std::unordered_map<uint32_t, MultiGetColumnFamilyData*> multiget_cf_data;
+ // fill up and allocate outside of mutex
+ for (auto cf : column_family) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(cf);
+ auto cfd = cfh->cfd();
+ if (multiget_cf_data.find(cfd->GetID()) == multiget_cf_data.end()) {
+ auto mgcfd = new MultiGetColumnFamilyData();
+ mgcfd->cfd = cfd;
+ multiget_cf_data.insert({cfd->GetID(), mgcfd});
+ }
+ }
+
+ mutex_.Lock();
+ if (read_options.snapshot != nullptr) {
+ snapshot = reinterpret_cast<const SnapshotImpl*>(
+ read_options.snapshot)->number_;
+ } else {
+ snapshot = versions_->LastSequence();
+ }
+ for (auto mgd_iter : multiget_cf_data) {
+ mgd_iter.second->super_version =
+ mgd_iter.second->cfd->GetSuperVersion()->Ref();
+ }
+ mutex_.Unlock();
+
+ // Contain a list of merge operations if merge occurs.
+ MergeContext merge_context;
+
+ // Note: this always resizes the values array
+ size_t num_keys = keys.size();
+ std::vector<Status> stat_list(num_keys);
+ values->resize(num_keys);
+
+ // Keep track of bytes that we read for statistics-recording later
+ uint64_t bytes_read = 0;
+ PERF_TIMER_STOP(get_snapshot_time);
+
+ // For each of the given keys, apply the entire "get" process as follows:
+ // First look in the memtable, then in the immutable memtable (if any).
+ // s is both in/out. When in, s could either be OK or MergeInProgress.
+ // merge_operands will contain the sequence of merges in the latter case.
+ for (size_t i = 0; i < num_keys; ++i) {
+ merge_context.Clear();
+ Status& s = stat_list[i];
+ std::string* value = &(*values)[i];
+
+ LookupKey lkey(keys[i], snapshot);
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family[i]);
+ RangeDelAggregator range_del_agg(cfh->cfd()->internal_comparator(),
+ snapshot);
+ auto mgd_iter = multiget_cf_data.find(cfh->cfd()->GetID());
+ assert(mgd_iter != multiget_cf_data.end());
+ auto mgd = mgd_iter->second;
+ auto super_version = mgd->super_version;
+ bool skip_memtable =
+ (read_options.read_tier == kPersistedTier &&
+ has_unpersisted_data_.load(std::memory_order_relaxed));
+ bool done = false;
+ if (!skip_memtable) {
+ if (super_version->mem->Get(lkey, value, &s, &merge_context,
+ &range_del_agg, read_options)) {
+ done = true;
+ // TODO(?): RecordTick(stats_, MEMTABLE_HIT)?
+ } else if (super_version->imm->Get(lkey, value, &s, &merge_context,
+ &range_del_agg, read_options)) {
+ done = true;
+ // TODO(?): RecordTick(stats_, MEMTABLE_HIT)?
+ }
+ }
+ if (!done) {
+ PinnableSlice pinnable_val;
+ PERF_TIMER_GUARD(get_from_output_files_time);
+ super_version->current->Get(read_options, lkey, &pinnable_val, &s,
+ &merge_context, &range_del_agg);
+ value->assign(pinnable_val.data(), pinnable_val.size());
+ // TODO(?): RecordTick(stats_, MEMTABLE_MISS)?
+ }
+
+ if (s.ok()) {
+ bytes_read += value->size();
+ }
+ }
+
+ // Post processing (decrement reference counts and record statistics)
+ PERF_TIMER_GUARD(get_post_process_time);
+ autovector<SuperVersion*> superversions_to_delete;
+
+ // TODO(icanadi) do we need lock here or just around Cleanup()?
+ mutex_.Lock();
+ for (auto mgd_iter : multiget_cf_data) {
+ auto mgd = mgd_iter.second;
+ if (mgd->super_version->Unref()) {
+ mgd->super_version->Cleanup();
+ superversions_to_delete.push_back(mgd->super_version);
+ }
+ }
+ mutex_.Unlock();
+
+ for (auto td : superversions_to_delete) {
+ delete td;
+ }
+ for (auto mgd : multiget_cf_data) {
+ delete mgd.second;
+ }
+
+ RecordTick(stats_, NUMBER_MULTIGET_CALLS);
+ RecordTick(stats_, NUMBER_MULTIGET_KEYS_READ, num_keys);
+ RecordTick(stats_, NUMBER_MULTIGET_BYTES_READ, bytes_read);
+ MeasureTime(stats_, BYTES_PER_MULTIGET, bytes_read);
+ PERF_TIMER_STOP(get_post_process_time);
+
+ return stat_list;
+}
+
+Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options,
+ const std::string& column_family,
+ ColumnFamilyHandle** handle) {
+ assert(handle != nullptr);
+ Status s = CreateColumnFamilyImpl(cf_options, column_family, handle);
+ if (s.ok()) {
+ s = WriteOptionsFile(true /*need_mutex_lock*/,
+ true /*need_enter_write_thread*/);
+ }
+ return s;
+}
+
+Status DBImpl::CreateColumnFamilies(
+ const ColumnFamilyOptions& cf_options,
+ const std::vector<std::string>& column_family_names,
+ std::vector<ColumnFamilyHandle*>* handles) {
+ assert(handles != nullptr);
+ handles->clear();
+ size_t num_cf = column_family_names.size();
+ Status s;
+ bool success_once = false;
+ for (size_t i = 0; i < num_cf; i++) {
+ ColumnFamilyHandle* handle;
+ s = CreateColumnFamilyImpl(cf_options, column_family_names[i], &handle);
+ if (!s.ok()) {
+ break;
+ }
+ handles->push_back(handle);
+ success_once = true;
+ }
+ if (success_once) {
+ Status persist_options_status = WriteOptionsFile(
+ true /*need_mutex_lock*/, true /*need_enter_write_thread*/);
+ if (s.ok() && !persist_options_status.ok()) {
+ s = persist_options_status;
+ }
+ }
+ return s;
+}
+
+Status DBImpl::CreateColumnFamilies(
+ const std::vector<ColumnFamilyDescriptor>& column_families,
+ std::vector<ColumnFamilyHandle*>* handles) {
+ assert(handles != nullptr);
+ handles->clear();
+ size_t num_cf = column_families.size();
+ Status s;
+ bool success_once = false;
+ for (size_t i = 0; i < num_cf; i++) {
+ ColumnFamilyHandle* handle;
+ s = CreateColumnFamilyImpl(column_families[i].options,
+ column_families[i].name, &handle);
+ if (!s.ok()) {
+ break;
+ }
+ handles->push_back(handle);
+ success_once = true;
+ }
+ if (success_once) {
+ Status persist_options_status = WriteOptionsFile(
+ true /*need_mutex_lock*/, true /*need_enter_write_thread*/);
+ if (s.ok() && !persist_options_status.ok()) {
+ s = persist_options_status;
+ }
+ }
+ return s;
+}
+
+Status DBImpl::CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options,
+ const std::string& column_family_name,
+ ColumnFamilyHandle** handle) {
+ Status s;
+ Status persist_options_status;
+ *handle = nullptr;
+
+ s = CheckCompressionSupported(cf_options);
+ if (s.ok() && immutable_db_options_.allow_concurrent_memtable_write) {
+ s = CheckConcurrentWritesSupported(cf_options);
+ }
+ if (!s.ok()) {
+ return s;
+ }
+
+ {
+ InstrumentedMutexLock l(&mutex_);
+
+ if (versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name) !=
+ nullptr) {
+ return Status::InvalidArgument("Column family already exists");
+ }
+ VersionEdit edit;
+ edit.AddColumnFamily(column_family_name);
+ uint32_t new_id = versions_->GetColumnFamilySet()->GetNextColumnFamilyID();
+ edit.SetColumnFamily(new_id);
+ edit.SetLogNumber(logfile_number_);
+ edit.SetComparatorName(cf_options.comparator->Name());
+
+ // LogAndApply will both write the creation in MANIFEST and create
+ // ColumnFamilyData object
+ { // write thread
+ WriteThread::Writer w;
+ write_thread_.EnterUnbatched(&w, &mutex_);
+ // LogAndApply will both write the creation in MANIFEST and create
+ // ColumnFamilyData object
+ s = versions_->LogAndApply(nullptr, MutableCFOptions(cf_options), &edit,
+ &mutex_, directories_.GetDbDir(), false,
+ &cf_options);
+ write_thread_.ExitUnbatched(&w);
+ }
+ if (s.ok()) {
+ single_column_family_mode_ = false;
+ auto* cfd =
+ versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name);
+ assert(cfd != nullptr);
+ delete InstallSuperVersionAndScheduleWork(
+ cfd, nullptr, *cfd->GetLatestMutableCFOptions());
+
+ if (!cfd->mem()->IsSnapshotSupported()) {
+ is_snapshot_supported_ = false;
+ }
+
+ cfd->set_initialized();
+
+ *handle = new ColumnFamilyHandleImpl(cfd, this, &mutex_);
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "Created column family [%s] (ID %u)",
+ column_family_name.c_str(), (unsigned)cfd->GetID());
+ } else {
+ ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+ "Creating column family [%s] FAILED -- %s",
+ column_family_name.c_str(), s.ToString().c_str());
+ }
+ } // InstrumentedMutexLock l(&mutex_)
+
+ // this is outside the mutex
+ if (s.ok()) {
+ NewThreadStatusCfInfo(
+ reinterpret_cast<ColumnFamilyHandleImpl*>(*handle)->cfd());
+ }
+ return s;
+}
+
+Status DBImpl::DropColumnFamily(ColumnFamilyHandle* column_family) {
+ assert(column_family != nullptr);
+ Status s = DropColumnFamilyImpl(column_family);
+ if (s.ok()) {
+ s = WriteOptionsFile(true /*need_mutex_lock*/,
+ true /*need_enter_write_thread*/);
+ }
+ return s;
+}
+
+Status DBImpl::DropColumnFamilies(
+ const std::vector<ColumnFamilyHandle*>& column_families) {
+ Status s;
+ bool success_once = false;
+ for (auto* handle : column_families) {
+ s = DropColumnFamilyImpl(handle);
+ if (!s.ok()) {
+ break;
+ }
+ success_once = true;
+ }
+ if (success_once) {
+ Status persist_options_status = WriteOptionsFile(
+ true /*need_mutex_lock*/, true /*need_enter_write_thread*/);
+ if (s.ok() && !persist_options_status.ok()) {
+ s = persist_options_status;
+ }
+ }
+ return s;
+}
+
+Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ auto cfd = cfh->cfd();
+ if (cfd->GetID() == 0) {
+ return Status::InvalidArgument("Can't drop default column family");
+ }
+
+ bool cf_support_snapshot = cfd->mem()->IsSnapshotSupported();
+
+ VersionEdit edit;
+ edit.DropColumnFamily();
+ edit.SetColumnFamily(cfd->GetID());
+
+ Status s;
+ {
+ InstrumentedMutexLock l(&mutex_);
+ if (cfd->IsDropped()) {
+ s = Status::InvalidArgument("Column family already dropped!\n");
+ }
+ if (s.ok()) {
+ // we drop column family from a single write thread
+ WriteThread::Writer w;
+ write_thread_.EnterUnbatched(&w, &mutex_);
+ s = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
+ &edit, &mutex_);
+ write_thread_.ExitUnbatched(&w);
+ }
+
+ if (!cf_support_snapshot) {
+ // Dropped Column Family doesn't support snapshot. Need to recalculate
+ // is_snapshot_supported_.
+ bool new_is_snapshot_supported = true;
+ for (auto c : *versions_->GetColumnFamilySet()) {
+ if (!c->IsDropped() && !c->mem()->IsSnapshotSupported()) {
+ new_is_snapshot_supported = false;
+ break;
+ }
+ }
+ is_snapshot_supported_ = new_is_snapshot_supported;
+ }
+ }
+
+ if (s.ok()) {
+ // Note that here we erase the associated cf_info of the to-be-dropped
+ // cfd before its ref-count goes to zero to avoid having to erase cf_info
+ // later inside db_mutex.
+ EraseThreadStatusCfInfo(cfd);
+ assert(cfd->IsDropped());
+ auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
+ max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
+ mutable_cf_options->max_write_buffer_number;
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "Dropped column family with id %u\n", cfd->GetID());
+ } else {
+ ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+ "Dropping column family with id %u FAILED -- %s\n",
+ cfd->GetID(), s.ToString().c_str());
+ }
+
+ return s;
+}
+
+bool DBImpl::KeyMayExist(const ReadOptions& read_options,
+ ColumnFamilyHandle* column_family, const Slice& key,
+ std::string* value, bool* value_found) {
+ assert(value != nullptr);
+ if (value_found != nullptr) {
+ // falsify later if key-may-exist but can't fetch value
+ *value_found = true;
+ }
+ ReadOptions roptions = read_options;
+ roptions.read_tier = kBlockCacheTier; // read from block cache only
+ PinnableSlice pinnable_val;
+ auto s = GetImpl(roptions, column_family, key, &pinnable_val, value_found);
+ value->assign(pinnable_val.data(), pinnable_val.size());
+
+ // If block_cache is enabled and the index block of the table didn't
+ // not present in block_cache, the return value will be Status::Incomplete.
+ // In this case, key may still exist in the table.
+ return s.ok() || s.IsIncomplete();
+}
+
+Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
+ ColumnFamilyHandle* column_family) {
+ if (read_options.read_tier == kPersistedTier) {
+ return NewErrorIterator(Status::NotSupported(
+ "ReadTier::kPersistedData is not yet supported in iterators."));
+ }
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ auto cfd = cfh->cfd();
+ if (read_options.managed) {
+#ifdef ROCKSDB_LITE
+ // not supported in lite version
+ return NewErrorIterator(Status::InvalidArgument(
+ "Managed Iterators not supported in RocksDBLite."));
+#else
+ if ((read_options.tailing) || (read_options.snapshot != nullptr) ||
+ (is_snapshot_supported_)) {
+ return new ManagedIterator(this, read_options, cfd);
+ }
+ // Managed iter not supported
+ return NewErrorIterator(Status::InvalidArgument(
+ "Managed Iterators not supported without snapshots."));
+#endif
+ } else if (read_options.tailing) {
+#ifdef ROCKSDB_LITE
+ // not supported in lite version
+ return nullptr;
+#else
+ SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+ auto iter = new ForwardIterator(this, read_options, cfd, sv);
+ return NewDBIterator(
+ env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
+ kMaxSequenceNumber,
+ sv->mutable_cf_options.max_sequential_skip_in_iterations);
+#endif
+ } else {
+ SequenceNumber latest_snapshot = versions_->LastSequence();
+ SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+
+ auto snapshot =
+ read_options.snapshot != nullptr
+ ? reinterpret_cast<const SnapshotImpl*>(
+ read_options.snapshot)->number_
+ : latest_snapshot;
+
+ // Try to generate a DB iterator tree in continuous memory area to be
+ // cache friendly. Here is an example of result:
+ // +-------------------------------+
+ // | |
+ // | ArenaWrappedDBIter |
+ // | + |
+ // | +---> Inner Iterator ------------+
+ // | | | |
+ // | | +-- -- -- -- -- -- -- --+ |
+ // | +--- | Arena | |
+ // | | | |
+ // | Allocated Memory: | |
+ // | | +-------------------+ |
+ // | | | DBIter | <---+
+ // | | + |
+ // | | | +-> iter_ ------------+
+ // | | | | |
+ // | | +-------------------+ |
+ // | | | MergingIterator | <---+
+ // | | + |
+ // | | | +->child iter1 ------------+
+ // | | | | | |
+ // | | +->child iter2 ----------+ |
+ // | | | | | | |
+ // | | | +->child iter3 --------+ | |
+ // | | | | | |
+ // | | +-------------------+ | | |
+ // | | | Iterator1 | <--------+
+ // | | +-------------------+ | |
+ // | | | Iterator2 | <------+
+ // | | +-------------------+ |
+ // | | | Iterator3 | <----+
+ // | | +-------------------+
+ // | | |
+ // +-------+-----------------------+
+ //
+ // ArenaWrappedDBIter inlines an arena area where all the iterators in
+ // the iterator tree are allocated in the order of being accessed when
+ // querying.
+ // Laying out the iterators in the order of being accessed makes it more
+ // likely that any iterator pointer is close to the iterator it points to so
+ // that they are likely to be in the same cache line and/or page.
+ ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
+ env_, read_options, *cfd->ioptions(), snapshot,
+ sv->mutable_cf_options.max_sequential_skip_in_iterations,
+ sv->version_number,
+ ((read_options.snapshot != nullptr) ? nullptr : this), cfd);
+
+ InternalIterator* internal_iter =
+ NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
+ db_iter->GetRangeDelAggregator());
+ db_iter->SetIterUnderDBIter(internal_iter);
+
+ return db_iter;
+ }
+ // To stop compiler from complaining
+ return nullptr;
+}
+
+Status DBImpl::NewIterators(
+ const ReadOptions& read_options,
+ const std::vector<ColumnFamilyHandle*>& column_families,
+ std::vector<Iterator*>* iterators) {
+ if (read_options.read_tier == kPersistedTier) {
+ return Status::NotSupported(
+ "ReadTier::kPersistedData is not yet supported in iterators.");
+ }
+ iterators->clear();
+ iterators->reserve(column_families.size());
+ if (read_options.managed) {
+#ifdef ROCKSDB_LITE
+ return Status::InvalidArgument(
+ "Managed interator not supported in RocksDB lite");
+#else
+ if ((!read_options.tailing) && (read_options.snapshot == nullptr) &&
+ (!is_snapshot_supported_)) {
+ return Status::InvalidArgument(
+ "Managed interator not supported without snapshots");
+ }
+ for (auto cfh : column_families) {
+ auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
+ auto iter = new ManagedIterator(this, read_options, cfd);
+ iterators->push_back(iter);
+ }
+#endif
+ } else if (read_options.tailing) {
+#ifdef ROCKSDB_LITE
+ return Status::InvalidArgument(
+ "Tailing interator not supported in RocksDB lite");
+#else
+ for (auto cfh : column_families) {
+ auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
+ SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+ auto iter = new ForwardIterator(this, read_options, cfd, sv);
+ iterators->push_back(NewDBIterator(
+ env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
+ kMaxSequenceNumber,
+ sv->mutable_cf_options.max_sequential_skip_in_iterations));
+ }
+#endif
+ } else {
+ SequenceNumber latest_snapshot = versions_->LastSequence();
+
+ for (size_t i = 0; i < column_families.size(); ++i) {
+ auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(
+ column_families[i])->cfd();
+ SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+
+ auto snapshot =
+ read_options.snapshot != nullptr
+ ? reinterpret_cast<const SnapshotImpl*>(
+ read_options.snapshot)->number_
+ : latest_snapshot;
+
+ ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
+ env_, read_options, *cfd->ioptions(), snapshot,
+ sv->mutable_cf_options.max_sequential_skip_in_iterations,
+ sv->version_number,
+ ((read_options.snapshot != nullptr) ? nullptr : this), cfd);
+ InternalIterator* internal_iter =
+ NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
+ db_iter->GetRangeDelAggregator());
+ db_iter->SetIterUnderDBIter(internal_iter);
+ iterators->push_back(db_iter);
+ }
+ }
+
+ return Status::OK();
+}
+
+const Snapshot* DBImpl::GetSnapshot() { return GetSnapshotImpl(false); }
+
+#ifndef ROCKSDB_LITE
+const Snapshot* DBImpl::GetSnapshotForWriteConflictBoundary() {
+ return GetSnapshotImpl(true);
+}
+#endif // ROCKSDB_LITE
+
+const Snapshot* DBImpl::GetSnapshotImpl(bool is_write_conflict_boundary) {
+ int64_t unix_time = 0;
+ env_->GetCurrentTime(&unix_time); // Ignore error
+ SnapshotImpl* s = new SnapshotImpl;
+
+ InstrumentedMutexLock l(&mutex_);
+ // returns null if the underlying memtable does not support snapshot.
+ if (!is_snapshot_supported_) {
+ delete s;
+ return nullptr;
+ }
+ return snapshots_.New(s, versions_->LastSequence(), unix_time,
+ is_write_conflict_boundary);
+}
+
+void DBImpl::ReleaseSnapshot(const Snapshot* s) {
+ const SnapshotImpl* casted_s = reinterpret_cast<const SnapshotImpl*>(s);
+ {
+ InstrumentedMutexLock l(&mutex_);
+ snapshots_.Delete(casted_s);
+ }
+ delete casted_s;
+}
+
+bool DBImpl::HasActiveSnapshotLaterThanSN(SequenceNumber sn) {
+ InstrumentedMutexLock l(&mutex_);
+ if (snapshots_.empty()) {
+ return false;
+ }
+ return (snapshots_.newest()->GetSequenceNumber() > sn);
+}
+
+#ifndef ROCKSDB_LITE
+Status DBImpl::GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
+ TablePropertiesCollection* props) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ auto cfd = cfh->cfd();
+
+ // Increment the ref count
+ mutex_.Lock();
+ auto version = cfd->current();
+ version->Ref();
+ mutex_.Unlock();
+
+ auto s = version->GetPropertiesOfAllTables(props);
+
+ // Decrement the ref count
+ mutex_.Lock();
+ version->Unref();
+ mutex_.Unlock();
+
+ return s;
+}
+
+Status DBImpl::GetPropertiesOfTablesInRange(ColumnFamilyHandle* column_family,
+ const Range* range, std::size_t n,
+ TablePropertiesCollection* props) {
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ auto cfd = cfh->cfd();
+
+ // Increment the ref count
+ mutex_.Lock();
+ auto version = cfd->current();
+ version->Ref();
+ mutex_.Unlock();
+
+ auto s = version->GetPropertiesOfTablesInRange(range, n, props);
+
+ // Decrement the ref count
+ mutex_.Lock();
+ version->Unref();
+ mutex_.Unlock();
+
+ return s;
+}
+
+#endif // ROCKSDB_LITE
+
+const std::string& DBImpl::GetName() const {
+ return dbname_;
+}
+
+Env* DBImpl::GetEnv() const {
+ return env_;
+}
+
+Options DBImpl::GetOptions(ColumnFamilyHandle* column_family) const {
+ InstrumentedMutexLock l(&mutex_);
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ return Options(BuildDBOptions(immutable_db_options_, mutable_db_options_),
+ cfh->cfd()->GetLatestCFOptions());
+}
+
+DBOptions DBImpl::GetDBOptions() const {
+ InstrumentedMutexLock l(&mutex_);
+ return BuildDBOptions(immutable_db_options_, mutable_db_options_);
+}
+
+bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
+ const Slice& property, std::string* value) {
+ const DBPropertyInfo* property_info = GetPropertyInfo(property);
+ value->clear();
+ auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
+ if (property_info == nullptr) {
+ return false;
+ } else if (property_info->handle_int) {
+ uint64_t int_value;
+ bool ret_value =
+ GetIntPropertyInternal(cfd, *property_info, false, &int_value);
+ if (ret_value) {
+ *value = ToString(int_value);
+ }
+ return ret_value;
+ } else if (property_info->handle_string) {
+ InstrumentedMutexLock l(&mutex_);
+ return cfd->internal_stats()->GetStringProperty(*property_info, property,
+ value);
+ }
+ // Shouldn't reach here since exactly one of handle_string and handle_int
+ // should be non-nullptr.
+ assert(false);
+ return false;
+}
+
+bool DBImpl::GetMapProperty(ColumnFamilyHandle* column_family,
+ const Slice& property,
+ std::map<std::string, double>* value) {
+ const DBPropertyInfo* property_info = GetPropertyInfo(property);
+ value->clear();
+ auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
+ if (property_info == nullptr) {
+ return false;
+ } else if (property_info->handle_map) {
+ InstrumentedMutexLock l(&mutex_);
+ return cfd->internal_stats()->GetMapProperty(*property_info, property,
+ value);
+ }
+ // If we reach this point it means that handle_map is not provided for the
+ // requested property
+ return false;
+}
+
+bool DBImpl::GetIntProperty(ColumnFamilyHandle* column_family,
+ const Slice& property, uint64_t* value) {
+ const DBPropertyInfo* property_info = GetPropertyInfo(property);
+ if (property_info == nullptr || property_info->handle_int == nullptr) {
+ return false;
+ }
+ auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
+ return GetIntPropertyInternal(cfd, *property_info, false, value);
+}
+
+bool DBImpl::GetIntPropertyInternal(ColumnFamilyData* cfd,
+ const DBPropertyInfo& property_info,
+ bool is_locked, uint64_t* value) {
+ assert(property_info.handle_int != nullptr);
+ if (!property_info.need_out_of_mutex) {
+ if (is_locked) {
+ mutex_.AssertHeld();
+ return cfd->internal_stats()->GetIntProperty(property_info, value, this);
+ } else {
+ InstrumentedMutexLock l(&mutex_);
+ return cfd->internal_stats()->GetIntProperty(property_info, value, this);
+ }
+ } else {
+ SuperVersion* sv = nullptr;
+ if (!is_locked) {
+ sv = GetAndRefSuperVersion(cfd);
+ } else {
+ sv = cfd->GetSuperVersion();
+ }
+
+ bool ret = cfd->internal_stats()->GetIntPropertyOutOfMutex(
+ property_info, sv->current, value);
+
+ if (!is_locked) {
+ ReturnAndCleanupSuperVersion(cfd, sv);
+ }
+
+ return ret;
+ }
+}
+
+#ifndef ROCKSDB_LITE
+Status DBImpl::ResetStats() {
+ InstrumentedMutexLock l(&mutex_);
+ for (auto* cfd : *versions_->GetColumnFamilySet()) {
+ if (cfd->initialized()) {
+ cfd->internal_stats()->Clear();
+ }
+ }
+ return Status::OK();
+}
+#endif // ROCKSDB_LITE
+
+bool DBImpl::GetAggregatedIntProperty(const Slice& property,
+ uint64_t* aggregated_value) {
+ const DBPropertyInfo* property_info = GetPropertyInfo(property);
+ if (property_info == nullptr || property_info->handle_int == nullptr) {
+ return false;
+ }
+
+ uint64_t sum = 0;
+ {
+ // Needs mutex to protect the list of column families.
+ InstrumentedMutexLock l(&mutex_);
+ uint64_t value;
+ for (auto* cfd : *versions_->GetColumnFamilySet()) {
+ if (!cfd->initialized()) {
+ continue;
+ }
+ if (GetIntPropertyInternal(cfd, *property_info, true, &value)) {
+ sum += value;
+ } else {
+ return false;
+ }
+ }
+ }
+ *aggregated_value = sum;
+ return true;
+}
+
+SuperVersion* DBImpl::GetAndRefSuperVersion(ColumnFamilyData* cfd) {
+ // TODO(ljin): consider using GetReferencedSuperVersion() directly
+ return cfd->GetThreadLocalSuperVersion(&mutex_);
+}
+
+// REQUIRED: this function should only be called on the write thread or if the
+// mutex is held.
+SuperVersion* DBImpl::GetAndRefSuperVersion(uint32_t column_family_id) {
+ auto column_family_set = versions_->GetColumnFamilySet();
+ auto cfd = column_family_set->GetColumnFamily(column_family_id);
+ if (!cfd) {
+ return nullptr;
+ }
+
+ return GetAndRefSuperVersion(cfd);
+}
+
+void DBImpl::ReturnAndCleanupSuperVersion(ColumnFamilyData* cfd,
+ SuperVersion* sv) {
+ bool unref_sv = !cfd->ReturnThreadLocalSuperVersion(sv);
+
+ if (unref_sv) {
+ // Release SuperVersion
+ if (sv->Unref()) {
+ {
+ InstrumentedMutexLock l(&mutex_);
+ sv->Cleanup();
+ }
+ delete sv;
+ RecordTick(stats_, NUMBER_SUPERVERSION_CLEANUPS);
+ }
+ RecordTick(stats_, NUMBER_SUPERVERSION_RELEASES);
+ }
+}
+
+// REQUIRED: this function should only be called on the write thread.
+void DBImpl::ReturnAndCleanupSuperVersion(uint32_t column_family_id,
+ SuperVersion* sv) {
+ auto column_family_set = versions_->GetColumnFamilySet();
+ auto cfd = column_family_set->GetColumnFamily(column_family_id);
+
+ // If SuperVersion is held, and we successfully fetched a cfd using
+ // GetAndRefSuperVersion(), it must still exist.
+ assert(cfd != nullptr);
+ ReturnAndCleanupSuperVersion(cfd, sv);
+}
+
+// REQUIRED: this function should only be called on the write thread or if the
+// mutex is held.
+ColumnFamilyHandle* DBImpl::GetColumnFamilyHandle(uint32_t column_family_id) {
+ ColumnFamilyMemTables* cf_memtables = column_family_memtables_.get();
+
+ if (!cf_memtables->Seek(column_family_id)) {
+ return nullptr;
+ }
+
+ return cf_memtables->GetColumnFamilyHandle();
+}
+
+// REQUIRED: mutex is NOT held.
+ColumnFamilyHandle* DBImpl::GetColumnFamilyHandleUnlocked(
+ uint32_t column_family_id) {
+ ColumnFamilyMemTables* cf_memtables = column_family_memtables_.get();
+
+ InstrumentedMutexLock l(&mutex_);
+
+ if (!cf_memtables->Seek(column_family_id)) {
+ return nullptr;
+ }
+
+ return cf_memtables->GetColumnFamilyHandle();
+}
+
+void DBImpl::GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
+ const Range& range,
+ uint64_t* const count,
+ uint64_t* const size) {
+ ColumnFamilyHandleImpl* cfh =
+ reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ ColumnFamilyData* cfd = cfh->cfd();
+ SuperVersion* sv = GetAndRefSuperVersion(cfd);
+
+ // Convert user_key into a corresponding internal key.
+ InternalKey k1(range.start, kMaxSequenceNumber, kValueTypeForSeek);
+ InternalKey k2(range.limit, kMaxSequenceNumber, kValueTypeForSeek);
+ MemTable::MemTableStats memStats =
+ sv->mem->ApproximateStats(k1.Encode(), k2.Encode());
+ MemTable::MemTableStats immStats =
+ sv->imm->ApproximateStats(k1.Encode(), k2.Encode());
+ *count = memStats.count + immStats.count;
+ *size = memStats.size + immStats.size;
+
+ ReturnAndCleanupSuperVersion(cfd, sv);
+}
+
+void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,
+ const Range* range, int n, uint64_t* sizes,
+ uint8_t include_flags) {
+ assert(include_flags & DB::SizeApproximationFlags::INCLUDE_FILES ||
+ include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES);
+ Version* v;
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ auto cfd = cfh->cfd();
+ SuperVersion* sv = GetAndRefSuperVersion(cfd);
+ v = sv->current;
+
+ for (int i = 0; i < n; i++) {
+ // Convert user_key into a corresponding internal key.
+ InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek);
+ InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek);
+ sizes[i] = 0;
+ if (include_flags & DB::SizeApproximationFlags::INCLUDE_FILES) {
+ sizes[i] += versions_->ApproximateSize(v, k1.Encode(), k2.Encode());
+ }
+ if (include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES) {
+ sizes[i] += sv->mem->ApproximateStats(k1.Encode(), k2.Encode()).size;
+ sizes[i] += sv->imm->ApproximateStats(k1.Encode(), k2.Encode()).size;
+ }
+ }
+
+ ReturnAndCleanupSuperVersion(cfd, sv);
+}
+
+std::list<uint64_t>::iterator
+DBImpl::CaptureCurrentFileNumberInPendingOutputs() {
+ // We need to remember the iterator of our insert, because after the
+ // background job is done, we need to remove that element from
+ // pending_outputs_.
+ pending_outputs_.push_back(versions_->current_next_file_number());
+ auto pending_outputs_inserted_elem = pending_outputs_.end();
+ --pending_outputs_inserted_elem;
+ return pending_outputs_inserted_elem;
+}
+
+void DBImpl::ReleaseFileNumberFromPendingOutputs(
+ std::list<uint64_t>::iterator v) {
+ pending_outputs_.erase(v);
+}
+
+#ifndef ROCKSDB_LITE
+Status DBImpl::GetUpdatesSince(
+ SequenceNumber seq, unique_ptr<TransactionLogIterator>* iter,
+ const TransactionLogIterator::ReadOptions& read_options) {
+
+ RecordTick(stats_, GET_UPDATES_SINCE_CALLS);
+ if (seq > versions_->LastSequence()) {
+ return Status::NotFound("Requested sequence not yet written in the db");
+ }
+ return wal_manager_.GetUpdatesSince(seq, iter, read_options, versions_.get());
+}
+
+Status DBImpl::DeleteFile(std::string name) {
+ uint64_t number;
+ FileType type;
+ WalFileType log_type;
+ if (!ParseFileName(name, &number, &type, &log_type) ||
+ (type != kTableFile && type != kLogFile)) {
+ ROCKS_LOG_ERROR(immutable_db_options_.info_log, "DeleteFile %s failed.\n",
+ name.c_str());
+ return Status::InvalidArgument("Invalid file name");
+ }
+
+ Status status;
+ if (type == kLogFile) {
+ // Only allow deleting archived log files
+ if (log_type != kArchivedLogFile) {
+ ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+ "DeleteFile %s failed - not archived log.\n",
+ name.c_str());
+ return Status::NotSupported("Delete only supported for archived logs");
+ }
+ status =
+ env_->DeleteFile(immutable_db_options_.wal_dir + "/" + name.c_str());
+ if (!status.ok()) {
+ ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+ "DeleteFile %s failed -- %s.\n", name.c_str(),
+ status.ToString().c_str());
+ }
+ return status;
+ }
+
+ int level;
+ FileMetaData* metadata;
+ ColumnFamilyData* cfd;
+ VersionEdit edit;
+ JobContext job_context(next_job_id_.fetch_add(1), true);
+ {
+ InstrumentedMutexLock l(&mutex_);
+ status = versions_->GetMetadataForFile(number, &level, &metadata, &cfd);
+ if (!status.ok()) {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "DeleteFile %s failed. File not found\n", name.c_str());
+ job_context.Clean();
+ return Status::InvalidArgument("File not found");
+ }
+ assert(level < cfd->NumberLevels());
+
+ // If the file is being compacted no need to delete.
+ if (metadata->being_compacted) {
+ ROCKS_LOG_INFO(immutable_db_options_.info_log,
+ "DeleteFile %s Skipped. File about to be compacted\n",
+ name.c_str());
+ job_context.Clean();
+ return Status::OK();
+ }
+
+ // Only the files in the last level can be deleted externally.
+ // This is to make sure that any deletion tombstones are not
+ // lost. Check that the level passed is the last level.
+ auto* vstoreage = cfd->current()->storage_info();
+ for (int i = level + 1; i < cfd->NumberLevels(); i++) {
+ if (vstoreage->NumLevelFiles(i) != 0) {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "DeleteFile %s FAILED. File not in last level\n",
+ name.c_str());
+ job_context.Clean();
+ return Status::InvalidArgument("File not in last level");
+ }
+ }
+ // if level == 0, it has to be the oldest file
+ if (level == 0 &&
+ vstoreage->LevelFiles(0).back()->fd.GetNumber() != number) {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "DeleteFile %s failed ---"
+ " target file in level 0 must be the oldest.",
+ name.c_str());
+ job_context.Clean();
+ return Status::InvalidArgument("File in level 0, but not oldest");
+ }
+ edit.SetColumnFamily(cfd->GetID());
+ edit.DeleteFile(level, number);
+ status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
+ &edit, &mutex_, directories_.GetDbDir());
+ if (status.ok()) {
+ InstallSuperVersionAndScheduleWorkWrapper(
+ cfd, &job_context, *cfd->GetLatestMutableCFOptions());
+ }
+ FindObsoleteFiles(&job_context, false);
+ } // lock released here
+
+ LogFlush(immutable_db_options_.info_log);
+ // remove files outside the db-lock
+ if (job_context.HaveSomethingToDelete()) {
+ // Call PurgeObsoleteFiles() without holding mutex.
+ PurgeObsoleteFiles(job_context);
+ }
+ job_context.Clean();
+ return status;
+}
+
+Status DBImpl::DeleteFilesInRange(ColumnFamilyHandle* column_family,
+ const Slice* begin, const Slice* end) {
+ Status status;
+ auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+ ColumnFamilyData* cfd = cfh->cfd();
+ VersionEdit edit;
+ std::vector<FileMetaData*> deleted_files;
+ JobContext job_context(next_job_id_.fetch_add(1), true);
+ {
+ InstrumentedMutexLock l(&mutex_);
+ Version* input_version = cfd->current();
+
+ auto* vstorage = input_version->storage_info();
+ for (int i = 1; i < cfd->NumberLevels(); i++) {
+ if (vstorage->LevelFiles(i).empty() ||
+ !vstorage->OverlapInLevel(i, begin, end)) {
+ continue;
+ }
+ std::vector<FileMetaData*> level_files;
+ InternalKey begin_storage, end_storage, *begin_key, *end_key;
+ if (begin == nullptr) {
+ begin_key = nullptr;
+ } else {
+ begin_storage.SetMaxPossibleForUserKey(*begin);
+ begin_key = &begin_storage;
+ }
+ if (end == nullptr) {
+ end_key = nullptr;
+ } else {
+ end_storage.SetMinPossibleForUserKey(*end);
+ end_key = &end_storage;
+ }
+
+ vstorage->GetOverlappingInputs(i, begin_key, end_key, &level_files, -1,
+ nullptr, false);
+ FileMetaData* level_file;
+ for (uint32_t j = 0; j < level_files.size(); j++) {
+ level_file = level_files[j];
+ if (((begin == nullptr) ||
+ (cfd->internal_comparator().user_comparator()->Compare(
+ level_file->smallest.user_key(), *begin) >= 0)) &&
+ ((end == nullptr) ||
+ (cfd->internal_comparator().user_comparator()->Compare(
+ level_file->largest.user_key(), *end) <= 0))) {
+ if (level_file->being_compacted) {
+ continue;
+ }
+ edit.SetColumnFamily(cfd->GetID());
+ edit.DeleteFile(i, level_file->fd.GetNumber());
+ deleted_files.push_back(level_file);
+ level_file->being_compacted = true;
+ }
+ }
+ }
+ if (edit.GetDeletedFiles().empty()) {
+ job_context.Clean();
+ return Status::OK();
+ }
+ input_version->Ref();
+ status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
+ &edit, &mutex_, directories_.GetDbDir());
+ if (status.ok()) {
+ InstallSuperVersionAndScheduleWorkWrapper(
+ cfd, &job_context, *cfd->GetLatestMutableCFOptions());
+ }
+ for (auto* deleted_file : deleted_files) {
+ deleted_file->being_compacted = false;
+ }
+ input_version->Unref();
+ FindObsoleteFiles(&job_context, false);
+ } // lock released here
+
+ LogFlush(immutable_db_options_.info_log);
+ // remove files outside the db-lock
+ if (job_context.HaveSomethingToDelete()) {
+ // Call PurgeObsoleteFiles() without holding mutex.
+ PurgeObsoleteFiles(job_context);
+ }
+ job_context.Clean();
+ return status;
+}
+
+void DBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
+ InstrumentedMutexLock l(&mutex_);
+ versions_->GetLiveFilesMetaData(metadata);
+}
+
+void DBImpl::GetColumnFamilyMetaData(
+ ColumnFamilyHandle* column_family,
+ ColumnFamilyMetaData* cf_meta) {
+ assert(column_family);
+ auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
+ auto* sv = GetAndRefSuperVersion(cfd);
+ sv->current->GetColumnFamilyMetaData(cf_meta);
+ ReturnAndCleanupSuperVersion(cfd, sv);
+}
+
+#endif // ROCKSDB_LITE
+
+Status DBImpl::CheckConsistency() {
+ mutex_.AssertHeld();
+ std::vector<LiveFileMetaData> metadata;
+ versions_->GetLiveFilesMetaData(&metadata);
+
+ std::string corruption_messages;
+ for (const auto& md : metadata) {
+ // md.name has a leading "/".
+ std::string file_path = md.db_path + md.name;
+
+ uint64_t fsize = 0;
+ Status s = env_->GetFileSize(file_path, &fsize);
+ if (!s.ok() &&
+ env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok()) {
+ s = Status::OK();
+ }
+ if (!s.ok()) {
+ corruption_messages +=
+ "Can't access " + md.name + ": " + s.ToString() + "\n";
+ } else if (fsize != md.size) {
+ corruption_messages += "Sst file size mismatch: " + file_path +
+ ". Size recorded in manifest " +
+ ToString(md.size) + ", actual size " +
+ ToString(fsize) + "\n";
+ }
+ }
+ if (corruption_messages.size() == 0) {
+ return Status::OK();
+ } else {
+ return Status::Corruption(corruption_messages);
+ }
+}
+
+Status DBImpl::GetDbIdentity(std::string& identity) const {
+ std::string idfilename = IdentityFileName(dbname_);
+ const EnvOptions soptions;
+ unique_ptr<SequentialFileReader> id_file_reader;
+ Status s;
+ {
+ unique_ptr<SequentialFile> idfile;
+ s = env_->NewSequentialFile(idfilename, &idfile, soptions);
+ if (!s.ok()) {
+ return s;
+ }
+ id_file_reader.reset(new SequentialFileReader(std::move(idfile)));
+ }
+
+ uint64_t file_size;
+ s = env_->GetFileSize(idfilename, &file_size);
+ if (!s.ok()) {
+ return s;
+ }
+ char* buffer = reinterpret_cast<char*>(alloca(file_size));
+ Slice id;
+ s = id_file_reader->Read(static_cast<size_t>(file_size), &id, buffer);
+ if (!s.ok()) {
+ return s;
+ }
+ identity.assign(id.ToString());
+ // If last character is '\n' remove it from identity
+ if (identity.size() > 0 && identity.back() == '\n') {
+ identity.pop_back();
+ }
+ return s;
+}
+
+// Default implementation -- returns not supported status
+Status DB::CreateColumnFamily(const ColumnFamilyOptions& cf_options,
+ const std::string& column_family_name,
+ ColumnFamilyHandle** handle) {
+ return Status::NotSupported("");
+}
+
+Status DB::CreateColumnFamilies(
+ const ColumnFamilyOptions& cf_options,
+ const std::vector<std::string>& column_family_names,
+ std::vector<ColumnFamilyHandle*>* handles) {
+ return Status::NotSupported("");
+}
+
+Status DB::CreateColumnFamilies(
+ const std::vector<ColumnFamilyDescriptor>& column_families,
+ std::vector<ColumnFamilyHandle*>* handles) {
+ return Status::NotSupported("");
+}
+
+Status DB::DropColumnFamily(ColumnFamilyHandle* column_family) {
+ return Status::NotSupported("");
+}
+
+Status DB::DropColumnFamilies(
+ const std::vector<ColumnFamilyHandle*>& column_families) {
+ return Status::NotSupported("");
+}
+
+Status DB::DestroyColumnFamilyHandle(ColumnFamilyHandle* column_family) {
+ delete column_family;
+ return Status::OK();
+}
+
+DB::~DB() { }
+
+Status DB::ListColumnFamilies(const DBOptions& db_options,
+ const std::string& name,
+ std::vector<std::string>* column_families) {
+ return VersionSet::ListColumnFamilies(column_families, name, db_options.env);
+}
+
+Snapshot::~Snapshot() {
+}
+
+Status DestroyDB(const std::string& dbname, const Options& options) {
+ const ImmutableDBOptions soptions(SanitizeOptions(dbname, options));
+ Env* env = soptions.env;
+ std::vector<std::string> filenames;
+
+ // Ignore error in case directory does not exist
+ env->GetChildren(dbname, &filenames);
+
+ FileLock* lock;
+ const std::string lockname = LockFileName(dbname);
+ Status result = env->LockFile(lockname, &lock);
+ if (result.ok()) {
+ uint64_t number;
+ FileType type;
+ InfoLogPrefix info_log_prefix(!soptions.db_log_dir.empty(), dbname);
+ for (size_t i = 0; i < filenames.size(); i++) {
+ if (ParseFileName(filenames[i], &number, info_log_prefix.prefix, &type) &&
+ type != kDBLockFile) { // Lock file will be deleted at end
+ Status del;
+ std::string path_to_delete = dbname + "/" + filenames[i];
+ if (type == kMetaDatabase) {
+ del = DestroyDB(path_to_delete, options);
+ } else if (type == kTableFile) {
+ del = DeleteSSTFile(&soptions, path_to_delete, 0);
+ } else {
+ del = env->DeleteFile(path_to_delete);
+ }
+ if (result.ok() && !del.ok()) {
+ result = del;
+ }
+ }
+ }
+
+ for (size_t path_id = 0; path_id < options.db_paths.size(); path_id++) {
+ const auto& db_path = options.db_paths[path_id];
+ env->GetChildren(db_path.path, &filenames);
+ for (size_t i = 0; i < filenames.size(); i++) {
+ if (ParseFileName(filenames[i], &number, &type) &&
+ type == kTableFile) { // Lock file will be deleted at end
+ std::string table_path = db_path.path + "/" + filenames[i];
+ Status del = DeleteSSTFile(&soptions, table_path,
+ static_cast<uint32_t>(path_id));
+ if (result.ok() && !del.ok()) {
+ result = del;
+ }
+ }
+ }
+ }
+
+ std::vector<std::string> walDirFiles;
+ std::string archivedir = ArchivalDirectory(dbname);
+ if (dbname != soptions.wal_dir) {
+ env->GetChildren(soptions.wal_dir, &walDirFiles);
+ archivedir = ArchivalDirectory(soptions.wal_dir);
+ }
+
+ // Delete log files in the WAL dir
+ for (const auto& file : walDirFiles) {
+ if (ParseFileName(file, &number, &type) && type == kLogFile) {
+ Status del = env->DeleteFile(LogFileName(soptions.wal_dir, number));
+ if (result.ok() && !del.ok()) {
+ result = del;
+ }
+ }
+ }
+
+ std::vector<std::string> archiveFiles;
+ env->GetChildren(archivedir, &archiveFiles);
+ // Delete archival files.
+ for (size_t i = 0; i < archiveFiles.size(); ++i) {
+ if (ParseFileName(archiveFiles[i], &number, &type) &&
+ type == kLogFile) {
+ Status del = env->DeleteFile(archivedir + "/" + archiveFiles[i]);
+ if (result.ok() && !del.ok()) {
+ result = del;
+ }
+ }
+ }
+
+ // ignore case where no archival directory is present
+ env->DeleteDir(archivedir);
+
+ env->UnlockFile(lock); // Ignore error since state is already gone
+ env->DeleteFile(lockname);
+ env->DeleteDir(dbname); // Ignore error in case dir contains other files
+ env->DeleteDir(soptions.wal_dir);
+ }
+ return result;
+}
+
+Status DBImpl::WriteOptionsFile(bool need_mutex_lock,
+ bool need_enter_write_thread) {
+#ifndef ROCKSDB_LITE
+ WriteThread::Writer w;
+ if (need_mutex_lock) {
+ mutex_.Lock();
+ } else {
+ mutex_.AssertHeld();
+ }
+ if (need_enter_write_thread) {
+ write_thread_.EnterUnbatched(&w, &mutex_);
+ }
+
+ std::vector<std::string> cf_names;
+ std::vector<ColumnFamilyOptions> cf_opts;
+
+ // This part requires mutex to protect the column family options
+ for (auto cfd : *versions_->GetColumnFamilySet()) {
+ if (cfd->IsDropped()) {
+ continue;
+ }
+ cf_names.push_back(cfd->GetName());
+ cf_opts.push_back(cfd->GetLatestCFOptions());
+ }
+
+ // Unlock during expensive operations. New writes cannot get here
+ // because the single write thread ensures all new writes get queued.
+ DBOptions db_options =
+ BuildDBOptions(immutable_db_options_, mutable_db_options_);
+ mutex_.Unlock();
+
+ TEST_SYNC_POINT("DBImpl::WriteOptionsFile:1");
+ TEST_SYNC_POINT("DBImpl::WriteOptionsFile:2");
+
+ std::string file_name =
+ TempOptionsFileName(GetName(), versions_->NewFileNumber());
+ Status s =
+ PersistRocksDBOptions(db_options, cf_names, cf_opts, file_name, GetEnv());
+
+ if (s.ok()) {
+ s = RenameTempFileToOptionsFile(file_name);
+ }
+ // restore lock
+ if (!need_mutex_lock) {
+ mutex_.Lock();
+ }
+ if (need_enter_write_thread) {
+ write_thread_.ExitUnbatched(&w);
+ }
+ if (!s.ok()) {
+ ROCKS_LOG_WARN(immutable_db_options_.info_log,
+ "Unnable to persist options -- %s", s.ToString().c_str());
+ if (immutable_db_options_.fail_if_options_file_error) {
+ return Status::IOError("Unable to persist options.",
+ s.ToString().c_str());
+ }
+ }
+#endif // !ROCKSDB_LITE
+ return Status::OK();
+}
+
+#ifndef ROCKSDB_LITE
+namespace {
+void DeleteOptionsFilesHelper(const std::map<uint64_t, std::string>& filenames,
+ const size_t num_files_to_keep,
+ const std::shared_ptr<Logger>& info_log,
+ Env* env) {
+ if (filenames.size() <= num_files_to_keep) {
+ return;
+ }
+ for (auto iter = std::next(filenames.begin(), num_files_to_keep);
+ iter != filenames.end(); ++iter) {
+ if (!env->DeleteFile(iter->second).ok()) {
+ ROCKS_LOG_WARN(info_log, "Unable to delete options file %s",
+ iter->second.c_str());
+ }
+ }
+}
+} // namespace
+#endif // !ROCKSDB_LITE
+
+Status DBImpl::DeleteObsoleteOptionsFiles() {
+#ifndef ROCKSDB_LITE
+ std::vector<std::string> filenames;
+ // use ordered map to store keep the filenames sorted from the newest
+ // to the oldest.
+ std::map<uint64_t, std::string> options_filenames;
+ Status s;
+ s = GetEnv()->GetChildren(GetName(), &filenames);
+ if (!s.ok()) {
+ return s;
+ }
+ for (auto& filename : filenames) {
+ uint64_t file_number;
+ FileType type;
+ if (ParseFileName(filename, &file_number, &type) && type == kOptionsFile) {
+ options_filenames.insert(
+ {std::numeric_limits<uint64_t>::max() - file_number,
+ GetName() + "/" + filename});
+ }
+ }
+
+ // Keeps the latest 2 Options file
+ const size_t kNumOptionsFilesKept = 2;
+ DeleteOptionsFilesHelper(options_filenames, kNumOptionsFilesKept,
+ immutable_db_options_.info_log, GetEnv());
+ return Status::OK();
+#else
+ return Status::OK();
+#endif // !ROCKSDB_LITE
+}
+
+Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name) {
+#ifndef ROCKSDB_LITE
+ Status s;
+
+ versions_->options_file_number_ = versions_->NewFileNumber();
+ std::string options_file_name =
+ OptionsFileName(GetName(), versions_->options_file_number_);
+ // Retry if the file name happen to conflict with an existing one.
+ s = GetEnv()->RenameFile(file_name, options_file_name);
+
+ DeleteObsoleteOptionsFiles();
+ return s;
+#else
+ return Status::OK();
+#endif // !ROCKSDB_LITE
+}
+
+#ifdef ROCKSDB_USING_THREAD_STATUS
+
+void DBImpl::NewThreadStatusCfInfo(
+ ColumnFamilyData* cfd) const {
+ if (immutable_db_options_.enable_thread_tracking) {
+ ThreadStatusUtil::NewColumnFamilyInfo(this, cfd, cfd->GetName(),
+ cfd->ioptions()->env);
+ }
+}
+
+void DBImpl::EraseThreadStatusCfInfo(
+ ColumnFamilyData* cfd) const {
+ if (immutable_db_options_.enable_thread_tracking) {
+ ThreadStatusUtil::EraseColumnFamilyInfo(cfd);
+ }
+}
+
+void DBImpl::EraseThreadStatusDbInfo() const {
+ if (immutable_db_options_.enable_thread_tracking) {
+ ThreadStatusUtil::EraseDatabaseInfo(this);
+ }
+}
+
+#else
+void DBImpl::NewThreadStatusCfInfo(
+ ColumnFamilyData* cfd) const {
+}
+
+void DBImpl::EraseThreadStatusCfInfo(
+ ColumnFamilyData* cfd) const {
+}
+
+void DBImpl::EraseThreadStatusDbInfo() const {
+}
+#endif // ROCKSDB_USING_THREAD_STATUS
+
+//
+// A global method that can dump out the build version
+void DumpRocksDBBuildVersion(Logger * log) {
+#if !defined(IOS_CROSS_COMPILE)
+ // if we compile with Xcode, we don't r
<TRUNCATED>
[09/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/version_set.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/version_set.cc b/thirdparty/rocksdb/db/version_set.cc
new file mode 100644
index 0000000..6b9611a
--- /dev/null
+++ b/thirdparty/rocksdb/db/version_set.cc
@@ -0,0 +1,3801 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/version_set.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <stdio.h>
+#include <algorithm>
+#include <climits>
+#include <map>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include "db/compaction.h"
+#include "db/internal_stats.h"
+#include "db/log_reader.h"
+#include "db/log_writer.h"
+#include "db/memtable.h"
+#include "db/merge_context.h"
+#include "db/merge_helper.h"
+#include "db/pinned_iterators_manager.h"
+#include "db/table_cache.h"
+#include "db/version_builder.h"
+#include "monitoring/file_read_sample.h"
+#include "monitoring/perf_context_imp.h"
+#include "rocksdb/env.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/write_buffer_manager.h"
+#include "table/format.h"
+#include "table/get_context.h"
+#include "table/internal_iterator.h"
+#include "table/merging_iterator.h"
+#include "table/meta_blocks.h"
+#include "table/plain_table_factory.h"
+#include "table/table_reader.h"
+#include "table/two_level_iterator.h"
+#include "util/coding.h"
+#include "util/file_reader_writer.h"
+#include "util/filename.h"
+#include "util/stop_watch.h"
+#include "util/string_util.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+namespace {
+
+// Find File in LevelFilesBrief data structure
+// Within an index range defined by left and right
+int FindFileInRange(const InternalKeyComparator& icmp,
+ const LevelFilesBrief& file_level,
+ const Slice& key,
+ uint32_t left,
+ uint32_t right) {
+ while (left < right) {
+ uint32_t mid = (left + right) / 2;
+ const FdWithKeyRange& f = file_level.files[mid];
+ if (icmp.InternalKeyComparator::Compare(f.largest_key, key) < 0) {
+ // Key at "mid.largest" is < "target". Therefore all
+ // files at or before "mid" are uninteresting.
+ left = mid + 1;
+ } else {
+ // Key at "mid.largest" is >= "target". Therefore all files
+ // after "mid" are uninteresting.
+ right = mid;
+ }
+ }
+ return right;
+}
+
+// Class to help choose the next file to search for the particular key.
+// Searches and returns files level by level.
+// We can search level-by-level since entries never hop across
+// levels. Therefore we are guaranteed that if we find data
+// in a smaller level, later levels are irrelevant (unless we
+// are MergeInProgress).
+class FilePicker {
+ public:
+ FilePicker(std::vector<FileMetaData*>* files, const Slice& user_key,
+ const Slice& ikey, autovector<LevelFilesBrief>* file_levels,
+ unsigned int num_levels, FileIndexer* file_indexer,
+ const Comparator* user_comparator,
+ const InternalKeyComparator* internal_comparator)
+ : num_levels_(num_levels),
+ curr_level_(static_cast<unsigned int>(-1)),
+ returned_file_level_(static_cast<unsigned int>(-1)),
+ hit_file_level_(static_cast<unsigned int>(-1)),
+ search_left_bound_(0),
+ search_right_bound_(FileIndexer::kLevelMaxIndex),
+#ifndef NDEBUG
+ files_(files),
+#endif
+ level_files_brief_(file_levels),
+ is_hit_file_last_in_level_(false),
+ user_key_(user_key),
+ ikey_(ikey),
+ file_indexer_(file_indexer),
+ user_comparator_(user_comparator),
+ internal_comparator_(internal_comparator) {
+ // Setup member variables to search first level.
+ search_ended_ = !PrepareNextLevel();
+ if (!search_ended_) {
+ // Prefetch Level 0 table data to avoid cache miss if possible.
+ for (unsigned int i = 0; i < (*level_files_brief_)[0].num_files; ++i) {
+ auto* r = (*level_files_brief_)[0].files[i].fd.table_reader;
+ if (r) {
+ r->Prepare(ikey);
+ }
+ }
+ }
+ }
+
+ int GetCurrentLevel() const { return curr_level_; }
+
+ FdWithKeyRange* GetNextFile() {
+ while (!search_ended_) { // Loops over different levels.
+ while (curr_index_in_curr_level_ < curr_file_level_->num_files) {
+ // Loops over all files in current level.
+ FdWithKeyRange* f = &curr_file_level_->files[curr_index_in_curr_level_];
+ hit_file_level_ = curr_level_;
+ is_hit_file_last_in_level_ =
+ curr_index_in_curr_level_ == curr_file_level_->num_files - 1;
+ int cmp_largest = -1;
+
+ // Do key range filtering of files or/and fractional cascading if:
+ // (1) not all the files are in level 0, or
+ // (2) there are more than 3 current level files
+ // If there are only 3 or less current level files in the system, we skip
+ // the key range filtering. In this case, more likely, the system is
+ // highly tuned to minimize number of tables queried by each query,
+ // so it is unlikely that key range filtering is more efficient than
+ // querying the files.
+ if (num_levels_ > 1 || curr_file_level_->num_files > 3) {
+ // Check if key is within a file's range. If search left bound and
+ // right bound point to the same find, we are sure key falls in
+ // range.
+ assert(
+ curr_level_ == 0 ||
+ curr_index_in_curr_level_ == start_index_in_curr_level_ ||
+ user_comparator_->Compare(user_key_,
+ ExtractUserKey(f->smallest_key)) <= 0);
+
+ int cmp_smallest = user_comparator_->Compare(user_key_,
+ ExtractUserKey(f->smallest_key));
+ if (cmp_smallest >= 0) {
+ cmp_largest = user_comparator_->Compare(user_key_,
+ ExtractUserKey(f->largest_key));
+ }
+
+ // Setup file search bound for the next level based on the
+ // comparison results
+ if (curr_level_ > 0) {
+ file_indexer_->GetNextLevelIndex(curr_level_,
+ curr_index_in_curr_level_,
+ cmp_smallest, cmp_largest,
+ &search_left_bound_,
+ &search_right_bound_);
+ }
+ // Key falls out of current file's range
+ if (cmp_smallest < 0 || cmp_largest > 0) {
+ if (curr_level_ == 0) {
+ ++curr_index_in_curr_level_;
+ continue;
+ } else {
+ // Search next level.
+ break;
+ }
+ }
+ }
+#ifndef NDEBUG
+ // Sanity check to make sure that the files are correctly sorted
+ if (prev_file_) {
+ if (curr_level_ != 0) {
+ int comp_sign = internal_comparator_->Compare(
+ prev_file_->largest_key, f->smallest_key);
+ assert(comp_sign < 0);
+ } else {
+ // level == 0, the current file cannot be newer than the previous
+ // one. Use compressed data structure, has no attribute seqNo
+ assert(curr_index_in_curr_level_ > 0);
+ assert(!NewestFirstBySeqNo(files_[0][curr_index_in_curr_level_],
+ files_[0][curr_index_in_curr_level_-1]));
+ }
+ }
+ prev_file_ = f;
+#endif
+ returned_file_level_ = curr_level_;
+ if (curr_level_ > 0 && cmp_largest < 0) {
+ // No more files to search in this level.
+ search_ended_ = !PrepareNextLevel();
+ } else {
+ ++curr_index_in_curr_level_;
+ }
+ return f;
+ }
+ // Start searching next level.
+ search_ended_ = !PrepareNextLevel();
+ }
+ // Search ended.
+ return nullptr;
+ }
+
+ // getter for current file level
+ // for GET_HIT_L0, GET_HIT_L1 & GET_HIT_L2_AND_UP counts
+ unsigned int GetHitFileLevel() { return hit_file_level_; }
+
+ // Returns true if the most recent "hit file" (i.e., one returned by
+ // GetNextFile()) is at the last index in its level.
+ bool IsHitFileLastInLevel() { return is_hit_file_last_in_level_; }
+
+ private:
+ unsigned int num_levels_;
+ unsigned int curr_level_;
+ unsigned int returned_file_level_;
+ unsigned int hit_file_level_;
+ int32_t search_left_bound_;
+ int32_t search_right_bound_;
+#ifndef NDEBUG
+ std::vector<FileMetaData*>* files_;
+#endif
+ autovector<LevelFilesBrief>* level_files_brief_;
+ bool search_ended_;
+ bool is_hit_file_last_in_level_;
+ LevelFilesBrief* curr_file_level_;
+ unsigned int curr_index_in_curr_level_;
+ unsigned int start_index_in_curr_level_;
+ Slice user_key_;
+ Slice ikey_;
+ FileIndexer* file_indexer_;
+ const Comparator* user_comparator_;
+ const InternalKeyComparator* internal_comparator_;
+#ifndef NDEBUG
+ FdWithKeyRange* prev_file_;
+#endif
+
+ // Setup local variables to search next level.
+ // Returns false if there are no more levels to search.
+ bool PrepareNextLevel() {
+ curr_level_++;
+ while (curr_level_ < num_levels_) {
+ curr_file_level_ = &(*level_files_brief_)[curr_level_];
+ if (curr_file_level_->num_files == 0) {
+ // When current level is empty, the search bound generated from upper
+ // level must be [0, -1] or [0, FileIndexer::kLevelMaxIndex] if it is
+ // also empty.
+ assert(search_left_bound_ == 0);
+ assert(search_right_bound_ == -1 ||
+ search_right_bound_ == FileIndexer::kLevelMaxIndex);
+ // Since current level is empty, it will need to search all files in
+ // the next level
+ search_left_bound_ = 0;
+ search_right_bound_ = FileIndexer::kLevelMaxIndex;
+ curr_level_++;
+ continue;
+ }
+
+ // Some files may overlap each other. We find
+ // all files that overlap user_key and process them in order from
+ // newest to oldest. In the context of merge-operator, this can occur at
+ // any level. Otherwise, it only occurs at Level-0 (since Put/Deletes
+ // are always compacted into a single entry).
+ int32_t start_index;
+ if (curr_level_ == 0) {
+ // On Level-0, we read through all files to check for overlap.
+ start_index = 0;
+ } else {
+ // On Level-n (n>=1), files are sorted. Binary search to find the
+ // earliest file whose largest key >= ikey. Search left bound and
+ // right bound are used to narrow the range.
+ if (search_left_bound_ == search_right_bound_) {
+ start_index = search_left_bound_;
+ } else if (search_left_bound_ < search_right_bound_) {
+ if (search_right_bound_ == FileIndexer::kLevelMaxIndex) {
+ search_right_bound_ =
+ static_cast<int32_t>(curr_file_level_->num_files) - 1;
+ }
+ start_index =
+ FindFileInRange(*internal_comparator_, *curr_file_level_, ikey_,
+ static_cast<uint32_t>(search_left_bound_),
+ static_cast<uint32_t>(search_right_bound_));
+ } else {
+ // search_left_bound > search_right_bound, key does not exist in
+ // this level. Since no comparison is done in this level, it will
+ // need to search all files in the next level.
+ search_left_bound_ = 0;
+ search_right_bound_ = FileIndexer::kLevelMaxIndex;
+ curr_level_++;
+ continue;
+ }
+ }
+ start_index_in_curr_level_ = start_index;
+ curr_index_in_curr_level_ = start_index;
+#ifndef NDEBUG
+ prev_file_ = nullptr;
+#endif
+ return true;
+ }
+ // curr_level_ = num_levels_. So, no more levels to search.
+ return false;
+ }
+};
+} // anonymous namespace
+
+VersionStorageInfo::~VersionStorageInfo() { delete[] files_; }
+
+Version::~Version() {
+ assert(refs_ == 0);
+
+ // Remove from linked list
+ prev_->next_ = next_;
+ next_->prev_ = prev_;
+
+ // Drop references to files
+ for (int level = 0; level < storage_info_.num_levels_; level++) {
+ for (size_t i = 0; i < storage_info_.files_[level].size(); i++) {
+ FileMetaData* f = storage_info_.files_[level][i];
+ assert(f->refs > 0);
+ f->refs--;
+ if (f->refs <= 0) {
+ vset_->obsolete_files_.push_back(f);
+ }
+ }
+ }
+}
+
+int FindFile(const InternalKeyComparator& icmp,
+ const LevelFilesBrief& file_level,
+ const Slice& key) {
+ return FindFileInRange(icmp, file_level, key, 0,
+ static_cast<uint32_t>(file_level.num_files));
+}
+
+void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
+ const std::vector<FileMetaData*>& files,
+ Arena* arena) {
+ assert(file_level);
+ assert(arena);
+
+ size_t num = files.size();
+ file_level->num_files = num;
+ char* mem = arena->AllocateAligned(num * sizeof(FdWithKeyRange));
+ file_level->files = new (mem)FdWithKeyRange[num];
+
+ for (size_t i = 0; i < num; i++) {
+ Slice smallest_key = files[i]->smallest.Encode();
+ Slice largest_key = files[i]->largest.Encode();
+
+ // Copy key slice to sequential memory
+ size_t smallest_size = smallest_key.size();
+ size_t largest_size = largest_key.size();
+ mem = arena->AllocateAligned(smallest_size + largest_size);
+ memcpy(mem, smallest_key.data(), smallest_size);
+ memcpy(mem + smallest_size, largest_key.data(), largest_size);
+
+ FdWithKeyRange& f = file_level->files[i];
+ f.fd = files[i]->fd;
+ f.file_metadata = files[i];
+ f.smallest_key = Slice(mem, smallest_size);
+ f.largest_key = Slice(mem + smallest_size, largest_size);
+ }
+}
+
+static bool AfterFile(const Comparator* ucmp,
+ const Slice* user_key, const FdWithKeyRange* f) {
+ // nullptr user_key occurs before all keys and is therefore never after *f
+ return (user_key != nullptr &&
+ ucmp->Compare(*user_key, ExtractUserKey(f->largest_key)) > 0);
+}
+
+static bool BeforeFile(const Comparator* ucmp,
+ const Slice* user_key, const FdWithKeyRange* f) {
+ // nullptr user_key occurs after all keys and is therefore never before *f
+ return (user_key != nullptr &&
+ ucmp->Compare(*user_key, ExtractUserKey(f->smallest_key)) < 0);
+}
+
+bool SomeFileOverlapsRange(
+ const InternalKeyComparator& icmp,
+ bool disjoint_sorted_files,
+ const LevelFilesBrief& file_level,
+ const Slice* smallest_user_key,
+ const Slice* largest_user_key) {
+ const Comparator* ucmp = icmp.user_comparator();
+ if (!disjoint_sorted_files) {
+ // Need to check against all files
+ for (size_t i = 0; i < file_level.num_files; i++) {
+ const FdWithKeyRange* f = &(file_level.files[i]);
+ if (AfterFile(ucmp, smallest_user_key, f) ||
+ BeforeFile(ucmp, largest_user_key, f)) {
+ // No overlap
+ } else {
+ return true; // Overlap
+ }
+ }
+ return false;
+ }
+
+ // Binary search over file list
+ uint32_t index = 0;
+ if (smallest_user_key != nullptr) {
+ // Find the earliest possible internal key for smallest_user_key
+ InternalKey small;
+ small.SetMaxPossibleForUserKey(*smallest_user_key);
+ index = FindFile(icmp, file_level, small.Encode());
+ }
+
+ if (index >= file_level.num_files) {
+ // beginning of range is after all files, so no overlap.
+ return false;
+ }
+
+ return !BeforeFile(ucmp, largest_user_key, &file_level.files[index]);
+}
+
+namespace {
+
+// An internal iterator. For a given version/level pair, yields
+// information about the files in the level. For a given entry, key()
+// is the largest key that occurs in the file, and value() is an
+// 16-byte value containing the file number and file size, both
+// encoded using EncodeFixed64.
+class LevelFileNumIterator : public InternalIterator {
+ public:
+ LevelFileNumIterator(const InternalKeyComparator& icmp,
+ const LevelFilesBrief* flevel, bool should_sample)
+ : icmp_(icmp),
+ flevel_(flevel),
+ index_(static_cast<uint32_t>(flevel->num_files)),
+ current_value_(0, 0, 0), // Marks as invalid
+ should_sample_(should_sample) {}
+ virtual bool Valid() const override { return index_ < flevel_->num_files; }
+ virtual void Seek(const Slice& target) override {
+ index_ = FindFile(icmp_, *flevel_, target);
+ }
+ virtual void SeekForPrev(const Slice& target) override {
+ SeekForPrevImpl(target, &icmp_);
+ }
+
+ virtual void SeekToFirst() override { index_ = 0; }
+ virtual void SeekToLast() override {
+ index_ = (flevel_->num_files == 0)
+ ? 0
+ : static_cast<uint32_t>(flevel_->num_files) - 1;
+ }
+ virtual void Next() override {
+ assert(Valid());
+ index_++;
+ }
+ virtual void Prev() override {
+ assert(Valid());
+ if (index_ == 0) {
+ index_ = static_cast<uint32_t>(flevel_->num_files); // Marks as invalid
+ } else {
+ index_--;
+ }
+ }
+ Slice key() const override {
+ assert(Valid());
+ return flevel_->files[index_].largest_key;
+ }
+ Slice value() const override {
+ assert(Valid());
+
+ auto file_meta = flevel_->files[index_];
+ if (should_sample_) {
+ sample_file_read_inc(file_meta.file_metadata);
+ }
+ current_value_ = file_meta.fd;
+ return Slice(reinterpret_cast<const char*>(¤t_value_),
+ sizeof(FileDescriptor));
+ }
+ virtual Status status() const override { return Status::OK(); }
+
+ private:
+ const InternalKeyComparator icmp_;
+ const LevelFilesBrief* flevel_;
+ uint32_t index_;
+ mutable FileDescriptor current_value_;
+ bool should_sample_;
+};
+
+class LevelFileIteratorState : public TwoLevelIteratorState {
+ public:
+ // @param skip_filters Disables loading/accessing the filter block
+ LevelFileIteratorState(TableCache* table_cache,
+ const ReadOptions& read_options,
+ const EnvOptions& env_options,
+ const InternalKeyComparator& icomparator,
+ HistogramImpl* file_read_hist, bool for_compaction,
+ bool prefix_enabled, bool skip_filters, int level,
+ RangeDelAggregator* range_del_agg)
+ : TwoLevelIteratorState(prefix_enabled),
+ table_cache_(table_cache),
+ read_options_(read_options),
+ env_options_(env_options),
+ icomparator_(icomparator),
+ file_read_hist_(file_read_hist),
+ for_compaction_(for_compaction),
+ skip_filters_(skip_filters),
+ level_(level),
+ range_del_agg_(range_del_agg) {}
+
+ InternalIterator* NewSecondaryIterator(const Slice& meta_handle) override {
+ if (meta_handle.size() != sizeof(FileDescriptor)) {
+ return NewErrorInternalIterator(
+ Status::Corruption("FileReader invoked with unexpected value"));
+ }
+ const FileDescriptor* fd =
+ reinterpret_cast<const FileDescriptor*>(meta_handle.data());
+ return table_cache_->NewIterator(
+ read_options_, env_options_, icomparator_, *fd, range_del_agg_,
+ nullptr /* don't need reference to table */, file_read_hist_,
+ for_compaction_, nullptr /* arena */, skip_filters_, level_);
+ }
+
+ bool PrefixMayMatch(const Slice& internal_key) override {
+ return true;
+ }
+
+ bool KeyReachedUpperBound(const Slice& internal_key) override {
+ return read_options_.iterate_upper_bound != nullptr &&
+ icomparator_.user_comparator()->Compare(
+ ExtractUserKey(internal_key),
+ *read_options_.iterate_upper_bound) >= 0;
+ }
+
+ private:
+ TableCache* table_cache_;
+ const ReadOptions read_options_;
+ const EnvOptions& env_options_;
+ const InternalKeyComparator& icomparator_;
+ HistogramImpl* file_read_hist_;
+ bool for_compaction_;
+ bool skip_filters_;
+ int level_;
+ RangeDelAggregator* range_del_agg_;
+};
+
+// A wrapper of version builder which references the current version in
+// constructor and unref it in the destructor.
+// Both of the constructor and destructor need to be called inside DB Mutex.
+class BaseReferencedVersionBuilder {
+ public:
+ explicit BaseReferencedVersionBuilder(ColumnFamilyData* cfd)
+ : version_builder_(new VersionBuilder(
+ cfd->current()->version_set()->env_options(), cfd->table_cache(),
+ cfd->current()->storage_info(), cfd->ioptions()->info_log)),
+ version_(cfd->current()) {
+ version_->Ref();
+ }
+ ~BaseReferencedVersionBuilder() {
+ delete version_builder_;
+ version_->Unref();
+ }
+ VersionBuilder* version_builder() { return version_builder_; }
+
+ private:
+ VersionBuilder* version_builder_;
+ Version* version_;
+};
+} // anonymous namespace
+
+Status Version::GetTableProperties(std::shared_ptr<const TableProperties>* tp,
+ const FileMetaData* file_meta,
+ const std::string* fname) const {
+ auto table_cache = cfd_->table_cache();
+ auto ioptions = cfd_->ioptions();
+ Status s = table_cache->GetTableProperties(
+ vset_->env_options_, cfd_->internal_comparator(), file_meta->fd,
+ tp, true /* no io */);
+ if (s.ok()) {
+ return s;
+ }
+
+ // We only ignore error type `Incomplete` since it's by design that we
+ // disallow table when it's not in table cache.
+ if (!s.IsIncomplete()) {
+ return s;
+ }
+
+ // 2. Table is not present in table cache, we'll read the table properties
+ // directly from the properties block in the file.
+ std::unique_ptr<RandomAccessFile> file;
+ std::string file_name;
+ if (fname != nullptr) {
+ file_name = *fname;
+ } else {
+ file_name =
+ TableFileName(vset_->db_options_->db_paths, file_meta->fd.GetNumber(),
+ file_meta->fd.GetPathId());
+ }
+ s = ioptions->env->NewRandomAccessFile(file_name, &file, vset_->env_options_);
+ if (!s.ok()) {
+ return s;
+ }
+
+ TableProperties* raw_table_properties;
+ // By setting the magic number to kInvalidTableMagicNumber, we can by
+ // pass the magic number check in the footer.
+ std::unique_ptr<RandomAccessFileReader> file_reader(
+ new RandomAccessFileReader(std::move(file), file_name));
+ s = ReadTableProperties(
+ file_reader.get(), file_meta->fd.GetFileSize(),
+ Footer::kInvalidTableMagicNumber /* table's magic number */, *ioptions, &raw_table_properties);
+ if (!s.ok()) {
+ return s;
+ }
+ RecordTick(ioptions->statistics, NUMBER_DIRECT_LOAD_TABLE_PROPERTIES);
+
+ *tp = std::shared_ptr<const TableProperties>(raw_table_properties);
+ return s;
+}
+
+Status Version::GetPropertiesOfAllTables(TablePropertiesCollection* props) {
+ Status s;
+ for (int level = 0; level < storage_info_.num_levels_; level++) {
+ s = GetPropertiesOfAllTables(props, level);
+ if (!s.ok()) {
+ return s;
+ }
+ }
+
+ return Status::OK();
+}
+
+Status Version::GetPropertiesOfAllTables(TablePropertiesCollection* props,
+ int level) {
+ for (const auto& file_meta : storage_info_.files_[level]) {
+ auto fname =
+ TableFileName(vset_->db_options_->db_paths, file_meta->fd.GetNumber(),
+ file_meta->fd.GetPathId());
+ // 1. If the table is already present in table cache, load table
+ // properties from there.
+ std::shared_ptr<const TableProperties> table_properties;
+ Status s = GetTableProperties(&table_properties, file_meta, &fname);
+ if (s.ok()) {
+ props->insert({fname, table_properties});
+ } else {
+ return s;
+ }
+ }
+
+ return Status::OK();
+}
+
+Status Version::GetPropertiesOfTablesInRange(
+ const Range* range, std::size_t n, TablePropertiesCollection* props) const {
+ for (int level = 0; level < storage_info_.num_non_empty_levels(); level++) {
+ for (decltype(n) i = 0; i < n; i++) {
+ // Convert user_key into a corresponding internal key.
+ InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek);
+ InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek);
+ std::vector<FileMetaData*> files;
+ storage_info_.GetOverlappingInputs(level, &k1, &k2, &files, -1, nullptr,
+ false);
+ for (const auto& file_meta : files) {
+ auto fname =
+ TableFileName(vset_->db_options_->db_paths,
+ file_meta->fd.GetNumber(), file_meta->fd.GetPathId());
+ if (props->count(fname) == 0) {
+ // 1. If the table is already present in table cache, load table
+ // properties from there.
+ std::shared_ptr<const TableProperties> table_properties;
+ Status s = GetTableProperties(&table_properties, file_meta, &fname);
+ if (s.ok()) {
+ props->insert({fname, table_properties});
+ } else {
+ return s;
+ }
+ }
+ }
+ }
+ }
+
+ return Status::OK();
+}
+
+Status Version::GetAggregatedTableProperties(
+ std::shared_ptr<const TableProperties>* tp, int level) {
+ TablePropertiesCollection props;
+ Status s;
+ if (level < 0) {
+ s = GetPropertiesOfAllTables(&props);
+ } else {
+ s = GetPropertiesOfAllTables(&props, level);
+ }
+ if (!s.ok()) {
+ return s;
+ }
+
+ auto* new_tp = new TableProperties();
+ for (const auto& item : props) {
+ new_tp->Add(*item.second);
+ }
+ tp->reset(new_tp);
+ return Status::OK();
+}
+
+size_t Version::GetMemoryUsageByTableReaders() {
+ size_t total_usage = 0;
+ for (auto& file_level : storage_info_.level_files_brief_) {
+ for (size_t i = 0; i < file_level.num_files; i++) {
+ total_usage += cfd_->table_cache()->GetMemoryUsageByTableReader(
+ vset_->env_options_, cfd_->internal_comparator(),
+ file_level.files[i].fd);
+ }
+ }
+ return total_usage;
+}
+
+void Version::GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta) {
+ assert(cf_meta);
+ assert(cfd_);
+
+ cf_meta->name = cfd_->GetName();
+ cf_meta->size = 0;
+ cf_meta->file_count = 0;
+ cf_meta->levels.clear();
+
+ auto* ioptions = cfd_->ioptions();
+ auto* vstorage = storage_info();
+
+ for (int level = 0; level < cfd_->NumberLevels(); level++) {
+ uint64_t level_size = 0;
+ cf_meta->file_count += vstorage->LevelFiles(level).size();
+ std::vector<SstFileMetaData> files;
+ for (const auto& file : vstorage->LevelFiles(level)) {
+ uint32_t path_id = file->fd.GetPathId();
+ std::string file_path;
+ if (path_id < ioptions->db_paths.size()) {
+ file_path = ioptions->db_paths[path_id].path;
+ } else {
+ assert(!ioptions->db_paths.empty());
+ file_path = ioptions->db_paths.back().path;
+ }
+ files.emplace_back(
+ MakeTableFileName("", file->fd.GetNumber()), file_path,
+ file->fd.GetFileSize(), file->smallest_seqno, file->largest_seqno,
+ file->smallest.user_key().ToString(),
+ file->largest.user_key().ToString(),
+ file->stats.num_reads_sampled.load(std::memory_order_relaxed),
+ file->being_compacted);
+ level_size += file->fd.GetFileSize();
+ }
+ cf_meta->levels.emplace_back(
+ level, level_size, std::move(files));
+ cf_meta->size += level_size;
+ }
+}
+
+
+uint64_t VersionStorageInfo::GetEstimatedActiveKeys() const {
+ // Estimation will be inaccurate when:
+ // (1) there exist merge keys
+ // (2) keys are directly overwritten
+ // (3) deletion on non-existing keys
+ // (4) low number of samples
+ if (current_num_samples_ == 0) {
+ return 0;
+ }
+
+ if (current_num_non_deletions_ <= current_num_deletions_) {
+ return 0;
+ }
+
+ uint64_t est = current_num_non_deletions_ - current_num_deletions_;
+
+ uint64_t file_count = 0;
+ for (int level = 0; level < num_levels_; ++level) {
+ file_count += files_[level].size();
+ }
+
+ if (current_num_samples_ < file_count) {
+ // casting to avoid overflowing
+ return
+ static_cast<uint64_t>(
+ (est * static_cast<double>(file_count) / current_num_samples_)
+ );
+ } else {
+ return est;
+ }
+}
+
+double VersionStorageInfo::GetEstimatedCompressionRatioAtLevel(
+ int level) const {
+ assert(level < num_levels_);
+ uint64_t sum_file_size_bytes = 0;
+ uint64_t sum_data_size_bytes = 0;
+ for (auto* file_meta : files_[level]) {
+ sum_file_size_bytes += file_meta->fd.GetFileSize();
+ sum_data_size_bytes += file_meta->raw_key_size + file_meta->raw_value_size;
+ }
+ if (sum_file_size_bytes == 0) {
+ return -1.0;
+ }
+ return static_cast<double>(sum_data_size_bytes) / sum_file_size_bytes;
+}
+
+void Version::AddIterators(const ReadOptions& read_options,
+ const EnvOptions& soptions,
+ MergeIteratorBuilder* merge_iter_builder,
+ RangeDelAggregator* range_del_agg) {
+ assert(storage_info_.finalized_);
+
+ for (int level = 0; level < storage_info_.num_non_empty_levels(); level++) {
+ AddIteratorsForLevel(read_options, soptions, merge_iter_builder, level,
+ range_del_agg);
+ }
+}
+
+void Version::AddIteratorsForLevel(const ReadOptions& read_options,
+ const EnvOptions& soptions,
+ MergeIteratorBuilder* merge_iter_builder,
+ int level,
+ RangeDelAggregator* range_del_agg) {
+ assert(storage_info_.finalized_);
+ if (level >= storage_info_.num_non_empty_levels()) {
+ // This is an empty level
+ return;
+ } else if (storage_info_.LevelFilesBrief(level).num_files == 0) {
+ // No files in this level
+ return;
+ }
+
+ bool should_sample = should_sample_file_read();
+
+ auto* arena = merge_iter_builder->GetArena();
+ if (level == 0) {
+ // Merge all level zero files together since they may overlap
+ for (size_t i = 0; i < storage_info_.LevelFilesBrief(0).num_files; i++) {
+ const auto& file = storage_info_.LevelFilesBrief(0).files[i];
+ merge_iter_builder->AddIterator(cfd_->table_cache()->NewIterator(
+ read_options, soptions, cfd_->internal_comparator(), file.fd,
+ range_del_agg, nullptr, cfd_->internal_stats()->GetFileReadHist(0),
+ false, arena, false /* skip_filters */, 0 /* level */));
+ }
+ if (should_sample) {
+ // Count ones for every L0 files. This is done per iterator creation
+ // rather than Seek(), while files in other levels are recored per seek.
+ // If users execute one range query per iterator, there may be some
+ // discrepancy here.
+ for (FileMetaData* meta : storage_info_.LevelFiles(0)) {
+ sample_file_read_inc(meta);
+ }
+ }
+ } else {
+ // For levels > 0, we can use a concatenating iterator that sequentially
+ // walks through the non-overlapping files in the level, opening them
+ // lazily.
+ auto* mem = arena->AllocateAligned(sizeof(LevelFileIteratorState));
+ auto* state = new (mem)
+ LevelFileIteratorState(cfd_->table_cache(), read_options, soptions,
+ cfd_->internal_comparator(),
+ cfd_->internal_stats()->GetFileReadHist(level),
+ false /* for_compaction */,
+ cfd_->ioptions()->prefix_extractor != nullptr,
+ IsFilterSkipped(level), level, range_del_agg);
+ mem = arena->AllocateAligned(sizeof(LevelFileNumIterator));
+ auto* first_level_iter = new (mem) LevelFileNumIterator(
+ cfd_->internal_comparator(), &storage_info_.LevelFilesBrief(level),
+ should_sample_file_read());
+ merge_iter_builder->AddIterator(
+ NewTwoLevelIterator(state, first_level_iter, arena, false));
+ }
+}
+
+void Version::AddRangeDelIteratorsForLevel(
+ const ReadOptions& read_options, const EnvOptions& soptions, int level,
+ std::vector<InternalIterator*>* range_del_iters) {
+ range_del_iters->clear();
+ for (size_t i = 0; i < storage_info_.LevelFilesBrief(level).num_files; i++) {
+ const auto& file = storage_info_.LevelFilesBrief(level).files[i];
+ auto* range_del_iter = cfd_->table_cache()->NewRangeTombstoneIterator(
+ read_options, soptions, cfd_->internal_comparator(), file.fd,
+ cfd_->internal_stats()->GetFileReadHist(level),
+ false /* skip_filters */, level);
+ if (range_del_iter != nullptr) {
+ range_del_iters->push_back(range_del_iter);
+ }
+ }
+}
+
+VersionStorageInfo::VersionStorageInfo(
+ const InternalKeyComparator* internal_comparator,
+ const Comparator* user_comparator, int levels,
+ CompactionStyle compaction_style, VersionStorageInfo* ref_vstorage,
+ bool _force_consistency_checks)
+ : internal_comparator_(internal_comparator),
+ user_comparator_(user_comparator),
+ // cfd is nullptr if Version is dummy
+ num_levels_(levels),
+ num_non_empty_levels_(0),
+ file_indexer_(user_comparator),
+ compaction_style_(compaction_style),
+ files_(new std::vector<FileMetaData*>[num_levels_]),
+ base_level_(num_levels_ == 1 ? -1 : 1),
+ files_by_compaction_pri_(num_levels_),
+ level0_non_overlapping_(false),
+ next_file_to_compact_by_size_(num_levels_),
+ compaction_score_(num_levels_),
+ compaction_level_(num_levels_),
+ l0_delay_trigger_count_(0),
+ accumulated_file_size_(0),
+ accumulated_raw_key_size_(0),
+ accumulated_raw_value_size_(0),
+ accumulated_num_non_deletions_(0),
+ accumulated_num_deletions_(0),
+ current_num_non_deletions_(0),
+ current_num_deletions_(0),
+ current_num_samples_(0),
+ estimated_compaction_needed_bytes_(0),
+ finalized_(false),
+ force_consistency_checks_(_force_consistency_checks) {
+ if (ref_vstorage != nullptr) {
+ accumulated_file_size_ = ref_vstorage->accumulated_file_size_;
+ accumulated_raw_key_size_ = ref_vstorage->accumulated_raw_key_size_;
+ accumulated_raw_value_size_ = ref_vstorage->accumulated_raw_value_size_;
+ accumulated_num_non_deletions_ =
+ ref_vstorage->accumulated_num_non_deletions_;
+ accumulated_num_deletions_ = ref_vstorage->accumulated_num_deletions_;
+ current_num_non_deletions_ = ref_vstorage->current_num_non_deletions_;
+ current_num_deletions_ = ref_vstorage->current_num_deletions_;
+ current_num_samples_ = ref_vstorage->current_num_samples_;
+ }
+}
+
+Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset,
+ uint64_t version_number)
+ : env_(vset->env_),
+ cfd_(column_family_data),
+ info_log_((cfd_ == nullptr) ? nullptr : cfd_->ioptions()->info_log),
+ db_statistics_((cfd_ == nullptr) ? nullptr
+ : cfd_->ioptions()->statistics),
+ table_cache_((cfd_ == nullptr) ? nullptr : cfd_->table_cache()),
+ merge_operator_((cfd_ == nullptr) ? nullptr
+ : cfd_->ioptions()->merge_operator),
+ storage_info_(
+ (cfd_ == nullptr) ? nullptr : &cfd_->internal_comparator(),
+ (cfd_ == nullptr) ? nullptr : cfd_->user_comparator(),
+ cfd_ == nullptr ? 0 : cfd_->NumberLevels(),
+ cfd_ == nullptr ? kCompactionStyleLevel
+ : cfd_->ioptions()->compaction_style,
+ (cfd_ == nullptr || cfd_->current() == nullptr)
+ ? nullptr
+ : cfd_->current()->storage_info(),
+ cfd_ == nullptr ? false : cfd_->ioptions()->force_consistency_checks),
+ vset_(vset),
+ next_(this),
+ prev_(this),
+ refs_(0),
+ version_number_(version_number) {}
+
+void Version::Get(const ReadOptions& read_options, const LookupKey& k,
+ PinnableSlice* value, Status* status,
+ MergeContext* merge_context,
+ RangeDelAggregator* range_del_agg, bool* value_found,
+ bool* key_exists, SequenceNumber* seq) {
+ Slice ikey = k.internal_key();
+ Slice user_key = k.user_key();
+
+ assert(status->ok() || status->IsMergeInProgress());
+
+ if (key_exists != nullptr) {
+ // will falsify below if not found
+ *key_exists = true;
+ }
+
+ PinnedIteratorsManager pinned_iters_mgr;
+ GetContext get_context(
+ user_comparator(), merge_operator_, info_log_, db_statistics_,
+ status->ok() ? GetContext::kNotFound : GetContext::kMerge, user_key,
+ value, value_found, merge_context, range_del_agg, this->env_, seq,
+ merge_operator_ ? &pinned_iters_mgr : nullptr);
+
+ // Pin blocks that we read to hold merge operands
+ if (merge_operator_) {
+ pinned_iters_mgr.StartPinning();
+ }
+
+ FilePicker fp(
+ storage_info_.files_, user_key, ikey, &storage_info_.level_files_brief_,
+ storage_info_.num_non_empty_levels_, &storage_info_.file_indexer_,
+ user_comparator(), internal_comparator());
+ FdWithKeyRange* f = fp.GetNextFile();
+ while (f != nullptr) {
+ if (get_context.sample()) {
+ sample_file_read_inc(f->file_metadata);
+ }
+ *status = table_cache_->Get(
+ read_options, *internal_comparator(), f->fd, ikey, &get_context,
+ cfd_->internal_stats()->GetFileReadHist(fp.GetHitFileLevel()),
+ IsFilterSkipped(static_cast<int>(fp.GetHitFileLevel()),
+ fp.IsHitFileLastInLevel()),
+ fp.GetCurrentLevel());
+ // TODO: examine the behavior for corrupted key
+ if (!status->ok()) {
+ return;
+ }
+
+ switch (get_context.State()) {
+ case GetContext::kNotFound:
+ // Keep searching in other files
+ break;
+ case GetContext::kFound:
+ if (fp.GetHitFileLevel() == 0) {
+ RecordTick(db_statistics_, GET_HIT_L0);
+ } else if (fp.GetHitFileLevel() == 1) {
+ RecordTick(db_statistics_, GET_HIT_L1);
+ } else if (fp.GetHitFileLevel() >= 2) {
+ RecordTick(db_statistics_, GET_HIT_L2_AND_UP);
+ }
+ return;
+ case GetContext::kDeleted:
+ // Use empty error message for speed
+ *status = Status::NotFound();
+ return;
+ case GetContext::kCorrupt:
+ *status = Status::Corruption("corrupted key for ", user_key);
+ return;
+ case GetContext::kMerge:
+ break;
+ }
+ f = fp.GetNextFile();
+ }
+
+ if (GetContext::kMerge == get_context.State()) {
+ if (!merge_operator_) {
+ *status = Status::InvalidArgument(
+ "merge_operator is not properly initialized.");
+ return;
+ }
+ // merge_operands are in saver and we hit the beginning of the key history
+ // do a final merge of nullptr and operands;
+ std::string* str_value = value != nullptr ? value->GetSelf() : nullptr;
+ *status = MergeHelper::TimedFullMerge(
+ merge_operator_, user_key, nullptr, merge_context->GetOperands(),
+ str_value, info_log_, db_statistics_, env_,
+ nullptr /* result_operand */, true);
+ if (LIKELY(value != nullptr)) {
+ value->PinSelf();
+ }
+ } else {
+ if (key_exists != nullptr) {
+ *key_exists = false;
+ }
+ *status = Status::NotFound(); // Use an empty error message for speed
+ }
+}
+
+bool Version::IsFilterSkipped(int level, bool is_file_last_in_level) {
+ // Reaching the bottom level implies misses at all upper levels, so we'll
+ // skip checking the filters when we predict a hit.
+ return cfd_->ioptions()->optimize_filters_for_hits &&
+ (level > 0 || is_file_last_in_level) &&
+ level == storage_info_.num_non_empty_levels() - 1;
+}
+
+void VersionStorageInfo::GenerateLevelFilesBrief() {
+ level_files_brief_.resize(num_non_empty_levels_);
+ for (int level = 0; level < num_non_empty_levels_; level++) {
+ DoGenerateLevelFilesBrief(
+ &level_files_brief_[level], files_[level], &arena_);
+ }
+}
+
+void Version::PrepareApply(
+ const MutableCFOptions& mutable_cf_options,
+ bool update_stats) {
+ UpdateAccumulatedStats(update_stats);
+ storage_info_.UpdateNumNonEmptyLevels();
+ storage_info_.CalculateBaseBytes(*cfd_->ioptions(), mutable_cf_options);
+ storage_info_.UpdateFilesByCompactionPri(cfd_->ioptions()->compaction_pri);
+ storage_info_.GenerateFileIndexer();
+ storage_info_.GenerateLevelFilesBrief();
+ storage_info_.GenerateLevel0NonOverlapping();
+}
+
+bool Version::MaybeInitializeFileMetaData(FileMetaData* file_meta) {
+ if (file_meta->init_stats_from_file ||
+ file_meta->compensated_file_size > 0) {
+ return false;
+ }
+ std::shared_ptr<const TableProperties> tp;
+ Status s = GetTableProperties(&tp, file_meta);
+ file_meta->init_stats_from_file = true;
+ if (!s.ok()) {
+ ROCKS_LOG_ERROR(vset_->db_options_->info_log,
+ "Unable to load table properties for file %" PRIu64
+ " --- %s\n",
+ file_meta->fd.GetNumber(), s.ToString().c_str());
+ return false;
+ }
+ if (tp.get() == nullptr) return false;
+ file_meta->num_entries = tp->num_entries;
+ file_meta->num_deletions = GetDeletedKeys(tp->user_collected_properties);
+ file_meta->raw_value_size = tp->raw_value_size;
+ file_meta->raw_key_size = tp->raw_key_size;
+
+ return true;
+}
+
+void VersionStorageInfo::UpdateAccumulatedStats(FileMetaData* file_meta) {
+ assert(file_meta->init_stats_from_file);
+ accumulated_file_size_ += file_meta->fd.GetFileSize();
+ accumulated_raw_key_size_ += file_meta->raw_key_size;
+ accumulated_raw_value_size_ += file_meta->raw_value_size;
+ accumulated_num_non_deletions_ +=
+ file_meta->num_entries - file_meta->num_deletions;
+ accumulated_num_deletions_ += file_meta->num_deletions;
+
+ current_num_non_deletions_ +=
+ file_meta->num_entries - file_meta->num_deletions;
+ current_num_deletions_ += file_meta->num_deletions;
+ current_num_samples_++;
+}
+
+void VersionStorageInfo::RemoveCurrentStats(FileMetaData* file_meta) {
+ if (file_meta->init_stats_from_file) {
+ current_num_non_deletions_ -=
+ file_meta->num_entries - file_meta->num_deletions;
+ current_num_deletions_ -= file_meta->num_deletions;
+ current_num_samples_--;
+ }
+}
+
+void Version::UpdateAccumulatedStats(bool update_stats) {
+ if (update_stats) {
+ // maximum number of table properties loaded from files.
+ const int kMaxInitCount = 20;
+ int init_count = 0;
+ // here only the first kMaxInitCount files which haven't been
+ // initialized from file will be updated with num_deletions.
+ // The motivation here is to cap the maximum I/O per Version creation.
+ // The reason for choosing files from lower-level instead of higher-level
+ // is that such design is able to propagate the initialization from
+ // lower-level to higher-level: When the num_deletions of lower-level
+ // files are updated, it will make the lower-level files have accurate
+ // compensated_file_size, making lower-level to higher-level compaction
+ // will be triggered, which creates higher-level files whose num_deletions
+ // will be updated here.
+ for (int level = 0;
+ level < storage_info_.num_levels_ && init_count < kMaxInitCount;
+ ++level) {
+ for (auto* file_meta : storage_info_.files_[level]) {
+ if (MaybeInitializeFileMetaData(file_meta)) {
+ // each FileMeta will be initialized only once.
+ storage_info_.UpdateAccumulatedStats(file_meta);
+ // when option "max_open_files" is -1, all the file metadata has
+ // already been read, so MaybeInitializeFileMetaData() won't incur
+ // any I/O cost. "max_open_files=-1" means that the table cache passed
+ // to the VersionSet and then to the ColumnFamilySet has a size of
+ // TableCache::kInfiniteCapacity
+ if (vset_->GetColumnFamilySet()->get_table_cache()->GetCapacity() ==
+ TableCache::kInfiniteCapacity) {
+ continue;
+ }
+ if (++init_count >= kMaxInitCount) {
+ break;
+ }
+ }
+ }
+ }
+ // In case all sampled-files contain only deletion entries, then we
+ // load the table-property of a file in higher-level to initialize
+ // that value.
+ for (int level = storage_info_.num_levels_ - 1;
+ storage_info_.accumulated_raw_value_size_ == 0 && level >= 0;
+ --level) {
+ for (int i = static_cast<int>(storage_info_.files_[level].size()) - 1;
+ storage_info_.accumulated_raw_value_size_ == 0 && i >= 0; --i) {
+ if (MaybeInitializeFileMetaData(storage_info_.files_[level][i])) {
+ storage_info_.UpdateAccumulatedStats(storage_info_.files_[level][i]);
+ }
+ }
+ }
+ }
+
+ storage_info_.ComputeCompensatedSizes();
+}
+
+void VersionStorageInfo::ComputeCompensatedSizes() {
+ static const int kDeletionWeightOnCompaction = 2;
+ uint64_t average_value_size = GetAverageValueSize();
+
+ // compute the compensated size
+ for (int level = 0; level < num_levels_; level++) {
+ for (auto* file_meta : files_[level]) {
+ // Here we only compute compensated_file_size for those file_meta
+ // which compensated_file_size is uninitialized (== 0). This is true only
+ // for files that have been created right now and no other thread has
+ // access to them. That's why we can safely mutate compensated_file_size.
+ if (file_meta->compensated_file_size == 0) {
+ file_meta->compensated_file_size = file_meta->fd.GetFileSize();
+ // Here we only boost the size of deletion entries of a file only
+ // when the number of deletion entries is greater than the number of
+ // non-deletion entries in the file. The motivation here is that in
+ // a stable workload, the number of deletion entries should be roughly
+ // equal to the number of non-deletion entries. If we compensate the
+ // size of deletion entries in a stable workload, the deletion
+ // compensation logic might introduce unwanted effet which changes the
+ // shape of LSM tree.
+ if (file_meta->num_deletions * 2 >= file_meta->num_entries) {
+ file_meta->compensated_file_size +=
+ (file_meta->num_deletions * 2 - file_meta->num_entries) *
+ average_value_size * kDeletionWeightOnCompaction;
+ }
+ }
+ }
+ }
+}
+
+int VersionStorageInfo::MaxInputLevel() const {
+ if (compaction_style_ == kCompactionStyleLevel) {
+ return num_levels() - 2;
+ }
+ return 0;
+}
+
+int VersionStorageInfo::MaxOutputLevel(bool allow_ingest_behind) const {
+ if (allow_ingest_behind) {
+ assert(num_levels() > 1);
+ return num_levels() - 2;
+ }
+ return num_levels() - 1;
+}
+
+void VersionStorageInfo::EstimateCompactionBytesNeeded(
+ const MutableCFOptions& mutable_cf_options) {
+ // Only implemented for level-based compaction
+ if (compaction_style_ != kCompactionStyleLevel) {
+ estimated_compaction_needed_bytes_ = 0;
+ return;
+ }
+
+ // Start from Level 0, if level 0 qualifies compaction to level 1,
+ // we estimate the size of compaction.
+ // Then we move on to the next level and see whether it qualifies compaction
+ // to the next level. The size of the level is estimated as the actual size
+ // on the level plus the input bytes from the previous level if there is any.
+ // If it exceeds, take the exceeded bytes as compaction input and add the size
+ // of the compaction size to tatal size.
+ // We keep doing it to Level 2, 3, etc, until the last level and return the
+ // accumulated bytes.
+
+ uint64_t bytes_compact_to_next_level = 0;
+ uint64_t level_size = 0;
+ for (auto* f : files_[0]) {
+ level_size += f->fd.GetFileSize();
+ }
+ // Level 0
+ bool level0_compact_triggered = false;
+ if (static_cast<int>(files_[0].size()) >=
+ mutable_cf_options.level0_file_num_compaction_trigger ||
+ level_size >= mutable_cf_options.max_bytes_for_level_base) {
+ level0_compact_triggered = true;
+ estimated_compaction_needed_bytes_ = level_size;
+ bytes_compact_to_next_level = level_size;
+ } else {
+ estimated_compaction_needed_bytes_ = 0;
+ }
+
+ // Level 1 and up.
+ uint64_t bytes_next_level = 0;
+ for (int level = base_level(); level <= MaxInputLevel(); level++) {
+ level_size = 0;
+ if (bytes_next_level > 0) {
+#ifndef NDEBUG
+ uint64_t level_size2 = 0;
+ for (auto* f : files_[level]) {
+ level_size2 += f->fd.GetFileSize();
+ }
+ assert(level_size2 == bytes_next_level);
+#endif
+ level_size = bytes_next_level;
+ bytes_next_level = 0;
+ } else {
+ for (auto* f : files_[level]) {
+ level_size += f->fd.GetFileSize();
+ }
+ }
+ if (level == base_level() && level0_compact_triggered) {
+ // Add base level size to compaction if level0 compaction triggered.
+ estimated_compaction_needed_bytes_ += level_size;
+ }
+ // Add size added by previous compaction
+ level_size += bytes_compact_to_next_level;
+ bytes_compact_to_next_level = 0;
+ uint64_t level_target = MaxBytesForLevel(level);
+ if (level_size > level_target) {
+ bytes_compact_to_next_level = level_size - level_target;
+ // Estimate the actual compaction fan-out ratio as size ratio between
+ // the two levels.
+
+ assert(bytes_next_level == 0);
+ if (level + 1 < num_levels_) {
+ for (auto* f : files_[level + 1]) {
+ bytes_next_level += f->fd.GetFileSize();
+ }
+ }
+ if (bytes_next_level > 0) {
+ assert(level_size > 0);
+ estimated_compaction_needed_bytes_ += static_cast<uint64_t>(
+ static_cast<double>(bytes_compact_to_next_level) *
+ (static_cast<double>(bytes_next_level) /
+ static_cast<double>(level_size) +
+ 1));
+ }
+ }
+ }
+}
+
+namespace {
+uint32_t GetExpiredTtlFilesCount(const ImmutableCFOptions& ioptions,
+ const std::vector<FileMetaData*>& files) {
+ uint32_t ttl_expired_files_count = 0;
+
+ int64_t _current_time;
+ auto status = ioptions.env->GetCurrentTime(&_current_time);
+ if (status.ok()) {
+ const uint64_t current_time = static_cast<uint64_t>(_current_time);
+ for (auto f : files) {
+ if (!f->being_compacted && f->fd.table_reader != nullptr &&
+ f->fd.table_reader->GetTableProperties() != nullptr) {
+ auto creation_time =
+ f->fd.table_reader->GetTableProperties()->creation_time;
+ if (creation_time > 0 &&
+ creation_time <
+ (current_time - ioptions.compaction_options_fifo.ttl)) {
+ ttl_expired_files_count++;
+ }
+ }
+ }
+ }
+ return ttl_expired_files_count;
+}
+} // anonymous namespace
+
+void VersionStorageInfo::ComputeCompactionScore(
+ const ImmutableCFOptions& immutable_cf_options,
+ const MutableCFOptions& mutable_cf_options) {
+ for (int level = 0; level <= MaxInputLevel(); level++) {
+ double score;
+ if (level == 0) {
+ // We treat level-0 specially by bounding the number of files
+ // instead of number of bytes for two reasons:
+ //
+ // (1) With larger write-buffer sizes, it is nice not to do too
+ // many level-0 compactions.
+ //
+ // (2) The files in level-0 are merged on every read and
+ // therefore we wish to avoid too many files when the individual
+ // file size is small (perhaps because of a small write-buffer
+ // setting, or very high compression ratios, or lots of
+ // overwrites/deletions).
+ int num_sorted_runs = 0;
+ uint64_t total_size = 0;
+ for (auto* f : files_[level]) {
+ if (!f->being_compacted) {
+ total_size += f->compensated_file_size;
+ num_sorted_runs++;
+ }
+ }
+ if (compaction_style_ == kCompactionStyleUniversal) {
+ // For universal compaction, we use level0 score to indicate
+ // compaction score for the whole DB. Adding other levels as if
+ // they are L0 files.
+ for (int i = 1; i < num_levels(); i++) {
+ if (!files_[i].empty() && !files_[i][0]->being_compacted) {
+ num_sorted_runs++;
+ }
+ }
+ }
+
+ if (compaction_style_ == kCompactionStyleFIFO) {
+ score =
+ static_cast<double>(total_size) /
+ immutable_cf_options.compaction_options_fifo.max_table_files_size;
+ if (immutable_cf_options.compaction_options_fifo.allow_compaction) {
+ score = std::max(
+ static_cast<double>(num_sorted_runs) /
+ mutable_cf_options.level0_file_num_compaction_trigger,
+ score);
+ }
+ if (immutable_cf_options.compaction_options_fifo.ttl > 0) {
+ score = std::max(static_cast<double>(GetExpiredTtlFilesCount(
+ immutable_cf_options, files_[level])),
+ score);
+ }
+
+ } else {
+ score = static_cast<double>(num_sorted_runs) /
+ mutable_cf_options.level0_file_num_compaction_trigger;
+ if (compaction_style_ == kCompactionStyleLevel && num_levels() > 1) {
+ // Level-based involves L0->L0 compactions that can lead to oversized
+ // L0 files. Take into account size as well to avoid later giant
+ // compactions to the base level.
+ score = std::max(
+ score, static_cast<double>(total_size) /
+ mutable_cf_options.max_bytes_for_level_base);
+ }
+ }
+ } else {
+ // Compute the ratio of current size to size limit.
+ uint64_t level_bytes_no_compacting = 0;
+ for (auto f : files_[level]) {
+ if (!f->being_compacted) {
+ level_bytes_no_compacting += f->compensated_file_size;
+ }
+ }
+ score = static_cast<double>(level_bytes_no_compacting) /
+ MaxBytesForLevel(level);
+ }
+ compaction_level_[level] = level;
+ compaction_score_[level] = score;
+ }
+
+ // sort all the levels based on their score. Higher scores get listed
+ // first. Use bubble sort because the number of entries are small.
+ for (int i = 0; i < num_levels() - 2; i++) {
+ for (int j = i + 1; j < num_levels() - 1; j++) {
+ if (compaction_score_[i] < compaction_score_[j]) {
+ double score = compaction_score_[i];
+ int level = compaction_level_[i];
+ compaction_score_[i] = compaction_score_[j];
+ compaction_level_[i] = compaction_level_[j];
+ compaction_score_[j] = score;
+ compaction_level_[j] = level;
+ }
+ }
+ }
+ ComputeFilesMarkedForCompaction();
+ EstimateCompactionBytesNeeded(mutable_cf_options);
+}
+
+void VersionStorageInfo::ComputeFilesMarkedForCompaction() {
+ files_marked_for_compaction_.clear();
+ int last_qualify_level = 0;
+
+ // Do not include files from the last level with data
+ // If table properties collector suggests a file on the last level,
+ // we should not move it to a new level.
+ for (int level = num_levels() - 1; level >= 1; level--) {
+ if (!files_[level].empty()) {
+ last_qualify_level = level - 1;
+ break;
+ }
+ }
+
+ for (int level = 0; level <= last_qualify_level; level++) {
+ for (auto* f : files_[level]) {
+ if (!f->being_compacted && f->marked_for_compaction) {
+ files_marked_for_compaction_.emplace_back(level, f);
+ }
+ }
+ }
+}
+
+namespace {
+
+// used to sort files by size
+struct Fsize {
+ size_t index;
+ FileMetaData* file;
+};
+
+// Compator that is used to sort files based on their size
+// In normal mode: descending size
+bool CompareCompensatedSizeDescending(const Fsize& first, const Fsize& second) {
+ return (first.file->compensated_file_size >
+ second.file->compensated_file_size);
+}
+} // anonymous namespace
+
+void VersionStorageInfo::AddFile(int level, FileMetaData* f, Logger* info_log) {
+ auto* level_files = &files_[level];
+ // Must not overlap
+#ifndef NDEBUG
+ if (level > 0 && !level_files->empty() &&
+ internal_comparator_->Compare(
+ (*level_files)[level_files->size() - 1]->largest, f->smallest) >= 0) {
+ auto* f2 = (*level_files)[level_files->size() - 1];
+ if (info_log != nullptr) {
+ Error(info_log, "Adding new file %" PRIu64
+ " range (%s, %s) to level %d but overlapping "
+ "with existing file %" PRIu64 " %s %s",
+ f->fd.GetNumber(), f->smallest.DebugString(true).c_str(),
+ f->largest.DebugString(true).c_str(), level, f2->fd.GetNumber(),
+ f2->smallest.DebugString(true).c_str(),
+ f2->largest.DebugString(true).c_str());
+ LogFlush(info_log);
+ }
+ assert(false);
+ }
+#endif
+ f->refs++;
+ level_files->push_back(f);
+}
+
+// Version::PrepareApply() need to be called before calling the function, or
+// following functions called:
+// 1. UpdateNumNonEmptyLevels();
+// 2. CalculateBaseBytes();
+// 3. UpdateFilesByCompactionPri();
+// 4. GenerateFileIndexer();
+// 5. GenerateLevelFilesBrief();
+// 6. GenerateLevel0NonOverlapping();
+void VersionStorageInfo::SetFinalized() {
+ finalized_ = true;
+#ifndef NDEBUG
+ if (compaction_style_ != kCompactionStyleLevel) {
+ // Not level based compaction.
+ return;
+ }
+ assert(base_level_ < 0 || num_levels() == 1 ||
+ (base_level_ >= 1 && base_level_ < num_levels()));
+ // Verify all levels newer than base_level are empty except L0
+ for (int level = 1; level < base_level(); level++) {
+ assert(NumLevelBytes(level) == 0);
+ }
+ uint64_t max_bytes_prev_level = 0;
+ for (int level = base_level(); level < num_levels() - 1; level++) {
+ if (LevelFiles(level).size() == 0) {
+ continue;
+ }
+ assert(MaxBytesForLevel(level) >= max_bytes_prev_level);
+ max_bytes_prev_level = MaxBytesForLevel(level);
+ }
+ int num_empty_non_l0_level = 0;
+ for (int level = 0; level < num_levels(); level++) {
+ assert(LevelFiles(level).size() == 0 ||
+ LevelFiles(level).size() == LevelFilesBrief(level).num_files);
+ if (level > 0 && NumLevelBytes(level) > 0) {
+ num_empty_non_l0_level++;
+ }
+ if (LevelFiles(level).size() > 0) {
+ assert(level < num_non_empty_levels());
+ }
+ }
+ assert(compaction_level_.size() > 0);
+ assert(compaction_level_.size() == compaction_score_.size());
+#endif
+}
+
+void VersionStorageInfo::UpdateNumNonEmptyLevels() {
+ num_non_empty_levels_ = num_levels_;
+ for (int i = num_levels_ - 1; i >= 0; i--) {
+ if (files_[i].size() != 0) {
+ return;
+ } else {
+ num_non_empty_levels_ = i;
+ }
+ }
+}
+
+namespace {
+// Sort `temp` based on ratio of overlapping size over file size
+void SortFileByOverlappingRatio(
+ const InternalKeyComparator& icmp, const std::vector<FileMetaData*>& files,
+ const std::vector<FileMetaData*>& next_level_files,
+ std::vector<Fsize>* temp) {
+ std::unordered_map<uint64_t, uint64_t> file_to_order;
+ auto next_level_it = next_level_files.begin();
+
+ for (auto& file : files) {
+ uint64_t overlapping_bytes = 0;
+ // Skip files in next level that is smaller than current file
+ while (next_level_it != next_level_files.end() &&
+ icmp.Compare((*next_level_it)->largest, file->smallest) < 0) {
+ next_level_it++;
+ }
+
+ while (next_level_it != next_level_files.end() &&
+ icmp.Compare((*next_level_it)->smallest, file->largest) < 0) {
+ overlapping_bytes += (*next_level_it)->fd.file_size;
+
+ if (icmp.Compare((*next_level_it)->largest, file->largest) > 0) {
+ // next level file cross large boundary of current file.
+ break;
+ }
+ next_level_it++;
+ }
+
+ assert(file->fd.file_size != 0);
+ file_to_order[file->fd.GetNumber()] =
+ overlapping_bytes * 1024u / file->fd.file_size;
+ }
+
+ std::sort(temp->begin(), temp->end(),
+ [&](const Fsize& f1, const Fsize& f2) -> bool {
+ return file_to_order[f1.file->fd.GetNumber()] <
+ file_to_order[f2.file->fd.GetNumber()];
+ });
+}
+} // namespace
+
+void VersionStorageInfo::UpdateFilesByCompactionPri(
+ CompactionPri compaction_pri) {
+ if (compaction_style_ == kCompactionStyleFIFO ||
+ compaction_style_ == kCompactionStyleUniversal) {
+ // don't need this
+ return;
+ }
+ // No need to sort the highest level because it is never compacted.
+ for (int level = 0; level < num_levels() - 1; level++) {
+ const std::vector<FileMetaData*>& files = files_[level];
+ auto& files_by_compaction_pri = files_by_compaction_pri_[level];
+ assert(files_by_compaction_pri.size() == 0);
+
+ // populate a temp vector for sorting based on size
+ std::vector<Fsize> temp(files.size());
+ for (size_t i = 0; i < files.size(); i++) {
+ temp[i].index = i;
+ temp[i].file = files[i];
+ }
+
+ // sort the top number_of_files_to_sort_ based on file size
+ size_t num = VersionStorageInfo::kNumberFilesToSort;
+ if (num > temp.size()) {
+ num = temp.size();
+ }
+ switch (compaction_pri) {
+ case kByCompensatedSize:
+ std::partial_sort(temp.begin(), temp.begin() + num, temp.end(),
+ CompareCompensatedSizeDescending);
+ break;
+ case kOldestLargestSeqFirst:
+ std::sort(temp.begin(), temp.end(),
+ [](const Fsize& f1, const Fsize& f2) -> bool {
+ return f1.file->largest_seqno < f2.file->largest_seqno;
+ });
+ break;
+ case kOldestSmallestSeqFirst:
+ std::sort(temp.begin(), temp.end(),
+ [](const Fsize& f1, const Fsize& f2) -> bool {
+ return f1.file->smallest_seqno < f2.file->smallest_seqno;
+ });
+ break;
+ case kMinOverlappingRatio:
+ SortFileByOverlappingRatio(*internal_comparator_, files_[level],
+ files_[level + 1], &temp);
+ break;
+ default:
+ assert(false);
+ }
+ assert(temp.size() == files.size());
+
+ // initialize files_by_compaction_pri_
+ for (size_t i = 0; i < temp.size(); i++) {
+ files_by_compaction_pri.push_back(static_cast<int>(temp[i].index));
+ }
+ next_file_to_compact_by_size_[level] = 0;
+ assert(files_[level].size() == files_by_compaction_pri_[level].size());
+ }
+}
+
+void VersionStorageInfo::GenerateLevel0NonOverlapping() {
+ assert(!finalized_);
+ level0_non_overlapping_ = true;
+ if (level_files_brief_.size() == 0) {
+ return;
+ }
+
+ // A copy of L0 files sorted by smallest key
+ std::vector<FdWithKeyRange> level0_sorted_file(
+ level_files_brief_[0].files,
+ level_files_brief_[0].files + level_files_brief_[0].num_files);
+ std::sort(level0_sorted_file.begin(), level0_sorted_file.end(),
+ [this](const FdWithKeyRange& f1, const FdWithKeyRange& f2) -> bool {
+ return (internal_comparator_->Compare(f1.smallest_key,
+ f2.smallest_key) < 0);
+ });
+
+ for (size_t i = 1; i < level0_sorted_file.size(); ++i) {
+ FdWithKeyRange& f = level0_sorted_file[i];
+ FdWithKeyRange& prev = level0_sorted_file[i - 1];
+ if (internal_comparator_->Compare(prev.largest_key, f.smallest_key) >= 0) {
+ level0_non_overlapping_ = false;
+ break;
+ }
+ }
+}
+
+void Version::Ref() {
+ ++refs_;
+}
+
+bool Version::Unref() {
+ assert(refs_ >= 1);
+ --refs_;
+ if (refs_ == 0) {
+ delete this;
+ return true;
+ }
+ return false;
+}
+
+bool VersionStorageInfo::OverlapInLevel(int level,
+ const Slice* smallest_user_key,
+ const Slice* largest_user_key) {
+ if (level >= num_non_empty_levels_) {
+ // empty level, no overlap
+ return false;
+ }
+ return SomeFileOverlapsRange(*internal_comparator_, (level > 0),
+ level_files_brief_[level], smallest_user_key,
+ largest_user_key);
+}
+
+// Store in "*inputs" all files in "level" that overlap [begin,end]
+// If hint_index is specified, then it points to a file in the
+// overlapping range.
+// The file_index returns a pointer to any file in an overlapping range.
+void VersionStorageInfo::GetOverlappingInputs(
+ int level, const InternalKey* begin, const InternalKey* end,
+ std::vector<FileMetaData*>* inputs, int hint_index, int* file_index,
+ bool expand_range) const {
+ if (level >= num_non_empty_levels_) {
+ // this level is empty, no overlapping inputs
+ return;
+ }
+
+ inputs->clear();
+ Slice user_begin, user_end;
+ if (begin != nullptr) {
+ user_begin = begin->user_key();
+ }
+ if (end != nullptr) {
+ user_end = end->user_key();
+ }
+ if (file_index) {
+ *file_index = -1;
+ }
+ const Comparator* user_cmp = user_comparator_;
+ if (begin != nullptr && end != nullptr && level > 0) {
+ GetOverlappingInputsRangeBinarySearch(level, user_begin, user_end, inputs,
+ hint_index, file_index);
+ return;
+ }
+
+ for (size_t i = 0; i < level_files_brief_[level].num_files; ) {
+ FdWithKeyRange* f = &(level_files_brief_[level].files[i++]);
+ const Slice file_start = ExtractUserKey(f->smallest_key);
+ const Slice file_limit = ExtractUserKey(f->largest_key);
+ if (begin != nullptr && user_cmp->Compare(file_limit, user_begin) < 0) {
+ // "f" is completely before specified range; skip it
+ } else if (end != nullptr && user_cmp->Compare(file_start, user_end) > 0) {
+ // "f" is completely after specified range; skip it
+ } else {
+ inputs->push_back(files_[level][i-1]);
+ if (level == 0 && expand_range) {
+ // Level-0 files may overlap each other. So check if the newly
+ // added file has expanded the range. If so, restart search.
+ if (begin != nullptr && user_cmp->Compare(file_start, user_begin) < 0) {
+ user_begin = file_start;
+ inputs->clear();
+ i = 0;
+ } else if (end != nullptr
+ && user_cmp->Compare(file_limit, user_end) > 0) {
+ user_end = file_limit;
+ inputs->clear();
+ i = 0;
+ }
+ } else if (file_index) {
+ *file_index = static_cast<int>(i) - 1;
+ }
+ }
+ }
+}
+
+// Store in "*inputs" files in "level" that within range [begin,end]
+// Guarantee a "clean cut" boundary between the files in inputs
+// and the surrounding files and the maxinum number of files.
+// This will ensure that no parts of a key are lost during compaction.
+// If hint_index is specified, then it points to a file in the range.
+// The file_index returns a pointer to any file in an overlapping range.
+void VersionStorageInfo::GetCleanInputsWithinInterval(
+ int level, const InternalKey* begin, const InternalKey* end,
+ std::vector<FileMetaData*>* inputs, int hint_index, int* file_index) const {
+ if (level >= num_non_empty_levels_) {
+ // this level is empty, no inputs within range
+ return;
+ }
+
+ inputs->clear();
+ Slice user_begin, user_end;
+ if (begin != nullptr) {
+ user_begin = begin->user_key();
+ }
+ if (end != nullptr) {
+ user_end = end->user_key();
+ }
+ if (file_index) {
+ *file_index = -1;
+ }
+ if (begin != nullptr && end != nullptr && level > 0) {
+ GetOverlappingInputsRangeBinarySearch(level, user_begin, user_end, inputs,
+ hint_index, file_index,
+ true /* within_interval */);
+ }
+}
+
+// Store in "*inputs" all files in "level" that overlap [begin,end]
+// Employ binary search to find at least one file that overlaps the
+// specified range. From that file, iterate backwards and
+// forwards to find all overlapping files.
+// if within_range is set, then only store the maximum clean inputs
+// within range [begin, end]. "clean" means there is a boudnary
+// between the files in "*inputs" and the surrounding files
+void VersionStorageInfo::GetOverlappingInputsRangeBinarySearch(
+ int level, const Slice& user_begin, const Slice& user_end,
+ std::vector<FileMetaData*>* inputs, int hint_index, int* file_index,
+ bool within_interval) const {
+ assert(level > 0);
+ int min = 0;
+ int mid = 0;
+ int max = static_cast<int>(files_[level].size()) - 1;
+ bool foundOverlap = false;
+ const Comparator* user_cmp = user_comparator_;
+
+ // if the caller already knows the index of a file that has overlap,
+ // then we can skip the binary search.
+ if (hint_index != -1) {
+ mid = hint_index;
+ foundOverlap = true;
+ }
+
+ while (!foundOverlap && min <= max) {
+ mid = (min + max)/2;
+ FdWithKeyRange* f = &(level_files_brief_[level].files[mid]);
+ const Slice file_start = ExtractUserKey(f->smallest_key);
+ const Slice file_limit = ExtractUserKey(f->largest_key);
+ if ((!within_interval && user_cmp->Compare(file_limit, user_begin) < 0) ||
+ (within_interval && user_cmp->Compare(file_start, user_begin) < 0)) {
+ min = mid + 1;
+ } else if ((!within_interval &&
+ user_cmp->Compare(user_end, file_start) < 0) ||
+ (within_interval &&
+ user_cmp->Compare(user_end, file_limit) < 0)) {
+ max = mid - 1;
+ } else {
+ foundOverlap = true;
+ break;
+ }
+ }
+
+ // If there were no overlapping files, return immediately.
+ if (!foundOverlap) {
+ return;
+ }
+ // returns the index where an overlap is found
+ if (file_index) {
+ *file_index = mid;
+ }
+
+ int start_index, end_index;
+ if (within_interval) {
+ ExtendFileRangeWithinInterval(level, user_begin, user_end, mid, &start_index,
+ &end_index);
+ } else {
+ ExtendFileRangeOverlappingInterval(level, user_begin, user_end, mid,
+ &start_index, &end_index);
+ }
+ assert(end_index >= start_index);
+ // insert overlapping files into vector
+ for (int i = start_index; i <= end_index; i++) {
+ inputs->push_back(files_[level][i]);
+ }
+}
+
+// Store in *start_index and *end_index the range of all files in
+// "level" that overlap [begin,end]
+// The mid_index specifies the index of at least one file that
+// overlaps the specified range. From that file, iterate backward
+// and forward to find all overlapping files.
+// Use FileLevel in searching, make it faster
+void VersionStorageInfo::ExtendFileRangeOverlappingInterval(
+ int level, const Slice& user_begin, const Slice& user_end,
+ unsigned int mid_index, int* start_index, int* end_index) const {
+ const Comparator* user_cmp = user_comparator_;
+ const FdWithKeyRange* files = level_files_brief_[level].files;
+#ifndef NDEBUG
+ {
+ // assert that the file at mid_index overlaps with the range
+ assert(mid_index < level_files_brief_[level].num_files);
+ const FdWithKeyRange* f = &files[mid_index];
+ const Slice fstart = ExtractUserKey(f->smallest_key);
+ const Slice flimit = ExtractUserKey(f->largest_key);
+ if (user_cmp->Compare(fstart, user_begin) >= 0) {
+ assert(user_cmp->Compare(fstart, user_end) <= 0);
+ } else {
+ assert(user_cmp->Compare(flimit, user_begin) >= 0);
+ }
+ }
+#endif
+ *start_index = mid_index + 1;
+ *end_index = mid_index;
+ int count __attribute__((unused)) = 0;
+
+ // check backwards from 'mid' to lower indices
+ for (int i = mid_index; i >= 0 ; i--) {
+ const FdWithKeyRange* f = &files[i];
+ const Slice file_limit = ExtractUserKey(f->largest_key);
+ if (user_cmp->Compare(file_limit, user_begin) >= 0) {
+ *start_index = i;
+ assert((count++, true));
+ } else {
+ break;
+ }
+ }
+ // check forward from 'mid+1' to higher indices
+ for (unsigned int i = mid_index+1;
+ i < level_files_brief_[level].num_files; i++) {
+ const FdWithKeyRange* f = &files[i];
+ const Slice file_start = ExtractUserKey(f->smallest_key);
+ if (user_cmp->Compare(file_start, user_end) <= 0) {
+ assert((count++, true));
+ *end_index = i;
+ } else {
+ break;
+ }
+ }
+ assert(count == *end_index - *start_index + 1);
+}
+
+// Store in *start_index and *end_index the clean range of all files in
+// "level" within [begin,end]
+// The mid_index specifies the index of at least one file within
+// the specified range. From that file, iterate backward
+// and forward to find all overlapping files and then "shrink" to
+// the clean range required.
+// Use FileLevel in searching, make it faster
+void VersionStorageInfo::ExtendFileRangeWithinInterval(
+ int level, const Slice& user_begin, const Slice& user_end,
+ unsigned int mid_index, int* start_index, int* end_index) const {
+ assert(level != 0);
+ const Comparator* user_cmp = user_comparator_;
+ const FdWithKeyRange* files = level_files_brief_[level].files;
+#ifndef NDEBUG
+ {
+ // assert that the file at mid_index is within the range
+ assert(mid_index < level_files_brief_[level].num_files);
+ const FdWithKeyRange* f = &files[mid_index];
+ const Slice fstart = ExtractUserKey(f->smallest_key);
+ const Slice flimit = ExtractUserKey(f->largest_key);
+ assert(user_cmp->Compare(fstart, user_begin) >= 0 &&
+ user_cmp->Compare(flimit, user_end) <= 0);
+ }
+#endif
+ ExtendFileRangeOverlappingInterval(level, user_begin, user_end, mid_index,
+ start_index, end_index);
+ int left = *start_index;
+ int right = *end_index;
+ // shrink from left to right
+ while (left <= right) {
+ const Slice& first_key_in_range = ExtractUserKey(files[left].smallest_key);
+ if (user_cmp->Compare(first_key_in_range, user_begin) < 0) {
+ left++;
+ continue;
+ }
+ if (left > 0) { // If not first file
+ const Slice& last_key_before =
+ ExtractUserKey(files[left - 1].largest_key);
+ if (user_cmp->Equal(first_key_in_range, last_key_before)) {
+ // The first user key in range overlaps with the previous file's last
+ // key
+ left++;
+ continue;
+ }
+ }
+ break;
+ }
+ // shrink from right to left
+ while (left <= right) {
+ const Slice last_key_in_range = ExtractUserKey(files[right].largest_key);
+ if (user_cmp->Compare(last_key_in_range, user_end) > 0) {
+ right--;
+ continue;
+ }
+ if (right < static_cast<int>(level_files_brief_[level].num_files) -
+ 1) { // If not the last file
+ const Slice first_key_after =
+ ExtractUserKey(files[right + 1].smallest_key);
+ if (user_cmp->Equal(last_key_in_range, first_key_after)) {
+ // The last user key in range overlaps with the next file's first key
+ right--;
+ continue;
+ }
+ }
+ break;
+ }
+
+ *start_index = left;
+ *end_index = right;
+}
+
+uint64_t VersionStorageInfo::NumLevelBytes(int level) const {
+ assert(level >= 0);
+ assert(level < num_levels());
+ return TotalFileSize(files_[level]);
+}
+
+const char* VersionStorageInfo::LevelSummary(
+ LevelSummaryStorage* scratch) const {
+ int len = 0;
+ if (compaction_style_ == kCompactionStyleLevel && num_levels() > 1) {
+ assert(base_level_ < static_cast<int>(level_max_bytes_.size()));
+ len = snprintf(scratch->buffer, sizeof(scratch->buffer),
+ "base level %d max bytes base %" PRIu64 " ", base_level_,
+ level_max_bytes_[base_level_]);
+ }
+ len +=
+ snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, "files[");
+ for (int i = 0; i < num_levels(); i++) {
+ int sz = sizeof(scratch->buffer) - len;
+ int ret = snprintf(scratch->buffer + len, sz, "%d ", int(files_[i].size()));
+ if (ret < 0 || ret >= sz) break;
+ len += ret;
+ }
+ if (len > 0) {
+ // overwrite the last space
+ --len;
+ }
+ len += snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len,
+ "] max score %.2f", compaction_score_[0]);
+
+ if (!files_marked_for_compaction_.empty()) {
+ snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len,
+ " (%" ROCKSDB_PRIszt " files need compaction)",
+ files_marked_for_compaction_.size());
+ }
+
+ return scratch->buffer;
+}
+
+const char* VersionStorageInfo::LevelFileSummary(FileSummaryStorage* scratch,
+ int level) const {
+ int len = snprintf(scratch->buffer, sizeof(scratch->buffer), "files_size[");
+ for (const auto& f : files_[level]) {
+ int sz = sizeof(scratch->buffer) - len;
+ char sztxt[16];
+ AppendHumanBytes(f->fd.GetFileSize(), sztxt, sizeof(sztxt));
+ int ret = snprintf(scratch->buffer + len, sz,
+ "#%" PRIu64 "(seq=%" PRIu64 ",sz=%s,%d) ",
+ f->fd.GetNumber(), f->smallest_seqno, sztxt,
+ static_cast<int>(f->being_compacted));
+ if (ret < 0 || ret >= sz)
+ break;
+ len += ret;
+ }
+ // overwrite the last space (only if files_[level].size() is non-zero)
+ if (files_[level].size() && len > 0) {
+ --len;
+ }
+ snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, "]");
+ return scratch->buffer;
+}
+
+int64_t VersionStorageInfo::MaxNextLevelOverlappingBytes() {
+ uint64_t result = 0;
+ std::vector<FileMetaData*> overlaps;
+ for (int level = 1; level < num_levels() - 1; level++) {
+ for (const auto& f : files_[level]) {
+ GetOverlappingInputs(level + 1, &f->smallest, &f->largest, &overlaps);
+ const uint64_t sum = TotalFileSize(overlaps);
+ if (sum > result) {
+ result = sum;
+ }
+ }
+ }
+ return result;
+}
+
+uint64_t VersionStorageInfo::MaxBytesForLevel(int level) const {
+ // Note: the result for level zero is not really used since we set
+ // the level-0 compaction threshold based on number of files.
+ assert(level >= 0);
+ assert(level < static_cast<int>(level_max_bytes_.size()));
+ return level_max_bytes_[level];
+}
+
+void VersionStorageInfo::CalculateBaseBytes(const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& options) {
+ // Special logic to set number of sorted runs.
+ // It is to match the previous behavior when all files are in L0.
+ int num_l0_count = static_cast<int>(files_[0].size());
+ if (compaction_style_ == kCompactionStyleUniversal) {
+ // For universal compaction, we use level0 score to indicate
+ // compaction score for the whole DB. Adding other levels as if
+ // they are L0 files.
+ for (int i = 1; i < num_levels(); i++) {
+ if (!files_[i].empty()) {
+ num_l0_count++;
+ }
+ }
+ }
+ set_l0_delay_trigger_count(num_l0_count);
+
+ level_max_bytes_.resize(ioptions.num_levels);
+ if (!ioptions.level_compaction_dynamic_level_bytes) {
+ base_level_ = (ioptions.compaction_style == kCompactionStyleLevel) ? 1 : -1;
+
+ // Calculate for static bytes base case
+ for (int i = 0; i < ioptions.num_levels; ++i) {
+ if (i == 0 && ioptions.compaction_style == kCompactionStyleUniversal) {
+ level_max_bytes_[i] = options.max_bytes_for_level_base;
+ } else if (i > 1) {
+ level_max_bytes_[i] = MultiplyCheckOverflow(
+ MultiplyCheckOverflow(level_max_bytes_[i - 1],
+ options.max_bytes_for_level_multiplier),
+ options.MaxBytesMultiplerAdditional(i - 1));
+ } else {
+ level_max_bytes_[i] = options.max_bytes_for_level_base;
+ }
+ }
+ } else {
+ uint64_t max_level_size = 0;
+
+ int first_non_empty_level = -1;
+ // Find size of non-L0 level of most data.
+ // Cannot use the size of the last level because it can be empty or less
+ // than previous levels after compaction.
+ for (int i = 1; i < num_levels_; i++) {
+ uint64_t total_size = 0;
+ for (const auto& f : files_[i]) {
+ total_size += f->fd.GetFileSize();
+ }
+ if (total_size > 0 && first_non_empty_level == -1) {
+ first_non_empty_level = i;
+ }
+ if (total_size > max_level_size) {
+ max_level_size = total_size;
+ }
+ }
+
+ // Prefill every level's max bytes to disallow compaction from there.
+ for (int i = 0; i < num_levels_; i++) {
+ level_max_bytes_[i] = std::numeric_limits<uint64_t>::max();
+ }
+
+ if (max_level_size == 0) {
+ // No data for L1 and up. L0 compacts to last level directly.
+ // No compaction from L1+ needs to be scheduled.
+ base_level_ = num_levels_ - 1;
+ } else {
+ uint64_t base_bytes_max = options.max_bytes_for_level_base;
+ uint64_t base_bytes_min = static_cast<uint64_t>(
+ base_bytes_max / options.max_bytes_for_level_multiplier);
+
+ // Try whether we can make last level's target size to be max_level_size
+ uint64_t cur_level_size = max_level_size;
+ for (int i = num_levels_ - 2; i >= first_non_empty_level; i--) {
+ // Round up after dividing
+ cur_level_size = static_cast<uint64_t>(
+ cur_level_size / options.max_bytes_for_level_multiplier);
+ }
+
+ // Calculate base level and its size.
+ uint64_t base_level_size;
+ if (cur_level_size <= base_bytes_min) {
+ // Case 1. If we make target size of last level to be max_level_size,
+ // target size of the first non-empty level would be smaller than
+ // base_bytes_min. We set it be base_bytes_min.
+ base_level_size = base_bytes_min + 1U;
+ base_level_ = first_non_empty_level;
+ ROCKS_LOG_WARN(ioptions.info_log,
+ "More existing levels in DB than needed. "
+ "max_bytes_for_level_multiplier may not be guaranteed.");
+ } else {
+ // Find base level (where L0 data is compacted to).
+ base_level_ = first_non_empty_level;
+ while (base_level_ > 1 && cur_level_size > base_bytes_max) {
+ --base_level_;
+ cur_level_size = static_cast<uint64_t>(
+ cur_level_size / options.max_bytes_for_level_multiplier);
+ }
+ if (cur_level_size > base_bytes_max) {
+ // Even L1 will be too large
+ assert(base_level_ == 1);
+ base_level_size = base_bytes_max;
+ } else {
+ base_level_size = cur_level_size;
+ }
+ }
+
+ uint64_t level_size = base_level_size;
+ for (int i = base_level_; i < num_levels_; i++) {
+ if (i > base_level_) {
+ level_size = MultiplyCheckOverflow(
+ level_size, options.max_bytes_for_level_multiplier);
+ }
+ // Don't set any level below base_bytes_max. Otherwise, the LSM can
+ // assume an hourglass shape where L1+ sizes are smaller than L0. This
+ // causes compaction scoring, which depends on level sizes, to favor L1+
+ // at the expense of L0, which may fill up and stall.
+ level_max_bytes_[i] = std::max(level_size, base_bytes_max);
+ }
+ }
+ }
+}
+
+uint64_t VersionStorageInfo::EstimateLiveDataSize() const {
+ // Estimate the live data size by adding up the size of the last level for all
+ // key ranges. Note: Estimate depends on the ordering of files in level 0
+ // because files in level 0 can be overlapping.
+ uint64_t size = 0;
+
+ auto ikey_lt = [this](InternalKey* x, InternalKey* y) {
+ return internal_comparator_->Compare(*x, *y) < 0;
+ };
+ // (Ordered) map of largest keys in non-overlapping files
+ std::map<InternalKey*, FileMetaData*, decltype(ikey_lt)> ranges(ikey_lt);
+
+ for (int l = num_levels_ - 1; l >= 0; l--) {
+ bool found_end = false;
+ for (auto file : files_[l]) {
+ // Find the first file where the largest key is larger than the smallest
+ // key of the current file. If this file does not overlap with the
+ // current file, none of the files in the map does. If there is
+ // no potential overlap, we can safely insert the rest of this level
+ // (if the level is not 0) into the map without checking again because
+ // the elements in the level are sorted and non-overlapping.
+ auto lb = (found_end && l != 0) ?
+ ranges.end() : ranges.lower_bound(&file->smallest);
+ found_end = (lb == ranges.end());
+ if (found_end || internal_comparator_->Compare(
+ file->largest, (*lb).second->smallest) < 0) {
+ ranges.emplace_hint(lb, &file->largest, file);
+ size += file->fd.file_size;
+ }
+ }
+ }
+ return size;
+}
+
+
+void Version::AddLiveFiles(std::vector<FileDescriptor>* live) {
+ for (int level = 0; level < storage_info_.num_levels(); level++) {
+ const std::vector<FileMetaData*>& files = storage_info_.files_[level];
+ for (const auto& file : files) {
+ live->push_back(file->fd);
+ }
+ }
+}
+
+std::string Version::DebugString(bool hex, bool print_stats) const {
+ std::string r;
+ for (int level = 0; level < storage_info_.num_levels_; level++) {
+ // E.g.,
+ // --- level 1 ---
+ // 17:123['a' .. 'd']
+ // 20:43['e' .. 'g']
+ //
+ // if print_stats=true:
+ // 17:123['a' .. 'd'](4096)
+ r.append("--- level ");
+ AppendNumberTo(&r, level);
+ r.append(" --- version# ");
+ AppendNumberTo(&r, version_number_);
+ r.append(" ---\n");
+ const std::vector<FileMetaData*>& files = storage_info_.files_[level];
+ for (size_t i = 0; i < files.size(); i++) {
+ r.push_back(' ');
+ AppendNumberTo(&r, files[i]->fd.GetNumber());
+ r.push_back(':');
+ AppendNumberTo(&r, files[i]->fd.GetFileSize());
+ r.append("[");
+ r.append(files[i]->smallest.DebugString(hex));
+ r.append(" .. ");
+ r.append(files[i]->largest.DebugString(hex));
+ r.append("]");
+ if (print_stats) {
+ r.append("(");
+ r.append(ToString(
+ files[i]->stats.num_reads_sampled.load(std::memory_order_relaxed)));
+ r.append(")");
+ }
+ r.append("\n");
+ }
+ }
+ return r;
+}
+
+// this is used to batch writes to the manifest file
+struct VersionSet::ManifestWriter {
+ Status status;
+ bool done;
+ InstrumentedCondVar cv;
+ ColumnFamilyData* cfd;
+ const autovector<VersionEdit*>& edit_list;
+
+ explicit ManifestWriter(InstrumentedMutex* mu, ColumnFamilyData* _cfd,
+ const autovector<VersionEdit*>& e)
+ : done(false), cv(mu), cfd(_cfd), edit_list(e) {}
+};
+
+VersionSet::VersionSet(const std::string& dbname,
+ const ImmutableDBOptions* db_options,
+ const EnvOptions& storage_options, Cache* table_cache,
+ WriteBufferManager* write_buffer_manager,
+ WriteController* write_controller)
+ : column_family_set_(
+ new ColumnFamilySet(dbname, db_options, storage_options, table_cache,
+ write_buffer_manager, write_controller)),
+ env_(db_options->env),
+ dbname_(dbname),
+ db_options_(db_options),
+ next_file_number_(2),
+ manifest_file_number_(0), // Filled by Recover()
+ pending_manifest_file_number_(0),
+ last_sequence_(0),
+ last_to_be_written_sequence_(0),
+ prev_log_number_(0),
+ current_version_number_(0),
+ manifest_file_size_(0),
+ env_options_(storage_options),
+ env_options_compactions_(
+ env_->OptimizeForCompactionTableRead(env_options_, *db_options_)) {}
+
+void CloseTables(void* ptr, size_t) {
+ TableReader* table_reader = reinterpret_cast<TableReader*>(ptr);
+ table_reader->Close();
+}
+
+VersionSet::~VersionSet() {
+ // we need to delete column_family_set_ because its destructor depends on
+ // VersionSet
+ Cache* table_cache = column_family_set_->get_table_cache();
+ table_cache->ApplyToAllCacheEntries(&CloseTables, false /* thread_safe */);
+ column_family_set_.reset();
+ for (auto file : obsolete_files_) {
+ if (file->table_reader_handle) {
+ table_cache->Release(file->table_reader_handle);
+ TableCache::Evict(table_cache, file->fd.GetNumber());
+ }
+ delete file;
+ }
+ obsolete_files_.clear();
+}
+
+void VersionSet::AppendVersion(ColumnFamilyData* column_family_data,
+ Version* v) {
+ // compute new compaction score
+ v->storage_info()->ComputeCompactionScore(
+ *column_family_data->ioptions(),
+ *column_family_data->GetLatestM
<TRUNCATED>
[29/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/c.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/c.cc b/thirdparty/rocksdb/db/c.cc
new file mode 100644
index 0000000..788eab6
--- /dev/null
+++ b/thirdparty/rocksdb/db/c.cc
@@ -0,0 +1,3549 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef ROCKSDB_LITE
+
+#include "rocksdb/c.h"
+
+#include <stdlib.h>
+#include "port/port.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/options.h"
+#include "rocksdb/status.h"
+#include "rocksdb/write_batch.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/universal_compaction.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/slice_transform.h"
+#include "rocksdb/table.h"
+#include "rocksdb/rate_limiter.h"
+#include "rocksdb/utilities/backupable_db.h"
+#include "rocksdb/utilities/write_batch_with_index.h"
+#include "utilities/merge_operators.h"
+#include "rocksdb/utilities/transaction.h"
+#include "rocksdb/utilities/transaction_db.h"
+#include "rocksdb/utilities/checkpoint.h"
+
+using rocksdb::BytewiseComparator;
+using rocksdb::Cache;
+using rocksdb::ColumnFamilyDescriptor;
+using rocksdb::ColumnFamilyHandle;
+using rocksdb::ColumnFamilyOptions;
+using rocksdb::CompactionFilter;
+using rocksdb::CompactionFilterFactory;
+using rocksdb::CompactionFilterContext;
+using rocksdb::CompactionOptionsFIFO;
+using rocksdb::Comparator;
+using rocksdb::CompressionType;
+using rocksdb::WALRecoveryMode;
+using rocksdb::DB;
+using rocksdb::DBOptions;
+using rocksdb::DbPath;
+using rocksdb::Env;
+using rocksdb::EnvOptions;
+using rocksdb::InfoLogLevel;
+using rocksdb::FileLock;
+using rocksdb::FilterPolicy;
+using rocksdb::FlushOptions;
+using rocksdb::IngestExternalFileOptions;
+using rocksdb::Iterator;
+using rocksdb::Logger;
+using rocksdb::MergeOperator;
+using rocksdb::MergeOperators;
+using rocksdb::NewBloomFilterPolicy;
+using rocksdb::NewLRUCache;
+using rocksdb::Options;
+using rocksdb::BlockBasedTableOptions;
+using rocksdb::CuckooTableOptions;
+using rocksdb::RandomAccessFile;
+using rocksdb::Range;
+using rocksdb::ReadOptions;
+using rocksdb::SequentialFile;
+using rocksdb::Slice;
+using rocksdb::SliceParts;
+using rocksdb::SliceTransform;
+using rocksdb::Snapshot;
+using rocksdb::SstFileWriter;
+using rocksdb::Status;
+using rocksdb::WritableFile;
+using rocksdb::WriteBatch;
+using rocksdb::WriteBatchWithIndex;
+using rocksdb::WriteOptions;
+using rocksdb::LiveFileMetaData;
+using rocksdb::BackupEngine;
+using rocksdb::BackupableDBOptions;
+using rocksdb::BackupInfo;
+using rocksdb::RestoreOptions;
+using rocksdb::CompactRangeOptions;
+using rocksdb::RateLimiter;
+using rocksdb::NewGenericRateLimiter;
+using rocksdb::PinnableSlice;
+using rocksdb::TransactionDBOptions;
+using rocksdb::TransactionDB;
+using rocksdb::TransactionOptions;
+using rocksdb::Transaction;
+using rocksdb::Checkpoint;
+
+using std::shared_ptr;
+
+extern "C" {
+
+struct rocksdb_t { DB* rep; };
+struct rocksdb_backup_engine_t { BackupEngine* rep; };
+struct rocksdb_backup_engine_info_t { std::vector<BackupInfo> rep; };
+struct rocksdb_restore_options_t { RestoreOptions rep; };
+struct rocksdb_iterator_t { Iterator* rep; };
+struct rocksdb_writebatch_t { WriteBatch rep; };
+struct rocksdb_writebatch_wi_t { WriteBatchWithIndex* rep; };
+struct rocksdb_snapshot_t { const Snapshot* rep; };
+struct rocksdb_flushoptions_t { FlushOptions rep; };
+struct rocksdb_fifo_compaction_options_t { CompactionOptionsFIFO rep; };
+struct rocksdb_readoptions_t {
+ ReadOptions rep;
+ Slice upper_bound; // stack variable to set pointer to in ReadOptions
+};
+struct rocksdb_writeoptions_t { WriteOptions rep; };
+struct rocksdb_options_t { Options rep; };
+struct rocksdb_compactoptions_t {
+ CompactRangeOptions rep;
+};
+struct rocksdb_block_based_table_options_t { BlockBasedTableOptions rep; };
+struct rocksdb_cuckoo_table_options_t { CuckooTableOptions rep; };
+struct rocksdb_seqfile_t { SequentialFile* rep; };
+struct rocksdb_randomfile_t { RandomAccessFile* rep; };
+struct rocksdb_writablefile_t { WritableFile* rep; };
+struct rocksdb_filelock_t { FileLock* rep; };
+struct rocksdb_logger_t { shared_ptr<Logger> rep; };
+struct rocksdb_cache_t { shared_ptr<Cache> rep; };
+struct rocksdb_livefiles_t { std::vector<LiveFileMetaData> rep; };
+struct rocksdb_column_family_handle_t { ColumnFamilyHandle* rep; };
+struct rocksdb_envoptions_t { EnvOptions rep; };
+struct rocksdb_ingestexternalfileoptions_t { IngestExternalFileOptions rep; };
+struct rocksdb_sstfilewriter_t { SstFileWriter* rep; };
+struct rocksdb_ratelimiter_t { RateLimiter* rep; };
+struct rocksdb_pinnableslice_t {
+ PinnableSlice rep;
+};
+struct rocksdb_transactiondb_options_t {
+ TransactionDBOptions rep;
+};
+struct rocksdb_transactiondb_t {
+ TransactionDB* rep;
+};
+struct rocksdb_transaction_options_t {
+ TransactionOptions rep;
+};
+struct rocksdb_transaction_t {
+ Transaction* rep;
+};
+struct rocksdb_checkpoint_t {
+ Checkpoint* rep;
+};
+
+struct rocksdb_compactionfiltercontext_t {
+ CompactionFilter::Context rep;
+};
+
+struct rocksdb_compactionfilter_t : public CompactionFilter {
+ void* state_;
+ void (*destructor_)(void*);
+ unsigned char (*filter_)(
+ void*,
+ int level,
+ const char* key, size_t key_length,
+ const char* existing_value, size_t value_length,
+ char** new_value, size_t *new_value_length,
+ unsigned char* value_changed);
+ const char* (*name_)(void*);
+ unsigned char ignore_snapshots_;
+
+ virtual ~rocksdb_compactionfilter_t() {
+ (*destructor_)(state_);
+ }
+
+ virtual bool Filter(int level, const Slice& key, const Slice& existing_value,
+ std::string* new_value,
+ bool* value_changed) const override {
+ char* c_new_value = nullptr;
+ size_t new_value_length = 0;
+ unsigned char c_value_changed = 0;
+ unsigned char result = (*filter_)(
+ state_,
+ level,
+ key.data(), key.size(),
+ existing_value.data(), existing_value.size(),
+ &c_new_value, &new_value_length, &c_value_changed);
+ if (c_value_changed) {
+ new_value->assign(c_new_value, new_value_length);
+ *value_changed = true;
+ }
+ return result;
+ }
+
+ virtual const char* Name() const override { return (*name_)(state_); }
+
+ virtual bool IgnoreSnapshots() const override { return ignore_snapshots_; }
+};
+
+struct rocksdb_compactionfilterfactory_t : public CompactionFilterFactory {
+ void* state_;
+ void (*destructor_)(void*);
+ rocksdb_compactionfilter_t* (*create_compaction_filter_)(
+ void*, rocksdb_compactionfiltercontext_t* context);
+ const char* (*name_)(void*);
+
+ virtual ~rocksdb_compactionfilterfactory_t() { (*destructor_)(state_); }
+
+ virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
+ const CompactionFilter::Context& context) override {
+ rocksdb_compactionfiltercontext_t ccontext;
+ ccontext.rep = context;
+ CompactionFilter* cf = (*create_compaction_filter_)(state_, &ccontext);
+ return std::unique_ptr<CompactionFilter>(cf);
+ }
+
+ virtual const char* Name() const override { return (*name_)(state_); }
+};
+
+struct rocksdb_comparator_t : public Comparator {
+ void* state_;
+ void (*destructor_)(void*);
+ int (*compare_)(
+ void*,
+ const char* a, size_t alen,
+ const char* b, size_t blen);
+ const char* (*name_)(void*);
+
+ virtual ~rocksdb_comparator_t() {
+ (*destructor_)(state_);
+ }
+
+ virtual int Compare(const Slice& a, const Slice& b) const override {
+ return (*compare_)(state_, a.data(), a.size(), b.data(), b.size());
+ }
+
+ virtual const char* Name() const override { return (*name_)(state_); }
+
+ // No-ops since the C binding does not support key shortening methods.
+ virtual void FindShortestSeparator(std::string*,
+ const Slice&) const override {}
+ virtual void FindShortSuccessor(std::string* key) const override {}
+};
+
+struct rocksdb_filterpolicy_t : public FilterPolicy {
+ void* state_;
+ void (*destructor_)(void*);
+ const char* (*name_)(void*);
+ char* (*create_)(
+ void*,
+ const char* const* key_array, const size_t* key_length_array,
+ int num_keys,
+ size_t* filter_length);
+ unsigned char (*key_match_)(
+ void*,
+ const char* key, size_t length,
+ const char* filter, size_t filter_length);
+ void (*delete_filter_)(
+ void*,
+ const char* filter, size_t filter_length);
+
+ virtual ~rocksdb_filterpolicy_t() {
+ (*destructor_)(state_);
+ }
+
+ virtual const char* Name() const override { return (*name_)(state_); }
+
+ virtual void CreateFilter(const Slice* keys, int n,
+ std::string* dst) const override {
+ std::vector<const char*> key_pointers(n);
+ std::vector<size_t> key_sizes(n);
+ for (int i = 0; i < n; i++) {
+ key_pointers[i] = keys[i].data();
+ key_sizes[i] = keys[i].size();
+ }
+ size_t len;
+ char* filter = (*create_)(state_, &key_pointers[0], &key_sizes[0], n, &len);
+ dst->append(filter, len);
+
+ if (delete_filter_ != nullptr) {
+ (*delete_filter_)(state_, filter, len);
+ } else {
+ free(filter);
+ }
+ }
+
+ virtual bool KeyMayMatch(const Slice& key,
+ const Slice& filter) const override {
+ return (*key_match_)(state_, key.data(), key.size(),
+ filter.data(), filter.size());
+ }
+};
+
+struct rocksdb_mergeoperator_t : public MergeOperator {
+ void* state_;
+ void (*destructor_)(void*);
+ const char* (*name_)(void*);
+ char* (*full_merge_)(
+ void*,
+ const char* key, size_t key_length,
+ const char* existing_value, size_t existing_value_length,
+ const char* const* operands_list, const size_t* operands_list_length,
+ int num_operands,
+ unsigned char* success, size_t* new_value_length);
+ char* (*partial_merge_)(void*, const char* key, size_t key_length,
+ const char* const* operands_list,
+ const size_t* operands_list_length, int num_operands,
+ unsigned char* success, size_t* new_value_length);
+ void (*delete_value_)(
+ void*,
+ const char* value, size_t value_length);
+
+ virtual ~rocksdb_mergeoperator_t() {
+ (*destructor_)(state_);
+ }
+
+ virtual const char* Name() const override { return (*name_)(state_); }
+
+ virtual bool FullMergeV2(const MergeOperationInput& merge_in,
+ MergeOperationOutput* merge_out) const override {
+ size_t n = merge_in.operand_list.size();
+ std::vector<const char*> operand_pointers(n);
+ std::vector<size_t> operand_sizes(n);
+ for (size_t i = 0; i < n; i++) {
+ Slice operand(merge_in.operand_list[i]);
+ operand_pointers[i] = operand.data();
+ operand_sizes[i] = operand.size();
+ }
+
+ const char* existing_value_data = nullptr;
+ size_t existing_value_len = 0;
+ if (merge_in.existing_value != nullptr) {
+ existing_value_data = merge_in.existing_value->data();
+ existing_value_len = merge_in.existing_value->size();
+ }
+
+ unsigned char success;
+ size_t new_value_len;
+ char* tmp_new_value = (*full_merge_)(
+ state_, merge_in.key.data(), merge_in.key.size(), existing_value_data,
+ existing_value_len, &operand_pointers[0], &operand_sizes[0],
+ static_cast<int>(n), &success, &new_value_len);
+ merge_out->new_value.assign(tmp_new_value, new_value_len);
+
+ if (delete_value_ != nullptr) {
+ (*delete_value_)(state_, tmp_new_value, new_value_len);
+ } else {
+ free(tmp_new_value);
+ }
+
+ return success;
+ }
+
+ virtual bool PartialMergeMulti(const Slice& key,
+ const std::deque<Slice>& operand_list,
+ std::string* new_value,
+ Logger* logger) const override {
+ size_t operand_count = operand_list.size();
+ std::vector<const char*> operand_pointers(operand_count);
+ std::vector<size_t> operand_sizes(operand_count);
+ for (size_t i = 0; i < operand_count; ++i) {
+ Slice operand(operand_list[i]);
+ operand_pointers[i] = operand.data();
+ operand_sizes[i] = operand.size();
+ }
+
+ unsigned char success;
+ size_t new_value_len;
+ char* tmp_new_value = (*partial_merge_)(
+ state_, key.data(), key.size(), &operand_pointers[0], &operand_sizes[0],
+ static_cast<int>(operand_count), &success, &new_value_len);
+ new_value->assign(tmp_new_value, new_value_len);
+
+ if (delete_value_ != nullptr) {
+ (*delete_value_)(state_, tmp_new_value, new_value_len);
+ } else {
+ free(tmp_new_value);
+ }
+
+ return success;
+ }
+};
+
+struct rocksdb_dbpath_t {
+ DbPath rep;
+};
+
+struct rocksdb_env_t {
+ Env* rep;
+ bool is_default;
+};
+
+struct rocksdb_slicetransform_t : public SliceTransform {
+ void* state_;
+ void (*destructor_)(void*);
+ const char* (*name_)(void*);
+ char* (*transform_)(
+ void*,
+ const char* key, size_t length,
+ size_t* dst_length);
+ unsigned char (*in_domain_)(
+ void*,
+ const char* key, size_t length);
+ unsigned char (*in_range_)(
+ void*,
+ const char* key, size_t length);
+
+ virtual ~rocksdb_slicetransform_t() {
+ (*destructor_)(state_);
+ }
+
+ virtual const char* Name() const override { return (*name_)(state_); }
+
+ virtual Slice Transform(const Slice& src) const override {
+ size_t len;
+ char* dst = (*transform_)(state_, src.data(), src.size(), &len);
+ return Slice(dst, len);
+ }
+
+ virtual bool InDomain(const Slice& src) const override {
+ return (*in_domain_)(state_, src.data(), src.size());
+ }
+
+ virtual bool InRange(const Slice& src) const override {
+ return (*in_range_)(state_, src.data(), src.size());
+ }
+};
+
+struct rocksdb_universal_compaction_options_t {
+ rocksdb::CompactionOptionsUniversal *rep;
+};
+
+static bool SaveError(char** errptr, const Status& s) {
+ assert(errptr != nullptr);
+ if (s.ok()) {
+ return false;
+ } else if (*errptr == nullptr) {
+ *errptr = strdup(s.ToString().c_str());
+ } else {
+ // TODO(sanjay): Merge with existing error?
+ // This is a bug if *errptr is not created by malloc()
+ free(*errptr);
+ *errptr = strdup(s.ToString().c_str());
+ }
+ return true;
+}
+
+static char* CopyString(const std::string& str) {
+ char* result = reinterpret_cast<char*>(malloc(sizeof(char) * str.size()));
+ memcpy(result, str.data(), sizeof(char) * str.size());
+ return result;
+}
+
+rocksdb_t* rocksdb_open(
+ const rocksdb_options_t* options,
+ const char* name,
+ char** errptr) {
+ DB* db;
+ if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) {
+ return nullptr;
+ }
+ rocksdb_t* result = new rocksdb_t;
+ result->rep = db;
+ return result;
+}
+
+rocksdb_t* rocksdb_open_for_read_only(
+ const rocksdb_options_t* options,
+ const char* name,
+ unsigned char error_if_log_file_exist,
+ char** errptr) {
+ DB* db;
+ if (SaveError(errptr, DB::OpenForReadOnly(options->rep, std::string(name), &db, error_if_log_file_exist))) {
+ return nullptr;
+ }
+ rocksdb_t* result = new rocksdb_t;
+ result->rep = db;
+ return result;
+}
+
+rocksdb_backup_engine_t* rocksdb_backup_engine_open(
+ const rocksdb_options_t* options, const char* path, char** errptr) {
+ BackupEngine* be;
+ if (SaveError(errptr, BackupEngine::Open(options->rep.env,
+ BackupableDBOptions(path,
+ nullptr,
+ true,
+ options->rep.info_log.get()),
+ &be))) {
+ return nullptr;
+ }
+ rocksdb_backup_engine_t* result = new rocksdb_backup_engine_t;
+ result->rep = be;
+ return result;
+}
+
+void rocksdb_backup_engine_create_new_backup(rocksdb_backup_engine_t* be,
+ rocksdb_t* db, char** errptr) {
+ SaveError(errptr, be->rep->CreateNewBackup(db->rep));
+}
+
+void rocksdb_backup_engine_purge_old_backups(rocksdb_backup_engine_t* be,
+ uint32_t num_backups_to_keep,
+ char** errptr) {
+ SaveError(errptr, be->rep->PurgeOldBackups(num_backups_to_keep));
+}
+
+rocksdb_restore_options_t* rocksdb_restore_options_create() {
+ return new rocksdb_restore_options_t;
+}
+
+void rocksdb_restore_options_destroy(rocksdb_restore_options_t* opt) {
+ delete opt;
+}
+
+void rocksdb_restore_options_set_keep_log_files(rocksdb_restore_options_t* opt,
+ int v) {
+ opt->rep.keep_log_files = v;
+}
+
+void rocksdb_backup_engine_restore_db_from_latest_backup(
+ rocksdb_backup_engine_t* be, const char* db_dir, const char* wal_dir,
+ const rocksdb_restore_options_t* restore_options, char** errptr) {
+ SaveError(errptr, be->rep->RestoreDBFromLatestBackup(std::string(db_dir),
+ std::string(wal_dir),
+ restore_options->rep));
+}
+
+const rocksdb_backup_engine_info_t* rocksdb_backup_engine_get_backup_info(
+ rocksdb_backup_engine_t* be) {
+ rocksdb_backup_engine_info_t* result = new rocksdb_backup_engine_info_t;
+ be->rep->GetBackupInfo(&result->rep);
+ return result;
+}
+
+int rocksdb_backup_engine_info_count(const rocksdb_backup_engine_info_t* info) {
+ return static_cast<int>(info->rep.size());
+}
+
+int64_t rocksdb_backup_engine_info_timestamp(
+ const rocksdb_backup_engine_info_t* info, int index) {
+ return info->rep[index].timestamp;
+}
+
+uint32_t rocksdb_backup_engine_info_backup_id(
+ const rocksdb_backup_engine_info_t* info, int index) {
+ return info->rep[index].backup_id;
+}
+
+uint64_t rocksdb_backup_engine_info_size(
+ const rocksdb_backup_engine_info_t* info, int index) {
+ return info->rep[index].size;
+}
+
+uint32_t rocksdb_backup_engine_info_number_files(
+ const rocksdb_backup_engine_info_t* info, int index) {
+ return info->rep[index].number_files;
+}
+
+void rocksdb_backup_engine_info_destroy(
+ const rocksdb_backup_engine_info_t* info) {
+ delete info;
+}
+
+void rocksdb_backup_engine_close(rocksdb_backup_engine_t* be) {
+ delete be->rep;
+ delete be;
+}
+
+rocksdb_checkpoint_t* rocksdb_checkpoint_object_create(rocksdb_t* db,
+ char** errptr) {
+ Checkpoint* checkpoint;
+ if (SaveError(errptr, Checkpoint::Create(db->rep, &checkpoint))) {
+ return nullptr;
+ }
+ rocksdb_checkpoint_t* result = new rocksdb_checkpoint_t;
+ result->rep = checkpoint;
+ return result;
+}
+
+void rocksdb_checkpoint_create(rocksdb_checkpoint_t* checkpoint,
+ const char* checkpoint_dir,
+ uint64_t log_size_for_flush, char** errptr) {
+ SaveError(errptr, checkpoint->rep->CreateCheckpoint(
+ std::string(checkpoint_dir), log_size_for_flush));
+}
+
+void rocksdb_checkpoint_object_destroy(rocksdb_checkpoint_t* checkpoint) {
+ delete checkpoint->rep;
+ delete checkpoint;
+}
+
+void rocksdb_close(rocksdb_t* db) {
+ delete db->rep;
+ delete db;
+}
+
+void rocksdb_options_set_uint64add_merge_operator(rocksdb_options_t* opt) {
+ opt->rep.merge_operator = rocksdb::MergeOperators::CreateUInt64AddOperator();
+}
+
+rocksdb_t* rocksdb_open_column_families(
+ const rocksdb_options_t* db_options,
+ const char* name,
+ int num_column_families,
+ const char** column_family_names,
+ const rocksdb_options_t** column_family_options,
+ rocksdb_column_family_handle_t** column_family_handles,
+ char** errptr) {
+ std::vector<ColumnFamilyDescriptor> column_families;
+ for (int i = 0; i < num_column_families; i++) {
+ column_families.push_back(ColumnFamilyDescriptor(
+ std::string(column_family_names[i]),
+ ColumnFamilyOptions(column_family_options[i]->rep)));
+ }
+
+ DB* db;
+ std::vector<ColumnFamilyHandle*> handles;
+ if (SaveError(errptr, DB::Open(DBOptions(db_options->rep),
+ std::string(name), column_families, &handles, &db))) {
+ return nullptr;
+ }
+
+ for (size_t i = 0; i < handles.size(); i++) {
+ rocksdb_column_family_handle_t* c_handle = new rocksdb_column_family_handle_t;
+ c_handle->rep = handles[i];
+ column_family_handles[i] = c_handle;
+ }
+ rocksdb_t* result = new rocksdb_t;
+ result->rep = db;
+ return result;
+}
+
+rocksdb_t* rocksdb_open_for_read_only_column_families(
+ const rocksdb_options_t* db_options,
+ const char* name,
+ int num_column_families,
+ const char** column_family_names,
+ const rocksdb_options_t** column_family_options,
+ rocksdb_column_family_handle_t** column_family_handles,
+ unsigned char error_if_log_file_exist,
+ char** errptr) {
+ std::vector<ColumnFamilyDescriptor> column_families;
+ for (int i = 0; i < num_column_families; i++) {
+ column_families.push_back(ColumnFamilyDescriptor(
+ std::string(column_family_names[i]),
+ ColumnFamilyOptions(column_family_options[i]->rep)));
+ }
+
+ DB* db;
+ std::vector<ColumnFamilyHandle*> handles;
+ if (SaveError(errptr, DB::OpenForReadOnly(DBOptions(db_options->rep),
+ std::string(name), column_families, &handles, &db, error_if_log_file_exist))) {
+ return nullptr;
+ }
+
+ for (size_t i = 0; i < handles.size(); i++) {
+ rocksdb_column_family_handle_t* c_handle = new rocksdb_column_family_handle_t;
+ c_handle->rep = handles[i];
+ column_family_handles[i] = c_handle;
+ }
+ rocksdb_t* result = new rocksdb_t;
+ result->rep = db;
+ return result;
+}
+
+char** rocksdb_list_column_families(
+ const rocksdb_options_t* options,
+ const char* name,
+ size_t* lencfs,
+ char** errptr) {
+ std::vector<std::string> fams;
+ SaveError(errptr,
+ DB::ListColumnFamilies(DBOptions(options->rep),
+ std::string(name), &fams));
+
+ *lencfs = fams.size();
+ char** column_families = static_cast<char**>(malloc(sizeof(char*) * fams.size()));
+ for (size_t i = 0; i < fams.size(); i++) {
+ column_families[i] = strdup(fams[i].c_str());
+ }
+ return column_families;
+}
+
+void rocksdb_list_column_families_destroy(char** list, size_t len) {
+ for (size_t i = 0; i < len; ++i) {
+ free(list[i]);
+ }
+ free(list);
+}
+
+rocksdb_column_family_handle_t* rocksdb_create_column_family(
+ rocksdb_t* db,
+ const rocksdb_options_t* column_family_options,
+ const char* column_family_name,
+ char** errptr) {
+ rocksdb_column_family_handle_t* handle = new rocksdb_column_family_handle_t;
+ SaveError(errptr,
+ db->rep->CreateColumnFamily(ColumnFamilyOptions(column_family_options->rep),
+ std::string(column_family_name), &(handle->rep)));
+ return handle;
+}
+
+void rocksdb_drop_column_family(
+ rocksdb_t* db,
+ rocksdb_column_family_handle_t* handle,
+ char** errptr) {
+ SaveError(errptr, db->rep->DropColumnFamily(handle->rep));
+}
+
+void rocksdb_column_family_handle_destroy(rocksdb_column_family_handle_t* handle) {
+ delete handle->rep;
+ delete handle;
+}
+
+void rocksdb_put(
+ rocksdb_t* db,
+ const rocksdb_writeoptions_t* options,
+ const char* key, size_t keylen,
+ const char* val, size_t vallen,
+ char** errptr) {
+ SaveError(errptr,
+ db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen)));
+}
+
+void rocksdb_put_cf(
+ rocksdb_t* db,
+ const rocksdb_writeoptions_t* options,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t keylen,
+ const char* val, size_t vallen,
+ char** errptr) {
+ SaveError(errptr,
+ db->rep->Put(options->rep, column_family->rep,
+ Slice(key, keylen), Slice(val, vallen)));
+}
+
+void rocksdb_delete(
+ rocksdb_t* db,
+ const rocksdb_writeoptions_t* options,
+ const char* key, size_t keylen,
+ char** errptr) {
+ SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen)));
+}
+
+void rocksdb_delete_cf(
+ rocksdb_t* db,
+ const rocksdb_writeoptions_t* options,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t keylen,
+ char** errptr) {
+ SaveError(errptr, db->rep->Delete(options->rep, column_family->rep,
+ Slice(key, keylen)));
+}
+
+void rocksdb_merge(
+ rocksdb_t* db,
+ const rocksdb_writeoptions_t* options,
+ const char* key, size_t keylen,
+ const char* val, size_t vallen,
+ char** errptr) {
+ SaveError(errptr,
+ db->rep->Merge(options->rep, Slice(key, keylen), Slice(val, vallen)));
+}
+
+void rocksdb_merge_cf(
+ rocksdb_t* db,
+ const rocksdb_writeoptions_t* options,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t keylen,
+ const char* val, size_t vallen,
+ char** errptr) {
+ SaveError(errptr,
+ db->rep->Merge(options->rep, column_family->rep,
+ Slice(key, keylen), Slice(val, vallen)));
+}
+
+void rocksdb_write(
+ rocksdb_t* db,
+ const rocksdb_writeoptions_t* options,
+ rocksdb_writebatch_t* batch,
+ char** errptr) {
+ SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
+}
+
+char* rocksdb_get(
+ rocksdb_t* db,
+ const rocksdb_readoptions_t* options,
+ const char* key, size_t keylen,
+ size_t* vallen,
+ char** errptr) {
+ char* result = nullptr;
+ std::string tmp;
+ Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp);
+ if (s.ok()) {
+ *vallen = tmp.size();
+ result = CopyString(tmp);
+ } else {
+ *vallen = 0;
+ if (!s.IsNotFound()) {
+ SaveError(errptr, s);
+ }
+ }
+ return result;
+}
+
+char* rocksdb_get_cf(
+ rocksdb_t* db,
+ const rocksdb_readoptions_t* options,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t keylen,
+ size_t* vallen,
+ char** errptr) {
+ char* result = nullptr;
+ std::string tmp;
+ Status s = db->rep->Get(options->rep, column_family->rep,
+ Slice(key, keylen), &tmp);
+ if (s.ok()) {
+ *vallen = tmp.size();
+ result = CopyString(tmp);
+ } else {
+ *vallen = 0;
+ if (!s.IsNotFound()) {
+ SaveError(errptr, s);
+ }
+ }
+ return result;
+}
+
+void rocksdb_multi_get(
+ rocksdb_t* db,
+ const rocksdb_readoptions_t* options,
+ size_t num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes,
+ char** values_list, size_t* values_list_sizes,
+ char** errs) {
+ std::vector<Slice> keys(num_keys);
+ for (size_t i = 0; i < num_keys; i++) {
+ keys[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ std::vector<std::string> values(num_keys);
+ std::vector<Status> statuses = db->rep->MultiGet(options->rep, keys, &values);
+ for (size_t i = 0; i < num_keys; i++) {
+ if (statuses[i].ok()) {
+ values_list[i] = CopyString(values[i]);
+ values_list_sizes[i] = values[i].size();
+ errs[i] = nullptr;
+ } else {
+ values_list[i] = nullptr;
+ values_list_sizes[i] = 0;
+ if (!statuses[i].IsNotFound()) {
+ errs[i] = strdup(statuses[i].ToString().c_str());
+ } else {
+ errs[i] = nullptr;
+ }
+ }
+ }
+}
+
+void rocksdb_multi_get_cf(
+ rocksdb_t* db,
+ const rocksdb_readoptions_t* options,
+ const rocksdb_column_family_handle_t* const* column_families,
+ size_t num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes,
+ char** values_list, size_t* values_list_sizes,
+ char** errs) {
+ std::vector<Slice> keys(num_keys);
+ std::vector<ColumnFamilyHandle*> cfs(num_keys);
+ for (size_t i = 0; i < num_keys; i++) {
+ keys[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ cfs[i] = column_families[i]->rep;
+ }
+ std::vector<std::string> values(num_keys);
+ std::vector<Status> statuses = db->rep->MultiGet(options->rep, cfs, keys, &values);
+ for (size_t i = 0; i < num_keys; i++) {
+ if (statuses[i].ok()) {
+ values_list[i] = CopyString(values[i]);
+ values_list_sizes[i] = values[i].size();
+ errs[i] = nullptr;
+ } else {
+ values_list[i] = nullptr;
+ values_list_sizes[i] = 0;
+ if (!statuses[i].IsNotFound()) {
+ errs[i] = strdup(statuses[i].ToString().c_str());
+ } else {
+ errs[i] = nullptr;
+ }
+ }
+ }
+}
+
+rocksdb_iterator_t* rocksdb_create_iterator(
+ rocksdb_t* db,
+ const rocksdb_readoptions_t* options) {
+ rocksdb_iterator_t* result = new rocksdb_iterator_t;
+ result->rep = db->rep->NewIterator(options->rep);
+ return result;
+}
+
+rocksdb_iterator_t* rocksdb_create_iterator_cf(
+ rocksdb_t* db,
+ const rocksdb_readoptions_t* options,
+ rocksdb_column_family_handle_t* column_family) {
+ rocksdb_iterator_t* result = new rocksdb_iterator_t;
+ result->rep = db->rep->NewIterator(options->rep, column_family->rep);
+ return result;
+}
+
+void rocksdb_create_iterators(
+ rocksdb_t *db,
+ rocksdb_readoptions_t* opts,
+ rocksdb_column_family_handle_t** column_families,
+ rocksdb_iterator_t** iterators,
+ size_t size,
+ char** errptr) {
+ std::vector<ColumnFamilyHandle*> column_families_vec;
+ for (size_t i = 0; i < size; i++) {
+ column_families_vec.push_back(column_families[i]->rep);
+ }
+
+ std::vector<Iterator*> res;
+ Status status = db->rep->NewIterators(opts->rep, column_families_vec, &res);
+ assert(res.size() == size);
+ if (SaveError(errptr, status)) {
+ return;
+ }
+
+ for (size_t i = 0; i < size; i++) {
+ iterators[i] = new rocksdb_iterator_t;
+ iterators[i]->rep = res[i];
+ }
+}
+
+const rocksdb_snapshot_t* rocksdb_create_snapshot(
+ rocksdb_t* db) {
+ rocksdb_snapshot_t* result = new rocksdb_snapshot_t;
+ result->rep = db->rep->GetSnapshot();
+ return result;
+}
+
+void rocksdb_release_snapshot(
+ rocksdb_t* db,
+ const rocksdb_snapshot_t* snapshot) {
+ db->rep->ReleaseSnapshot(snapshot->rep);
+ delete snapshot;
+}
+
+char* rocksdb_property_value(
+ rocksdb_t* db,
+ const char* propname) {
+ std::string tmp;
+ if (db->rep->GetProperty(Slice(propname), &tmp)) {
+ // We use strdup() since we expect human readable output.
+ return strdup(tmp.c_str());
+ } else {
+ return nullptr;
+ }
+}
+
+int rocksdb_property_int(
+ rocksdb_t* db,
+ const char* propname,
+ uint64_t *out_val) {
+ if (db->rep->GetIntProperty(Slice(propname), out_val)) {
+ return 0;
+ } else {
+ return -1;
+ }
+}
+
+char* rocksdb_property_value_cf(
+ rocksdb_t* db,
+ rocksdb_column_family_handle_t* column_family,
+ const char* propname) {
+ std::string tmp;
+ if (db->rep->GetProperty(column_family->rep, Slice(propname), &tmp)) {
+ // We use strdup() since we expect human readable output.
+ return strdup(tmp.c_str());
+ } else {
+ return nullptr;
+ }
+}
+
+void rocksdb_approximate_sizes(
+ rocksdb_t* db,
+ int num_ranges,
+ const char* const* range_start_key, const size_t* range_start_key_len,
+ const char* const* range_limit_key, const size_t* range_limit_key_len,
+ uint64_t* sizes) {
+ Range* ranges = new Range[num_ranges];
+ for (int i = 0; i < num_ranges; i++) {
+ ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]);
+ ranges[i].limit = Slice(range_limit_key[i], range_limit_key_len[i]);
+ }
+ db->rep->GetApproximateSizes(ranges, num_ranges, sizes);
+ delete[] ranges;
+}
+
+void rocksdb_approximate_sizes_cf(
+ rocksdb_t* db,
+ rocksdb_column_family_handle_t* column_family,
+ int num_ranges,
+ const char* const* range_start_key, const size_t* range_start_key_len,
+ const char* const* range_limit_key, const size_t* range_limit_key_len,
+ uint64_t* sizes) {
+ Range* ranges = new Range[num_ranges];
+ for (int i = 0; i < num_ranges; i++) {
+ ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]);
+ ranges[i].limit = Slice(range_limit_key[i], range_limit_key_len[i]);
+ }
+ db->rep->GetApproximateSizes(column_family->rep, ranges, num_ranges, sizes);
+ delete[] ranges;
+}
+
+void rocksdb_delete_file(
+ rocksdb_t* db,
+ const char* name) {
+ db->rep->DeleteFile(name);
+}
+
+const rocksdb_livefiles_t* rocksdb_livefiles(
+ rocksdb_t* db) {
+ rocksdb_livefiles_t* result = new rocksdb_livefiles_t;
+ db->rep->GetLiveFilesMetaData(&result->rep);
+ return result;
+}
+
+void rocksdb_compact_range(
+ rocksdb_t* db,
+ const char* start_key, size_t start_key_len,
+ const char* limit_key, size_t limit_key_len) {
+ Slice a, b;
+ db->rep->CompactRange(
+ CompactRangeOptions(),
+ // Pass nullptr Slice if corresponding "const char*" is nullptr
+ (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr),
+ (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr));
+}
+
+void rocksdb_compact_range_cf(
+ rocksdb_t* db,
+ rocksdb_column_family_handle_t* column_family,
+ const char* start_key, size_t start_key_len,
+ const char* limit_key, size_t limit_key_len) {
+ Slice a, b;
+ db->rep->CompactRange(
+ CompactRangeOptions(), column_family->rep,
+ // Pass nullptr Slice if corresponding "const char*" is nullptr
+ (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr),
+ (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr));
+}
+
+void rocksdb_compact_range_opt(rocksdb_t* db, rocksdb_compactoptions_t* opt,
+ const char* start_key, size_t start_key_len,
+ const char* limit_key, size_t limit_key_len) {
+ Slice a, b;
+ db->rep->CompactRange(
+ opt->rep,
+ // Pass nullptr Slice if corresponding "const char*" is nullptr
+ (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr),
+ (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr));
+}
+
+void rocksdb_compact_range_cf_opt(rocksdb_t* db,
+ rocksdb_column_family_handle_t* column_family,
+ rocksdb_compactoptions_t* opt,
+ const char* start_key, size_t start_key_len,
+ const char* limit_key, size_t limit_key_len) {
+ Slice a, b;
+ db->rep->CompactRange(
+ opt->rep, column_family->rep,
+ // Pass nullptr Slice if corresponding "const char*" is nullptr
+ (start_key ? (a = Slice(start_key, start_key_len), &a) : nullptr),
+ (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr));
+}
+
+void rocksdb_flush(
+ rocksdb_t* db,
+ const rocksdb_flushoptions_t* options,
+ char** errptr) {
+ SaveError(errptr, db->rep->Flush(options->rep));
+}
+
+void rocksdb_disable_file_deletions(
+ rocksdb_t* db,
+ char** errptr) {
+ SaveError(errptr, db->rep->DisableFileDeletions());
+}
+
+void rocksdb_enable_file_deletions(
+ rocksdb_t* db,
+ unsigned char force,
+ char** errptr) {
+ SaveError(errptr, db->rep->EnableFileDeletions(force));
+}
+
+void rocksdb_destroy_db(
+ const rocksdb_options_t* options,
+ const char* name,
+ char** errptr) {
+ SaveError(errptr, DestroyDB(name, options->rep));
+}
+
+void rocksdb_repair_db(
+ const rocksdb_options_t* options,
+ const char* name,
+ char** errptr) {
+ SaveError(errptr, RepairDB(name, options->rep));
+}
+
+void rocksdb_iter_destroy(rocksdb_iterator_t* iter) {
+ delete iter->rep;
+ delete iter;
+}
+
+unsigned char rocksdb_iter_valid(const rocksdb_iterator_t* iter) {
+ return iter->rep->Valid();
+}
+
+void rocksdb_iter_seek_to_first(rocksdb_iterator_t* iter) {
+ iter->rep->SeekToFirst();
+}
+
+void rocksdb_iter_seek_to_last(rocksdb_iterator_t* iter) {
+ iter->rep->SeekToLast();
+}
+
+void rocksdb_iter_seek(rocksdb_iterator_t* iter, const char* k, size_t klen) {
+ iter->rep->Seek(Slice(k, klen));
+}
+
+void rocksdb_iter_seek_for_prev(rocksdb_iterator_t* iter, const char* k,
+ size_t klen) {
+ iter->rep->SeekForPrev(Slice(k, klen));
+}
+
+void rocksdb_iter_next(rocksdb_iterator_t* iter) {
+ iter->rep->Next();
+}
+
+void rocksdb_iter_prev(rocksdb_iterator_t* iter) {
+ iter->rep->Prev();
+}
+
+const char* rocksdb_iter_key(const rocksdb_iterator_t* iter, size_t* klen) {
+ Slice s = iter->rep->key();
+ *klen = s.size();
+ return s.data();
+}
+
+const char* rocksdb_iter_value(const rocksdb_iterator_t* iter, size_t* vlen) {
+ Slice s = iter->rep->value();
+ *vlen = s.size();
+ return s.data();
+}
+
+void rocksdb_iter_get_error(const rocksdb_iterator_t* iter, char** errptr) {
+ SaveError(errptr, iter->rep->status());
+}
+
+rocksdb_writebatch_t* rocksdb_writebatch_create() {
+ return new rocksdb_writebatch_t;
+}
+
+rocksdb_writebatch_t* rocksdb_writebatch_create_from(const char* rep,
+ size_t size) {
+ rocksdb_writebatch_t* b = new rocksdb_writebatch_t;
+ b->rep = WriteBatch(std::string(rep, size));
+ return b;
+}
+
+void rocksdb_writebatch_destroy(rocksdb_writebatch_t* b) {
+ delete b;
+}
+
+void rocksdb_writebatch_clear(rocksdb_writebatch_t* b) {
+ b->rep.Clear();
+}
+
+int rocksdb_writebatch_count(rocksdb_writebatch_t* b) {
+ return b->rep.Count();
+}
+
+void rocksdb_writebatch_put(
+ rocksdb_writebatch_t* b,
+ const char* key, size_t klen,
+ const char* val, size_t vlen) {
+ b->rep.Put(Slice(key, klen), Slice(val, vlen));
+}
+
+void rocksdb_writebatch_put_cf(
+ rocksdb_writebatch_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t klen,
+ const char* val, size_t vlen) {
+ b->rep.Put(column_family->rep, Slice(key, klen), Slice(val, vlen));
+}
+
+void rocksdb_writebatch_putv(
+ rocksdb_writebatch_t* b,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes,
+ int num_values, const char* const* values_list,
+ const size_t* values_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ std::vector<Slice> value_slices(num_values);
+ for (int i = 0; i < num_values; i++) {
+ value_slices[i] = Slice(values_list[i], values_list_sizes[i]);
+ }
+ b->rep.Put(SliceParts(key_slices.data(), num_keys),
+ SliceParts(value_slices.data(), num_values));
+}
+
+void rocksdb_writebatch_putv_cf(
+ rocksdb_writebatch_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes,
+ int num_values, const char* const* values_list,
+ const size_t* values_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ std::vector<Slice> value_slices(num_values);
+ for (int i = 0; i < num_values; i++) {
+ value_slices[i] = Slice(values_list[i], values_list_sizes[i]);
+ }
+ b->rep.Put(column_family->rep, SliceParts(key_slices.data(), num_keys),
+ SliceParts(value_slices.data(), num_values));
+}
+
+void rocksdb_writebatch_merge(
+ rocksdb_writebatch_t* b,
+ const char* key, size_t klen,
+ const char* val, size_t vlen) {
+ b->rep.Merge(Slice(key, klen), Slice(val, vlen));
+}
+
+void rocksdb_writebatch_merge_cf(
+ rocksdb_writebatch_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t klen,
+ const char* val, size_t vlen) {
+ b->rep.Merge(column_family->rep, Slice(key, klen), Slice(val, vlen));
+}
+
+void rocksdb_writebatch_mergev(
+ rocksdb_writebatch_t* b,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes,
+ int num_values, const char* const* values_list,
+ const size_t* values_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ std::vector<Slice> value_slices(num_values);
+ for (int i = 0; i < num_values; i++) {
+ value_slices[i] = Slice(values_list[i], values_list_sizes[i]);
+ }
+ b->rep.Merge(SliceParts(key_slices.data(), num_keys),
+ SliceParts(value_slices.data(), num_values));
+}
+
+void rocksdb_writebatch_mergev_cf(
+ rocksdb_writebatch_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes,
+ int num_values, const char* const* values_list,
+ const size_t* values_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ std::vector<Slice> value_slices(num_values);
+ for (int i = 0; i < num_values; i++) {
+ value_slices[i] = Slice(values_list[i], values_list_sizes[i]);
+ }
+ b->rep.Merge(column_family->rep, SliceParts(key_slices.data(), num_keys),
+ SliceParts(value_slices.data(), num_values));
+}
+
+void rocksdb_writebatch_delete(
+ rocksdb_writebatch_t* b,
+ const char* key, size_t klen) {
+ b->rep.Delete(Slice(key, klen));
+}
+
+void rocksdb_writebatch_delete_cf(
+ rocksdb_writebatch_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t klen) {
+ b->rep.Delete(column_family->rep, Slice(key, klen));
+}
+
+void rocksdb_writebatch_deletev(
+ rocksdb_writebatch_t* b,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ b->rep.Delete(SliceParts(key_slices.data(), num_keys));
+}
+
+void rocksdb_writebatch_deletev_cf(
+ rocksdb_writebatch_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ b->rep.Delete(column_family->rep, SliceParts(key_slices.data(), num_keys));
+}
+
+void rocksdb_writebatch_delete_range(rocksdb_writebatch_t* b,
+ const char* start_key,
+ size_t start_key_len, const char* end_key,
+ size_t end_key_len) {
+ b->rep.DeleteRange(Slice(start_key, start_key_len),
+ Slice(end_key, end_key_len));
+}
+
+void rocksdb_writebatch_delete_range_cf(
+ rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family,
+ const char* start_key, size_t start_key_len, const char* end_key,
+ size_t end_key_len) {
+ b->rep.DeleteRange(column_family->rep, Slice(start_key, start_key_len),
+ Slice(end_key, end_key_len));
+}
+
+void rocksdb_writebatch_delete_rangev(rocksdb_writebatch_t* b, int num_keys,
+ const char* const* start_keys_list,
+ const size_t* start_keys_list_sizes,
+ const char* const* end_keys_list,
+ const size_t* end_keys_list_sizes) {
+ std::vector<Slice> start_key_slices(num_keys);
+ std::vector<Slice> end_key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ start_key_slices[i] = Slice(start_keys_list[i], start_keys_list_sizes[i]);
+ end_key_slices[i] = Slice(end_keys_list[i], end_keys_list_sizes[i]);
+ }
+ b->rep.DeleteRange(SliceParts(start_key_slices.data(), num_keys),
+ SliceParts(end_key_slices.data(), num_keys));
+}
+
+void rocksdb_writebatch_delete_rangev_cf(
+ rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family,
+ int num_keys, const char* const* start_keys_list,
+ const size_t* start_keys_list_sizes, const char* const* end_keys_list,
+ const size_t* end_keys_list_sizes) {
+ std::vector<Slice> start_key_slices(num_keys);
+ std::vector<Slice> end_key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ start_key_slices[i] = Slice(start_keys_list[i], start_keys_list_sizes[i]);
+ end_key_slices[i] = Slice(end_keys_list[i], end_keys_list_sizes[i]);
+ }
+ b->rep.DeleteRange(column_family->rep,
+ SliceParts(start_key_slices.data(), num_keys),
+ SliceParts(end_key_slices.data(), num_keys));
+}
+
+void rocksdb_writebatch_put_log_data(
+ rocksdb_writebatch_t* b,
+ const char* blob, size_t len) {
+ b->rep.PutLogData(Slice(blob, len));
+}
+
+void rocksdb_writebatch_iterate(
+ rocksdb_writebatch_t* b,
+ void* state,
+ void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
+ void (*deleted)(void*, const char* k, size_t klen)) {
+ class H : public WriteBatch::Handler {
+ public:
+ void* state_;
+ void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+ void (*deleted_)(void*, const char* k, size_t klen);
+ virtual void Put(const Slice& key, const Slice& value) override {
+ (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+ }
+ virtual void Delete(const Slice& key) override {
+ (*deleted_)(state_, key.data(), key.size());
+ }
+ };
+ H handler;
+ handler.state_ = state;
+ handler.put_ = put;
+ handler.deleted_ = deleted;
+ b->rep.Iterate(&handler);
+}
+
+const char* rocksdb_writebatch_data(rocksdb_writebatch_t* b, size_t* size) {
+ *size = b->rep.GetDataSize();
+ return b->rep.Data().c_str();
+}
+
+void rocksdb_writebatch_set_save_point(rocksdb_writebatch_t* b) {
+ b->rep.SetSavePoint();
+}
+
+void rocksdb_writebatch_rollback_to_save_point(rocksdb_writebatch_t* b,
+ char** errptr) {
+ SaveError(errptr, b->rep.RollbackToSavePoint());
+}
+
+void rocksdb_writebatch_pop_save_point(rocksdb_writebatch_t* b, char** errptr) {
+ SaveError(errptr, b->rep.PopSavePoint());
+}
+
+rocksdb_writebatch_wi_t* rocksdb_writebatch_wi_create(size_t reserved_bytes, unsigned char overwrite_key) {
+ rocksdb_writebatch_wi_t* b = new rocksdb_writebatch_wi_t;
+ b->rep = new WriteBatchWithIndex(BytewiseComparator(), reserved_bytes, overwrite_key);
+ return b;
+}
+
+void rocksdb_writebatch_wi_destroy(rocksdb_writebatch_wi_t* b) {
+ if (b->rep) {
+ delete b->rep;
+ }
+ delete b;
+}
+
+void rocksdb_writebatch_wi_clear(rocksdb_writebatch_wi_t* b) {
+ b->rep->Clear();
+}
+
+int rocksdb_writebatch_wi_count(rocksdb_writebatch_wi_t* b) {
+ return b->rep->GetWriteBatch()->Count();
+}
+
+void rocksdb_writebatch_wi_put(
+ rocksdb_writebatch_wi_t* b,
+ const char* key, size_t klen,
+ const char* val, size_t vlen) {
+ b->rep->Put(Slice(key, klen), Slice(val, vlen));
+}
+
+void rocksdb_writebatch_wi_put_cf(
+ rocksdb_writebatch_wi_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t klen,
+ const char* val, size_t vlen) {
+ b->rep->Put(column_family->rep, Slice(key, klen), Slice(val, vlen));
+}
+
+void rocksdb_writebatch_wi_putv(
+ rocksdb_writebatch_wi_t* b,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes,
+ int num_values, const char* const* values_list,
+ const size_t* values_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ std::vector<Slice> value_slices(num_values);
+ for (int i = 0; i < num_values; i++) {
+ value_slices[i] = Slice(values_list[i], values_list_sizes[i]);
+ }
+ b->rep->Put(SliceParts(key_slices.data(), num_keys),
+ SliceParts(value_slices.data(), num_values));
+}
+
+void rocksdb_writebatch_wi_putv_cf(
+ rocksdb_writebatch_wi_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes,
+ int num_values, const char* const* values_list,
+ const size_t* values_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ std::vector<Slice> value_slices(num_values);
+ for (int i = 0; i < num_values; i++) {
+ value_slices[i] = Slice(values_list[i], values_list_sizes[i]);
+ }
+ b->rep->Put(column_family->rep, SliceParts(key_slices.data(), num_keys),
+ SliceParts(value_slices.data(), num_values));
+}
+
+void rocksdb_writebatch_wi_merge(
+ rocksdb_writebatch_wi_t* b,
+ const char* key, size_t klen,
+ const char* val, size_t vlen) {
+ b->rep->Merge(Slice(key, klen), Slice(val, vlen));
+}
+
+void rocksdb_writebatch_wi_merge_cf(
+ rocksdb_writebatch_wi_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t klen,
+ const char* val, size_t vlen) {
+ b->rep->Merge(column_family->rep, Slice(key, klen), Slice(val, vlen));
+}
+
+void rocksdb_writebatch_wi_mergev(
+ rocksdb_writebatch_wi_t* b,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes,
+ int num_values, const char* const* values_list,
+ const size_t* values_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ std::vector<Slice> value_slices(num_values);
+ for (int i = 0; i < num_values; i++) {
+ value_slices[i] = Slice(values_list[i], values_list_sizes[i]);
+ }
+ b->rep->Merge(SliceParts(key_slices.data(), num_keys),
+ SliceParts(value_slices.data(), num_values));
+}
+
+void rocksdb_writebatch_wi_mergev_cf(
+ rocksdb_writebatch_wi_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes,
+ int num_values, const char* const* values_list,
+ const size_t* values_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ std::vector<Slice> value_slices(num_values);
+ for (int i = 0; i < num_values; i++) {
+ value_slices[i] = Slice(values_list[i], values_list_sizes[i]);
+ }
+ b->rep->Merge(column_family->rep, SliceParts(key_slices.data(), num_keys),
+ SliceParts(value_slices.data(), num_values));
+}
+
+void rocksdb_writebatch_wi_delete(
+ rocksdb_writebatch_wi_t* b,
+ const char* key, size_t klen) {
+ b->rep->Delete(Slice(key, klen));
+}
+
+void rocksdb_writebatch_wi_delete_cf(
+ rocksdb_writebatch_wi_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t klen) {
+ b->rep->Delete(column_family->rep, Slice(key, klen));
+}
+
+void rocksdb_writebatch_wi_deletev(
+ rocksdb_writebatch_wi_t* b,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ b->rep->Delete(SliceParts(key_slices.data(), num_keys));
+}
+
+void rocksdb_writebatch_wi_deletev_cf(
+ rocksdb_writebatch_wi_t* b,
+ rocksdb_column_family_handle_t* column_family,
+ int num_keys, const char* const* keys_list,
+ const size_t* keys_list_sizes) {
+ std::vector<Slice> key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ key_slices[i] = Slice(keys_list[i], keys_list_sizes[i]);
+ }
+ b->rep->Delete(column_family->rep, SliceParts(key_slices.data(), num_keys));
+}
+
+void rocksdb_writebatch_wi_delete_range(rocksdb_writebatch_wi_t* b,
+ const char* start_key,
+ size_t start_key_len, const char* end_key,
+ size_t end_key_len) {
+ b->rep->DeleteRange(Slice(start_key, start_key_len),
+ Slice(end_key, end_key_len));
+}
+
+void rocksdb_writebatch_wi_delete_range_cf(
+ rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family,
+ const char* start_key, size_t start_key_len, const char* end_key,
+ size_t end_key_len) {
+ b->rep->DeleteRange(column_family->rep, Slice(start_key, start_key_len),
+ Slice(end_key, end_key_len));
+}
+
+void rocksdb_writebatch_wi_delete_rangev(rocksdb_writebatch_wi_t* b, int num_keys,
+ const char* const* start_keys_list,
+ const size_t* start_keys_list_sizes,
+ const char* const* end_keys_list,
+ const size_t* end_keys_list_sizes) {
+ std::vector<Slice> start_key_slices(num_keys);
+ std::vector<Slice> end_key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ start_key_slices[i] = Slice(start_keys_list[i], start_keys_list_sizes[i]);
+ end_key_slices[i] = Slice(end_keys_list[i], end_keys_list_sizes[i]);
+ }
+ b->rep->DeleteRange(SliceParts(start_key_slices.data(), num_keys),
+ SliceParts(end_key_slices.data(), num_keys));
+}
+
+void rocksdb_writebatch_wi_delete_rangev_cf(
+ rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family,
+ int num_keys, const char* const* start_keys_list,
+ const size_t* start_keys_list_sizes, const char* const* end_keys_list,
+ const size_t* end_keys_list_sizes) {
+ std::vector<Slice> start_key_slices(num_keys);
+ std::vector<Slice> end_key_slices(num_keys);
+ for (int i = 0; i < num_keys; i++) {
+ start_key_slices[i] = Slice(start_keys_list[i], start_keys_list_sizes[i]);
+ end_key_slices[i] = Slice(end_keys_list[i], end_keys_list_sizes[i]);
+ }
+ b->rep->DeleteRange(column_family->rep,
+ SliceParts(start_key_slices.data(), num_keys),
+ SliceParts(end_key_slices.data(), num_keys));
+}
+
+void rocksdb_writebatch_wi_put_log_data(
+ rocksdb_writebatch_wi_t* b,
+ const char* blob, size_t len) {
+ b->rep->PutLogData(Slice(blob, len));
+}
+
+void rocksdb_writebatch_wi_iterate(
+ rocksdb_writebatch_wi_t* b,
+ void* state,
+ void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
+ void (*deleted)(void*, const char* k, size_t klen)) {
+ class H : public WriteBatch::Handler {
+ public:
+ void* state_;
+ void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+ void (*deleted_)(void*, const char* k, size_t klen);
+ virtual void Put(const Slice& key, const Slice& value) override {
+ (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+ }
+ virtual void Delete(const Slice& key) override {
+ (*deleted_)(state_, key.data(), key.size());
+ }
+ };
+ H handler;
+ handler.state_ = state;
+ handler.put_ = put;
+ handler.deleted_ = deleted;
+ b->rep->GetWriteBatch()->Iterate(&handler);
+}
+
+const char* rocksdb_writebatch_wi_data(rocksdb_writebatch_wi_t* b, size_t* size) {
+ WriteBatch* wb = b->rep->GetWriteBatch();
+ *size = wb->GetDataSize();
+ return wb->Data().c_str();
+}
+
+void rocksdb_writebatch_wi_set_save_point(rocksdb_writebatch_wi_t* b) {
+ b->rep->SetSavePoint();
+}
+
+void rocksdb_writebatch_wi_rollback_to_save_point(rocksdb_writebatch_wi_t* b,
+ char** errptr) {
+ SaveError(errptr, b->rep->RollbackToSavePoint());
+}
+
+rocksdb_iterator_t* rocksdb_writebatch_wi_create_iterator_with_base(
+ rocksdb_writebatch_wi_t* wbwi,
+ rocksdb_iterator_t* base_iterator) {
+ rocksdb_iterator_t* result = new rocksdb_iterator_t;
+ result->rep = wbwi->rep->NewIteratorWithBase(base_iterator->rep);
+ delete base_iterator;
+ return result;
+}
+
+rocksdb_iterator_t* rocksdb_writebatch_wi_create_iterator_with_base_cf(
+ rocksdb_writebatch_wi_t* wbwi,
+ rocksdb_iterator_t* base_iterator,
+ rocksdb_column_family_handle_t* column_family) {
+ rocksdb_iterator_t* result = new rocksdb_iterator_t;
+ result->rep = wbwi->rep->NewIteratorWithBase(column_family->rep, base_iterator->rep);
+ delete base_iterator;
+ return result;
+}
+
+char* rocksdb_writebatch_wi_get_from_batch(
+ rocksdb_writebatch_wi_t* wbwi,
+ const rocksdb_options_t* options,
+ const char* key, size_t keylen,
+ size_t* vallen,
+ char** errptr) {
+ char* result = nullptr;
+ std::string tmp;
+ Status s = wbwi->rep->GetFromBatch(options->rep, Slice(key, keylen), &tmp);
+ if (s.ok()) {
+ *vallen = tmp.size();
+ result = CopyString(tmp);
+ } else {
+ *vallen = 0;
+ if (!s.IsNotFound()) {
+ SaveError(errptr, s);
+ }
+ }
+ return result;
+}
+
+char* rocksdb_writebatch_wi_get_from_batch_cf(
+ rocksdb_writebatch_wi_t* wbwi,
+ const rocksdb_options_t* options,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t keylen,
+ size_t* vallen,
+ char** errptr) {
+ char* result = nullptr;
+ std::string tmp;
+ Status s = wbwi->rep->GetFromBatch(column_family->rep, options->rep,
+ Slice(key, keylen), &tmp);
+ if (s.ok()) {
+ *vallen = tmp.size();
+ result = CopyString(tmp);
+ } else {
+ *vallen = 0;
+ if (!s.IsNotFound()) {
+ SaveError(errptr, s);
+ }
+ }
+ return result;
+}
+
+char* rocksdb_writebatch_wi_get_from_batch_and_db(
+ rocksdb_writebatch_wi_t* wbwi,
+ rocksdb_t* db,
+ const rocksdb_readoptions_t* options,
+ const char* key, size_t keylen,
+ size_t* vallen,
+ char** errptr) {
+ char* result = nullptr;
+ std::string tmp;
+ Status s = wbwi->rep->GetFromBatchAndDB(db->rep, options->rep, Slice(key, keylen), &tmp);
+ if (s.ok()) {
+ *vallen = tmp.size();
+ result = CopyString(tmp);
+ } else {
+ *vallen = 0;
+ if (!s.IsNotFound()) {
+ SaveError(errptr, s);
+ }
+ }
+ return result;
+}
+
+char* rocksdb_writebatch_wi_get_from_batch_and_db_cf(
+ rocksdb_writebatch_wi_t* wbwi,
+ rocksdb_t* db,
+ const rocksdb_readoptions_t* options,
+ rocksdb_column_family_handle_t* column_family,
+ const char* key, size_t keylen,
+ size_t* vallen,
+ char** errptr) {
+ char* result = nullptr;
+ std::string tmp;
+ Status s = wbwi->rep->GetFromBatchAndDB(db->rep, options->rep, column_family->rep,
+ Slice(key, keylen), &tmp);
+ if (s.ok()) {
+ *vallen = tmp.size();
+ result = CopyString(tmp);
+ } else {
+ *vallen = 0;
+ if (!s.IsNotFound()) {
+ SaveError(errptr, s);
+ }
+ }
+ return result;
+}
+
+void rocksdb_write_writebatch_wi(
+ rocksdb_t* db,
+ const rocksdb_writeoptions_t* options,
+ rocksdb_writebatch_wi_t* wbwi,
+ char** errptr) {
+ WriteBatch* wb = wbwi->rep->GetWriteBatch();
+ SaveError(errptr, db->rep->Write(options->rep, wb));
+}
+
+rocksdb_block_based_table_options_t*
+rocksdb_block_based_options_create() {
+ return new rocksdb_block_based_table_options_t;
+}
+
+void rocksdb_block_based_options_destroy(
+ rocksdb_block_based_table_options_t* options) {
+ delete options;
+}
+
+void rocksdb_block_based_options_set_block_size(
+ rocksdb_block_based_table_options_t* options, size_t block_size) {
+ options->rep.block_size = block_size;
+}
+
+void rocksdb_block_based_options_set_block_size_deviation(
+ rocksdb_block_based_table_options_t* options, int block_size_deviation) {
+ options->rep.block_size_deviation = block_size_deviation;
+}
+
+void rocksdb_block_based_options_set_block_restart_interval(
+ rocksdb_block_based_table_options_t* options, int block_restart_interval) {
+ options->rep.block_restart_interval = block_restart_interval;
+}
+
+void rocksdb_block_based_options_set_filter_policy(
+ rocksdb_block_based_table_options_t* options,
+ rocksdb_filterpolicy_t* filter_policy) {
+ options->rep.filter_policy.reset(filter_policy);
+}
+
+void rocksdb_block_based_options_set_no_block_cache(
+ rocksdb_block_based_table_options_t* options,
+ unsigned char no_block_cache) {
+ options->rep.no_block_cache = no_block_cache;
+}
+
+void rocksdb_block_based_options_set_block_cache(
+ rocksdb_block_based_table_options_t* options,
+ rocksdb_cache_t* block_cache) {
+ if (block_cache) {
+ options->rep.block_cache = block_cache->rep;
+ }
+}
+
+void rocksdb_block_based_options_set_block_cache_compressed(
+ rocksdb_block_based_table_options_t* options,
+ rocksdb_cache_t* block_cache_compressed) {
+ if (block_cache_compressed) {
+ options->rep.block_cache_compressed = block_cache_compressed->rep;
+ }
+}
+
+void rocksdb_block_based_options_set_whole_key_filtering(
+ rocksdb_block_based_table_options_t* options, unsigned char v) {
+ options->rep.whole_key_filtering = v;
+}
+
+void rocksdb_block_based_options_set_format_version(
+ rocksdb_block_based_table_options_t* options, int v) {
+ options->rep.format_version = v;
+}
+
+void rocksdb_block_based_options_set_index_type(
+ rocksdb_block_based_table_options_t* options, int v) {
+ options->rep.index_type = static_cast<BlockBasedTableOptions::IndexType>(v);
+}
+
+void rocksdb_block_based_options_set_hash_index_allow_collision(
+ rocksdb_block_based_table_options_t* options, unsigned char v) {
+ options->rep.hash_index_allow_collision = v;
+}
+
+void rocksdb_block_based_options_set_cache_index_and_filter_blocks(
+ rocksdb_block_based_table_options_t* options, unsigned char v) {
+ options->rep.cache_index_and_filter_blocks = v;
+}
+
+void rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(
+ rocksdb_block_based_table_options_t* options, unsigned char v) {
+ options->rep.pin_l0_filter_and_index_blocks_in_cache = v;
+}
+
+void rocksdb_options_set_block_based_table_factory(
+ rocksdb_options_t *opt,
+ rocksdb_block_based_table_options_t* table_options) {
+ if (table_options) {
+ opt->rep.table_factory.reset(
+ rocksdb::NewBlockBasedTableFactory(table_options->rep));
+ }
+}
+
+
+rocksdb_cuckoo_table_options_t*
+rocksdb_cuckoo_options_create() {
+ return new rocksdb_cuckoo_table_options_t;
+}
+
+void rocksdb_cuckoo_options_destroy(
+ rocksdb_cuckoo_table_options_t* options) {
+ delete options;
+}
+
+void rocksdb_cuckoo_options_set_hash_ratio(
+ rocksdb_cuckoo_table_options_t* options, double v) {
+ options->rep.hash_table_ratio = v;
+}
+
+void rocksdb_cuckoo_options_set_max_search_depth(
+ rocksdb_cuckoo_table_options_t* options, uint32_t v) {
+ options->rep.max_search_depth = v;
+}
+
+void rocksdb_cuckoo_options_set_cuckoo_block_size(
+ rocksdb_cuckoo_table_options_t* options, uint32_t v) {
+ options->rep.cuckoo_block_size = v;
+}
+
+void rocksdb_cuckoo_options_set_identity_as_first_hash(
+ rocksdb_cuckoo_table_options_t* options, unsigned char v) {
+ options->rep.identity_as_first_hash = v;
+}
+
+void rocksdb_cuckoo_options_set_use_module_hash(
+ rocksdb_cuckoo_table_options_t* options, unsigned char v) {
+ options->rep.use_module_hash = v;
+}
+
+void rocksdb_options_set_cuckoo_table_factory(
+ rocksdb_options_t *opt,
+ rocksdb_cuckoo_table_options_t* table_options) {
+ if (table_options) {
+ opt->rep.table_factory.reset(
+ rocksdb::NewCuckooTableFactory(table_options->rep));
+ }
+}
+
+void rocksdb_set_options(
+ rocksdb_t* db, int count, const char* const keys[], const char* const values[], char** errptr) {
+ std::unordered_map<std::string, std::string> options_map;
+ for (int i=0; i<count; i++)
+ options_map[keys[i]] = values[i];
+ SaveError(errptr,
+ db->rep->SetOptions(options_map));
+ }
+
+rocksdb_options_t* rocksdb_options_create() {
+ return new rocksdb_options_t;
+}
+
+void rocksdb_options_destroy(rocksdb_options_t* options) {
+ delete options;
+}
+
+void rocksdb_options_increase_parallelism(
+ rocksdb_options_t* opt, int total_threads) {
+ opt->rep.IncreaseParallelism(total_threads);
+}
+
+void rocksdb_options_optimize_for_point_lookup(
+ rocksdb_options_t* opt, uint64_t block_cache_size_mb) {
+ opt->rep.OptimizeForPointLookup(block_cache_size_mb);
+}
+
+void rocksdb_options_optimize_level_style_compaction(
+ rocksdb_options_t* opt, uint64_t memtable_memory_budget) {
+ opt->rep.OptimizeLevelStyleCompaction(memtable_memory_budget);
+}
+
+void rocksdb_options_optimize_universal_style_compaction(
+ rocksdb_options_t* opt, uint64_t memtable_memory_budget) {
+ opt->rep.OptimizeUniversalStyleCompaction(memtable_memory_budget);
+}
+
+void rocksdb_options_set_compaction_filter(
+ rocksdb_options_t* opt,
+ rocksdb_compactionfilter_t* filter) {
+ opt->rep.compaction_filter = filter;
+}
+
+void rocksdb_options_set_compaction_filter_factory(
+ rocksdb_options_t* opt, rocksdb_compactionfilterfactory_t* factory) {
+ opt->rep.compaction_filter_factory =
+ std::shared_ptr<CompactionFilterFactory>(factory);
+}
+
+void rocksdb_options_compaction_readahead_size(
+ rocksdb_options_t* opt, size_t s) {
+ opt->rep.compaction_readahead_size = s;
+}
+
+void rocksdb_options_set_comparator(
+ rocksdb_options_t* opt,
+ rocksdb_comparator_t* cmp) {
+ opt->rep.comparator = cmp;
+}
+
+void rocksdb_options_set_merge_operator(
+ rocksdb_options_t* opt,
+ rocksdb_mergeoperator_t* merge_operator) {
+ opt->rep.merge_operator = std::shared_ptr<MergeOperator>(merge_operator);
+}
+
+
+void rocksdb_options_set_create_if_missing(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.create_if_missing = v;
+}
+
+void rocksdb_options_set_create_missing_column_families(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.create_missing_column_families = v;
+}
+
+void rocksdb_options_set_error_if_exists(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.error_if_exists = v;
+}
+
+void rocksdb_options_set_paranoid_checks(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.paranoid_checks = v;
+}
+
+void rocksdb_options_set_db_paths(rocksdb_options_t* opt,
+ const rocksdb_dbpath_t** dbpath_values,
+ size_t num_paths) {
+ std::vector<DbPath> db_paths(num_paths);
+ for (size_t i = 0; i < num_paths; ++i) {
+ db_paths[i] = dbpath_values[i]->rep;
+ }
+ opt->rep.db_paths = db_paths;
+}
+
+void rocksdb_options_set_env(rocksdb_options_t* opt, rocksdb_env_t* env) {
+ opt->rep.env = (env ? env->rep : nullptr);
+}
+
+void rocksdb_options_set_info_log(rocksdb_options_t* opt, rocksdb_logger_t* l) {
+ if (l) {
+ opt->rep.info_log = l->rep;
+ }
+}
+
+void rocksdb_options_set_info_log_level(
+ rocksdb_options_t* opt, int v) {
+ opt->rep.info_log_level = static_cast<InfoLogLevel>(v);
+}
+
+void rocksdb_options_set_db_write_buffer_size(rocksdb_options_t* opt,
+ size_t s) {
+ opt->rep.db_write_buffer_size = s;
+}
+
+void rocksdb_options_set_write_buffer_size(rocksdb_options_t* opt, size_t s) {
+ opt->rep.write_buffer_size = s;
+}
+
+void rocksdb_options_set_max_open_files(rocksdb_options_t* opt, int n) {
+ opt->rep.max_open_files = n;
+}
+
+void rocksdb_options_set_max_file_opening_threads(rocksdb_options_t* opt, int n) {
+ opt->rep.max_file_opening_threads = n;
+}
+
+void rocksdb_options_set_max_total_wal_size(rocksdb_options_t* opt, uint64_t n) {
+ opt->rep.max_total_wal_size = n;
+}
+
+void rocksdb_options_set_target_file_size_base(
+ rocksdb_options_t* opt, uint64_t n) {
+ opt->rep.target_file_size_base = n;
+}
+
+void rocksdb_options_set_target_file_size_multiplier(
+ rocksdb_options_t* opt, int n) {
+ opt->rep.target_file_size_multiplier = n;
+}
+
+void rocksdb_options_set_max_bytes_for_level_base(
+ rocksdb_options_t* opt, uint64_t n) {
+ opt->rep.max_bytes_for_level_base = n;
+}
+
+void rocksdb_options_set_level_compaction_dynamic_level_bytes(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.level_compaction_dynamic_level_bytes = v;
+}
+
+void rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t* opt,
+ double n) {
+ opt->rep.max_bytes_for_level_multiplier = n;
+}
+
+void rocksdb_options_set_max_compaction_bytes(rocksdb_options_t* opt,
+ uint64_t n) {
+ opt->rep.max_compaction_bytes = n;
+}
+
+void rocksdb_options_set_max_bytes_for_level_multiplier_additional(
+ rocksdb_options_t* opt, int* level_values, size_t num_levels) {
+ opt->rep.max_bytes_for_level_multiplier_additional.resize(num_levels);
+ for (size_t i = 0; i < num_levels; ++i) {
+ opt->rep.max_bytes_for_level_multiplier_additional[i] = level_values[i];
+ }
+}
+
+void rocksdb_options_enable_statistics(rocksdb_options_t* opt) {
+ opt->rep.statistics = rocksdb::CreateDBStatistics();
+}
+
+void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, unsigned char val) {
+ opt->rep.skip_stats_update_on_db_open = val;
+}
+
+void rocksdb_options_set_num_levels(rocksdb_options_t* opt, int n) {
+ opt->rep.num_levels = n;
+}
+
+void rocksdb_options_set_level0_file_num_compaction_trigger(
+ rocksdb_options_t* opt, int n) {
+ opt->rep.level0_file_num_compaction_trigger = n;
+}
+
+void rocksdb_options_set_level0_slowdown_writes_trigger(
+ rocksdb_options_t* opt, int n) {
+ opt->rep.level0_slowdown_writes_trigger = n;
+}
+
+void rocksdb_options_set_level0_stop_writes_trigger(
+ rocksdb_options_t* opt, int n) {
+ opt->rep.level0_stop_writes_trigger = n;
+}
+
+void rocksdb_options_set_max_mem_compaction_level(rocksdb_options_t* opt,
+ int n) {}
+
+void rocksdb_options_set_wal_recovery_mode(rocksdb_options_t* opt,int mode) {
+ opt->rep.wal_recovery_mode = static_cast<WALRecoveryMode>(mode);
+}
+
+void rocksdb_options_set_compression(rocksdb_options_t* opt, int t) {
+ opt->rep.compression = static_cast<CompressionType>(t);
+}
+
+void rocksdb_options_set_compression_per_level(rocksdb_options_t* opt,
+ int* level_values,
+ size_t num_levels) {
+ opt->rep.compression_per_level.resize(num_levels);
+ for (size_t i = 0; i < num_levels; ++i) {
+ opt->rep.compression_per_level[i] =
+ static_cast<CompressionType>(level_values[i]);
+ }
+}
+
+void rocksdb_options_set_compression_options(rocksdb_options_t* opt, int w_bits,
+ int level, int strategy,
+ int max_dict_bytes) {
+ opt->rep.compression_opts.window_bits = w_bits;
+ opt->rep.compression_opts.level = level;
+ opt->rep.compression_opts.strategy = strategy;
+ opt->rep.compression_opts.max_dict_bytes = max_dict_bytes;
+}
+
+void rocksdb_options_set_prefix_extractor(
+ rocksdb_options_t* opt, rocksdb_slicetransform_t* prefix_extractor) {
+ opt->rep.prefix_extractor.reset(prefix_extractor);
+}
+
+void rocksdb_options_set_use_fsync(
+ rocksdb_options_t* opt, int use_fsync) {
+ opt->rep.use_fsync = use_fsync;
+}
+
+void rocksdb_options_set_db_log_dir(
+ rocksdb_options_t* opt, const char* db_log_dir) {
+ opt->rep.db_log_dir = db_log_dir;
+}
+
+void rocksdb_options_set_wal_dir(
+ rocksdb_options_t* opt, const char* v) {
+ opt->rep.wal_dir = v;
+}
+
+void rocksdb_options_set_WAL_ttl_seconds(rocksdb_options_t* opt, uint64_t ttl) {
+ opt->rep.WAL_ttl_seconds = ttl;
+}
+
+void rocksdb_options_set_WAL_size_limit_MB(
+ rocksdb_options_t* opt, uint64_t limit) {
+ opt->rep.WAL_size_limit_MB = limit;
+}
+
+void rocksdb_options_set_manifest_preallocation_size(
+ rocksdb_options_t* opt, size_t v) {
+ opt->rep.manifest_preallocation_size = v;
+}
+
+// noop
+void rocksdb_options_set_purge_redundant_kvs_while_flush(rocksdb_options_t* opt,
+ unsigned char v) {}
+
+void rocksdb_options_set_use_direct_reads(rocksdb_options_t* opt,
+ unsigned char v) {
+ opt->rep.use_direct_reads = v;
+}
+
+void rocksdb_options_set_use_direct_io_for_flush_and_compaction(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.use_direct_io_for_flush_and_compaction = v;
+}
+
+void rocksdb_options_set_allow_mmap_reads(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.allow_mmap_reads = v;
+}
+
+void rocksdb_options_set_allow_mmap_writes(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.allow_mmap_writes = v;
+}
+
+void rocksdb_options_set_is_fd_close_on_exec(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.is_fd_close_on_exec = v;
+}
+
+void rocksdb_options_set_skip_log_error_on_recovery(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.skip_log_error_on_recovery = v;
+}
+
+void rocksdb_options_set_stats_dump_period_sec(
+ rocksdb_options_t* opt, unsigned int v) {
+ opt->rep.stats_dump_period_sec = v;
+}
+
+void rocksdb_options_set_advise_random_on_open(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.advise_random_on_open = v;
+}
+
+void rocksdb_options_set_access_hint_on_compaction_start(
+ rocksdb_options_t* opt, int v) {
+ switch(v) {
+ case 0:
+ opt->rep.access_hint_on_compaction_start = rocksdb::Options::NONE;
+ break;
+ case 1:
+ opt->rep.access_hint_on_compaction_start = rocksdb::Options::NORMAL;
+ break;
+ case 2:
+ opt->rep.access_hint_on_compaction_start = rocksdb::Options::SEQUENTIAL;
+ break;
+ case 3:
+ opt->rep.access_hint_on_compaction_start = rocksdb::Options::WILLNEED;
+ break;
+ }
+}
+
+void rocksdb_options_set_use_adaptive_mutex(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.use_adaptive_mutex = v;
+}
+
+void rocksdb_options_set_bytes_per_sync(
+ rocksdb_options_t* opt, uint64_t v) {
+ opt->rep.bytes_per_sync = v;
+}
+
+void rocksdb_options_set_allow_concurrent_memtable_write(rocksdb_options_t* opt,
+ unsigned char v) {
+ opt->rep.allow_concurrent_memtable_write = v;
+}
+
+void rocksdb_options_set_enable_write_thread_adaptive_yield(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.enable_write_thread_adaptive_yield = v;
+}
+
+void rocksdb_options_set_max_sequential_skip_in_iterations(
+ rocksdb_options_t* opt, uint64_t v) {
+ opt->rep.max_sequential_skip_in_iterations = v;
+}
+
+void rocksdb_options_set_max_write_buffer_number(rocksdb_options_t* opt, int n) {
+ opt->rep.max_write_buffer_number = n;
+}
+
+void rocksdb_options_set_min_write_buffer_number_to_merge(rocksdb_options_t* opt, int n) {
+ opt->rep.min_write_buffer_number_to_merge = n;
+}
+
+void rocksdb_options_set_max_write_buffer_number_to_maintain(
+ rocksdb_options_t* opt, int n) {
+ opt->rep.max_write_buffer_number_to_maintain = n;
+}
+
+void rocksdb_options_set_max_background_compactions(rocksdb_options_t* opt, int n) {
+ opt->rep.max_background_compactions = n;
+}
+
+void rocksdb_options_set_base_background_compactions(rocksdb_options_t* opt,
+ int n) {
+ opt->rep.base_background_compactions = n;
+}
+
+void rocksdb_options_set_max_background_flushes(rocksdb_options_t* opt, int n) {
+ opt->rep.max_background_flushes = n;
+}
+
+void rocksdb_options_set_max_log_file_size(rocksdb_options_t* opt, size_t v) {
+ opt->rep.max_log_file_size = v;
+}
+
+void rocksdb_options_set_log_file_time_to_roll(rocksdb_options_t* opt, size_t v) {
+ opt->rep.log_file_time_to_roll = v;
+}
+
+void rocksdb_options_set_keep_log_file_num(rocksdb_options_t* opt, size_t v) {
+ opt->rep.keep_log_file_num = v;
+}
+
+void rocksdb_options_set_recycle_log_file_num(rocksdb_options_t* opt,
+ size_t v) {
+ opt->rep.recycle_log_file_num = v;
+}
+
+void rocksdb_options_set_soft_rate_limit(rocksdb_options_t* opt, double v) {
+ opt->rep.soft_rate_limit = v;
+}
+
+void rocksdb_options_set_hard_rate_limit(rocksdb_options_t* opt, double v) {
+ opt->rep.hard_rate_limit = v;
+}
+
+void rocksdb_options_set_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt, size_t v) {
+ opt->rep.soft_pending_compaction_bytes_limit = v;
+}
+
+void rocksdb_options_set_hard_pending_compaction_bytes_limit(rocksdb_options_t* opt, size_t v) {
+ opt->rep.hard_pending_compaction_bytes_limit = v;
+}
+
+void rocksdb_options_set_rate_limit_delay_max_milliseconds(
+ rocksdb_options_t* opt, unsigned int v) {
+ opt->rep.rate_limit_delay_max_milliseconds = v;
+}
+
+void rocksdb_options_set_max_manifest_file_size(
+ rocksdb_options_t* opt, size_t v) {
+ opt->rep.max_manifest_file_size = v;
+}
+
+void rocksdb_options_set_table_cache_numshardbits(
+ rocksdb_options_t* opt, int v) {
+ opt->rep.table_cache_numshardbits = v;
+}
+
+void rocksdb_options_set_table_cache_remove_scan_count_limit(
+ rocksdb_options_t* opt, int v) {
+ // this option is deprecated
+}
+
+void rocksdb_options_set_arena_block_size(
+ rocksdb_options_t* opt, size_t v) {
+ opt->rep.arena_block_size = v;
+}
+
+void rocksdb_options_set_disable_auto_compactions(rocksdb_options_t* opt, int disable) {
+ opt->rep.disable_auto_compactions = disable;
+}
+
+void rocksdb_options_set_optimize_filters_for_hits(rocksdb_options_t* opt, int v) {
+ opt->rep.optimize_filters_for_hits = v;
+}
+
+void rocksdb_options_set_delete_obsolete_files_period_micros(
+ rocksdb_options_t* opt, uint64_t v) {
+ opt->rep.delete_obsolete_files_period_micros = v;
+}
+
+void rocksdb_options_prepare_for_bulk_load(rocksdb_options_t* opt) {
+ opt->rep.PrepareForBulkLoad();
+}
+
+void rocksdb_options_set_memtable_vector_rep(rocksdb_options_t *opt) {
+ opt->rep.memtable_factory.reset(new rocksdb::VectorRepFactory);
+}
+
+void rocksdb_options_set_memtable_prefix_bloom_size_ratio(
+ rocksdb_options_t* opt, double v) {
+ opt->rep.memtable_prefix_bloom_size_ratio = v;
+}
+
+void rocksdb_options_set_memtable_huge_page_size(rocksdb_options_t* opt,
+ size_t v) {
+ opt->rep.memtable_huge_page_size = v;
+}
+
+void rocksdb_options_set_hash_skip_list_rep(
+ rocksdb_options_t *opt, size_t bucket_count,
+ int32_t skiplist_height, int32_t skiplist_branching_factor) {
+ rocksdb::MemTableRepFactory* factory = rocksdb::NewHashSkipListRepFactory(
+ bucket_count, skiplist_height, skiplist_branching_factor);
+ opt->rep.memtable_factory.reset(factory);
+}
+
+void rocksdb_options_set_hash_link_list_rep(
+ rocksdb_options_t *opt, size_t bucket_count) {
+ opt->rep.memtable_factory.reset(rocksdb::NewHashLinkListRepFactory(bucket_count));
+}
+
+void rocksdb_options_set_plain_table_factory(
+ rocksdb_options_t *opt, uint32_t user_key_len, int bloom_bits_per_key,
+ double hash_table_ratio, size_t index_sparseness) {
+ rocksdb::PlainTableOptions options;
+ options.user_key_len = user_key_len;
+ options.bloom_bits_per_key = bloom_bits_per_key;
+ options.hash_table_ratio = hash_table_ratio;
+ options.index_sparseness = index_sparseness;
+
+ rocksdb::TableFactory* factory = rocksdb::NewPlainTableFactory(options);
+ opt->rep.table_factory.reset(factory);
+}
+
+void rocksdb_options_set_max_successive_merges(
+ rocksdb_options_t* opt, size_t v) {
+ opt->rep.max_successive_merges = v;
+}
+
+void rocksdb_options_set_bloom_locality(
+ rocksdb_options_t* opt, uint32_t v) {
+ opt->rep.bloom_locality = v;
+}
+
+void rocksdb_options_set_inplace_update_support(
+ rocksdb_options_t* opt, unsigned char v) {
+ opt->rep.inplace_update_support = v;
+}
+
+void rocksdb_options_set_inplace_update_num_locks(
+ rocksdb_options_t* opt, size_t v) {
+ opt->rep.inplace_update_num_locks = v;
+}
+
+void rocksdb_options_set_report_bg_io_stats(
+ rocksdb_options_t* opt, int v) {
+ opt->rep.report_bg_io_stats = v;
+}
+
+void rocksdb_options_set_compaction_style(rocksdb_options_t *opt, int style) {
+ opt->rep.compaction_style = static_cast<rocksdb::CompactionStyle>(style);
+}
+
+void rocksdb_options_set_universal_compaction_options(rocksdb_options_t *opt, rocksdb_universal_compaction_options_t *uco) {
+ opt->rep.compaction_options_universal = *(uco->rep);
+}
+
+void rocksdb_options_set_fifo_compaction_options(
+ rocksdb_options_t* opt,
+ rocksdb_fifo_compaction_options_t* fifo) {
+ opt->rep.compaction_options_fifo = fifo->rep;
+}
+
+char *rocksdb_options_statistics_get_string(rocksdb_options_t *opt) {
+ rocksdb::Statistics *statistics = opt->rep.statistics.get();
+ if (statistics) {
+ return strdup(statistics->ToString().c_str());
+ }
+ return nullptr;
+}
+
+void rocksdb_options_set_ratelimiter(rocksdb_options_t *opt, rocksdb_ratelimiter_t *limiter) {
+ opt->rep.rate_limiter.reset(limiter->rep);
+ limiter->rep = nullptr;
+}
+
+rocksdb_ratelimiter_t* rocksdb_ratelimiter_create(
+ int64_t rate_bytes_per_sec,
+ int64_t refill_period_us,
+ int32_t fairness) {
+ rocksdb_ratelimiter_t* rate_limiter = new rocksdb_ratelimiter_t;
+ rate_limiter->rep = NewGenericRateLimiter(rate_bytes_per_sec,
+ refill_period_us, fairness);
+ return rate_limiter;
+}
+
+void rocksdb_ratelimiter_destroy(rocksdb_ratelimiter_t *limiter) {
+ if (limiter->rep) {
+ delete limiter->rep;
+ }
+ delete limiter;
+}
+
+/*
+TODO:
+DB::OpenForReadOnly
+DB::KeyMayExist
+DB::GetOptions
+DB::GetSortedWalFiles
+DB::GetLatestSequenceNumber
+DB::GetUpdatesSince
+DB::GetDbIdentity
+DB::RunManualCompaction
+custom cache
+table_properties_collectors
+*/
+
+rocksdb_compactionfilter_t* rocksdb_compactionfilter_create(
+ void* state,
+ void (*destructor)(void*),
+ unsigned char (*filter)(
+ void*,
+ int level,
+ const char* key, size_t key_length,
+ const char* existing_value, size_t value_length,
+ char** new_value, size_t *new_value_length,
+ unsigned char* value_changed),
+ const char* (*name)(void*)) {
+ rocksdb_compactionfilter_t* result = new rocksdb_compactionfilter_t;
+ result->state_ = state;
+ result->destructor_ = destructor;
+ result->filter_ = filter;
+ result->ignore_snapshots_ = false;
+ result->name_ = name;
+ return result;
+}
+
+void rocksdb_compactionfilter_set_ignore_snapshots(
+ rocksdb_compactionfilter_t* filter,
+ unsigned char whether_ignore) {
+ filter->ignore_snapshots_ = whether_ignore;
+}
+
+void rocksdb_compactionfilter_destroy(rocksdb_compactionfilter_t* filter) {
+ delete filter;
+}
+
+unsigned char rocksdb_compactionfiltercontext_is_full_compaction(
+ rocksdb_compactionfiltercontext_t* context) {
+ return context->rep.is_full_compaction;
+}
+
+unsigned char rocksdb_compactionfiltercontext_is_manual_compaction(
+ rocksdb_compactionfiltercontext_t* context) {
+ return context->rep.is_manual_compaction;
+}
+
+rocksdb_compactionfilterfactory_t* rocksdb_compactionfilterfactory_create(
+ void* state, void (*destructor)(void*),
+ rocksdb_compactionfilter_t* (*create_compaction_filter)(
+ void*, rocksdb_compactionfiltercontext_t* context),
+ const char* (*name)(void*)) {
+ rocksdb_compactionfilterfactory_t* result =
+ new rocksdb_compactionfilterfactory_t;
+ result->state_ = state;
+ result->destructor_ = destructor;
+ result->create_compaction_filter_ = create_compaction_filter;
+ result->name_ = name;
+ return result;
+}
+
+void rocksdb_compactionfilterfactory_destroy(
+ rocksdb_compactionfilterfactory_t* factory) {
+ delete factory;
+}
+
+rocksdb_comparator_t* rocksdb_comparator_create(
+ void* state,
+ void (*destructor)(void*),
+ int (*compare)(
+ void*,
+ const char* a, size_t alen,
+ const char* b, size_t blen),
+ const char* (*name)(void*)) {
+ rocksdb_comparator_t* result = new rocksdb_comparator_t;
+ result->state_ = state;
+ result->destructor_ = destructor;
+ result->compare_ = compare;
+ result->name_ = name;
+ return result;
+}
+
+void rocksdb_comparator_destroy(rocksdb_comparator_t* cmp) {
+ delete cmp;
+}
+
+rocksdb_filterpolicy_t* rocksdb_filterpolicy_create(
+ void* state,
+ void (*destructor)(void*),
+ char* (*create_filter)(
+ void*,
+ const char* const* key_array, const size_t* key_length_array,
+ int num_keys,
+ size_t* filter_length),
+ unsigned char (*key_may_match)(
+ void*,
+ const char* key, size_t length,
+ const char* filter, size_t filter_length),
+ void (*delete_filter)(
+ void*,
+ const char* filter, size_t filter_length),
+ const char* (*name)(void*)) {
+ rocksdb_filterpolicy_t* result = new rocksdb_filterpolicy_t;
+ result->state_ = state;
+ result->destructor_ = destructor;
+ result->create_ = create_filter;
+ result->key_match_ = key_may_match;
+ result->delete_filter_ = delete_filter;
+ result->name_ = name;
+ return result;
+}
+
+void rocksdb_filterpolicy_destroy(rocksdb_filterpolicy_t* filter) {
+ delete filter;
+}
+
+rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_bloom_format(int bits_per_key, bool original_format) {
+ // Make a rocksdb_filterpolicy_t, but override all of its methods so
+ // they delegate to a NewBloomFilterPolicy() instead of user
+ // supplied C functions.
+ struct Wrapper : public rocksdb_filterpolicy_t {
+ const FilterPolicy* rep_;
+ ~Wrapper() { delete rep_; }
+ const char* Name() const override { return rep_->Name(); }
+ void CreateFilter(const Slice* keys, int n,
+ std::string* dst) const override {
+ return rep_->CreateFilter(keys, n, dst);
+ }
+ bool KeyMayMatch(const Slice& key, const Slice& filter) const override {
+ return rep_->KeyMayMatch(key, filter);
+ }
+ static void DoNothing(void*) { }
+ };
+ Wrapper* wrapper = new Wrapper;
+ wrapper->rep_ = NewBloomFilterPolicy(bits_per_key, original_format);
+ wrapper->state_ = nullptr;
+ wrapper->delete_filter_ = nullptr;
+ wrapper->destructor_ = &Wrapper::DoNothing;
+ return wrapper;
+}
+
+rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_bloom_full(int bits_per_key) {
+ return rocksdb_filterpolicy_create_bloom_format(bits_per_key, false);
+}
+
+rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_bloom(int bits_per_key) {
+ return rocksdb_filterpolicy_create_bloom_format(bits_per_key, true);
+}
+
+rocksdb_mergeoperator_t* rocksdb_mergeoperator_create(
+ void* state, void (*destructor)(void*),
+ char* (*full_merge)(void*, const char* key, size_t key_length,
+ const char* existing_value,
+ size_t existing_value_length,
+ const char* const* operands_list,
+ const size_t* operands_list_length, int num_operands,
+ unsigned char* success, size_t* new_value_length),
+ char* (*partial_merge)(void*, const char* key, size_t key_length,
+ const char* const* operands_list,
+ const size_t* operands_list_length, int num_operands,
+ unsigned char* success, size_t* new_value_length),
+ void (*delete_value)(void*, const char* value, size_t value_length),
+ const char* (*name)(void*)) {
+ rocksdb_mergeoperator_t* result = new rocksdb_mergeoperator_t;
+ result->state_ = state;
+ result->destructor_ = destructor;
+ result->full_merge_ = full_merge;
+ result->partial_merge_ = partial_merge;
+ result->delete_value_ = delete_value;
+ result->name_ = name;
+ return result;
+}
+
+void rocksdb_mergeoperator_destroy(rocksdb_mergeoperator_t* merge_operator) {
+ delete merge_operator;
+}
+
+rocksdb_readoptions_t* rocksdb_readoptions_create() {
+ return new rocksdb_readoptions_t;
+}
+
+void rocksdb_readoptions_destroy(rocksdb_readoptions_t* opt) {
+ delete opt;
+}
+
+void rocksdb_readoptions_set_verify_checksums(
+ rocksdb_readoptions_t* opt,
+ unsigned char v) {
+ opt->rep.verify_checksums = v;
+}
+
+void rocksdb_readoptions_set_fill_cache(
+ rocksdb_readoptions_t* opt, unsigned char v) {
+ opt->rep.fill_cache = v;
+}
+
+void rocksdb_readoptions_set_snapshot(
+ rocksdb_readoptions_t* opt,
+ const rocksdb_snapshot_t* snap) {
+ opt->rep.snapsh
<TRUNCATED>
[16/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/dbformat.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/dbformat.h b/thirdparty/rocksdb/db/dbformat.h
new file mode 100644
index 0000000..d9fd5f3
--- /dev/null
+++ b/thirdparty/rocksdb/db/dbformat.h
@@ -0,0 +1,596 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <stdio.h>
+#include <string>
+#include <utility>
+#include "rocksdb/comparator.h"
+#include "rocksdb/db.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/slice_transform.h"
+#include "rocksdb/table.h"
+#include "rocksdb/types.h"
+#include "util/coding.h"
+#include "util/logging.h"
+
+namespace rocksdb {
+
+class InternalKey;
+
+// Value types encoded as the last component of internal keys.
+// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk
+// data structures.
+// The highest bit of the value type needs to be reserved to SST tables
+// for them to do more flexible encoding.
+enum ValueType : unsigned char {
+ kTypeDeletion = 0x0,
+ kTypeValue = 0x1,
+ kTypeMerge = 0x2,
+ kTypeLogData = 0x3, // WAL only.
+ kTypeColumnFamilyDeletion = 0x4, // WAL only.
+ kTypeColumnFamilyValue = 0x5, // WAL only.
+ kTypeColumnFamilyMerge = 0x6, // WAL only.
+ kTypeSingleDeletion = 0x7,
+ kTypeColumnFamilySingleDeletion = 0x8, // WAL only.
+ kTypeBeginPrepareXID = 0x9, // WAL only.
+ kTypeEndPrepareXID = 0xA, // WAL only.
+ kTypeCommitXID = 0xB, // WAL only.
+ kTypeRollbackXID = 0xC, // WAL only.
+ kTypeNoop = 0xD, // WAL only.
+ kTypeColumnFamilyRangeDeletion = 0xE, // WAL only.
+ kTypeRangeDeletion = 0xF, // meta block
+ kMaxValue = 0x7F // Not used for storing records.
+};
+
+// Defined in dbformat.cc
+extern const ValueType kValueTypeForSeek;
+extern const ValueType kValueTypeForSeekForPrev;
+
+// Checks whether a type is an inline value type
+// (i.e. a type used in memtable skiplist and sst file datablock).
+inline bool IsValueType(ValueType t) {
+ return t <= kTypeMerge || t == kTypeSingleDeletion;
+}
+
+// Checks whether a type is from user operation
+// kTypeRangeDeletion is in meta block so this API is separated from above
+inline bool IsExtendedValueType(ValueType t) {
+ return IsValueType(t) || t == kTypeRangeDeletion;
+}
+
+// We leave eight bits empty at the bottom so a type and sequence#
+// can be packed together into 64-bits.
+static const SequenceNumber kMaxSequenceNumber =
+ ((0x1ull << 56) - 1);
+
+static const SequenceNumber kDisableGlobalSequenceNumber = port::kMaxUint64;
+
+struct ParsedInternalKey {
+ Slice user_key;
+ SequenceNumber sequence;
+ ValueType type;
+
+ ParsedInternalKey()
+ : sequence(kMaxSequenceNumber) // Make code analyzer happy
+ {} // Intentionally left uninitialized (for speed)
+ ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
+ : user_key(u), sequence(seq), type(t) { }
+ std::string DebugString(bool hex = false) const;
+
+ void clear() {
+ user_key.clear();
+ sequence = 0;
+ type = kTypeDeletion;
+ }
+};
+
+// Return the length of the encoding of "key".
+inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) {
+ return key.user_key.size() + 8;
+}
+
+// Pack a sequence number and a ValueType into a uint64_t
+extern uint64_t PackSequenceAndType(uint64_t seq, ValueType t);
+
+// Given the result of PackSequenceAndType, store the sequence number in *seq
+// and the ValueType in *t.
+extern void UnPackSequenceAndType(uint64_t packed, uint64_t* seq, ValueType* t);
+
+// Append the serialization of "key" to *result.
+extern void AppendInternalKey(std::string* result,
+ const ParsedInternalKey& key);
+// Serialized internal key consists of user key followed by footer.
+// This function appends the footer to *result, assuming that *result already
+// contains the user key at the end.
+extern void AppendInternalKeyFooter(std::string* result, SequenceNumber s,
+ ValueType t);
+
+// Attempt to parse an internal key from "internal_key". On success,
+// stores the parsed data in "*result", and returns true.
+//
+// On error, returns false, leaves "*result" in an undefined state.
+extern bool ParseInternalKey(const Slice& internal_key,
+ ParsedInternalKey* result);
+
+// Returns the user key portion of an internal key.
+inline Slice ExtractUserKey(const Slice& internal_key) {
+ assert(internal_key.size() >= 8);
+ return Slice(internal_key.data(), internal_key.size() - 8);
+}
+
+inline ValueType ExtractValueType(const Slice& internal_key) {
+ assert(internal_key.size() >= 8);
+ const size_t n = internal_key.size();
+ uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
+ unsigned char c = num & 0xff;
+ return static_cast<ValueType>(c);
+}
+
+// A comparator for internal keys that uses a specified comparator for
+// the user key portion and breaks ties by decreasing sequence number.
+class InternalKeyComparator : public Comparator {
+ private:
+ const Comparator* user_comparator_;
+ std::string name_;
+ public:
+ explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c),
+ name_("rocksdb.InternalKeyComparator:" +
+ std::string(user_comparator_->Name())) {
+ }
+ virtual ~InternalKeyComparator() {}
+
+ virtual const char* Name() const override;
+ virtual int Compare(const Slice& a, const Slice& b) const override;
+ virtual void FindShortestSeparator(std::string* start,
+ const Slice& limit) const override;
+ virtual void FindShortSuccessor(std::string* key) const override;
+
+ const Comparator* user_comparator() const { return user_comparator_; }
+
+ int Compare(const InternalKey& a, const InternalKey& b) const;
+ int Compare(const ParsedInternalKey& a, const ParsedInternalKey& b) const;
+ virtual const Comparator* GetRootComparator() const override {
+ return user_comparator_->GetRootComparator();
+ }
+};
+
+// Modules in this directory should keep internal keys wrapped inside
+// the following class instead of plain strings so that we do not
+// incorrectly use string comparisons instead of an InternalKeyComparator.
+class InternalKey {
+ private:
+ std::string rep_;
+ public:
+ InternalKey() { } // Leave rep_ as empty to indicate it is invalid
+ InternalKey(const Slice& _user_key, SequenceNumber s, ValueType t) {
+ AppendInternalKey(&rep_, ParsedInternalKey(_user_key, s, t));
+ }
+
+ // sets the internal key to be bigger or equal to all internal keys with this
+ // user key
+ void SetMaxPossibleForUserKey(const Slice& _user_key) {
+ AppendInternalKey(&rep_, ParsedInternalKey(_user_key, kMaxSequenceNumber,
+ kValueTypeForSeek));
+ }
+
+ // sets the internal key to be smaller or equal to all internal keys with this
+ // user key
+ void SetMinPossibleForUserKey(const Slice& _user_key) {
+ AppendInternalKey(
+ &rep_, ParsedInternalKey(_user_key, 0, static_cast<ValueType>(0)));
+ }
+
+ bool Valid() const {
+ ParsedInternalKey parsed;
+ return ParseInternalKey(Slice(rep_), &parsed);
+ }
+
+ void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); }
+ Slice Encode() const {
+ assert(!rep_.empty());
+ return rep_;
+ }
+
+ Slice user_key() const { return ExtractUserKey(rep_); }
+ size_t size() { return rep_.size(); }
+
+ void Set(const Slice& _user_key, SequenceNumber s, ValueType t) {
+ SetFrom(ParsedInternalKey(_user_key, s, t));
+ }
+
+ void SetFrom(const ParsedInternalKey& p) {
+ rep_.clear();
+ AppendInternalKey(&rep_, p);
+ }
+
+ void Clear() { rep_.clear(); }
+
+ // The underlying representation.
+ // Intended only to be used together with ConvertFromUserKey().
+ std::string* rep() { return &rep_; }
+
+ // Assuming that *rep() contains a user key, this method makes internal key
+ // out of it in-place. This saves a memcpy compared to Set()/SetFrom().
+ void ConvertFromUserKey(SequenceNumber s, ValueType t) {
+ AppendInternalKeyFooter(&rep_, s, t);
+ }
+
+ std::string DebugString(bool hex = false) const;
+};
+
+inline int InternalKeyComparator::Compare(
+ const InternalKey& a, const InternalKey& b) const {
+ return Compare(a.Encode(), b.Encode());
+}
+
+inline bool ParseInternalKey(const Slice& internal_key,
+ ParsedInternalKey* result) {
+ const size_t n = internal_key.size();
+ if (n < 8) return false;
+ uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
+ unsigned char c = num & 0xff;
+ result->sequence = num >> 8;
+ result->type = static_cast<ValueType>(c);
+ assert(result->type <= ValueType::kMaxValue);
+ result->user_key = Slice(internal_key.data(), n - 8);
+ return IsExtendedValueType(result->type);
+}
+
+// Update the sequence number in the internal key.
+// Guarantees not to invalidate ikey.data().
+inline void UpdateInternalKey(std::string* ikey, uint64_t seq, ValueType t) {
+ size_t ikey_sz = ikey->size();
+ assert(ikey_sz >= 8);
+ uint64_t newval = (seq << 8) | t;
+
+ // Note: Since C++11, strings are guaranteed to be stored contiguously and
+ // string::operator[]() is guaranteed not to change ikey.data().
+ EncodeFixed64(&(*ikey)[ikey_sz - 8], newval);
+}
+
+// Get the sequence number from the internal key
+inline uint64_t GetInternalKeySeqno(const Slice& internal_key) {
+ const size_t n = internal_key.size();
+ assert(n >= 8);
+ uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
+ return num >> 8;
+}
+
+
+// A helper class useful for DBImpl::Get()
+class LookupKey {
+ public:
+ // Initialize *this for looking up user_key at a snapshot with
+ // the specified sequence number.
+ LookupKey(const Slice& _user_key, SequenceNumber sequence);
+
+ ~LookupKey();
+
+ // Return a key suitable for lookup in a MemTable.
+ Slice memtable_key() const {
+ return Slice(start_, static_cast<size_t>(end_ - start_));
+ }
+
+ // Return an internal key (suitable for passing to an internal iterator)
+ Slice internal_key() const {
+ return Slice(kstart_, static_cast<size_t>(end_ - kstart_));
+ }
+
+ // Return the user key
+ Slice user_key() const {
+ return Slice(kstart_, static_cast<size_t>(end_ - kstart_ - 8));
+ }
+
+ private:
+ // We construct a char array of the form:
+ // klength varint32 <-- start_
+ // userkey char[klength] <-- kstart_
+ // tag uint64
+ // <-- end_
+ // The array is a suitable MemTable key.
+ // The suffix starting with "userkey" can be used as an InternalKey.
+ const char* start_;
+ const char* kstart_;
+ const char* end_;
+ char space_[200]; // Avoid allocation for short keys
+
+ // No copying allowed
+ LookupKey(const LookupKey&);
+ void operator=(const LookupKey&);
+};
+
+inline LookupKey::~LookupKey() {
+ if (start_ != space_) delete[] start_;
+}
+
+class IterKey {
+ public:
+ IterKey()
+ : buf_(space_),
+ buf_size_(sizeof(space_)),
+ key_(buf_),
+ key_size_(0),
+ is_user_key_(true) {}
+
+ ~IterKey() { ResetBuffer(); }
+
+ Slice GetInternalKey() const {
+ assert(!IsUserKey());
+ return Slice(key_, key_size_);
+ }
+
+ Slice GetUserKey() const {
+ if (IsUserKey()) {
+ return Slice(key_, key_size_);
+ } else {
+ assert(key_size_ >= 8);
+ return Slice(key_, key_size_ - 8);
+ }
+ }
+
+ size_t Size() const { return key_size_; }
+
+ void Clear() { key_size_ = 0; }
+
+ // Append "non_shared_data" to its back, from "shared_len"
+ // This function is used in Block::Iter::ParseNextKey
+ // shared_len: bytes in [0, shard_len-1] would be remained
+ // non_shared_data: data to be append, its length must be >= non_shared_len
+ void TrimAppend(const size_t shared_len, const char* non_shared_data,
+ const size_t non_shared_len) {
+ assert(shared_len <= key_size_);
+ size_t total_size = shared_len + non_shared_len;
+
+ if (IsKeyPinned() /* key is not in buf_ */) {
+ // Copy the key from external memory to buf_ (copy shared_len bytes)
+ EnlargeBufferIfNeeded(total_size);
+ memcpy(buf_, key_, shared_len);
+ } else if (total_size > buf_size_) {
+ // Need to allocate space, delete previous space
+ char* p = new char[total_size];
+ memcpy(p, key_, shared_len);
+
+ if (buf_ != space_) {
+ delete[] buf_;
+ }
+
+ buf_ = p;
+ buf_size_ = total_size;
+ }
+
+ memcpy(buf_ + shared_len, non_shared_data, non_shared_len);
+ key_ = buf_;
+ key_size_ = total_size;
+ }
+
+ Slice SetUserKey(const Slice& key, bool copy = true) {
+ is_user_key_ = true;
+ return SetKeyImpl(key, copy);
+ }
+
+ Slice SetInternalKey(const Slice& key, bool copy = true) {
+ is_user_key_ = false;
+ return SetKeyImpl(key, copy);
+ }
+
+ // Copies the content of key, updates the reference to the user key in ikey
+ // and returns a Slice referencing the new copy.
+ Slice SetInternalKey(const Slice& key, ParsedInternalKey* ikey) {
+ size_t key_n = key.size();
+ assert(key_n >= 8);
+ SetInternalKey(key);
+ ikey->user_key = Slice(key_, key_n - 8);
+ return Slice(key_, key_n);
+ }
+
+ // Copy the key into IterKey own buf_
+ void OwnKey() {
+ assert(IsKeyPinned() == true);
+
+ Reserve(key_size_);
+ memcpy(buf_, key_, key_size_);
+ key_ = buf_;
+ }
+
+ // Update the sequence number in the internal key. Guarantees not to
+ // invalidate slices to the key (and the user key).
+ void UpdateInternalKey(uint64_t seq, ValueType t) {
+ assert(!IsKeyPinned());
+ assert(key_size_ >= 8);
+ uint64_t newval = (seq << 8) | t;
+ EncodeFixed64(&buf_[key_size_ - 8], newval);
+ }
+
+ bool IsKeyPinned() const { return (key_ != buf_); }
+
+ void SetInternalKey(const Slice& key_prefix, const Slice& user_key,
+ SequenceNumber s,
+ ValueType value_type = kValueTypeForSeek) {
+ size_t psize = key_prefix.size();
+ size_t usize = user_key.size();
+ EnlargeBufferIfNeeded(psize + usize + sizeof(uint64_t));
+ if (psize > 0) {
+ memcpy(buf_, key_prefix.data(), psize);
+ }
+ memcpy(buf_ + psize, user_key.data(), usize);
+ EncodeFixed64(buf_ + usize + psize, PackSequenceAndType(s, value_type));
+
+ key_ = buf_;
+ key_size_ = psize + usize + sizeof(uint64_t);
+ is_user_key_ = false;
+ }
+
+ void SetInternalKey(const Slice& user_key, SequenceNumber s,
+ ValueType value_type = kValueTypeForSeek) {
+ SetInternalKey(Slice(), user_key, s, value_type);
+ }
+
+ void Reserve(size_t size) {
+ EnlargeBufferIfNeeded(size);
+ key_size_ = size;
+ }
+
+ void SetInternalKey(const ParsedInternalKey& parsed_key) {
+ SetInternalKey(Slice(), parsed_key);
+ }
+
+ void SetInternalKey(const Slice& key_prefix,
+ const ParsedInternalKey& parsed_key_suffix) {
+ SetInternalKey(key_prefix, parsed_key_suffix.user_key,
+ parsed_key_suffix.sequence, parsed_key_suffix.type);
+ }
+
+ void EncodeLengthPrefixedKey(const Slice& key) {
+ auto size = key.size();
+ EnlargeBufferIfNeeded(size + static_cast<size_t>(VarintLength(size)));
+ char* ptr = EncodeVarint32(buf_, static_cast<uint32_t>(size));
+ memcpy(ptr, key.data(), size);
+ key_ = buf_;
+ is_user_key_ = true;
+ }
+
+ bool IsUserKey() const { return is_user_key_; }
+
+ private:
+ char* buf_;
+ size_t buf_size_;
+ const char* key_;
+ size_t key_size_;
+ char space_[32]; // Avoid allocation for short keys
+ bool is_user_key_;
+
+ Slice SetKeyImpl(const Slice& key, bool copy) {
+ size_t size = key.size();
+ if (copy) {
+ // Copy key to buf_
+ EnlargeBufferIfNeeded(size);
+ memcpy(buf_, key.data(), size);
+ key_ = buf_;
+ } else {
+ // Update key_ to point to external memory
+ key_ = key.data();
+ }
+ key_size_ = size;
+ return Slice(key_, key_size_);
+ }
+
+ void ResetBuffer() {
+ if (buf_ != space_) {
+ delete[] buf_;
+ buf_ = space_;
+ }
+ buf_size_ = sizeof(space_);
+ key_size_ = 0;
+ }
+
+ // Enlarge the buffer size if needed based on key_size.
+ // By default, static allocated buffer is used. Once there is a key
+ // larger than the static allocated buffer, another buffer is dynamically
+ // allocated, until a larger key buffer is requested. In that case, we
+ // reallocate buffer and delete the old one.
+ void EnlargeBufferIfNeeded(size_t key_size) {
+ // If size is smaller than buffer size, continue using current buffer,
+ // or the static allocated one, as default
+ if (key_size > buf_size_) {
+ // Need to enlarge the buffer.
+ ResetBuffer();
+ buf_ = new char[key_size];
+ buf_size_ = key_size;
+ }
+ }
+
+ // No copying allowed
+ IterKey(const IterKey&) = delete;
+ void operator=(const IterKey&) = delete;
+};
+
+class InternalKeySliceTransform : public SliceTransform {
+ public:
+ explicit InternalKeySliceTransform(const SliceTransform* transform)
+ : transform_(transform) {}
+
+ virtual const char* Name() const override { return transform_->Name(); }
+
+ virtual Slice Transform(const Slice& src) const override {
+ auto user_key = ExtractUserKey(src);
+ return transform_->Transform(user_key);
+ }
+
+ virtual bool InDomain(const Slice& src) const override {
+ auto user_key = ExtractUserKey(src);
+ return transform_->InDomain(user_key);
+ }
+
+ virtual bool InRange(const Slice& dst) const override {
+ auto user_key = ExtractUserKey(dst);
+ return transform_->InRange(user_key);
+ }
+
+ const SliceTransform* user_prefix_extractor() const { return transform_; }
+
+ private:
+ // Like comparator, InternalKeySliceTransform will not take care of the
+ // deletion of transform_
+ const SliceTransform* const transform_;
+};
+
+// Read the key of a record from a write batch.
+// if this record represent the default column family then cf_record
+// must be passed as false, otherwise it must be passed as true.
+extern bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key,
+ bool cf_record);
+
+// Read record from a write batch piece from input.
+// tag, column_family, key, value and blob are return values. Callers own the
+// Slice they point to.
+// Tag is defined as ValueType.
+// input will be advanced to after the record.
+extern Status ReadRecordFromWriteBatch(Slice* input, char* tag,
+ uint32_t* column_family, Slice* key,
+ Slice* value, Slice* blob, Slice* xid);
+
+// When user call DeleteRange() to delete a range of keys,
+// we will store a serialized RangeTombstone in MemTable and SST.
+// the struct here is a easy-understood form
+// start/end_key_ is the start/end user key of the range to be deleted
+struct RangeTombstone {
+ Slice start_key_;
+ Slice end_key_;
+ SequenceNumber seq_;
+ RangeTombstone() = default;
+ RangeTombstone(Slice sk, Slice ek, SequenceNumber sn)
+ : start_key_(sk), end_key_(ek), seq_(sn) {}
+
+ RangeTombstone(ParsedInternalKey parsed_key, Slice value) {
+ start_key_ = parsed_key.user_key;
+ seq_ = parsed_key.sequence;
+ end_key_ = value;
+ }
+
+ // be careful to use Serialize(), allocates new memory
+ std::pair<InternalKey, Slice> Serialize() const {
+ auto key = InternalKey(start_key_, seq_, kTypeRangeDeletion);
+ Slice value = end_key_;
+ return std::make_pair(std::move(key), std::move(value));
+ }
+
+ // be careful to use SerializeKey(), allocates new memory
+ InternalKey SerializeKey() const {
+ return InternalKey(start_key_, seq_, kTypeRangeDeletion);
+ }
+
+ // be careful to use SerializeEndKey(), allocates new memory
+ InternalKey SerializeEndKey() const {
+ return InternalKey(end_key_, seq_, kTypeRangeDeletion);
+ }
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/event_helpers.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/event_helpers.cc b/thirdparty/rocksdb/db/event_helpers.cc
new file mode 100644
index 0000000..1b79acb
--- /dev/null
+++ b/thirdparty/rocksdb/db/event_helpers.cc
@@ -0,0 +1,155 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "db/event_helpers.h"
+
+namespace rocksdb {
+
+namespace {
+template<class T>
+inline T SafeDivide(T a, T b) {
+ return b == 0 ? 0 : a / b;
+}
+} // namespace
+
+void EventHelpers::AppendCurrentTime(JSONWriter* jwriter) {
+ *jwriter << "time_micros"
+ << std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+}
+
+#ifndef ROCKSDB_LITE
+void EventHelpers::NotifyTableFileCreationStarted(
+ const std::vector<std::shared_ptr<EventListener>>& listeners,
+ const std::string& db_name, const std::string& cf_name,
+ const std::string& file_path, int job_id, TableFileCreationReason reason) {
+ TableFileCreationBriefInfo info;
+ info.db_name = db_name;
+ info.cf_name = cf_name;
+ info.file_path = file_path;
+ info.job_id = job_id;
+ info.reason = reason;
+ for (auto& listener : listeners) {
+ listener->OnTableFileCreationStarted(info);
+ }
+}
+#endif // !ROCKSDB_LITE
+
+void EventHelpers::NotifyOnBackgroundError(
+ const std::vector<std::shared_ptr<EventListener>>& listeners,
+ BackgroundErrorReason reason, Status* bg_error,
+ InstrumentedMutex* db_mutex) {
+#ifndef ROCKSDB_LITE
+ if (listeners.size() == 0U) {
+ return;
+ }
+ db_mutex->AssertHeld();
+ // release lock while notifying events
+ db_mutex->Unlock();
+ for (auto& listener : listeners) {
+ listener->OnBackgroundError(reason, bg_error);
+ }
+ db_mutex->Lock();
+#endif // ROCKSDB_LITE
+}
+
+void EventHelpers::LogAndNotifyTableFileCreationFinished(
+ EventLogger* event_logger,
+ const std::vector<std::shared_ptr<EventListener>>& listeners,
+ const std::string& db_name, const std::string& cf_name,
+ const std::string& file_path, int job_id, const FileDescriptor& fd,
+ const TableProperties& table_properties, TableFileCreationReason reason,
+ const Status& s) {
+ if (s.ok() && event_logger) {
+ JSONWriter jwriter;
+ AppendCurrentTime(&jwriter);
+ jwriter << "cf_name" << cf_name << "job" << job_id << "event"
+ << "table_file_creation"
+ << "file_number" << fd.GetNumber() << "file_size"
+ << fd.GetFileSize();
+
+ // table_properties
+ {
+ jwriter << "table_properties";
+ jwriter.StartObject();
+
+ // basic properties:
+ jwriter << "data_size" << table_properties.data_size << "index_size"
+ << table_properties.index_size << "filter_size"
+ << table_properties.filter_size << "raw_key_size"
+ << table_properties.raw_key_size << "raw_average_key_size"
+ << SafeDivide(table_properties.raw_key_size,
+ table_properties.num_entries)
+ << "raw_value_size" << table_properties.raw_value_size
+ << "raw_average_value_size"
+ << SafeDivide(table_properties.raw_value_size,
+ table_properties.num_entries)
+ << "num_data_blocks" << table_properties.num_data_blocks
+ << "num_entries" << table_properties.num_entries
+ << "filter_policy_name" << table_properties.filter_policy_name;
+
+ // user collected properties
+ for (const auto& prop : table_properties.readable_properties) {
+ jwriter << prop.first << prop.second;
+ }
+ jwriter.EndObject();
+ }
+ jwriter.EndObject();
+
+ event_logger->Log(jwriter);
+ }
+
+#ifndef ROCKSDB_LITE
+ if (listeners.size() == 0) {
+ return;
+ }
+ TableFileCreationInfo info;
+ info.db_name = db_name;
+ info.cf_name = cf_name;
+ info.file_path = file_path;
+ info.file_size = fd.file_size;
+ info.job_id = job_id;
+ info.table_properties = table_properties;
+ info.reason = reason;
+ info.status = s;
+ for (auto& listener : listeners) {
+ listener->OnTableFileCreated(info);
+ }
+#endif // !ROCKSDB_LITE
+}
+
+void EventHelpers::LogAndNotifyTableFileDeletion(
+ EventLogger* event_logger, int job_id,
+ uint64_t file_number, const std::string& file_path,
+ const Status& status, const std::string& dbname,
+ const std::vector<std::shared_ptr<EventListener>>& listeners) {
+
+ JSONWriter jwriter;
+ AppendCurrentTime(&jwriter);
+
+ jwriter << "job" << job_id
+ << "event" << "table_file_deletion"
+ << "file_number" << file_number;
+ if (!status.ok()) {
+ jwriter << "status" << status.ToString();
+ }
+
+ jwriter.EndObject();
+
+ event_logger->Log(jwriter);
+
+#ifndef ROCKSDB_LITE
+ TableFileDeletionInfo info;
+ info.db_name = dbname;
+ info.job_id = job_id;
+ info.file_path = file_path;
+ info.status = status;
+ for (auto& listener : listeners) {
+ listener->OnTableFileDeleted(info);
+ }
+#endif // !ROCKSDB_LITE
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/event_helpers.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/event_helpers.h b/thirdparty/rocksdb/db/event_helpers.h
new file mode 100644
index 0000000..674e6c5
--- /dev/null
+++ b/thirdparty/rocksdb/db/event_helpers.h
@@ -0,0 +1,52 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "db/column_family.h"
+#include "db/version_edit.h"
+#include "rocksdb/listener.h"
+#include "rocksdb/table_properties.h"
+#include "util/event_logger.h"
+
+namespace rocksdb {
+
+class EventHelpers {
+ public:
+ static void AppendCurrentTime(JSONWriter* json_writer);
+#ifndef ROCKSDB_LITE
+ static void NotifyTableFileCreationStarted(
+ const std::vector<std::shared_ptr<EventListener>>& listeners,
+ const std::string& db_name, const std::string& cf_name,
+ const std::string& file_path, int job_id, TableFileCreationReason reason);
+#endif // !ROCKSDB_LITE
+ static void NotifyOnBackgroundError(
+ const std::vector<std::shared_ptr<EventListener>>& listeners,
+ BackgroundErrorReason reason, Status* bg_error,
+ InstrumentedMutex* db_mutex);
+ static void LogAndNotifyTableFileCreationFinished(
+ EventLogger* event_logger,
+ const std::vector<std::shared_ptr<EventListener>>& listeners,
+ const std::string& db_name, const std::string& cf_name,
+ const std::string& file_path, int job_id, const FileDescriptor& fd,
+ const TableProperties& table_properties, TableFileCreationReason reason,
+ const Status& s);
+ static void LogAndNotifyTableFileDeletion(
+ EventLogger* event_logger, int job_id,
+ uint64_t file_number, const std::string& file_path,
+ const Status& status, const std::string& db_name,
+ const std::vector<std::shared_ptr<EventListener>>& listeners);
+
+ private:
+ static void LogAndNotifyTableFileCreation(
+ EventLogger* event_logger,
+ const std::vector<std::shared_ptr<EventListener>>& listeners,
+ const FileDescriptor& fd, const TableFileCreationInfo& info);
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/experimental.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/experimental.cc b/thirdparty/rocksdb/db/experimental.cc
new file mode 100644
index 0000000..effe9d7
--- /dev/null
+++ b/thirdparty/rocksdb/db/experimental.cc
@@ -0,0 +1,49 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "rocksdb/experimental.h"
+
+#include "db/db_impl.h"
+
+namespace rocksdb {
+namespace experimental {
+
+#ifndef ROCKSDB_LITE
+
+Status SuggestCompactRange(DB* db, ColumnFamilyHandle* column_family,
+ const Slice* begin, const Slice* end) {
+ if (db == nullptr) {
+ return Status::InvalidArgument("DB is empty");
+ }
+
+ return db->SuggestCompactRange(column_family, begin, end);
+}
+
+Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) {
+ if (db == nullptr) {
+ return Status::InvalidArgument("Didn't recognize DB object");
+ }
+ return db->PromoteL0(column_family, target_level);
+}
+
+#else // ROCKSDB_LITE
+
+Status SuggestCompactRange(DB* db, ColumnFamilyHandle* column_family,
+ const Slice* begin, const Slice* end) {
+ return Status::NotSupported("Not supported in RocksDB LITE");
+}
+
+Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) {
+ return Status::NotSupported("Not supported in RocksDB LITE");
+}
+
+#endif // ROCKSDB_LITE
+
+Status SuggestCompactRange(DB* db, const Slice* begin, const Slice* end) {
+ return SuggestCompactRange(db, db->DefaultColumnFamily(), begin, end);
+}
+
+} // namespace experimental
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/external_sst_file_ingestion_job.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/external_sst_file_ingestion_job.cc b/thirdparty/rocksdb/db/external_sst_file_ingestion_job.cc
new file mode 100644
index 0000000..58fa354
--- /dev/null
+++ b/thirdparty/rocksdb/db/external_sst_file_ingestion_job.cc
@@ -0,0 +1,665 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+
+#include "db/external_sst_file_ingestion_job.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#include "db/version_edit.h"
+#include "table/merging_iterator.h"
+#include "table/scoped_arena_iterator.h"
+#include "table/sst_file_writer_collectors.h"
+#include "table/table_builder.h"
+#include "util/file_reader_writer.h"
+#include "util/file_util.h"
+#include "util/stop_watch.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+Status ExternalSstFileIngestionJob::Prepare(
+ const std::vector<std::string>& external_files_paths) {
+ Status status;
+
+ // Read the information of files we are ingesting
+ for (const std::string& file_path : external_files_paths) {
+ IngestedFileInfo file_to_ingest;
+ status = GetIngestedFileInfo(file_path, &file_to_ingest);
+ if (!status.ok()) {
+ return status;
+ }
+ files_to_ingest_.push_back(file_to_ingest);
+ }
+
+ for (const IngestedFileInfo& f : files_to_ingest_) {
+ if (f.cf_id !=
+ TablePropertiesCollectorFactory::Context::kUnknownColumnFamily &&
+ f.cf_id != cfd_->GetID()) {
+ return Status::InvalidArgument(
+ "External file column family id dont match");
+ }
+ }
+
+ const Comparator* ucmp = cfd_->internal_comparator().user_comparator();
+ auto num_files = files_to_ingest_.size();
+ if (num_files == 0) {
+ return Status::InvalidArgument("The list of files is empty");
+ } else if (num_files > 1) {
+ // Verify that passed files dont have overlapping ranges
+ autovector<const IngestedFileInfo*> sorted_files;
+ for (size_t i = 0; i < num_files; i++) {
+ sorted_files.push_back(&files_to_ingest_[i]);
+ }
+
+ std::sort(
+ sorted_files.begin(), sorted_files.end(),
+ [&ucmp](const IngestedFileInfo* info1, const IngestedFileInfo* info2) {
+ return ucmp->Compare(info1->smallest_user_key,
+ info2->smallest_user_key) < 0;
+ });
+
+ for (size_t i = 0; i < num_files - 1; i++) {
+ if (ucmp->Compare(sorted_files[i]->largest_user_key,
+ sorted_files[i + 1]->smallest_user_key) >= 0) {
+ return Status::NotSupported("Files have overlapping ranges");
+ }
+ }
+ }
+
+ for (IngestedFileInfo& f : files_to_ingest_) {
+ if (f.num_entries == 0) {
+ return Status::InvalidArgument("File contain no entries");
+ }
+
+ if (!f.smallest_internal_key().Valid() ||
+ !f.largest_internal_key().Valid()) {
+ return Status::Corruption("Generated table have corrupted keys");
+ }
+ }
+
+ // Copy/Move external files into DB
+ for (IngestedFileInfo& f : files_to_ingest_) {
+ f.fd = FileDescriptor(versions_->NewFileNumber(), 0, f.file_size);
+
+ const std::string path_outside_db = f.external_file_path;
+ const std::string path_inside_db =
+ TableFileName(db_options_.db_paths, f.fd.GetNumber(), f.fd.GetPathId());
+
+ if (ingestion_options_.move_files) {
+ status = env_->LinkFile(path_outside_db, path_inside_db);
+ if (status.IsNotSupported()) {
+ // Original file is on a different FS, use copy instead of hard linking
+ status = CopyFile(env_, path_outside_db, path_inside_db, 0,
+ db_options_.use_fsync);
+ }
+ } else {
+ status = CopyFile(env_, path_outside_db, path_inside_db, 0,
+ db_options_.use_fsync);
+ }
+ TEST_SYNC_POINT("DBImpl::AddFile:FileCopied");
+ if (!status.ok()) {
+ break;
+ }
+ f.internal_file_path = path_inside_db;
+ }
+
+ if (!status.ok()) {
+ // We failed, remove all files that we copied into the db
+ for (IngestedFileInfo& f : files_to_ingest_) {
+ if (f.internal_file_path == "") {
+ break;
+ }
+ Status s = env_->DeleteFile(f.internal_file_path);
+ if (!s.ok()) {
+ ROCKS_LOG_WARN(db_options_.info_log,
+ "AddFile() clean up for file %s failed : %s",
+ f.internal_file_path.c_str(), s.ToString().c_str());
+ }
+ }
+ }
+
+ return status;
+}
+
+Status ExternalSstFileIngestionJob::NeedsFlush(bool* flush_needed) {
+ SuperVersion* super_version = cfd_->GetSuperVersion();
+ Status status =
+ IngestedFilesOverlapWithMemtables(super_version, flush_needed);
+
+ if (status.ok() && *flush_needed &&
+ !ingestion_options_.allow_blocking_flush) {
+ status = Status::InvalidArgument("External file requires flush");
+ }
+ return status;
+}
+
+// REQUIRES: we have become the only writer by entering both write_thread_ and
+// nonmem_write_thread_
+Status ExternalSstFileIngestionJob::Run() {
+ Status status;
+#ifndef NDEBUG
+ // We should never run the job with a memtable that is overlapping
+ // with the files we are ingesting
+ bool need_flush = false;
+ status = NeedsFlush(&need_flush);
+ assert(status.ok() && need_flush == false);
+#endif
+
+ bool consumed_seqno = false;
+ bool force_global_seqno = false;
+
+ if (ingestion_options_.snapshot_consistency && !db_snapshots_->empty()) {
+ // We need to assign a global sequence number to all the files even
+ // if the dont overlap with any ranges since we have snapshots
+ force_global_seqno = true;
+ }
+ // It is safe to use this instead of LastToBeWrittenSequence since we are
+ // the only active writer, and hence they are equal
+ const SequenceNumber last_seqno = versions_->LastSequence();
+ SuperVersion* super_version = cfd_->GetSuperVersion();
+ edit_.SetColumnFamily(cfd_->GetID());
+ // The levels that the files will be ingested into
+
+ for (IngestedFileInfo& f : files_to_ingest_) {
+ SequenceNumber assigned_seqno = 0;
+ if (ingestion_options_.ingest_behind) {
+ status = CheckLevelForIngestedBehindFile(&f);
+ } else {
+ status = AssignLevelAndSeqnoForIngestedFile(
+ super_version, force_global_seqno, cfd_->ioptions()->compaction_style,
+ &f, &assigned_seqno);
+ }
+ if (!status.ok()) {
+ return status;
+ }
+ status = AssignGlobalSeqnoForIngestedFile(&f, assigned_seqno);
+ TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::Run",
+ &assigned_seqno);
+ if (assigned_seqno == last_seqno + 1) {
+ consumed_seqno = true;
+ }
+ if (!status.ok()) {
+ return status;
+ }
+ edit_.AddFile(f.picked_level, f.fd.GetNumber(), f.fd.GetPathId(),
+ f.fd.GetFileSize(), f.smallest_internal_key(),
+ f.largest_internal_key(), f.assigned_seqno, f.assigned_seqno,
+ false);
+ }
+
+ if (consumed_seqno) {
+ versions_->SetLastToBeWrittenSequence(last_seqno + 1);
+ versions_->SetLastSequence(last_seqno + 1);
+ }
+
+ return status;
+}
+
+void ExternalSstFileIngestionJob::UpdateStats() {
+ // Update internal stats for new ingested files
+ uint64_t total_keys = 0;
+ uint64_t total_l0_files = 0;
+ uint64_t total_time = env_->NowMicros() - job_start_time_;
+ for (IngestedFileInfo& f : files_to_ingest_) {
+ InternalStats::CompactionStats stats(1);
+ stats.micros = total_time;
+ stats.bytes_written = f.fd.GetFileSize();
+ stats.num_output_files = 1;
+ cfd_->internal_stats()->AddCompactionStats(f.picked_level, stats);
+ cfd_->internal_stats()->AddCFStats(InternalStats::BYTES_INGESTED_ADD_FILE,
+ f.fd.GetFileSize());
+ total_keys += f.num_entries;
+ if (f.picked_level == 0) {
+ total_l0_files += 1;
+ }
+ ROCKS_LOG_INFO(
+ db_options_.info_log,
+ "[AddFile] External SST file %s was ingested in L%d with path %s "
+ "(global_seqno=%" PRIu64 ")\n",
+ f.external_file_path.c_str(), f.picked_level,
+ f.internal_file_path.c_str(), f.assigned_seqno);
+ }
+ cfd_->internal_stats()->AddCFStats(InternalStats::INGESTED_NUM_KEYS_TOTAL,
+ total_keys);
+ cfd_->internal_stats()->AddCFStats(InternalStats::INGESTED_NUM_FILES_TOTAL,
+ files_to_ingest_.size());
+ cfd_->internal_stats()->AddCFStats(
+ InternalStats::INGESTED_LEVEL0_NUM_FILES_TOTAL, total_l0_files);
+}
+
+void ExternalSstFileIngestionJob::Cleanup(const Status& status) {
+ if (!status.ok()) {
+ // We failed to add the files to the database
+ // remove all the files we copied
+ for (IngestedFileInfo& f : files_to_ingest_) {
+ Status s = env_->DeleteFile(f.internal_file_path);
+ if (!s.ok()) {
+ ROCKS_LOG_WARN(db_options_.info_log,
+ "AddFile() clean up for file %s failed : %s",
+ f.internal_file_path.c_str(), s.ToString().c_str());
+ }
+ }
+ } else if (status.ok() && ingestion_options_.move_files) {
+ // The files were moved and added successfully, remove original file links
+ for (IngestedFileInfo& f : files_to_ingest_) {
+ Status s = env_->DeleteFile(f.external_file_path);
+ if (!s.ok()) {
+ ROCKS_LOG_WARN(
+ db_options_.info_log,
+ "%s was added to DB successfully but failed to remove original "
+ "file link : %s",
+ f.external_file_path.c_str(), s.ToString().c_str());
+ }
+ }
+ }
+}
+
+Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
+ const std::string& external_file, IngestedFileInfo* file_to_ingest) {
+ file_to_ingest->external_file_path = external_file;
+
+ // Get external file size
+ Status status = env_->GetFileSize(external_file, &file_to_ingest->file_size);
+ if (!status.ok()) {
+ return status;
+ }
+
+ // Create TableReader for external file
+ std::unique_ptr<TableReader> table_reader;
+ std::unique_ptr<RandomAccessFile> sst_file;
+ std::unique_ptr<RandomAccessFileReader> sst_file_reader;
+
+ status = env_->NewRandomAccessFile(external_file, &sst_file, env_options_);
+ if (!status.ok()) {
+ return status;
+ }
+ sst_file_reader.reset(new RandomAccessFileReader(std::move(sst_file),
+ external_file));
+
+ status = cfd_->ioptions()->table_factory->NewTableReader(
+ TableReaderOptions(*cfd_->ioptions(), env_options_,
+ cfd_->internal_comparator()),
+ std::move(sst_file_reader), file_to_ingest->file_size, &table_reader);
+ if (!status.ok()) {
+ return status;
+ }
+
+ // Get the external file properties
+ auto props = table_reader->GetTableProperties();
+ const auto& uprops = props->user_collected_properties;
+
+ // Get table version
+ auto version_iter = uprops.find(ExternalSstFilePropertyNames::kVersion);
+ if (version_iter == uprops.end()) {
+ return Status::Corruption("External file version not found");
+ }
+ file_to_ingest->version = DecodeFixed32(version_iter->second.c_str());
+
+ auto seqno_iter = uprops.find(ExternalSstFilePropertyNames::kGlobalSeqno);
+ if (file_to_ingest->version == 2) {
+ // version 2 imply that we have global sequence number
+ if (seqno_iter == uprops.end()) {
+ return Status::Corruption(
+ "External file global sequence number not found");
+ }
+
+ // Set the global sequence number
+ file_to_ingest->original_seqno = DecodeFixed64(seqno_iter->second.c_str());
+ file_to_ingest->global_seqno_offset = props->properties_offsets.at(
+ ExternalSstFilePropertyNames::kGlobalSeqno);
+
+ if (file_to_ingest->global_seqno_offset == 0) {
+ return Status::Corruption("Was not able to find file global seqno field");
+ }
+ } else if (file_to_ingest->version == 1) {
+ // SST file V1 should not have global seqno field
+ assert(seqno_iter == uprops.end());
+ file_to_ingest->original_seqno = 0;
+ if (ingestion_options_.allow_blocking_flush ||
+ ingestion_options_.allow_global_seqno) {
+ return Status::InvalidArgument(
+ "External SST file V1 does not support global seqno");
+ }
+ } else {
+ return Status::InvalidArgument("External file version is not supported");
+ }
+ // Get number of entries in table
+ file_to_ingest->num_entries = props->num_entries;
+
+ ParsedInternalKey key;
+ ReadOptions ro;
+ // During reading the external file we can cache blocks that we read into
+ // the block cache, if we later change the global seqno of this file, we will
+ // have block in cache that will include keys with wrong seqno.
+ // We need to disable fill_cache so that we read from the file without
+ // updating the block cache.
+ ro.fill_cache = false;
+ std::unique_ptr<InternalIterator> iter(table_reader->NewIterator(ro));
+
+ // Get first (smallest) key from file
+ iter->SeekToFirst();
+ if (!ParseInternalKey(iter->key(), &key)) {
+ return Status::Corruption("external file have corrupted keys");
+ }
+ if (key.sequence != 0) {
+ return Status::Corruption("external file have non zero sequence number");
+ }
+ file_to_ingest->smallest_user_key = key.user_key.ToString();
+
+ // Get last (largest) key from file
+ iter->SeekToLast();
+ if (!ParseInternalKey(iter->key(), &key)) {
+ return Status::Corruption("external file have corrupted keys");
+ }
+ if (key.sequence != 0) {
+ return Status::Corruption("external file have non zero sequence number");
+ }
+ file_to_ingest->largest_user_key = key.user_key.ToString();
+
+ file_to_ingest->cf_id = static_cast<uint32_t>(props->column_family_id);
+
+ file_to_ingest->table_properties = *props;
+
+ return status;
+}
+
+Status ExternalSstFileIngestionJob::IngestedFilesOverlapWithMemtables(
+ SuperVersion* sv, bool* overlap) {
+ // Create an InternalIterator over all memtables
+ Arena arena;
+ ReadOptions ro;
+ ro.total_order_seek = true;
+ MergeIteratorBuilder merge_iter_builder(&cfd_->internal_comparator(), &arena);
+ merge_iter_builder.AddIterator(sv->mem->NewIterator(ro, &arena));
+ sv->imm->AddIterators(ro, &merge_iter_builder);
+ ScopedArenaIterator memtable_iter(merge_iter_builder.Finish());
+
+ std::vector<InternalIterator*> memtable_range_del_iters;
+ auto* active_range_del_iter = sv->mem->NewRangeTombstoneIterator(ro);
+ if (active_range_del_iter != nullptr) {
+ memtable_range_del_iters.push_back(active_range_del_iter);
+ }
+ sv->imm->AddRangeTombstoneIterators(ro, &memtable_range_del_iters);
+ std::unique_ptr<InternalIterator> memtable_range_del_iter(NewMergingIterator(
+ &cfd_->internal_comparator(),
+ memtable_range_del_iters.empty() ? nullptr : &memtable_range_del_iters[0],
+ static_cast<int>(memtable_range_del_iters.size())));
+
+ Status status;
+ *overlap = false;
+ for (IngestedFileInfo& f : files_to_ingest_) {
+ status =
+ IngestedFileOverlapWithIteratorRange(&f, memtable_iter.get(), overlap);
+ if (!status.ok() || *overlap == true) {
+ break;
+ }
+ status = IngestedFileOverlapWithRangeDeletions(
+ &f, memtable_range_del_iter.get(), overlap);
+ if (!status.ok() || *overlap == true) {
+ break;
+ }
+ }
+
+ return status;
+}
+
+Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
+ SuperVersion* sv, bool force_global_seqno, CompactionStyle compaction_style,
+ IngestedFileInfo* file_to_ingest, SequenceNumber* assigned_seqno) {
+ Status status;
+ *assigned_seqno = 0;
+ const SequenceNumber last_seqno = versions_->LastSequence();
+ if (force_global_seqno) {
+ *assigned_seqno = last_seqno + 1;
+ if (compaction_style == kCompactionStyleUniversal) {
+ file_to_ingest->picked_level = 0;
+ return status;
+ }
+ }
+
+ bool overlap_with_db = false;
+ Arena arena;
+ ReadOptions ro;
+ ro.total_order_seek = true;
+ int target_level = 0;
+ auto* vstorage = cfd_->current()->storage_info();
+
+ for (int lvl = 0; lvl < cfd_->NumberLevels(); lvl++) {
+ if (lvl > 0 && lvl < vstorage->base_level()) {
+ continue;
+ }
+
+ if (vstorage->NumLevelFiles(lvl) > 0) {
+ bool overlap_with_level = false;
+ status = IngestedFileOverlapWithLevel(sv, file_to_ingest, lvl,
+ &overlap_with_level);
+ if (!status.ok()) {
+ return status;
+ }
+ if (overlap_with_level) {
+ // We must use L0 or any level higher than `lvl` to be able to overwrite
+ // the keys that we overlap with in this level, We also need to assign
+ // this file a seqno to overwrite the existing keys in level `lvl`
+ overlap_with_db = true;
+ break;
+ }
+
+ if (compaction_style == kCompactionStyleUniversal && lvl != 0) {
+ const std::vector<FileMetaData*>& level_files =
+ vstorage->LevelFiles(lvl);
+ const SequenceNumber level_largest_seqno =
+ (*max_element(level_files.begin(), level_files.end(),
+ [](FileMetaData* f1, FileMetaData* f2) {
+ return f1->largest_seqno < f2->largest_seqno;
+ }))
+ ->largest_seqno;
+ if (level_largest_seqno != 0) {
+ *assigned_seqno = level_largest_seqno;
+ } else {
+ continue;
+ }
+ }
+ } else if (compaction_style == kCompactionStyleUniversal) {
+ continue;
+ }
+
+ // We dont overlap with any keys in this level, but we still need to check
+ // if our file can fit in it
+ if (IngestedFileFitInLevel(file_to_ingest, lvl)) {
+ target_level = lvl;
+ }
+ }
+ TEST_SYNC_POINT_CALLBACK(
+ "ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile",
+ &overlap_with_db);
+ file_to_ingest->picked_level = target_level;
+ if (overlap_with_db && *assigned_seqno == 0) {
+ *assigned_seqno = last_seqno + 1;
+ }
+ return status;
+}
+
+Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
+ IngestedFileInfo* file_to_ingest) {
+ auto* vstorage = cfd_->current()->storage_info();
+ // first check if new files fit in the bottommost level
+ int bottom_lvl = cfd_->NumberLevels() - 1;
+ if(!IngestedFileFitInLevel(file_to_ingest, bottom_lvl)) {
+ return Status::InvalidArgument(
+ "Can't ingest_behind file as it doesn't fit "
+ "at the bottommost level!");
+ }
+
+ // second check if despite allow_ingest_behind=true we still have 0 seqnums
+ // at some upper level
+ for (int lvl = 0; lvl < cfd_->NumberLevels() - 1; lvl++) {
+ for (auto file : vstorage->LevelFiles(lvl)) {
+ if (file->smallest_seqno == 0) {
+ return Status::InvalidArgument(
+ "Can't ingest_behind file as despite allow_ingest_behind=true "
+ "there are files with 0 seqno in database at upper levels!");
+ }
+ }
+ }
+
+ file_to_ingest->picked_level = bottom_lvl;
+ return Status::OK();
+}
+
+Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
+ IngestedFileInfo* file_to_ingest, SequenceNumber seqno) {
+ if (file_to_ingest->original_seqno == seqno) {
+ // This file already have the correct global seqno
+ return Status::OK();
+ } else if (!ingestion_options_.allow_global_seqno) {
+ return Status::InvalidArgument("Global seqno is required, but disabled");
+ } else if (file_to_ingest->global_seqno_offset == 0) {
+ return Status::InvalidArgument(
+ "Trying to set global seqno for a file that dont have a global seqno "
+ "field");
+ }
+
+ std::unique_ptr<RandomRWFile> rwfile;
+ Status status = env_->NewRandomRWFile(file_to_ingest->internal_file_path,
+ &rwfile, env_options_);
+ if (!status.ok()) {
+ return status;
+ }
+
+ // Write the new seqno in the global sequence number field in the file
+ std::string seqno_val;
+ PutFixed64(&seqno_val, seqno);
+ status = rwfile->Write(file_to_ingest->global_seqno_offset, seqno_val);
+ if (status.ok()) {
+ file_to_ingest->assigned_seqno = seqno;
+ }
+ return status;
+}
+
+Status ExternalSstFileIngestionJob::IngestedFileOverlapWithIteratorRange(
+ const IngestedFileInfo* file_to_ingest, InternalIterator* iter,
+ bool* overlap) {
+ auto* vstorage = cfd_->current()->storage_info();
+ auto* ucmp = vstorage->InternalComparator()->user_comparator();
+ InternalKey range_start(file_to_ingest->smallest_user_key, kMaxSequenceNumber,
+ kValueTypeForSeek);
+ iter->Seek(range_start.Encode());
+ if (!iter->status().ok()) {
+ return iter->status();
+ }
+
+ *overlap = false;
+ if (iter->Valid()) {
+ ParsedInternalKey seek_result;
+ if (!ParseInternalKey(iter->key(), &seek_result)) {
+ return Status::Corruption("DB have corrupted keys");
+ }
+
+ if (ucmp->Compare(seek_result.user_key, file_to_ingest->largest_user_key) <=
+ 0) {
+ *overlap = true;
+ }
+ }
+
+ return iter->status();
+}
+
+Status ExternalSstFileIngestionJob::IngestedFileOverlapWithRangeDeletions(
+ const IngestedFileInfo* file_to_ingest, InternalIterator* range_del_iter,
+ bool* overlap) {
+ auto* vstorage = cfd_->current()->storage_info();
+ auto* ucmp = vstorage->InternalComparator()->user_comparator();
+
+ *overlap = false;
+ if (range_del_iter != nullptr) {
+ for (range_del_iter->SeekToFirst(); range_del_iter->Valid();
+ range_del_iter->Next()) {
+ ParsedInternalKey parsed_key;
+ if (!ParseInternalKey(range_del_iter->key(), &parsed_key)) {
+ return Status::Corruption("corrupted range deletion key: " +
+ range_del_iter->key().ToString());
+ }
+ RangeTombstone range_del(parsed_key, range_del_iter->value());
+ if (ucmp->Compare(range_del.start_key_,
+ file_to_ingest->largest_user_key) <= 0 &&
+ ucmp->Compare(file_to_ingest->smallest_user_key,
+ range_del.end_key_) <= 0) {
+ *overlap = true;
+ break;
+ }
+ }
+ }
+ return Status::OK();
+}
+
+bool ExternalSstFileIngestionJob::IngestedFileFitInLevel(
+ const IngestedFileInfo* file_to_ingest, int level) {
+ if (level == 0) {
+ // Files can always fit in L0
+ return true;
+ }
+
+ auto* vstorage = cfd_->current()->storage_info();
+ Slice file_smallest_user_key(file_to_ingest->smallest_user_key);
+ Slice file_largest_user_key(file_to_ingest->largest_user_key);
+
+ if (vstorage->OverlapInLevel(level, &file_smallest_user_key,
+ &file_largest_user_key)) {
+ // File overlap with another files in this level, we cannot
+ // add it to this level
+ return false;
+ }
+ if (cfd_->RangeOverlapWithCompaction(file_smallest_user_key,
+ file_largest_user_key, level)) {
+ // File overlap with a running compaction output that will be stored
+ // in this level, we cannot add this file to this level
+ return false;
+ }
+
+ // File did not overlap with level files, our compaction output
+ return true;
+}
+
+Status ExternalSstFileIngestionJob::IngestedFileOverlapWithLevel(
+ SuperVersion* sv, IngestedFileInfo* file_to_ingest, int lvl,
+ bool* overlap_with_level) {
+ Arena arena;
+ ReadOptions ro;
+ ro.total_order_seek = true;
+ MergeIteratorBuilder merge_iter_builder(&cfd_->internal_comparator(),
+ &arena);
+ sv->current->AddIteratorsForLevel(ro, env_options_, &merge_iter_builder, lvl,
+ nullptr /* range_del_agg */);
+ ScopedArenaIterator level_iter(merge_iter_builder.Finish());
+
+ std::vector<InternalIterator*> level_range_del_iters;
+ sv->current->AddRangeDelIteratorsForLevel(ro, env_options_, lvl,
+ &level_range_del_iters);
+ std::unique_ptr<InternalIterator> level_range_del_iter(NewMergingIterator(
+ &cfd_->internal_comparator(),
+ level_range_del_iters.empty() ? nullptr : &level_range_del_iters[0],
+ static_cast<int>(level_range_del_iters.size())));
+
+ Status status = IngestedFileOverlapWithIteratorRange(
+ file_to_ingest, level_iter.get(), overlap_with_level);
+ if (status.ok() && *overlap_with_level == false) {
+ status = IngestedFileOverlapWithRangeDeletions(
+ file_to_ingest, level_range_del_iter.get(), overlap_with_level);
+ }
+ return status;
+}
+
+} // namespace rocksdb
+
+#endif // !ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/external_sst_file_ingestion_job.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/external_sst_file_ingestion_job.h b/thirdparty/rocksdb/db/external_sst_file_ingestion_job.h
new file mode 100644
index 0000000..2d0fade
--- /dev/null
+++ b/thirdparty/rocksdb/db/external_sst_file_ingestion_job.h
@@ -0,0 +1,171 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "db/column_family.h"
+#include "db/dbformat.h"
+#include "db/internal_stats.h"
+#include "db/snapshot_impl.h"
+#include "options/db_options.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/sst_file_writer.h"
+#include "util/autovector.h"
+
+namespace rocksdb {
+
+struct IngestedFileInfo {
+ // External file path
+ std::string external_file_path;
+ // Smallest user key in external file
+ std::string smallest_user_key;
+ // Largest user key in external file
+ std::string largest_user_key;
+ // Sequence number for keys in external file
+ SequenceNumber original_seqno;
+ // Offset of the global sequence number field in the file, will
+ // be zero if version is 1 (global seqno is not supported)
+ size_t global_seqno_offset;
+ // External file size
+ uint64_t file_size;
+ // total number of keys in external file
+ uint64_t num_entries;
+ // Id of column family this file shoule be ingested into
+ uint32_t cf_id;
+ // TableProperties read from external file
+ TableProperties table_properties;
+ // Version of external file
+ int version;
+
+ // FileDescriptor for the file inside the DB
+ FileDescriptor fd;
+ // file path that we picked for file inside the DB
+ std::string internal_file_path = "";
+ // Global sequence number that we picked for the file inside the DB
+ SequenceNumber assigned_seqno = 0;
+ // Level inside the DB we picked for the external file.
+ int picked_level = 0;
+
+ InternalKey smallest_internal_key() const {
+ return InternalKey(smallest_user_key, assigned_seqno,
+ ValueType::kTypeValue);
+ }
+
+ InternalKey largest_internal_key() const {
+ return InternalKey(largest_user_key, assigned_seqno, ValueType::kTypeValue);
+ }
+};
+
+class ExternalSstFileIngestionJob {
+ public:
+ ExternalSstFileIngestionJob(
+ Env* env, VersionSet* versions, ColumnFamilyData* cfd,
+ const ImmutableDBOptions& db_options, const EnvOptions& env_options,
+ SnapshotList* db_snapshots,
+ const IngestExternalFileOptions& ingestion_options)
+ : env_(env),
+ versions_(versions),
+ cfd_(cfd),
+ db_options_(db_options),
+ env_options_(env_options),
+ db_snapshots_(db_snapshots),
+ ingestion_options_(ingestion_options),
+ job_start_time_(env_->NowMicros()) {}
+
+ // Prepare the job by copying external files into the DB.
+ Status Prepare(const std::vector<std::string>& external_files_paths);
+
+ // Check if we need to flush the memtable before running the ingestion job
+ // This will be true if the files we are ingesting are overlapping with any
+ // key range in the memtable.
+ // REQUIRES: Mutex held
+ Status NeedsFlush(bool* flush_needed);
+
+ // Will execute the ingestion job and prepare edit() to be applied.
+ // REQUIRES: Mutex held
+ Status Run();
+
+ // Update column family stats.
+ // REQUIRES: Mutex held
+ void UpdateStats();
+
+ // Cleanup after successful/failed job
+ void Cleanup(const Status& status);
+
+ VersionEdit* edit() { return &edit_; }
+
+ const autovector<IngestedFileInfo>& files_to_ingest() const {
+ return files_to_ingest_;
+ }
+
+ private:
+ // Open the external file and populate `file_to_ingest` with all the
+ // external information we need to ingest this file.
+ Status GetIngestedFileInfo(const std::string& external_file,
+ IngestedFileInfo* file_to_ingest);
+
+ // Check if the files we are ingesting overlap with any memtable.
+ // REQUIRES: Mutex held
+ Status IngestedFilesOverlapWithMemtables(SuperVersion* sv, bool* overlap);
+
+ // Assign `file_to_ingest` the appropriate sequence number and the lowest
+ // possible level that it can be ingested to according to compaction_style.
+ // REQUIRES: Mutex held
+ Status AssignLevelAndSeqnoForIngestedFile(SuperVersion* sv,
+ bool force_global_seqno,
+ CompactionStyle compaction_style,
+ IngestedFileInfo* file_to_ingest,
+ SequenceNumber* assigned_seqno);
+
+ // File that we want to ingest behind always goes to the lowest level;
+ // we just check that it fits in the level, that DB allows ingest_behind,
+ // and that we don't have 0 seqnums at the upper levels.
+ // REQUIRES: Mutex held
+ Status CheckLevelForIngestedBehindFile(IngestedFileInfo* file_to_ingest);
+
+ // Set the file global sequence number to `seqno`
+ Status AssignGlobalSeqnoForIngestedFile(IngestedFileInfo* file_to_ingest,
+ SequenceNumber seqno);
+
+ // Check if `file_to_ingest` key range overlap with the range `iter` represent
+ // REQUIRES: Mutex held
+ Status IngestedFileOverlapWithIteratorRange(
+ const IngestedFileInfo* file_to_ingest, InternalIterator* iter,
+ bool* overlap);
+
+ // Check if `file_to_ingest` key range overlaps with any range deletions
+ // specified by `iter`.
+ // REQUIRES: Mutex held
+ Status IngestedFileOverlapWithRangeDeletions(
+ const IngestedFileInfo* file_to_ingest, InternalIterator* range_del_iter,
+ bool* overlap);
+
+ // Check if `file_to_ingest` key range overlap with level
+ // REQUIRES: Mutex held
+ Status IngestedFileOverlapWithLevel(SuperVersion* sv,
+ IngestedFileInfo* file_to_ingest, int lvl, bool* overlap_with_level);
+
+ // Check if `file_to_ingest` can fit in level `level`
+ // REQUIRES: Mutex held
+ bool IngestedFileFitInLevel(const IngestedFileInfo* file_to_ingest,
+ int level);
+
+ Env* env_;
+ VersionSet* versions_;
+ ColumnFamilyData* cfd_;
+ const ImmutableDBOptions& db_options_;
+ const EnvOptions& env_options_;
+ SnapshotList* db_snapshots_;
+ autovector<IngestedFileInfo> files_to_ingest_;
+ const IngestExternalFileOptions& ingestion_options_;
+ VersionEdit edit_;
+ uint64_t job_start_time_;
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/file_indexer.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/file_indexer.cc b/thirdparty/rocksdb/db/file_indexer.cc
new file mode 100644
index 0000000..abfa7cf
--- /dev/null
+++ b/thirdparty/rocksdb/db/file_indexer.cc
@@ -0,0 +1,214 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/file_indexer.h"
+#include <algorithm>
+#include <functional>
+#include "db/version_edit.h"
+#include "rocksdb/comparator.h"
+
+namespace rocksdb {
+
+FileIndexer::FileIndexer(const Comparator* ucmp)
+ : num_levels_(0), ucmp_(ucmp), level_rb_(nullptr) {}
+
+size_t FileIndexer::NumLevelIndex() const { return next_level_index_.size(); }
+
+size_t FileIndexer::LevelIndexSize(size_t level) const {
+ if (level >= next_level_index_.size()) {
+ return 0;
+ }
+ return next_level_index_[level].num_index;
+}
+
+void FileIndexer::GetNextLevelIndex(const size_t level, const size_t file_index,
+ const int cmp_smallest,
+ const int cmp_largest, int32_t* left_bound,
+ int32_t* right_bound) const {
+ assert(level > 0);
+
+ // Last level, no hint
+ if (level == num_levels_ - 1) {
+ *left_bound = 0;
+ *right_bound = -1;
+ return;
+ }
+
+ assert(level < num_levels_ - 1);
+ assert(static_cast<int32_t>(file_index) <= level_rb_[level]);
+
+ const IndexUnit* index_units = next_level_index_[level].index_units;
+ const auto& index = index_units[file_index];
+
+ if (cmp_smallest < 0) {
+ *left_bound = (level > 0 && file_index > 0)
+ ? index_units[file_index - 1].largest_lb
+ : 0;
+ *right_bound = index.smallest_rb;
+ } else if (cmp_smallest == 0) {
+ *left_bound = index.smallest_lb;
+ *right_bound = index.smallest_rb;
+ } else if (cmp_smallest > 0 && cmp_largest < 0) {
+ *left_bound = index.smallest_lb;
+ *right_bound = index.largest_rb;
+ } else if (cmp_largest == 0) {
+ *left_bound = index.largest_lb;
+ *right_bound = index.largest_rb;
+ } else if (cmp_largest > 0) {
+ *left_bound = index.largest_lb;
+ *right_bound = level_rb_[level + 1];
+ } else {
+ assert(false);
+ }
+
+ assert(*left_bound >= 0);
+ assert(*left_bound <= *right_bound + 1);
+ assert(*right_bound <= level_rb_[level + 1]);
+}
+
+void FileIndexer::UpdateIndex(Arena* arena, const size_t num_levels,
+ std::vector<FileMetaData*>* const files) {
+ if (files == nullptr) {
+ return;
+ }
+ if (num_levels == 0) { // uint_32 0-1 would cause bad behavior
+ num_levels_ = num_levels;
+ return;
+ }
+ assert(level_rb_ == nullptr); // level_rb_ should be init here
+
+ num_levels_ = num_levels;
+ next_level_index_.resize(num_levels);
+
+ char* mem = arena->AllocateAligned(num_levels_ * sizeof(int32_t));
+ level_rb_ = new (mem) int32_t[num_levels_];
+ for (size_t i = 0; i < num_levels_; i++) {
+ level_rb_[i] = -1;
+ }
+
+ // L1 - Ln-1
+ for (size_t level = 1; level < num_levels_ - 1; ++level) {
+ const auto& upper_files = files[level];
+ const int32_t upper_size = static_cast<int32_t>(upper_files.size());
+ const auto& lower_files = files[level + 1];
+ level_rb_[level] = static_cast<int32_t>(upper_files.size()) - 1;
+ if (upper_size == 0) {
+ continue;
+ }
+ IndexLevel& index_level = next_level_index_[level];
+ index_level.num_index = upper_size;
+ mem = arena->AllocateAligned(upper_size * sizeof(IndexUnit));
+ index_level.index_units = new (mem) IndexUnit[upper_size];
+
+ CalculateLB(
+ upper_files, lower_files, &index_level,
+ [this](const FileMetaData * a, const FileMetaData * b)->int {
+ return ucmp_->Compare(a->smallest.user_key(), b->largest.user_key());
+ },
+ [](IndexUnit* index, int32_t f_idx) { index->smallest_lb = f_idx; });
+ CalculateLB(
+ upper_files, lower_files, &index_level,
+ [this](const FileMetaData * a, const FileMetaData * b)->int {
+ return ucmp_->Compare(a->largest.user_key(), b->largest.user_key());
+ },
+ [](IndexUnit* index, int32_t f_idx) { index->largest_lb = f_idx; });
+ CalculateRB(
+ upper_files, lower_files, &index_level,
+ [this](const FileMetaData * a, const FileMetaData * b)->int {
+ return ucmp_->Compare(a->smallest.user_key(), b->smallest.user_key());
+ },
+ [](IndexUnit* index, int32_t f_idx) { index->smallest_rb = f_idx; });
+ CalculateRB(
+ upper_files, lower_files, &index_level,
+ [this](const FileMetaData * a, const FileMetaData * b)->int {
+ return ucmp_->Compare(a->largest.user_key(), b->smallest.user_key());
+ },
+ [](IndexUnit* index, int32_t f_idx) { index->largest_rb = f_idx; });
+ }
+
+ level_rb_[num_levels_ - 1] =
+ static_cast<int32_t>(files[num_levels_ - 1].size()) - 1;
+}
+
+void FileIndexer::CalculateLB(
+ const std::vector<FileMetaData*>& upper_files,
+ const std::vector<FileMetaData*>& lower_files, IndexLevel* index_level,
+ std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
+ std::function<void(IndexUnit*, int32_t)> set_index) {
+ const int32_t upper_size = static_cast<int32_t>(upper_files.size());
+ const int32_t lower_size = static_cast<int32_t>(lower_files.size());
+ int32_t upper_idx = 0;
+ int32_t lower_idx = 0;
+
+ IndexUnit* index = index_level->index_units;
+ while (upper_idx < upper_size && lower_idx < lower_size) {
+ int cmp = cmp_op(upper_files[upper_idx], lower_files[lower_idx]);
+
+ if (cmp == 0) {
+ set_index(&index[upper_idx], lower_idx);
+ ++upper_idx;
+ ++lower_idx;
+ } else if (cmp > 0) {
+ // Lower level's file (largest) is smaller, a key won't hit in that
+ // file. Move to next lower file
+ ++lower_idx;
+ } else {
+ // Lower level's file becomes larger, update the index, and
+ // move to the next upper file
+ set_index(&index[upper_idx], lower_idx);
+ ++upper_idx;
+ }
+ }
+
+ while (upper_idx < upper_size) {
+ // Lower files are exhausted, that means the remaining upper files are
+ // greater than any lower files. Set the index to be the lower level size.
+ set_index(&index[upper_idx], lower_size);
+ ++upper_idx;
+ }
+}
+
+void FileIndexer::CalculateRB(
+ const std::vector<FileMetaData*>& upper_files,
+ const std::vector<FileMetaData*>& lower_files, IndexLevel* index_level,
+ std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
+ std::function<void(IndexUnit*, int32_t)> set_index) {
+ const int32_t upper_size = static_cast<int32_t>(upper_files.size());
+ const int32_t lower_size = static_cast<int32_t>(lower_files.size());
+ int32_t upper_idx = upper_size - 1;
+ int32_t lower_idx = lower_size - 1;
+
+ IndexUnit* index = index_level->index_units;
+ while (upper_idx >= 0 && lower_idx >= 0) {
+ int cmp = cmp_op(upper_files[upper_idx], lower_files[lower_idx]);
+
+ if (cmp == 0) {
+ set_index(&index[upper_idx], lower_idx);
+ --upper_idx;
+ --lower_idx;
+ } else if (cmp < 0) {
+ // Lower level's file (smallest) is larger, a key won't hit in that
+ // file. Move to next lower file.
+ --lower_idx;
+ } else {
+ // Lower level's file becomes smaller, update the index, and move to
+ // the next the upper file
+ set_index(&index[upper_idx], lower_idx);
+ --upper_idx;
+ }
+ }
+ while (upper_idx >= 0) {
+ // Lower files are exhausted, that means the remaining upper files are
+ // smaller than any lower files. Set it to -1.
+ set_index(&index[upper_idx], -1);
+ --upper_idx;
+ }
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/file_indexer.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/file_indexer.h b/thirdparty/rocksdb/db/file_indexer.h
new file mode 100644
index 0000000..1bef3aa
--- /dev/null
+++ b/thirdparty/rocksdb/db/file_indexer.h
@@ -0,0 +1,142 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <cstdint>
+#include <functional>
+#include <limits>
+#include <vector>
+#include "port/port.h"
+#include "util/arena.h"
+#include "util/autovector.h"
+
+namespace rocksdb {
+
+class Comparator;
+struct FileMetaData;
+struct FdWithKeyRange;
+struct FileLevel;
+
+// The file tree structure in Version is prebuilt and the range of each file
+// is known. On Version::Get(), it uses binary search to find a potential file
+// and then check if a target key can be found in the file by comparing the key
+// to each file's smallest and largest key. The results of these comparisons
+// can be reused beyond checking if a key falls into a file's range.
+// With some pre-calculated knowledge, each key comparison that has been done
+// can serve as a hint to narrow down further searches: if a key compared to
+// be smaller than a file's smallest or largest, that comparison can be used
+// to find out the right bound of next binary search. Similarly, if a key
+// compared to be larger than a file's smallest or largest, it can be utilized
+// to find out the left bound of next binary search.
+// With these hints: it can greatly reduce the range of binary search,
+// especially for bottom levels, given that one file most likely overlaps with
+// only N files from level below (where N is max_bytes_for_level_multiplier).
+// So on level L, we will only look at ~N files instead of N^L files on the
+// naive approach.
+class FileIndexer {
+ public:
+ explicit FileIndexer(const Comparator* ucmp);
+
+ size_t NumLevelIndex() const;
+
+ size_t LevelIndexSize(size_t level) const;
+
+ // Return a file index range in the next level to search for a key based on
+ // smallest and largest key comparison for the current file specified by
+ // level and file_index. When *left_index < *right_index, both index should
+ // be valid and fit in the vector size.
+ void GetNextLevelIndex(const size_t level, const size_t file_index,
+ const int cmp_smallest, const int cmp_largest,
+ int32_t* left_bound, int32_t* right_bound) const;
+
+ void UpdateIndex(Arena* arena, const size_t num_levels,
+ std::vector<FileMetaData*>* const files);
+
+ enum {
+ // MSVC version 1800 still does not have constexpr for ::max()
+ kLevelMaxIndex = rocksdb::port::kMaxInt32
+ };
+
+ private:
+ size_t num_levels_;
+ const Comparator* ucmp_;
+
+ struct IndexUnit {
+ IndexUnit()
+ : smallest_lb(0), largest_lb(0), smallest_rb(-1), largest_rb(-1) {}
+ // During file search, a key is compared against smallest and largest
+ // from a FileMetaData. It can have 3 possible outcomes:
+ // (1) key is smaller than smallest, implying it is also smaller than
+ // larger. Precalculated index based on "smallest < smallest" can
+ // be used to provide right bound.
+ // (2) key is in between smallest and largest.
+ // Precalculated index based on "smallest > greatest" can be used to
+ // provide left bound.
+ // Precalculated index based on "largest < smallest" can be used to
+ // provide right bound.
+ // (3) key is larger than largest, implying it is also larger than smallest.
+ // Precalculated index based on "largest > largest" can be used to
+ // provide left bound.
+ //
+ // As a result, we will need to do:
+ // Compare smallest (<=) and largest keys from upper level file with
+ // smallest key from lower level to get a right bound.
+ // Compare smallest (>=) and largest keys from upper level file with
+ // largest key from lower level to get a left bound.
+ //
+ // Example:
+ // level 1: [50 - 60]
+ // level 2: [1 - 40], [45 - 55], [58 - 80]
+ // A key 35, compared to be less than 50, 3rd file on level 2 can be
+ // skipped according to rule (1). LB = 0, RB = 1.
+ // A key 53, sits in the middle 50 and 60. 1st file on level 2 can be
+ // skipped according to rule (2)-a, but the 3rd file cannot be skipped
+ // because 60 is greater than 58. LB = 1, RB = 2.
+ // A key 70, compared to be larger than 60. 1st and 2nd file can be skipped
+ // according to rule (3). LB = 2, RB = 2.
+ //
+ // Point to a left most file in a lower level that may contain a key,
+ // which compares greater than smallest of a FileMetaData (upper level)
+ int32_t smallest_lb;
+ // Point to a left most file in a lower level that may contain a key,
+ // which compares greater than largest of a FileMetaData (upper level)
+ int32_t largest_lb;
+ // Point to a right most file in a lower level that may contain a key,
+ // which compares smaller than smallest of a FileMetaData (upper level)
+ int32_t smallest_rb;
+ // Point to a right most file in a lower level that may contain a key,
+ // which compares smaller than largest of a FileMetaData (upper level)
+ int32_t largest_rb;
+ };
+
+ // Data structure to store IndexUnits in a whole level
+ struct IndexLevel {
+ size_t num_index;
+ IndexUnit* index_units;
+
+ IndexLevel() : num_index(0), index_units(nullptr) {}
+ };
+
+ void CalculateLB(
+ const std::vector<FileMetaData*>& upper_files,
+ const std::vector<FileMetaData*>& lower_files, IndexLevel* index_level,
+ std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
+ std::function<void(IndexUnit*, int32_t)> set_index);
+
+ void CalculateRB(
+ const std::vector<FileMetaData*>& upper_files,
+ const std::vector<FileMetaData*>& lower_files, IndexLevel* index_level,
+ std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
+ std::function<void(IndexUnit*, int32_t)> set_index);
+
+ autovector<IndexLevel> next_level_index_;
+ int32_t* level_rb_;
+};
+
+} // namespace rocksdb
[37/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/crc32c.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/crc32c.cc b/thirdparty/leveldb-1.18/util/crc32c.cc
deleted file mode 100755
index 6db9e77..0000000
--- a/thirdparty/leveldb-1.18/util/crc32c.cc
+++ /dev/null
@@ -1,332 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A portable implementation of crc32c, optimized to handle
-// four bytes at a time.
-
-#include "util/crc32c.h"
-
-#include <stdint.h>
-#include "util/coding.h"
-
-namespace leveldb {
-namespace crc32c {
-
-static const uint32_t table0_[256] = {
- 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4,
- 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb,
- 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b,
- 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24,
- 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b,
- 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384,
- 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54,
- 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b,
- 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a,
- 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35,
- 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5,
- 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa,
- 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45,
- 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a,
- 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a,
- 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595,
- 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48,
- 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957,
- 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687,
- 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198,
- 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927,
- 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38,
- 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8,
- 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7,
- 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096,
- 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789,
- 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859,
- 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46,
- 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9,
- 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6,
- 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36,
- 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829,
- 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c,
- 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93,
- 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043,
- 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c,
- 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3,
- 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc,
- 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c,
- 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033,
- 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652,
- 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d,
- 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d,
- 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982,
- 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d,
- 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622,
- 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2,
- 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed,
- 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530,
- 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f,
- 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff,
- 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0,
- 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f,
- 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540,
- 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90,
- 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f,
- 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee,
- 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1,
- 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321,
- 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e,
- 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81,
- 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e,
- 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e,
- 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351
-};
-static const uint32_t table1_[256] = {
- 0x00000000, 0x13a29877, 0x274530ee, 0x34e7a899,
- 0x4e8a61dc, 0x5d28f9ab, 0x69cf5132, 0x7a6dc945,
- 0x9d14c3b8, 0x8eb65bcf, 0xba51f356, 0xa9f36b21,
- 0xd39ea264, 0xc03c3a13, 0xf4db928a, 0xe7790afd,
- 0x3fc5f181, 0x2c6769f6, 0x1880c16f, 0x0b225918,
- 0x714f905d, 0x62ed082a, 0x560aa0b3, 0x45a838c4,
- 0xa2d13239, 0xb173aa4e, 0x859402d7, 0x96369aa0,
- 0xec5b53e5, 0xfff9cb92, 0xcb1e630b, 0xd8bcfb7c,
- 0x7f8be302, 0x6c297b75, 0x58ced3ec, 0x4b6c4b9b,
- 0x310182de, 0x22a31aa9, 0x1644b230, 0x05e62a47,
- 0xe29f20ba, 0xf13db8cd, 0xc5da1054, 0xd6788823,
- 0xac154166, 0xbfb7d911, 0x8b507188, 0x98f2e9ff,
- 0x404e1283, 0x53ec8af4, 0x670b226d, 0x74a9ba1a,
- 0x0ec4735f, 0x1d66eb28, 0x298143b1, 0x3a23dbc6,
- 0xdd5ad13b, 0xcef8494c, 0xfa1fe1d5, 0xe9bd79a2,
- 0x93d0b0e7, 0x80722890, 0xb4958009, 0xa737187e,
- 0xff17c604, 0xecb55e73, 0xd852f6ea, 0xcbf06e9d,
- 0xb19da7d8, 0xa23f3faf, 0x96d89736, 0x857a0f41,
- 0x620305bc, 0x71a19dcb, 0x45463552, 0x56e4ad25,
- 0x2c896460, 0x3f2bfc17, 0x0bcc548e, 0x186eccf9,
- 0xc0d23785, 0xd370aff2, 0xe797076b, 0xf4359f1c,
- 0x8e585659, 0x9dface2e, 0xa91d66b7, 0xbabffec0,
- 0x5dc6f43d, 0x4e646c4a, 0x7a83c4d3, 0x69215ca4,
- 0x134c95e1, 0x00ee0d96, 0x3409a50f, 0x27ab3d78,
- 0x809c2506, 0x933ebd71, 0xa7d915e8, 0xb47b8d9f,
- 0xce1644da, 0xddb4dcad, 0xe9537434, 0xfaf1ec43,
- 0x1d88e6be, 0x0e2a7ec9, 0x3acdd650, 0x296f4e27,
- 0x53028762, 0x40a01f15, 0x7447b78c, 0x67e52ffb,
- 0xbf59d487, 0xacfb4cf0, 0x981ce469, 0x8bbe7c1e,
- 0xf1d3b55b, 0xe2712d2c, 0xd69685b5, 0xc5341dc2,
- 0x224d173f, 0x31ef8f48, 0x050827d1, 0x16aabfa6,
- 0x6cc776e3, 0x7f65ee94, 0x4b82460d, 0x5820de7a,
- 0xfbc3faf9, 0xe861628e, 0xdc86ca17, 0xcf245260,
- 0xb5499b25, 0xa6eb0352, 0x920cabcb, 0x81ae33bc,
- 0x66d73941, 0x7575a136, 0x419209af, 0x523091d8,
- 0x285d589d, 0x3bffc0ea, 0x0f186873, 0x1cbaf004,
- 0xc4060b78, 0xd7a4930f, 0xe3433b96, 0xf0e1a3e1,
- 0x8a8c6aa4, 0x992ef2d3, 0xadc95a4a, 0xbe6bc23d,
- 0x5912c8c0, 0x4ab050b7, 0x7e57f82e, 0x6df56059,
- 0x1798a91c, 0x043a316b, 0x30dd99f2, 0x237f0185,
- 0x844819fb, 0x97ea818c, 0xa30d2915, 0xb0afb162,
- 0xcac27827, 0xd960e050, 0xed8748c9, 0xfe25d0be,
- 0x195cda43, 0x0afe4234, 0x3e19eaad, 0x2dbb72da,
- 0x57d6bb9f, 0x447423e8, 0x70938b71, 0x63311306,
- 0xbb8de87a, 0xa82f700d, 0x9cc8d894, 0x8f6a40e3,
- 0xf50789a6, 0xe6a511d1, 0xd242b948, 0xc1e0213f,
- 0x26992bc2, 0x353bb3b5, 0x01dc1b2c, 0x127e835b,
- 0x68134a1e, 0x7bb1d269, 0x4f567af0, 0x5cf4e287,
- 0x04d43cfd, 0x1776a48a, 0x23910c13, 0x30339464,
- 0x4a5e5d21, 0x59fcc556, 0x6d1b6dcf, 0x7eb9f5b8,
- 0x99c0ff45, 0x8a626732, 0xbe85cfab, 0xad2757dc,
- 0xd74a9e99, 0xc4e806ee, 0xf00fae77, 0xe3ad3600,
- 0x3b11cd7c, 0x28b3550b, 0x1c54fd92, 0x0ff665e5,
- 0x759baca0, 0x663934d7, 0x52de9c4e, 0x417c0439,
- 0xa6050ec4, 0xb5a796b3, 0x81403e2a, 0x92e2a65d,
- 0xe88f6f18, 0xfb2df76f, 0xcfca5ff6, 0xdc68c781,
- 0x7b5fdfff, 0x68fd4788, 0x5c1aef11, 0x4fb87766,
- 0x35d5be23, 0x26772654, 0x12908ecd, 0x013216ba,
- 0xe64b1c47, 0xf5e98430, 0xc10e2ca9, 0xd2acb4de,
- 0xa8c17d9b, 0xbb63e5ec, 0x8f844d75, 0x9c26d502,
- 0x449a2e7e, 0x5738b609, 0x63df1e90, 0x707d86e7,
- 0x0a104fa2, 0x19b2d7d5, 0x2d557f4c, 0x3ef7e73b,
- 0xd98eedc6, 0xca2c75b1, 0xfecbdd28, 0xed69455f,
- 0x97048c1a, 0x84a6146d, 0xb041bcf4, 0xa3e32483
-};
-static const uint32_t table2_[256] = {
- 0x00000000, 0xa541927e, 0x4f6f520d, 0xea2ec073,
- 0x9edea41a, 0x3b9f3664, 0xd1b1f617, 0x74f06469,
- 0x38513ec5, 0x9d10acbb, 0x773e6cc8, 0xd27ffeb6,
- 0xa68f9adf, 0x03ce08a1, 0xe9e0c8d2, 0x4ca15aac,
- 0x70a27d8a, 0xd5e3eff4, 0x3fcd2f87, 0x9a8cbdf9,
- 0xee7cd990, 0x4b3d4bee, 0xa1138b9d, 0x045219e3,
- 0x48f3434f, 0xedb2d131, 0x079c1142, 0xa2dd833c,
- 0xd62de755, 0x736c752b, 0x9942b558, 0x3c032726,
- 0xe144fb14, 0x4405696a, 0xae2ba919, 0x0b6a3b67,
- 0x7f9a5f0e, 0xdadbcd70, 0x30f50d03, 0x95b49f7d,
- 0xd915c5d1, 0x7c5457af, 0x967a97dc, 0x333b05a2,
- 0x47cb61cb, 0xe28af3b5, 0x08a433c6, 0xade5a1b8,
- 0x91e6869e, 0x34a714e0, 0xde89d493, 0x7bc846ed,
- 0x0f382284, 0xaa79b0fa, 0x40577089, 0xe516e2f7,
- 0xa9b7b85b, 0x0cf62a25, 0xe6d8ea56, 0x43997828,
- 0x37691c41, 0x92288e3f, 0x78064e4c, 0xdd47dc32,
- 0xc76580d9, 0x622412a7, 0x880ad2d4, 0x2d4b40aa,
- 0x59bb24c3, 0xfcfab6bd, 0x16d476ce, 0xb395e4b0,
- 0xff34be1c, 0x5a752c62, 0xb05bec11, 0x151a7e6f,
- 0x61ea1a06, 0xc4ab8878, 0x2e85480b, 0x8bc4da75,
- 0xb7c7fd53, 0x12866f2d, 0xf8a8af5e, 0x5de93d20,
- 0x29195949, 0x8c58cb37, 0x66760b44, 0xc337993a,
- 0x8f96c396, 0x2ad751e8, 0xc0f9919b, 0x65b803e5,
- 0x1148678c, 0xb409f5f2, 0x5e273581, 0xfb66a7ff,
- 0x26217bcd, 0x8360e9b3, 0x694e29c0, 0xcc0fbbbe,
- 0xb8ffdfd7, 0x1dbe4da9, 0xf7908dda, 0x52d11fa4,
- 0x1e704508, 0xbb31d776, 0x511f1705, 0xf45e857b,
- 0x80aee112, 0x25ef736c, 0xcfc1b31f, 0x6a802161,
- 0x56830647, 0xf3c29439, 0x19ec544a, 0xbcadc634,
- 0xc85da25d, 0x6d1c3023, 0x8732f050, 0x2273622e,
- 0x6ed23882, 0xcb93aafc, 0x21bd6a8f, 0x84fcf8f1,
- 0xf00c9c98, 0x554d0ee6, 0xbf63ce95, 0x1a225ceb,
- 0x8b277743, 0x2e66e53d, 0xc448254e, 0x6109b730,
- 0x15f9d359, 0xb0b84127, 0x5a968154, 0xffd7132a,
- 0xb3764986, 0x1637dbf8, 0xfc191b8b, 0x595889f5,
- 0x2da8ed9c, 0x88e97fe2, 0x62c7bf91, 0xc7862def,
- 0xfb850ac9, 0x5ec498b7, 0xb4ea58c4, 0x11abcaba,
- 0x655baed3, 0xc01a3cad, 0x2a34fcde, 0x8f756ea0,
- 0xc3d4340c, 0x6695a672, 0x8cbb6601, 0x29faf47f,
- 0x5d0a9016, 0xf84b0268, 0x1265c21b, 0xb7245065,
- 0x6a638c57, 0xcf221e29, 0x250cde5a, 0x804d4c24,
- 0xf4bd284d, 0x51fcba33, 0xbbd27a40, 0x1e93e83e,
- 0x5232b292, 0xf77320ec, 0x1d5de09f, 0xb81c72e1,
- 0xccec1688, 0x69ad84f6, 0x83834485, 0x26c2d6fb,
- 0x1ac1f1dd, 0xbf8063a3, 0x55aea3d0, 0xf0ef31ae,
- 0x841f55c7, 0x215ec7b9, 0xcb7007ca, 0x6e3195b4,
- 0x2290cf18, 0x87d15d66, 0x6dff9d15, 0xc8be0f6b,
- 0xbc4e6b02, 0x190ff97c, 0xf321390f, 0x5660ab71,
- 0x4c42f79a, 0xe90365e4, 0x032da597, 0xa66c37e9,
- 0xd29c5380, 0x77ddc1fe, 0x9df3018d, 0x38b293f3,
- 0x7413c95f, 0xd1525b21, 0x3b7c9b52, 0x9e3d092c,
- 0xeacd6d45, 0x4f8cff3b, 0xa5a23f48, 0x00e3ad36,
- 0x3ce08a10, 0x99a1186e, 0x738fd81d, 0xd6ce4a63,
- 0xa23e2e0a, 0x077fbc74, 0xed517c07, 0x4810ee79,
- 0x04b1b4d5, 0xa1f026ab, 0x4bdee6d8, 0xee9f74a6,
- 0x9a6f10cf, 0x3f2e82b1, 0xd50042c2, 0x7041d0bc,
- 0xad060c8e, 0x08479ef0, 0xe2695e83, 0x4728ccfd,
- 0x33d8a894, 0x96993aea, 0x7cb7fa99, 0xd9f668e7,
- 0x9557324b, 0x3016a035, 0xda386046, 0x7f79f238,
- 0x0b899651, 0xaec8042f, 0x44e6c45c, 0xe1a75622,
- 0xdda47104, 0x78e5e37a, 0x92cb2309, 0x378ab177,
- 0x437ad51e, 0xe63b4760, 0x0c158713, 0xa954156d,
- 0xe5f54fc1, 0x40b4ddbf, 0xaa9a1dcc, 0x0fdb8fb2,
- 0x7b2bebdb, 0xde6a79a5, 0x3444b9d6, 0x91052ba8
-};
-static const uint32_t table3_[256] = {
- 0x00000000, 0xdd45aab8, 0xbf672381, 0x62228939,
- 0x7b2231f3, 0xa6679b4b, 0xc4451272, 0x1900b8ca,
- 0xf64463e6, 0x2b01c95e, 0x49234067, 0x9466eadf,
- 0x8d665215, 0x5023f8ad, 0x32017194, 0xef44db2c,
- 0xe964b13d, 0x34211b85, 0x560392bc, 0x8b463804,
- 0x924680ce, 0x4f032a76, 0x2d21a34f, 0xf06409f7,
- 0x1f20d2db, 0xc2657863, 0xa047f15a, 0x7d025be2,
- 0x6402e328, 0xb9474990, 0xdb65c0a9, 0x06206a11,
- 0xd725148b, 0x0a60be33, 0x6842370a, 0xb5079db2,
- 0xac072578, 0x71428fc0, 0x136006f9, 0xce25ac41,
- 0x2161776d, 0xfc24ddd5, 0x9e0654ec, 0x4343fe54,
- 0x5a43469e, 0x8706ec26, 0xe524651f, 0x3861cfa7,
- 0x3e41a5b6, 0xe3040f0e, 0x81268637, 0x5c632c8f,
- 0x45639445, 0x98263efd, 0xfa04b7c4, 0x27411d7c,
- 0xc805c650, 0x15406ce8, 0x7762e5d1, 0xaa274f69,
- 0xb327f7a3, 0x6e625d1b, 0x0c40d422, 0xd1057e9a,
- 0xaba65fe7, 0x76e3f55f, 0x14c17c66, 0xc984d6de,
- 0xd0846e14, 0x0dc1c4ac, 0x6fe34d95, 0xb2a6e72d,
- 0x5de23c01, 0x80a796b9, 0xe2851f80, 0x3fc0b538,
- 0x26c00df2, 0xfb85a74a, 0x99a72e73, 0x44e284cb,
- 0x42c2eeda, 0x9f874462, 0xfda5cd5b, 0x20e067e3,
- 0x39e0df29, 0xe4a57591, 0x8687fca8, 0x5bc25610,
- 0xb4868d3c, 0x69c32784, 0x0be1aebd, 0xd6a40405,
- 0xcfa4bccf, 0x12e11677, 0x70c39f4e, 0xad8635f6,
- 0x7c834b6c, 0xa1c6e1d4, 0xc3e468ed, 0x1ea1c255,
- 0x07a17a9f, 0xdae4d027, 0xb8c6591e, 0x6583f3a6,
- 0x8ac7288a, 0x57828232, 0x35a00b0b, 0xe8e5a1b3,
- 0xf1e51979, 0x2ca0b3c1, 0x4e823af8, 0x93c79040,
- 0x95e7fa51, 0x48a250e9, 0x2a80d9d0, 0xf7c57368,
- 0xeec5cba2, 0x3380611a, 0x51a2e823, 0x8ce7429b,
- 0x63a399b7, 0xbee6330f, 0xdcc4ba36, 0x0181108e,
- 0x1881a844, 0xc5c402fc, 0xa7e68bc5, 0x7aa3217d,
- 0x52a0c93f, 0x8fe56387, 0xedc7eabe, 0x30824006,
- 0x2982f8cc, 0xf4c75274, 0x96e5db4d, 0x4ba071f5,
- 0xa4e4aad9, 0x79a10061, 0x1b838958, 0xc6c623e0,
- 0xdfc69b2a, 0x02833192, 0x60a1b8ab, 0xbde41213,
- 0xbbc47802, 0x6681d2ba, 0x04a35b83, 0xd9e6f13b,
- 0xc0e649f1, 0x1da3e349, 0x7f816a70, 0xa2c4c0c8,
- 0x4d801be4, 0x90c5b15c, 0xf2e73865, 0x2fa292dd,
- 0x36a22a17, 0xebe780af, 0x89c50996, 0x5480a32e,
- 0x8585ddb4, 0x58c0770c, 0x3ae2fe35, 0xe7a7548d,
- 0xfea7ec47, 0x23e246ff, 0x41c0cfc6, 0x9c85657e,
- 0x73c1be52, 0xae8414ea, 0xcca69dd3, 0x11e3376b,
- 0x08e38fa1, 0xd5a62519, 0xb784ac20, 0x6ac10698,
- 0x6ce16c89, 0xb1a4c631, 0xd3864f08, 0x0ec3e5b0,
- 0x17c35d7a, 0xca86f7c2, 0xa8a47efb, 0x75e1d443,
- 0x9aa50f6f, 0x47e0a5d7, 0x25c22cee, 0xf8878656,
- 0xe1873e9c, 0x3cc29424, 0x5ee01d1d, 0x83a5b7a5,
- 0xf90696d8, 0x24433c60, 0x4661b559, 0x9b241fe1,
- 0x8224a72b, 0x5f610d93, 0x3d4384aa, 0xe0062e12,
- 0x0f42f53e, 0xd2075f86, 0xb025d6bf, 0x6d607c07,
- 0x7460c4cd, 0xa9256e75, 0xcb07e74c, 0x16424df4,
- 0x106227e5, 0xcd278d5d, 0xaf050464, 0x7240aedc,
- 0x6b401616, 0xb605bcae, 0xd4273597, 0x09629f2f,
- 0xe6264403, 0x3b63eebb, 0x59416782, 0x8404cd3a,
- 0x9d0475f0, 0x4041df48, 0x22635671, 0xff26fcc9,
- 0x2e238253, 0xf36628eb, 0x9144a1d2, 0x4c010b6a,
- 0x5501b3a0, 0x88441918, 0xea669021, 0x37233a99,
- 0xd867e1b5, 0x05224b0d, 0x6700c234, 0xba45688c,
- 0xa345d046, 0x7e007afe, 0x1c22f3c7, 0xc167597f,
- 0xc747336e, 0x1a0299d6, 0x782010ef, 0xa565ba57,
- 0xbc65029d, 0x6120a825, 0x0302211c, 0xde478ba4,
- 0x31035088, 0xec46fa30, 0x8e647309, 0x5321d9b1,
- 0x4a21617b, 0x9764cbc3, 0xf54642fa, 0x2803e842
-};
-
-// Used to fetch a naturally-aligned 32-bit word in little endian byte-order
-static inline uint32_t LE_LOAD32(const uint8_t *p) {
- return DecodeFixed32(reinterpret_cast<const char*>(p));
-}
-
-uint32_t Extend(uint32_t crc, const char* buf, size_t size) {
- const uint8_t *p = reinterpret_cast<const uint8_t *>(buf);
- const uint8_t *e = p + size;
- uint32_t l = crc ^ 0xffffffffu;
-
-#define STEP1 do { \
- int c = (l & 0xff) ^ *p++; \
- l = table0_[c] ^ (l >> 8); \
-} while (0)
-#define STEP4 do { \
- uint32_t c = l ^ LE_LOAD32(p); \
- p += 4; \
- l = table3_[c & 0xff] ^ \
- table2_[(c >> 8) & 0xff] ^ \
- table1_[(c >> 16) & 0xff] ^ \
- table0_[c >> 24]; \
-} while (0)
-
- // Point x at first 4-byte aligned byte in string. This might be
- // just past the end of the string.
- const uintptr_t pval = reinterpret_cast<uintptr_t>(p);
- const uint8_t* x = reinterpret_cast<const uint8_t*>(((pval + 3) >> 2) << 2);
- if (x <= e) {
- // Process bytes until finished or p is 4-byte aligned
- while (p != x) {
- STEP1;
- }
- }
- // Process bytes 16 at a time
- while ((e-p) >= 16) {
- STEP4; STEP4; STEP4; STEP4;
- }
- // Process bytes 4 at a time
- while ((e-p) >= 4) {
- STEP4;
- }
- // Process the last few bytes
- while (p != e) {
- STEP1;
- }
-#undef STEP4
-#undef STEP1
- return l ^ 0xffffffffu;
-}
-
-} // namespace crc32c
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/crc32c.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/crc32c.h b/thirdparty/leveldb-1.18/util/crc32c.h
deleted file mode 100755
index 1d7e5c0..0000000
--- a/thirdparty/leveldb-1.18/util/crc32c.h
+++ /dev/null
@@ -1,45 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_UTIL_CRC32C_H_
-#define STORAGE_LEVELDB_UTIL_CRC32C_H_
-
-#include <stddef.h>
-#include <stdint.h>
-
-namespace leveldb {
-namespace crc32c {
-
-// Return the crc32c of concat(A, data[0,n-1]) where init_crc is the
-// crc32c of some string A. Extend() is often used to maintain the
-// crc32c of a stream of data.
-extern uint32_t Extend(uint32_t init_crc, const char* data, size_t n);
-
-// Return the crc32c of data[0,n-1]
-inline uint32_t Value(const char* data, size_t n) {
- return Extend(0, data, n);
-}
-
-static const uint32_t kMaskDelta = 0xa282ead8ul;
-
-// Return a masked representation of crc.
-//
-// Motivation: it is problematic to compute the CRC of a string that
-// contains embedded CRCs. Therefore we recommend that CRCs stored
-// somewhere (e.g., in files) should be masked before being stored.
-inline uint32_t Mask(uint32_t crc) {
- // Rotate right by 15 bits and add a constant.
- return ((crc >> 15) | (crc << 17)) + kMaskDelta;
-}
-
-// Return the crc whose masked representation is masked_crc.
-inline uint32_t Unmask(uint32_t masked_crc) {
- uint32_t rot = masked_crc - kMaskDelta;
- return ((rot >> 17) | (rot << 15));
-}
-
-} // namespace crc32c
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_UTIL_CRC32C_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/crc32c_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/crc32c_test.cc b/thirdparty/leveldb-1.18/util/crc32c_test.cc
deleted file mode 100755
index 78ae483..0000000
--- a/thirdparty/leveldb-1.18/util/crc32c_test.cc
+++ /dev/null
@@ -1,72 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "util/crc32c.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-namespace crc32c {
-
-class CRC { };
-
-TEST(CRC, StandardResults) {
- // From rfc3720 section B.4.
- char buf[32];
-
- memset(buf, 0, sizeof(buf));
- ASSERT_EQ(static_cast<uint32_t>(0x8a9136aa), Value(buf, sizeof(buf)));
-
- memset(buf, 0xff, sizeof(buf));
- ASSERT_EQ(static_cast<uint32_t>(0x62a8ab43), Value(buf, sizeof(buf)));
-
- for (int i = 0; i < 32; i++) {
- buf[i] = i;
- }
- ASSERT_EQ(static_cast<uint32_t>(0x46dd794e), Value(buf, sizeof(buf)));
-
- for (int i = 0; i < 32; i++) {
- buf[i] = 31 - i;
- }
- ASSERT_EQ(static_cast<uint32_t>(0x113fdb5c), Value(buf, sizeof(buf)));
-
- unsigned char data[48] = {
- 0x01, 0xc0, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x14, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x04, 0x00,
- 0x00, 0x00, 0x00, 0x14,
- 0x00, 0x00, 0x00, 0x18,
- 0x28, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x02, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- };
- ASSERT_EQ(static_cast<uint32_t>(0xd9963a56), Value(reinterpret_cast<char*>(data), sizeof(data)));
-}
-
-TEST(CRC, Values) {
- ASSERT_NE(Value("a", 1), Value("foo", 3));
-}
-
-TEST(CRC, Extend) {
- ASSERT_EQ(Value("hello world", 11),
- Extend(Value("hello ", 6), "world", 5));
-}
-
-TEST(CRC, Mask) {
- uint32_t crc = Value("foo", 3);
- ASSERT_NE(crc, Mask(crc));
- ASSERT_NE(crc, Mask(Mask(crc)));
- ASSERT_EQ(crc, Unmask(Mask(crc)));
- ASSERT_EQ(crc, Unmask(Unmask(Mask(Mask(crc)))));
-}
-
-} // namespace crc32c
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/env.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/env.cc b/thirdparty/leveldb-1.18/util/env.cc
deleted file mode 100755
index c2600e9..0000000
--- a/thirdparty/leveldb-1.18/util/env.cc
+++ /dev/null
@@ -1,96 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/env.h"
-
-namespace leveldb {
-
-Env::~Env() {
-}
-
-SequentialFile::~SequentialFile() {
-}
-
-RandomAccessFile::~RandomAccessFile() {
-}
-
-WritableFile::~WritableFile() {
-}
-
-Logger::~Logger() {
-}
-
-FileLock::~FileLock() {
-}
-
-void Log(Logger* info_log, const char* format, ...) {
- if (info_log != NULL) {
- va_list ap;
- va_start(ap, format);
- info_log->Logv(format, ap);
- va_end(ap);
- }
-}
-
-static Status DoWriteStringToFile(Env* env, const Slice& data,
- const std::string& fname,
- bool should_sync) {
- WritableFile* file;
- Status s = env->NewWritableFile(fname, &file);
- if (!s.ok()) {
- return s;
- }
- s = file->Append(data);
- if (s.ok() && should_sync) {
- s = file->Sync();
- }
- if (s.ok()) {
- s = file->Close();
- }
- delete file; // Will auto-close if we did not close above
- if (!s.ok()) {
- env->DeleteFile(fname);
- }
- return s;
-}
-
-Status WriteStringToFile(Env* env, const Slice& data,
- const std::string& fname) {
- return DoWriteStringToFile(env, data, fname, false);
-}
-
-Status WriteStringToFileSync(Env* env, const Slice& data,
- const std::string& fname) {
- return DoWriteStringToFile(env, data, fname, true);
-}
-
-Status ReadFileToString(Env* env, const std::string& fname, std::string* data) {
- data->clear();
- SequentialFile* file;
- Status s = env->NewSequentialFile(fname, &file);
- if (!s.ok()) {
- return s;
- }
- static const int kBufferSize = 8192;
- char* space = new char[kBufferSize];
- while (true) {
- Slice fragment;
- s = file->Read(kBufferSize, &fragment, space);
- if (!s.ok()) {
- break;
- }
- data->append(fragment.data(), fragment.size());
- if (fragment.empty()) {
- break;
- }
- }
- delete[] space;
- delete file;
- return s;
-}
-
-EnvWrapper::~EnvWrapper() {
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/env_boost.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/env_boost.cc b/thirdparty/leveldb-1.18/util/env_boost.cc
deleted file mode 100755
index 9149706..0000000
--- a/thirdparty/leveldb-1.18/util/env_boost.cc
+++ /dev/null
@@ -1,726 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <atomic>
-#include <chrono>
-#include <cstdint>
-#include <deque>
-#include <set>
-#include <thread>
-#include <functional>
-#include <memory>
-#include <condition_variable>
-#include <mutex>
-
-#ifdef WIN32
-#include <windows.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <time.h>
-#include <io.h>
-#else
-#include <dirent.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/param.h>
-#include <time.h>
-#include <unistd.h>
-#endif
-#if defined(LEVELDB_PLATFORM_ANDROID)
-#include <sys/stat.h>
-#endif
-#include "leveldb/env.h"
-#include "leveldb/slice.h"
-
-#ifdef WIN32
-#include "util/win_logger.h"
-#else
-#include "util/posix_logger.h"
-#endif
-#include "port/port.h"
-#include "util/logging.h"
-
-#ifdef __linux
-#include <sys/sysinfo.h>
-#include <linux/unistd.h>
-#endif
-
-#include <fstream>
-
-// Boost includes - see WINDOWS file to see which modules to install
-#include <boost/filesystem/convenience.hpp>
-#include <boost/interprocess/sync/file_lock.hpp>
-#include <boost/lexical_cast.hpp>
-
-namespace leveldb {
-namespace {
-
-static Status IOError(const std::string& context, int err_number) {
- return Status::IOError(context, strerror(err_number));
-}
-
-static Status WinIOError(const std::string & context, DWORD err)
-{
- std::string err_mess;
- LPTSTR lpErrorText = NULL;
- if (!::FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER,
- 0,
- err,
- 0,
- (LPTSTR)&lpErrorText,
- 32000,
- 0))
- {
- err_mess = "unknown error";
- }
- else
- {
- err_mess = lpErrorText;
- ::LocalFree(lpErrorText);
- }
- return Status::IOError(context, err_mess);
-}
-
-// returns the ID of the current process
-static std::uint32_t current_process_id(void) {
-#ifdef _WIN32
- return static_cast<std::uint32_t>(::GetCurrentProcessId());
-#else
- return static_cast<std::uint32_t>(::getpid());
-#endif
-}
-
-// returns the ID of the current thread
-static std::uint32_t current_thread_id(void) {
-#ifdef _WIN32
- return static_cast<std::uint32_t>(::GetCurrentThreadId());
-#else
-#ifdef __linux
- return static_cast<std::uint32_t>(::syscall(__NR_gettid));
-#else
- // just return the pid
- return current_process_id();
-#endif
-#endif
-}
-
-class WindowsSequentialFile: public SequentialFile {
- private:
- std::string filename_;
- HANDLE handle_;
-
- public:
- WindowsSequentialFile(const std::string& fname, HANDLE h)
- : filename_(fname), handle_(h) { }
- virtual ~WindowsSequentialFile() { CloseHandle(handle_); }
-
- virtual Status Read(size_t n, Slice* result, char* scratch) {
- size_t r = 0;
-
- if (!::ReadFile(handle_, scratch, static_cast<DWORD>(n), reinterpret_cast<DWORD *>(&r), NULL))
- {
- return WinIOError(filename_, GetLastError());
- }
- *result = Slice(scratch, r);
- return Status::OK();
- }
-
- virtual Status Skip(uint64_t n) {
- const LONG lo = static_cast<LONG>(n & 0xffffffff);
- LONG hi = static_cast<LONG>(n >> 32);
- if (SetFilePointer(handle_, lo, &hi, FILE_BEGIN) == INVALID_SET_FILE_POINTER)
- {
- return WinIOError(filename_, GetLastError());
- }
- return Status::OK();
- }
-};
-
-class WindowsRandomAccessFile: public RandomAccessFile {
- private:
- std::string filename_;
- const void * region_;
- uint64_t length_;
- HANDLE handle_;
-
- public:
- WindowsRandomAccessFile(const std::string& fname, const void * base, uint64_t l, HANDLE h)
- : filename_(fname), region_(base), length_(l), handle_(h) { }
- virtual ~WindowsRandomAccessFile()
- {
- if (region_)
- {
- UnmapViewOfFile(region_);
- }
- CloseHandle(handle_);
- }
-
- virtual Status Read(uint64_t offset, size_t n, Slice* result,
- char* scratch) const {
- Status s;
- if (offset + static_cast<uint64_t>(n) > length_) {
- *result = Slice();
- s = IOError(filename_, EINVAL);
- } else {
- *result = Slice(reinterpret_cast<const char*>(region_) + offset, n);
- }
- return s;
- }
-};
-
-class WindowsWritableFile : public WritableFile {
-
-private:
- boost::filesystem::path filename_;
- HANDLE handle_;
-
-public:
- WindowsWritableFile(const std::string& fname, HANDLE h)
- : filename_(fname), handle_(h) { }
-
- ~WindowsWritableFile() { CloseHandle(handle_); }
-
-public:
- virtual Status Append(const Slice& data) {
-
- DWORD written = 0;
-
- if (!::WriteFile(handle_, data.data(), static_cast<DWORD>(data.size()), &written, NULL))
- {
- return WinIOError(filename_.string(), ::GetLastError());
- }
-
- if (static_cast<size_t>(written) != data.size())
- {
- return Status::IOError(filename_.string(), "could not write all bytes to disk");
- }
-
- return Status::OK();
- }
-
- virtual Status Close() {
- ::CloseHandle(handle_);
- handle_ = NULL;
- return Status::OK();
- }
-
- virtual Status Flush() {
- return Status::OK();
- }
-
- virtual Status Sync() {
- return Status::OK();
- }
-
-
-};
-
-class BoostFileLock : public FileLock {
- public:
- std::ofstream file_;
- boost::interprocess::file_lock fl_;
- std::string name_;
-};
-
-// Set of locked files. We keep a separate set instead of just
-// relying on fcntrl(F_SETLK) since fcntl(F_SETLK) does not provide
-// any protection against multiple uses from the same process.
-class BoostLockTable {
- private:
- std::mutex mu_;
- std::set<std::string> locked_files_;
- public:
- bool Insert(const std::string& fname) {
- std::unique_lock<std::mutex> l(mu_);
- return locked_files_.insert(fname).second;
- }
- void Remove(const std::string& fname) {
- std::unique_lock<std::mutex> l(mu_);
- locked_files_.erase(fname);
- }
-};
-
-class PosixEnv : public Env {
- public:
- PosixEnv();
- virtual ~PosixEnv()
- {
- bool expected = true;
- if (run_bg_thread_.compare_exchange_strong(expected, false))
- {
- bgsignal_.notify_one();
- }
-
- std::thread * t = nullptr;
-
- {
- std::unique_lock<std::mutex> lock(mu_);
- t = bgthread_.get();
- }
-
- if (t)
- {
- t->join();
- }
-
- std::unique_lock<std::mutex> lock(mu_);
- bgthread_.reset();
-
- queue_.clear();
- }
-
- virtual Status NewSequentialFile(const std::string& fname,
- SequentialFile** result) {
- HANDLE h = ::CreateFile(fname.c_str(),
- GENERIC_READ,
- FILE_SHARE_READ,
- NULL,
- OPEN_EXISTING,
- FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,
- NULL);
-
- if (h == INVALID_HANDLE_VALUE)
- {
- return WinIOError(fname, ::GetLastError());
- }
- *result = new WindowsSequentialFile(fname, h);
- return Status::OK();
- }
-
- virtual Status NewRandomAccessFile(const std::string& fname,
- RandomAccessFile** result) {
- HANDLE h = ::CreateFile(fname.c_str(),
- GENERIC_READ,
- FILE_SHARE_READ,
- NULL,
- OPEN_EXISTING,
- FILE_ATTRIBUTE_NORMAL,
- NULL);
-
- if (h == INVALID_HANDLE_VALUE)
- {
- return WinIOError(fname, ::GetLastError());
- }
-
- DWORD hi_size = 0;
-
- const DWORD lo_size = ::GetFileSize(h, &hi_size);
-
- Status s;
-
- if (lo_size == INVALID_FILE_SIZE)
- {
- s = WinIOError(fname, ::GetLastError());
- }
-
- const uint64_t file_size = (static_cast<uint64_t>(hi_size) << 32uLL) + static_cast<uint64_t>(lo_size);
-
- HANDLE map = NULL;
- const void * base = nullptr;
-
- if (s.ok())
- {
- map = ::CreateFileMapping(h, NULL, PAGE_READONLY, 0, 0, NULL);
-
- if (map == NULL)
- {
- s = WinIOError(fname, ::GetLastError());
- }
-
- // handle of the file no longer needed whether we knew success or failure
- ::CloseHandle(h);
-
- if (map)
- {
- base = ::MapViewOfFile(map, FILE_MAP_READ, 0, 0, 0);
- if (!base)
- {
- s = WinIOError(fname, ::GetLastError());
- ::CloseHandle(map);
- }
- }
- }
-
- if (s.ok())
- {
- *result = new WindowsRandomAccessFile(fname, base, file_size, map);
- }
-
- return s;
- }
-
- virtual Status NewWritableFile(const std::string& fname,
- WritableFile** result) {
-
- HANDLE h = ::CreateFile(fname.c_str(),
- GENERIC_WRITE,
- 0,
- NULL,
- CREATE_ALWAYS,
- FILE_ATTRIBUTE_NORMAL,
- NULL);
-
- if (h == INVALID_HANDLE_VALUE)
- {
- return WinIOError(fname, ::GetLastError());
- }
-
- *result = new WindowsWritableFile(fname, h);
- return Status::OK();
- }
-
- virtual bool FileExists(const std::string& fname) {
- boost::system::error_code ec;
- return boost::filesystem::exists(fname, ec);
- }
-
- virtual Status GetChildren(const std::string& dir,
- std::vector<std::string>* result) {
- result->clear();
-
- boost::system::error_code ec;
- boost::filesystem::directory_iterator current(dir, ec);
- if (ec != 0) {
- return Status::IOError(dir, ec.message());
- }
-
- boost::filesystem::directory_iterator end;
-
- for(; current != end; ++current) {
- result->push_back(current->path().filename().generic_string());
- }
-
- return Status::OK();
- }
-
- virtual Status DeleteFile(const std::string& fname) {
- boost::system::error_code ec;
-
- boost::filesystem::remove(fname, ec);
-
- Status result;
-
- if (ec != 0) {
- result = Status::IOError(fname, ec.message());
- }
-
- return result;
- }
-
- virtual Status CreateDir(const std::string& name) {
- Status result;
-
- boost::system::error_code ec;
-
- if (boost::filesystem::exists(name, ec) &&
- boost::filesystem::is_directory(name, ec)) {
- return result;
- }
-
- if (!boost::filesystem::create_directories(name, ec)) {
- result = Status::IOError(name, ec.message());
- }
-
- return result;
- };
-
- virtual Status DeleteDir(const std::string& name) {
- Status result;
-
- boost::system::error_code ec;
- if (!boost::filesystem::remove_all(name, ec)) {
- result = Status::IOError(name, ec.message());
- }
-
- return result;
- };
-
- virtual Status GetFileSize(const std::string& fname, uint64_t* size) {
- boost::system::error_code ec;
-
- Status result;
-
- *size = static_cast<uint64_t>(boost::filesystem::file_size(fname, ec));
- if (ec != 0) {
- *size = 0;
- result = Status::IOError(fname, ec.message());
- }
-
- return result;
- }
-
- virtual Status RenameFile(const std::string& src, const std::string& target) {
- boost::system::error_code ec;
-
- boost::filesystem::rename(src, target, ec);
-
- Status result;
-
- if (ec != 0) {
- result = Status::IOError(src, ec.message());
- }
-
- return result;
- }
-
-
- virtual Status LockFile(const std::string& fname, FileLock** lock) {
- *lock = nullptr;
- std::ofstream of(fname.c_str(), std::ios_base::trunc | std::ios_base::out);
- if (of.bad()) {
- return Status::IOError("lock " + fname, "cannot create lock file");
- }
- if (!locks_.Insert(fname)) {
- of.close();
- return Status::IOError("lock " + fname, "already held by process");
- }
- boost::interprocess::file_lock fl(fname.c_str());
- if (!fl.try_lock()) {
- of.close();
- locks_.Remove(fname);
- return Status::IOError("lock " + fname, "database already in use: could not acquire exclusive lock" );
- }
- BoostFileLock * my_lock = new BoostFileLock();
- my_lock->name_ = fname;
- my_lock->file_ = std::move(of);
- my_lock->fl_ = std::move(fl);
- *lock = my_lock;
- return Status();
- }
-
-
-
- virtual Status UnlockFile(FileLock* lock) {
- BoostFileLock * my_lock = static_cast<BoostFileLock *>(lock);
- Status result;
- try {
- my_lock->fl_.unlock();
- } catch (const std::exception & e) {
- result = Status::IOError("unlock " + my_lock->name_, e.what());
- }
- locks_.Remove(my_lock->name_);
- my_lock->file_.close();
- delete my_lock;
- return result;
- }
-
- virtual void Schedule(void (*function)(void*), void* arg);
-
- virtual Status GetTestDirectory(std::string* result) {
- boost::system::error_code ec;
- boost::filesystem::path temp_dir =
- boost::filesystem::temp_directory_path(ec);
- if (ec != 0) {
- temp_dir = "tmp";
- }
-
- temp_dir /= "leveldb_tests";
- temp_dir /= boost::lexical_cast<std::string>(current_process_id());
-
- // Directory may already exist
- CreateDir(temp_dir.generic_string());
-
- *result = temp_dir.generic_string();
-
- return Status::OK();
- }
-
-#ifndef WIN32
- static uint64_t gettid() {
- pthread_t tid = pthread_self();
- uint64_t thread_id = 0;
- memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid)));
- return thread_id;
- }
-#endif
-
- virtual Status NewLogger(const std::string& fname, Logger** result) {
- FILE* f = fopen(fname.c_str(), "wt");
- if (f == NULL) {
- *result = NULL;
- return Status::IOError(fname, strerror(errno));
- } else {
-#ifdef WIN32
- *result = new WinLogger(f);
-#else
- *result = new PosixLogger(f, &PosixEnv::gettid);
-#endif
- return Status::OK();
- }
- }
-
- virtual uint64_t NowMicros() {
- return static_cast<uint64_t>(
- std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now().time_since_epoch()).count());
- }
-
- virtual void SleepForMicroseconds(int micros) {
- std::this_thread::sleep_for(std::chrono::microseconds(micros));
- }
-
- public:
- virtual std::thread::native_handle_type GetBackgroundThreadHandle() {
- return bgthread_ ? bgthread_->native_handle() : std::thread::native_handle_type();
- }
-
- // BGThread() is the body of the background thread
- void BGThread();
-
- static void* BGThreadWrapper(void* arg) {
- reinterpret_cast<PosixEnv*>(arg)->BGThread();
- return NULL;
- }
-
- std::mutex mu_;
- std::condition_variable bgsignal_;
- std::unique_ptr<std::thread> bgthread_;
-
- // Entry per Schedule() call
- struct BGItem { void* arg; void (*function)(void*); };
- typedef std::deque<BGItem> BGQueue;
- BGQueue queue_;
- std::atomic<bool> run_bg_thread_;
-
- BoostLockTable locks_;
-};
-
-PosixEnv::PosixEnv() { }
-
-void PosixEnv::Schedule(void (*function)(void*), void* arg) {
- std::unique_lock<std::mutex> lock(mu_);
-
- // Start background thread if necessary
- if (!bgthread_) {
- run_bg_thread_ = true;
- bgthread_.reset(new std::thread(std::bind(&PosixEnv::BGThreadWrapper, this)));
- }
-
- // Add to priority queue
- queue_.push_back(BGItem());
- queue_.back().function = function;
- queue_.back().arg = arg;
-
- lock.unlock();
-
- bgsignal_.notify_one();
-
-}
-
-#ifdef _MSC_VER
-// we use SEH, we cannot have a destructor, therefore we pass a pure pointer
-static void name_this_thread(const char * thread_name)
-{
- if (!::IsDebuggerPresent())
- {
- // no need to do this if no debugger is present, remember threads in Windows don't really have name
- // this is just a secret handshake with Visual Studio to name threads and make debugging more pleasant
- // on UNIXES however the name will appear in utilities such as htop
- return;
- }
-
- static const DWORD MS_VC_EXCEPTION = 0x406D1388;
-
-#pragma pack(push,8)
- struct THREADNAME_INFO
- {
- DWORD dwType; // Must be 0x1000.
- LPCSTR szName; // Pointer to name (in user addr space).
- DWORD dwThreadID; // Thread ID (-1=caller thread).
- DWORD dwFlags; // Reserved for future use, must be zero.
- };
-#pragma pack(pop)
-
- THREADNAME_INFO info;
- info.dwType = 0x1000;
- info.szName = thread_name;
- info.dwThreadID = ::GetCurrentThreadId();
- info.dwFlags = 0;
-
- __try
- {
- RaiseException(MS_VC_EXCEPTION, 0, sizeof(info) / sizeof(ULONG_PTR), (ULONG_PTR*)&info);
- }
- __except (EXCEPTION_EXECUTE_HANDLER)
- {
- }
-
-}
-#else
-
-// useful for debugging to name the thread, since we only use env_boost.cc for Windows
-// we have this macro
-// for env_posix.cc we also do the right thing for FreeBSD, Linux and MacOS
-
-static void name_this_thread(const char *) {}
-#endif
-
-void PosixEnv::BGThread() {
-
- name_this_thread("leveldb background");
-
- try
- {
- while (run_bg_thread_) {
- // Wait until there is an item that is ready to run
- std::unique_lock<std::mutex> lock(mu_);
-
- while (queue_.empty() && run_bg_thread_) {
- bgsignal_.wait(lock);
- }
-
- if (!run_bg_thread_)
- {
- break;
- }
-
- void (*function)(void*) = queue_.front().function;
- void* arg = queue_.front().arg;
- queue_.pop_front();
-
- lock.unlock();
- (*function)(arg);
- }
- }
- catch (...) {}
-
-}
-
-}
-
-static std::once_flag once;
-static Env* default_env = nullptr;
-static void InitDefaultEnv() {
- default_env = new PosixEnv();
-
- // force background thread creation so that thread affinity can work
- default_env->Schedule([](void *) {}, nullptr);
-}
-
-Env* Env::Default() {
- std::call_once(once, InitDefaultEnv);
- return default_env;
-}
-
-void Env::UnsafeDeallocate() {
- // will not be able to call again, but the purpose of this function is to get rid of fake alerts by analyzers
- delete default_env;
- default_env = nullptr;
-}
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/env_posix.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/env_posix.cc b/thirdparty/leveldb-1.18/util/env_posix.cc
deleted file mode 100755
index 4e9503b..0000000
--- a/thirdparty/leveldb-1.18/util/env_posix.cc
+++ /dev/null
@@ -1,654 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <dirent.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <time.h>
-#include <unistd.h>
-#include <deque>
-#include <set>
-#include "leveldb/env.h"
-#include "leveldb/slice.h"
-#include "port/port.h"
-#include "util/logging.h"
-#include "util/mutexlock.h"
-#include "util/posix_logger.h"
-
-#ifdef OS_FREEBSD
-#include <pthread_np.h>
-#endif
-
-#ifdef OS_LINUX
-#include <sys/prctl.h>
-#endif
-
-namespace leveldb {
-
-namespace {
-
-static Status IOError(const std::string& context, int err_number) {
- return Status::IOError(context, strerror(err_number));
-}
-
-class PosixSequentialFile: public SequentialFile {
- private:
- std::string filename_;
- FILE* file_;
-
- public:
- PosixSequentialFile(const std::string& fname, FILE* f)
- : filename_(fname), file_(f) { }
- virtual ~PosixSequentialFile() { fclose(file_); }
-
- virtual Status Read(size_t n, Slice* result, char* scratch) {
- Status s;
- size_t r = fread_unlocked(scratch, 1, n, file_);
- *result = Slice(scratch, r);
- if (r < n) {
- if (feof(file_)) {
- // We leave status as ok if we hit the end of the file
- } else {
- // A partial read with an error: return a non-ok status
- s = IOError(filename_, errno);
- }
- }
- return s;
- }
-
- virtual Status Skip(uint64_t n) {
- if (fseek(file_, n, SEEK_CUR)) {
- return IOError(filename_, errno);
- }
- return Status::OK();
- }
-};
-
-// pread() based random-access
-class PosixRandomAccessFile: public RandomAccessFile {
- private:
- std::string filename_;
- int fd_;
-
- public:
- PosixRandomAccessFile(const std::string& fname, int fd)
- : filename_(fname), fd_(fd) { }
- virtual ~PosixRandomAccessFile() { close(fd_); }
-
- virtual Status Read(uint64_t offset, size_t n, Slice* result,
- char* scratch) const {
- Status s;
- ssize_t r = pread(fd_, scratch, n, static_cast<off_t>(offset));
- *result = Slice(scratch, (r < 0) ? 0 : r);
- if (r < 0) {
- // An error: return a non-ok status
- s = IOError(filename_, errno);
- }
- return s;
- }
-};
-
-// Helper class to limit mmap file usage so that we do not end up
-// running out virtual memory or running into kernel performance
-// problems for very large databases.
-class MmapLimiter {
- public:
- // Up to 1000 mmaps for 64-bit binaries; none for smaller pointer sizes.
- MmapLimiter() {
- SetAllowed(sizeof(void*) >= 8 ? 1000 : 0);
- }
-
- // If another mmap slot is available, acquire it and return true.
- // Else return false.
- bool Acquire() {
- if (GetAllowed() <= 0) {
- return false;
- }
- MutexLock l(&mu_);
- intptr_t x = GetAllowed();
- if (x <= 0) {
- return false;
- } else {
- SetAllowed(x - 1);
- return true;
- }
- }
-
- // Release a slot acquired by a previous call to Acquire() that returned true.
- void Release() {
- MutexLock l(&mu_);
- SetAllowed(GetAllowed() + 1);
- }
-
- private:
- port::Mutex mu_;
- port::AtomicPointer allowed_;
-
- intptr_t GetAllowed() const {
- return reinterpret_cast<intptr_t>(allowed_.Acquire_Load());
- }
-
- // REQUIRES: mu_ must be held
- void SetAllowed(intptr_t v) {
- allowed_.Release_Store(reinterpret_cast<void*>(v));
- }
-
- MmapLimiter(const MmapLimiter&);
- void operator=(const MmapLimiter&);
-};
-
-// mmap() based random-access
-class PosixMmapReadableFile: public RandomAccessFile {
- private:
- std::string filename_;
- void* mmapped_region_;
- size_t length_;
- MmapLimiter* limiter_;
-
- public:
- // base[0,length-1] contains the mmapped contents of the file.
- PosixMmapReadableFile(const std::string& fname, void* base, size_t length,
- MmapLimiter* limiter)
- : filename_(fname), mmapped_region_(base), length_(length),
- limiter_(limiter) {
- }
-
- virtual ~PosixMmapReadableFile() {
- munmap(mmapped_region_, length_);
- limiter_->Release();
- }
-
- virtual Status Read(uint64_t offset, size_t n, Slice* result,
- char* scratch) const {
- Status s;
- if (offset + n > length_) {
- *result = Slice();
- s = IOError(filename_, EINVAL);
- } else {
- *result = Slice(reinterpret_cast<char*>(mmapped_region_) + offset, n);
- }
- return s;
- }
-};
-
-class PosixWritableFile : public WritableFile {
- private:
- std::string filename_;
- FILE* file_;
-
- public:
- PosixWritableFile(const std::string& fname, FILE* f)
- : filename_(fname), file_(f) { }
-
- ~PosixWritableFile() {
- if (file_ != NULL) {
- // Ignoring any potential errors
- fclose(file_);
- }
- }
-
- virtual Status Append(const Slice& data) {
- size_t r = fwrite_unlocked(data.data(), 1, data.size(), file_);
- if (r != data.size()) {
- return IOError(filename_, errno);
- }
- return Status::OK();
- }
-
- virtual Status Close() {
- Status result;
- if (fclose(file_) != 0) {
- result = IOError(filename_, errno);
- }
- file_ = NULL;
- return result;
- }
-
- virtual Status Flush() {
- if (fflush_unlocked(file_) != 0) {
- return IOError(filename_, errno);
- }
- return Status::OK();
- }
-
- Status SyncDirIfManifest() {
- const char* f = filename_.c_str();
- const char* sep = strrchr(f, '/');
- Slice basename;
- std::string dir;
- if (sep == NULL) {
- dir = ".";
- basename = f;
- } else {
- dir = std::string(f, sep - f);
- basename = sep + 1;
- }
- Status s;
- if (basename.starts_with("MANIFEST")) {
- int fd = open(dir.c_str(), O_RDONLY);
- if (fd < 0) {
- s = IOError(dir, errno);
- } else {
- if (fsync(fd) < 0) {
- s = IOError(dir, errno);
- }
- close(fd);
- }
- }
- return s;
- }
-
- virtual Status Sync() {
- // Ensure new files referred to by the manifest are in the filesystem.
- Status s = SyncDirIfManifest();
- if (!s.ok()) {
- return s;
- }
- if (fflush_unlocked(file_) != 0 ||
- fdatasync(fileno(file_)) != 0) {
- s = Status::IOError(filename_, strerror(errno));
- }
- return s;
- }
-};
-
-static int LockOrUnlock(int fd, bool lock) {
- errno = 0;
- struct flock f;
- memset(&f, 0, sizeof(f));
- f.l_type = (lock ? F_WRLCK : F_UNLCK);
- f.l_whence = SEEK_SET;
- f.l_start = 0;
- f.l_len = 0; // Lock/unlock entire file
- return fcntl(fd, F_SETLK, &f);
-}
-
-class PosixFileLock : public FileLock {
- public:
- int fd_;
- std::string name_;
-};
-
-// Set of locked files. We keep a separate set instead of just
-// relying on fcntrl(F_SETLK) since fcntl(F_SETLK) does not provide
-// any protection against multiple uses from the same process.
-class PosixLockTable {
- private:
- port::Mutex mu_;
- std::set<std::string> locked_files_;
- public:
- bool Insert(const std::string& fname) {
- MutexLock l(&mu_);
- return locked_files_.insert(fname).second;
- }
- void Remove(const std::string& fname) {
- MutexLock l(&mu_);
- locked_files_.erase(fname);
- }
-};
-
-class PosixEnv : public Env {
- public:
- PosixEnv();
- virtual ~PosixEnv()
- {
- bool expected = true;
- if (started_bgthread_.compare_exchange_strong(expected, false))
- {
- // signal the thread that we want to stop
- PthreadCall("lock", pthread_mutex_lock(&mu_));
- if (running_)
- {
- PthreadCall("signal", pthread_cond_signal(&bgsignal_));
- running_ = false;
- }
- PthreadCall("unlock", pthread_mutex_unlock(&mu_));
-
- // wait for thread termination
- PthreadCall("join", pthread_join(bgthread_, nullptr));
- }
- }
-
- virtual Status NewSequentialFile(const std::string& fname,
- SequentialFile** result) {
- FILE* f = fopen(fname.c_str(), "r");
- if (f == NULL) {
- *result = NULL;
- return IOError(fname, errno);
- } else {
- *result = new PosixSequentialFile(fname, f);
- return Status::OK();
- }
- }
-
- virtual Status NewRandomAccessFile(const std::string& fname,
- RandomAccessFile** result) {
- *result = NULL;
- Status s;
- int fd = open(fname.c_str(), O_RDONLY);
- if (fd < 0) {
- s = IOError(fname, errno);
- } else if (mmap_limit_.Acquire()) {
- uint64_t size;
- s = GetFileSize(fname, &size);
- if (s.ok()) {
- void* base = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, 0);
- if (base != MAP_FAILED) {
- *result = new PosixMmapReadableFile(fname, base, size, &mmap_limit_);
- } else {
- s = IOError(fname, errno);
- }
- }
- close(fd);
- if (!s.ok()) {
- mmap_limit_.Release();
- }
- } else {
- *result = new PosixRandomAccessFile(fname, fd);
- }
- return s;
- }
-
- virtual Status NewWritableFile(const std::string& fname,
- WritableFile** result) {
- Status s;
- FILE* f = fopen(fname.c_str(), "w");
- if (f == NULL) {
- *result = NULL;
- s = IOError(fname, errno);
- } else {
- *result = new PosixWritableFile(fname, f);
- }
- return s;
- }
-
- virtual bool FileExists(const std::string& fname) {
- return access(fname.c_str(), F_OK) == 0;
- }
-
- virtual Status GetChildren(const std::string& dir,
- std::vector<std::string>* result) {
- result->clear();
- DIR* d = opendir(dir.c_str());
- if (d == NULL) {
- return IOError(dir, errno);
- }
- struct dirent* entry;
- while ((entry = readdir(d)) != NULL) {
- result->push_back(entry->d_name);
- }
- closedir(d);
- return Status::OK();
- }
-
- virtual Status DeleteFile(const std::string& fname) {
- Status result;
- if (unlink(fname.c_str()) != 0) {
- result = IOError(fname, errno);
- }
- return result;
- }
-
- virtual Status CreateDir(const std::string& name) {
- Status result;
- if (mkdir(name.c_str(), 0755) != 0) {
- result = IOError(name, errno);
- }
- return result;
- }
-
- virtual Status DeleteDir(const std::string& name) {
- Status result;
- if (rmdir(name.c_str()) != 0) {
- result = IOError(name, errno);
- }
- return result;
- }
-
- virtual Status GetFileSize(const std::string& fname, uint64_t* size) {
- Status s;
- struct stat sbuf;
- if (stat(fname.c_str(), &sbuf) != 0) {
- *size = 0;
- s = IOError(fname, errno);
- } else {
- *size = sbuf.st_size;
- }
- return s;
- }
-
- virtual Status RenameFile(const std::string& src, const std::string& target) {
- Status result;
- if (rename(src.c_str(), target.c_str()) != 0) {
- result = IOError(src, errno);
- }
- return result;
- }
-
- virtual Status LockFile(const std::string& fname, FileLock** lock) {
- *lock = NULL;
- Status result;
- int fd = open(fname.c_str(), O_RDWR | O_CREAT, 0644);
- if (fd < 0) {
- result = IOError(fname, errno);
- } else if (!locks_.Insert(fname)) {
- close(fd);
- result = Status::IOError("lock " + fname, "already held by process");
- } else if (LockOrUnlock(fd, true) == -1) {
- result = IOError("lock " + fname, errno);
- close(fd);
- locks_.Remove(fname);
- } else {
- PosixFileLock* my_lock = new PosixFileLock;
- my_lock->fd_ = fd;
- my_lock->name_ = fname;
- *lock = my_lock;
- }
- return result;
- }
-
- virtual Status UnlockFile(FileLock* lock) {
- PosixFileLock* my_lock = reinterpret_cast<PosixFileLock*>(lock);
- Status result;
- if (LockOrUnlock(my_lock->fd_, false) == -1) {
- result = IOError("unlock", errno);
- }
- locks_.Remove(my_lock->name_);
- close(my_lock->fd_);
- delete my_lock;
- return result;
- }
-
- virtual void Schedule(void (*function)(void*), void* arg);
-
- virtual Status GetTestDirectory(std::string* result) {
- const char* env = getenv("TEST_TMPDIR");
- if (env && env[0] != '\0') {
- *result = env;
- } else {
- char buf[100];
- snprintf(buf, sizeof(buf), "/tmp/leveldbtest-%d", int(geteuid()));
- *result = buf;
- }
- // Directory may already exist
- CreateDir(*result);
- return Status::OK();
- }
-
- static uint64_t gettid() {
- pthread_t tid = pthread_self();
- uint64_t thread_id = 0;
- memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid)));
- return thread_id;
- }
-
- virtual Status NewLogger(const std::string& fname, Logger** result) {
- FILE* f = fopen(fname.c_str(), "w");
- if (f == NULL) {
- *result = NULL;
- return IOError(fname, errno);
- } else {
- *result = new PosixLogger(f, &PosixEnv::gettid);
- return Status::OK();
- }
- }
-
- virtual uint64_t NowMicros() {
- struct timeval tv;
- gettimeofday(&tv, NULL);
- return static_cast<uint64_t>(tv.tv_sec) * 1000000 + tv.tv_usec;
- }
-
- virtual void SleepForMicroseconds(int micros) {
- usleep(micros);
- }
-
- private:
- void PthreadCall(const char* label, int result) {
- if (result != 0) {
- fprintf(stderr, "pthread %s: %s\n", label, strerror(result));
- exit(1);
- }
- }
-
- public:
- virtual std::thread::native_handle_type GetBackgroundThreadHandle() {
- return bgthread_;
- }
-
- // BGThread() is the body of the background thread
- void BGThread();
- static void* BGThreadWrapper(void* arg) {
- reinterpret_cast<PosixEnv*>(arg)->BGThread();
- return NULL;
- }
-
-
- pthread_mutex_t mu_;
- pthread_cond_t bgsignal_;
- pthread_t bgthread_;
-
- std::atomic<bool> started_bgthread_;
- volatile bool running_;
-
- // Entry per Schedule() call
- struct BGItem { void* arg; void (*function)(void*); };
- typedef std::deque<BGItem> BGQueue;
- BGQueue queue_;
-
- PosixLockTable locks_;
- MmapLimiter mmap_limit_;
-};
-
-PosixEnv::PosixEnv() : bgthread_(0),
- started_bgthread_(false),
- running_(false) {
- PthreadCall("mutex_init", pthread_mutex_init(&mu_, NULL));
- PthreadCall("cvar_init", pthread_cond_init(&bgsignal_, NULL));
-}
-
-void PosixEnv::Schedule(void (*function)(void*), void* arg) {
- PthreadCall("lock", pthread_mutex_lock(&mu_));
-
- bool expected = false;
-
- // Start background thread if necessary
- // compare_exchange_strong returns true iff the value was changed
- if (started_bgthread_.compare_exchange_strong(expected, /*desired=*/true)) {
- running_ = true;
- PthreadCall(
- "create thread",
- pthread_create(&bgthread_, NULL, &PosixEnv::BGThreadWrapper, this));
- }
-
- // If the queue is currently empty, the background thread may currently be
- // waiting.
- if (queue_.empty()) {
- PthreadCall("signal", pthread_cond_signal(&bgsignal_));
- }
-
- // Add to priority queue
- queue_.push_back(BGItem());
- queue_.back().function = function;
- queue_.back().arg = arg;
-
- PthreadCall("unlock", pthread_mutex_unlock(&mu_));
-}
-
-void PosixEnv::BGThread() {
-
- // useful for debugging to name the thread, but for each platform, it changes...
-
-#ifdef OS_MACOSX
- pthread_setname_np("leveldb background");
-#endif
-
-#ifdef OS_FREEBSD
- pthread_set_name_np(pthread_self(), "leveldb background");
-#endif
-
-#ifdef OS_LINUX
- prctl(PR_SET_NAME, "leveldb background", 0, 0, 0);
-#endif
-
- while (running_) {
- // Wait until there is an item that is ready to run
- PthreadCall("lock", pthread_mutex_lock(&mu_));
- while (queue_.empty() && running_) {
- PthreadCall("wait", pthread_cond_wait(&bgsignal_, &mu_));
- }
-
- void (*function)(void*) = nullptr;
- void* arg = nullptr;
-
- if (running_)
- {
- function = queue_.front().function;
- arg = queue_.front().arg;
- queue_.pop_front();
- }
-
- PthreadCall("unlock", pthread_mutex_unlock(&mu_));
-
- if (function)
- {
- (*function)(arg);
- }
- }
-}
-
-} // namespace
-
-static const pthread_once_t init_value = PTHREAD_ONCE_INIT;
-static pthread_once_t once = PTHREAD_ONCE_INIT;
-static Env* default_env = nullptr;
-static void InitDefaultEnv()
-{
- default_env = new PosixEnv();
-
- // force background thread creation so that thread affinity can work
- default_env->Schedule([](void *) {}, nullptr);
-}
-
-Env* Env::Default() {
- pthread_once(&once, InitDefaultEnv);
- return default_env;
-}
-
-void Env::UnsafeDeallocate()
-{
- once = init_value;
- delete default_env;
- default_env = nullptr;
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/env_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/env_test.cc b/thirdparty/leveldb-1.18/util/env_test.cc
deleted file mode 100755
index 2b97165..0000000
--- a/thirdparty/leveldb-1.18/util/env_test.cc
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/env.h"
-
-#include "port/port.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-static const int kDelayMicros = 100000;
-
-class EnvPosixTest {
- private:
- port::Mutex mu_;
- std::string events_;
-
- public:
- Env* env_;
- EnvPosixTest() : env_(Env::Default()) { }
-};
-
-static void SetBool(void* ptr) {
- reinterpret_cast<port::AtomicPointer*>(ptr)->NoBarrier_Store(ptr);
-}
-
-TEST(EnvPosixTest, RunImmediately) {
- port::AtomicPointer called (NULL);
- env_->Schedule(&SetBool, &called);
- Env::Default()->SleepForMicroseconds(kDelayMicros);
- ASSERT_TRUE(called.NoBarrier_Load() != NULL);
-}
-
-TEST(EnvPosixTest, RunMany) {
- port::AtomicPointer last_id (NULL);
-
- struct CB {
- port::AtomicPointer* last_id_ptr; // Pointer to shared slot
- uintptr_t id; // Order# for the execution of this callback
-
- CB(port::AtomicPointer* p, int i) : last_id_ptr(p), id(i) { }
-
- static void Run(void* v) {
- CB* cb = reinterpret_cast<CB*>(v);
- void* cur = cb->last_id_ptr->NoBarrier_Load();
- ASSERT_EQ(cb->id-1, reinterpret_cast<uintptr_t>(cur));
- cb->last_id_ptr->Release_Store(reinterpret_cast<void*>(cb->id));
- }
- };
-
- // Schedule in different order than start time
- CB cb1(&last_id, 1);
- CB cb2(&last_id, 2);
- CB cb3(&last_id, 3);
- CB cb4(&last_id, 4);
- env_->Schedule(&CB::Run, &cb1);
- env_->Schedule(&CB::Run, &cb2);
- env_->Schedule(&CB::Run, &cb3);
- env_->Schedule(&CB::Run, &cb4);
-
- Env::Default()->SleepForMicroseconds(kDelayMicros);
- void* cur = last_id.Acquire_Load();
- ASSERT_EQ(4, reinterpret_cast<uintptr_t>(cur));
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/filter_policy.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/filter_policy.cc b/thirdparty/leveldb-1.18/util/filter_policy.cc
deleted file mode 100755
index 7b045c8..0000000
--- a/thirdparty/leveldb-1.18/util/filter_policy.cc
+++ /dev/null
@@ -1,11 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/filter_policy.h"
-
-namespace leveldb {
-
-FilterPolicy::~FilterPolicy() { }
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/hash.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/hash.cc b/thirdparty/leveldb-1.18/util/hash.cc
deleted file mode 100755
index f4aaa21..0000000
--- a/thirdparty/leveldb-1.18/util/hash.cc
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <string.h>
-#include "util/coding.h"
-#include "util/hash.h"
-
-// The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through
-// between switch labels. The real definition should be provided externally.
-// This one is a fallback version for unsupported compilers.
-#ifndef FALLTHROUGH_INTENDED
-#define FALLTHROUGH_INTENDED do { } while (0)
-#endif
-
-namespace leveldb {
-
-uint32_t Hash(const void * data, size_t n, uint32_t seed) {
-
- const unsigned char * p = static_cast<const unsigned char *>(data);
-
- // Similar to murmur hash
- const uint32_t m = 0xc6a4a793;
- const uint32_t r = 24;
- const unsigned char * limit = p + n;
- uint32_t h = seed ^ (static_cast<uint32_t>(n) * m);
-
- // Pick up four bytes at a time
- while (p + 4 <= limit) {
- uint32_t w = DecodeFixed32(p);
- p += 4;
- h += w;
- h *= m;
- h ^= (h >> 16);
- }
-
- // Pick up remaining bytes
- switch (limit - p) {
- case 3:
- h += p[2] << 16;
- FALLTHROUGH_INTENDED;
- case 2:
- h += p[1] << 8;
- FALLTHROUGH_INTENDED;
- case 1:
- h += p[0];
- h *= m;
- h ^= (h >> r);
- break;
- }
- return h;
-}
-
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/hash.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/hash.h b/thirdparty/leveldb-1.18/util/hash.h
deleted file mode 100755
index 5b12af7..0000000
--- a/thirdparty/leveldb-1.18/util/hash.h
+++ /dev/null
@@ -1,19 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Simple hash function used for internal data structures
-
-#ifndef STORAGE_LEVELDB_UTIL_HASH_H_
-#define STORAGE_LEVELDB_UTIL_HASH_H_
-
-#include <stddef.h>
-#include <stdint.h>
-
-namespace leveldb {
-
-extern uint32_t Hash(const void * data, size_t n, uint32_t seed);
-
-}
-
-#endif // STORAGE_LEVELDB_UTIL_HASH_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/hash_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/hash_test.cc b/thirdparty/leveldb-1.18/util/hash_test.cc
deleted file mode 100755
index 7b47e03..0000000
--- a/thirdparty/leveldb-1.18/util/hash_test.cc
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "util/hash.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-class HASH { };
-
-TEST(HASH, SignedUnsignedIssue) {
- const unsigned char data1[1] = {0x62};
- const unsigned char data2[2] = {0xc3, 0x97};
- const unsigned char data3[3] = {0xe2, 0x99, 0xa5};
- const unsigned char data4[4] = {0xe1, 0x80, 0xb9, 0x32};
- const unsigned char data5[48] = {
- 0x01, 0xc0, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x14, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x04, 0x00,
- 0x00, 0x00, 0x00, 0x14,
- 0x00, 0x00, 0x00, 0x18,
- 0x28, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x02, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- };
-
- ASSERT_EQ(Hash(0, 0, 0xbc9f1d34), 0xbc9f1d34);
- ASSERT_EQ(
- Hash(data1, sizeof(data1), 0xbc9f1d34),
- 0xef1345c4);
- ASSERT_EQ(
- Hash(data2, sizeof(data2), 0xbc9f1d34),
- 0x5b663814);
- ASSERT_EQ(
- Hash(data3, sizeof(data3), 0xbc9f1d34),
- 0x323c078f);
- ASSERT_EQ(
- Hash(data4, sizeof(data4), 0xbc9f1d34),
- 0xed21633a);
- ASSERT_EQ(
- Hash(data5, sizeof(data5), 0x12345678),
- 0xf333dabb);
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/histogram.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/histogram.cc b/thirdparty/leveldb-1.18/util/histogram.cc
deleted file mode 100755
index bb95f58..0000000
--- a/thirdparty/leveldb-1.18/util/histogram.cc
+++ /dev/null
@@ -1,139 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <math.h>
-#include <stdio.h>
-#include "port/port.h"
-#include "util/histogram.h"
-
-namespace leveldb {
-
-const double Histogram::kBucketLimit[kNumBuckets] = {
- 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20, 25, 30, 35, 40, 45,
- 50, 60, 70, 80, 90, 100, 120, 140, 160, 180, 200, 250, 300, 350, 400, 450,
- 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000, 2500, 3000,
- 3500, 4000, 4500, 5000, 6000, 7000, 8000, 9000, 10000, 12000, 14000,
- 16000, 18000, 20000, 25000, 30000, 35000, 40000, 45000, 50000, 60000,
- 70000, 80000, 90000, 100000, 120000, 140000, 160000, 180000, 200000,
- 250000, 300000, 350000, 400000, 450000, 500000, 600000, 700000, 800000,
- 900000, 1000000, 1200000, 1400000, 1600000, 1800000, 2000000, 2500000,
- 3000000, 3500000, 4000000, 4500000, 5000000, 6000000, 7000000, 8000000,
- 9000000, 10000000, 12000000, 14000000, 16000000, 18000000, 20000000,
- 25000000, 30000000, 35000000, 40000000, 45000000, 50000000, 60000000,
- 70000000, 80000000, 90000000, 100000000, 120000000, 140000000, 160000000,
- 180000000, 200000000, 250000000, 300000000, 350000000, 400000000,
- 450000000, 500000000, 600000000, 700000000, 800000000, 900000000,
- 1000000000, 1200000000, 1400000000, 1600000000, 1800000000, 2000000000,
- 2500000000.0, 3000000000.0, 3500000000.0, 4000000000.0, 4500000000.0,
- 5000000000.0, 6000000000.0, 7000000000.0, 8000000000.0, 9000000000.0,
- 1e200,
-};
-
-void Histogram::Clear() {
- min_ = kBucketLimit[kNumBuckets-1];
- max_ = 0;
- num_ = 0;
- sum_ = 0;
- sum_squares_ = 0;
- for (int i = 0; i < kNumBuckets; i++) {
- buckets_[i] = 0;
- }
-}
-
-void Histogram::Add(double value) {
- // Linear search is fast enough for our usage in db_bench
- int b = 0;
- while (b < kNumBuckets - 1 && kBucketLimit[b] <= value) {
- b++;
- }
- buckets_[b] += 1.0;
- if (min_ > value) min_ = value;
- if (max_ < value) max_ = value;
- num_++;
- sum_ += value;
- sum_squares_ += (value * value);
-}
-
-void Histogram::Merge(const Histogram& other) {
- if (other.min_ < min_) min_ = other.min_;
- if (other.max_ > max_) max_ = other.max_;
- num_ += other.num_;
- sum_ += other.sum_;
- sum_squares_ += other.sum_squares_;
- for (int b = 0; b < kNumBuckets; b++) {
- buckets_[b] += other.buckets_[b];
- }
-}
-
-double Histogram::Median() const {
- return Percentile(50.0);
-}
-
-double Histogram::Percentile(double p) const {
- double threshold = num_ * (p / 100.0);
- double sum = 0;
- for (int b = 0; b < kNumBuckets; b++) {
- sum += buckets_[b];
- if (sum >= threshold) {
- // Scale linearly within this bucket
- double left_point = (b == 0) ? 0 : kBucketLimit[b-1];
- double right_point = kBucketLimit[b];
- double left_sum = sum - buckets_[b];
- double right_sum = sum;
- double pos = (threshold - left_sum) / (right_sum - left_sum);
- double r = left_point + (right_point - left_point) * pos;
- if (r < min_) r = min_;
- if (r > max_) r = max_;
- return r;
- }
- }
- return max_;
-}
-
-double Histogram::Average() const {
- if (num_ == 0.0) return 0;
- return sum_ / num_;
-}
-
-double Histogram::StandardDeviation() const {
- if (num_ == 0.0) return 0;
- double variance = (sum_squares_ * num_ - sum_ * sum_) / (num_ * num_);
- return sqrt(variance);
-}
-
-std::string Histogram::ToString() const {
- std::string r;
- char buf[200];
- snprintf(buf, sizeof(buf),
- "Count: %.0f Average: %.4f StdDev: %.2f\n",
- num_, Average(), StandardDeviation());
- r.append(buf);
- snprintf(buf, sizeof(buf),
- "Min: %.4f Median: %.4f Max: %.4f\n",
- (num_ == 0.0 ? 0.0 : min_), Median(), max_);
- r.append(buf);
- r.append("------------------------------------------------------\n");
- const double mult = 100.0 / num_;
- double sum = 0;
- for (int b = 0; b < kNumBuckets; b++) {
- if (buckets_[b] <= 0.0) continue;
- sum += buckets_[b];
- snprintf(buf, sizeof(buf),
- "[ %7.0f, %7.0f ) %7.0f %7.3f%% %7.3f%% ",
- ((b == 0) ? 0.0 : kBucketLimit[b-1]), // left
- kBucketLimit[b], // right
- buckets_[b], // count
- mult * buckets_[b], // percentage
- mult * sum); // cumulative percentage
- r.append(buf);
-
- // Add hash marks based on percentage; 20 marks for 100%.
- int marks = static_cast<int>(20*(buckets_[b] / num_) + 0.5);
- r.append(marks, '#');
- r.push_back('\n');
- }
- return r;
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/histogram.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/histogram.h b/thirdparty/leveldb-1.18/util/histogram.h
deleted file mode 100755
index 1ef9f3c..0000000
--- a/thirdparty/leveldb-1.18/util/histogram.h
+++ /dev/null
@@ -1,42 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_UTIL_HISTOGRAM_H_
-#define STORAGE_LEVELDB_UTIL_HISTOGRAM_H_
-
-#include <string>
-
-namespace leveldb {
-
-class Histogram {
- public:
- Histogram() { }
- ~Histogram() { }
-
- void Clear();
- void Add(double value);
- void Merge(const Histogram& other);
-
- std::string ToString() const;
-
- private:
- double min_;
- double max_;
- double num_;
- double sum_;
- double sum_squares_;
-
- enum { kNumBuckets = 154 };
- static const double kBucketLimit[kNumBuckets];
- double buckets_[kNumBuckets];
-
- double Median() const;
- double Percentile(double p) const;
- double Average() const;
- double StandardDeviation() const;
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_UTIL_HISTOGRAM_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/logging.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/logging.cc b/thirdparty/leveldb-1.18/util/logging.cc
deleted file mode 100755
index ca6b324..0000000
--- a/thirdparty/leveldb-1.18/util/logging.cc
+++ /dev/null
@@ -1,72 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "util/logging.h"
-
-#include <errno.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include "leveldb/env.h"
-#include "leveldb/slice.h"
-
-namespace leveldb {
-
-void AppendNumberTo(std::string* str, uint64_t num) {
- char buf[30];
- snprintf(buf, sizeof(buf), "%llu", (unsigned long long) num);
- str->append(buf);
-}
-
-void AppendEscapedStringTo(std::string* str, const Slice& value) {
- for (size_t i = 0; i < value.size(); i++) {
- char c = value[i];
- if (c >= ' ' && c <= '~') {
- str->push_back(c);
- } else {
- char buf[10];
- snprintf(buf, sizeof(buf), "\\x%02x",
- static_cast<unsigned int>(c) & 0xff);
- str->append(buf);
- }
- }
-}
-
-std::string NumberToString(uint64_t num) {
- std::string r;
- AppendNumberTo(&r, num);
- return r;
-}
-
-std::string EscapeString(const Slice& value) {
- std::string r;
- AppendEscapedStringTo(&r, value);
- return r;
-}
-
-bool ConsumeDecimalNumber(Slice* in, uint64_t* val) {
- uint64_t v = 0;
- int digits = 0;
- while (!in->empty()) {
- char c = (*in)[0];
- if (c >= '0' && c <= '9') {
- ++digits;
- const int delta = (c - '0');
- static const uint64_t kMaxUint64 = ~static_cast<uint64_t>(0);
- if (v > kMaxUint64/10 ||
- (v == kMaxUint64/10 && delta > kMaxUint64%10)) {
- // Overflow
- return false;
- }
- v = (v * 10) + delta;
- in->remove_prefix(1);
- } else {
- break;
- }
- }
- *val = v;
- return (digits > 0);
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/logging.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/logging.h b/thirdparty/leveldb-1.18/util/logging.h
deleted file mode 100755
index 1b450d2..0000000
--- a/thirdparty/leveldb-1.18/util/logging.h
+++ /dev/null
@@ -1,43 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Must not be included from any .h files to avoid polluting the namespace
-// with macros.
-
-#ifndef STORAGE_LEVELDB_UTIL_LOGGING_H_
-#define STORAGE_LEVELDB_UTIL_LOGGING_H_
-
-#include <stdio.h>
-#include <stdint.h>
-#include <string>
-#include "port/port.h"
-
-namespace leveldb {
-
-class Slice;
-class WritableFile;
-
-// Append a human-readable printout of "num" to *str
-extern void AppendNumberTo(std::string* str, uint64_t num);
-
-// Append a human-readable printout of "value" to *str.
-// Escapes any non-printable characters found in "value".
-extern void AppendEscapedStringTo(std::string* str, const Slice& value);
-
-// Return a human-readable printout of "num"
-extern std::string NumberToString(uint64_t num);
-
-// Return a human-readable version of "value".
-// Escapes any non-printable characters found in "value".
-extern std::string EscapeString(const Slice& value);
-
-// Parse a human-readable number from "*in" into *value. On success,
-// advances "*in" past the consumed number and sets "*val" to the
-// numeric value. Otherwise, returns false and leaves *in in an
-// unspecified state.
-extern bool ConsumeDecimalNumber(Slice* in, uint64_t* val);
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_UTIL_LOGGING_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/mutexlock.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/mutexlock.h b/thirdparty/leveldb-1.18/util/mutexlock.h
deleted file mode 100755
index 1ff5a9e..0000000
--- a/thirdparty/leveldb-1.18/util/mutexlock.h
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_
-#define STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_
-
-#include "port/port.h"
-#include "port/thread_annotations.h"
-
-namespace leveldb {
-
-// Helper class that locks a mutex on construction and unlocks the mutex when
-// the destructor of the MutexLock object is invoked.
-//
-// Typical usage:
-//
-// void MyClass::MyMethod() {
-// MutexLock l(&mu_); // mu_ is an instance variable
-// ... some complex code, possibly with multiple return paths ...
-// }
-
-class SCOPED_LOCKABLE MutexLock {
- public:
- explicit MutexLock(port::Mutex *mu) EXCLUSIVE_LOCK_FUNCTION(mu)
- : mu_(mu) {
- this->mu_->Lock();
- }
- ~MutexLock() UNLOCK_FUNCTION() { this->mu_->Unlock(); }
-
- private:
- port::Mutex *const mu_;
- // No copying allowed
- MutexLock(const MutexLock&);
- void operator=(const MutexLock&);
-};
-
-} // namespace leveldb
-
-
-#endif // STORAGE_LEVELDB_UTIL_MUTEXLOCK_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/options.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/options.cc b/thirdparty/leveldb-1.18/util/options.cc
deleted file mode 100755
index 76af5b9..0000000
--- a/thirdparty/leveldb-1.18/util/options.cc
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/options.h"
-
-#include "leveldb/comparator.h"
-#include "leveldb/env.h"
-
-namespace leveldb {
-
-Options::Options()
- : comparator(BytewiseComparator()),
- create_if_missing(false),
- error_if_exists(false),
- paranoid_checks(false),
- env(Env::Default()),
- info_log(NULL),
- write_buffer_size(4<<20),
- max_open_files(1000),
- block_cache(NULL),
- block_size(4096),
- block_restart_interval(16),
- compression(kSnappyCompression),
- filter_policy(NULL) {
-}
-
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/posix_logger.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/posix_logger.h b/thirdparty/leveldb-1.18/util/posix_logger.h
deleted file mode 100755
index 9741b1a..0000000
--- a/thirdparty/leveldb-1.18/util/posix_logger.h
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Logger implementation that can be shared by all environments
-// where enough posix functionality is available.
-
-#ifndef STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_
-#define STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_
-
-#include <algorithm>
-#include <stdio.h>
-#include <sys/time.h>
-#include <time.h>
-#include "leveldb/env.h"
-
-namespace leveldb {
-
-class PosixLogger : public Logger {
- private:
- FILE* file_;
- uint64_t (*gettid_)(); // Return the thread id for the current thread
- public:
- PosixLogger(FILE* f, uint64_t (*gettid)()) : file_(f), gettid_(gettid) { }
- virtual ~PosixLogger() {
- fclose(file_);
- }
- virtual void Logv(const char* format, va_list ap) {
- const uint64_t thread_id = (*gettid_)();
-
- // We try twice: the first time with a fixed-size stack allocated buffer,
- // and the second time with a much larger dynamically allocated buffer.
- char buffer[500];
- for (int iter = 0; iter < 2; iter++) {
- char* base;
- int bufsize;
- if (iter == 0) {
- bufsize = sizeof(buffer);
- base = buffer;
- } else {
- bufsize = 30000;
- base = new char[bufsize];
- }
- char* p = base;
- char* limit = base + bufsize;
-
- struct timeval now_tv;
- gettimeofday(&now_tv, NULL);
- const time_t seconds = now_tv.tv_sec;
- struct tm t;
- localtime_r(&seconds, &t);
- p += snprintf(p, limit - p,
- "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ",
- t.tm_year + 1900,
- t.tm_mon + 1,
- t.tm_mday,
- t.tm_hour,
- t.tm_min,
- t.tm_sec,
- static_cast<int>(now_tv.tv_usec),
- static_cast<long long unsigned int>(thread_id));
-
- // Print the message
- if (p < limit) {
- va_list backup_ap;
- va_copy(backup_ap, ap);
- p += vsnprintf(p, limit - p, format, backup_ap);
- va_end(backup_ap);
- }
-
- // Truncate to available space if necessary
- if (p >= limit) {
- if (iter == 0) {
- continue; // Try again with larger buffer
- } else {
- p = limit - 1;
- }
- }
-
- // Add newline if necessary
- if (p == base || p[-1] != '\n') {
- *p++ = '\n';
- }
-
- assert(p <= limit);
- fwrite(base, 1, p - base, file_);
- fflush(file_);
- if (base != buffer) {
- delete[] base;
- }
- break;
- }
- }
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/random.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/random.h b/thirdparty/leveldb-1.18/util/random.h
deleted file mode 100755
index ddd51b1..0000000
--- a/thirdparty/leveldb-1.18/util/random.h
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_UTIL_RANDOM_H_
-#define STORAGE_LEVELDB_UTIL_RANDOM_H_
-
-#include <stdint.h>
-
-namespace leveldb {
-
-// A very simple random number generator. Not especially good at
-// generating truly random bits, but good enough for our needs in this
-// package.
-class Random {
- private:
- uint32_t seed_;
- public:
- explicit Random(uint32_t s) : seed_(s & 0x7fffffffu) {
- // Avoid bad seeds.
- if (seed_ == 0 || seed_ == 2147483647L) {
- seed_ = 1;
- }
- }
- uint32_t Next() {
- static const uint32_t M = 2147483647L; // 2^31-1
- static const uint64_t A = 16807; // bits 14, 8, 7, 5, 2, 1, 0
- // We are computing
- // seed_ = (seed_ * A) % M, where M = 2^31-1
- //
- // seed_ must not be zero or M, or else all subsequent computed values
- // will be zero or M respectively. For all other values, seed_ will end
- // up cycling through every number in [1,M-1]
- uint64_t product = seed_ * A;
-
- // Compute (product % M) using the fact that ((x << 31) % M) == x.
- seed_ = static_cast<uint32_t>((product >> 31) + (product & M));
- // The first reduction may overflow by 1 bit, so we may need to
- // repeat. mod == M is not possible; using > allows the faster
- // sign-bit-based test.
- if (seed_ > M) {
- seed_ -= M;
- }
- return seed_;
- }
- // Returns a uniformly distributed value in the range [0..n-1]
- // REQUIRES: n > 0
- uint32_t Uniform(int n) { return Next() % n; }
-
- // Randomly returns true ~"1/n" of the time, and false otherwise.
- // REQUIRES: n > 0
- bool OneIn(int n) { return (Next() % n) == 0; }
-
- // Skewed: pick "base" uniformly from range [0,max_log] and then
- // return "base" random bits. The effect is to pick a number in the
- // range [0,2^max_log-1] with exponential bias towards smaller numbers.
- uint32_t Skewed(int max_log) {
- return Uniform(1 << Uniform(max_log + 1));
- }
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_UTIL_RANDOM_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/status.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/status.cc b/thirdparty/leveldb-1.18/util/status.cc
deleted file mode 100755
index eb0d31b..0000000
--- a/thirdparty/leveldb-1.18/util/status.cc
+++ /dev/null
@@ -1,75 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <stdio.h>
-#include "port/port.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-const char* Status::CopyState(const char* state) {
- uint32_t size;
- memcpy(&size, state, sizeof(size));
- char* result = new char[size + 5];
- memcpy(result, state, size + 5);
- return result;
-}
-
-Status::Status(Code code, const Slice& msg, const Slice& msg2) {
- assert(code != kOk);
- const uint32_t len1 = static_cast<uint32_t>(msg.size());
- const uint32_t len2 = static_cast<uint32_t>(msg2.size());
- const uint32_t size = len1 + (len2 ? (2 + len2) : 0);
- char* result = new char[size + 5];
- memcpy(result, &size, sizeof(size));
- result[4] = static_cast<char>(code);
- memcpy(result + 5, msg.data(), len1);
- if (len2) {
- result[5 + len1] = ':';
- result[6 + len1] = ' ';
- memcpy(result + 7 + len1, msg2.data(), len2);
- }
- state_ = result;
-}
-
-std::string Status::ToString() const {
- if (state_ == NULL) {
- return "OK";
- } else {
- char tmp[30];
- const char* type;
- switch (code()) {
- case kOk:
- type = "OK";
- break;
- case kNotFound:
- type = "NotFound: ";
- break;
- case kCorruption:
- type = "Corruption: ";
- break;
- case kNotSupported:
- type = "Not implemented: ";
- break;
- case kInvalidArgument:
- type = "Invalid argument: ";
- break;
- case kIOError:
- type = "IO error: ";
- break;
- default:
- snprintf(tmp, sizeof(tmp), "Unknown code(%d): ",
- static_cast<int>(code()));
- type = tmp;
- break;
- }
- std::string result(type);
- uint32_t length;
- memcpy(&length, state_, sizeof(length));
- result.append(state_ + 5, length);
- return result;
- }
-}
-
-} // namespace leveldb
[08/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/version_set.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/version_set.h b/thirdparty/rocksdb/db/version_set.h
new file mode 100644
index 0000000..9fb000c
--- /dev/null
+++ b/thirdparty/rocksdb/db/version_set.h
@@ -0,0 +1,862 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// The representation of a DBImpl consists of a set of Versions. The
+// newest version is called "current". Older versions may be kept
+// around to provide a consistent view to live iterators.
+//
+// Each Version keeps track of a set of Table files per level. The
+// entire set of versions is maintained in a VersionSet.
+//
+// Version,VersionSet are thread-compatible, but require external
+// synchronization on all accesses.
+
+#pragma once
+#include <atomic>
+#include <deque>
+#include <limits>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "db/column_family.h"
+#include "db/compaction.h"
+#include "db/compaction_picker.h"
+#include "db/dbformat.h"
+#include "db/file_indexer.h"
+#include "db/log_reader.h"
+#include "db/range_del_aggregator.h"
+#include "db/table_cache.h"
+#include "db/version_builder.h"
+#include "db/version_edit.h"
+#include "db/write_controller.h"
+#include "monitoring/instrumented_mutex.h"
+#include "options/db_options.h"
+#include "port/port.h"
+#include "rocksdb/env.h"
+
+namespace rocksdb {
+
+namespace log {
+class Writer;
+}
+
+class Compaction;
+class InternalIterator;
+class LogBuffer;
+class LookupKey;
+class MemTable;
+class Version;
+class VersionSet;
+class WriteBufferManager;
+class MergeContext;
+class ColumnFamilySet;
+class TableCache;
+class MergeIteratorBuilder;
+
+// Return the smallest index i such that file_level.files[i]->largest >= key.
+// Return file_level.num_files if there is no such file.
+// REQUIRES: "file_level.files" contains a sorted list of
+// non-overlapping files.
+extern int FindFile(const InternalKeyComparator& icmp,
+ const LevelFilesBrief& file_level, const Slice& key);
+
+// Returns true iff some file in "files" overlaps the user key range
+// [*smallest,*largest].
+// smallest==nullptr represents a key smaller than all keys in the DB.
+// largest==nullptr represents a key largest than all keys in the DB.
+// REQUIRES: If disjoint_sorted_files, file_level.files[]
+// contains disjoint ranges in sorted order.
+extern bool SomeFileOverlapsRange(const InternalKeyComparator& icmp,
+ bool disjoint_sorted_files,
+ const LevelFilesBrief& file_level,
+ const Slice* smallest_user_key,
+ const Slice* largest_user_key);
+
+// Generate LevelFilesBrief from vector<FdWithKeyRange*>
+// Would copy smallest_key and largest_key data to sequential memory
+// arena: Arena used to allocate the memory
+extern void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
+ const std::vector<FileMetaData*>& files,
+ Arena* arena);
+
+class VersionStorageInfo {
+ public:
+ VersionStorageInfo(const InternalKeyComparator* internal_comparator,
+ const Comparator* user_comparator, int num_levels,
+ CompactionStyle compaction_style,
+ VersionStorageInfo* src_vstorage,
+ bool _force_consistency_checks);
+ ~VersionStorageInfo();
+
+ void Reserve(int level, size_t size) { files_[level].reserve(size); }
+
+ void AddFile(int level, FileMetaData* f, Logger* info_log = nullptr);
+
+ void SetFinalized();
+
+ // Update num_non_empty_levels_.
+ void UpdateNumNonEmptyLevels();
+
+ void GenerateFileIndexer() {
+ file_indexer_.UpdateIndex(&arena_, num_non_empty_levels_, files_);
+ }
+
+ // Update the accumulated stats from a file-meta.
+ void UpdateAccumulatedStats(FileMetaData* file_meta);
+
+ // Decrease the current stat form a to-be-delected file-meta
+ void RemoveCurrentStats(FileMetaData* file_meta);
+
+ void ComputeCompensatedSizes();
+
+ // Updates internal structures that keep track of compaction scores
+ // We use compaction scores to figure out which compaction to do next
+ // REQUIRES: db_mutex held!!
+ // TODO find a better way to pass compaction_options_fifo.
+ void ComputeCompactionScore(const ImmutableCFOptions& immutable_cf_options,
+ const MutableCFOptions& mutable_cf_options);
+
+ // Estimate est_comp_needed_bytes_
+ void EstimateCompactionBytesNeeded(
+ const MutableCFOptions& mutable_cf_options);
+
+ // This computes files_marked_for_compaction_ and is called by
+ // ComputeCompactionScore()
+ void ComputeFilesMarkedForCompaction();
+
+ // Generate level_files_brief_ from files_
+ void GenerateLevelFilesBrief();
+ // Sort all files for this version based on their file size and
+ // record results in files_by_compaction_pri_. The largest files are listed
+ // first.
+ void UpdateFilesByCompactionPri(CompactionPri compaction_pri);
+
+ void GenerateLevel0NonOverlapping();
+ bool level0_non_overlapping() const {
+ return level0_non_overlapping_;
+ }
+
+ int MaxInputLevel() const;
+ int MaxOutputLevel(bool allow_ingest_behind) const;
+
+ // Return level number that has idx'th highest score
+ int CompactionScoreLevel(int idx) const { return compaction_level_[idx]; }
+
+ // Return idx'th highest score
+ double CompactionScore(int idx) const { return compaction_score_[idx]; }
+
+ void GetOverlappingInputs(
+ int level, const InternalKey* begin, // nullptr means before all keys
+ const InternalKey* end, // nullptr means after all keys
+ std::vector<FileMetaData*>* inputs,
+ int hint_index = -1, // index of overlap file
+ int* file_index = nullptr, // return index of overlap file
+ bool expand_range = true) // if set, returns files which overlap the
+ const; // range and overlap each other. If false,
+ // then just files intersecting the range
+ void GetCleanInputsWithinInterval(
+ int level, const InternalKey* begin, // nullptr means before all keys
+ const InternalKey* end, // nullptr means after all keys
+ std::vector<FileMetaData*>* inputs,
+ int hint_index = -1, // index of overlap file
+ int* file_index = nullptr) // return index of overlap file
+ const;
+
+ void GetOverlappingInputsRangeBinarySearch(
+ int level, // level > 0
+ const Slice& begin, // nullptr means before all keys
+ const Slice& end, // nullptr means after all keys
+ std::vector<FileMetaData*>* inputs,
+ int hint_index, // index of overlap file
+ int* file_index, // return index of overlap file
+ bool within_interval = false) // if set, force the inputs within interval
+ const;
+
+ void ExtendFileRangeOverlappingInterval(
+ int level,
+ const Slice& begin, // nullptr means before all keys
+ const Slice& end, // nullptr means after all keys
+ unsigned int index, // start extending from this index
+ int* startIndex, // return the startIndex of input range
+ int* endIndex) // return the endIndex of input range
+ const;
+
+ void ExtendFileRangeWithinInterval(
+ int level,
+ const Slice& begin, // nullptr means before all keys
+ const Slice& end, // nullptr means after all keys
+ unsigned int index, // start extending from this index
+ int* startIndex, // return the startIndex of input range
+ int* endIndex) // return the endIndex of input range
+ const;
+
+ // Returns true iff some file in the specified level overlaps
+ // some part of [*smallest_user_key,*largest_user_key].
+ // smallest_user_key==NULL represents a key smaller than all keys in the DB.
+ // largest_user_key==NULL represents a key largest than all keys in the DB.
+ bool OverlapInLevel(int level, const Slice* smallest_user_key,
+ const Slice* largest_user_key);
+
+ // Returns true iff the first or last file in inputs contains
+ // an overlapping user key to the file "just outside" of it (i.e.
+ // just after the last file, or just before the first file)
+ // REQUIRES: "*inputs" is a sorted list of non-overlapping files
+ bool HasOverlappingUserKey(const std::vector<FileMetaData*>* inputs,
+ int level);
+
+ int num_levels() const { return num_levels_; }
+
+ // REQUIRES: This version has been saved (see VersionSet::SaveTo)
+ int num_non_empty_levels() const {
+ assert(finalized_);
+ return num_non_empty_levels_;
+ }
+
+ // REQUIRES: This version has been finalized.
+ // (CalculateBaseBytes() is called)
+ // This may or may not return number of level files. It is to keep backward
+ // compatible behavior in universal compaction.
+ int l0_delay_trigger_count() const { return l0_delay_trigger_count_; }
+
+ void set_l0_delay_trigger_count(int v) { l0_delay_trigger_count_ = v; }
+
+ // REQUIRES: This version has been saved (see VersionSet::SaveTo)
+ int NumLevelFiles(int level) const {
+ assert(finalized_);
+ return static_cast<int>(files_[level].size());
+ }
+
+ // Return the combined file size of all files at the specified level.
+ uint64_t NumLevelBytes(int level) const;
+
+ // REQUIRES: This version has been saved (see VersionSet::SaveTo)
+ const std::vector<FileMetaData*>& LevelFiles(int level) const {
+ return files_[level];
+ }
+
+ const rocksdb::LevelFilesBrief& LevelFilesBrief(int level) const {
+ assert(level < static_cast<int>(level_files_brief_.size()));
+ return level_files_brief_[level];
+ }
+
+ // REQUIRES: This version has been saved (see VersionSet::SaveTo)
+ const std::vector<int>& FilesByCompactionPri(int level) const {
+ assert(finalized_);
+ return files_by_compaction_pri_[level];
+ }
+
+ // REQUIRES: This version has been saved (see VersionSet::SaveTo)
+ // REQUIRES: DB mutex held during access
+ const autovector<std::pair<int, FileMetaData*>>& FilesMarkedForCompaction()
+ const {
+ assert(finalized_);
+ return files_marked_for_compaction_;
+ }
+
+ int base_level() const { return base_level_; }
+
+ // REQUIRES: lock is held
+ // Set the index that is used to offset into files_by_compaction_pri_ to find
+ // the next compaction candidate file.
+ void SetNextCompactionIndex(int level, int index) {
+ next_file_to_compact_by_size_[level] = index;
+ }
+
+ // REQUIRES: lock is held
+ int NextCompactionIndex(int level) const {
+ return next_file_to_compact_by_size_[level];
+ }
+
+ // REQUIRES: This version has been saved (see VersionSet::SaveTo)
+ const FileIndexer& file_indexer() const {
+ assert(finalized_);
+ return file_indexer_;
+ }
+
+ // Only the first few entries of files_by_compaction_pri_ are sorted.
+ // There is no need to sort all the files because it is likely
+ // that on a running system, we need to look at only the first
+ // few largest files because a new version is created every few
+ // seconds/minutes (because of concurrent compactions).
+ static const size_t kNumberFilesToSort = 50;
+
+ // Return a human-readable short (single-line) summary of the number
+ // of files per level. Uses *scratch as backing store.
+ struct LevelSummaryStorage {
+ char buffer[1000];
+ };
+ struct FileSummaryStorage {
+ char buffer[3000];
+ };
+ const char* LevelSummary(LevelSummaryStorage* scratch) const;
+ // Return a human-readable short (single-line) summary of files
+ // in a specified level. Uses *scratch as backing store.
+ const char* LevelFileSummary(FileSummaryStorage* scratch, int level) const;
+
+ // Return the maximum overlapping data (in bytes) at next level for any
+ // file at a level >= 1.
+ int64_t MaxNextLevelOverlappingBytes();
+
+ // Return a human readable string that describes this version's contents.
+ std::string DebugString(bool hex = false) const;
+
+ uint64_t GetAverageValueSize() const {
+ if (accumulated_num_non_deletions_ == 0) {
+ return 0;
+ }
+ assert(accumulated_raw_key_size_ + accumulated_raw_value_size_ > 0);
+ assert(accumulated_file_size_ > 0);
+ return accumulated_raw_value_size_ / accumulated_num_non_deletions_ *
+ accumulated_file_size_ /
+ (accumulated_raw_key_size_ + accumulated_raw_value_size_);
+ }
+
+ uint64_t GetEstimatedActiveKeys() const;
+
+ double GetEstimatedCompressionRatioAtLevel(int level) const;
+
+ // re-initializes the index that is used to offset into
+ // files_by_compaction_pri_
+ // to find the next compaction candidate file.
+ void ResetNextCompactionIndex(int level) {
+ next_file_to_compact_by_size_[level] = 0;
+ }
+
+ const InternalKeyComparator* InternalComparator() {
+ return internal_comparator_;
+ }
+
+ // Returns maximum total bytes of data on a given level.
+ uint64_t MaxBytesForLevel(int level) const;
+
+ // Must be called after any change to MutableCFOptions.
+ void CalculateBaseBytes(const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& options);
+
+ // Returns an estimate of the amount of live data in bytes.
+ uint64_t EstimateLiveDataSize() const;
+
+ uint64_t estimated_compaction_needed_bytes() const {
+ return estimated_compaction_needed_bytes_;
+ }
+
+ void TEST_set_estimated_compaction_needed_bytes(uint64_t v) {
+ estimated_compaction_needed_bytes_ = v;
+ }
+
+ bool force_consistency_checks() const { return force_consistency_checks_; }
+
+ private:
+ const InternalKeyComparator* internal_comparator_;
+ const Comparator* user_comparator_;
+ int num_levels_; // Number of levels
+ int num_non_empty_levels_; // Number of levels. Any level larger than it
+ // is guaranteed to be empty.
+ // Per-level max bytes
+ std::vector<uint64_t> level_max_bytes_;
+
+ // A short brief metadata of files per level
+ autovector<rocksdb::LevelFilesBrief> level_files_brief_;
+ FileIndexer file_indexer_;
+ Arena arena_; // Used to allocate space for file_levels_
+
+ CompactionStyle compaction_style_;
+
+ // List of files per level, files in each level are arranged
+ // in increasing order of keys
+ std::vector<FileMetaData*>* files_;
+
+ // Level that L0 data should be compacted to. All levels < base_level_ should
+ // be empty. -1 if it is not level-compaction so it's not applicable.
+ int base_level_;
+
+ // A list for the same set of files that are stored in files_,
+ // but files in each level are now sorted based on file
+ // size. The file with the largest size is at the front.
+ // This vector stores the index of the file from files_.
+ std::vector<std::vector<int>> files_by_compaction_pri_;
+
+ // If true, means that files in L0 have keys with non overlapping ranges
+ bool level0_non_overlapping_;
+
+ // An index into files_by_compaction_pri_ that specifies the first
+ // file that is not yet compacted
+ std::vector<int> next_file_to_compact_by_size_;
+
+ // Only the first few entries of files_by_compaction_pri_ are sorted.
+ // There is no need to sort all the files because it is likely
+ // that on a running system, we need to look at only the first
+ // few largest files because a new version is created every few
+ // seconds/minutes (because of concurrent compactions).
+ static const size_t number_of_files_to_sort_ = 50;
+
+ // This vector contains list of files marked for compaction and also not
+ // currently being compacted. It is protected by DB mutex. It is calculated in
+ // ComputeCompactionScore()
+ autovector<std::pair<int, FileMetaData*>> files_marked_for_compaction_;
+
+ // Level that should be compacted next and its compaction score.
+ // Score < 1 means compaction is not strictly needed. These fields
+ // are initialized by Finalize().
+ // The most critical level to be compacted is listed first
+ // These are used to pick the best compaction level
+ std::vector<double> compaction_score_;
+ std::vector<int> compaction_level_;
+ int l0_delay_trigger_count_ = 0; // Count used to trigger slow down and stop
+ // for number of L0 files.
+
+ // the following are the sampled temporary stats.
+ // the current accumulated size of sampled files.
+ uint64_t accumulated_file_size_;
+ // the current accumulated size of all raw keys based on the sampled files.
+ uint64_t accumulated_raw_key_size_;
+ // the current accumulated size of all raw keys based on the sampled files.
+ uint64_t accumulated_raw_value_size_;
+ // total number of non-deletion entries
+ uint64_t accumulated_num_non_deletions_;
+ // total number of deletion entries
+ uint64_t accumulated_num_deletions_;
+ // current number of non_deletion entries
+ uint64_t current_num_non_deletions_;
+ // current number of delection entries
+ uint64_t current_num_deletions_;
+ // current number of file samples
+ uint64_t current_num_samples_;
+ // Estimated bytes needed to be compacted until all levels' size is down to
+ // target sizes.
+ uint64_t estimated_compaction_needed_bytes_;
+
+ bool finalized_;
+
+ // If set to true, we will run consistency checks even if RocksDB
+ // is compiled in release mode
+ bool force_consistency_checks_;
+
+ friend class Version;
+ friend class VersionSet;
+ // No copying allowed
+ VersionStorageInfo(const VersionStorageInfo&) = delete;
+ void operator=(const VersionStorageInfo&) = delete;
+};
+
+class Version {
+ public:
+ // Append to *iters a sequence of iterators that will
+ // yield the contents of this Version when merged together.
+ // REQUIRES: This version has been saved (see VersionSet::SaveTo)
+ void AddIterators(const ReadOptions&, const EnvOptions& soptions,
+ MergeIteratorBuilder* merger_iter_builder,
+ RangeDelAggregator* range_del_agg);
+
+ void AddIteratorsForLevel(const ReadOptions&, const EnvOptions& soptions,
+ MergeIteratorBuilder* merger_iter_builder,
+ int level, RangeDelAggregator* range_del_agg);
+
+ void AddRangeDelIteratorsForLevel(
+ const ReadOptions& read_options, const EnvOptions& soptions, int level,
+ std::vector<InternalIterator*>* range_del_iters);
+
+ // Lookup the value for key. If found, store it in *val and
+ // return OK. Else return a non-OK status.
+ // Uses *operands to store merge_operator operations to apply later.
+ //
+ // If the ReadOptions.read_tier is set to do a read-only fetch, then
+ // *value_found will be set to false if it cannot be determined whether
+ // this value exists without doing IO.
+ //
+ // If the key is Deleted, *status will be set to NotFound and
+ // *key_exists will be set to true.
+ // If no key was found, *status will be set to NotFound and
+ // *key_exists will be set to false.
+ // If seq is non-null, *seq will be set to the sequence number found
+ // for the key if a key was found.
+ //
+ // REQUIRES: lock is not held
+ void Get(const ReadOptions&, const LookupKey& key, PinnableSlice* value,
+ Status* status, MergeContext* merge_context,
+ RangeDelAggregator* range_del_agg, bool* value_found = nullptr,
+ bool* key_exists = nullptr, SequenceNumber* seq = nullptr);
+
+ // Loads some stats information from files. Call without mutex held. It needs
+ // to be called before applying the version to the version set.
+ void PrepareApply(const MutableCFOptions& mutable_cf_options,
+ bool update_stats);
+
+ // Reference count management (so Versions do not disappear out from
+ // under live iterators)
+ void Ref();
+ // Decrease reference count. Delete the object if no reference left
+ // and return true. Otherwise, return false.
+ bool Unref();
+
+ // Add all files listed in the current version to *live.
+ void AddLiveFiles(std::vector<FileDescriptor>* live);
+
+ // Return a human readable string that describes this version's contents.
+ std::string DebugString(bool hex = false, bool print_stats = false) const;
+
+ // Returns the version nuber of this version
+ uint64_t GetVersionNumber() const { return version_number_; }
+
+ // REQUIRES: lock is held
+ // On success, "tp" will contains the table properties of the file
+ // specified in "file_meta". If the file name of "file_meta" is
+ // known ahread, passing it by a non-null "fname" can save a
+ // file-name conversion.
+ Status GetTableProperties(std::shared_ptr<const TableProperties>* tp,
+ const FileMetaData* file_meta,
+ const std::string* fname = nullptr) const;
+
+ // REQUIRES: lock is held
+ // On success, *props will be populated with all SSTables' table properties.
+ // The keys of `props` are the sst file name, the values of `props` are the
+ // tables' propertis, represented as shared_ptr.
+ Status GetPropertiesOfAllTables(TablePropertiesCollection* props);
+ Status GetPropertiesOfAllTables(TablePropertiesCollection* props, int level);
+ Status GetPropertiesOfTablesInRange(const Range* range, std::size_t n,
+ TablePropertiesCollection* props) const;
+
+ // REQUIRES: lock is held
+ // On success, "tp" will contains the aggregated table property amoug
+ // the table properties of all sst files in this version.
+ Status GetAggregatedTableProperties(
+ std::shared_ptr<const TableProperties>* tp, int level = -1);
+
+ uint64_t GetEstimatedActiveKeys() {
+ return storage_info_.GetEstimatedActiveKeys();
+ }
+
+ size_t GetMemoryUsageByTableReaders();
+
+ ColumnFamilyData* cfd() const { return cfd_; }
+
+ // Return the next Version in the linked list. Used for debug only
+ Version* TEST_Next() const {
+ return next_;
+ }
+
+ int TEST_refs() const { return refs_; }
+
+ VersionStorageInfo* storage_info() { return &storage_info_; }
+
+ VersionSet* version_set() { return vset_; }
+
+ void GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta);
+
+ private:
+ Env* env_;
+ friend class VersionSet;
+
+ const InternalKeyComparator* internal_comparator() const {
+ return storage_info_.internal_comparator_;
+ }
+ const Comparator* user_comparator() const {
+ return storage_info_.user_comparator_;
+ }
+
+ bool PrefixMayMatch(const ReadOptions& read_options,
+ InternalIterator* level_iter,
+ const Slice& internal_prefix) const;
+
+ // Returns true if the filter blocks in the specified level will not be
+ // checked during read operations. In certain cases (trivial move or preload),
+ // the filter block may already be cached, but we still do not access it such
+ // that it eventually expires from the cache.
+ bool IsFilterSkipped(int level, bool is_file_last_in_level = false);
+
+ // The helper function of UpdateAccumulatedStats, which may fill the missing
+ // fields of file_mata from its associated TableProperties.
+ // Returns true if it does initialize FileMetaData.
+ bool MaybeInitializeFileMetaData(FileMetaData* file_meta);
+
+ // Update the accumulated stats associated with the current version.
+ // This accumulated stats will be used in compaction.
+ void UpdateAccumulatedStats(bool update_stats);
+
+ // Sort all files for this version based on their file size and
+ // record results in files_by_compaction_pri_. The largest files are listed
+ // first.
+ void UpdateFilesByCompactionPri();
+
+ ColumnFamilyData* cfd_; // ColumnFamilyData to which this Version belongs
+ Logger* info_log_;
+ Statistics* db_statistics_;
+ TableCache* table_cache_;
+ const MergeOperator* merge_operator_;
+
+ VersionStorageInfo storage_info_;
+ VersionSet* vset_; // VersionSet to which this Version belongs
+ Version* next_; // Next version in linked list
+ Version* prev_; // Previous version in linked list
+ int refs_; // Number of live refs to this version
+
+ // A version number that uniquely represents this version. This is
+ // used for debugging and logging purposes only.
+ uint64_t version_number_;
+
+ Version(ColumnFamilyData* cfd, VersionSet* vset, uint64_t version_number = 0);
+
+ ~Version();
+
+ // No copying allowed
+ Version(const Version&);
+ void operator=(const Version&);
+};
+
+class VersionSet {
+ public:
+ VersionSet(const std::string& dbname, const ImmutableDBOptions* db_options,
+ const EnvOptions& env_options, Cache* table_cache,
+ WriteBufferManager* write_buffer_manager,
+ WriteController* write_controller);
+ ~VersionSet();
+
+ // Apply *edit to the current version to form a new descriptor that
+ // is both saved to persistent state and installed as the new
+ // current version. Will release *mu while actually writing to the file.
+ // column_family_options has to be set if edit is column family add
+ // REQUIRES: *mu is held on entry.
+ // REQUIRES: no other thread concurrently calls LogAndApply()
+ Status LogAndApply(
+ ColumnFamilyData* column_family_data,
+ const MutableCFOptions& mutable_cf_options, VersionEdit* edit,
+ InstrumentedMutex* mu, Directory* db_directory = nullptr,
+ bool new_descriptor_log = false,
+ const ColumnFamilyOptions* column_family_options = nullptr) {
+ autovector<VersionEdit*> edit_list;
+ edit_list.push_back(edit);
+ return LogAndApply(column_family_data, mutable_cf_options, edit_list, mu,
+ db_directory, new_descriptor_log, column_family_options);
+ }
+ // The batch version. If edit_list.size() > 1, caller must ensure that
+ // no edit in the list column family add or drop
+ Status LogAndApply(
+ ColumnFamilyData* column_family_data,
+ const MutableCFOptions& mutable_cf_options,
+ const autovector<VersionEdit*>& edit_list, InstrumentedMutex* mu,
+ Directory* db_directory = nullptr, bool new_descriptor_log = false,
+ const ColumnFamilyOptions* column_family_options = nullptr);
+
+ // Recover the last saved descriptor from persistent storage.
+ // If read_only == true, Recover() will not complain if some column families
+ // are not opened
+ Status Recover(const std::vector<ColumnFamilyDescriptor>& column_families,
+ bool read_only = false);
+
+ // Reads a manifest file and returns a list of column families in
+ // column_families.
+ static Status ListColumnFamilies(std::vector<std::string>* column_families,
+ const std::string& dbname, Env* env);
+
+#ifndef ROCKSDB_LITE
+ // Try to reduce the number of levels. This call is valid when
+ // only one level from the new max level to the old
+ // max level containing files.
+ // The call is static, since number of levels is immutable during
+ // the lifetime of a RocksDB instance. It reduces number of levels
+ // in a DB by applying changes to manifest.
+ // For example, a db currently has 7 levels [0-6], and a call to
+ // to reduce to 5 [0-4] can only be executed when only one level
+ // among [4-6] contains files.
+ static Status ReduceNumberOfLevels(const std::string& dbname,
+ const Options* options,
+ const EnvOptions& env_options,
+ int new_levels);
+
+ // printf contents (for debugging)
+ Status DumpManifest(Options& options, std::string& manifestFileName,
+ bool verbose, bool hex = false, bool json = false);
+
+#endif // ROCKSDB_LITE
+
+ // Return the current manifest file number
+ uint64_t manifest_file_number() const { return manifest_file_number_; }
+
+ uint64_t options_file_number() const { return options_file_number_; }
+
+ uint64_t pending_manifest_file_number() const {
+ return pending_manifest_file_number_;
+ }
+
+ uint64_t current_next_file_number() const { return next_file_number_.load(); }
+
+ // Allocate and return a new file number
+ uint64_t NewFileNumber() { return next_file_number_.fetch_add(1); }
+
+ // Return the last sequence number.
+ uint64_t LastSequence() const {
+ return last_sequence_.load(std::memory_order_acquire);
+ }
+
+ // Note: memory_order_acquire must be sufficient.
+ uint64_t LastToBeWrittenSequence() const {
+ return last_to_be_written_sequence_.load(std::memory_order_seq_cst);
+ }
+
+ // Set the last sequence number to s.
+ void SetLastSequence(uint64_t s) {
+ assert(s >= last_sequence_);
+ // Last visible seqeunce must always be less than last written seq
+ assert(!db_options_->concurrent_prepare ||
+ s <= last_to_be_written_sequence_);
+ last_sequence_.store(s, std::memory_order_release);
+ }
+
+ // Note: memory_order_release must be sufficient
+ void SetLastToBeWrittenSequence(uint64_t s) {
+ assert(s >= last_to_be_written_sequence_);
+ last_to_be_written_sequence_.store(s, std::memory_order_seq_cst);
+ }
+
+ // Note: memory_order_release must be sufficient
+ uint64_t FetchAddLastToBeWrittenSequence(uint64_t s) {
+ return last_to_be_written_sequence_.fetch_add(s, std::memory_order_seq_cst);
+ }
+
+ // Mark the specified file number as used.
+ // REQUIRED: this is only called during single-threaded recovery
+ void MarkFileNumberUsedDuringRecovery(uint64_t number);
+
+ // Return the log file number for the log file that is currently
+ // being compacted, or zero if there is no such log file.
+ uint64_t prev_log_number() const { return prev_log_number_; }
+
+ // Returns the minimum log number such that all
+ // log numbers less than or equal to it can be deleted
+ uint64_t MinLogNumber() const {
+ uint64_t min_log_num = std::numeric_limits<uint64_t>::max();
+ for (auto cfd : *column_family_set_) {
+ // It's safe to ignore dropped column families here:
+ // cfd->IsDropped() becomes true after the drop is persisted in MANIFEST.
+ if (min_log_num > cfd->GetLogNumber() && !cfd->IsDropped()) {
+ min_log_num = cfd->GetLogNumber();
+ }
+ }
+ return min_log_num;
+ }
+
+ // Create an iterator that reads over the compaction inputs for "*c".
+ // The caller should delete the iterator when no longer needed.
+ InternalIterator* MakeInputIterator(const Compaction* c,
+ RangeDelAggregator* range_del_agg);
+
+ // Add all files listed in any live version to *live.
+ void AddLiveFiles(std::vector<FileDescriptor>* live_list);
+
+ // Return the approximate size of data to be scanned for range [start, end)
+ // in levels [start_level, end_level). If end_level == 0 it will search
+ // through all non-empty levels
+ uint64_t ApproximateSize(Version* v, const Slice& start, const Slice& end,
+ int start_level = 0, int end_level = -1);
+
+ // Return the size of the current manifest file
+ uint64_t manifest_file_size() const { return manifest_file_size_; }
+
+ // verify that the files that we started with for a compaction
+ // still exist in the current version and in the same original level.
+ // This ensures that a concurrent compaction did not erroneously
+ // pick the same files to compact.
+ bool VerifyCompactionFileConsistency(Compaction* c);
+
+ Status GetMetadataForFile(uint64_t number, int* filelevel,
+ FileMetaData** metadata, ColumnFamilyData** cfd);
+
+ // This function doesn't support leveldb SST filenames
+ void GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata);
+
+ void GetObsoleteFiles(std::vector<FileMetaData*>* files,
+ std::vector<std::string>* manifest_filenames,
+ uint64_t min_pending_output);
+
+ ColumnFamilySet* GetColumnFamilySet() { return column_family_set_.get(); }
+ const EnvOptions& env_options() { return env_options_; }
+
+ static uint64_t GetNumLiveVersions(Version* dummy_versions);
+
+ static uint64_t GetTotalSstFilesSize(Version* dummy_versions);
+
+ private:
+ struct ManifestWriter;
+
+ friend class Version;
+ friend class DBImpl;
+
+ struct LogReporter : public log::Reader::Reporter {
+ Status* status;
+ virtual void Corruption(size_t bytes, const Status& s) override {
+ if (this->status->ok()) *this->status = s;
+ }
+ };
+
+ // ApproximateSize helper
+ uint64_t ApproximateSizeLevel0(Version* v, const LevelFilesBrief& files_brief,
+ const Slice& start, const Slice& end);
+
+ uint64_t ApproximateSize(Version* v, const FdWithKeyRange& f,
+ const Slice& key);
+
+ // Save current contents to *log
+ Status WriteSnapshot(log::Writer* log);
+
+ void AppendVersion(ColumnFamilyData* column_family_data, Version* v);
+
+ ColumnFamilyData* CreateColumnFamily(const ColumnFamilyOptions& cf_options,
+ VersionEdit* edit);
+
+ std::unique_ptr<ColumnFamilySet> column_family_set_;
+
+ Env* const env_;
+ const std::string dbname_;
+ const ImmutableDBOptions* const db_options_;
+ std::atomic<uint64_t> next_file_number_;
+ uint64_t manifest_file_number_;
+ uint64_t options_file_number_;
+ uint64_t pending_manifest_file_number_;
+ // The last seq visible to reads
+ std::atomic<uint64_t> last_sequence_;
+ // The last seq with which a writer has written/will write.
+ std::atomic<uint64_t> last_to_be_written_sequence_;
+ uint64_t prev_log_number_; // 0 or backing store for memtable being compacted
+
+ // Opened lazily
+ unique_ptr<log::Writer> descriptor_log_;
+
+ // generates a increasing version number for every new version
+ uint64_t current_version_number_;
+
+ // Queue of writers to the manifest file
+ std::deque<ManifestWriter*> manifest_writers_;
+
+ // Current size of manifest file
+ uint64_t manifest_file_size_;
+
+ std::vector<FileMetaData*> obsolete_files_;
+ std::vector<std::string> obsolete_manifests_;
+
+ // env options for all reads and writes except compactions
+ const EnvOptions& env_options_;
+
+ // env options used for compactions. This is a copy of
+ // env_options_ but with readaheads set to readahead_compactions_.
+ const EnvOptions env_options_compactions_;
+
+ // No copying allowed
+ VersionSet(const VersionSet&);
+ void operator=(const VersionSet&);
+
+ void LogAndApplyCFHelper(VersionEdit* edit);
+ void LogAndApplyHelper(ColumnFamilyData* cfd, VersionBuilder* b, Version* v,
+ VersionEdit* edit, InstrumentedMutex* mu);
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/wal_manager.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/wal_manager.cc b/thirdparty/rocksdb/db/wal_manager.cc
new file mode 100644
index 0000000..4a9ecbf
--- /dev/null
+++ b/thirdparty/rocksdb/db/wal_manager.cc
@@ -0,0 +1,476 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/wal_manager.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <algorithm>
+#include <vector>
+#include <memory>
+
+#include "db/log_reader.h"
+#include "db/log_writer.h"
+#include "db/transaction_log_impl.h"
+#include "db/write_batch_internal.h"
+#include "port/port.h"
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+#include "rocksdb/write_batch.h"
+#include "util/cast_util.h"
+#include "util/coding.h"
+#include "util/file_reader_writer.h"
+#include "util/filename.h"
+#include "util/logging.h"
+#include "util/mutexlock.h"
+#include "util/string_util.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+#ifndef ROCKSDB_LITE
+
+Status WalManager::GetSortedWalFiles(VectorLogPtr& files) {
+ // First get sorted files in db dir, then get sorted files from archived
+ // dir, to avoid a race condition where a log file is moved to archived
+ // dir in between.
+ Status s;
+ // list wal files in main db dir.
+ VectorLogPtr logs;
+ s = GetSortedWalsOfType(db_options_.wal_dir, logs, kAliveLogFile);
+ if (!s.ok()) {
+ return s;
+ }
+
+ // Reproduce the race condition where a log file is moved
+ // to archived dir, between these two sync points, used in
+ // (DBTest,TransactionLogIteratorRace)
+ TEST_SYNC_POINT("WalManager::GetSortedWalFiles:1");
+ TEST_SYNC_POINT("WalManager::GetSortedWalFiles:2");
+
+ files.clear();
+ // list wal files in archive dir.
+ std::string archivedir = ArchivalDirectory(db_options_.wal_dir);
+ Status exists = env_->FileExists(archivedir);
+ if (exists.ok()) {
+ s = GetSortedWalsOfType(archivedir, files, kArchivedLogFile);
+ if (!s.ok()) {
+ return s;
+ }
+ } else if (!exists.IsNotFound()) {
+ assert(s.IsIOError());
+ return s;
+ }
+
+ uint64_t latest_archived_log_number = 0;
+ if (!files.empty()) {
+ latest_archived_log_number = files.back()->LogNumber();
+ ROCKS_LOG_INFO(db_options_.info_log, "Latest Archived log: %" PRIu64,
+ latest_archived_log_number);
+ }
+
+ files.reserve(files.size() + logs.size());
+ for (auto& log : logs) {
+ if (log->LogNumber() > latest_archived_log_number) {
+ files.push_back(std::move(log));
+ } else {
+ // When the race condition happens, we could see the
+ // same log in both db dir and archived dir. Simply
+ // ignore the one in db dir. Note that, if we read
+ // archived dir first, we would have missed the log file.
+ ROCKS_LOG_WARN(db_options_.info_log, "%s already moved to archive",
+ log->PathName().c_str());
+ }
+ }
+
+ return s;
+}
+
+Status WalManager::GetUpdatesSince(
+ SequenceNumber seq, std::unique_ptr<TransactionLogIterator>* iter,
+ const TransactionLogIterator::ReadOptions& read_options,
+ VersionSet* version_set) {
+
+ // Get all sorted Wal Files.
+ // Do binary search and open files and find the seq number.
+
+ std::unique_ptr<VectorLogPtr> wal_files(new VectorLogPtr);
+ Status s = GetSortedWalFiles(*wal_files);
+ if (!s.ok()) {
+ return s;
+ }
+
+ s = RetainProbableWalFiles(*wal_files, seq);
+ if (!s.ok()) {
+ return s;
+ }
+ iter->reset(new TransactionLogIteratorImpl(
+ db_options_.wal_dir, &db_options_, read_options, env_options_, seq,
+ std::move(wal_files), version_set));
+ return (*iter)->status();
+}
+
+// 1. Go through all archived files and
+// a. if ttl is enabled, delete outdated files
+// b. if archive size limit is enabled, delete empty files,
+// compute file number and size.
+// 2. If size limit is enabled:
+// a. compute how many files should be deleted
+// b. get sorted non-empty archived logs
+// c. delete what should be deleted
+void WalManager::PurgeObsoleteWALFiles() {
+ bool const ttl_enabled = db_options_.wal_ttl_seconds > 0;
+ bool const size_limit_enabled = db_options_.wal_size_limit_mb > 0;
+ if (!ttl_enabled && !size_limit_enabled) {
+ return;
+ }
+
+ int64_t current_time;
+ Status s = env_->GetCurrentTime(¤t_time);
+ if (!s.ok()) {
+ ROCKS_LOG_ERROR(db_options_.info_log, "Can't get current time: %s",
+ s.ToString().c_str());
+ assert(false);
+ return;
+ }
+ uint64_t const now_seconds = static_cast<uint64_t>(current_time);
+ uint64_t const time_to_check = (ttl_enabled && !size_limit_enabled)
+ ? db_options_.wal_ttl_seconds / 2
+ : kDefaultIntervalToDeleteObsoleteWAL;
+
+ if (purge_wal_files_last_run_ + time_to_check > now_seconds) {
+ return;
+ }
+
+ purge_wal_files_last_run_ = now_seconds;
+
+ std::string archival_dir = ArchivalDirectory(db_options_.wal_dir);
+ std::vector<std::string> files;
+ s = env_->GetChildren(archival_dir, &files);
+ if (!s.ok()) {
+ ROCKS_LOG_ERROR(db_options_.info_log, "Can't get archive files: %s",
+ s.ToString().c_str());
+ assert(false);
+ return;
+ }
+
+ size_t log_files_num = 0;
+ uint64_t log_file_size = 0;
+
+ for (auto& f : files) {
+ uint64_t number;
+ FileType type;
+ if (ParseFileName(f, &number, &type) && type == kLogFile) {
+ std::string const file_path = archival_dir + "/" + f;
+ if (ttl_enabled) {
+ uint64_t file_m_time;
+ s = env_->GetFileModificationTime(file_path, &file_m_time);
+ if (!s.ok()) {
+ ROCKS_LOG_WARN(db_options_.info_log,
+ "Can't get file mod time: %s: %s", file_path.c_str(),
+ s.ToString().c_str());
+ continue;
+ }
+ if (now_seconds - file_m_time > db_options_.wal_ttl_seconds) {
+ s = env_->DeleteFile(file_path);
+ if (!s.ok()) {
+ ROCKS_LOG_WARN(db_options_.info_log, "Can't delete file: %s: %s",
+ file_path.c_str(), s.ToString().c_str());
+ continue;
+ } else {
+ MutexLock l(&read_first_record_cache_mutex_);
+ read_first_record_cache_.erase(number);
+ }
+ continue;
+ }
+ }
+
+ if (size_limit_enabled) {
+ uint64_t file_size;
+ s = env_->GetFileSize(file_path, &file_size);
+ if (!s.ok()) {
+ ROCKS_LOG_ERROR(db_options_.info_log,
+ "Unable to get file size: %s: %s", file_path.c_str(),
+ s.ToString().c_str());
+ return;
+ } else {
+ if (file_size > 0) {
+ log_file_size = std::max(log_file_size, file_size);
+ ++log_files_num;
+ } else {
+ s = env_->DeleteFile(file_path);
+ if (!s.ok()) {
+ ROCKS_LOG_WARN(db_options_.info_log,
+ "Unable to delete file: %s: %s", file_path.c_str(),
+ s.ToString().c_str());
+ continue;
+ } else {
+ MutexLock l(&read_first_record_cache_mutex_);
+ read_first_record_cache_.erase(number);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (0 == log_files_num || !size_limit_enabled) {
+ return;
+ }
+
+ size_t const files_keep_num =
+ db_options_.wal_size_limit_mb * 1024 * 1024 / log_file_size;
+ if (log_files_num <= files_keep_num) {
+ return;
+ }
+
+ size_t files_del_num = log_files_num - files_keep_num;
+ VectorLogPtr archived_logs;
+ GetSortedWalsOfType(archival_dir, archived_logs, kArchivedLogFile);
+
+ if (files_del_num > archived_logs.size()) {
+ ROCKS_LOG_WARN(db_options_.info_log,
+ "Trying to delete more archived log files than "
+ "exist. Deleting all");
+ files_del_num = archived_logs.size();
+ }
+
+ for (size_t i = 0; i < files_del_num; ++i) {
+ std::string const file_path = archived_logs[i]->PathName();
+ s = env_->DeleteFile(db_options_.wal_dir + "/" + file_path);
+ if (!s.ok()) {
+ ROCKS_LOG_WARN(db_options_.info_log, "Unable to delete file: %s: %s",
+ file_path.c_str(), s.ToString().c_str());
+ continue;
+ } else {
+ MutexLock l(&read_first_record_cache_mutex_);
+ read_first_record_cache_.erase(archived_logs[i]->LogNumber());
+ }
+ }
+}
+
+void WalManager::ArchiveWALFile(const std::string& fname, uint64_t number) {
+ auto archived_log_name = ArchivedLogFileName(db_options_.wal_dir, number);
+ // The sync point below is used in (DBTest,TransactionLogIteratorRace)
+ TEST_SYNC_POINT("WalManager::PurgeObsoleteFiles:1");
+ Status s = env_->RenameFile(fname, archived_log_name);
+ // The sync point below is used in (DBTest,TransactionLogIteratorRace)
+ TEST_SYNC_POINT("WalManager::PurgeObsoleteFiles:2");
+ ROCKS_LOG_INFO(db_options_.info_log, "Move log file %s to %s -- %s\n",
+ fname.c_str(), archived_log_name.c_str(),
+ s.ToString().c_str());
+}
+
+namespace {
+struct CompareLogByPointer {
+ bool operator()(const std::unique_ptr<LogFile>& a,
+ const std::unique_ptr<LogFile>& b) {
+ LogFileImpl* a_impl = static_cast_with_check<LogFileImpl, LogFile>(a.get());
+ LogFileImpl* b_impl = static_cast_with_check<LogFileImpl, LogFile>(b.get());
+ return *a_impl < *b_impl;
+ }
+};
+}
+
+Status WalManager::GetSortedWalsOfType(const std::string& path,
+ VectorLogPtr& log_files,
+ WalFileType log_type) {
+ std::vector<std::string> all_files;
+ const Status status = env_->GetChildren(path, &all_files);
+ if (!status.ok()) {
+ return status;
+ }
+ log_files.reserve(all_files.size());
+ for (const auto& f : all_files) {
+ uint64_t number;
+ FileType type;
+ if (ParseFileName(f, &number, &type) && type == kLogFile) {
+ SequenceNumber sequence;
+ Status s = ReadFirstRecord(log_type, number, &sequence);
+ if (!s.ok()) {
+ return s;
+ }
+ if (sequence == 0) {
+ // empty file
+ continue;
+ }
+
+ // Reproduce the race condition where a log file is moved
+ // to archived dir, between these two sync points, used in
+ // (DBTest,TransactionLogIteratorRace)
+ TEST_SYNC_POINT("WalManager::GetSortedWalsOfType:1");
+ TEST_SYNC_POINT("WalManager::GetSortedWalsOfType:2");
+
+ uint64_t size_bytes;
+ s = env_->GetFileSize(LogFileName(path, number), &size_bytes);
+ // re-try in case the alive log file has been moved to archive.
+ std::string archived_file = ArchivedLogFileName(path, number);
+ if (!s.ok() && log_type == kAliveLogFile &&
+ env_->FileExists(archived_file).ok()) {
+ s = env_->GetFileSize(archived_file, &size_bytes);
+ if (!s.ok() && env_->FileExists(archived_file).IsNotFound()) {
+ // oops, the file just got deleted from archived dir! move on
+ s = Status::OK();
+ continue;
+ }
+ }
+ if (!s.ok()) {
+ return s;
+ }
+
+ log_files.push_back(std::unique_ptr<LogFile>(
+ new LogFileImpl(number, log_type, sequence, size_bytes)));
+ }
+ }
+ CompareLogByPointer compare_log_files;
+ std::sort(log_files.begin(), log_files.end(), compare_log_files);
+ return status;
+}
+
+Status WalManager::RetainProbableWalFiles(VectorLogPtr& all_logs,
+ const SequenceNumber target) {
+ int64_t start = 0; // signed to avoid overflow when target is < first file.
+ int64_t end = static_cast<int64_t>(all_logs.size()) - 1;
+ // Binary Search. avoid opening all files.
+ while (end >= start) {
+ int64_t mid = start + (end - start) / 2; // Avoid overflow.
+ SequenceNumber current_seq_num = all_logs.at(mid)->StartSequence();
+ if (current_seq_num == target) {
+ end = mid;
+ break;
+ } else if (current_seq_num < target) {
+ start = mid + 1;
+ } else {
+ end = mid - 1;
+ }
+ }
+ // end could be -ve.
+ size_t start_index = std::max(static_cast<int64_t>(0), end);
+ // The last wal file is always included
+ all_logs.erase(all_logs.begin(), all_logs.begin() + start_index);
+ return Status::OK();
+}
+
+Status WalManager::ReadFirstRecord(const WalFileType type,
+ const uint64_t number,
+ SequenceNumber* sequence) {
+ *sequence = 0;
+ if (type != kAliveLogFile && type != kArchivedLogFile) {
+ ROCKS_LOG_ERROR(db_options_.info_log, "[WalManger] Unknown file type %s",
+ ToString(type).c_str());
+ return Status::NotSupported(
+ "File Type Not Known " + ToString(type));
+ }
+ {
+ MutexLock l(&read_first_record_cache_mutex_);
+ auto itr = read_first_record_cache_.find(number);
+ if (itr != read_first_record_cache_.end()) {
+ *sequence = itr->second;
+ return Status::OK();
+ }
+ }
+ Status s;
+ if (type == kAliveLogFile) {
+ std::string fname = LogFileName(db_options_.wal_dir, number);
+ s = ReadFirstLine(fname, number, sequence);
+ if (env_->FileExists(fname).ok() && !s.ok()) {
+ // return any error that is not caused by non-existing file
+ return s;
+ }
+ }
+
+ if (type == kArchivedLogFile || !s.ok()) {
+ // check if the file got moved to archive.
+ std::string archived_file =
+ ArchivedLogFileName(db_options_.wal_dir, number);
+ s = ReadFirstLine(archived_file, number, sequence);
+ // maybe the file was deleted from archive dir. If that's the case, return
+ // Status::OK(). The caller with identify this as empty file because
+ // *sequence == 0
+ if (!s.ok() && env_->FileExists(archived_file).IsNotFound()) {
+ return Status::OK();
+ }
+ }
+
+ if (s.ok() && *sequence != 0) {
+ MutexLock l(&read_first_record_cache_mutex_);
+ read_first_record_cache_.insert({number, *sequence});
+ }
+ return s;
+}
+
+// the function returns status.ok() and sequence == 0 if the file exists, but is
+// empty
+Status WalManager::ReadFirstLine(const std::string& fname,
+ const uint64_t number,
+ SequenceNumber* sequence) {
+ struct LogReporter : public log::Reader::Reporter {
+ Env* env;
+ Logger* info_log;
+ const char* fname;
+
+ Status* status;
+ bool ignore_error; // true if db_options_.paranoid_checks==false
+ virtual void Corruption(size_t bytes, const Status& s) override {
+ ROCKS_LOG_WARN(info_log, "[WalManager] %s%s: dropping %d bytes; %s",
+ (this->ignore_error ? "(ignoring error) " : ""), fname,
+ static_cast<int>(bytes), s.ToString().c_str());
+ if (this->status->ok()) {
+ // only keep the first error
+ *this->status = s;
+ }
+ }
+ };
+
+ std::unique_ptr<SequentialFile> file;
+ Status status = env_->NewSequentialFile(
+ fname, &file, env_->OptimizeForLogRead(env_options_));
+ unique_ptr<SequentialFileReader> file_reader(
+ new SequentialFileReader(std::move(file)));
+
+ if (!status.ok()) {
+ return status;
+ }
+
+ LogReporter reporter;
+ reporter.env = env_;
+ reporter.info_log = db_options_.info_log.get();
+ reporter.fname = fname.c_str();
+ reporter.status = &status;
+ reporter.ignore_error = !db_options_.paranoid_checks;
+ log::Reader reader(db_options_.info_log, std::move(file_reader), &reporter,
+ true /*checksum*/, 0 /*initial_offset*/, number);
+ std::string scratch;
+ Slice record;
+
+ if (reader.ReadRecord(&record, &scratch) &&
+ (status.ok() || !db_options_.paranoid_checks)) {
+ if (record.size() < WriteBatchInternal::kHeader) {
+ reporter.Corruption(record.size(),
+ Status::Corruption("log record too small"));
+ // TODO read record's till the first no corrupt entry?
+ } else {
+ WriteBatch batch;
+ WriteBatchInternal::SetContents(&batch, record);
+ *sequence = WriteBatchInternal::Sequence(&batch);
+ return Status::OK();
+ }
+ }
+
+ // ReadRecord returns false on EOF, which means that the log file is empty. we
+ // return status.ok() in that case and set sequence number to 0
+ *sequence = 0;
+ return status;
+}
+
+#endif // ROCKSDB_LITE
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/wal_manager.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/wal_manager.h b/thirdparty/rocksdb/db/wal_manager.h
new file mode 100644
index 0000000..aa62d79
--- /dev/null
+++ b/thirdparty/rocksdb/db/wal_manager.h
@@ -0,0 +1,95 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#pragma once
+
+#include <atomic>
+#include <deque>
+#include <limits>
+#include <set>
+#include <utility>
+#include <vector>
+#include <string>
+#include <memory>
+
+#include "db/version_set.h"
+#include "options/db_options.h"
+#include "port/port.h"
+#include "rocksdb/env.h"
+#include "rocksdb/status.h"
+#include "rocksdb/transaction_log.h"
+#include "rocksdb/types.h"
+
+namespace rocksdb {
+
+#ifndef ROCKSDB_LITE
+class WalManager {
+ public:
+ WalManager(const ImmutableDBOptions& db_options,
+ const EnvOptions& env_options)
+ : db_options_(db_options),
+ env_options_(env_options),
+ env_(db_options.env),
+ purge_wal_files_last_run_(0) {}
+
+ Status GetSortedWalFiles(VectorLogPtr& files);
+
+ Status GetUpdatesSince(
+ SequenceNumber seq_number, std::unique_ptr<TransactionLogIterator>* iter,
+ const TransactionLogIterator::ReadOptions& read_options,
+ VersionSet* version_set);
+
+ void PurgeObsoleteWALFiles();
+
+ void ArchiveWALFile(const std::string& fname, uint64_t number);
+
+ Status TEST_ReadFirstRecord(const WalFileType type, const uint64_t number,
+ SequenceNumber* sequence) {
+ return ReadFirstRecord(type, number, sequence);
+ }
+
+ Status TEST_ReadFirstLine(const std::string& fname, const uint64_t number,
+ SequenceNumber* sequence) {
+ return ReadFirstLine(fname, number, sequence);
+ }
+
+ private:
+ Status GetSortedWalsOfType(const std::string& path, VectorLogPtr& log_files,
+ WalFileType type);
+ // Requires: all_logs should be sorted with earliest log file first
+ // Retains all log files in all_logs which contain updates with seq no.
+ // Greater Than or Equal to the requested SequenceNumber.
+ Status RetainProbableWalFiles(VectorLogPtr& all_logs,
+ const SequenceNumber target);
+
+ Status ReadFirstRecord(const WalFileType type, const uint64_t number,
+ SequenceNumber* sequence);
+
+ Status ReadFirstLine(const std::string& fname, const uint64_t number,
+ SequenceNumber* sequence);
+
+ // ------- state from DBImpl ------
+ const ImmutableDBOptions& db_options_;
+ const EnvOptions& env_options_;
+ Env* env_;
+
+ // ------- WalManager state -------
+ // cache for ReadFirstRecord() calls
+ std::unordered_map<uint64_t, SequenceNumber> read_first_record_cache_;
+ port::Mutex read_first_record_cache_mutex_;
+
+ // last time when PurgeObsoleteWALFiles ran.
+ uint64_t purge_wal_files_last_run_;
+
+ // obsolete files will be deleted every this seconds if ttl deletion is
+ // enabled and archive size_limit is disabled.
+ static const uint64_t kDefaultIntervalToDeleteObsoleteWAL = 600;
+};
+
+#endif // ROCKSDB_LITE
+} // namespace rocksdb
[02/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_search.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_search.scss b/thirdparty/rocksdb/docs/_sass/_search.scss
new file mode 100644
index 0000000..eadfa11
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_search.scss
@@ -0,0 +1,142 @@
+input[type="search"] {
+ -moz-appearance: none;
+ -webkit-appearance: none;
+}
+
+.navSearchWrapper {
+ align-self: center;
+ position: relative;
+
+ &::before {
+ border: 3px solid $primary-overlay-special;
+ border-radius: 50%;
+ content: " ";
+ display: block;
+ height: 6px;
+ left: 15px;
+ width: 6px;
+ position: absolute;
+ top: 4px;
+ z-index: 1;
+ }
+
+ &::after {
+ background: $primary-overlay-special;
+ content: " ";
+ height: 7px;
+ left: 24px;
+ position: absolute;
+ transform: rotate(-45deg);
+ top: 12px;
+ width: 3px;
+ z-index: 1;
+ }
+
+ .aa-dropdown-menu {
+ background: $secondary-bg;
+ border: 3px solid rgba($text, 0.25);
+ color: $text;
+ font-size: 14px;
+ left: auto !important;
+ line-height: 1.2em;
+ right: 0 !important;
+
+ .algolia-docsearch-suggestion--category-header {
+ background: $primary-overlay-special;
+ color: $primary-bg;
+
+ .algolia-docsearch-suggestion--highlight {
+ background-color: $primary-bg;
+ color: $primary-overlay;
+ }
+ }
+
+ .algolia-docsearch-suggestion--title .algolia-docsearch-suggestion--highlight,
+ .algolia-docsearch-suggestion--subcategory-column .algolia-docsearch-suggestion--highlight {
+ color: $primary-bg;
+ }
+
+ .algolia-docsearch-suggestion__secondary,
+ .algolia-docsearch-suggestion--subcategory-column {
+ border-color: rgba($text, 0.3);
+ }
+ }
+}
+
+input#search_input {
+ padding-left: 25px;
+ font-size: 14px;
+ line-height: 20px;
+ border-radius: 20px;
+ background-color: rgba($primary-overlay-special, 0.25);
+ border: none;
+ color: rgba($primary-overlay-special, 0);
+ outline: none;
+ position: relative;
+ transition: background-color .2s cubic-bezier(0.68, -0.55, 0.265, 1.55), width .2s cubic-bezier(0.68, -0.55, 0.265, 1.55), color .2s ease;
+ width: 60px;
+
+ &:focus, &:active {
+ background-color: $secondary-bg;
+ color: $text;
+ width: 240px;
+ }
+}
+
+.navigationSlider {
+ .navSearchWrapper {
+ &::before {
+ left: 6px;
+ top: 6px;
+ }
+
+ &::after {
+ left: 15px;
+ top: 14px;
+ }
+ }
+
+ input#search_input_react {
+ box-sizing: border-box;
+ padding-left: 25px;
+ font-size: 14px;
+ line-height: 20px;
+ border-radius: 20px;
+ background-color: rgba($primary-overlay-special, 0.25);
+ border: none;
+ color: $text;
+ outline: none;
+ position: relative;
+ transition: background-color .2s cubic-bezier(0.68, -0.55, 0.265, 1.55), width .2s cubic-bezier(0.68, -0.55, 0.265, 1.55), color .2s ease;
+ width: 100%;
+
+ &:focus, &:active {
+ background-color: $primary-bg;
+ color: $primary-overlay;
+ }
+ }
+
+ .algolia-docsearch-suggestion--subcategory-inline {
+ display: none;
+ }
+
+ & > span {
+ width: 100%;
+ }
+
+ .aa-dropdown-menu {
+ background: $secondary-bg;
+ border: 0px solid $secondary-bg;
+ color: $text;
+ font-size: 12px;
+ line-height: 2em;
+ max-height: 140px;
+ min-width: auto;
+ overflow-y: scroll;
+ -webkit-overflow-scrolling: touch;
+ padding: 0;
+ border-radius: 0;
+ position: relative !important;
+ width: 100%;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_slideshow.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_slideshow.scss b/thirdparty/rocksdb/docs/_sass/_slideshow.scss
new file mode 100644
index 0000000..cd98a6c
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_slideshow.scss
@@ -0,0 +1,48 @@
+.slideshow {
+ position: relative;
+
+ .slide {
+ display: none;
+
+ img {
+ display: block;
+ margin: 0 auto;
+ }
+
+ &.slideActive {
+ display: block;
+ }
+
+ a {
+ border: none;
+ display: block;
+ }
+ }
+
+ .pagination {
+ display: block;
+ margin: -10px;
+ padding: 1em 0;
+ text-align: center;
+ width: 100%;
+
+ .pager {
+ background: transparent;
+ border: 2px solid rgba(255, 255, 255, 0.5);
+ border-radius: 50%;
+ cursor: pointer;
+ display: inline-block;
+ height: 12px;
+ margin: 10px;
+ transition: background-color 0.3s, border-color 0.3s;
+ width: 12px;
+
+ &.pagerActive {
+ background: rgba(255, 255, 255, 0.5);
+ border-width: 4px;
+ height: 8px;
+ width: 8px;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_syntax-highlighting.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_syntax-highlighting.scss b/thirdparty/rocksdb/docs/_sass/_syntax-highlighting.scss
new file mode 100644
index 0000000..e55c88a
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_syntax-highlighting.scss
@@ -0,0 +1,129 @@
+
+
+.rougeHighlight { background-color: $code-bg; color: #93a1a1 }
+.rougeHighlight .c { color: #586e75 } /* Comment */
+.rougeHighlight .err { color: #93a1a1 } /* Error */
+.rougeHighlight .g { color: #93a1a1 } /* Generic */
+.rougeHighlight .k { color: #859900 } /* Keyword */
+.rougeHighlight .l { color: #93a1a1 } /* Literal */
+.rougeHighlight .n { color: #93a1a1 } /* Name */
+.rougeHighlight .o { color: #859900 } /* Operator */
+.rougeHighlight .x { color: #cb4b16 } /* Other */
+.rougeHighlight .p { color: #93a1a1 } /* Punctuation */
+.rougeHighlight .cm { color: #586e75 } /* Comment.Multiline */
+.rougeHighlight .cp { color: #859900 } /* Comment.Preproc */
+.rougeHighlight .c1 { color: #72c02c; } /* Comment.Single */
+.rougeHighlight .cs { color: #859900 } /* Comment.Special */
+.rougeHighlight .gd { color: #2aa198 } /* Generic.Deleted */
+.rougeHighlight .ge { color: #93a1a1; font-style: italic } /* Generic.Emph */
+.rougeHighlight .gr { color: #dc322f } /* Generic.Error */
+.rougeHighlight .gh { color: #cb4b16 } /* Generic.Heading */
+.rougeHighlight .gi { color: #859900 } /* Generic.Inserted */
+.rougeHighlight .go { color: #93a1a1 } /* Generic.Output */
+.rougeHighlight .gp { color: #93a1a1 } /* Generic.Prompt */
+.rougeHighlight .gs { color: #93a1a1; font-weight: bold } /* Generic.Strong */
+.rougeHighlight .gu { color: #cb4b16 } /* Generic.Subheading */
+.rougeHighlight .gt { color: #93a1a1 } /* Generic.Traceback */
+.rougeHighlight .kc { color: #cb4b16 } /* Keyword.Constant */
+.rougeHighlight .kd { color: #268bd2 } /* Keyword.Declaration */
+.rougeHighlight .kn { color: #859900 } /* Keyword.Namespace */
+.rougeHighlight .kp { color: #859900 } /* Keyword.Pseudo */
+.rougeHighlight .kr { color: #268bd2 } /* Keyword.Reserved */
+.rougeHighlight .kt { color: #dc322f } /* Keyword.Type */
+.rougeHighlight .ld { color: #93a1a1 } /* Literal.Date */
+.rougeHighlight .m { color: #2aa198 } /* Literal.Number */
+.rougeHighlight .s { color: #2aa198 } /* Literal.String */
+.rougeHighlight .na { color: #93a1a1 } /* Name.Attribute */
+.rougeHighlight .nb { color: #B58900 } /* Name.Builtin */
+.rougeHighlight .nc { color: #268bd2 } /* Name.Class */
+.rougeHighlight .no { color: #cb4b16 } /* Name.Constant */
+.rougeHighlight .nd { color: #268bd2 } /* Name.Decorator */
+.rougeHighlight .ni { color: #cb4b16 } /* Name.Entity */
+.rougeHighlight .ne { color: #cb4b16 } /* Name.Exception */
+.rougeHighlight .nf { color: #268bd2 } /* Name.Function */
+.rougeHighlight .nl { color: #93a1a1 } /* Name.Label */
+.rougeHighlight .nn { color: #93a1a1 } /* Name.Namespace */
+.rougeHighlight .nx { color: #93a1a1 } /* Name.Other */
+.rougeHighlight .py { color: #93a1a1 } /* Name.Property */
+.rougeHighlight .nt { color: #268bd2 } /* Name.Tag */
+.rougeHighlight .nv { color: #268bd2 } /* Name.Variable */
+.rougeHighlight .ow { color: #859900 } /* Operator.Word */
+.rougeHighlight .w { color: #93a1a1 } /* Text.Whitespace */
+.rougeHighlight .mf { color: #2aa198 } /* Literal.Number.Float */
+.rougeHighlight .mh { color: #2aa198 } /* Literal.Number.Hex */
+.rougeHighlight .mi { color: #2aa198 } /* Literal.Number.Integer */
+.rougeHighlight .mo { color: #2aa198 } /* Literal.Number.Oct */
+.rougeHighlight .sb { color: #586e75 } /* Literal.String.Backtick */
+.rougeHighlight .sc { color: #2aa198 } /* Literal.String.Char */
+.rougeHighlight .sd { color: #93a1a1 } /* Literal.String.Doc */
+.rougeHighlight .s2 { color: #2aa198 } /* Literal.String.Double */
+.rougeHighlight .se { color: #cb4b16 } /* Literal.String.Escape */
+.rougeHighlight .sh { color: #93a1a1 } /* Literal.String.Heredoc */
+.rougeHighlight .si { color: #2aa198 } /* Literal.String.Interpol */
+.rougeHighlight .sx { color: #2aa198 } /* Literal.String.Other */
+.rougeHighlight .sr { color: #dc322f } /* Literal.String.Regex */
+.rougeHighlight .s1 { color: #2aa198 } /* Literal.String.Single */
+.rougeHighlight .ss { color: #2aa198 } /* Literal.String.Symbol */
+.rougeHighlight .bp { color: #268bd2 } /* Name.Builtin.Pseudo */
+.rougeHighlight .vc { color: #268bd2 } /* Name.Variable.Class */
+.rougeHighlight .vg { color: #268bd2 } /* Name.Variable.Global */
+.rougeHighlight .vi { color: #268bd2 } /* Name.Variable.Instance */
+.rougeHighlight .il { color: #2aa198 } /* Literal.Number.Integer.Long */
+
+.highlighter-rouge {
+ color: darken(#72c02c, 8%);
+ font: 800 12px/1.5em Hack, monospace;
+ max-width: 100%;
+
+ .rougeHighlight {
+ border-radius: 3px;
+ margin: 20px 0;
+ padding: 0px;
+ overflow-x: scroll;
+ -webkit-overflow-scrolling: touch;
+
+ table {
+ background: none;
+ border: none;
+
+ tbody {
+ tr {
+ background: none;
+ display: flex;
+ flex-flow: row nowrap;
+
+ td {
+ display: block;
+ flex: 1 1;
+
+ &.gutter {
+ border-right: 1px solid lighten($code-bg, 10%);
+ color: lighten($code-bg, 15%);
+ margin-right: 10px;
+ max-width: 40px;
+ padding-right: 10px;
+
+ pre {
+ max-width: 20px;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+p > .highlighter-rouge,
+li > .highlighter-rouge,
+a > .highlighter-rouge {
+ font-size: 16px;
+ font-weight: 400;
+ line-height: inherit;
+}
+
+a:hover {
+ .highlighter-rouge {
+ color: white;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_sass/_tables.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_sass/_tables.scss b/thirdparty/rocksdb/docs/_sass/_tables.scss
new file mode 100644
index 0000000..f847c70
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_sass/_tables.scss
@@ -0,0 +1,47 @@
+table {
+ background: $lightergrey;
+ border: 1px solid $lightgrey;
+ border-collapse: collapse;
+ display:table;
+ margin: 20px 0;
+
+ thead {
+ border-bottom: 1px solid $lightgrey;
+ display: table-header-group;
+ }
+ tbody {
+ display: table-row-group;
+ }
+ tr {
+ display: table-row;
+ &:nth-of-type(odd) {
+ background: $greyish;
+ }
+
+ th, td {
+ border-right: 1px dotted $lightgrey;
+ display: table-cell;
+ font-size: 14px;
+ line-height: 1.3em;
+ padding: 10px;
+ text-align: left;
+
+ &:last-of-type {
+ border-right: 0;
+ }
+
+ code {
+ color: $green;
+ display: inline-block;
+ font-size: 12px;
+ }
+ }
+
+ th {
+ color: #000000;
+ font-weight: bold;
+ font-family: $header-font-family;
+ text-transform: uppercase;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_top-level/support.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_top-level/support.md b/thirdparty/rocksdb/docs/_top-level/support.md
new file mode 100644
index 0000000..6416575
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_top-level/support.md
@@ -0,0 +1,22 @@
+---
+layout: top-level
+title: Support
+id: support
+category: support
+---
+
+## Need help?
+
+Do not hesitate to ask questions if you are having trouble with RocksDB.
+
+### GitHub issues
+
+Use [GitHub issues](https://github.com/facebook/rocksdb/issues) to report bugs, issues and feature requests for the RocksDB codebase.
+
+### Facebook Group
+
+Use the [RocksDB Facebook group](https://www.facebook.com/groups/rocksdb.dev/) for general questions and discussion about RocksDB.
+
+### FAQ
+
+Check out a list of [commonly asked questions](/docs/support/faq) about RocksDB.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/blog/all.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/blog/all.html b/thirdparty/rocksdb/docs/blog/all.html
new file mode 100644
index 0000000..3be2d3b
--- /dev/null
+++ b/thirdparty/rocksdb/docs/blog/all.html
@@ -0,0 +1,20 @@
+---
+id: all
+layout: blog
+category: blog
+---
+
+<div class="posts">
+ <div class="post">
+ <h1>All Posts</h1>
+ {% for post in site.posts %}
+ {% assign author = site.data.authors[post.author] %}
+ <p>
+ <strong>
+ <a href="{{ post.url }}">{{ post.title }}</a>
+ </strong>
+ on {{ post.date | date: "%B %e, %Y" }} by {{ author.display_name }}
+ </p>
+ {% endfor %}
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/blog/index.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/blog/index.html b/thirdparty/rocksdb/docs/blog/index.html
new file mode 100644
index 0000000..9f6b25d
--- /dev/null
+++ b/thirdparty/rocksdb/docs/blog/index.html
@@ -0,0 +1,12 @@
+---
+id: blog
+title: Blog
+layout: blog
+category: blog
+---
+
+<div class="posts">
+ {% for page in site.posts %}
+ {% include post.html truncate=true %}
+ {% endfor %}
+</div>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/css/main.scss
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/css/main.scss b/thirdparty/rocksdb/docs/css/main.scss
new file mode 100644
index 0000000..48a3e14
--- /dev/null
+++ b/thirdparty/rocksdb/docs/css/main.scss
@@ -0,0 +1,149 @@
+---
+# Only the main Sass file needs front matter (the dashes are enough)
+---
+@charset "utf-8";
+
+@font-face {
+ font-family: 'Lato';
+ src: url("{{ '/static/fonts/LatoLatin-Italic.woff2' }}") format('woff2'),
+ url("{{ '/static/fonts/LatoLatin-Italic.woff' }}") format('woff');
+ font-weight: normal;
+ font-style: italic;
+}
+
+@font-face {
+ font-family: 'Lato';
+ src: url("{{ '/static/fonts/LatoLatin-Black.woff2' }}") format('woff2'),
+ url("{{ '/static/fonts/LatoLatin-Black.woff' }}") format('woff');
+ font-weight: 900;
+ font-style: normal;
+}
+
+@font-face {
+ font-family: 'Lato';
+ src: url("{{ '/static/fonts/LatoLatin-BlackItalic.woff2' }}") format('woff2'),
+ url("{{ '/static/fonts/LatoLatin-BlackItalic.woff' }}") format('woff');
+ font-weight: 900;
+ font-style: italic;
+}
+
+@font-face {
+ font-family: 'Lato';
+ src: url("{{ '/static/fonts/LatoLatin-Light.woff2' }}") format('woff2'),
+ url("{{ '/static/fonts/LatoLatin-Light.woff' }}") format('woff');
+ font-weight: 300;
+ font-style: normal;
+}
+
+@font-face {
+ font-family: 'Lato';
+ src: url("{{ '/static/fonts/LatoLatin-Regular.woff2' }}") format('woff2'),
+ url("{{ '/static/fonts/LatoLatin-Regular.woff' }}") format('woff');
+ font-weight: normal;
+ font-style: normal;
+}
+
+// Our variables
+$base-font-family: 'Lato', Calibri, Arial, sans-serif;
+$header-font-family: 'Lato', 'Helvetica Neue', Arial, sans-serif;
+$base-font-size: 18px;
+$small-font-size: $base-font-size * 0.875;
+$base-line-height: 1.4em;
+
+$spacing-unit: 12px;
+
+// Two configured colors (see _config.yml)
+$primary-bg: {{ site.color.primary }};
+$secondary-bg: {{ site.color.secondary }};
+
+// $primary-bg overlays
+{% if site.color.primary-overlay == 'light' %}
+$primary-overlay: darken($primary-bg, 70%);
+$primary-overlay-special: darken($primary-bg, 40%);
+{% else %}
+$primary-overlay: #fff;
+$primary-overlay-special: lighten($primary-bg, 30%);
+{% endif %}
+
+// $secondary-bg overlays
+{% if site.color.secondary-overlay == 'light' %}
+$text: #393939;
+$sidenav: darken($secondary-bg, 20%);
+$sidenav-text: $text;
+$sidenav-overlay: darken($sidenav, 10%);
+$sidenav-active: lighten($sidenav, 10%);
+{% else %}
+$text: #fff;
+$sidenav: lighten($secondary-bg, 20%);
+$sidenav-text: $text;
+$sidenav-overlay: lighten($sidenav, 10%);
+$sidenav-active: darken($sidenav, 10%);
+{% endif %}
+
+$code-bg: #002b36;
+
+$header-height: 34px;
+$header-ptop: 10px;
+$header-pbot: 8px;
+
+// Width of the content area
+$content-width: 900px;
+
+// Table setting variables
+$lightergrey: #F8F8F8;
+$greyish: #E8E8E8;
+$lightgrey: #B0B0B0;
+$green: #2db04b;
+
+// Using media queries with like this:
+// @include media-query($on-palm) {
+// .wrapper {
+// padding-right: $spacing-unit / 2;
+// padding-left: $spacing-unit / 2;
+// }
+// }
+@mixin media-query($device) {
+ @media screen and (max-width: $device) {
+ @content;
+ }
+}
+
+
+
+// Import partials from `sass_dir` (defaults to `_sass`)
+@import
+ "reset",
+ "base",
+ "header",
+ "search",
+ "syntax-highlighting",
+ "promo",
+ "buttons",
+ "gridBlock",
+ "poweredby",
+ "footer",
+ "react_header_nav",
+ "react_docs_nav",
+ "tables",
+ "blog"
+;
+
+// Anchor links
+// http://ben.balter.com/2014/03/13/pages-anchor-links/
+.header-link {
+ position: absolute;
+ margin-left: 0.2em;
+ opacity: 0;
+
+ -webkit-transition: opacity 0.2s ease-in-out 0.1s;
+ -moz-transition: opacity 0.2s ease-in-out 0.1s;
+ -ms-transition: opacity 0.2s ease-in-out 0.1s;
+}
+
+h2:hover .header-link,
+h3:hover .header-link,
+h4:hover .header-link,
+h5:hover .header-link,
+h6:hover .header-link {
+ opacity: 1;
+}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/doc-type-examples/2016-04-07-blog-post-example.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/doc-type-examples/2016-04-07-blog-post-example.md b/thirdparty/rocksdb/docs/doc-type-examples/2016-04-07-blog-post-example.md
new file mode 100644
index 0000000..ef954d6
--- /dev/null
+++ b/thirdparty/rocksdb/docs/doc-type-examples/2016-04-07-blog-post-example.md
@@ -0,0 +1,21 @@
+---
+title: Blog Post Example
+layout: post
+author: exampleauthor
+category: blog
+---
+
+Any local blog posts would go in the `_posts` directory.
+
+This is an example blog post introduction, try to keep it short and about a paragraph long, to encourage people to click through to read the entire post.
+
+<!--truncate-->
+
+Everything below the `<!--truncate-->` tag will only show on the actual blog post page, not on the `/blog/` index.
+
+Author is defined in `_data/authors.yml`
+
+
+## No posts?
+
+If you have no blog for your site, you can remove the entire `_posts` folder. Otherwise add markdown files in here. See CONTRIBUTING.md for details.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/doc-type-examples/docs-hello-world.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/doc-type-examples/docs-hello-world.md b/thirdparty/rocksdb/docs/doc-type-examples/docs-hello-world.md
new file mode 100644
index 0000000..c7094ba
--- /dev/null
+++ b/thirdparty/rocksdb/docs/doc-type-examples/docs-hello-world.md
@@ -0,0 +1,12 @@
+---
+docid: hello-world
+title: Hello, World!
+layout: docs
+permalink: /docs/hello-world.html
+---
+
+Any local docs would go in the `_docs` directory.
+
+## No documentation?
+
+If you have no documentation for your site, you can remove the entire `_docs` folder. Otherwise add markdown files in here. See CONTRIBUTING.md for details.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/doc-type-examples/top-level-example.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/doc-type-examples/top-level-example.md b/thirdparty/rocksdb/docs/doc-type-examples/top-level-example.md
new file mode 100644
index 0000000..67b1fa7
--- /dev/null
+++ b/thirdparty/rocksdb/docs/doc-type-examples/top-level-example.md
@@ -0,0 +1,8 @@
+---
+layout: top-level
+title: Support Example
+id: top-level-example
+category: top-level
+---
+
+This is a static page disconnected from the blog or docs collections that can be added at a top-level (i.e., the same level as `index.md`).
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/docs/index.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/docs/index.html b/thirdparty/rocksdb/docs/docs/index.html
new file mode 100644
index 0000000..fa6ec8b
--- /dev/null
+++ b/thirdparty/rocksdb/docs/docs/index.html
@@ -0,0 +1,6 @@
+---
+id: docs
+title: Docs
+layout: redirect
+destination: getting-started.html
+---
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/feed.xml
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/feed.xml b/thirdparty/rocksdb/docs/feed.xml
new file mode 100644
index 0000000..590cbd9
--- /dev/null
+++ b/thirdparty/rocksdb/docs/feed.xml
@@ -0,0 +1,30 @@
+---
+layout: null
+---
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
+ <channel>
+ <title>{{ site.title | xml_escape }}</title>
+ <description>{{ site.description | xml_escape }}</description>
+ <link>{{ absolute_url }}/</link>
+ <atom:link href="{{ '/feed.xml' | absolute_url }}" rel="self" type="application/rss+xml"/>
+ <pubDate>{{ site.time | date_to_rfc822 }}</pubDate>
+ <lastBuildDate>{{ site.time | date_to_rfc822 }}</lastBuildDate>
+ <generator>Jekyll v{{ jekyll.version }}</generator>
+ {% for post in site.posts limit:10 %}
+ <item>
+ <title>{{ post.title | xml_escape }}</title>
+ <description>{{ post.content | xml_escape }}</description>
+ <pubDate>{{ post.date | date_to_rfc822 }}</pubDate>
+ <link>{{ post.url | absolute_url }}</link>
+ <guid isPermaLink="true">{{ post.url | absolute_url }}</guid>
+ {% for tag in post.tags %}
+ <category>{{ tag | xml_escape }}</category>
+ {% endfor %}
+ {% for cat in post.categories %}
+ <category>{{ cat | xml_escape }}</category>
+ {% endfor %}
+ </item>
+ {% endfor %}
+ </channel>
+</rss>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/index.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/index.md b/thirdparty/rocksdb/docs/index.md
new file mode 100644
index 0000000..2b9570d
--- /dev/null
+++ b/thirdparty/rocksdb/docs/index.md
@@ -0,0 +1,9 @@
+---
+layout: home
+title: RocksDB | A persistent key-value store
+id: home
+---
+
+## Features
+
+{% include content/gridblocks.html data_source=site.data.features align="center" %}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/favicon.png
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/favicon.png b/thirdparty/rocksdb/docs/static/favicon.png
new file mode 100644
index 0000000..7f668f3
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/favicon.png differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Black.woff
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Black.woff b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Black.woff
new file mode 100644
index 0000000..d1e2579
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Black.woff differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Black.woff2
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Black.woff2 b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Black.woff2
new file mode 100644
index 0000000..4127b4d
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Black.woff2 differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff
new file mode 100644
index 0000000..142c1c9
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff2
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff2 b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff2
new file mode 100644
index 0000000..e9862e6
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-BlackItalic.woff2 differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Italic.woff
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Italic.woff b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Italic.woff
new file mode 100644
index 0000000..d8cf84c
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Italic.woff differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Italic.woff2
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Italic.woff2 b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Italic.woff2
new file mode 100644
index 0000000..aaa5a35
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Italic.woff2 differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Light.woff
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Light.woff b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Light.woff
new file mode 100644
index 0000000..e7d4278
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Light.woff differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Light.woff2
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Light.woff2 b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Light.woff2
new file mode 100644
index 0000000..b6d0288
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Light.woff2 differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Regular.woff
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Regular.woff b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Regular.woff
new file mode 100644
index 0000000..bf73a6d
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Regular.woff differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Regular.woff2
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Regular.woff2 b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Regular.woff2
new file mode 100644
index 0000000..a4d084b
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/fonts/LatoLatin-Regular.woff2 differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/Resize-of-20140327_200754-300x225.jpg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/Resize-of-20140327_200754-300x225.jpg b/thirdparty/rocksdb/docs/static/images/Resize-of-20140327_200754-300x225.jpg
new file mode 100644
index 0000000..9f93151
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/Resize-of-20140327_200754-300x225.jpg differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/compaction/full-range.png
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/compaction/full-range.png b/thirdparty/rocksdb/docs/static/images/compaction/full-range.png
new file mode 100644
index 0000000..5b2c9fc
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/compaction/full-range.png differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/compaction/l0-l1-contend.png
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/compaction/l0-l1-contend.png b/thirdparty/rocksdb/docs/static/images/compaction/l0-l1-contend.png
new file mode 100644
index 0000000..bcf8ec7
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/compaction/l0-l1-contend.png differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/compaction/l1-l2-contend.png
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/compaction/l1-l2-contend.png b/thirdparty/rocksdb/docs/static/images/compaction/l1-l2-contend.png
new file mode 100644
index 0000000..6dafbbb
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/compaction/l1-l2-contend.png differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/compaction/part-range-old.png
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/compaction/part-range-old.png b/thirdparty/rocksdb/docs/static/images/compaction/part-range-old.png
new file mode 100644
index 0000000..1cc723d
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/compaction/part-range-old.png differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/pcache-blockindex.jpg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/pcache-blockindex.jpg b/thirdparty/rocksdb/docs/static/images/pcache-blockindex.jpg
new file mode 100644
index 0000000..9c18bde
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/pcache-blockindex.jpg differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/pcache-fileindex.jpg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/pcache-fileindex.jpg b/thirdparty/rocksdb/docs/static/images/pcache-fileindex.jpg
new file mode 100644
index 0000000..51f4e09
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/pcache-fileindex.jpg differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/pcache-filelayout.jpg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/pcache-filelayout.jpg b/thirdparty/rocksdb/docs/static/images/pcache-filelayout.jpg
new file mode 100644
index 0000000..771ee60
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/pcache-filelayout.jpg differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/pcache-readiopath.jpg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/pcache-readiopath.jpg b/thirdparty/rocksdb/docs/static/images/pcache-readiopath.jpg
new file mode 100644
index 0000000..4993f00
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/pcache-readiopath.jpg differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/pcache-tieredstorage.jpg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/pcache-tieredstorage.jpg b/thirdparty/rocksdb/docs/static/images/pcache-tieredstorage.jpg
new file mode 100644
index 0000000..c362a2d
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/pcache-tieredstorage.jpg differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/pcache-writeiopath.jpg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/pcache-writeiopath.jpg b/thirdparty/rocksdb/docs/static/images/pcache-writeiopath.jpg
new file mode 100644
index 0000000..561b551
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/pcache-writeiopath.jpg differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/promo-adapt.svg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/promo-adapt.svg b/thirdparty/rocksdb/docs/static/images/promo-adapt.svg
new file mode 100644
index 0000000..7cd4443
--- /dev/null
+++ b/thirdparty/rocksdb/docs/static/images/promo-adapt.svg
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg id="svg4136" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://www.w3.org/2000/svg" height="256px" width="256px" version="1.1" xmlns:cc="http://creativecommons.org/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" viewBox="0 0 2691.7869 1102.369">
+ <g id="layer1" transform="translate(2299.2 62.602)">
+ <path id="path4167" style="color-rendering:auto;text-decoration-color:#000000;color:#000000;isolation:auto;mix-blend-mode:normal;shape-rendering:auto;solid-color:#000000;block-progression:tb;text-decoration-line:none;text-decoration-style:solid;image-rendering:auto;white-space:normal;text-indent:0;text-transform:none" d="m392.59 123.17c0.0121-1.9688-0.10333-3.9361-0.34718-5.8898-0.15395-1.3644-0.36978-2.7214-0.64725-4.0662-0.12442-0.56729-0.25999-1.1325-0.40601-1.6946-0.17896-0.75016-0.37707-1.4955-0.59429-2.2357-0.34263-1.1286-0.72934-2.2438-1.1592-3.3423-0.0693-0.18679-0.14009-0.37234-0.21179-0.5591-4.65-11.643-13.94-20.829-25.63-25.358l-235.94-136.22c-21.65-12.935-49.703-5.641-62.314 16.201-12.61 21.841-4.902 49.778 17.123 62.066l95.659 55.229-590.22 0.00006c-278.88 0.00003-504.81 225.93-504.81 504.81l-0.68255 171.28 91.222 0.15305-0.15877-171.43c-0.00003-230.38 184.06-414.43 414.43-414.43l590.22 0.00003-95.659 55.229c-22.025 12.288-29.733 40.224-17.123 62.066s40.659 29.136 62.
313 16.205l236.18-136.36c12.515-4.9182 22.19-15.154 26.396-27.926 0.17775-0.55981 0.34438-1.1222 0.5001-1.6886 0.22565-0.75133 0.43169-1.5088 0.61787-2.2712 0.26151-1.1286 0.47932-2.2671 0.65315-3.4128 0.0346-0.20366 0.0681-0.40725 0.10003-0.61218 0.28861-1.8998 0.45557-3.8159 0.50016-5.737zm-1219 630.38-493.8-0.00012c-199.29-0.00006-363.89-137.75-404.69-324.05l283.68-0.00003-95.659 55.228c-22.023 12.288-29.728 40.224-17.119 62.064 12.61 21.841 40.656 29.135 62.309 16.207l235.94-136.22c11.696-4.5293 20.985-13.716 25.642-25.361 0.072-0.18618 0.1416-0.37291 0.2109-0.56005 0.4299-1.0986 0.8167-2.2135 1.1593-3.3423 0.2169-0.74012 0.4154-1.4857 0.5944-2.236 0.1446-0.56217 0.2801-1.1271 0.4061-1.6945 0.2771-1.3448 0.4934-2.7015 0.6471-4.066 0.244-1.9537 0.36-3.9213 0.3474-5.8901-0.045-1.9211-0.2109-3.8374-0.5005-5.7372-0.033-0.20396-0.066-0.40801-0.099-0.61167-0.1747-1.1456-0.3919-2.2841-0.6534-3.413-0.1868-0.76233-0.3923-1.5197-0.6176-2.2713-0.1567-0.56618-0.3227-1.1292-0.5004-1.6889-4.2
06-12.772-13.881-23.008-26.396-27.926l-236.18-136.36c-21.654-12.936-49.706-5.6434-62.318 16.201s-4.9022 49.785 17.128 62.07l95.659 55.229-293.42-0.00006h-90.38l-428.37 0.00009c-25.208-0.35649-45.834 19.98-45.834 45.19s20.626 45.547 45.834 45.19l436.44-0.00015c42.578 235.73 248.71 414.43 496.74 414.43l910.97-0.00009-95.659 55.229c-22.025 12.287-29.727 40.224-17.117 62.066s40.654 29.136 62.307 16.205l236.16-136.23c11.697-4.529 20.985-13.715 25.643-25.361 0.0717-0.18589 0.1425-0.37219 0.21179-0.55904 0.42982-1.0985 0.81653-2.2134 1.1592-3.3422 0.21722-0.74013 0.41533-1.4857 0.59431-2.236 0.14612-0.56229 0.28169-1.1272 0.40602-1.6946 0.27747-1.3448 0.49336-2.7015 0.64725-4.066 0.24373-1.9537 0.35986-3.9212 0.34715-5.89-0.0446-1.921-0.21149-3.8373-0.50013-5.737-0.0319-0.20396-0.0654-0.40828-0.10003-0.61194-0.17383-1.1456-0.39174-2.284-0.65315-3.4128-0.18618-0.76233-0.39219-1.5197-0.61781-2.2713-0.15575-0.56609-0.32248-1.1291-0.50016-1.6887-4.2062-12.772-13.881-23.008-26.396-27.926l-236.1
8-136.36c-21.654-12.931-49.697-5.6367-62.307 16.205s-4.908 49.779 17.117 62.066l95.659 55.229h-417.17z" fill-rule="evenodd"/>
+ <path id="path6042" d="m-1139.3 565.22v-376.9" stroke="#000" stroke-linecap="round" stroke-width="90.381" fill="none"/>
+ <path id="path6044" d="m-111.15 993.93v-376.9" stroke="#000" stroke-linecap="round" stroke-width="90.381" fill="none"/>
+ </g>
+</svg>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/promo-flash.svg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/promo-flash.svg b/thirdparty/rocksdb/docs/static/images/promo-flash.svg
new file mode 100644
index 0000000..79810c3
--- /dev/null
+++ b/thirdparty/rocksdb/docs/static/images/promo-flash.svg
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 15.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd" [
+ <!ENTITY ns_extend "http://ns.adobe.com/Extensibility/1.0/">
+ <!ENTITY ns_ai "http://ns.adobe.com/AdobeIllustrator/10.0/">
+ <!ENTITY ns_graphs "http://ns.adobe.com/Graphs/1.0/">
+ <!ENTITY ns_vars "http://ns.adobe.com/Variables/1.0/">
+ <!ENTITY ns_imrep "http://ns.adobe.com/ImageReplacement/1.0/">
+ <!ENTITY ns_sfw "http://ns.adobe.com/SaveForWeb/1.0/">
+ <!ENTITY ns_custom "http://ns.adobe.com/GenericCustomNamespace/1.0/">
+ <!ENTITY ns_adobe_xpath "http://ns.adobe.com/XPath/1.0/">
+]>
+<svg version="1.1" id="Ebene_1" xmlns:x="&ns_extend;" xmlns:i="&ns_ai;" xmlns:graph="&ns_graphs;"
+ xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" width="256px"
+ height="256px" viewBox="0 0 815.261 584.491" enable-background="new 0 0 815.261 584.491" xml:space="preserve">
+<switch>
+ <foreignObject requiredExtensions="&ns_ai;" x="0" y="0" width="1" height="1">
+ <i:pgfRef xlink:href="#adobe_illustrator_pgf">
+ </i:pgfRef>
+ </foreignObject>
+ <g i:extraneous="self">
+ <path fill-rule="evenodd" clip-rule="evenodd" d="M571.106,229.357l86.136,62.324l-500.458,292.81l15.751-116.254L0,392.661
+ l346.5-136.874l-79.521-80.771l332.31-123.92L538.864,0h276.397l-73.986,207.161l-59.189-36.993L571.106,229.357z M630.295,51.79
+ L312.155,170.168l96.182,81.385L60.602,384.729l133.175,51.79l-14.797,96.183l421.722-251.554l-81.385-66.588l170.168-96.182
+ l44.392,22.195l66.587-125.776H593.302L630.295,51.79z"/>
+ </g>
+</switch>
+</svg>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/promo-operations.svg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/promo-operations.svg b/thirdparty/rocksdb/docs/static/images/promo-operations.svg
new file mode 100644
index 0000000..3036294
--- /dev/null
+++ b/thirdparty/rocksdb/docs/static/images/promo-operations.svg
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg height="256px" width="256px" viewBox="0 0 154.875 154.91782" xmlns="http://www.w3.org/2000/svg">
+<g transform="translate(-263.3125,-563.76968)">
+<path d="m336.77535,565.51158c-1.59726.10936-3.15002.35671-4.71936.53629h-.1063l-3.75379,20.48605c-6.11896,1.39345-11.87541,3.75824-17.05404,6.97168l-16.83926-12.12002c-4.55215,3.53414-8.69458,7.65592-12.3345,12.12002l11.69091,17.05386c-3.54965,5.42465-6.21914,11.6188-7.72253,18.12639-.00018.031-.00018.10198 0,.10702l-20.37883,3.2177c-.3725,3.04312-.53624,6.18809-.53624,9.33134 0,2.57176.071,5.10917.32165,7.61524l20.37883,3.64673c1.44933,7.07687 4.20261,13.68602 8.04414,19.52075l-12.11991,16.6248c3.4711,4.30922 7.47839,8.23258 11.79812,11.69099l17.16106-11.79826c5.9977,3.82597 12.69269,6.50875 19.94983,7.82975l3.21758,20.27155c2.28662.20798 4.63161.2145 6.97192.2145 3.30389,0 6.46004-.12522 9.65312-.53628l3.86135-20.70056c6.89012-1.71472 13.36295-4.68941 18.98427-8.68781l16.51747,12.01276c4.28351-3.64433 8.20054-7.83321 11.5837-12.33452l-12.0127-17.37561c3.25344-5.61849 5.50726-11.8176 6.64976-18.44817l20.2718-3.21771c.17838-2.11543.21297-4.16701.21297-6.32815 0-3.75557-.43675-7.437
87-.96556-11.04745l-20.59342-3.75397c-1.61384-5.95909-4.26171-11.51888-7.61497-16.51756l12.11974-16.6248c-3.75686-4.59442-8.04235-8.83858-12.76333-12.4418l-17.48303,12.01278c-5.02475-2.97177-10.43184-5.25192-16.30306-6.54268l-3.21759-20.37879c-2.92858-.34452-5.88149-.53628-8.90214-.53628-.81656,0-1.65672-.024-2.46715,0-.39495.0126-.78593-.024-1.17962,0-.1063.007-.21621-.007-.32269,0zm2.78876,52.1268c.39207-.0213.78323,0 1.17998,0 12.69611,0 23.06003,10.36401 23.06003,23.06023s-10.36392,22.95297-23.06003,22.95297-22.95301-10.25675-22.95301-22.95297c0-12.29946 9.6261-22.44383 21.77303-23.06023z" fill="#333" stroke="#333" stroke-width="3.43222"/>
+</g>
+</svg>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/promo-performance.svg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/promo-performance.svg b/thirdparty/rocksdb/docs/static/images/promo-performance.svg
new file mode 100644
index 0000000..be8a101
--- /dev/null
+++ b/thirdparty/rocksdb/docs/static/images/promo-performance.svg
@@ -0,0 +1,134 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg viewBox="0 0 64 64" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<defs>
+<linearGradient id="linearGradient4121">
+<stop offset="0" stop-color="#fff"/>
+<stop offset="1" stop-color="#fff" stop-opacity="0"/>
+</linearGradient>
+<linearGradient gradientTransform="matrix(.30081,0,0,.20986,-1650.25,-38.87783)" gradientUnits="userSpaceOnUse" id="linearGradient4107" x1="-342.85715" x2="-376.17944" xlink:href="#linearGradient4121" y1="-397.01691" y2="-915.50836"/>
+<linearGradient gradientUnits="userSpaceOnUse" id="linearGradient15735" x1="797.06111" x2="788.0298" xlink:href="#linearGradient4121" y1="-667.70464" y2="-819.35937"/>
+</defs>
+<title>netalloy chequered flag</title>
+<g transform="matrix(.08343,-.03199,0,.08343,-73.61005,-8.84057)">
+<path d="m1688.925,115.67413h30.08058v20.9864h-30.08058z" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m1719.6503,115.67413h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m1658.8445,136.51062h30.08058v20.9864h-30.08058z" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m1689.5697,136.51062h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m136.5856-1749.1506h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m158.0217-1749.1506h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m156.97238-1718.8552h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m178.40849-1718.8552h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m1718.5759,177.7339h30.08058v20.9864h-30.08058z" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m1719.2205,198.5704h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m219.03217-1748.5061h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m1748.2268,157.64693h30.08058v20.9864h-30.08058z" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m1748.8713,178.48341h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m198.94519-1778.157h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m220.3813-1778.157h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m1777.8777,137.26013h30.08058v20.9864h-30.08058z" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m1778.5222,158.09662h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m177.95879-1808.6672h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m199.99451-1807.8077h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m1808.8176,157.34711h30.08058v20.9864h-30.08058z" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m1809.4622,178.18361h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m199.24498-1838.7478h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m220.08148-1838.7478h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m1657.9851,178.93314h30.08058v20.9864h-30.08058z" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m1658.6296,199.76964h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m220.2314-1687.9152h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m1688.0657,199.31993h30.08058v20.9864h-30.08058z" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m1688.7102,220.15643h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m220.00832-1868.8394h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m221.05762-1837.2548h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m-1807.9261-241.48938h30.08058v20.9864h-30.08058z" transform="matrix(-.87909,-.47665,.59562,-.80326,0,0)"/>
+<path d="m-1777.201-241.48938h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(-.87909,-.47665,.59562,-.80326,0,0)"/>
+<path d="m115.67591-1837.6415h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m116.12562-1807.7758h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m-1778.4473-136.55739h30.08058v20.9864h-30.08058z" fill="url(#linearGradient4107)" transform="matrix(-.87909,-.47665,.59562,-.80326,0,0)"/>
+<path d="m-1776.9432-156.36673h27.94272v19.09188h-27.94272z" fill="#fff" opacity=".60096" transform="matrix(-.87909,-.47665,.59562,-.80326,0,0)"/>
+<path d="m-1869.0726-199.46429h30.08058v20.9864h-30.08058z" transform="matrix(-.87909,-.47665,.59562,-.80326,0,0)"/>
+<path d="m-1868.4281-178.62779h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(-.87909,-.47665,.59562,-.80326,0,0)"/>
+<path d="m-158.16603,1839.1426h20.9864v30.08058h-20.9864z" transform="matrix(.59562,-.80326,.87909,.47665,0,0)"/>
+<path d="m-136.72992,1838.5409h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(.59562,-.80326,.87909,.47665,0,0)"/>
+<path d="m116.27553-1899.0918h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m-1899.7549-220.4507h30.08058v20.9864h-30.08058z" transform="matrix(-.87909,-.47665,.59562,-.80326,0,0)"/>
+<path d="m-1899.1104-199.6142h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(-.87909,-.47665,.59562,-.80326,0,0)"/>
+<path d="m-179.15244,1869.8247h20.9864v30.08058h-20.9864z" transform="matrix(.59562,-.80326,.87909,.47665,0,0)"/>
+<path d="m-157.71632,1869.2231h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(.59562,-.80326,.87909,.47665,0,0)"/>
+<path d="m239.53888-1718.5974h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m1719.3923,239.46391h30.08058v20.9864h-30.08058z" fill="#fff" opacity=".60096" transform="matrix(.87909,.47665,-.59562,.80326,0,0)"/>
+<path d="m239.91541-1897.8026h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m240.36511-1867.9369h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m-1838.6084-260.79688h30.08058v20.9864h-30.08058z" transform="matrix(-.87909,-.47665,.59562,-.80326,0,0)"/>
+<path d="m240.21521-1778.3398h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m115.63581-1776.621h20.9864v30.08058h-20.9864z" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m-1837.9637-155.30208h27.82166v17.65242h-27.82166z" fill="#fff" opacity=".60096" transform="matrix(-.87909,-.47665,.59562,-.80326,0,0)"/>
+<path d="m1389.2368,874.53916-207.8251,263.56844" fill="none" stroke="#000" stroke-linecap="round" stroke-width="14.84459"/>
+<path d="m199.22183-1868.7964h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m156.79575-1690.0319h21.6532v29.62879h-21.6532z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m221.44484-1899.1632h18.00162v29.54019h-18.00162z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m241.58183-1687.8284h18.77428v29.87089h-18.77428z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m240.00063-1806.0569h20.9864v30.08058h-20.9864z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m114.93913-1688.511h21.01802v24.93378h-21.01802z" fill="#fff" opacity=".60096" transform="matrix(-.59562,.80326,-.87909,-.47665,0,0)"/>
+<path d="m-855.13867,658.61304h27.1029v29.24923h-27.1029z" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m-827.45502,658.61304h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m-882.24158,687.65332h27.1029v29.24923h-27.1029z" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m-854.55792,687.65332h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m687.75781,800.87488h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m717.63379,800.87488h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m716.17133,828.17133h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m746.0473,828.17133h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m-828.42303,745.10712h27.1029v29.24923h-27.1029z" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m-827.84222,774.14746h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m802.66547,801.45563h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m-801.70728,717.11145h27.1029v29.24923h-27.1029z" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m-801.12653,746.15173h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m774.66974,774.73993h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m804.54578,774.73993h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m-774.99152,688.69794h27.1029v29.24923h-27.1029z" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m-774.41077,717.73822h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m745.42053,747.24982h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m776.1322,748.02417h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m-747.11432,716.6936h27.1029v29.24923h-27.1029z" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m-746.53357,745.73389h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m775.08759,720.14691h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m804.12793,720.14691h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m-883.01599,746.7785h27.1029v29.24923h-27.1029z" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m-882.43518,775.81885h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m804.33685,856.04858h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m-855.91309,775.19208h27.1029v29.24923h-27.1029z" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m-855.33228,804.23236h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m804.02594,693.03412h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m805.48834,721.49219h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m747.91748-833.9646h27.1029v29.24923h-27.1029z" transform="matrix(.97568,.21922,-.42736,-.90408,0,0)"/>
+<path d="m775.6012-833.9646h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(.97568,.21922,-.42736,-.90408,0,0)"/>
+<path d="m658.61548,721.14368h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m659.24225,748.05304h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m774.47833-687.71844h27.1029v29.24923h-27.1029z" fill="url(#linearGradient15735)" transform="matrix(.97568,.21922,-.42736,-.90408,0,0)"/>
+<path d="m775.83344-715.32721h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(.97568,.21922,-.42736,-.90408,0,0)"/>
+<path d="m692.82385-775.39325h27.1029v29.24923h-27.1029z" transform="matrix(.97568,.21922,-.42736,-.90408,0,0)"/>
+<path d="m693.40466-746.35297h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(.97568,.21922,-.42736,-.90408,0,0)"/>
+<path d="m-717.83496-719.79126h29.24923v27.1029h-29.24923z" transform="matrix(-.42736,-.90408,-.97568,-.21922,0,0)"/>
+<path d="m-687.95892-720.33331h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(-.42736,-.90408,-.97568,-.21922,0,0)"/>
+<path d="m659.45117,665.77631h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m665.17896-804.64252h27.1029v29.24923h-27.1029z" transform="matrix(.97568,.21922,-.42736,-.90408,0,0)"/>
+<path d="m665.7597-775.60217h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(.97568,.21922,-.42736,-.90408,0,0)"/>
+<path d="m-747.08417-692.1463h29.24923v27.1029h-29.24923z" transform="matrix(-.42736,-.90408,-.97568,-.21922,0,0)"/>
+<path d="m-717.20813-692.68835h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(-.42736,-.90408,-.97568,-.21922,0,0)"/>
+<path d="m831.24615,828.40363h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m-827.68732,831.14166h27.1029v29.24923h-27.1029z" fill="#fff" opacity=".60096" transform="matrix(-.97568,-.21922,.42736,.90408,0,0)"/>
+<path d="m831.77087,666.93793h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m832.39764,693.84723h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m720.27252-860.8739h27.1029v29.24923h-27.1029z" transform="matrix(.97568,.21922,-.42736,-.90408,0,0)"/>
+<path d="m832.18872,774.57513h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m658.55963,776.12384h29.24923v27.1029h-29.24923z" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m720.85333-713.84338h27.18274v24.15996h-27.18274z" fill="#fff" opacity=".60096" transform="matrix(.97568,.21922,-.42736,-.90408,0,0)"/>
+<path d="m1142.0758,779.44917 192.2989,413.58843" fill="none" stroke="#000" stroke-linecap="round" stroke-width="14.20834"/>
+<path d="m775.05536,693.07288h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m715.92511,854.14148h29.44084v23.17059h-29.44084z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m805.88055,665.00702h23.23064v27.43012h-23.23064z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m833.31165,857.23889h28.21643v22.16755h-28.21643z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m833.60077,748.08258h29.24923v27.1029h-29.24923z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+<path d="m657.4411,854.80676h28.65906v24.2827h-28.65906z" fill="#fff" opacity=".60096" transform="matrix(.42736,.90408,.97568,.21922,0,0)"/>
+</g>
+</svg>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/images/tree_example1.png
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/images/tree_example1.png b/thirdparty/rocksdb/docs/static/images/tree_example1.png
new file mode 100644
index 0000000..9f72586
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/images/tree_example1.png differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/logo.svg
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/logo.svg b/thirdparty/rocksdb/docs/static/logo.svg
new file mode 100644
index 0000000..e6e1e8a
--- /dev/null
+++ b/thirdparty/rocksdb/docs/static/logo.svg
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 18.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+ viewBox="0 0 841.9 595.3" enable-background="new 0 0 841.9 595.3" xml:space="preserve">
+<g id="Layer_2" display="none">
+ <g display="inline">
+ <path fill="#0F2647" d="M213,86.8h415.9c17,0,31,13.9,31,31v359.7c0,17-14,31-31,31H213c-17.1,0-31-14-31-31V117.8
+ C182,100.7,195.9,86.8,213,86.8z"/>
+ </g>
+</g>
+<g id="Layer_4">
+ <g>
+ <path fill="#FFBE00" d="M501.9,398l-64.7,1.6l-24.4,56l-63.6,16.6l-39.8,36.3h320.4c16.6,0,30.1-13.5,30.1-30.1V319.2l-113.6,28.4
+ L501.9,398z M268.5,402.7L214.6,449c-0.5-3.3-1.8-6.6-4.1-9.4l-28.3,23.8c6,7.1,16.1,8.5,23.8,3.8l52.7-23.9L268.5,402.7z
+ M581.4,240.2l6.5,17.2c10.1-2.6,25.6-13.7,36.5-22.3c-0.4,3-0.2,6,0.9,9l34.7-12.8c-3.6-9.6-14.2-14.4-23.8-10.9
+ c-0.3,0.1-0.6,0.3-0.9,0.5l0.1-0.1L581.4,240.2z M641.2,269.2c-0.1,0-0.1,0-0.2,0l-63.6-5.5l-14.5-38.1v-40.4
+ c2.9,1.6,6.1,2.5,9.6,2.5c10.5,0,19.1-8.1,20.1-18.3l15.2-10.1l-0.7-20.5c-0.1-2.1-1.7-3.7-3.8-3.7l-19-0.3c-4-5.6-11.4-8-17.9-5
+ l-41.8,19.6c0.1,0.1,0.1,0.3,0.1,0.4c1.4,2.9,3.5,5.1,6,6.7c-50.9,26.3-72.5,61.8-72.5,61.8L263.6,323.1
+ c-11.4,6.1-25.7,1.8-31.8-9.6c-6.1-11.4-1.8-25.7,9.6-31.8l55.2-29.7c14-7.5,22.2-21.5,23.2-36.2l-33.1,17.8l-55.2,29.7
+ c-21.6,11.6-29.7,38.6-18.1,60.1c8,14.9,23.3,23.4,39.2,23.4c7.1,0,14.3-1.7,20.9-5.3l24.9-13.4c-1.8,9.8-1,20.3,2.9,30.2
+ c1.3,3.4,2.2,5.5,2.2,5.5c8.8,19.1-2.8,34.1-18,34.1h-4.8l-17.5,76.2c-2.3-2.4-5.3-4.2-8.8-5.1l-8.7,35.9
+ c8.8,2.1,17.5-2.4,21.1-10.4l39.7-71.2c50.2-4.9,76.6-38.9,75.4-86.8c52,3.2,121.3-29.5,152.1-74.2c23.4,29.7,67.6,25.5,87.3,18
+ l3.8-1.3c-1.4,2.6-2.2,5.6-2.2,8.8l37-0.4C659.7,277.3,651.4,269.1,641.2,269.2z M296.3,201.8c1.8,3.4,2.7,7,2.8,10.6l19.5-10.5
+ c-0.9-3.4-2.1-6.8-3.9-10l-1-1.9l-18.4,9.9L296.3,201.8z M289.2,188.6l18.4-9.9l-2.4-4.4c-2.7-5.1-9.1-7-14.2-4.3
+ c-5.1,2.7-7,9.1-4.3,14.2L289.2,188.6z"/>
+ <path fill="#0F2647" d="M571.4,139.9c-3.3,0-5.9,2.7-5.9,6c0,3.3,2.7,6,5.9,6c3.3,0,6-2.7,6-6
+ C577.4,142.6,574.7,139.9,571.4,139.9z M536.6,184.6c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9
+ s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3s-1.5-1.4-3.7-2.3c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9
+ s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3S534.4,183.8,536.6,184.6z M516.4,191.9c0,0-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9
+ s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9
+ s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3s-1.5-1.4-3.7-2.3C515.5,192.9,516.4,191.9,516.4,191.9z M543,220.2
+ c0,0-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3s-1.5-1.4-3.7-2.3c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9
+ s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9
+ C540.4,223.1,543,220.2,543,220.2z M512.7,230.3c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9
+ s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9
+ s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3S515,231.2,512.7,230.3z M484.1,221.6c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9
+ c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9
+ c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3S486.3,222.5,484.1,221.6z M473,255.6c0,0-0.9-1.1-2.3-1.9
+ c2.2-0.9,3.7-2.3,3.7-2.3s-1.5-1.4-3.7-2.3c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9
+ c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9C470.5,258.5,473,255.6,473,255.6z
+ M423.9,260.6c0,0,2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3s-1.5-1.4-3.7-2.3
+ c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3
+ C424.8,259.6,423.9,260.6,423.9,260.6z M450.9,277.8c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9
+ s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9
+ s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3S453.1,278.6,450.9,277.8z M480.5,284.3c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9
+ c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9
+ c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3S482.7,285.2,480.5,284.3z M504.7,260.9c1.4-0.8,2.3-1.9,2.3-1.9
+ s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9
+ s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3S506.9,261.7,504.7,260.9z M548.1,241.4
+ c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3
+ c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3
+ S550.3,242.2,548.1,241.4z M405.7,278.2c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9
+ c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9
+ c2.2-0.9,3.7-2.3,3.7-2.3S407.9,279.1,405.7,278.2z M434.6,306.7c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9
+ c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9
+ c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3S436.8,307.5,434.6,306.7z M397.8,311.9c1.4-0.8,2.3-1.9,2.3-1.9
+ s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9
+ s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3S400.1,312.7,397.8,311.9z M367.8,290.7
+ c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3
+ c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3
+ S370,291.5,367.8,290.7z M325.2,318.1c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9
+ c2.2-0.9,3.7-2.3,3.7-2.3s-1.5-1.4-3.7-2.3c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9
+ c-2.2,0.9-3.7,2.3-3.7,2.3S323,317.2,325.2,318.1z M363.5,328.3c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9
+ s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9
+ s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3S365.7,329.1,363.5,328.3z M357.2,359.5c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9
+ c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9
+ c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3S359.4,360.4,357.2,359.5z M328,355c0,0-0.9-1.1-2.3-1.9
+ c2.2-0.9,3.7-2.3,3.7-2.3s-1.5-1.4-3.7-2.3c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9
+ c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9C325.4,358,328,355,328,355z M335.6,384.5
+ c1.4-0.8,2.3-1.9,2.3-1.9s-2.5-2.9-5.6-2.9c-3.1,0-5.6,2.9-5.6,2.9s0.9,1.1,2.3,1.9c-2.2,0.9-3.7,2.3-3.7,2.3s1.5,1.4,3.7,2.3
+ c-1.4,0.8-2.3,1.9-2.3,1.9s2.5,2.9,5.6,2.9c3.1,0,5.6-2.9,5.6-2.9s-0.9-1.1-2.3-1.9c2.2-0.9,3.7-2.3,3.7-2.3
+ S337.8,385.3,335.6,384.5z"/>
+ </g>
+</g>
+</svg>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/static/og_image.png
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/static/og_image.png b/thirdparty/rocksdb/docs/static/og_image.png
new file mode 100644
index 0000000..4e2759e
Binary files /dev/null and b/thirdparty/rocksdb/docs/static/og_image.png differ
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/env/env.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/env/env.cc b/thirdparty/rocksdb/env/env.cc
new file mode 100644
index 0000000..ae0b111
--- /dev/null
+++ b/thirdparty/rocksdb/env/env.cc
@@ -0,0 +1,381 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "rocksdb/env.h"
+
+#include <thread>
+#include "options/db_options.h"
+#include "port/port.h"
+#include "port/sys_time.h"
+#include "rocksdb/options.h"
+#include "util/arena.h"
+#include "util/autovector.h"
+
+namespace rocksdb {
+
+Env::~Env() {
+}
+
+uint64_t Env::GetThreadID() const {
+ std::hash<std::thread::id> hasher;
+ return hasher(std::this_thread::get_id());
+}
+
+Status Env::ReuseWritableFile(const std::string& fname,
+ const std::string& old_fname,
+ unique_ptr<WritableFile>* result,
+ const EnvOptions& options) {
+ Status s = RenameFile(old_fname, fname);
+ if (!s.ok()) {
+ return s;
+ }
+ return NewWritableFile(fname, result, options);
+}
+
+Status Env::GetChildrenFileAttributes(const std::string& dir,
+ std::vector<FileAttributes>* result) {
+ assert(result != nullptr);
+ std::vector<std::string> child_fnames;
+ Status s = GetChildren(dir, &child_fnames);
+ if (!s.ok()) {
+ return s;
+ }
+ result->resize(child_fnames.size());
+ size_t result_size = 0;
+ for (size_t i = 0; i < child_fnames.size(); ++i) {
+ const std::string path = dir + "/" + child_fnames[i];
+ if (!(s = GetFileSize(path, &(*result)[result_size].size_bytes)).ok()) {
+ if (FileExists(path).IsNotFound()) {
+ // The file may have been deleted since we listed the directory
+ continue;
+ }
+ return s;
+ }
+ (*result)[result_size].name = std::move(child_fnames[i]);
+ result_size++;
+ }
+ result->resize(result_size);
+ return Status::OK();
+}
+
+SequentialFile::~SequentialFile() {
+}
+
+RandomAccessFile::~RandomAccessFile() {
+}
+
+WritableFile::~WritableFile() {
+}
+
+Logger::~Logger() {
+}
+
+FileLock::~FileLock() {
+}
+
+void LogFlush(Logger *info_log) {
+ if (info_log) {
+ info_log->Flush();
+ }
+}
+
+void Log(Logger* info_log, const char* format, ...) {
+ if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::INFO_LEVEL) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+
+void Logger::Logv(const InfoLogLevel log_level, const char* format, va_list ap) {
+ static const char* kInfoLogLevelNames[5] = { "DEBUG", "INFO", "WARN",
+ "ERROR", "FATAL" };
+ if (log_level < log_level_) {
+ return;
+ }
+
+ if (log_level == InfoLogLevel::INFO_LEVEL) {
+ // Doesn't print log level if it is INFO level.
+ // This is to avoid unexpected performance regression after we add
+ // the feature of log level. All the logs before we add the feature
+ // are INFO level. We don't want to add extra costs to those existing
+ // logging.
+ Logv(format, ap);
+ } else {
+ char new_format[500];
+ snprintf(new_format, sizeof(new_format) - 1, "[%s] %s",
+ kInfoLogLevelNames[log_level], format);
+ Logv(new_format, ap);
+ }
+}
+
+
+void Log(const InfoLogLevel log_level, Logger* info_log, const char* format,
+ ...) {
+ if (info_log && info_log->GetInfoLogLevel() <= log_level) {
+ va_list ap;
+ va_start(ap, format);
+
+ if (log_level == InfoLogLevel::HEADER_LEVEL) {
+ info_log->LogHeader(format, ap);
+ } else {
+ info_log->Logv(log_level, format, ap);
+ }
+
+ va_end(ap);
+ }
+}
+
+void Header(Logger* info_log, const char* format, ...) {
+ if (info_log) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->LogHeader(format, ap);
+ va_end(ap);
+ }
+}
+
+void Debug(Logger* info_log, const char* format, ...) {
+ if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::DEBUG_LEVEL) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::DEBUG_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+
+void Info(Logger* info_log, const char* format, ...) {
+ if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::INFO_LEVEL) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+
+void Warn(Logger* info_log, const char* format, ...) {
+ if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::WARN_LEVEL) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::WARN_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+void Error(Logger* info_log, const char* format, ...) {
+ if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::ERROR_LEVEL) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::ERROR_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+void Fatal(Logger* info_log, const char* format, ...) {
+ if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::FATAL_LEVEL) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::FATAL_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+
+void LogFlush(const shared_ptr<Logger>& info_log) {
+ if (info_log) {
+ info_log->Flush();
+ }
+}
+
+void Log(const InfoLogLevel log_level, const shared_ptr<Logger>& info_log,
+ const char* format, ...) {
+ if (info_log) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(log_level, format, ap);
+ va_end(ap);
+ }
+}
+
+void Header(const shared_ptr<Logger>& info_log, const char* format, ...) {
+ if (info_log) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->LogHeader(format, ap);
+ va_end(ap);
+ }
+}
+
+void Debug(const shared_ptr<Logger>& info_log, const char* format, ...) {
+ if (info_log) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::DEBUG_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+
+void Info(const shared_ptr<Logger>& info_log, const char* format, ...) {
+ if (info_log) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+
+void Warn(const shared_ptr<Logger>& info_log, const char* format, ...) {
+ if (info_log) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::WARN_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+
+void Error(const shared_ptr<Logger>& info_log, const char* format, ...) {
+ if (info_log) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::ERROR_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+
+void Fatal(const shared_ptr<Logger>& info_log, const char* format, ...) {
+ if (info_log) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::FATAL_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+
+void Log(const shared_ptr<Logger>& info_log, const char* format, ...) {
+ if (info_log) {
+ va_list ap;
+ va_start(ap, format);
+ info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap);
+ va_end(ap);
+ }
+}
+
+Status WriteStringToFile(Env* env, const Slice& data, const std::string& fname,
+ bool should_sync) {
+ unique_ptr<WritableFile> file;
+ EnvOptions soptions;
+ Status s = env->NewWritableFile(fname, &file, soptions);
+ if (!s.ok()) {
+ return s;
+ }
+ s = file->Append(data);
+ if (s.ok() && should_sync) {
+ s = file->Sync();
+ }
+ if (!s.ok()) {
+ env->DeleteFile(fname);
+ }
+ return s;
+}
+
+Status ReadFileToString(Env* env, const std::string& fname, std::string* data) {
+ EnvOptions soptions;
+ data->clear();
+ unique_ptr<SequentialFile> file;
+ Status s = env->NewSequentialFile(fname, &file, soptions);
+ if (!s.ok()) {
+ return s;
+ }
+ static const int kBufferSize = 8192;
+ char* space = new char[kBufferSize];
+ while (true) {
+ Slice fragment;
+ s = file->Read(kBufferSize, &fragment, space);
+ if (!s.ok()) {
+ break;
+ }
+ data->append(fragment.data(), fragment.size());
+ if (fragment.empty()) {
+ break;
+ }
+ }
+ delete[] space;
+ return s;
+}
+
+EnvWrapper::~EnvWrapper() {
+}
+
+namespace { // anonymous namespace
+
+void AssignEnvOptions(EnvOptions* env_options, const DBOptions& options) {
+ env_options->use_mmap_reads = options.allow_mmap_reads;
+ env_options->use_mmap_writes = options.allow_mmap_writes;
+ env_options->use_direct_reads = options.use_direct_reads;
+ env_options->set_fd_cloexec = options.is_fd_close_on_exec;
+ env_options->bytes_per_sync = options.bytes_per_sync;
+ env_options->compaction_readahead_size = options.compaction_readahead_size;
+ env_options->random_access_max_buffer_size =
+ options.random_access_max_buffer_size;
+ env_options->rate_limiter = options.rate_limiter.get();
+ env_options->writable_file_max_buffer_size =
+ options.writable_file_max_buffer_size;
+ env_options->allow_fallocate = options.allow_fallocate;
+}
+
+}
+
+EnvOptions Env::OptimizeForLogWrite(const EnvOptions& env_options,
+ const DBOptions& db_options) const {
+ EnvOptions optimized_env_options(env_options);
+ optimized_env_options.bytes_per_sync = db_options.wal_bytes_per_sync;
+ return optimized_env_options;
+}
+
+EnvOptions Env::OptimizeForManifestWrite(const EnvOptions& env_options) const {
+ return env_options;
+}
+
+EnvOptions Env::OptimizeForLogRead(const EnvOptions& env_options) const {
+ EnvOptions optimized_env_options(env_options);
+ optimized_env_options.use_direct_reads = false;
+ return optimized_env_options;
+}
+
+EnvOptions Env::OptimizeForManifestRead(const EnvOptions& env_options) const {
+ EnvOptions optimized_env_options(env_options);
+ optimized_env_options.use_direct_reads = false;
+ return optimized_env_options;
+}
+
+EnvOptions Env::OptimizeForCompactionTableWrite(
+ const EnvOptions& env_options, const ImmutableDBOptions& db_options) const {
+ EnvOptions optimized_env_options(env_options);
+ optimized_env_options.use_direct_writes =
+ db_options.use_direct_io_for_flush_and_compaction;
+ return optimized_env_options;
+}
+
+EnvOptions Env::OptimizeForCompactionTableRead(
+ const EnvOptions& env_options, const ImmutableDBOptions& db_options) const {
+ EnvOptions optimized_env_options(env_options);
+ optimized_env_options.use_direct_reads =
+ db_options.use_direct_io_for_flush_and_compaction;
+ return optimized_env_options;
+}
+
+EnvOptions::EnvOptions(const DBOptions& options) {
+ AssignEnvOptions(this, options);
+}
+
+EnvOptions::EnvOptions() {
+ DBOptions options;
+ AssignEnvOptions(this, options);
+}
+
+
+} // namespace rocksdb
[32/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/gnu_parallel
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/gnu_parallel b/thirdparty/rocksdb/build_tools/gnu_parallel
new file mode 100755
index 0000000..abbf8f1
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/gnu_parallel
@@ -0,0 +1,7936 @@
+#!/usr/bin/env perl
+
+# Copyright (C) 2007,2008,2009,2010,2011,2012,2013,2014 Ole Tange and
+# Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+# or write to the Free Software Foundation, Inc., 51 Franklin St,
+# Fifth Floor, Boston, MA 02110-1301 USA
+
+# open3 used in Job::start
+use IPC::Open3;
+# &WNOHANG used in reaper
+use POSIX qw(:sys_wait_h setsid ceil :errno_h);
+# gensym used in Job::start
+use Symbol qw(gensym);
+# tempfile used in Job::start
+use File::Temp qw(tempfile tempdir);
+# mkpath used in openresultsfile
+use File::Path;
+# GetOptions used in get_options_from_array
+use Getopt::Long;
+# Used to ensure code quality
+use strict;
+use File::Basename;
+
+if(not $ENV{HOME}) {
+ # $ENV{HOME} is sometimes not set if called from PHP
+ ::warning("\$HOME not set. Using /tmp\n");
+ $ENV{HOME} = "/tmp";
+}
+
+save_stdin_stdout_stderr();
+save_original_signal_handler();
+parse_options();
+::debug("init", "Open file descriptors: ", join(" ",keys %Global::fd), "\n");
+my $number_of_args;
+if($Global::max_number_of_args) {
+ $number_of_args=$Global::max_number_of_args;
+} elsif ($opt::X or $opt::m or $opt::xargs) {
+ $number_of_args = undef;
+} else {
+ $number_of_args = 1;
+}
+
+my @command;
+@command = @ARGV;
+
+my @fhlist;
+if($opt::pipepart) {
+ @fhlist = map { open_or_exit($_) } "/dev/null";
+} else {
+ @fhlist = map { open_or_exit($_) } @opt::a;
+ if(not @fhlist and not $opt::pipe) {
+ @fhlist = (*STDIN);
+ }
+}
+
+if($opt::skip_first_line) {
+ # Skip the first line for the first file handle
+ my $fh = $fhlist[0];
+ <$fh>;
+}
+if($opt::header and not $opt::pipe) {
+ my $fh = $fhlist[0];
+ # split with colsep or \t
+ # $header force $colsep = \t if undef?
+ my $delimiter = $opt::colsep;
+ $delimiter ||= "\$";
+ my $id = 1;
+ for my $fh (@fhlist) {
+ my $line = <$fh>;
+ chomp($line);
+ ::debug("init", "Delimiter: '$delimiter'");
+ for my $s (split /$delimiter/o, $line) {
+ ::debug("init", "Colname: '$s'");
+ # Replace {colname} with {2}
+ # TODO accept configurable short hands
+ # TODO how to deal with headers in {=...=}
+ for(@command) {
+ s:\{$s(|/|//|\.|/\.)\}:\{$id$1\}:g;
+ }
+ $Global::input_source_header{$id} = $s;
+ $id++;
+ }
+ }
+} else {
+ my $id = 1;
+ for my $fh (@fhlist) {
+ $Global::input_source_header{$id} = $id;
+ $id++;
+ }
+}
+
+if($opt::filter_hosts and (@opt::sshlogin or @opt::sshloginfile)) {
+ # Parallel check all hosts are up. Remove hosts that are down
+ filter_hosts();
+}
+
+if($opt::nonall or $opt::onall) {
+ onall(@command);
+ wait_and_exit(min(undef_as_zero($Global::exitstatus),254));
+}
+
+# TODO --transfer foo/./bar --cleanup
+# multiple --transfer and --basefile with different /./
+
+$Global::JobQueue = JobQueue->new(
+ \@command,\@fhlist,$Global::ContextReplace,$number_of_args,\@Global::ret_files);
+
+if($opt::eta or $opt::bar) {
+ # Count the number of jobs before starting any
+ $Global::JobQueue->total_jobs();
+}
+if($opt::pipepart) {
+ @Global::cat_partials = map { pipe_part_files($_) } @opt::a;
+ # Unget the command as many times as there are parts
+ $Global::JobQueue->{'commandlinequeue'}->unget(
+ map { $Global::JobQueue->{'commandlinequeue'}->get() } @Global::cat_partials
+ );
+}
+for my $sshlogin (values %Global::host) {
+ $sshlogin->max_jobs_running();
+}
+
+init_run_jobs();
+my $sem;
+if($Global::semaphore) {
+ $sem = acquire_semaphore();
+}
+$SIG{TERM} = \&start_no_new_jobs;
+
+start_more_jobs();
+if(not $opt::pipepart) {
+ if($opt::pipe) {
+ spreadstdin();
+ }
+}
+::debug("init", "Start draining\n");
+drain_job_queue();
+::debug("init", "Done draining\n");
+reaper();
+::debug("init", "Done reaping\n");
+if($opt::pipe and @opt::a) {
+ for my $job (@Global::tee_jobs) {
+ unlink $job->fh(2,"name");
+ $job->set_fh(2,"name","");
+ $job->print();
+ unlink $job->fh(1,"name");
+ }
+}
+::debug("init", "Cleaning\n");
+cleanup();
+if($Global::semaphore) {
+ $sem->release();
+}
+for(keys %Global::sshmaster) {
+ kill "TERM", $_;
+}
+::debug("init", "Halt\n");
+if($opt::halt_on_error) {
+ wait_and_exit($Global::halt_on_error_exitstatus);
+} else {
+ wait_and_exit(min(undef_as_zero($Global::exitstatus),254));
+}
+
+sub __PIPE_MODE__ {}
+
+sub pipe_part_files {
+ # Input:
+ # $file = the file to read
+ # Returns:
+ # @commands that will cat_partial each part
+ my ($file) = @_;
+ my $buf = "";
+ my $header = find_header(\$buf,open_or_exit($file));
+ # find positions
+ my @pos = find_split_positions($file,$opt::blocksize,length $header);
+ # Make @cat_partials
+ my @cat_partials = ();
+ for(my $i=0; $i<$#pos; $i++) {
+ push @cat_partials, cat_partial($file, 0, length($header), $pos[$i], $pos[$i+1]);
+ }
+ # Remote exec should look like:
+ # ssh -oLogLevel=quiet lo 'eval `echo $SHELL | grep "/t\{0,1\}csh" > /dev/null && echo setenv PARALLEL_SEQ '$PARALLEL_SEQ'\; setenv PARALLEL_PID '$PARALLEL_PID' || echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ FOO\ /tmp/foo\ \|\|\ export\ FOO=/tmp/foo\; \(wc\ -\ \$FOO\)
+ # ssh -tt not allowed. Remote will die due to broken pipe anyway.
+ # TODO test remote with --fifo / --cat
+ return @cat_partials;
+}
+
+sub find_header {
+ # Input:
+ # $buf_ref = reference to read-in buffer
+ # $fh = filehandle to read from
+ # Uses:
+ # $opt::header
+ # $opt::blocksize
+ # Returns:
+ # $header string
+ my ($buf_ref, $fh) = @_;
+ my $header = "";
+ if($opt::header) {
+ if($opt::header eq ":") { $opt::header = "(.*\n)"; }
+ # Number = number of lines
+ $opt::header =~ s/^(\d+)$/"(.*\n)"x$1/e;
+ while(read($fh,substr($$buf_ref,length $$buf_ref,0),$opt::blocksize)) {
+ if($$buf_ref=~s/^($opt::header)//) {
+ $header = $1;
+ last;
+ }
+ }
+ }
+ return $header;
+}
+
+sub find_split_positions {
+ # Input:
+ # $file = the file to read
+ # $block = (minimal) --block-size of each chunk
+ # $headerlen = length of header to be skipped
+ # Uses:
+ # $opt::recstart
+ # $opt::recend
+ # Returns:
+ # @positions of block start/end
+ my($file, $block, $headerlen) = @_;
+ my $size = -s $file;
+ $block = int $block;
+ # The optimal dd blocksize for mint, redhat, solaris, openbsd = 2^17..2^20
+ # The optimal dd blocksize for freebsd = 2^15..2^17
+ my $dd_block_size = 131072; # 2^17
+ my @pos;
+ my ($recstart,$recend) = recstartrecend();
+ my $recendrecstart = $recend.$recstart;
+ my $fh = ::open_or_exit($file);
+ push(@pos,$headerlen);
+ for(my $pos = $block+$headerlen; $pos < $size; $pos += $block) {
+ my $buf;
+ seek($fh, $pos, 0) || die;
+ while(read($fh,substr($buf,length $buf,0),$dd_block_size)) {
+ if($opt::regexp) {
+ # If match /$recend$recstart/ => Record position
+ if($buf =~ /(.*$recend)$recstart/os) {
+ my $i = length($1);
+ push(@pos,$pos+$i);
+ # Start looking for next record _after_ this match
+ $pos += $i;
+ last;
+ }
+ } else {
+ # If match $recend$recstart => Record position
+ my $i = index($buf,$recendrecstart);
+ if($i != -1) {
+ push(@pos,$pos+$i);
+ # Start looking for next record _after_ this match
+ $pos += $i;
+ last;
+ }
+ }
+ }
+ }
+ push(@pos,$size);
+ close $fh;
+ return @pos;
+}
+
+sub cat_partial {
+ # Input:
+ # $file = the file to read
+ # ($start, $end, [$start2, $end2, ...]) = start byte, end byte
+ # Returns:
+ # Efficient perl command to copy $start..$end, $start2..$end2, ... to stdout
+ my($file, @start_end) = @_;
+ my($start, $i);
+ # Convert start_end to start_len
+ my @start_len = map { if(++$i % 2) { $start = $_; } else { $_-$start } } @start_end;
+ return "<". shell_quote_scalar($file) .
+ q{ perl -e 'while(@ARGV) { sysseek(STDIN,shift,0) || die; $left = shift; while($read = sysread(STDIN,$buf, ($left > 32768 ? 32768 : $left))){ $left -= $read; syswrite(STDOUT,$buf); } }' } .
+ " @start_len";
+}
+
+sub spreadstdin {
+ # read a record
+ # Spawn a job and print the record to it.
+ # Uses:
+ # $opt::blocksize
+ # STDIN
+ # $opr::r
+ # $Global::max_lines
+ # $Global::max_number_of_args
+ # $opt::regexp
+ # $Global::start_no_new_jobs
+ # $opt::roundrobin
+ # %Global::running
+
+ my $buf = "";
+ my ($recstart,$recend) = recstartrecend();
+ my $recendrecstart = $recend.$recstart;
+ my $chunk_number = 1;
+ my $one_time_through;
+ my $blocksize = $opt::blocksize;
+ my $in = *STDIN;
+ my $header = find_header(\$buf,$in);
+ while(1) {
+ my $anything_written = 0;
+ if(not read($in,substr($buf,length $buf,0),$blocksize)) {
+ # End-of-file
+ $chunk_number != 1 and last;
+ # Force the while-loop once if everything was read by header reading
+ $one_time_through++ and last;
+ }
+ if($opt::r) {
+ # Remove empty lines
+ $buf =~ s/^\s*\n//gm;
+ if(length $buf == 0) {
+ next;
+ }
+ }
+ if($Global::max_lines and not $Global::max_number_of_args) {
+ # Read n-line records
+ my $n_lines = $buf =~ tr/\n/\n/;
+ my $last_newline_pos = rindex($buf,"\n");
+ while($n_lines % $Global::max_lines) {
+ $n_lines--;
+ $last_newline_pos = rindex($buf,"\n",$last_newline_pos-1);
+ }
+ # Chop at $last_newline_pos as that is where n-line record ends
+ $anything_written +=
+ write_record_to_pipe($chunk_number++,\$header,\$buf,
+ $recstart,$recend,$last_newline_pos+1);
+ substr($buf,0,$last_newline_pos+1) = "";
+ } elsif($opt::regexp) {
+ if($Global::max_number_of_args) {
+ # -N => (start..*?end){n}
+ # -L -N => (start..*?end){n*l}
+ my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1);
+ while($buf =~ s/((?:$recstart.*?$recend){$read_n_lines})($recstart.*)$/$2/os) {
+ # Copy to modifiable variable
+ my $b = $1;
+ $anything_written +=
+ write_record_to_pipe($chunk_number++,\$header,\$b,
+ $recstart,$recend,length $1);
+ }
+ } else {
+ # Find the last recend-recstart in $buf
+ if($buf =~ s/(.*$recend)($recstart.*?)$/$2/os) {
+ # Copy to modifiable variable
+ my $b = $1;
+ $anything_written +=
+ write_record_to_pipe($chunk_number++,\$header,\$b,
+ $recstart,$recend,length $1);
+ }
+ }
+ } else {
+ if($Global::max_number_of_args) {
+ # -N => (start..*?end){n}
+ my $i = 0;
+ my $read_n_lines = $Global::max_number_of_args * ($Global::max_lines || 1);
+ while(($i = nindex(\$buf,$recendrecstart,$read_n_lines)) != -1) {
+ $i += length $recend; # find the actual splitting location
+ $anything_written +=
+ write_record_to_pipe($chunk_number++,\$header,\$buf,
+ $recstart,$recend,$i);
+ substr($buf,0,$i) = "";
+ }
+ } else {
+ # Find the last recend-recstart in $buf
+ my $i = rindex($buf,$recendrecstart);
+ if($i != -1) {
+ $i += length $recend; # find the actual splitting location
+ $anything_written +=
+ write_record_to_pipe($chunk_number++,\$header,\$buf,
+ $recstart,$recend,$i);
+ substr($buf,0,$i) = "";
+ }
+ }
+ }
+ if(not $anything_written and not eof($in)) {
+ # Nothing was written - maybe the block size < record size?
+ # Increase blocksize exponentially
+ my $old_blocksize = $blocksize;
+ $blocksize = ceil($blocksize * 1.3 + 1);
+ ::warning("A record was longer than $old_blocksize. " .
+ "Increasing to --blocksize $blocksize\n");
+ }
+ }
+ ::debug("init", "Done reading input\n");
+
+ # If there is anything left in the buffer write it
+ substr($buf,0,0) = "";
+ write_record_to_pipe($chunk_number++,\$header,\$buf,$recstart,$recend,length $buf);
+
+ $Global::start_no_new_jobs ||= 1;
+ if($opt::roundrobin) {
+ for my $job (values %Global::running) {
+ close $job->fh(0,"w");
+ }
+ my %incomplete_jobs = %Global::running;
+ my $sleep = 1;
+ while(keys %incomplete_jobs) {
+ my $something_written = 0;
+ for my $pid (keys %incomplete_jobs) {
+ my $job = $incomplete_jobs{$pid};
+ if($job->stdin_buffer_length()) {
+ $something_written += $job->non_block_write();
+ } else {
+ delete $incomplete_jobs{$pid}
+ }
+ }
+ if($something_written) {
+ $sleep = $sleep/2+0.001;
+ }
+ $sleep = ::reap_usleep($sleep);
+ }
+ }
+}
+
+sub recstartrecend {
+ # Uses:
+ # $opt::recstart
+ # $opt::recend
+ # Returns:
+ # $recstart,$recend with default values and regexp conversion
+ my($recstart,$recend);
+ if(defined($opt::recstart) and defined($opt::recend)) {
+ # If both --recstart and --recend is given then both must match
+ $recstart = $opt::recstart;
+ $recend = $opt::recend;
+ } elsif(defined($opt::recstart)) {
+ # If --recstart is given it must match start of record
+ $recstart = $opt::recstart;
+ $recend = "";
+ } elsif(defined($opt::recend)) {
+ # If --recend is given then it must match end of record
+ $recstart = "";
+ $recend = $opt::recend;
+ }
+
+ if($opt::regexp) {
+ # If $recstart/$recend contains '|' this should only apply to the regexp
+ $recstart = "(?:".$recstart.")";
+ $recend = "(?:".$recend.")";
+ } else {
+ # $recstart/$recend = printf strings (\n)
+ $recstart =~ s/\\([0rnt\'\"\\])/"qq|\\$1|"/gee;
+ $recend =~ s/\\([0rnt\'\"\\])/"qq|\\$1|"/gee;
+ }
+ return ($recstart,$recend);
+}
+
+sub nindex {
+ # See if string is in buffer N times
+ # Returns:
+ # the position where the Nth copy is found
+ my ($buf_ref, $str, $n) = @_;
+ my $i = 0;
+ for(1..$n) {
+ $i = index($$buf_ref,$str,$i+1);
+ if($i == -1) { last }
+ }
+ return $i;
+}
+
+{
+ my @robin_queue;
+
+ sub round_robin_write {
+ # Input:
+ # $header_ref = ref to $header string
+ # $block_ref = ref to $block to be written
+ # $recstart = record start string
+ # $recend = record end string
+ # $endpos = end position of $block
+ # Uses:
+ # %Global::running
+ my ($header_ref,$block_ref,$recstart,$recend,$endpos) = @_;
+ my $something_written = 0;
+ my $block_passed = 0;
+ my $sleep = 1;
+ while(not $block_passed) {
+ # Continue flushing existing buffers
+ # until one is empty and a new block is passed
+ # Make a queue to spread the blocks evenly
+ if(not @robin_queue) {
+ push @robin_queue, values %Global::running;
+ }
+ while(my $job = shift @robin_queue) {
+ if($job->stdin_buffer_length() > 0) {
+ $something_written += $job->non_block_write();
+ } else {
+ $job->set_stdin_buffer($header_ref,$block_ref,$endpos,$recstart,$recend);
+ $block_passed = 1;
+ $job->set_virgin(0);
+ $something_written += $job->non_block_write();
+ last;
+ }
+ }
+ $sleep = ::reap_usleep($sleep);
+ }
+ return $something_written;
+ }
+}
+
+sub write_record_to_pipe {
+ # Fork then
+ # Write record from pos 0 .. $endpos to pipe
+ # Input:
+ # $chunk_number = sequence number - to see if already run
+ # $header_ref = reference to header string to prepend
+ # $record_ref = reference to record to write
+ # $recstart = start string of record
+ # $recend = end string of record
+ # $endpos = position in $record_ref where record ends
+ # Uses:
+ # $Global::job_already_run
+ # $opt::roundrobin
+ # @Global::virgin_jobs
+ # Returns:
+ # Number of chunks written (0 or 1)
+ my ($chunk_number,$header_ref,$record_ref,$recstart,$recend,$endpos) = @_;
+ if($endpos == 0) { return 0; }
+ if(vec($Global::job_already_run,$chunk_number,1)) { return 1; }
+ if($opt::roundrobin) {
+ return round_robin_write($header_ref,$record_ref,$recstart,$recend,$endpos);
+ }
+ # If no virgin found, backoff
+ my $sleep = 0.0001; # 0.01 ms - better performance on highend
+ while(not @Global::virgin_jobs) {
+ ::debug("pipe", "No virgin jobs");
+ $sleep = ::reap_usleep($sleep);
+ # Jobs may not be started because of loadavg
+ # or too little time between each ssh login.
+ start_more_jobs();
+ }
+ my $job = shift @Global::virgin_jobs;
+ # Job is no longer virgin
+ $job->set_virgin(0);
+ if(fork()) {
+ # Skip
+ } else {
+ # Chop of at $endpos as we do not know how many rec_sep will
+ # be removed.
+ substr($$record_ref,$endpos,length $$record_ref) = "";
+ # Remove rec_sep
+ if($opt::remove_rec_sep) {
+ Job::remove_rec_sep($record_ref,$recstart,$recend);
+ }
+ $job->write($header_ref);
+ $job->write($record_ref);
+ close $job->fh(0,"w");
+ exit(0);
+ }
+ close $job->fh(0,"w");
+ return 1;
+}
+
+sub __SEM_MODE__ {}
+
+sub acquire_semaphore {
+ # Acquires semaphore. If needed: spawns to the background
+ # Uses:
+ # @Global::host
+ # Returns:
+ # The semaphore to be released when jobs is complete
+ $Global::host{':'} = SSHLogin->new(":");
+ my $sem = Semaphore->new($Semaphore::name,$Global::host{':'}->max_jobs_running());
+ $sem->acquire();
+ if($Semaphore::fg) {
+ # skip
+ } else {
+ # If run in the background, the PID will change
+ # therefore release and re-acquire the semaphore
+ $sem->release();
+ if(fork()) {
+ exit(0);
+ } else {
+ # child
+ # Get a semaphore for this pid
+ ::die_bug("Can't start a new session: $!") if setsid() == -1;
+ $sem = Semaphore->new($Semaphore::name,$Global::host{':'}->max_jobs_running());
+ $sem->acquire();
+ }
+ }
+ return $sem;
+}
+
+sub __PARSE_OPTIONS__ {}
+
+sub options_hash {
+ # Returns:
+ # %hash = the GetOptions config
+ return
+ ("debug|D=s" => \$opt::D,
+ "xargs" => \$opt::xargs,
+ "m" => \$opt::m,
+ "X" => \$opt::X,
+ "v" => \@opt::v,
+ "joblog=s" => \$opt::joblog,
+ "results|result|res=s" => \$opt::results,
+ "resume" => \$opt::resume,
+ "resume-failed|resumefailed" => \$opt::resume_failed,
+ "silent" => \$opt::silent,
+ #"silent-error|silenterror" => \$opt::silent_error,
+ "keep-order|keeporder|k" => \$opt::keeporder,
+ "group" => \$opt::group,
+ "g" => \$opt::retired,
+ "ungroup|u" => \$opt::ungroup,
+ "linebuffer|linebuffered|line-buffer|line-buffered" => \$opt::linebuffer,
+ "tmux" => \$opt::tmux,
+ "null|0" => \$opt::0,
+ "quote|q" => \$opt::q,
+ # Replacement strings
+ "parens=s" => \$opt::parens,
+ "rpl=s" => \@opt::rpl,
+ "plus" => \$opt::plus,
+ "I=s" => \$opt::I,
+ "extensionreplace|er=s" => \$opt::U,
+ "U=s" => \$opt::retired,
+ "basenamereplace|bnr=s" => \$opt::basenamereplace,
+ "dirnamereplace|dnr=s" => \$opt::dirnamereplace,
+ "basenameextensionreplace|bner=s" => \$opt::basenameextensionreplace,
+ "seqreplace=s" => \$opt::seqreplace,
+ "slotreplace=s" => \$opt::slotreplace,
+ "jobs|j=s" => \$opt::jobs,
+ "delay=f" => \$opt::delay,
+ "sshdelay=f" => \$opt::sshdelay,
+ "load=s" => \$opt::load,
+ "noswap" => \$opt::noswap,
+ "max-line-length-allowed" => \$opt::max_line_length_allowed,
+ "number-of-cpus" => \$opt::number_of_cpus,
+ "number-of-cores" => \$opt::number_of_cores,
+ "use-cpus-instead-of-cores" => \$opt::use_cpus_instead_of_cores,
+ "shellquote|shell_quote|shell-quote" => \$opt::shellquote,
+ "nice=i" => \$opt::nice,
+ "timeout=s" => \$opt::timeout,
+ "tag" => \$opt::tag,
+ "tagstring|tag-string=s" => \$opt::tagstring,
+ "onall" => \$opt::onall,
+ "nonall" => \$opt::nonall,
+ "filter-hosts|filterhosts|filter-host" => \$opt::filter_hosts,
+ "sshlogin|S=s" => \@opt::sshlogin,
+ "sshloginfile|slf=s" => \@opt::sshloginfile,
+ "controlmaster|M" => \$opt::controlmaster,
+ "return=s" => \@opt::return,
+ "trc=s" => \@opt::trc,
+ "transfer" => \$opt::transfer,
+ "cleanup" => \$opt::cleanup,
+ "basefile|bf=s" => \@opt::basefile,
+ "B=s" => \$opt::retired,
+ "ctrlc|ctrl-c" => \$opt::ctrlc,
+ "noctrlc|no-ctrlc|no-ctrl-c" => \$opt::noctrlc,
+ "workdir|work-dir|wd=s" => \$opt::workdir,
+ "W=s" => \$opt::retired,
+ "tmpdir=s" => \$opt::tmpdir,
+ "tempdir=s" => \$opt::tmpdir,
+ "use-compress-program|compress-program=s" => \$opt::compress_program,
+ "use-decompress-program|decompress-program=s" => \$opt::decompress_program,
+ "compress" => \$opt::compress,
+ "tty" => \$opt::tty,
+ "T" => \$opt::retired,
+ "halt-on-error|halt=s" => \$opt::halt_on_error,
+ "H=i" => \$opt::retired,
+ "retries=i" => \$opt::retries,
+ "dry-run|dryrun" => \$opt::dryrun,
+ "progress" => \$opt::progress,
+ "eta" => \$opt::eta,
+ "bar" => \$opt::bar,
+ "arg-sep|argsep=s" => \$opt::arg_sep,
+ "arg-file-sep|argfilesep=s" => \$opt::arg_file_sep,
+ "trim=s" => \$opt::trim,
+ "env=s" => \@opt::env,
+ "recordenv|record-env" => \$opt::record_env,
+ "plain" => \$opt::plain,
+ "profile|J=s" => \@opt::profile,
+ "pipe|spreadstdin" => \$opt::pipe,
+ "robin|round-robin|roundrobin" => \$opt::roundrobin,
+ "recstart=s" => \$opt::recstart,
+ "recend=s" => \$opt::recend,
+ "regexp|regex" => \$opt::regexp,
+ "remove-rec-sep|removerecsep|rrs" => \$opt::remove_rec_sep,
+ "files|output-as-files|outputasfiles" => \$opt::files,
+ "block|block-size|blocksize=s" => \$opt::blocksize,
+ "tollef" => \$opt::retired,
+ "gnu" => \$opt::gnu,
+ "xapply" => \$opt::xapply,
+ "bibtex" => \$opt::bibtex,
+ "nn|nonotice|no-notice" => \$opt::no_notice,
+ # xargs-compatibility - implemented, man, testsuite
+ "max-procs|P=s" => \$opt::jobs,
+ "delimiter|d=s" => \$opt::d,
+ "max-chars|s=i" => \$opt::max_chars,
+ "arg-file|a=s" => \@opt::a,
+ "no-run-if-empty|r" => \$opt::r,
+ "replace|i:s" => \$opt::i,
+ "E=s" => \$opt::eof,
+ "eof|e:s" => \$opt::eof,
+ "max-args|n=i" => \$opt::max_args,
+ "max-replace-args|N=i" => \$opt::max_replace_args,
+ "colsep|col-sep|C=s" => \$opt::colsep,
+ "help|h" => \$opt::help,
+ "L=f" => \$opt::L,
+ "max-lines|l:f" => \$opt::max_lines,
+ "interactive|p" => \$opt::p,
+ "verbose|t" => \$opt::verbose,
+ "version|V" => \$opt::version,
+ "minversion|min-version=i" => \$opt::minversion,
+ "show-limits|showlimits" => \$opt::show_limits,
+ "exit|x" => \$opt::x,
+ # Semaphore
+ "semaphore" => \$opt::semaphore,
+ "semaphoretimeout=i" => \$opt::semaphoretimeout,
+ "semaphorename|id=s" => \$opt::semaphorename,
+ "fg" => \$opt::fg,
+ "bg" => \$opt::bg,
+ "wait" => \$opt::wait,
+ # Shebang #!/usr/bin/parallel --shebang
+ "shebang|hashbang" => \$opt::shebang,
+ "internal-pipe-means-argfiles" => \$opt::internal_pipe_means_argfiles,
+ "Y" => \$opt::retired,
+ "skip-first-line" => \$opt::skip_first_line,
+ "header=s" => \$opt::header,
+ "cat" => \$opt::cat,
+ "fifo" => \$opt::fifo,
+ "pipepart|pipe-part" => \$opt::pipepart,
+ "hgrp|hostgroup|hostgroups" => \$opt::hostgroups,
+ );
+}
+
+sub get_options_from_array {
+ # Run GetOptions on @array
+ # Input:
+ # $array_ref = ref to @ARGV to parse
+ # @keep_only = Keep only these options
+ # Uses:
+ # @ARGV
+ # Returns:
+ # true if parsing worked
+ # false if parsing failed
+ # @$array_ref is changed
+ my ($array_ref, @keep_only) = @_;
+ if(not @$array_ref) {
+ # Empty array: No need to look more at that
+ return 1;
+ }
+ # A bit of shuffling of @ARGV needed as GetOptionsFromArray is not
+ # supported everywhere
+ my @save_argv;
+ my $this_is_ARGV = (\@::ARGV == $array_ref);
+ if(not $this_is_ARGV) {
+ @save_argv = @::ARGV;
+ @::ARGV = @{$array_ref};
+ }
+ # If @keep_only set: Ignore all values except @keep_only
+ my %options = options_hash();
+ if(@keep_only) {
+ my (%keep,@dummy);
+ @keep{@keep_only} = @keep_only;
+ for my $k (grep { not $keep{$_} } keys %options) {
+ # Store the value of the option in @dummy
+ $options{$k} = \@dummy;
+ }
+ }
+ my $retval = GetOptions(%options);
+ if(not $this_is_ARGV) {
+ @{$array_ref} = @::ARGV;
+ @::ARGV = @save_argv;
+ }
+ return $retval;
+}
+
+sub parse_options {
+ # Returns: N/A
+ # Defaults:
+ $Global::version = 20141122;
+ $Global::progname = 'parallel';
+ $Global::infinity = 2**31;
+ $Global::debug = 0;
+ $Global::verbose = 0;
+ $Global::quoting = 0;
+ # Read only table with default --rpl values
+ %Global::replace =
+ (
+ '{}' => '',
+ '{#}' => '1 $_=$job->seq()',
+ '{%}' => '1 $_=$job->slot()',
+ '{/}' => 's:.*/::',
+ '{//}' => '$Global::use{"File::Basename"} ||= eval "use File::Basename; 1;"; $_ = dirname($_);',
+ '{/.}' => 's:.*/::; s:\.[^/.]+$::;',
+ '{.}' => 's:\.[^/.]+$::',
+ );
+ %Global::plus =
+ (
+ # {} = {+/}/{/}
+ # = {.}.{+.} = {+/}/{/.}.{+.}
+ # = {..}.{+..} = {+/}/{/..}.{+..}
+ # = {...}.{+...} = {+/}/{/...}.{+...}
+ '{+/}' => 's:/[^/]*$::',
+ '{+.}' => 's:.*\.::',
+ '{+..}' => 's:.*\.([^.]*\.):$1:',
+ '{+...}' => 's:.*\.([^.]*\.[^.]*\.):$1:',
+ '{..}' => 's:\.[^/.]+$::; s:\.[^/.]+$::',
+ '{...}' => 's:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
+ '{/..}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::',
+ '{/...}' => 's:.*/::; s:\.[^/.]+$::; s:\.[^/.]+$::; s:\.[^/.]+$::',
+ );
+ # Modifiable copy of %Global::replace
+ %Global::rpl = %Global::replace;
+ $Global::parens = "{==}";
+ $/="\n";
+ $Global::ignore_empty = 0;
+ $Global::interactive = 0;
+ $Global::stderr_verbose = 0;
+ $Global::default_simultaneous_sshlogins = 9;
+ $Global::exitstatus = 0;
+ $Global::halt_on_error_exitstatus = 0;
+ $Global::arg_sep = ":::";
+ $Global::arg_file_sep = "::::";
+ $Global::trim = 'n';
+ $Global::max_jobs_running = 0;
+ $Global::job_already_run = '';
+ $ENV{'TMPDIR'} ||= "/tmp";
+
+ @ARGV=read_options();
+
+ if(@opt::v) { $Global::verbose = $#opt::v+1; } # Convert -v -v to v=2
+ $Global::debug = $opt::D;
+ $Global::shell = $ENV{'PARALLEL_SHELL'} || parent_shell($$) || $ENV{'SHELL'} || "/bin/sh";
+ if(defined $opt::X) { $Global::ContextReplace = 1; }
+ if(defined $opt::silent) { $Global::verbose = 0; }
+ if(defined $opt::0) { $/ = "\0"; }
+ if(defined $opt::d) { my $e="sprintf \"$opt::d\""; $/ = eval $e; }
+ if(defined $opt::p) { $Global::interactive = $opt::p; }
+ if(defined $opt::q) { $Global::quoting = 1; }
+ if(defined $opt::r) { $Global::ignore_empty = 1; }
+ if(defined $opt::verbose) { $Global::stderr_verbose = 1; }
+ # Deal with --rpl
+ sub rpl {
+ # Modify %Global::rpl
+ # Replace $old with $new
+ my ($old,$new) = @_;
+ if($old ne $new) {
+ $Global::rpl{$new} = $Global::rpl{$old};
+ delete $Global::rpl{$old};
+ }
+ }
+ if(defined $opt::parens) { $Global::parens = $opt::parens; }
+ my $parenslen = 0.5*length $Global::parens;
+ $Global::parensleft = substr($Global::parens,0,$parenslen);
+ $Global::parensright = substr($Global::parens,$parenslen);
+ if(defined $opt::plus) { %Global::rpl = (%Global::plus,%Global::rpl); }
+ if(defined $opt::I) { rpl('{}',$opt::I); }
+ if(defined $opt::U) { rpl('{.}',$opt::U); }
+ if(defined $opt::i and $opt::i) { rpl('{}',$opt::i); }
+ if(defined $opt::basenamereplace) { rpl('{/}',$opt::basenamereplace); }
+ if(defined $opt::dirnamereplace) { rpl('{//}',$opt::dirnamereplace); }
+ if(defined $opt::seqreplace) { rpl('{#}',$opt::seqreplace); }
+ if(defined $opt::slotreplace) { rpl('{%}',$opt::slotreplace); }
+ if(defined $opt::basenameextensionreplace) {
+ rpl('{/.}',$opt::basenameextensionreplace);
+ }
+ for(@opt::rpl) {
+ # Create $Global::rpl entries for --rpl options
+ # E.g: "{..} s:\.[^.]+$:;s:\.[^.]+$:;"
+ my ($shorthand,$long) = split/ /,$_,2;
+ $Global::rpl{$shorthand} = $long;
+ }
+ if(defined $opt::eof) { $Global::end_of_file_string = $opt::eof; }
+ if(defined $opt::max_args) { $Global::max_number_of_args = $opt::max_args; }
+ if(defined $opt::timeout) { $Global::timeoutq = TimeoutQueue->new($opt::timeout); }
+ if(defined $opt::tmpdir) { $ENV{'TMPDIR'} = $opt::tmpdir; }
+ if(defined $opt::help) { die_usage(); }
+ if(defined $opt::colsep) { $Global::trim = 'lr'; }
+ if(defined $opt::header) { $opt::colsep = defined $opt::colsep ? $opt::colsep : "\t"; }
+ if(defined $opt::trim) { $Global::trim = $opt::trim; }
+ if(defined $opt::arg_sep) { $Global::arg_sep = $opt::arg_sep; }
+ if(defined $opt::arg_file_sep) { $Global::arg_file_sep = $opt::arg_file_sep; }
+ if(defined $opt::number_of_cpus) { print SSHLogin::no_of_cpus(),"\n"; wait_and_exit(0); }
+ if(defined $opt::number_of_cores) {
+ print SSHLogin::no_of_cores(),"\n"; wait_and_exit(0);
+ }
+ if(defined $opt::max_line_length_allowed) {
+ print Limits::Command::real_max_length(),"\n"; wait_and_exit(0);
+ }
+ if(defined $opt::version) { version(); wait_and_exit(0); }
+ if(defined $opt::bibtex) { bibtex(); wait_and_exit(0); }
+ if(defined $opt::record_env) { record_env(); wait_and_exit(0); }
+ if(defined $opt::show_limits) { show_limits(); }
+ if(@opt::sshlogin) { @Global::sshlogin = @opt::sshlogin; }
+ if(@opt::sshloginfile) { read_sshloginfiles(@opt::sshloginfile); }
+ if(@opt::return) { push @Global::ret_files, @opt::return; }
+ if(not defined $opt::recstart and
+ not defined $opt::recend) { $opt::recend = "\n"; }
+ if(not defined $opt::blocksize) { $opt::blocksize = "1M"; }
+ $opt::blocksize = multiply_binary_prefix($opt::blocksize);
+ if(defined $opt::controlmaster) { $opt::noctrlc = 1; }
+ if(defined $opt::semaphore) { $Global::semaphore = 1; }
+ if(defined $opt::semaphoretimeout) { $Global::semaphore = 1; }
+ if(defined $opt::semaphorename) { $Global::semaphore = 1; }
+ if(defined $opt::fg) { $Global::semaphore = 1; }
+ if(defined $opt::bg) { $Global::semaphore = 1; }
+ if(defined $opt::wait) { $Global::semaphore = 1; }
+ if(defined $opt::halt_on_error and
+ $opt::halt_on_error=~/%/) { $opt::halt_on_error /= 100; }
+ if(defined $opt::timeout and $opt::timeout !~ /^\d+(\.\d+)?%?$/) {
+ ::error("--timeout must be seconds or percentage\n");
+ wait_and_exit(255);
+ }
+ if(defined $opt::minversion) {
+ print $Global::version,"\n";
+ if($Global::version < $opt::minversion) {
+ wait_and_exit(255);
+ } else {
+ wait_and_exit(0);
+ }
+ }
+ if(not defined $opt::delay) {
+ # Set --delay to --sshdelay if not set
+ $opt::delay = $opt::sshdelay;
+ }
+ if($opt::compress_program) {
+ $opt::compress = 1;
+ $opt::decompress_program ||= $opt::compress_program." -dc";
+ }
+ if($opt::compress) {
+ my ($compress, $decompress) = find_compression_program();
+ $opt::compress_program ||= $compress;
+ $opt::decompress_program ||= $decompress;
+ }
+ if(defined $opt::nonall) {
+ # Append a dummy empty argument
+ push @ARGV, $Global::arg_sep, "";
+ }
+ if(defined $opt::tty) {
+ # Defaults for --tty: -j1 -u
+ # Can be overridden with -jXXX -g
+ if(not defined $opt::jobs) {
+ $opt::jobs = 1;
+ }
+ if(not defined $opt::group) {
+ $opt::ungroup = 0;
+ }
+ }
+ if(@opt::trc) {
+ push @Global::ret_files, @opt::trc;
+ $opt::transfer = 1;
+ $opt::cleanup = 1;
+ }
+ if(defined $opt::max_lines) {
+ if($opt::max_lines eq "-0") {
+ # -l -0 (swallowed -0)
+ $opt::max_lines = 1;
+ $opt::0 = 1;
+ $/ = "\0";
+ } elsif ($opt::max_lines == 0) {
+ # If not given (or if 0 is given) => 1
+ $opt::max_lines = 1;
+ }
+ $Global::max_lines = $opt::max_lines;
+ if(not $opt::pipe) {
+ # --pipe -L means length of record - not max_number_of_args
+ $Global::max_number_of_args ||= $Global::max_lines;
+ }
+ }
+
+ # Read more than one arg at a time (-L, -N)
+ if(defined $opt::L) {
+ $Global::max_lines = $opt::L;
+ if(not $opt::pipe) {
+ # --pipe -L means length of record - not max_number_of_args
+ $Global::max_number_of_args ||= $Global::max_lines;
+ }
+ }
+ if(defined $opt::max_replace_args) {
+ $Global::max_number_of_args = $opt::max_replace_args;
+ $Global::ContextReplace = 1;
+ }
+ if((defined $opt::L or defined $opt::max_replace_args)
+ and
+ not ($opt::xargs or $opt::m)) {
+ $Global::ContextReplace = 1;
+ }
+ if(defined $opt::tag and not defined $opt::tagstring) {
+ $opt::tagstring = "\257<\257>"; # Default = {}
+ }
+ if(defined $opt::pipepart and
+ (defined $opt::L or defined $opt::max_lines
+ or defined $opt::max_replace_args)) {
+ ::error("--pipepart is incompatible with --max-replace-args, ",
+ "--max-lines, and -L.\n");
+ wait_and_exit(255);
+ }
+ if(grep /^$Global::arg_sep$|^$Global::arg_file_sep$/o, @ARGV) {
+ # Deal with ::: and ::::
+ @ARGV=read_args_from_command_line();
+ }
+
+ # Semaphore defaults
+ # Must be done before computing number of processes and max_line_length
+ # because when running as a semaphore GNU Parallel does not read args
+ $Global::semaphore ||= ($0 =~ m:(^|/)sem$:); # called as 'sem'
+ if($Global::semaphore) {
+ # A semaphore does not take input from neither stdin nor file
+ @opt::a = ("/dev/null");
+ push(@Global::unget_argv, [Arg->new("")]);
+ $Semaphore::timeout = $opt::semaphoretimeout || 0;
+ if(defined $opt::semaphorename) {
+ $Semaphore::name = $opt::semaphorename;
+ } else {
+ $Semaphore::name = `tty`;
+ chomp $Semaphore::name;
+ }
+ $Semaphore::fg = $opt::fg;
+ $Semaphore::wait = $opt::wait;
+ $Global::default_simultaneous_sshlogins = 1;
+ if(not defined $opt::jobs) {
+ $opt::jobs = 1;
+ }
+ if($Global::interactive and $opt::bg) {
+ ::error("Jobs running in the ".
+ "background cannot be interactive.\n");
+ ::wait_and_exit(255);
+ }
+ }
+ if(defined $opt::eta) {
+ $opt::progress = $opt::eta;
+ }
+ if(defined $opt::bar) {
+ $opt::progress = $opt::bar;
+ }
+ if(defined $opt::retired) {
+ ::error("-g has been retired. Use --group.\n");
+ ::error("-B has been retired. Use --bf.\n");
+ ::error("-T has been retired. Use --tty.\n");
+ ::error("-U has been retired. Use --er.\n");
+ ::error("-W has been retired. Use --wd.\n");
+ ::error("-Y has been retired. Use --shebang.\n");
+ ::error("-H has been retired. Use --halt.\n");
+ ::error("--tollef has been retired. Use -u -q --arg-sep -- and --load for -l.\n");
+ ::wait_and_exit(255);
+ }
+ citation_notice();
+
+ parse_sshlogin();
+ parse_env_var();
+
+ if(remote_hosts() and ($opt::X or $opt::m or $opt::xargs)) {
+ # As we do not know the max line length on the remote machine
+ # long commands generated by xargs may fail
+ # If opt_N is set, it is probably safe
+ ::warning("Using -X or -m with --sshlogin may fail.\n");
+ }
+
+ if(not defined $opt::jobs) {
+ $opt::jobs = "100%";
+ }
+ open_joblog();
+}
+
+sub env_quote {
+ # Input:
+ # $v = value to quote
+ # Returns:
+ # $v = value quoted as environment variable
+ my $v = $_[0];
+ $v =~ s/([\\])/\\$1/g;
+ $v =~ s/([\[\] \#\'\&\<\>\(\)\;\{\}\t\"\$\`\*\174\!\?\~])/\\$1/g;
+ $v =~ s/\n/"\n"/g;
+ return $v;
+}
+
+sub record_env {
+ # Record current %ENV-keys in ~/.parallel/ignored_vars
+ # Returns: N/A
+ my $ignore_filename = $ENV{'HOME'} . "/.parallel/ignored_vars";
+ if(open(my $vars_fh, ">", $ignore_filename)) {
+ print $vars_fh map { $_,"\n" } keys %ENV;
+ } else {
+ ::error("Cannot write to $ignore_filename\n");
+ ::wait_and_exit(255);
+ }
+}
+
+sub parse_env_var {
+ # Parse --env and set $Global::envvar, $Global::envwarn and $Global::envvarlen
+ #
+ # Bash functions must be parsed to export them remotely
+ # Pre-shellshock style bash function:
+ # myfunc=() {...
+ # Post-shellshock style bash function:
+ # BASH_FUNC_myfunc()=() {...
+ #
+ # Uses:
+ # $Global::envvar = eval string that will set variables in both bash and csh
+ # $Global::envwarn = If functions are used: Give warning in csh
+ # $Global::envvarlen = length of $Global::envvar
+ # @opt::env
+ # $Global::shell
+ # %ENV
+ # Returns: N/A
+ $Global::envvar = "";
+ $Global::envwarn = "";
+ my @vars = ('parallel_bash_environment');
+ for my $varstring (@opt::env) {
+ # Split up --env VAR1,VAR2
+ push @vars, split /,/, $varstring;
+ }
+ if(grep { /^_$/ } @vars) {
+ # --env _
+ # Include all vars that are not in a clean environment
+ if(open(my $vars_fh, "<", $ENV{'HOME'} . "/.parallel/ignored_vars")) {
+ my @ignore = <$vars_fh>;
+ chomp @ignore;
+ my %ignore;
+ @ignore{@ignore} = @ignore;
+ close $vars_fh;
+ push @vars, grep { not defined $ignore{$_} } keys %ENV;
+ @vars = grep { not /^_$/ } @vars;
+ } else {
+ ::error("Run '$Global::progname --record-env' in a clean environment first.\n");
+ ::wait_and_exit(255);
+ }
+ }
+ # Duplicate vars as BASH functions to include post-shellshock functions.
+ # So --env myfunc should also look for BASH_FUNC_myfunc()
+ @vars = map { $_, "BASH_FUNC_$_()" } @vars;
+ # Keep only defined variables
+ @vars = grep { defined($ENV{$_}) } @vars;
+ # Pre-shellshock style bash function:
+ # myfunc=() { echo myfunc
+ # }
+ # Post-shellshock style bash function:
+ # BASH_FUNC_myfunc()=() { echo myfunc
+ # }
+ my @bash_functions = grep { substr($ENV{$_},0,4) eq "() {" } @vars;
+ my @non_functions = grep { substr($ENV{$_},0,4) ne "() {" } @vars;
+ if(@bash_functions) {
+ # Functions are not supported for all shells
+ if($Global::shell !~ m:/(bash|rbash|zsh|rzsh|dash|ksh):) {
+ ::warning("Shell functions may not be supported in $Global::shell\n");
+ }
+ }
+
+ # Pre-shellschock names are without ()
+ my @bash_pre_shellshock = grep { not /\(\)/ } @bash_functions;
+ # Post-shellschock names are with ()
+ my @bash_post_shellshock = grep { /\(\)/ } @bash_functions;
+
+ my @qcsh = (map { my $a=$_; "setenv $a " . env_quote($ENV{$a}) }
+ grep { not /^parallel_bash_environment$/ } @non_functions);
+ my @qbash = (map { my $a=$_; "export $a=" . env_quote($ENV{$a}) }
+ @non_functions, @bash_pre_shellshock);
+
+ push @qbash, map { my $a=$_; "eval $a\"\$$a\"" } @bash_pre_shellshock;
+ push @qbash, map { /BASH_FUNC_(.*)\(\)/; "$1 $ENV{$_}" } @bash_post_shellshock;
+
+ #ssh -tt -oLogLevel=quiet lo 'eval `echo PARALLEL_SEQ='$PARALLEL_SEQ'\;export PARALLEL_SEQ\; PARALLEL_PID='$PARALLEL_PID'\;export PARALLEL_PID` ;' tty\ \>/dev/null\ \&\&\ stty\ isig\ -onlcr\ -echo\;echo\ \$SHELL\ \|\ grep\ \"/t\\\{0,1\\\}csh\"\ \>\ /dev/null\ \&\&\ setenv\ BASH_FUNC_myfunc\ \\\(\\\)\\\ \\\{\\\ \\\ echo\\\ a\"'
+ #'\"\\\}\ \|\|\ myfunc\(\)\ \{\ \ echo\ a'
+ #'\}\ \;myfunc\ 1;
+
+ # Check if any variables contain \n
+ if(my @v = map { s/BASH_FUNC_(.*)\(\)/$1/; $_ } grep { $ENV{$_}=~/\n/ } @vars) {
+ # \n is bad for csh and will cause it to fail.
+ $Global::envwarn = ::shell_quote_scalar(q{echo $SHELL | egrep "/t?csh" > /dev/null && echo CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset }."@v".q{ && exec false;}."\n\n") . $Global::envwarn;
+ }
+
+ if(not @qcsh) { push @qcsh, "true"; }
+ if(not @qbash) { push @qbash, "true"; }
+ # Create lines like:
+ # echo $SHELL | grep "/t\\{0,1\\}csh" >/dev/null && setenv V1 val1 && setenv V2 val2 || export V1=val1 && export V2=val2 ; echo "$V1$V2"
+ if(@vars) {
+ $Global::envvar .=
+ join"",
+ (q{echo $SHELL | grep "/t\\{0,1\\}csh" > /dev/null && }
+ . join(" && ", @qcsh)
+ . q{ || }
+ . join(" && ", @qbash)
+ .q{;});
+ if($ENV{'parallel_bash_environment'}) {
+ $Global::envvar .= 'eval "$parallel_bash_environment";'."\n";
+ }
+ }
+ $Global::envvarlen = length $Global::envvar;
+}
+
+sub open_joblog {
+ # Open joblog as specified by --joblog
+ # Uses:
+ # $opt::resume
+ # $opt::resume_failed
+ # $opt::joblog
+ # $opt::results
+ # $Global::job_already_run
+ # %Global::fd
+ my $append = 0;
+ if(($opt::resume or $opt::resume_failed)
+ and
+ not ($opt::joblog or $opt::results)) {
+ ::error("--resume and --resume-failed require --joblog or --results.\n");
+ ::wait_and_exit(255);
+ }
+ if($opt::joblog) {
+ if($opt::resume || $opt::resume_failed) {
+ if(open(my $joblog_fh, "<", $opt::joblog)) {
+ # Read the joblog
+ $append = <$joblog_fh>; # If there is a header: Open as append later
+ my $joblog_regexp;
+ if($opt::resume_failed) {
+ # Make a regexp that only matches commands with exit+signal=0
+ # 4 host 1360490623.067 3.445 1023 1222 0 0 command
+ $joblog_regexp='^(\d+)(?:\t[^\t]+){5}\t0\t0\t';
+ } else {
+ # Just match the job number
+ $joblog_regexp='^(\d+)';
+ }
+ while(<$joblog_fh>) {
+ if(/$joblog_regexp/o) {
+ # This is 30% faster than set_job_already_run($1);
+ vec($Global::job_already_run,($1||0),1) = 1;
+ } elsif(not /\d+\s+[^\s]+\s+([0-9.]+\s+){6}/) {
+ ::error("Format of '$opt::joblog' is wrong: $_");
+ ::wait_and_exit(255);
+ }
+ }
+ close $joblog_fh;
+ }
+ }
+ if($append) {
+ # Append to joblog
+ if(not open($Global::joblog, ">>", $opt::joblog)) {
+ ::error("Cannot append to --joblog $opt::joblog.\n");
+ ::wait_and_exit(255);
+ }
+ } else {
+ if($opt::joblog eq "-") {
+ # Use STDOUT as joblog
+ $Global::joblog = $Global::fd{1};
+ } elsif(not open($Global::joblog, ">", $opt::joblog)) {
+ # Overwrite the joblog
+ ::error("Cannot write to --joblog $opt::joblog.\n");
+ ::wait_and_exit(255);
+ }
+ print $Global::joblog
+ join("\t", "Seq", "Host", "Starttime", "JobRuntime",
+ "Send", "Receive", "Exitval", "Signal", "Command"
+ ). "\n";
+ }
+ }
+}
+
+sub find_compression_program {
+ # Find a fast compression program
+ # Returns:
+ # $compress_program = compress program with options
+ # $decompress_program = decompress program with options
+
+ # Search for these. Sorted by speed
+ my @prg = qw(lzop pigz pxz gzip plzip pbzip2 lzma xz lzip bzip2);
+ for my $p (@prg) {
+ if(which($p)) {
+ return ("$p -c -1","$p -dc");
+ }
+ }
+ # Fall back to cat
+ return ("cat","cat");
+}
+
+
+sub read_options {
+ # Read options from command line, profile and $PARALLEL
+ # Uses:
+ # $opt::shebang_wrap
+ # $opt::shebang
+ # @ARGV
+ # $opt::plain
+ # @opt::profile
+ # $ENV{'HOME'}
+ # $ENV{'PARALLEL'}
+ # Returns:
+ # @ARGV_no_opt = @ARGV without --options
+
+ # This must be done first as this may exec myself
+ if(defined $ARGV[0] and ($ARGV[0] =~ /^--shebang/ or
+ $ARGV[0] =~ /^--shebang-?wrap/ or
+ $ARGV[0] =~ /^--hashbang/)) {
+ # Program is called from #! line in script
+ # remove --shebang-wrap if it is set
+ $opt::shebang_wrap = ($ARGV[0] =~ s/^--shebang-?wrap *//);
+ # remove --shebang if it is set
+ $opt::shebang = ($ARGV[0] =~ s/^--shebang *//);
+ # remove --hashbang if it is set
+ $opt::shebang .= ($ARGV[0] =~ s/^--hashbang *//);
+ if($opt::shebang) {
+ my $argfile = shell_quote_scalar(pop @ARGV);
+ # exec myself to split $ARGV[0] into separate fields
+ exec "$0 --skip-first-line -a $argfile @ARGV";
+ }
+ if($opt::shebang_wrap) {
+ my @options;
+ my @parser;
+ if ($^O eq 'freebsd') {
+ # FreeBSD's #! puts different values in @ARGV than Linux' does.
+ my @nooptions = @ARGV;
+ get_options_from_array(\@nooptions);
+ while($#ARGV > $#nooptions) {
+ push @options, shift @ARGV;
+ }
+ while(@ARGV and $ARGV[0] ne ":::") {
+ push @parser, shift @ARGV;
+ }
+ if(@ARGV and $ARGV[0] eq ":::") {
+ shift @ARGV;
+ }
+ } else {
+ @options = shift @ARGV;
+ }
+ my $script = shell_quote_scalar(shift @ARGV);
+ # exec myself to split $ARGV[0] into separate fields
+ exec "$0 --internal-pipe-means-argfiles @options @parser $script ::: @ARGV";
+ }
+ }
+
+ Getopt::Long::Configure("bundling","require_order");
+ my @ARGV_copy = @ARGV;
+ # Check if there is a --profile to set @opt::profile
+ get_options_from_array(\@ARGV_copy,"profile|J=s","plain") || die_usage();
+ my @ARGV_profile = ();
+ my @ARGV_env = ();
+ if(not $opt::plain) {
+ # Add options from .parallel/config and other profiles
+ my @config_profiles = (
+ "/etc/parallel/config",
+ $ENV{'HOME'}."/.parallel/config",
+ $ENV{'HOME'}."/.parallelrc");
+ my @profiles = @config_profiles;
+ if(@opt::profile) {
+ # --profile overrides default profiles
+ @profiles = ();
+ for my $profile (@opt::profile) {
+ if(-r $profile) {
+ push @profiles, $profile;
+ } else {
+ push @profiles, $ENV{'HOME'}."/.parallel/".$profile;
+ }
+ }
+ }
+ for my $profile (@profiles) {
+ if(-r $profile) {
+ open (my $in_fh, "<", $profile) || ::die_bug("read-profile: $profile");
+ while(<$in_fh>) {
+ /^\s*\#/ and next;
+ chomp;
+ push @ARGV_profile, shellwords($_);
+ }
+ close $in_fh;
+ } else {
+ if(grep /^$profile$/, @config_profiles) {
+ # config file is not required to exist
+ } else {
+ ::error("$profile not readable.\n");
+ wait_and_exit(255);
+ }
+ }
+ }
+ # Add options from shell variable $PARALLEL
+ if($ENV{'PARALLEL'}) {
+ @ARGV_env = shellwords($ENV{'PARALLEL'});
+ }
+ }
+ Getopt::Long::Configure("bundling","require_order");
+ get_options_from_array(\@ARGV_profile) || die_usage();
+ get_options_from_array(\@ARGV_env) || die_usage();
+ get_options_from_array(\@ARGV) || die_usage();
+
+ # Prepend non-options to @ARGV (such as commands like 'nice')
+ unshift @ARGV, @ARGV_profile, @ARGV_env;
+ return @ARGV;
+}
+
+sub read_args_from_command_line {
+ # Arguments given on the command line after:
+ # ::: ($Global::arg_sep)
+ # :::: ($Global::arg_file_sep)
+ # Removes the arguments from @ARGV and:
+ # - puts filenames into -a
+ # - puts arguments into files and add the files to -a
+ # Input:
+ # @::ARGV = command option ::: arg arg arg :::: argfiles
+ # Uses:
+ # $Global::arg_sep
+ # $Global::arg_file_sep
+ # $opt::internal_pipe_means_argfiles
+ # $opt::pipe
+ # @opt::a
+ # Returns:
+ # @argv_no_argsep = @::ARGV without ::: and :::: and following args
+ my @new_argv = ();
+ for(my $arg = shift @ARGV; @ARGV; $arg = shift @ARGV) {
+ if($arg eq $Global::arg_sep
+ or
+ $arg eq $Global::arg_file_sep) {
+ my $group = $arg; # This group of arguments is args or argfiles
+ my @group;
+ while(defined ($arg = shift @ARGV)) {
+ if($arg eq $Global::arg_sep
+ or
+ $arg eq $Global::arg_file_sep) {
+ # exit while loop if finding new separator
+ last;
+ } else {
+ # If not hitting ::: or ::::
+ # Append it to the group
+ push @group, $arg;
+ }
+ }
+
+ if($group eq $Global::arg_file_sep
+ or ($opt::internal_pipe_means_argfiles and $opt::pipe)
+ ) {
+ # Group of file names on the command line.
+ # Append args into -a
+ push @opt::a, @group;
+ } elsif($group eq $Global::arg_sep) {
+ # Group of arguments on the command line.
+ # Put them into a file.
+ # Create argfile
+ my ($outfh,$name) = ::tmpfile(SUFFIX => ".arg");
+ unlink($name);
+ # Put args into argfile
+ print $outfh map { $_,$/ } @group;
+ seek $outfh, 0, 0;
+ # Append filehandle to -a
+ push @opt::a, $outfh;
+ } else {
+ ::die_bug("Unknown command line group: $group");
+ }
+ if(defined($arg)) {
+ # $arg is ::: or ::::
+ redo;
+ } else {
+ # $arg is undef -> @ARGV empty
+ last;
+ }
+ }
+ push @new_argv, $arg;
+ }
+ # Output: @ARGV = command to run with options
+ return @new_argv;
+}
+
+sub cleanup {
+ # Returns: N/A
+ if(@opt::basefile) { cleanup_basefile(); }
+}
+
+sub __QUOTING_ARGUMENTS_FOR_SHELL__ {}
+
+sub shell_quote {
+ # Input:
+ # @strings = strings to be quoted
+ # Output:
+ # @shell_quoted_strings = string quoted with \ as needed by the shell
+ my @strings = (@_);
+ for my $a (@strings) {
+ $a =~ s/([\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\*\>\<\~\|\; \"\!\$\&\'\202-\377])/\\$1/g;
+ $a =~ s/[\n]/'\n'/g; # filenames with '\n' is quoted using \'
+ }
+ return wantarray ? @strings : "@strings";
+}
+
+sub shell_quote_empty {
+ # Inputs:
+ # @strings = strings to be quoted
+ # Returns:
+ # @quoted_strings = empty strings quoted as ''.
+ my @strings = shell_quote(@_);
+ for my $a (@strings) {
+ if($a eq "") {
+ $a = "''";
+ }
+ }
+ return wantarray ? @strings : "@strings";
+}
+
+sub shell_quote_scalar {
+ # Quote the string so shell will not expand any special chars
+ # Inputs:
+ # $string = string to be quoted
+ # Returns:
+ # $shell_quoted = string quoted with \ as needed by the shell
+ my $a = $_[0];
+ if(defined $a) {
+ # $a =~ s/([\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\*\>\<\~\|\; \"\!\$\&\'\202-\377])/\\$1/g;
+ # This is 1% faster than the above
+ $a =~ s/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\*\>\<\~\|\; \"\!\$\&\'\202-\377]/\\$&/go;
+ $a =~ s/[\n]/'\n'/go; # filenames with '\n' is quoted using \'
+ }
+ return $a;
+}
+
+sub shell_quote_file {
+ # Quote the string so shell will not expand any special chars and prepend ./ if needed
+ # Input:
+ # $filename = filename to be shell quoted
+ # Returns:
+ # $quoted_filename = filename quoted with \ as needed by the shell and ./ if needed
+ my $a = shell_quote_scalar(shift);
+ if(defined $a) {
+ if($a =~ m:^/: or $a =~ m:^\./:) {
+ # /abs/path or ./rel/path => skip
+ } else {
+ # rel/path => ./rel/path
+ $a = "./".$a;
+ }
+ }
+ return $a;
+}
+
+sub shellwords {
+ # Input:
+ # $string = shell line
+ # Returns:
+ # @shell_words = $string split into words as shell would do
+ $Global::use{"Text::ParseWords"} ||= eval "use Text::ParseWords; 1;";
+ return Text::ParseWords::shellwords(@_);
+}
+
+
+sub __FILEHANDLES__ {}
+
+
+sub save_stdin_stdout_stderr {
+ # Remember the original STDIN, STDOUT and STDERR
+ # and file descriptors opened by the shell (e.g. 3>/tmp/foo)
+ # Uses:
+ # %Global::fd
+ # $Global::original_stderr
+ # $Global::original_stdin
+ # Returns: N/A
+
+ # Find file descriptors that are already opened (by the shell)
+ for my $fdno (1..61) {
+ # /dev/fd/62 and above are used by bash for <(cmd)
+ my $fh;
+ # 2-argument-open is used to be compatible with old perl 5.8.0
+ # bug #43570: Perl 5.8.0 creates 61 files
+ if(open($fh,">&=$fdno")) {
+ $Global::fd{$fdno}=$fh;
+ }
+ }
+ open $Global::original_stderr, ">&", "STDERR" or
+ ::die_bug("Can't dup STDERR: $!");
+ open $Global::original_stdin, "<&", "STDIN" or
+ ::die_bug("Can't dup STDIN: $!");
+}
+
+sub enough_file_handles {
+ # Check that we have enough filehandles available for starting
+ # another job
+ # Uses:
+ # $opt::ungroup
+ # %Global::fd
+ # Returns:
+ # 1 if ungrouped (thus not needing extra filehandles)
+ # 0 if too few filehandles
+ # 1 if enough filehandles
+ if(not $opt::ungroup) {
+ my %fh;
+ my $enough_filehandles = 1;
+ # perl uses 7 filehandles for something?
+ # open3 uses 2 extra filehandles temporarily
+ # We need a filehandle for each redirected file descriptor
+ # (normally just STDOUT and STDERR)
+ for my $i (1..(7+2+keys %Global::fd)) {
+ $enough_filehandles &&= open($fh{$i}, "<", "/dev/null");
+ }
+ for (values %fh) { close $_; }
+ return $enough_filehandles;
+ } else {
+ # Ungrouped does not need extra file handles
+ return 1;
+ }
+}
+
+sub open_or_exit {
+ # Open a file name or exit if the file cannot be opened
+ # Inputs:
+ # $file = filehandle or filename to open
+ # Uses:
+ # $Global::stdin_in_opt_a
+ # $Global::original_stdin
+ # Returns:
+ # $fh = file handle to read-opened file
+ my $file = shift;
+ if($file eq "-") {
+ $Global::stdin_in_opt_a = 1;
+ return ($Global::original_stdin || *STDIN);
+ }
+ if(ref $file eq "GLOB") {
+ # This is an open filehandle
+ return $file;
+ }
+ my $fh = gensym;
+ if(not open($fh, "<", $file)) {
+ ::error("Cannot open input file `$file': No such file or directory.\n");
+ wait_and_exit(255);
+ }
+ return $fh;
+}
+
+sub __RUNNING_THE_JOBS_AND_PRINTING_PROGRESS__ {}
+
+# Variable structure:
+#
+# $Global::running{$pid} = Pointer to Job-object
+# @Global::virgin_jobs = Pointer to Job-object that have received no input
+# $Global::host{$sshlogin} = Pointer to SSHLogin-object
+# $Global::total_running = total number of running jobs
+# $Global::total_started = total jobs started
+
+sub init_run_jobs {
+ $Global::total_running = 0;
+ $Global::total_started = 0;
+ $Global::tty_taken = 0;
+ $SIG{USR1} = \&list_running_jobs;
+ $SIG{USR2} = \&toggle_progress;
+ if(@opt::basefile) { setup_basefile(); }
+}
+
+{
+ my $last_time;
+ my %last_mtime;
+
+sub start_more_jobs {
+ # Run start_another_job() but only if:
+ # * not $Global::start_no_new_jobs set
+ # * not JobQueue is empty
+ # * not load on server is too high
+ # * not server swapping
+ # * not too short time since last remote login
+ # Uses:
+ # $Global::max_procs_file
+ # $Global::max_procs_file_last_mod
+ # %Global::host
+ # @opt::sshloginfile
+ # $Global::start_no_new_jobs
+ # $opt::filter_hosts
+ # $Global::JobQueue
+ # $opt::pipe
+ # $opt::load
+ # $opt::noswap
+ # $opt::delay
+ # $Global::newest_starttime
+ # Returns:
+ # $jobs_started = number of jobs started
+ my $jobs_started = 0;
+ my $jobs_started_this_round = 0;
+ if($Global::start_no_new_jobs) {
+ return $jobs_started;
+ }
+ if(time - ($last_time||0) > 1) {
+ # At most do this every second
+ $last_time = time;
+ if($Global::max_procs_file) {
+ # --jobs filename
+ my $mtime = (stat($Global::max_procs_file))[9];
+ if($mtime > $Global::max_procs_file_last_mod) {
+ # file changed: Force re-computing max_jobs_running
+ $Global::max_procs_file_last_mod = $mtime;
+ for my $sshlogin (values %Global::host) {
+ $sshlogin->set_max_jobs_running(undef);
+ }
+ }
+ }
+ if(@opt::sshloginfile) {
+ # Is --sshloginfile changed?
+ for my $slf (@opt::sshloginfile) {
+ my $actual_file = expand_slf_shorthand($slf);
+ my $mtime = (stat($actual_file))[9];
+ $last_mtime{$actual_file} ||= $mtime;
+ if($mtime - $last_mtime{$actual_file} > 1) {
+ ::debug("run","--sshloginfile $actual_file changed. reload\n");
+ $last_mtime{$actual_file} = $mtime;
+ # Reload $slf
+ # Empty sshlogins
+ @Global::sshlogin = ();
+ for (values %Global::host) {
+ # Don't start new jobs on any host
+ # except the ones added back later
+ $_->set_max_jobs_running(0);
+ }
+ # This will set max_jobs_running on the SSHlogins
+ read_sshloginfile($actual_file);
+ parse_sshlogin();
+ $opt::filter_hosts and filter_hosts();
+ setup_basefile();
+ }
+ }
+ }
+ }
+ do {
+ $jobs_started_this_round = 0;
+ # This will start 1 job on each --sshlogin (if possible)
+ # thus distribute the jobs on the --sshlogins round robin
+
+ for my $sshlogin (values %Global::host) {
+ if($Global::JobQueue->empty() and not $opt::pipe) {
+ # No more jobs in the queue
+ last;
+ }
+ debug("run", "Running jobs before on ", $sshlogin->string(), ": ",
+ $sshlogin->jobs_running(), "\n");
+ if ($sshlogin->jobs_running() < $sshlogin->max_jobs_running()) {
+ if($opt::load and $sshlogin->loadavg_too_high()) {
+ # The load is too high or unknown
+ next;
+ }
+ if($opt::noswap and $sshlogin->swapping()) {
+ # The server is swapping
+ next;
+ }
+ if($sshlogin->too_fast_remote_login()) {
+ # It has been too short since
+ next;
+ }
+ if($opt::delay and $opt::delay > ::now() - $Global::newest_starttime) {
+ # It has been too short since last start
+ next;
+ }
+ debug("run", $sshlogin->string(), " has ", $sshlogin->jobs_running(),
+ " out of ", $sshlogin->max_jobs_running(),
+ " jobs running. Start another.\n");
+ if(start_another_job($sshlogin) == 0) {
+ # No more jobs to start on this $sshlogin
+ debug("run","No jobs started on ", $sshlogin->string(), "\n");
+ next;
+ }
+ $sshlogin->inc_jobs_running();
+ $sshlogin->set_last_login_at(::now());
+ $jobs_started++;
+ $jobs_started_this_round++;
+ }
+ debug("run","Running jobs after on ", $sshlogin->string(), ": ",
+ $sshlogin->jobs_running(), " of ",
+ $sshlogin->max_jobs_running(), "\n");
+ }
+ } while($jobs_started_this_round);
+
+ return $jobs_started;
+}
+}
+
+{
+ my $no_more_file_handles_warned;
+
+sub start_another_job {
+ # If there are enough filehandles
+ # and JobQueue not empty
+ # and not $job is in joblog
+ # Then grab a job from Global::JobQueue,
+ # start it at sshlogin
+ # mark it as virgin_job
+ # Inputs:
+ # $sshlogin = the SSHLogin to start the job on
+ # Uses:
+ # $Global::JobQueue
+ # $opt::pipe
+ # $opt::results
+ # $opt::resume
+ # @Global::virgin_jobs
+ # Returns:
+ # 1 if another jobs was started
+ # 0 otherwise
+ my $sshlogin = shift;
+ # Do we have enough file handles to start another job?
+ if(enough_file_handles()) {
+ if($Global::JobQueue->empty() and not $opt::pipe) {
+ # No more commands to run
+ debug("start", "Not starting: JobQueue empty\n");
+ return 0;
+ } else {
+ my $job;
+ # Skip jobs already in job log
+ # Skip jobs already in results
+ do {
+ $job = get_job_with_sshlogin($sshlogin);
+ if(not defined $job) {
+ # No command available for that sshlogin
+ debug("start", "Not starting: no jobs available for ",
+ $sshlogin->string(), "\n");
+ return 0;
+ }
+ } while ($job->is_already_in_joblog()
+ or
+ ($opt::results and $opt::resume and $job->is_already_in_results()));
+ debug("start", "Command to run on '", $job->sshlogin()->string(), "': '",
+ $job->replaced(),"'\n");
+ if($job->start()) {
+ if($opt::pipe) {
+ push(@Global::virgin_jobs,$job);
+ }
+ debug("start", "Started as seq ", $job->seq(),
+ " pid:", $job->pid(), "\n");
+ return 1;
+ } else {
+ # Not enough processes to run the job.
+ # Put it back on the queue.
+ $Global::JobQueue->unget($job);
+ # Count down the number of jobs to run for this SSHLogin.
+ my $max = $sshlogin->max_jobs_running();
+ if($max > 1) { $max--; } else {
+ ::error("No more processes: cannot run a single job. Something is wrong.\n");
+ ::wait_and_exit(255);
+ }
+ $sshlogin->set_max_jobs_running($max);
+ # Sleep up to 300 ms to give other processes time to die
+ ::usleep(rand()*300);
+ ::warning("No more processes: ",
+ "Decreasing number of running jobs to $max. ",
+ "Raising ulimit -u or /etc/security/limits.conf may help.\n");
+ return 0;
+ }
+ }
+ } else {
+ # No more file handles
+ $no_more_file_handles_warned++ or
+ ::warning("No more file handles. ",
+ "Raising ulimit -n or /etc/security/limits.conf may help.\n");
+ return 0;
+ }
+}
+}
+
+sub init_progress {
+ # Uses:
+ # $opt::bar
+ # Returns:
+ # list of computers for progress output
+ $|=1;
+ if($opt::bar) {
+ return("","");
+ }
+ my %progress = progress();
+ return ("\nComputers / CPU cores / Max jobs to run\n",
+ $progress{'workerlist'});
+}
+
+sub drain_job_queue {
+ # Uses:
+ # $opt::progress
+ # $Global::original_stderr
+ # $Global::total_running
+ # $Global::max_jobs_running
+ # %Global::running
+ # $Global::JobQueue
+ # %Global::host
+ # $Global::start_no_new_jobs
+ # Returns: N/A
+ if($opt::progress) {
+ print $Global::original_stderr init_progress();
+ }
+ my $last_header="";
+ my $sleep = 0.2;
+ do {
+ while($Global::total_running > 0) {
+ debug($Global::total_running, "==", scalar
+ keys %Global::running," slots: ", $Global::max_jobs_running);
+ if($opt::pipe) {
+ # When using --pipe sometimes file handles are not closed properly
+ for my $job (values %Global::running) {
+ close $job->fh(0,"w");
+ }
+ }
+ if($opt::progress) {
+ my %progress = progress();
+ if($last_header ne $progress{'header'}) {
+ print $Global::original_stderr "\n", $progress{'header'}, "\n";
+ $last_header = $progress{'header'};
+ }
+ print $Global::original_stderr "\r",$progress{'status'};
+ flush $Global::original_stderr;
+ }
+ if($Global::total_running < $Global::max_jobs_running
+ and not $Global::JobQueue->empty()) {
+ # These jobs may not be started because of loadavg
+ # or too little time between each ssh login.
+ if(start_more_jobs() > 0) {
+ # Exponential back-on if jobs were started
+ $sleep = $sleep/2+0.001;
+ }
+ }
+ # Sometimes SIGCHLD is not registered, so force reaper
+ $sleep = ::reap_usleep($sleep);
+ }
+ if(not $Global::JobQueue->empty()) {
+ # These jobs may not be started:
+ # * because there the --filter-hosts has removed all
+ if(not %Global::host) {
+ ::error("There are no hosts left to run on.\n");
+ ::wait_and_exit(255);
+ }
+ # * because of loadavg
+ # * because of too little time between each ssh login.
+ start_more_jobs();
+ $sleep = ::reap_usleep($sleep);
+ if($Global::max_jobs_running == 0) {
+ ::warning("There are no job slots available. Increase --jobs.\n");
+ }
+ }
+ } while ($Global::total_running > 0
+ or
+ not $Global::start_no_new_jobs and not $Global::JobQueue->empty());
+ if($opt::progress) {
+ my %progress = progress();
+ print $Global::original_stderr "\r", $progress{'status'}, "\n";
+ flush $Global::original_stderr;
+ }
+}
+
+sub toggle_progress {
+ # Turn on/off progress view
+ # Uses:
+ # $opt::progress
+ # $Global::original_stderr
+ # Returns: N/A
+ $opt::progress = not $opt::progress;
+ if($opt::progress) {
+ print $Global::original_stderr init_progress();
+ }
+}
+
+sub progress {
+ # Uses:
+ # $opt::bar
+ # $opt::eta
+ # %Global::host
+ # $Global::total_started
+ # Returns:
+ # $workerlist = list of workers
+ # $header = that will fit on the screen
+ # $status = message that will fit on the screen
+ if($opt::bar) {
+ return ("workerlist" => "", "header" => "", "status" => bar());
+ }
+ my $eta = "";
+ my ($status,$header)=("","");
+ if($opt::eta) {
+ my($total, $completed, $left, $pctcomplete, $avgtime, $this_eta) =
+ compute_eta();
+ $eta = sprintf("ETA: %ds Left: %d AVG: %.2fs ",
+ $this_eta, $left, $avgtime);
+ }
+ my $termcols = terminal_columns();
+ my @workers = sort keys %Global::host;
+ my %sshlogin = map { $_ eq ":" ? ($_=>"local") : ($_=>$_) } @workers;
+ my $workerno = 1;
+ my %workerno = map { ($_=>$workerno++) } @workers;
+ my $workerlist = "";
+ for my $w (@workers) {
+ $workerlist .=
+ $workerno{$w}.":".$sshlogin{$w} ." / ".
+ ($Global::host{$w}->ncpus() || "-")." / ".
+ $Global::host{$w}->max_jobs_running()."\n";
+ }
+ $status = "x"x($termcols+1);
+ if(length $status > $termcols) {
+ # sshlogin1:XX/XX/XX%/XX.Xs sshlogin2:XX/XX/XX%/XX.Xs sshlogin3:XX/XX/XX%/XX.Xs
+ $header = "Computer:jobs running/jobs completed/%of started jobs/Average seconds to complete";
+ $status = $eta .
+ join(" ",map
+ {
+ if($Global::total_started) {
+ my $completed = ($Global::host{$_}->jobs_completed()||0);
+ my $running = $Global::host{$_}->jobs_running();
+ my $time = $completed ? (time-$^T)/($completed) : "0";
+ sprintf("%s:%d/%d/%d%%/%.1fs ",
+ $sshlogin{$_}, $running, $completed,
+ ($running+$completed)*100
+ / $Global::total_started, $time);
+ }
+ } @workers);
+ }
+ if(length $status > $termcols) {
+ # 1:XX/XX/XX%/XX.Xs 2:XX/XX/XX%/XX.Xs 3:XX/XX/XX%/XX.Xs 4:XX/XX/XX%/XX.Xs
+ $header = "Computer:jobs running/jobs completed/%of started jobs";
+ $status = $eta .
+ join(" ",map
+ {
+ my $completed = ($Global::host{$_}->jobs_completed()||0);
+ my $running = $Global::host{$_}->jobs_running();
+ my $time = $completed ? (time-$^T)/($completed) : "0";
+ sprintf("%s:%d/%d/%d%%/%.1fs ",
+ $workerno{$_}, $running, $completed,
+ ($running+$completed)*100
+ / $Global::total_started, $time);
+ } @workers);
+ }
+ if(length $status > $termcols) {
+ # sshlogin1:XX/XX/XX% sshlogin2:XX/XX/XX% sshlogin3:XX/XX/XX%
+ $header = "Computer:jobs running/jobs completed/%of started jobs";
+ $status = $eta .
+ join(" ",map
+ { sprintf("%s:%d/%d/%d%%",
+ $sshlogin{$_},
+ $Global::host{$_}->jobs_running(),
+ ($Global::host{$_}->jobs_completed()||0),
+ ($Global::host{$_}->jobs_running()+
+ ($Global::host{$_}->jobs_completed()||0))*100
+ / $Global::total_started) }
+ @workers);
+ }
+ if(length $status > $termcols) {
+ # 1:XX/XX/XX% 2:XX/XX/XX% 3:XX/XX/XX% 4:XX/XX/XX% 5:XX/XX/XX% 6:XX/XX/XX%
+ $header = "Computer:jobs running/jobs completed/%of started jobs";
+ $status = $eta .
+ join(" ",map
+ { sprintf("%s:%d/%d/%d%%",
+ $workerno{$_},
+ $Global::host{$_}->jobs_running(),
+ ($Global::host{$_}->jobs_completed()||0),
+ ($Global::host{$_}->jobs_running()+
+ ($Global::host{$_}->jobs_completed()||0))*100
+ / $Global::total_started) }
+ @workers);
+ }
+ if(length $status > $termcols) {
+ # sshlogin1:XX/XX/XX% sshlogin2:XX/XX/XX% sshlogin3:XX/XX sshlogin4:XX/XX
+ $header = "Computer:jobs running/jobs completed";
+ $status = $eta .
+ join(" ",map
+ { sprintf("%s:%d/%d",
+ $sshlogin{$_}, $Global::host{$_}->jobs_running(),
+ ($Global::host{$_}->jobs_completed()||0)) }
+ @workers);
+ }
+ if(length $status > $termcols) {
+ # sshlogin1:XX/XX sshlogin2:XX/XX sshlogin3:XX/XX sshlogin4:XX/XX
+ $header = "Computer:jobs running/jobs completed";
+ $status = $eta .
+ join(" ",map
+ { sprintf("%s:%d/%d",
+ $sshlogin{$_}, $Global::host{$_}->jobs_running(),
+ ($Global::host{$_}->jobs_completed()||0)) }
+ @workers);
+ }
+ if(length $status > $termcols) {
+ # 1:XX/XX 2:XX/XX 3:XX/XX 4:XX/XX 5:XX/XX 6:XX/XX
+ $header = "Computer:jobs running/jobs completed";
+ $status = $eta .
+ join(" ",map
+ { sprintf("%s:%d/%d",
+ $workerno{$_}, $Global::host{$_}->jobs_running(),
+ ($Global::host{$_}->jobs_completed()||0)) }
+ @workers);
+ }
+ if(length $status > $termcols) {
+ # sshlogin1:XX sshlogin2:XX sshlogin3:XX sshlogin4:XX sshlogin5:XX
+ $header = "Computer:jobs completed";
+ $status = $eta .
+ join(" ",map
+ { sprintf("%s:%d",
+ $sshlogin{$_},
+ ($Global::host{$_}->jobs_completed()||0)) }
+ @workers);
+ }
+ if(length $status > $termcols) {
+ # 1:XX 2:XX 3:XX 4:XX 5:XX 6:XX
+ $header = "Computer:jobs completed";
+ $status = $eta .
+ join(" ",map
+ { sprintf("%s:%d",
+ $workerno{$_},
+ ($Global::host{$_}->jobs_completed()||0)) }
+ @workers);
+ }
+ return ("workerlist" => $workerlist, "header" => $header, "status" => $status);
+}
+
+{
+ my ($total, $first_completed, $smoothed_avg_time);
+
+ sub compute_eta {
+ # Calculate important numbers for ETA
+ # Returns:
+ # $total = number of jobs in total
+ # $completed = number of jobs completed
+ # $left = number of jobs left
+ # $pctcomplete = percent of jobs completed
+ # $avgtime = averaged time
+ # $eta = smoothed eta
+ $total ||= $Global::JobQueue->total_jobs();
+ my $completed = 0;
+ for(values %Global::host) { $completed += $_->jobs_completed() }
+ my $left = $total - $completed;
+ if(not $completed) {
+ return($total, $completed, $left, 0, 0, 0);
+ }
+ my $pctcomplete = $completed / $total;
+ $first_completed ||= time;
+ my $timepassed = (time - $first_completed);
+ my $avgtime = $timepassed / $completed;
+ $smoothed_avg_time ||= $avgtime;
+ # Smooth the eta so it does not jump wildly
+ $smoothed_avg_time = (1 - $pctcomplete) * $smoothed_avg_time +
+ $pctcomplete * $avgtime;
+ my $eta = int($left * $smoothed_avg_time);
+ return($total, $completed, $left, $pctcomplete, $avgtime, $eta);
+ }
+}
+
+{
+ my ($rev,$reset);
+
+ sub bar {
+ # Return:
+ # $status = bar with eta, completed jobs, arg and pct
+ $rev ||= "\033[7m";
+ $reset ||= "\033[0m";
+ my($total, $completed, $left, $pctcomplete, $avgtime, $eta) =
+ compute_eta();
+ my $arg = $Global::newest_job ?
+ $Global::newest_job->{'commandline'}->replace_placeholders(["\257<\257>"],0,0) : "";
+ # These chars mess up display in the terminal
+ $arg =~ tr/[\011-\016\033\302-\365]//d;
+ my $bar_text =
+ sprintf("%d%% %d:%d=%ds %s",
+ $pctcomplete*100, $completed, $left, $eta, $arg);
+ my $terminal_width = terminal_columns();
+ my $s = sprintf("%-${terminal_width}s",
+ substr($bar_text." "x$terminal_width,
+ 0,$terminal_width));
+ my $width = int($terminal_width * $pctcomplete);
+ substr($s,$width,0) = $reset;
+ my $zenity = sprintf("%-${terminal_width}s",
+ substr("# $eta sec $arg",
+ 0,$terminal_width));
+ $s = "\r" . $zenity . "\r" . $pctcomplete*100 . # Prefix with zenity header
+ "\r" . $rev . $s . $reset;
+ return $s;
+ }
+}
+
+{
+ my ($columns,$last_column_time);
+
+ sub terminal_columns {
+ # Get the number of columns of the display
+ # Returns:
+ # number of columns of the screen
+ if(not $columns or $last_column_time < time) {
+ $last_column_time = time;
+ $columns = $ENV{'COLUMNS'};
+ if(not $columns) {
+ my $resize = qx{ resize 2>/dev/null };
+ $resize =~ /COLUMNS=(\d+);/ and do { $columns = $1; };
+ }
+ $columns ||= 80;
+ }
+ return $columns;
+ }
+}
+
+sub get_job_with_sshlogin {
+ # Returns:
+ # next job object for $sshlogin if any available
+ my $sshlogin = shift;
+ my $job = undef;
+
+ if ($opt::hostgroups) {
+ my @other_hostgroup_jobs = ();
+
+ while($job = $Global::JobQueue->get()) {
+ if($sshlogin->in_hostgroups($job->hostgroups())) {
+ # Found a job for this hostgroup
+ last;
+ } else {
+ # This job was not in the hostgroups of $sshlogin
+ push @other_hostgroup_jobs, $job;
+ }
+ }
+ $Global::JobQueue->unget(@other_hostgroup_jobs);
+ if(not defined $job) {
+ # No more jobs
+ return undef;
+ }
+ } else {
+ $job = $Global::JobQueue->get();
+ if(not defined $job) {
+ # No more jobs
+ ::debug("start", "No more jobs: JobQueue empty\n");
+ return undef;
+ }
+ }
+
+ my $clean_command = $job->replaced();
+ if($clean_command =~ /^\s*$/) {
+ # Do not run empty lines
+ if(not $Global::JobQueue->empty()) {
+ return get_job_with_sshlogin($sshlogin);
+ } else {
+ return undef;
+ }
+ }
+ $job->set_sshlogin($sshlogin);
+ if($opt::retries and $clean_command and
+ $job->failed_here()) {
+ # This command with these args failed for this sshlogin
+ my ($no_of_failed_sshlogins,$min_failures) = $job->min_failed();
+ # Only look at the Global::host that have > 0 jobslots
+ if($no_of_failed_sshlogins == grep { $_->max_jobs_running() > 0 } values %Global::host
+ and $job->failed_here() == $min_failures) {
+ # It failed the same or more times on another host:
+ # run it on this host
+ } else {
+ # If it failed fewer times on another host:
+ # Find another job to run
+ my $nextjob;
+ if(not $Global::JobQueue->empty()) {
+ # This can potentially recurse for all args
+ no warnings 'recursion';
+ $nextjob = get_job_with_sshlogin($sshlogin);
+ }
+ # Push the command back on the queue
+ $Global::JobQueue->unget($job);
+ return $nextjob;
+ }
+ }
+ return $job;
+}
+
+sub __REMOTE_SSH__ {}
+
+sub read_sshloginfiles {
+ # Returns: N/A
+ for my $s (@_) {
+ read_sshloginfile(expand_slf_shorthand($s));
+ }
+}
+
+sub expand_slf_shorthand {
+ my $file = shift;
+ if($file eq "-") {
+ # skip: It is stdin
+ } elsif($file eq "..") {
+ $file = $ENV{'HOME'}."/.parallel/sshloginfile";
+ } elsif($file eq ".") {
+ $file = "/etc/parallel/sshloginfile";
+ } elsif(not -r $file) {
+ if(not -r $ENV{'HOME'}."/.parallel/".$file) {
+ # Try prepending ~/.parallel
+ ::error("Cannot open $file.\n");
+ ::wait_and_exit(255);
+ } else {
+ $file = $ENV{'HOME'}."/.parallel/".$file;
+ }
+ }
+ return $file;
+}
+
+sub read_sshloginfile {
+ # Returns: N/A
+ my $file = shift;
+ my $close = 1;
+ my $in_fh;
+ ::debug("init","--slf ",$file);
+ if($file eq "-") {
+ $in_fh = *STDIN;
+ $close = 0;
+ } else {
+ if(not open($in_fh, "<", $file)) {
+ # Try the filename
+ ::error("Cannot open $file.\n");
+ ::wait_and_exit(255);
+ }
+ }
+ while(<$in_fh>) {
+ chomp;
+ /^\s*#/ and next;
+ /^\s*$/ and next;
+ push @Global::sshlogin, $_;
+ }
+ if($close) {
+ close $in_fh;
+ }
+}
+
+sub parse_sshlogin {
+ # Returns: N/A
+ my @login;
+ if(not @Global::sshlogin) { @Global::sshlogin = (":"); }
+ for my $sshlogin (@Global::sshlogin) {
+ # Split up -S sshlogin,sshlogin
+ for my $s (split /,/, $sshlogin) {
+ if ($s eq ".." or $s eq "-") {
+ # This may add to @Global::sshlogin - possibly bug
+ read_sshloginfile(expand_slf_shorthand($s));
+ } else {
+ push (@login, $s);
+ }
+ }
+ }
+ $Global::minimal_command_line_length = 8_000_000;
+ my @allowed_hostgroups;
+ for my $ncpu_sshlogin_string (::uniq(@login)) {
+ my $sshlogin = SSHLogin->new($ncpu_sshlogin_string);
+ my $sshlogin_string = $sshlogin->string();
+ if($sshlogin_string eq "") {
+ # This is an ssh group: -S @webservers
+ push @allowed_hostgroups, $sshlogin->hostgroups();
+ next;
+ }
+ if($Global::host{$sshlogin_string}) {
+ # This sshlogin has already been added:
+ # It is probably a host that has come back
+ # Set the max_jobs_running back to the original
+ debug("run","Already seen $sshlogin_string\n");
+ if($sshlogin->{'ncpus'}) {
+ # If ncpus set by '#/' of the sshlogin, overwrite it:
+ $Global::host{$sshlogin_string}->set_ncpus($sshlogin->ncpus());
+ }
+ $Global::host{$sshlogin_string}->set_max_jobs_running(undef);
+ next;
+ }
+ if($sshlogin_string eq ":") {
+ $sshlogin->set_maxlength(Limits::Command::max_length());
+ } else {
+ # If all chars needs to be quoted, every other character will be \
+ $sshlogin->set_maxlength(int(Limits::Command::max_length()/2));
+ }
+ $Global::minimal_command_line_length =
+ ::min($Global::minimal_command_line_length, $sshlogin->maxlength());
+ $Global::host{$sshlogin_string} = $sshlogin;
+ }
+ if(@allowed_hostgroups) {
+ # Remove hosts that are not in these groups
+ while (my ($string, $sshlogin) = each %Global::host) {
+ if(not $sshlogin->in_hostgroups(@allowed_hostgroups)) {
+ delete $Global::host{$string};
+ }
+ }
+ }
+
+ # debug("start", "sshlogin: ", my_dump(%Global::host),"\n");
+ if($opt::transfer or @opt::return or $opt::cleanup or @opt::basefile) {
+ if(not remote_hosts()) {
+ # There are no remote hosts
+ if(@opt::trc) {
+ ::warning("--trc ignored as there are no remote --sshlogin.\n");
+ } elsif (defined $opt::transfer) {
+ ::warning("--transfer ignored as there are no remote --sshlogin.\n");
+ } elsif (@opt::return) {
+ ::warning("--return ignored as there are no remote --sshlogin.\n");
+ } elsif (defined $opt::cleanup) {
+ ::warning("--cleanup ignored as there are no remote --sshlogin.\n");
+ } elsif (@opt::basefile) {
+ ::warning("--basefile ignored as there are no remote --sshlogin.\n");
+ }
+ }
+ }
+}
+
+sub remote_hosts {
+ # Return sshlogins that are not ':'
+ # Returns:
+ # list of sshlogins with ':' removed
+ return grep !/^:$/, keys %Global::host;
+}
+
+sub setup_basefile {
+ # Transfer basefiles to each $sshlogin
+ # This needs to be done before first jobs on $sshlogin is run
+ # Returns: N/A
+ my $cmd = "";
+ my $rsync_destdir;
+ my $workdir;
+ for my $sshlogin (values %Global::host) {
+ if($sshlogin->string() eq ":") { next }
+ for my $file (@opt::basefile) {
+ if($file !~ m:^/: and $opt::workdir eq "...") {
+ ::error("Work dir '...' will not work with relative basefiles\n");
+ ::wait_and_exit(255);
+ }
+ $workdir ||= Job->new("")->workdir();
+ $cmd .= $sshlogin->rsync_transfer_cmd($file,$workdir) . "&";
+ }
+ }
+ $cmd .= "wait;";
+ debug("init", "basesetup: $cmd\n");
+ print `$cmd`;
+}
+
+sub cleanup_basefile {
+ # Remove the basefiles transferred
+ # Returns: N/A
+ my $cmd="";
+ my $workdir = Job->new("")->workdir();
+ for my $sshlogin (values %Global::host) {
+ if($sshlogin->string() eq ":") { next }
+ for my $file (@opt::basefile) {
+ $cmd .= $sshlogin->cleanup_cmd($file,$workdir)."&";
+ }
+ }
+ $cmd .= "wait;";
+ debug("init", "basecleanup: $cmd\n");
+ print `$cmd`;
+}
+
+sub filter_hosts {
+ my(@cores, @cpus, @maxline, @echo);
+ my $envvar = ::shell_quote_scalar($Global::envvar);
+ while (my ($host, $sshlogin) = each %Global::host) {
+ if($host eq ":") { next }
+ # The 'true' is used to get the $host out later
+ my $sshcmd = "true $host;" . $sshlogin->sshcommand()." ".$sshlogin->serverlogin();
+ push(@cores, $host."\t".$sshcmd." ".$envvar." parallel --number-of-cores\n\0");
+ push(@cpus, $host."\t".$sshcmd." ".$envvar." parallel --number-of-cpus\n\0");
+ push(@maxline, $host."\t".$sshcmd." ".$envvar." parallel --max-line-length-allowed\n\0");
+ # 'echo' is used to get the best possible value for an ssh login time
+ push(@echo, $host."\t".$sshcmd." echo\n\0");
+ }
+ my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".ssh");
+ print $fh @cores, @cpus, @maxline, @echo;
+ close $fh;
+ # --timeout 5: Setting up an SSH connection and running a simple
+ # command should never take > 5 sec.
+ # --delay 0.1: If multiple sshlogins use the same proxy the delay
+ # will make it less likely to overload the ssh daemon.
+ # --retries 3: If the ssh daemon it overloaded, try 3 times
+ # -s 16000: Half of the max line on UnixWare
+ my $cmd = "cat $tmpfile | $0 -j0 --timeout 5 -s 16000 --joblog - --plain --delay 0.1 --retries 3 --tag --tagstring {1} -0 --colsep '\t' -k eval {2} 2>/dev/null";
+ ::debug("init", $cmd, "\n");
+ open(my $host_fh, "-|", $cmd) || ::die_bug("parallel host check: $cmd");
+ my (%ncores, %ncpus, %time_to_login, %maxlen, %echo, @down_hosts);
+ my $prepend = "";
+ while(<$host_fh>) {
+ if(/\'$/) {
+ # if last char = ' then append next line
+ # This may be due to quoting of $Global::envvar
+ $prepend .= $_;
+ next;
+ }
+ $_ = $prepend . $_;
+ $prepend = "";
+ chomp;
+ my @col = split /\t/, $_;
+ if(defined $col[6]) {
+ # This is a line from --joblog
+ # seq host time spent sent received exit signal command
+ # 2 : 1372607672.654 0.675 0 0 0 0 eval true\ m\;ssh\ m\ parallel\ --number-of-cores
+ if($col[0] eq "Seq" and $col[1] eq "Host" and
+ $col[2] eq "Starttime") {
+ # Header => skip
+ next;
+ }
+ # Get server from: eval true server\;
+ $col[8] =~ /eval true..([^;]+).;/ or ::die_bug("col8 does not contain host: $col[8]");
+ my $host = $1;
+ $host =~ tr/\\//d;
+ $Global::host{$host} or next;
+ if($col[6] eq "255" or $col[7] eq "15") {
+ # exit == 255 or signal == 15: ssh failed
+ # Remove sshlogin
+ ::debug("init", "--filtered $host\n");
+ push(@down_hosts, $host);
+ @down_hosts = uniq(@down_hosts);
+ } elsif($col[6] eq "127") {
+ # signal == 127: parallel not installed remote
+ # Set ncpus and ncores = 1
+ ::warning("Could not figure out ",
+ "number of cpus on $host. Using 1.\n");
+ $ncores{$host} = 1;
+ $ncpus{$host} = 1;
+ $maxlen{$host} = Limits::Command::max_length();
+ } elsif($col[0] =~ /^\d+$/ and $Global::host{$host}) {
+ # Remember how log it took to log in
+ # 2 : 1372607672.654 0.675 0 0 0 0 eval true\ m\;ssh\ m\ echo
+ $time_to_login{$host} = ::min($time_to_login{$host},$col[3]);
+ } else {
+ ::die_bug("host check unmatched long jobline: $_");
+ }
+ } elsif($Global::host{$col[0]}) {
+ # This output from --number-of-cores, --number-of-cpus,
+ # --max-line-length-allowed
+ # ncores: server 8
+ # ncpus: server 2
+ # maxlen: server 131071
+ if(not $ncores{$col[0]}) {
+ $ncores{$col[0]} = $col[1];
+ } elsif(not $ncpus{$col[0]}) {
+ $ncpus{$col[0]} = $col[1];
+ } elsif(not $maxlen{$col[0]}) {
+ $maxlen{$col[0]} = $col[1];
+ } elsif(not $echo{$col[0]}) {
+ $echo{$col[0]} = $col[1];
+ } elsif(m/perl: warning:|LANGUAGE =|LC_ALL =|LANG =|are supported and installed/) {
+ # Skip these:
+ # perl: warning: Setting locale failed.
+ # perl: warning: Please check that your locale settings:
+ # LANGUAGE = (unset),
+ # LC_ALL = (unset),
+ # LANG = "en_US.UTF-8"
+ # are supported and installed on your system.
+ # perl: warning: Falling back to the standard locale ("C").
+ } else {
+ ::die_bug("host check too many col0: $_");
+ }
+ } else {
+ ::die_bug("host check unmatched short jobline ($col[0]): $_");
+ }
+ }
+ close $host_fh;
+ $Global::debug or unlink $tmpfile;
+ delete @Global::host{@down_hosts};
+ @down_hosts and ::warning("Removed @down_hosts\n");
+ $Global::minimal_command_line_length = 8_000_000;
+ while (my ($sshlogin, $obj) = each %Global::host) {
+ if($sshlogin eq ":") { next }
+ $ncpus{$sshlogin} or ::die_bug("ncpus missing: ".$obj->serverlogin());
+ $ncores{$sshlogin} or ::die_bug("ncores missing: ".$obj->serverlogin());
+ $time_to_login{$sshlogin} or ::die_bug("time_to_login missing: ".$obj->serverlogin());
+ $maxlen{$sshlogin} or ::die_bug("maxlen missing: ".$obj->serverlogin());
+ if($opt::use_cpus_instead_of_cores) {
+ $obj->set_ncpus($ncpus{$sshlogin});
+ } else {
+ $obj->set_ncpus($ncores{$sshlogin});
+ }
+ $obj->set_time_to_login($time_to_login{$sshlogin});
+ $obj->set_maxlength($maxlen{$sshlogin});
+ $Global::minimal_command_line_length =
+ ::min($Global::minimal_command_line_length,
+ int($maxlen{$sshlogin}/2));
+ ::debug("init", "Timing from -S:$sshlogin ncpus:",$ncpus{$sshlogin},
+ " ncores:", $ncores{$sshlogin},
+ " time_to_login:", $time_to_login{$sshlogin},
+ " maxlen:", $maxlen{$sshlogin},
+ " min_max_len:", $Global::minimal_command_line_length,"\n");
+ }
+}
+
+sub onall {
+ sub tmp_joblog {
+ my $joblog = shift;
+ if(not defined $joblog) {
+ return undef;
+ }
+ my ($fh, $tmpfile) = ::tmpfile(SUFFIX => ".log");
+ close $fh;
+ return $tmpfile;
+ }
+ my @command = @_;
+ if($Global::quoting) {
+ @command = shell_quote_empty(@command);
+ }
+
+ # Copy all @fhlist into tempfiles
+ my @argfiles = ();
+ for my $fh (@fhlist) {
+ my ($outfh, $name) = ::tmpfile(SUFFIX => ".all", UNLINK => 1);
+ print $outfh (<$fh>);
+ close $outfh;
+ push @argfiles, $name;
+ }
+ if(@opt::basefile) { setup_basefile(); }
+ # for each sshlogin do:
+ # parallel -S $sshlogin $command :::: @argfiles
+ #
+ # Pass some of the options to the sub-parallels, not all of them as
+ # -P should only go to the first, and -S should not be copied at all.
+ my $options =
+ join(" ",
+ ((defined $opt::jobs) ? "-P $opt::jobs" : ""),
+ ((defined $opt::linebuffer) ? "--linebuffer" : ""),
+ ((defined $opt::ungroup) ? "-u" : ""),
+ ((defined $opt::group) ? "-g" : ""),
+ ((defined $opt::keeporder) ? "--keeporder" : ""),
+ ((defined $opt::D) ? "-D $opt::D" : ""),
+ ((defined $opt::plain) ? "--plain" : ""),
+ ((defined $opt::max_chars) ? "--max-chars ".$opt::max_chars : ""),
+ );
+ my $suboptions =
+ join(" ",
+ ((defined $opt::ungroup) ? "-u" : ""),
+ ((defined $opt::linebuffer) ? "--linebuffer" : ""),
+ ((defined $opt::group) ? "-g" : ""),
+ ((defined $opt::files) ? "--files" : ""),
+ ((defined $opt::keeporder) ? "--keeporder" : ""),
+ ((defined $opt::colsep) ? "--colsep ".shell_quote($opt::colsep) : ""),
+ ((@opt::v) ? "-vv" : ""),
+ ((defined $opt::D) ? "-D $opt::D" : ""),
+ ((defined $opt::timeout) ? "--timeout ".$opt::timeout : ""),
+ ((defined $opt::plain) ? "--plain" : ""),
+ ((defined $opt::retries) ? "--retries ".$opt::retries : ""),
+ ((defined $opt::max_chars) ? "--max-chars ".$opt::max_chars : ""),
+ ((defined $opt::arg_sep) ? "--arg-sep ".$opt::arg_sep : ""),
+ ((defined $opt::arg_file_sep) ? "--arg-file-sep ".$opt::arg_file_sep : ""),
+ (@opt::env ? map { "--env ".::shell_quote_scalar($_) } @opt::env : ""),
+ );
+ ::debug("init", "| $0 $options\n");
+ open(my $parallel_fh, "|-", "$0 --no-notice -j0 $options") ||
+ ::die_bug("This does not run GNU Parallel: $0 $options");
+ my @joblogs;
+ for my $host (sort keys %Global::host) {
+ my $sshlogin = $Global::host{$host};
+ my $joblog = tmp_joblog($opt::joblog);
+ if($joblog) {
+ push @joblogs, $joblog;
+ $joblog = "--joblog $joblog";
+ }
+ my $quad = $opt::arg_file_sep || "::::";
+ ::debug("init", "$0 $suboptions -j1 $joblog ",
+ ((defined $opt::tag) ?
+ "--tagstring ".shell_quote_scalar($sshlogin->string()) : ""),
+ " -S ", shell_quote_scalar($sshlogin->string())," ",
+ join(" ",shell_quote(@command))," $quad @argfiles\n");
+ print $parallel_fh "$0 $suboptions -j1 $joblog ",
+ ((defined $opt::tag) ?
+ "--tagstring ".shell_quote_scalar($sshlogin->string()) : ""),
+ " -S ", shell_quote_scalar($sshlogin->string()),"
<TRUNCATED>
[17/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_test_util.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_test_util.cc b/thirdparty/rocksdb/db/db_test_util.cc
new file mode 100644
index 0000000..c4d465b
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_test_util.cc
@@ -0,0 +1,1395 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/db_test_util.h"
+#include "db/forward_iterator.h"
+#include "rocksdb/env_encryption.h"
+
+namespace rocksdb {
+
+// Special Env used to delay background operations
+
+SpecialEnv::SpecialEnv(Env* base)
+ : EnvWrapper(base),
+ rnd_(301),
+ sleep_counter_(this),
+ addon_time_(0),
+ time_elapse_only_sleep_(false),
+ no_slowdown_(false) {
+ delay_sstable_sync_.store(false, std::memory_order_release);
+ drop_writes_.store(false, std::memory_order_release);
+ no_space_.store(false, std::memory_order_release);
+ non_writable_.store(false, std::memory_order_release);
+ count_random_reads_ = false;
+ count_sequential_reads_ = false;
+ manifest_sync_error_.store(false, std::memory_order_release);
+ manifest_write_error_.store(false, std::memory_order_release);
+ log_write_error_.store(false, std::memory_order_release);
+ random_file_open_counter_.store(0, std::memory_order_relaxed);
+ delete_count_.store(0, std::memory_order_relaxed);
+ num_open_wal_file_.store(0);
+ log_write_slowdown_ = 0;
+ bytes_written_ = 0;
+ sync_counter_ = 0;
+ non_writeable_rate_ = 0;
+ new_writable_count_ = 0;
+ non_writable_count_ = 0;
+ table_write_callback_ = nullptr;
+}
+#ifndef ROCKSDB_LITE
+ROT13BlockCipher rot13Cipher_(16);
+#endif // ROCKSDB_LITE
+
+DBTestBase::DBTestBase(const std::string path)
+ : mem_env_(!getenv("MEM_ENV") ? nullptr : new MockEnv(Env::Default())),
+#ifndef ROCKSDB_LITE
+ encrypted_env_(
+ !getenv("ENCRYPTED_ENV")
+ ? nullptr
+ : NewEncryptedEnv(mem_env_ ? mem_env_ : Env::Default(),
+ new CTREncryptionProvider(rot13Cipher_))),
+#else
+ encrypted_env_(nullptr),
+#endif // ROCKSDB_LITE
+ env_(new SpecialEnv(encrypted_env_
+ ? encrypted_env_
+ : (mem_env_ ? mem_env_ : Env::Default()))),
+ option_config_(kDefault) {
+ env_->SetBackgroundThreads(1, Env::LOW);
+ env_->SetBackgroundThreads(1, Env::HIGH);
+ dbname_ = test::TmpDir(env_) + path;
+ alternative_wal_dir_ = dbname_ + "/wal";
+ alternative_db_log_dir_ = dbname_ + "/db_log_dir";
+ auto options = CurrentOptions();
+ options.env = env_;
+ auto delete_options = options;
+ delete_options.wal_dir = alternative_wal_dir_;
+ EXPECT_OK(DestroyDB(dbname_, delete_options));
+ // Destroy it for not alternative WAL dir is used.
+ EXPECT_OK(DestroyDB(dbname_, options));
+ db_ = nullptr;
+ Reopen(options);
+ Random::GetTLSInstance()->Reset(0xdeadbeef);
+}
+
+DBTestBase::~DBTestBase() {
+ rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+ rocksdb::SyncPoint::GetInstance()->LoadDependency({});
+ rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
+ Close();
+ Options options;
+ options.db_paths.emplace_back(dbname_, 0);
+ options.db_paths.emplace_back(dbname_ + "_2", 0);
+ options.db_paths.emplace_back(dbname_ + "_3", 0);
+ options.db_paths.emplace_back(dbname_ + "_4", 0);
+ options.env = env_;
+
+ if (getenv("KEEP_DB")) {
+ printf("DB is still at %s\n", dbname_.c_str());
+ } else {
+ EXPECT_OK(DestroyDB(dbname_, options));
+ }
+ delete env_;
+}
+
+bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) {
+#ifdef ROCKSDB_LITE
+ // These options are not supported in ROCKSDB_LITE
+ if (option_config == kHashSkipList ||
+ option_config == kPlainTableFirstBytePrefix ||
+ option_config == kPlainTableCappedPrefix ||
+ option_config == kPlainTableCappedPrefixNonMmap ||
+ option_config == kPlainTableAllBytesPrefix ||
+ option_config == kVectorRep || option_config == kHashLinkList ||
+ option_config == kHashCuckoo || option_config == kUniversalCompaction ||
+ option_config == kUniversalCompactionMultiLevel ||
+ option_config == kUniversalSubcompactions ||
+ option_config == kFIFOCompaction ||
+ option_config == kConcurrentSkipList) {
+ return true;
+ }
+#endif
+
+ if ((skip_mask & kSkipUniversalCompaction) &&
+ (option_config == kUniversalCompaction ||
+ option_config == kUniversalCompactionMultiLevel)) {
+ return true;
+ }
+ if ((skip_mask & kSkipMergePut) && option_config == kMergePut) {
+ return true;
+ }
+ if ((skip_mask & kSkipNoSeekToLast) &&
+ (option_config == kHashLinkList || option_config == kHashSkipList)) {
+ return true;
+ }
+ if ((skip_mask & kSkipPlainTable) &&
+ (option_config == kPlainTableAllBytesPrefix ||
+ option_config == kPlainTableFirstBytePrefix ||
+ option_config == kPlainTableCappedPrefix ||
+ option_config == kPlainTableCappedPrefixNonMmap)) {
+ return true;
+ }
+ if ((skip_mask & kSkipHashIndex) &&
+ (option_config == kBlockBasedTableWithPrefixHashIndex ||
+ option_config == kBlockBasedTableWithWholeKeyHashIndex)) {
+ return true;
+ }
+ if ((skip_mask & kSkipHashCuckoo) && (option_config == kHashCuckoo)) {
+ return true;
+ }
+ if ((skip_mask & kSkipFIFOCompaction) && option_config == kFIFOCompaction) {
+ return true;
+ }
+ if ((skip_mask & kSkipMmapReads) && option_config == kWalDirAndMmapReads) {
+ return true;
+ }
+ return false;
+}
+
+// Switch to a fresh database with the next option configuration to
+// test. Return false if there are no more configurations to test.
+bool DBTestBase::ChangeOptions(int skip_mask) {
+ for (option_config_++; option_config_ < kEnd; option_config_++) {
+ if (ShouldSkipOptions(option_config_, skip_mask)) {
+ continue;
+ }
+ break;
+ }
+
+ if (option_config_ >= kEnd) {
+ Destroy(last_options_);
+ return false;
+ } else {
+ auto options = CurrentOptions();
+ options.create_if_missing = true;
+ DestroyAndReopen(options);
+ return true;
+ }
+}
+
+// Switch between different compaction styles.
+bool DBTestBase::ChangeCompactOptions() {
+ if (option_config_ == kDefault) {
+ option_config_ = kUniversalCompaction;
+ Destroy(last_options_);
+ auto options = CurrentOptions();
+ options.create_if_missing = true;
+ TryReopen(options);
+ return true;
+ } else if (option_config_ == kUniversalCompaction) {
+ option_config_ = kUniversalCompactionMultiLevel;
+ Destroy(last_options_);
+ auto options = CurrentOptions();
+ options.create_if_missing = true;
+ TryReopen(options);
+ return true;
+ } else if (option_config_ == kUniversalCompactionMultiLevel) {
+ option_config_ = kLevelSubcompactions;
+ Destroy(last_options_);
+ auto options = CurrentOptions();
+ assert(options.max_subcompactions > 1);
+ TryReopen(options);
+ return true;
+ } else if (option_config_ == kLevelSubcompactions) {
+ option_config_ = kUniversalSubcompactions;
+ Destroy(last_options_);
+ auto options = CurrentOptions();
+ assert(options.max_subcompactions > 1);
+ TryReopen(options);
+ return true;
+ } else {
+ return false;
+ }
+}
+
+// Switch between different WAL settings
+bool DBTestBase::ChangeWalOptions() {
+ if (option_config_ == kDefault) {
+ option_config_ = kDBLogDir;
+ Destroy(last_options_);
+ auto options = CurrentOptions();
+ Destroy(options);
+ options.create_if_missing = true;
+ TryReopen(options);
+ return true;
+ } else if (option_config_ == kDBLogDir) {
+ option_config_ = kWalDirAndMmapReads;
+ Destroy(last_options_);
+ auto options = CurrentOptions();
+ Destroy(options);
+ options.create_if_missing = true;
+ TryReopen(options);
+ return true;
+ } else if (option_config_ == kWalDirAndMmapReads) {
+ option_config_ = kRecycleLogFiles;
+ Destroy(last_options_);
+ auto options = CurrentOptions();
+ Destroy(options);
+ TryReopen(options);
+ return true;
+ } else {
+ return false;
+ }
+}
+
+// Switch between different filter policy
+// Jump from kDefault to kFilter to kFullFilter
+bool DBTestBase::ChangeFilterOptions() {
+ if (option_config_ == kDefault) {
+ option_config_ = kFilter;
+ } else if (option_config_ == kFilter) {
+ option_config_ = kFullFilterWithNewTableReaderForCompactions;
+ } else if (option_config_ == kFullFilterWithNewTableReaderForCompactions) {
+ option_config_ = kPartitionedFilterWithNewTableReaderForCompactions;
+ } else {
+ return false;
+ }
+ Destroy(last_options_);
+
+ auto options = CurrentOptions();
+ options.create_if_missing = true;
+ TryReopen(options);
+ return true;
+}
+
+// Return the current option configuration.
+Options DBTestBase::CurrentOptions(
+ const anon::OptionsOverride& options_override) const {
+ return GetOptions(option_config_, GetDefaultOptions(), options_override);
+}
+
+Options DBTestBase::CurrentOptions(
+ const Options& default_options,
+ const anon::OptionsOverride& options_override) const {
+ return GetOptions(option_config_, default_options, options_override);
+}
+
+Options DBTestBase::GetDefaultOptions() {
+ Options options;
+ options.write_buffer_size = 4090 * 4096;
+ options.target_file_size_base = 2 * 1024 * 1024;
+ options.max_bytes_for_level_base = 10 * 1024 * 1024;
+ options.max_open_files = 5000;
+ options.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
+ options.compaction_pri = CompactionPri::kByCompensatedSize;
+ return options;
+}
+
+Options DBTestBase::GetOptions(
+ int option_config, const Options& default_options,
+ const anon::OptionsOverride& options_override) const {
+ // this redundant copy is to minimize code change w/o having lint error.
+ Options options = default_options;
+ BlockBasedTableOptions table_options;
+ bool set_block_based_table_factory = true;
+#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \
+ !defined(OS_AIX)
+ rocksdb::SyncPoint::GetInstance()->ClearCallBack(
+ "NewRandomAccessFile:O_DIRECT");
+ rocksdb::SyncPoint::GetInstance()->ClearCallBack(
+ "NewWritableFile:O_DIRECT");
+#endif
+
+ bool can_allow_mmap = IsMemoryMappedAccessSupported();
+ switch (option_config) {
+#ifndef ROCKSDB_LITE
+ case kHashSkipList:
+ options.prefix_extractor.reset(NewFixedPrefixTransform(1));
+ options.memtable_factory.reset(NewHashSkipListRepFactory(16));
+ options.allow_concurrent_memtable_write = false;
+ break;
+ case kPlainTableFirstBytePrefix:
+ options.table_factory.reset(new PlainTableFactory());
+ options.prefix_extractor.reset(NewFixedPrefixTransform(1));
+ options.allow_mmap_reads = can_allow_mmap;
+ options.max_sequential_skip_in_iterations = 999999;
+ set_block_based_table_factory = false;
+ break;
+ case kPlainTableCappedPrefix:
+ options.table_factory.reset(new PlainTableFactory());
+ options.prefix_extractor.reset(NewCappedPrefixTransform(8));
+ options.allow_mmap_reads = can_allow_mmap;
+ options.max_sequential_skip_in_iterations = 999999;
+ set_block_based_table_factory = false;
+ break;
+ case kPlainTableCappedPrefixNonMmap:
+ options.table_factory.reset(new PlainTableFactory());
+ options.prefix_extractor.reset(NewCappedPrefixTransform(8));
+ options.allow_mmap_reads = false;
+ options.max_sequential_skip_in_iterations = 999999;
+ set_block_based_table_factory = false;
+ break;
+ case kPlainTableAllBytesPrefix:
+ options.table_factory.reset(new PlainTableFactory());
+ options.prefix_extractor.reset(NewNoopTransform());
+ options.allow_mmap_reads = can_allow_mmap;
+ options.max_sequential_skip_in_iterations = 999999;
+ set_block_based_table_factory = false;
+ break;
+ case kVectorRep:
+ options.memtable_factory.reset(new VectorRepFactory(100));
+ options.allow_concurrent_memtable_write = false;
+ break;
+ case kHashLinkList:
+ options.prefix_extractor.reset(NewFixedPrefixTransform(1));
+ options.memtable_factory.reset(
+ NewHashLinkListRepFactory(4, 0, 3, true, 4));
+ options.allow_concurrent_memtable_write = false;
+ break;
+ case kHashCuckoo:
+ options.memtable_factory.reset(
+ NewHashCuckooRepFactory(options.write_buffer_size));
+ options.allow_concurrent_memtable_write = false;
+ break;
+#endif // ROCKSDB_LITE
+ case kMergePut:
+ options.merge_operator = MergeOperators::CreatePutOperator();
+ break;
+ case kFilter:
+ table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
+ break;
+ case kFullFilterWithNewTableReaderForCompactions:
+ table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
+ options.new_table_reader_for_compaction_inputs = true;
+ options.compaction_readahead_size = 10 * 1024 * 1024;
+ break;
+ case kPartitionedFilterWithNewTableReaderForCompactions:
+ table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
+ table_options.partition_filters = true;
+ table_options.index_type =
+ BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+ options.new_table_reader_for_compaction_inputs = true;
+ options.compaction_readahead_size = 10 * 1024 * 1024;
+ break;
+ case kUncompressed:
+ options.compression = kNoCompression;
+ break;
+ case kNumLevel_3:
+ options.num_levels = 3;
+ break;
+ case kDBLogDir:
+ options.db_log_dir = alternative_db_log_dir_;
+ break;
+ case kWalDirAndMmapReads:
+ options.wal_dir = alternative_wal_dir_;
+ // mmap reads should be orthogonal to WalDir setting, so we piggyback to
+ // this option config to test mmap reads as well
+ options.allow_mmap_reads = can_allow_mmap;
+ break;
+ case kManifestFileSize:
+ options.max_manifest_file_size = 50; // 50 bytes
+ break;
+ case kPerfOptions:
+ options.soft_rate_limit = 2.0;
+ options.delayed_write_rate = 8 * 1024 * 1024;
+ options.report_bg_io_stats = true;
+ // TODO(3.13) -- test more options
+ break;
+ case kUniversalCompaction:
+ options.compaction_style = kCompactionStyleUniversal;
+ options.num_levels = 1;
+ break;
+ case kUniversalCompactionMultiLevel:
+ options.compaction_style = kCompactionStyleUniversal;
+ options.num_levels = 8;
+ break;
+ case kCompressedBlockCache:
+ options.allow_mmap_writes = can_allow_mmap;
+ table_options.block_cache_compressed = NewLRUCache(8 * 1024 * 1024);
+ break;
+ case kInfiniteMaxOpenFiles:
+ options.max_open_files = -1;
+ break;
+ case kxxHashChecksum: {
+ table_options.checksum = kxxHash;
+ break;
+ }
+ case kFIFOCompaction: {
+ options.compaction_style = kCompactionStyleFIFO;
+ break;
+ }
+ case kBlockBasedTableWithPrefixHashIndex: {
+ table_options.index_type = BlockBasedTableOptions::kHashSearch;
+ options.prefix_extractor.reset(NewFixedPrefixTransform(1));
+ break;
+ }
+ case kBlockBasedTableWithWholeKeyHashIndex: {
+ table_options.index_type = BlockBasedTableOptions::kHashSearch;
+ options.prefix_extractor.reset(NewNoopTransform());
+ break;
+ }
+ case kBlockBasedTableWithPartitionedIndex: {
+ table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch;
+ options.prefix_extractor.reset(NewNoopTransform());
+ break;
+ }
+ case kBlockBasedTableWithIndexRestartInterval: {
+ table_options.index_block_restart_interval = 8;
+ break;
+ }
+ case kOptimizeFiltersForHits: {
+ options.optimize_filters_for_hits = true;
+ set_block_based_table_factory = true;
+ break;
+ }
+ case kRowCache: {
+ options.row_cache = NewLRUCache(1024 * 1024);
+ break;
+ }
+ case kRecycleLogFiles: {
+ options.recycle_log_file_num = 2;
+ break;
+ }
+ case kLevelSubcompactions: {
+ options.max_subcompactions = 4;
+ break;
+ }
+ case kUniversalSubcompactions: {
+ options.compaction_style = kCompactionStyleUniversal;
+ options.num_levels = 8;
+ options.max_subcompactions = 4;
+ break;
+ }
+ case kConcurrentSkipList: {
+ options.allow_concurrent_memtable_write = true;
+ options.enable_write_thread_adaptive_yield = true;
+ break;
+ }
+ case kDirectIO: {
+ options.use_direct_reads = true;
+ options.use_direct_io_for_flush_and_compaction = true;
+ options.compaction_readahead_size = 2 * 1024 * 1024;
+#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) && \
+ !defined(OS_AIX)
+ rocksdb::SyncPoint::GetInstance()->SetCallBack(
+ "NewWritableFile:O_DIRECT", [&](void* arg) {
+ int* val = static_cast<int*>(arg);
+ *val &= ~O_DIRECT;
+ });
+ rocksdb::SyncPoint::GetInstance()->SetCallBack(
+ "NewRandomAccessFile:O_DIRECT", [&](void* arg) {
+ int* val = static_cast<int*>(arg);
+ *val &= ~O_DIRECT;
+ });
+ rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+#endif
+ break;
+ }
+ case kPipelinedWrite: {
+ options.enable_pipelined_write = true;
+ break;
+ }
+ case kConcurrentWALWrites: {
+ // This options optimize 2PC commit path
+ options.concurrent_prepare = true;
+ options.manual_wal_flush = true;
+ break;
+ }
+
+ default:
+ break;
+ }
+
+ if (options_override.filter_policy) {
+ table_options.filter_policy = options_override.filter_policy;
+ table_options.partition_filters = options_override.partition_filters;
+ table_options.metadata_block_size = options_override.metadata_block_size;
+ }
+ if (set_block_based_table_factory) {
+ options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+ }
+ options.env = env_;
+ options.create_if_missing = true;
+ options.fail_if_options_file_error = true;
+ return options;
+}
+
+void DBTestBase::CreateColumnFamilies(const std::vector<std::string>& cfs,
+ const Options& options) {
+ ColumnFamilyOptions cf_opts(options);
+ size_t cfi = handles_.size();
+ handles_.resize(cfi + cfs.size());
+ for (auto cf : cfs) {
+ ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++]));
+ }
+}
+
+void DBTestBase::CreateAndReopenWithCF(const std::vector<std::string>& cfs,
+ const Options& options) {
+ CreateColumnFamilies(cfs, options);
+ std::vector<std::string> cfs_plus_default = cfs;
+ cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName);
+ ReopenWithColumnFamilies(cfs_plus_default, options);
+}
+
+void DBTestBase::ReopenWithColumnFamilies(const std::vector<std::string>& cfs,
+ const std::vector<Options>& options) {
+ ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
+}
+
+void DBTestBase::ReopenWithColumnFamilies(const std::vector<std::string>& cfs,
+ const Options& options) {
+ ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
+}
+
+Status DBTestBase::TryReopenWithColumnFamilies(
+ const std::vector<std::string>& cfs, const std::vector<Options>& options) {
+ Close();
+ EXPECT_EQ(cfs.size(), options.size());
+ std::vector<ColumnFamilyDescriptor> column_families;
+ for (size_t i = 0; i < cfs.size(); ++i) {
+ column_families.push_back(ColumnFamilyDescriptor(cfs[i], options[i]));
+ }
+ DBOptions db_opts = DBOptions(options[0]);
+ return DB::Open(db_opts, dbname_, column_families, &handles_, &db_);
+}
+
+Status DBTestBase::TryReopenWithColumnFamilies(
+ const std::vector<std::string>& cfs, const Options& options) {
+ Close();
+ std::vector<Options> v_opts(cfs.size(), options);
+ return TryReopenWithColumnFamilies(cfs, v_opts);
+}
+
+void DBTestBase::Reopen(const Options& options) {
+ ASSERT_OK(TryReopen(options));
+}
+
+void DBTestBase::Close() {
+ for (auto h : handles_) {
+ db_->DestroyColumnFamilyHandle(h);
+ }
+ handles_.clear();
+ delete db_;
+ db_ = nullptr;
+}
+
+void DBTestBase::DestroyAndReopen(const Options& options) {
+ // Destroy using last options
+ Destroy(last_options_);
+ ASSERT_OK(TryReopen(options));
+}
+
+void DBTestBase::Destroy(const Options& options) {
+ Close();
+ ASSERT_OK(DestroyDB(dbname_, options));
+}
+
+Status DBTestBase::ReadOnlyReopen(const Options& options) {
+ return DB::OpenForReadOnly(options, dbname_, &db_);
+}
+
+Status DBTestBase::TryReopen(const Options& options) {
+ Close();
+ last_options_.table_factory.reset();
+ // Note: operator= is an unsafe approach here since it destructs shared_ptr in
+ // the same order of their creation, in contrast to destructors which
+ // destructs them in the opposite order of creation. One particular problme is
+ // that the cache destructor might invoke callback functions that use Option
+ // members such as statistics. To work around this problem, we manually call
+ // destructor of table_facotry which eventually clears the block cache.
+ last_options_ = options;
+ return DB::Open(options, dbname_, &db_);
+}
+
+bool DBTestBase::IsDirectIOSupported() {
+ EnvOptions env_options;
+ env_options.use_mmap_writes = false;
+ env_options.use_direct_writes = true;
+ std::string tmp = TempFileName(dbname_, 999);
+ Status s;
+ {
+ unique_ptr<WritableFile> file;
+ s = env_->NewWritableFile(tmp, &file, env_options);
+ }
+ if (s.ok()) {
+ s = env_->DeleteFile(tmp);
+ }
+ return s.ok();
+}
+
+bool DBTestBase::IsMemoryMappedAccessSupported() const {
+ return (!encrypted_env_);
+}
+
+Status DBTestBase::Flush(int cf) {
+ if (cf == 0) {
+ return db_->Flush(FlushOptions());
+ } else {
+ return db_->Flush(FlushOptions(), handles_[cf]);
+ }
+}
+
+Status DBTestBase::Put(const Slice& k, const Slice& v, WriteOptions wo) {
+ if (kMergePut == option_config_) {
+ return db_->Merge(wo, k, v);
+ } else {
+ return db_->Put(wo, k, v);
+ }
+}
+
+Status DBTestBase::Put(int cf, const Slice& k, const Slice& v,
+ WriteOptions wo) {
+ if (kMergePut == option_config_) {
+ return db_->Merge(wo, handles_[cf], k, v);
+ } else {
+ return db_->Put(wo, handles_[cf], k, v);
+ }
+}
+
+Status DBTestBase::Merge(const Slice& k, const Slice& v, WriteOptions wo) {
+ return db_->Merge(wo, k, v);
+}
+
+Status DBTestBase::Merge(int cf, const Slice& k, const Slice& v,
+ WriteOptions wo) {
+ return db_->Merge(wo, handles_[cf], k, v);
+}
+
+Status DBTestBase::Delete(const std::string& k) {
+ return db_->Delete(WriteOptions(), k);
+}
+
+Status DBTestBase::Delete(int cf, const std::string& k) {
+ return db_->Delete(WriteOptions(), handles_[cf], k);
+}
+
+Status DBTestBase::SingleDelete(const std::string& k) {
+ return db_->SingleDelete(WriteOptions(), k);
+}
+
+Status DBTestBase::SingleDelete(int cf, const std::string& k) {
+ return db_->SingleDelete(WriteOptions(), handles_[cf], k);
+}
+
+std::string DBTestBase::Get(const std::string& k, const Snapshot* snapshot) {
+ ReadOptions options;
+ options.verify_checksums = true;
+ options.snapshot = snapshot;
+ std::string result;
+ Status s = db_->Get(options, k, &result);
+ if (s.IsNotFound()) {
+ result = "NOT_FOUND";
+ } else if (!s.ok()) {
+ result = s.ToString();
+ }
+ return result;
+}
+
+std::string DBTestBase::Get(int cf, const std::string& k,
+ const Snapshot* snapshot) {
+ ReadOptions options;
+ options.verify_checksums = true;
+ options.snapshot = snapshot;
+ std::string result;
+ Status s = db_->Get(options, handles_[cf], k, &result);
+ if (s.IsNotFound()) {
+ result = "NOT_FOUND";
+ } else if (!s.ok()) {
+ result = s.ToString();
+ }
+ return result;
+}
+
+Status DBTestBase::Get(const std::string& k, PinnableSlice* v) {
+ ReadOptions options;
+ options.verify_checksums = true;
+ Status s = dbfull()->Get(options, dbfull()->DefaultColumnFamily(), k, v);
+ return s;
+}
+
+uint64_t DBTestBase::GetNumSnapshots() {
+ uint64_t int_num;
+ EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.num-snapshots", &int_num));
+ return int_num;
+}
+
+uint64_t DBTestBase::GetTimeOldestSnapshots() {
+ uint64_t int_num;
+ EXPECT_TRUE(
+ dbfull()->GetIntProperty("rocksdb.oldest-snapshot-time", &int_num));
+ return int_num;
+}
+
+// Return a string that contains all key,value pairs in order,
+// formatted like "(k1->v1)(k2->v2)".
+std::string DBTestBase::Contents(int cf) {
+ std::vector<std::string> forward;
+ std::string result;
+ Iterator* iter = (cf == 0) ? db_->NewIterator(ReadOptions())
+ : db_->NewIterator(ReadOptions(), handles_[cf]);
+ for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+ std::string s = IterStatus(iter);
+ result.push_back('(');
+ result.append(s);
+ result.push_back(')');
+ forward.push_back(s);
+ }
+
+ // Check reverse iteration results are the reverse of forward results
+ unsigned int matched = 0;
+ for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
+ EXPECT_LT(matched, forward.size());
+ EXPECT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]);
+ matched++;
+ }
+ EXPECT_EQ(matched, forward.size());
+
+ delete iter;
+ return result;
+}
+
+std::string DBTestBase::AllEntriesFor(const Slice& user_key, int cf) {
+ Arena arena;
+ auto options = CurrentOptions();
+ InternalKeyComparator icmp(options.comparator);
+ RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
+ ScopedArenaIterator iter;
+ if (cf == 0) {
+ iter.set(dbfull()->NewInternalIterator(&arena, &range_del_agg));
+ } else {
+ iter.set(
+ dbfull()->NewInternalIterator(&arena, &range_del_agg, handles_[cf]));
+ }
+ InternalKey target(user_key, kMaxSequenceNumber, kTypeValue);
+ iter->Seek(target.Encode());
+ std::string result;
+ if (!iter->status().ok()) {
+ result = iter->status().ToString();
+ } else {
+ result = "[ ";
+ bool first = true;
+ while (iter->Valid()) {
+ ParsedInternalKey ikey(Slice(), 0, kTypeValue);
+ if (!ParseInternalKey(iter->key(), &ikey)) {
+ result += "CORRUPTED";
+ } else {
+ if (!last_options_.comparator->Equal(ikey.user_key, user_key)) {
+ break;
+ }
+ if (!first) {
+ result += ", ";
+ }
+ first = false;
+ switch (ikey.type) {
+ case kTypeValue:
+ result += iter->value().ToString();
+ break;
+ case kTypeMerge:
+ // keep it the same as kTypeValue for testing kMergePut
+ result += iter->value().ToString();
+ break;
+ case kTypeDeletion:
+ result += "DEL";
+ break;
+ case kTypeSingleDeletion:
+ result += "SDEL";
+ break;
+ default:
+ assert(false);
+ break;
+ }
+ }
+ iter->Next();
+ }
+ if (!first) {
+ result += " ";
+ }
+ result += "]";
+ }
+ return result;
+}
+
+#ifndef ROCKSDB_LITE
+int DBTestBase::NumSortedRuns(int cf) {
+ ColumnFamilyMetaData cf_meta;
+ if (cf == 0) {
+ db_->GetColumnFamilyMetaData(&cf_meta);
+ } else {
+ db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta);
+ }
+ int num_sr = static_cast<int>(cf_meta.levels[0].files.size());
+ for (size_t i = 1U; i < cf_meta.levels.size(); i++) {
+ if (cf_meta.levels[i].files.size() > 0) {
+ num_sr++;
+ }
+ }
+ return num_sr;
+}
+
+uint64_t DBTestBase::TotalSize(int cf) {
+ ColumnFamilyMetaData cf_meta;
+ if (cf == 0) {
+ db_->GetColumnFamilyMetaData(&cf_meta);
+ } else {
+ db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta);
+ }
+ return cf_meta.size;
+}
+
+uint64_t DBTestBase::SizeAtLevel(int level) {
+ std::vector<LiveFileMetaData> metadata;
+ db_->GetLiveFilesMetaData(&metadata);
+ uint64_t sum = 0;
+ for (const auto& m : metadata) {
+ if (m.level == level) {
+ sum += m.size;
+ }
+ }
+ return sum;
+}
+
+size_t DBTestBase::TotalLiveFiles(int cf) {
+ ColumnFamilyMetaData cf_meta;
+ if (cf == 0) {
+ db_->GetColumnFamilyMetaData(&cf_meta);
+ } else {
+ db_->GetColumnFamilyMetaData(handles_[cf], &cf_meta);
+ }
+ size_t num_files = 0;
+ for (auto& level : cf_meta.levels) {
+ num_files += level.files.size();
+ }
+ return num_files;
+}
+
+size_t DBTestBase::CountLiveFiles() {
+ std::vector<LiveFileMetaData> metadata;
+ db_->GetLiveFilesMetaData(&metadata);
+ return metadata.size();
+}
+#endif // ROCKSDB_LITE
+
+int DBTestBase::NumTableFilesAtLevel(int level, int cf) {
+ std::string property;
+ if (cf == 0) {
+ // default cfd
+ EXPECT_TRUE(db_->GetProperty(
+ "rocksdb.num-files-at-level" + NumberToString(level), &property));
+ } else {
+ EXPECT_TRUE(db_->GetProperty(
+ handles_[cf], "rocksdb.num-files-at-level" + NumberToString(level),
+ &property));
+ }
+ return atoi(property.c_str());
+}
+
+double DBTestBase::CompressionRatioAtLevel(int level, int cf) {
+ std::string property;
+ if (cf == 0) {
+ // default cfd
+ EXPECT_TRUE(db_->GetProperty(
+ "rocksdb.compression-ratio-at-level" + NumberToString(level),
+ &property));
+ } else {
+ EXPECT_TRUE(db_->GetProperty(
+ handles_[cf],
+ "rocksdb.compression-ratio-at-level" + NumberToString(level),
+ &property));
+ }
+ return std::stod(property);
+}
+
+int DBTestBase::TotalTableFiles(int cf, int levels) {
+ if (levels == -1) {
+ levels = (cf == 0) ? db_->NumberLevels() : db_->NumberLevels(handles_[1]);
+ }
+ int result = 0;
+ for (int level = 0; level < levels; level++) {
+ result += NumTableFilesAtLevel(level, cf);
+ }
+ return result;
+}
+
+// Return spread of files per level
+std::string DBTestBase::FilesPerLevel(int cf) {
+ int num_levels =
+ (cf == 0) ? db_->NumberLevels() : db_->NumberLevels(handles_[1]);
+ std::string result;
+ size_t last_non_zero_offset = 0;
+ for (int level = 0; level < num_levels; level++) {
+ int f = NumTableFilesAtLevel(level, cf);
+ char buf[100];
+ snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
+ result += buf;
+ if (f > 0) {
+ last_non_zero_offset = result.size();
+ }
+ }
+ result.resize(last_non_zero_offset);
+ return result;
+}
+
+size_t DBTestBase::CountFiles() {
+ std::vector<std::string> files;
+ env_->GetChildren(dbname_, &files);
+
+ std::vector<std::string> logfiles;
+ if (dbname_ != last_options_.wal_dir) {
+ env_->GetChildren(last_options_.wal_dir, &logfiles);
+ }
+
+ return files.size() + logfiles.size();
+}
+
+uint64_t DBTestBase::Size(const Slice& start, const Slice& limit, int cf) {
+ Range r(start, limit);
+ uint64_t size;
+ if (cf == 0) {
+ db_->GetApproximateSizes(&r, 1, &size);
+ } else {
+ db_->GetApproximateSizes(handles_[1], &r, 1, &size);
+ }
+ return size;
+}
+
+void DBTestBase::Compact(int cf, const Slice& start, const Slice& limit,
+ uint32_t target_path_id) {
+ CompactRangeOptions compact_options;
+ compact_options.target_path_id = target_path_id;
+ ASSERT_OK(db_->CompactRange(compact_options, handles_[cf], &start, &limit));
+}
+
+void DBTestBase::Compact(int cf, const Slice& start, const Slice& limit) {
+ ASSERT_OK(
+ db_->CompactRange(CompactRangeOptions(), handles_[cf], &start, &limit));
+}
+
+void DBTestBase::Compact(const Slice& start, const Slice& limit) {
+ ASSERT_OK(db_->CompactRange(CompactRangeOptions(), &start, &limit));
+}
+
+// Do n memtable compactions, each of which produces an sstable
+// covering the range [small,large].
+void DBTestBase::MakeTables(int n, const std::string& small,
+ const std::string& large, int cf) {
+ for (int i = 0; i < n; i++) {
+ ASSERT_OK(Put(cf, small, "begin"));
+ ASSERT_OK(Put(cf, large, "end"));
+ ASSERT_OK(Flush(cf));
+ MoveFilesToLevel(n - i - 1, cf);
+ }
+}
+
+// Prevent pushing of new sstables into deeper levels by adding
+// tables that cover a specified range to all levels.
+void DBTestBase::FillLevels(const std::string& smallest,
+ const std::string& largest, int cf) {
+ MakeTables(db_->NumberLevels(handles_[cf]), smallest, largest, cf);
+}
+
+void DBTestBase::MoveFilesToLevel(int level, int cf) {
+ for (int l = 0; l < level; ++l) {
+ if (cf > 0) {
+ dbfull()->TEST_CompactRange(l, nullptr, nullptr, handles_[cf]);
+ } else {
+ dbfull()->TEST_CompactRange(l, nullptr, nullptr);
+ }
+ }
+}
+
+void DBTestBase::DumpFileCounts(const char* label) {
+ fprintf(stderr, "---\n%s:\n", label);
+ fprintf(stderr, "maxoverlap: %" PRIu64 "\n",
+ dbfull()->TEST_MaxNextLevelOverlappingBytes());
+ for (int level = 0; level < db_->NumberLevels(); level++) {
+ int num = NumTableFilesAtLevel(level);
+ if (num > 0) {
+ fprintf(stderr, " level %3d : %d files\n", level, num);
+ }
+ }
+}
+
+std::string DBTestBase::DumpSSTableList() {
+ std::string property;
+ db_->GetProperty("rocksdb.sstables", &property);
+ return property;
+}
+
+void DBTestBase::GetSstFiles(std::string path,
+ std::vector<std::string>* files) {
+ env_->GetChildren(path, files);
+
+ files->erase(
+ std::remove_if(files->begin(), files->end(), [](std::string name) {
+ uint64_t number;
+ FileType type;
+ return !(ParseFileName(name, &number, &type) && type == kTableFile);
+ }), files->end());
+}
+
+int DBTestBase::GetSstFileCount(std::string path) {
+ std::vector<std::string> files;
+ GetSstFiles(path, &files);
+ return static_cast<int>(files.size());
+}
+
+// this will generate non-overlapping files since it keeps increasing key_idx
+void DBTestBase::GenerateNewFile(int cf, Random* rnd, int* key_idx,
+ bool nowait) {
+ for (int i = 0; i < KNumKeysByGenerateNewFile; i++) {
+ ASSERT_OK(Put(cf, Key(*key_idx), RandomString(rnd, (i == 99) ? 1 : 990)));
+ (*key_idx)++;
+ }
+ if (!nowait) {
+ dbfull()->TEST_WaitForFlushMemTable();
+ dbfull()->TEST_WaitForCompact();
+ }
+}
+
+// this will generate non-overlapping files since it keeps increasing key_idx
+void DBTestBase::GenerateNewFile(Random* rnd, int* key_idx, bool nowait) {
+ for (int i = 0; i < KNumKeysByGenerateNewFile; i++) {
+ ASSERT_OK(Put(Key(*key_idx), RandomString(rnd, (i == 99) ? 1 : 990)));
+ (*key_idx)++;
+ }
+ if (!nowait) {
+ dbfull()->TEST_WaitForFlushMemTable();
+ dbfull()->TEST_WaitForCompact();
+ }
+}
+
+const int DBTestBase::kNumKeysByGenerateNewRandomFile = 51;
+
+void DBTestBase::GenerateNewRandomFile(Random* rnd, bool nowait) {
+ for (int i = 0; i < kNumKeysByGenerateNewRandomFile; i++) {
+ ASSERT_OK(Put("key" + RandomString(rnd, 7), RandomString(rnd, 2000)));
+ }
+ ASSERT_OK(Put("key" + RandomString(rnd, 7), RandomString(rnd, 200)));
+ if (!nowait) {
+ dbfull()->TEST_WaitForFlushMemTable();
+ dbfull()->TEST_WaitForCompact();
+ }
+}
+
+std::string DBTestBase::IterStatus(Iterator* iter) {
+ std::string result;
+ if (iter->Valid()) {
+ result = iter->key().ToString() + "->" + iter->value().ToString();
+ } else {
+ result = "(invalid)";
+ }
+ return result;
+}
+
+Options DBTestBase::OptionsForLogIterTest() {
+ Options options = CurrentOptions();
+ options.create_if_missing = true;
+ options.WAL_ttl_seconds = 1000;
+ return options;
+}
+
+std::string DBTestBase::DummyString(size_t len, char c) {
+ return std::string(len, c);
+}
+
+void DBTestBase::VerifyIterLast(std::string expected_key, int cf) {
+ Iterator* iter;
+ ReadOptions ro;
+ if (cf == 0) {
+ iter = db_->NewIterator(ro);
+ } else {
+ iter = db_->NewIterator(ro, handles_[cf]);
+ }
+ iter->SeekToLast();
+ ASSERT_EQ(IterStatus(iter), expected_key);
+ delete iter;
+}
+
+// Used to test InplaceUpdate
+
+// If previous value is nullptr or delta is > than previous value,
+// sets newValue with delta
+// If previous value is not empty,
+// updates previous value with 'b' string of previous value size - 1.
+UpdateStatus DBTestBase::updateInPlaceSmallerSize(char* prevValue,
+ uint32_t* prevSize,
+ Slice delta,
+ std::string* newValue) {
+ if (prevValue == nullptr) {
+ *newValue = std::string(delta.size(), 'c');
+ return UpdateStatus::UPDATED;
+ } else {
+ *prevSize = *prevSize - 1;
+ std::string str_b = std::string(*prevSize, 'b');
+ memcpy(prevValue, str_b.c_str(), str_b.size());
+ return UpdateStatus::UPDATED_INPLACE;
+ }
+}
+
+UpdateStatus DBTestBase::updateInPlaceSmallerVarintSize(char* prevValue,
+ uint32_t* prevSize,
+ Slice delta,
+ std::string* newValue) {
+ if (prevValue == nullptr) {
+ *newValue = std::string(delta.size(), 'c');
+ return UpdateStatus::UPDATED;
+ } else {
+ *prevSize = 1;
+ std::string str_b = std::string(*prevSize, 'b');
+ memcpy(prevValue, str_b.c_str(), str_b.size());
+ return UpdateStatus::UPDATED_INPLACE;
+ }
+}
+
+UpdateStatus DBTestBase::updateInPlaceLargerSize(char* prevValue,
+ uint32_t* prevSize,
+ Slice delta,
+ std::string* newValue) {
+ *newValue = std::string(delta.size(), 'c');
+ return UpdateStatus::UPDATED;
+}
+
+UpdateStatus DBTestBase::updateInPlaceNoAction(char* prevValue,
+ uint32_t* prevSize, Slice delta,
+ std::string* newValue) {
+ return UpdateStatus::UPDATE_FAILED;
+}
+
+// Utility method to test InplaceUpdate
+void DBTestBase::validateNumberOfEntries(int numValues, int cf) {
+ ScopedArenaIterator iter;
+ Arena arena;
+ auto options = CurrentOptions();
+ InternalKeyComparator icmp(options.comparator);
+ RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
+ if (cf != 0) {
+ iter.set(
+ dbfull()->NewInternalIterator(&arena, &range_del_agg, handles_[cf]));
+ } else {
+ iter.set(dbfull()->NewInternalIterator(&arena, &range_del_agg));
+ }
+ iter->SeekToFirst();
+ ASSERT_EQ(iter->status().ok(), true);
+ int seq = numValues;
+ while (iter->Valid()) {
+ ParsedInternalKey ikey;
+ ikey.sequence = -1;
+ ASSERT_EQ(ParseInternalKey(iter->key(), &ikey), true);
+
+ // checks sequence number for updates
+ ASSERT_EQ(ikey.sequence, (unsigned)seq--);
+ iter->Next();
+ }
+ ASSERT_EQ(0, seq);
+}
+
+void DBTestBase::CopyFile(const std::string& source,
+ const std::string& destination, uint64_t size) {
+ const EnvOptions soptions;
+ unique_ptr<SequentialFile> srcfile;
+ ASSERT_OK(env_->NewSequentialFile(source, &srcfile, soptions));
+ unique_ptr<WritableFile> destfile;
+ ASSERT_OK(env_->NewWritableFile(destination, &destfile, soptions));
+
+ if (size == 0) {
+ // default argument means copy everything
+ ASSERT_OK(env_->GetFileSize(source, &size));
+ }
+
+ char buffer[4096];
+ Slice slice;
+ while (size > 0) {
+ uint64_t one = std::min(uint64_t(sizeof(buffer)), size);
+ ASSERT_OK(srcfile->Read(one, &slice, buffer));
+ ASSERT_OK(destfile->Append(slice));
+ size -= slice.size();
+ }
+ ASSERT_OK(destfile->Close());
+}
+
+std::unordered_map<std::string, uint64_t> DBTestBase::GetAllSSTFiles(
+ uint64_t* total_size) {
+ std::unordered_map<std::string, uint64_t> res;
+
+ if (total_size) {
+ *total_size = 0;
+ }
+ std::vector<std::string> files;
+ env_->GetChildren(dbname_, &files);
+ for (auto& file_name : files) {
+ uint64_t number;
+ FileType type;
+ std::string file_path = dbname_ + "/" + file_name;
+ if (ParseFileName(file_name, &number, &type) && type == kTableFile) {
+ uint64_t file_size = 0;
+ env_->GetFileSize(file_path, &file_size);
+ res[file_path] = file_size;
+ if (total_size) {
+ *total_size += file_size;
+ }
+ }
+ }
+ return res;
+}
+
+std::vector<std::uint64_t> DBTestBase::ListTableFiles(Env* env,
+ const std::string& path) {
+ std::vector<std::string> files;
+ std::vector<uint64_t> file_numbers;
+ env->GetChildren(path, &files);
+ uint64_t number;
+ FileType type;
+ for (size_t i = 0; i < files.size(); ++i) {
+ if (ParseFileName(files[i], &number, &type)) {
+ if (type == kTableFile) {
+ file_numbers.push_back(number);
+ }
+ }
+ }
+ return file_numbers;
+}
+
+void DBTestBase::VerifyDBFromMap(std::map<std::string, std::string> true_data,
+ size_t* total_reads_res, bool tailing_iter,
+ std::map<std::string, Status> status) {
+ size_t total_reads = 0;
+
+ for (auto& kv : true_data) {
+ Status s = status[kv.first];
+ if (s.ok()) {
+ ASSERT_EQ(Get(kv.first), kv.second);
+ } else {
+ std::string value;
+ ASSERT_EQ(s, db_->Get(ReadOptions(), kv.first, &value));
+ }
+ total_reads++;
+ }
+
+ // Normal Iterator
+ {
+ int iter_cnt = 0;
+ ReadOptions ro;
+ ro.total_order_seek = true;
+ Iterator* iter = db_->NewIterator(ro);
+ // Verify Iterator::Next()
+ iter_cnt = 0;
+ auto data_iter = true_data.begin();
+ Status s;
+ for (iter->SeekToFirst(); iter->Valid(); iter->Next(), data_iter++) {
+ ASSERT_EQ(iter->key().ToString(), data_iter->first);
+ Status current_status = status[data_iter->first];
+ if (!current_status.ok()) {
+ s = current_status;
+ }
+ ASSERT_EQ(iter->status(), s);
+ if (current_status.ok()) {
+ ASSERT_EQ(iter->value().ToString(), data_iter->second);
+ }
+ iter_cnt++;
+ total_reads++;
+ }
+ ASSERT_EQ(data_iter, true_data.end()) << iter_cnt << " / "
+ << true_data.size();
+ delete iter;
+
+ // Verify Iterator::Prev()
+ // Use a new iterator to make sure its status is clean.
+ iter = db_->NewIterator(ro);
+ iter_cnt = 0;
+ s = Status::OK();
+ auto data_rev = true_data.rbegin();
+ for (iter->SeekToLast(); iter->Valid(); iter->Prev(), data_rev++) {
+ ASSERT_EQ(iter->key().ToString(), data_rev->first);
+ Status current_status = status[data_rev->first];
+ if (!current_status.ok()) {
+ s = current_status;
+ }
+ ASSERT_EQ(iter->status(), s);
+ if (current_status.ok()) {
+ ASSERT_EQ(iter->value().ToString(), data_rev->second);
+ }
+ iter_cnt++;
+ total_reads++;
+ }
+ ASSERT_EQ(data_rev, true_data.rend()) << iter_cnt << " / "
+ << true_data.size();
+
+ // Verify Iterator::Seek()
+ for (auto kv : true_data) {
+ iter->Seek(kv.first);
+ ASSERT_EQ(kv.first, iter->key().ToString());
+ ASSERT_EQ(kv.second, iter->value().ToString());
+ total_reads++;
+ }
+ delete iter;
+ }
+
+ if (tailing_iter) {
+#ifndef ROCKSDB_LITE
+ // Tailing iterator
+ int iter_cnt = 0;
+ ReadOptions ro;
+ ro.tailing = true;
+ ro.total_order_seek = true;
+ Iterator* iter = db_->NewIterator(ro);
+
+ // Verify ForwardIterator::Next()
+ iter_cnt = 0;
+ auto data_iter = true_data.begin();
+ for (iter->SeekToFirst(); iter->Valid(); iter->Next(), data_iter++) {
+ ASSERT_EQ(iter->key().ToString(), data_iter->first);
+ ASSERT_EQ(iter->value().ToString(), data_iter->second);
+ iter_cnt++;
+ total_reads++;
+ }
+ ASSERT_EQ(data_iter, true_data.end()) << iter_cnt << " / "
+ << true_data.size();
+
+ // Verify ForwardIterator::Seek()
+ for (auto kv : true_data) {
+ iter->Seek(kv.first);
+ ASSERT_EQ(kv.first, iter->key().ToString());
+ ASSERT_EQ(kv.second, iter->value().ToString());
+ total_reads++;
+ }
+
+ delete iter;
+#endif // ROCKSDB_LITE
+ }
+
+ if (total_reads_res) {
+ *total_reads_res = total_reads;
+ }
+}
+
+void DBTestBase::VerifyDBInternal(
+ std::vector<std::pair<std::string, std::string>> true_data) {
+ Arena arena;
+ InternalKeyComparator icmp(last_options_.comparator);
+ RangeDelAggregator range_del_agg(icmp, {});
+ auto iter = dbfull()->NewInternalIterator(&arena, &range_del_agg);
+ iter->SeekToFirst();
+ for (auto p : true_data) {
+ ASSERT_TRUE(iter->Valid());
+ ParsedInternalKey ikey;
+ ASSERT_TRUE(ParseInternalKey(iter->key(), &ikey));
+ ASSERT_EQ(p.first, ikey.user_key);
+ ASSERT_EQ(p.second, iter->value());
+ iter->Next();
+ };
+ ASSERT_FALSE(iter->Valid());
+ iter->~InternalIterator();
+}
+
+#ifndef ROCKSDB_LITE
+
+uint64_t DBTestBase::GetNumberOfSstFilesForColumnFamily(
+ DB* db, std::string column_family_name) {
+ std::vector<LiveFileMetaData> metadata;
+ db->GetLiveFilesMetaData(&metadata);
+ uint64_t result = 0;
+ for (auto& fileMetadata : metadata) {
+ result += (fileMetadata.column_family_name == column_family_name);
+ }
+ return result;
+}
+#endif // ROCKSDB_LITE
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_test_util.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_test_util.h b/thirdparty/rocksdb/db/db_test_util.h
new file mode 100644
index 0000000..cd1265e
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_test_util.h
@@ -0,0 +1,939 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <fcntl.h>
+#include <inttypes.h>
+
+#include <algorithm>
+#include <map>
+#include <set>
+#include <string>
+#include <thread>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "db/db_impl.h"
+#include "db/dbformat.h"
+#include "env/mock_env.h"
+#include "memtable/hash_linklist_rep.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/convenience.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/options.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/sst_file_writer.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/table.h"
+#include "rocksdb/utilities/checkpoint.h"
+#include "table/block_based_table_factory.h"
+#include "table/mock_table.h"
+#include "table/plain_table_factory.h"
+#include "table/scoped_arena_iterator.h"
+#include "util/compression.h"
+#include "util/filename.h"
+#include "util/mutexlock.h"
+
+#include "util/string_util.h"
+#include "util/sync_point.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+#include "utilities/merge_operators.h"
+
+namespace rocksdb {
+
+namespace anon {
+class AtomicCounter {
+ public:
+ explicit AtomicCounter(Env* env = NULL)
+ : env_(env), cond_count_(&mu_), count_(0) {}
+
+ void Increment() {
+ MutexLock l(&mu_);
+ count_++;
+ cond_count_.SignalAll();
+ }
+
+ int Read() {
+ MutexLock l(&mu_);
+ return count_;
+ }
+
+ bool WaitFor(int count) {
+ MutexLock l(&mu_);
+
+ uint64_t start = env_->NowMicros();
+ while (count_ < count) {
+ uint64_t now = env_->NowMicros();
+ cond_count_.TimedWait(now + /*1s*/ 1 * 1000 * 1000);
+ if (env_->NowMicros() - start > /*10s*/ 10 * 1000 * 1000) {
+ return false;
+ }
+ if (count_ < count) {
+ GTEST_LOG_(WARNING) << "WaitFor is taking more time than usual";
+ }
+ }
+
+ return true;
+ }
+
+ void Reset() {
+ MutexLock l(&mu_);
+ count_ = 0;
+ cond_count_.SignalAll();
+ }
+
+ private:
+ Env* env_;
+ port::Mutex mu_;
+ port::CondVar cond_count_;
+ int count_;
+};
+
+struct OptionsOverride {
+ std::shared_ptr<const FilterPolicy> filter_policy = nullptr;
+ // These will be used only if filter_policy is set
+ bool partition_filters = false;
+ uint64_t metadata_block_size = 1024;
+ BlockBasedTableOptions::IndexType index_type =
+ BlockBasedTableOptions::IndexType::kBinarySearch;
+
+ // Used as a bit mask of individual enums in which to skip an XF test point
+ int skip_policy = 0;
+};
+
+} // namespace anon
+
+enum SkipPolicy { kSkipNone = 0, kSkipNoSnapshot = 1, kSkipNoPrefix = 2 };
+
+// A hacky skip list mem table that triggers flush after number of entries.
+class SpecialMemTableRep : public MemTableRep {
+ public:
+ explicit SpecialMemTableRep(Allocator* allocator, MemTableRep* memtable,
+ int num_entries_flush)
+ : MemTableRep(allocator),
+ memtable_(memtable),
+ num_entries_flush_(num_entries_flush),
+ num_entries_(0) {}
+
+ virtual KeyHandle Allocate(const size_t len, char** buf) override {
+ return memtable_->Allocate(len, buf);
+ }
+
+ // Insert key into the list.
+ // REQUIRES: nothing that compares equal to key is currently in the list.
+ virtual void Insert(KeyHandle handle) override {
+ memtable_->Insert(handle);
+ num_entries_++;
+ }
+
+ // Returns true iff an entry that compares equal to key is in the list.
+ virtual bool Contains(const char* key) const override {
+ return memtable_->Contains(key);
+ }
+
+ virtual size_t ApproximateMemoryUsage() override {
+ // Return a high memory usage when number of entries exceeds the threshold
+ // to trigger a flush.
+ return (num_entries_ < num_entries_flush_) ? 0 : 1024 * 1024 * 1024;
+ }
+
+ virtual void Get(const LookupKey& k, void* callback_args,
+ bool (*callback_func)(void* arg,
+ const char* entry)) override {
+ memtable_->Get(k, callback_args, callback_func);
+ }
+
+ uint64_t ApproximateNumEntries(const Slice& start_ikey,
+ const Slice& end_ikey) override {
+ return memtable_->ApproximateNumEntries(start_ikey, end_ikey);
+ }
+
+ virtual MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override {
+ return memtable_->GetIterator(arena);
+ }
+
+ virtual ~SpecialMemTableRep() override {}
+
+ private:
+ unique_ptr<MemTableRep> memtable_;
+ int num_entries_flush_;
+ int num_entries_;
+};
+
+// The factory for the hacky skip list mem table that triggers flush after
+// number of entries exceeds a threshold.
+class SpecialSkipListFactory : public MemTableRepFactory {
+ public:
+ // After number of inserts exceeds `num_entries_flush` in a mem table, trigger
+ // flush.
+ explicit SpecialSkipListFactory(int num_entries_flush)
+ : num_entries_flush_(num_entries_flush) {}
+
+ using MemTableRepFactory::CreateMemTableRep;
+ virtual MemTableRep* CreateMemTableRep(
+ const MemTableRep::KeyComparator& compare, Allocator* allocator,
+ const SliceTransform* transform, Logger* logger) override {
+ return new SpecialMemTableRep(
+ allocator, factory_.CreateMemTableRep(compare, allocator, transform, 0),
+ num_entries_flush_);
+ }
+ virtual const char* Name() const override { return "SkipListFactory"; }
+
+ bool IsInsertConcurrentlySupported() const override {
+ return factory_.IsInsertConcurrentlySupported();
+ }
+
+ private:
+ SkipListFactory factory_;
+ int num_entries_flush_;
+};
+
+// Special Env used to delay background operations
+class SpecialEnv : public EnvWrapper {
+ public:
+ explicit SpecialEnv(Env* base);
+
+ Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
+ const EnvOptions& soptions) override {
+ class SSTableFile : public WritableFile {
+ private:
+ SpecialEnv* env_;
+ unique_ptr<WritableFile> base_;
+
+ public:
+ SSTableFile(SpecialEnv* env, unique_ptr<WritableFile>&& base)
+ : env_(env), base_(std::move(base)) {}
+ Status Append(const Slice& data) override {
+ if (env_->table_write_callback_) {
+ (*env_->table_write_callback_)();
+ }
+ if (env_->drop_writes_.load(std::memory_order_acquire)) {
+ // Drop writes on the floor
+ return Status::OK();
+ } else if (env_->no_space_.load(std::memory_order_acquire)) {
+ return Status::NoSpace("No space left on device");
+ } else {
+ env_->bytes_written_ += data.size();
+ return base_->Append(data);
+ }
+ }
+ Status PositionedAppend(const Slice& data, uint64_t offset) override {
+ if (env_->table_write_callback_) {
+ (*env_->table_write_callback_)();
+ }
+ if (env_->drop_writes_.load(std::memory_order_acquire)) {
+ // Drop writes on the floor
+ return Status::OK();
+ } else if (env_->no_space_.load(std::memory_order_acquire)) {
+ return Status::NoSpace("No space left on device");
+ } else {
+ env_->bytes_written_ += data.size();
+ return base_->PositionedAppend(data, offset);
+ }
+ }
+ Status Truncate(uint64_t size) override { return base_->Truncate(size); }
+ Status RangeSync(uint64_t offset, uint64_t nbytes) override {
+ Status s = base_->RangeSync(offset, nbytes);
+#if !(defined NDEBUG) || !defined(OS_WIN)
+ TEST_SYNC_POINT_CALLBACK("SpecialEnv::SStableFile::RangeSync", &s);
+#endif // !(defined NDEBUG) || !defined(OS_WIN)
+ return s;
+ }
+ Status Close() override {
+// SyncPoint is not supported in Released Windows Mode.
+#if !(defined NDEBUG) || !defined(OS_WIN)
+ // Check preallocation size
+ // preallocation size is never passed to base file.
+ size_t preallocation_size = preallocation_block_size();
+ TEST_SYNC_POINT_CALLBACK("DBTestWritableFile.GetPreallocationStatus",
+ &preallocation_size);
+#endif // !(defined NDEBUG) || !defined(OS_WIN)
+ Status s = base_->Close();
+#if !(defined NDEBUG) || !defined(OS_WIN)
+ TEST_SYNC_POINT_CALLBACK("SpecialEnv::SStableFile::Close", &s);
+#endif // !(defined NDEBUG) || !defined(OS_WIN)
+ return s;
+ }
+ Status Flush() override { return base_->Flush(); }
+ Status Sync() override {
+ ++env_->sync_counter_;
+ while (env_->delay_sstable_sync_.load(std::memory_order_acquire)) {
+ env_->SleepForMicroseconds(100000);
+ }
+ Status s = base_->Sync();
+#if !(defined NDEBUG) || !defined(OS_WIN)
+ TEST_SYNC_POINT_CALLBACK("SpecialEnv::SStableFile::Sync", &s);
+#endif // !(defined NDEBUG) || !defined(OS_WIN)
+ return s;
+ }
+ void SetIOPriority(Env::IOPriority pri) override {
+ base_->SetIOPriority(pri);
+ }
+ Env::IOPriority GetIOPriority() override {
+ return base_->GetIOPriority();
+ }
+ bool use_direct_io() const override {
+ return base_->use_direct_io();
+ }
+ Status Allocate(uint64_t offset, uint64_t len) override {
+ return base_->Allocate(offset, len);
+ }
+ };
+ class ManifestFile : public WritableFile {
+ public:
+ ManifestFile(SpecialEnv* env, unique_ptr<WritableFile>&& b)
+ : env_(env), base_(std::move(b)) {}
+ Status Append(const Slice& data) override {
+ if (env_->manifest_write_error_.load(std::memory_order_acquire)) {
+ return Status::IOError("simulated writer error");
+ } else {
+ return base_->Append(data);
+ }
+ }
+ Status Truncate(uint64_t size) override { return base_->Truncate(size); }
+ Status Close() override { return base_->Close(); }
+ Status Flush() override { return base_->Flush(); }
+ Status Sync() override {
+ ++env_->sync_counter_;
+ if (env_->manifest_sync_error_.load(std::memory_order_acquire)) {
+ return Status::IOError("simulated sync error");
+ } else {
+ return base_->Sync();
+ }
+ }
+ uint64_t GetFileSize() override { return base_->GetFileSize(); }
+
+ private:
+ SpecialEnv* env_;
+ unique_ptr<WritableFile> base_;
+ };
+ class WalFile : public WritableFile {
+ public:
+ WalFile(SpecialEnv* env, unique_ptr<WritableFile>&& b)
+ : env_(env), base_(std::move(b)) {
+ env_->num_open_wal_file_.fetch_add(1);
+ }
+ virtual ~WalFile() { env_->num_open_wal_file_.fetch_add(-1); }
+ Status Append(const Slice& data) override {
+#if !(defined NDEBUG) || !defined(OS_WIN)
+ TEST_SYNC_POINT("SpecialEnv::WalFile::Append:1");
+#endif
+ Status s;
+ if (env_->log_write_error_.load(std::memory_order_acquire)) {
+ s = Status::IOError("simulated writer error");
+ } else {
+ int slowdown =
+ env_->log_write_slowdown_.load(std::memory_order_acquire);
+ if (slowdown > 0) {
+ env_->SleepForMicroseconds(slowdown);
+ }
+ s = base_->Append(data);
+ }
+#if !(defined NDEBUG) || !defined(OS_WIN)
+ TEST_SYNC_POINT("SpecialEnv::WalFile::Append:2");
+#endif
+ return s;
+ }
+ Status Truncate(uint64_t size) override { return base_->Truncate(size); }
+ Status Close() override {
+// SyncPoint is not supported in Released Windows Mode.
+#if !(defined NDEBUG) || !defined(OS_WIN)
+ // Check preallocation size
+ // preallocation size is never passed to base file.
+ size_t preallocation_size = preallocation_block_size();
+ TEST_SYNC_POINT_CALLBACK("DBTestWalFile.GetPreallocationStatus",
+ &preallocation_size);
+#endif // !(defined NDEBUG) || !defined(OS_WIN)
+
+ return base_->Close();
+ }
+ Status Flush() override { return base_->Flush(); }
+ Status Sync() override {
+ ++env_->sync_counter_;
+ return base_->Sync();
+ }
+ bool IsSyncThreadSafe() const override {
+ return env_->is_wal_sync_thread_safe_.load();
+ }
+
+ private:
+ SpecialEnv* env_;
+ unique_ptr<WritableFile> base_;
+ };
+
+ if (non_writeable_rate_.load(std::memory_order_acquire) > 0) {
+ uint32_t random_number;
+ {
+ MutexLock l(&rnd_mutex_);
+ random_number = rnd_.Uniform(100);
+ }
+ if (random_number < non_writeable_rate_.load()) {
+ return Status::IOError("simulated random write error");
+ }
+ }
+
+ new_writable_count_++;
+
+ if (non_writable_count_.load() > 0) {
+ non_writable_count_--;
+ return Status::IOError("simulated write error");
+ }
+
+ EnvOptions optimized = soptions;
+ if (strstr(f.c_str(), "MANIFEST") != nullptr ||
+ strstr(f.c_str(), "log") != nullptr) {
+ optimized.use_mmap_writes = false;
+ optimized.use_direct_writes = false;
+ }
+
+ Status s = target()->NewWritableFile(f, r, optimized);
+ if (s.ok()) {
+ if (strstr(f.c_str(), ".sst") != nullptr) {
+ r->reset(new SSTableFile(this, std::move(*r)));
+ } else if (strstr(f.c_str(), "MANIFEST") != nullptr) {
+ r->reset(new ManifestFile(this, std::move(*r)));
+ } else if (strstr(f.c_str(), "log") != nullptr) {
+ r->reset(new WalFile(this, std::move(*r)));
+ }
+ }
+ return s;
+ }
+
+ Status NewRandomAccessFile(const std::string& f,
+ unique_ptr<RandomAccessFile>* r,
+ const EnvOptions& soptions) override {
+ class CountingFile : public RandomAccessFile {
+ public:
+ CountingFile(unique_ptr<RandomAccessFile>&& target,
+ anon::AtomicCounter* counter,
+ std::atomic<size_t>* bytes_read)
+ : target_(std::move(target)),
+ counter_(counter),
+ bytes_read_(bytes_read) {}
+ virtual Status Read(uint64_t offset, size_t n, Slice* result,
+ char* scratch) const override {
+ counter_->Increment();
+ Status s = target_->Read(offset, n, result, scratch);
+ *bytes_read_ += result->size();
+ return s;
+ }
+
+ private:
+ unique_ptr<RandomAccessFile> target_;
+ anon::AtomicCounter* counter_;
+ std::atomic<size_t>* bytes_read_;
+ };
+
+ Status s = target()->NewRandomAccessFile(f, r, soptions);
+ random_file_open_counter_++;
+ if (s.ok() && count_random_reads_) {
+ r->reset(new CountingFile(std::move(*r), &random_read_counter_,
+ &random_read_bytes_counter_));
+ }
+ return s;
+ }
+
+ Status NewSequentialFile(const std::string& f, unique_ptr<SequentialFile>* r,
+ const EnvOptions& soptions) override {
+ class CountingFile : public SequentialFile {
+ public:
+ CountingFile(unique_ptr<SequentialFile>&& target,
+ anon::AtomicCounter* counter)
+ : target_(std::move(target)), counter_(counter) {}
+ virtual Status Read(size_t n, Slice* result, char* scratch) override {
+ counter_->Increment();
+ return target_->Read(n, result, scratch);
+ }
+ virtual Status Skip(uint64_t n) override { return target_->Skip(n); }
+
+ private:
+ unique_ptr<SequentialFile> target_;
+ anon::AtomicCounter* counter_;
+ };
+
+ Status s = target()->NewSequentialFile(f, r, soptions);
+ if (s.ok() && count_sequential_reads_) {
+ r->reset(new CountingFile(std::move(*r), &sequential_read_counter_));
+ }
+ return s;
+ }
+
+ virtual void SleepForMicroseconds(int micros) override {
+ sleep_counter_.Increment();
+ if (no_slowdown_ || time_elapse_only_sleep_) {
+ addon_time_.fetch_add(micros);
+ }
+ if (!no_slowdown_) {
+ target()->SleepForMicroseconds(micros);
+ }
+ }
+
+ virtual Status GetCurrentTime(int64_t* unix_time) override {
+ Status s;
+ if (!time_elapse_only_sleep_) {
+ s = target()->GetCurrentTime(unix_time);
+ }
+ if (s.ok()) {
+ *unix_time += addon_time_.load();
+ }
+ return s;
+ }
+
+ virtual uint64_t NowNanos() override {
+ return (time_elapse_only_sleep_ ? 0 : target()->NowNanos()) +
+ addon_time_.load() * 1000;
+ }
+
+ virtual uint64_t NowMicros() override {
+ return (time_elapse_only_sleep_ ? 0 : target()->NowMicros()) +
+ addon_time_.load();
+ }
+
+ virtual Status DeleteFile(const std::string& fname) override {
+ delete_count_.fetch_add(1);
+ return target()->DeleteFile(fname);
+ }
+
+ Random rnd_;
+ port::Mutex rnd_mutex_; // Lock to pretect rnd_
+
+ // sstable Sync() calls are blocked while this pointer is non-nullptr.
+ std::atomic<bool> delay_sstable_sync_;
+
+ // Drop writes on the floor while this pointer is non-nullptr.
+ std::atomic<bool> drop_writes_;
+
+ // Simulate no-space errors while this pointer is non-nullptr.
+ std::atomic<bool> no_space_;
+
+ // Simulate non-writable file system while this pointer is non-nullptr
+ std::atomic<bool> non_writable_;
+
+ // Force sync of manifest files to fail while this pointer is non-nullptr
+ std::atomic<bool> manifest_sync_error_;
+
+ // Force write to manifest files to fail while this pointer is non-nullptr
+ std::atomic<bool> manifest_write_error_;
+
+ // Force write to log files to fail while this pointer is non-nullptr
+ std::atomic<bool> log_write_error_;
+
+ // Slow down every log write, in micro-seconds.
+ std::atomic<int> log_write_slowdown_;
+
+ // Number of WAL files that are still open for write.
+ std::atomic<int> num_open_wal_file_;
+
+ bool count_random_reads_;
+ anon::AtomicCounter random_read_counter_;
+ std::atomic<size_t> random_read_bytes_counter_;
+ std::atomic<int> random_file_open_counter_;
+
+ bool count_sequential_reads_;
+ anon::AtomicCounter sequential_read_counter_;
+
+ anon::AtomicCounter sleep_counter_;
+
+ std::atomic<int64_t> bytes_written_;
+
+ std::atomic<int> sync_counter_;
+
+ std::atomic<uint32_t> non_writeable_rate_;
+
+ std::atomic<uint32_t> new_writable_count_;
+
+ std::atomic<uint32_t> non_writable_count_;
+
+ std::function<void()>* table_write_callback_;
+
+ std::atomic<int64_t> addon_time_;
+
+ std::atomic<int> delete_count_;
+
+ bool time_elapse_only_sleep_;
+
+ bool no_slowdown_;
+
+ std::atomic<bool> is_wal_sync_thread_safe_{true};
+};
+
+#ifndef ROCKSDB_LITE
+class OnFileDeletionListener : public EventListener {
+ public:
+ OnFileDeletionListener() : matched_count_(0), expected_file_name_("") {}
+
+ void SetExpectedFileName(const std::string file_name) {
+ expected_file_name_ = file_name;
+ }
+
+ void VerifyMatchedCount(size_t expected_value) {
+ ASSERT_EQ(matched_count_, expected_value);
+ }
+
+ void OnTableFileDeleted(const TableFileDeletionInfo& info) override {
+ if (expected_file_name_ != "") {
+ ASSERT_EQ(expected_file_name_, info.file_path);
+ expected_file_name_ = "";
+ matched_count_++;
+ }
+ }
+
+ private:
+ size_t matched_count_;
+ std::string expected_file_name_;
+};
+#endif
+
+// A test merge operator mimics put but also fails if one of merge operands is
+// "corrupted".
+class TestPutOperator : public MergeOperator {
+ public:
+ virtual bool FullMergeV2(const MergeOperationInput& merge_in,
+ MergeOperationOutput* merge_out) const override {
+ if (merge_in.existing_value != nullptr &&
+ *(merge_in.existing_value) == "corrupted") {
+ return false;
+ }
+ for (auto value : merge_in.operand_list) {
+ if (value == "corrupted") {
+ return false;
+ }
+ }
+ merge_out->existing_operand = merge_in.operand_list.back();
+ return true;
+ }
+
+ virtual const char* Name() const override { return "TestPutOperator"; }
+};
+
+class DBTestBase : public testing::Test {
+ public:
+ // Sequence of option configurations to try
+ enum OptionConfig : int {
+ kDefault = 0,
+ kBlockBasedTableWithPrefixHashIndex = 1,
+ kBlockBasedTableWithWholeKeyHashIndex = 2,
+ kPlainTableFirstBytePrefix = 3,
+ kPlainTableCappedPrefix = 4,
+ kPlainTableCappedPrefixNonMmap = 5,
+ kPlainTableAllBytesPrefix = 6,
+ kVectorRep = 7,
+ kHashLinkList = 8,
+ kHashCuckoo = 9,
+ kMergePut = 10,
+ kFilter = 11,
+ kFullFilterWithNewTableReaderForCompactions = 12,
+ kUncompressed = 13,
+ kNumLevel_3 = 14,
+ kDBLogDir = 15,
+ kWalDirAndMmapReads = 16,
+ kManifestFileSize = 17,
+ kPerfOptions = 18,
+ kHashSkipList = 19,
+ kUniversalCompaction = 20,
+ kUniversalCompactionMultiLevel = 21,
+ kCompressedBlockCache = 22,
+ kInfiniteMaxOpenFiles = 23,
+ kxxHashChecksum = 24,
+ kFIFOCompaction = 25,
+ kOptimizeFiltersForHits = 26,
+ kRowCache = 27,
+ kRecycleLogFiles = 28,
+ kConcurrentSkipList = 29,
+ kPipelinedWrite = 30,
+ kConcurrentWALWrites = 31,
+ kEnd = 32,
+ kDirectIO = 33,
+ kLevelSubcompactions = 34,
+ kUniversalSubcompactions = 35,
+ kBlockBasedTableWithIndexRestartInterval = 36,
+ kBlockBasedTableWithPartitionedIndex = 37,
+ kPartitionedFilterWithNewTableReaderForCompactions = 38,
+ };
+
+ public:
+ std::string dbname_;
+ std::string alternative_wal_dir_;
+ std::string alternative_db_log_dir_;
+ MockEnv* mem_env_;
+ Env* encrypted_env_;
+ SpecialEnv* env_;
+ DB* db_;
+ std::vector<ColumnFamilyHandle*> handles_;
+
+ int option_config_;
+ Options last_options_;
+
+ // Skip some options, as they may not be applicable to a specific test.
+ // To add more skip constants, use values 4, 8, 16, etc.
+ enum OptionSkip {
+ kNoSkip = 0,
+ kSkipDeletesFilterFirst = 1,
+ kSkipUniversalCompaction = 2,
+ kSkipMergePut = 4,
+ kSkipPlainTable = 8,
+ kSkipHashIndex = 16,
+ kSkipNoSeekToLast = 32,
+ kSkipHashCuckoo = 64,
+ kSkipFIFOCompaction = 128,
+ kSkipMmapReads = 256,
+ };
+
+ explicit DBTestBase(const std::string path);
+
+ ~DBTestBase();
+
+ static std::string RandomString(Random* rnd, int len) {
+ std::string r;
+ test::RandomString(rnd, len, &r);
+ return r;
+ }
+
+ static std::string Key(int i) {
+ char buf[100];
+ snprintf(buf, sizeof(buf), "key%06d", i);
+ return std::string(buf);
+ }
+
+ static bool ShouldSkipOptions(int option_config, int skip_mask = kNoSkip);
+
+ // Switch to a fresh database with the next option configuration to
+ // test. Return false if there are no more configurations to test.
+ bool ChangeOptions(int skip_mask = kNoSkip);
+
+ // Switch between different compaction styles.
+ bool ChangeCompactOptions();
+
+ // Switch between different WAL-realted options.
+ bool ChangeWalOptions();
+
+ // Switch between different filter policy
+ // Jump from kDefault to kFilter to kFullFilter
+ bool ChangeFilterOptions();
+
+ // Return the current option configuration.
+ Options CurrentOptions(const anon::OptionsOverride& options_override =
+ anon::OptionsOverride()) const;
+
+ Options CurrentOptions(const Options& default_options,
+ const anon::OptionsOverride& options_override =
+ anon::OptionsOverride()) const;
+
+ static Options GetDefaultOptions();
+
+ Options GetOptions(int option_config,
+ const Options& default_options = GetDefaultOptions(),
+ const anon::OptionsOverride& options_override =
+ anon::OptionsOverride()) const;
+
+ DBImpl* dbfull() { return reinterpret_cast<DBImpl*>(db_); }
+
+ void CreateColumnFamilies(const std::vector<std::string>& cfs,
+ const Options& options);
+
+ void CreateAndReopenWithCF(const std::vector<std::string>& cfs,
+ const Options& options);
+
+ void ReopenWithColumnFamilies(const std::vector<std::string>& cfs,
+ const std::vector<Options>& options);
+
+ void ReopenWithColumnFamilies(const std::vector<std::string>& cfs,
+ const Options& options);
+
+ Status TryReopenWithColumnFamilies(const std::vector<std::string>& cfs,
+ const std::vector<Options>& options);
+
+ Status TryReopenWithColumnFamilies(const std::vector<std::string>& cfs,
+ const Options& options);
+
+ void Reopen(const Options& options);
+
+ void Close();
+
+ void DestroyAndReopen(const Options& options);
+
+ void Destroy(const Options& options);
+
+ Status ReadOnlyReopen(const Options& options);
+
+ Status TryReopen(const Options& options);
+
+ bool IsDirectIOSupported();
+
+ bool IsMemoryMappedAccessSupported() const;
+
+ Status Flush(int cf = 0);
+
+ Status Put(const Slice& k, const Slice& v, WriteOptions wo = WriteOptions());
+
+ Status Put(int cf, const Slice& k, const Slice& v,
+ WriteOptions wo = WriteOptions());
+
+ Status Merge(const Slice& k, const Slice& v,
+ WriteOptions wo = WriteOptions());
+
+ Status Merge(int cf, const Slice& k, const Slice& v,
+ WriteOptions wo = WriteOptions());
+
+ Status Delete(const std::string& k);
+
+ Status Delete(int cf, const std::string& k);
+
+ Status SingleDelete(const std::string& k);
+
+ Status SingleDelete(int cf, const std::string& k);
+
+ std::string Get(const std::string& k, const Snapshot* snapshot = nullptr);
+
+ std::string Get(int cf, const std::string& k,
+ const Snapshot* snapshot = nullptr);
+
+ Status Get(const std::string& k, PinnableSlice* v);
+
+ uint64_t GetNumSnapshots();
+
+ uint64_t GetTimeOldestSnapshots();
+
+ // Return a string that contains all key,value pairs in order,
+ // formatted like "(k1->v1)(k2->v2)".
+ std::string Contents(int cf = 0);
+
+ std::string AllEntriesFor(const Slice& user_key, int cf = 0);
+
+#ifndef ROCKSDB_LITE
+ int NumSortedRuns(int cf = 0);
+
+ uint64_t TotalSize(int cf = 0);
+
+ uint64_t SizeAtLevel(int level);
+
+ size_t TotalLiveFiles(int cf = 0);
+
+ size_t CountLiveFiles();
+#endif // ROCKSDB_LITE
+
+ int NumTableFilesAtLevel(int level, int cf = 0);
+
+ double CompressionRatioAtLevel(int level, int cf = 0);
+
+ int TotalTableFiles(int cf = 0, int levels = -1);
+
+ // Return spread of files per level
+ std::string FilesPerLevel(int cf = 0);
+
+ size_t CountFiles();
+
+ uint64_t Size(const Slice& start, const Slice& limit, int cf = 0);
+
+ void Compact(int cf, const Slice& start, const Slice& limit,
+ uint32_t target_path_id);
+
+ void Compact(int cf, const Slice& start, const Slice& limit);
+
+ void Compact(const Slice& start, const Slice& limit);
+
+ // Do n memtable compactions, each of which produces an sstable
+ // covering the range [small,large].
+ void MakeTables(int n, const std::string& small, const std::string& large,
+ int cf = 0);
+
+ // Prevent pushing of new sstables into deeper levels by adding
+ // tables that cover a specified range to all levels.
+ void FillLevels(const std::string& smallest, const std::string& largest,
+ int cf);
+
+ void MoveFilesToLevel(int level, int cf = 0);
+
+ void DumpFileCounts(const char* label);
+
+ std::string DumpSSTableList();
+
+ void GetSstFiles(std::string path, std::vector<std::string>* files);
+
+ int GetSstFileCount(std::string path);
+
+ // this will generate non-overlapping files since it keeps increasing key_idx
+ void GenerateNewFile(Random* rnd, int* key_idx, bool nowait = false);
+
+ void GenerateNewFile(int fd, Random* rnd, int* key_idx, bool nowait = false);
+
+ static const int kNumKeysByGenerateNewRandomFile;
+ static const int KNumKeysByGenerateNewFile = 100;
+
+ void GenerateNewRandomFile(Random* rnd, bool nowait = false);
+
+ std::string IterStatus(Iterator* iter);
+
+ Options OptionsForLogIterTest();
+
+ std::string DummyString(size_t len, char c = 'a');
+
+ void VerifyIterLast(std::string expected_key, int cf = 0);
+
+ // Used to test InplaceUpdate
+
+ // If previous value is nullptr or delta is > than previous value,
+ // sets newValue with delta
+ // If previous value is not empty,
+ // updates previous value with 'b' string of previous value size - 1.
+ static UpdateStatus updateInPlaceSmallerSize(char* prevValue,
+ uint32_t* prevSize, Slice delta,
+ std::string* newValue);
+
+ static UpdateStatus updateInPlaceSmallerVarintSize(char* prevValue,
+ uint32_t* prevSize,
+ Slice delta,
+ std::string* newValue);
+
+ static UpdateStatus updateInPlaceLargerSize(char* prevValue,
+ uint32_t* prevSize, Slice delta,
+ std::string* newValue);
+
+ static UpdateStatus updateInPlaceNoAction(char* prevValue, uint32_t* prevSize,
+ Slice delta, std::string* newValue);
+
+ // Utility method to test InplaceUpdate
+ void validateNumberOfEntries(int numValues, int cf = 0);
+
+ void CopyFile(const std::string& source, const std::string& destination,
+ uint64_t size = 0);
+
+ std::unordered_map<std::string, uint64_t> GetAllSSTFiles(
+ uint64_t* total_size = nullptr);
+
+ std::vector<std::uint64_t> ListTableFiles(Env* env, const std::string& path);
+
+ void VerifyDBFromMap(
+ std::map<std::string, std::string> true_data,
+ size_t* total_reads_res = nullptr, bool tailing_iter = false,
+ std::map<std::string, Status> status = std::map<std::string, Status>());
+
+ void VerifyDBInternal(
+ std::vector<std::pair<std::string, std::string>> true_data);
+
+#ifndef ROCKSDB_LITE
+ uint64_t GetNumberOfSstFilesForColumnFamily(DB* db,
+ std::string column_family_name);
+#endif // ROCKSDB_LITE
+
+ uint64_t TestGetTickerCount(const Options& options, Tickers ticker_type) {
+ return options.statistics->getTickerCount(ticker_type);
+ }
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/dbformat.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/dbformat.cc b/thirdparty/rocksdb/db/dbformat.cc
new file mode 100644
index 0000000..20c5449
--- /dev/null
+++ b/thirdparty/rocksdb/db/dbformat.cc
@@ -0,0 +1,177 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include "db/dbformat.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <stdio.h>
+#include "monitoring/perf_context_imp.h"
+#include "port/port.h"
+#include "util/coding.h"
+#include "util/string_util.h"
+
+namespace rocksdb {
+
+// kValueTypeForSeek defines the ValueType that should be passed when
+// constructing a ParsedInternalKey object for seeking to a particular
+// sequence number (since we sort sequence numbers in decreasing order
+// and the value type is embedded as the low 8 bits in the sequence
+// number in internal keys, we need to use the highest-numbered
+// ValueType, not the lowest).
+const ValueType kValueTypeForSeek = kTypeSingleDeletion;
+const ValueType kValueTypeForSeekForPrev = kTypeDeletion;
+
+uint64_t PackSequenceAndType(uint64_t seq, ValueType t) {
+ assert(seq <= kMaxSequenceNumber);
+ assert(IsExtendedValueType(t));
+ return (seq << 8) | t;
+}
+
+void UnPackSequenceAndType(uint64_t packed, uint64_t* seq, ValueType* t) {
+ *seq = packed >> 8;
+ *t = static_cast<ValueType>(packed & 0xff);
+
+ assert(*seq <= kMaxSequenceNumber);
+ assert(IsExtendedValueType(*t));
+}
+
+void AppendInternalKey(std::string* result, const ParsedInternalKey& key) {
+ result->append(key.user_key.data(), key.user_key.size());
+ PutFixed64(result, PackSequenceAndType(key.sequence, key.type));
+}
+
+void AppendInternalKeyFooter(std::string* result, SequenceNumber s,
+ ValueType t) {
+ PutFixed64(result, PackSequenceAndType(s, t));
+}
+
+std::string ParsedInternalKey::DebugString(bool hex) const {
+ char buf[50];
+ snprintf(buf, sizeof(buf), "' seq:%" PRIu64 ", type:%d", sequence,
+ static_cast<int>(type));
+ std::string result = "'";
+ result += user_key.ToString(hex);
+ result += buf;
+ return result;
+}
+
+std::string InternalKey::DebugString(bool hex) const {
+ std::string result;
+ ParsedInternalKey parsed;
+ if (ParseInternalKey(rep_, &parsed)) {
+ result = parsed.DebugString(hex);
+ } else {
+ result = "(bad)";
+ result.append(EscapeString(rep_));
+ }
+ return result;
+}
+
+const char* InternalKeyComparator::Name() const {
+ return name_.c_str();
+}
+
+int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
+ // Order by:
+ // increasing user key (according to user-supplied comparator)
+ // decreasing sequence number
+ // decreasing type (though sequence# should be enough to disambiguate)
+ int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
+ PERF_COUNTER_ADD(user_key_comparison_count, 1);
+ if (r == 0) {
+ const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
+ const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
+ if (anum > bnum) {
+ r = -1;
+ } else if (anum < bnum) {
+ r = +1;
+ }
+ }
+ return r;
+}
+
+int InternalKeyComparator::Compare(const ParsedInternalKey& a,
+ const ParsedInternalKey& b) const {
+ // Order by:
+ // increasing user key (according to user-supplied comparator)
+ // decreasing sequence number
+ // decreasing type (though sequence# should be enough to disambiguate)
+ int r = user_comparator_->Compare(a.user_key, b.user_key);
+ PERF_COUNTER_ADD(user_key_comparison_count, 1);
+ if (r == 0) {
+ if (a.sequence > b.sequence) {
+ r = -1;
+ } else if (a.sequence < b.sequence) {
+ r = +1;
+ } else if (a.type > b.type) {
+ r = -1;
+ } else if (a.type < b.type) {
+ r = +1;
+ }
+ }
+ return r;
+}
+
+void InternalKeyComparator::FindShortestSeparator(
+ std::string* start,
+ const Slice& limit) const {
+ // Attempt to shorten the user portion of the key
+ Slice user_start = ExtractUserKey(*start);
+ Slice user_limit = ExtractUserKey(limit);
+ std::string tmp(user_start.data(), user_start.size());
+ user_comparator_->FindShortestSeparator(&tmp, user_limit);
+ if (tmp.size() <= user_start.size() &&
+ user_comparator_->Compare(user_start, tmp) < 0) {
+ // User key has become shorter physically, but larger logically.
+ // Tack on the earliest possible number to the shortened user key.
+ PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
+ assert(this->Compare(*start, tmp) < 0);
+ assert(this->Compare(tmp, limit) < 0);
+ start->swap(tmp);
+ }
+}
+
+void InternalKeyComparator::FindShortSuccessor(std::string* key) const {
+ Slice user_key = ExtractUserKey(*key);
+ std::string tmp(user_key.data(), user_key.size());
+ user_comparator_->FindShortSuccessor(&tmp);
+ if (tmp.size() <= user_key.size() &&
+ user_comparator_->Compare(user_key, tmp) < 0) {
+ // User key has become shorter physically, but larger logically.
+ // Tack on the earliest possible number to the shortened user key.
+ PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
+ assert(this->Compare(*key, tmp) < 0);
+ key->swap(tmp);
+ }
+}
+
+LookupKey::LookupKey(const Slice& _user_key, SequenceNumber s) {
+ size_t usize = _user_key.size();
+ size_t needed = usize + 13; // A conservative estimate
+ char* dst;
+ if (needed <= sizeof(space_)) {
+ dst = space_;
+ } else {
+ dst = new char[needed];
+ }
+ start_ = dst;
+ // NOTE: We don't support users keys of more than 2GB :)
+ dst = EncodeVarint32(dst, static_cast<uint32_t>(usize + 8));
+ kstart_ = dst;
+ memcpy(dst, _user_key.data(), usize);
+ dst += usize;
+ EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek));
+ dst += 8;
+ end_ = dst;
+}
+
+} // namespace rocksdb
[39/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/thread_annotations.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/thread_annotations.h b/thirdparty/leveldb-1.18/port/thread_annotations.h
deleted file mode 100755
index aa2d1f5..0000000
--- a/thirdparty/leveldb-1.18/port/thread_annotations.h
+++ /dev/null
@@ -1,60 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_
-#define STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_
-
-// Some environments provide custom macros to aid in static thread-safety
-// analysis. Provide empty definitions of such macros unless they are already
-// defined.
-
-#ifndef EXCLUSIVE_LOCKS_REQUIRED
-#define EXCLUSIVE_LOCKS_REQUIRED(...)
-#endif
-
-#ifndef SHARED_LOCKS_REQUIRED
-#define SHARED_LOCKS_REQUIRED(...)
-#endif
-
-#ifndef LOCKS_EXCLUDED
-#define LOCKS_EXCLUDED(...)
-#endif
-
-#ifndef LOCK_RETURNED
-#define LOCK_RETURNED(x)
-#endif
-
-#ifndef LOCKABLE
-#define LOCKABLE
-#endif
-
-#ifndef SCOPED_LOCKABLE
-#define SCOPED_LOCKABLE
-#endif
-
-#ifndef EXCLUSIVE_LOCK_FUNCTION
-#define EXCLUSIVE_LOCK_FUNCTION(...)
-#endif
-
-#ifndef SHARED_LOCK_FUNCTION
-#define SHARED_LOCK_FUNCTION(...)
-#endif
-
-#ifndef EXCLUSIVE_TRYLOCK_FUNCTION
-#define EXCLUSIVE_TRYLOCK_FUNCTION(...)
-#endif
-
-#ifndef SHARED_TRYLOCK_FUNCTION
-#define SHARED_TRYLOCK_FUNCTION(...)
-#endif
-
-#ifndef UNLOCK_FUNCTION
-#define UNLOCK_FUNCTION(...)
-#endif
-
-#ifndef NO_THREAD_SAFETY_ANALYSIS
-#define NO_THREAD_SAFETY_ANALYSIS
-#endif
-
-#endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/win/stdint.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/win/stdint.h b/thirdparty/leveldb-1.18/port/win/stdint.h
deleted file mode 100755
index 39edd0d..0000000
--- a/thirdparty/leveldb-1.18/port/win/stdint.h
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-// MSVC didn't ship with this file until the 2010 version.
-
-#ifndef STORAGE_LEVELDB_PORT_WIN_STDINT_H_
-#define STORAGE_LEVELDB_PORT_WIN_STDINT_H_
-
-#if !defined(_MSC_VER)
-#error This file should only be included when compiling with MSVC.
-#endif
-
-// Define C99 equivalent types.
-typedef signed char int8_t;
-typedef signed short int16_t;
-typedef signed int int32_t;
-typedef signed long long int64_t;
-typedef unsigned char uint8_t;
-typedef unsigned short uint16_t;
-typedef unsigned int uint32_t;
-typedef unsigned long long uint64_t;
-
-#endif // STORAGE_LEVELDB_PORT_WIN_STDINT_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/block.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/block.cc b/thirdparty/leveldb-1.18/table/block.cc
deleted file mode 100755
index 1ead640..0000000
--- a/thirdparty/leveldb-1.18/table/block.cc
+++ /dev/null
@@ -1,268 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Decodes the blocks generated by block_builder.cc.
-
-#include "table/block.h"
-
-#include <vector>
-#include <algorithm>
-#include "leveldb/comparator.h"
-#include "table/format.h"
-#include "util/coding.h"
-#include "util/logging.h"
-
-namespace leveldb {
-
-inline uint32_t Block::NumRestarts() const {
- assert(size_ >= sizeof(uint32_t));
- return DecodeFixed32(data_ + size_ - sizeof(uint32_t));
-}
-
-Block::Block(const BlockContents& contents)
- : data_(contents.data.data()),
- size_(contents.data.size()),
- owned_(contents.heap_allocated) {
- if (size_ < sizeof(uint32_t)) {
- size_ = 0; // Error marker
- } else {
- size_t max_restarts_allowed = (size_-sizeof(uint32_t)) / sizeof(uint32_t);
- if (NumRestarts() > max_restarts_allowed) {
- // The size is too small for NumRestarts()
- size_ = 0;
- } else {
- restart_offset_ = static_cast<uint32_t>(size_ - (1 + NumRestarts()) * sizeof(uint32_t));
- }
- }
-}
-
-Block::~Block() {
- if (owned_) {
- delete[] data_;
- }
-}
-
-// Helper routine: decode the next block entry starting at "p",
-// storing the number of shared key bytes, non_shared key bytes,
-// and the length of the value in "*shared", "*non_shared", and
-// "*value_length", respectively. Will not dereference past "limit".
-//
-// If any errors are detected, returns NULL. Otherwise, returns a
-// pointer to the key delta (just past the three decoded values).
-static inline const char* DecodeEntry(const char* p, const char* limit,
- uint32_t* shared,
- uint32_t* non_shared,
- uint32_t* value_length) {
- if (limit - p < 3) return NULL;
- *shared = reinterpret_cast<const unsigned char*>(p)[0];
- *non_shared = reinterpret_cast<const unsigned char*>(p)[1];
- *value_length = reinterpret_cast<const unsigned char*>(p)[2];
- if ((*shared | *non_shared | *value_length) < 128) {
- // Fast path: all three values are encoded in one byte each
- p += 3;
- } else {
- if ((p = GetVarint32Ptr(p, limit, shared)) == NULL) return NULL;
- if ((p = GetVarint32Ptr(p, limit, non_shared)) == NULL) return NULL;
- if ((p = GetVarint32Ptr(p, limit, value_length)) == NULL) return NULL;
- }
-
- if (static_cast<uint32_t>(limit - p) < (*non_shared + *value_length)) {
- return NULL;
- }
- return p;
-}
-
-class Block::Iter : public Iterator {
- private:
- const Comparator* const comparator_;
- const char* const data_; // underlying block contents
- uint32_t const restarts_; // Offset of restart array (list of fixed32)
- uint32_t const num_restarts_; // Number of uint32_t entries in restart array
-
- // current_ is offset in data_ of current entry. >= restarts_ if !Valid
- uint32_t current_;
- uint32_t restart_index_; // Index of restart block in which current_ falls
- std::string key_;
- Slice value_;
- Status status_;
-
- inline int Compare(const Slice& a, const Slice& b) const {
- return comparator_->Compare(a, b);
- }
-
- // Return the offset in data_ just past the end of the current entry.
- inline uint32_t NextEntryOffset() const {
- return static_cast<uint32_t>((value_.data() + value_.size()) - data_);
- }
-
- uint32_t GetRestartPoint(uint32_t index) {
- assert(index < num_restarts_);
- return DecodeFixed32(data_ + restarts_ + index * sizeof(uint32_t));
- }
-
- void SeekToRestartPoint(uint32_t index) {
- key_.clear();
- restart_index_ = index;
- // current_ will be fixed by ParseNextKey();
-
- // ParseNextKey() starts at the end of value_, so set value_ accordingly
- uint32_t offset = GetRestartPoint(index);
- value_ = Slice(data_ + offset, 0);
- }
-
- public:
- Iter(const Comparator* comparator,
- const char* data,
- uint32_t restarts,
- uint32_t num_restarts)
- : comparator_(comparator),
- data_(data),
- restarts_(restarts),
- num_restarts_(num_restarts),
- current_(restarts_),
- restart_index_(num_restarts_) {
- assert(num_restarts_ > 0);
- }
-
- virtual bool Valid() const { return current_ < restarts_; }
- virtual Status status() const { return status_; }
- virtual Slice key() const {
- assert(Valid());
- return key_;
- }
- virtual Slice value() const {
- assert(Valid());
- return value_;
- }
-
- virtual void Next() {
- assert(Valid());
- ParseNextKey();
- }
-
- virtual void Prev() {
- assert(Valid());
-
- // Scan backwards to a restart point before current_
- const uint32_t original = current_;
- while (GetRestartPoint(restart_index_) >= original) {
- if (restart_index_ == 0) {
- // No more entries
- current_ = restarts_;
- restart_index_ = num_restarts_;
- return;
- }
- restart_index_--;
- }
-
- SeekToRestartPoint(restart_index_);
- do {
- // Loop until end of current entry hits the start of original entry
- } while (ParseNextKey() && NextEntryOffset() < original);
- }
-
- virtual void Seek(const Slice& target) {
- // Binary search in restart array to find the last restart point
- // with a key < target
- uint32_t left = 0;
- uint32_t right = num_restarts_ - 1;
- while (left < right) {
- uint32_t mid = (left + right + 1) / 2;
- uint32_t region_offset = GetRestartPoint(mid);
- uint32_t shared, non_shared, value_length;
- const char* key_ptr = DecodeEntry(data_ + region_offset,
- data_ + restarts_,
- &shared, &non_shared, &value_length);
- if (key_ptr == NULL || (shared != 0)) {
- CorruptionError();
- return;
- }
- Slice mid_key(key_ptr, non_shared);
- if (Compare(mid_key, target) < 0) {
- // Key at "mid" is smaller than "target". Therefore all
- // blocks before "mid" are uninteresting.
- left = mid;
- } else {
- // Key at "mid" is >= "target". Therefore all blocks at or
- // after "mid" are uninteresting.
- right = mid - 1;
- }
- }
-
- // Linear search (within restart block) for first key >= target
- SeekToRestartPoint(left);
- while (true) {
- if (!ParseNextKey()) {
- return;
- }
- if (Compare(key_, target) >= 0) {
- return;
- }
- }
- }
-
- virtual void SeekToFirst() {
- SeekToRestartPoint(0);
- ParseNextKey();
- }
-
- virtual void SeekToLast() {
- SeekToRestartPoint(num_restarts_ - 1);
- while (ParseNextKey() && NextEntryOffset() < restarts_) {
- // Keep skipping
- }
- }
-
- private:
- void CorruptionError() {
- current_ = restarts_;
- restart_index_ = num_restarts_;
- status_ = Status::Corruption("bad entry in block");
- key_.clear();
- value_.clear();
- }
-
- bool ParseNextKey() {
- current_ = NextEntryOffset();
- const char* p = data_ + current_;
- const char* limit = data_ + restarts_; // Restarts come right after data
- if (p >= limit) {
- // No more entries to return. Mark as invalid.
- current_ = restarts_;
- restart_index_ = num_restarts_;
- return false;
- }
-
- // Decode next entry
- uint32_t shared, non_shared, value_length;
- p = DecodeEntry(p, limit, &shared, &non_shared, &value_length);
- if (p == NULL || key_.size() < shared) {
- CorruptionError();
- return false;
- } else {
- key_.resize(shared);
- key_.append(p, non_shared);
- value_ = Slice(p + non_shared, value_length);
- while (restart_index_ + 1 < num_restarts_ &&
- GetRestartPoint(restart_index_ + 1) < current_) {
- ++restart_index_;
- }
- return true;
- }
- }
-};
-
-Iterator* Block::NewIterator(const Comparator* cmp) {
- if (size_ < sizeof(uint32_t)) {
- return NewErrorIterator(Status::Corruption("bad block contents"));
- }
- const uint32_t num_restarts = NumRestarts();
- if (num_restarts == 0) {
- return NewEmptyIterator();
- } else {
- return new Iter(cmp, data_, restart_offset_, num_restarts);
- }
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/block.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/block.h b/thirdparty/leveldb-1.18/table/block.h
deleted file mode 100755
index 2493eb9..0000000
--- a/thirdparty/leveldb-1.18/table/block.h
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_TABLE_BLOCK_H_
-#define STORAGE_LEVELDB_TABLE_BLOCK_H_
-
-#include <stddef.h>
-#include <stdint.h>
-#include "leveldb/iterator.h"
-
-namespace leveldb {
-
-struct BlockContents;
-class Comparator;
-
-class Block {
- public:
- // Initialize the block with the specified contents.
- explicit Block(const BlockContents& contents);
-
- ~Block();
-
- size_t size() const { return size_; }
- Iterator* NewIterator(const Comparator* comparator);
-
- private:
- uint32_t NumRestarts() const;
-
- const char* data_;
- size_t size_;
- uint32_t restart_offset_; // Offset in data_ of restart array
- bool owned_; // Block owns data_[]
-
- // No copying allowed
- Block(const Block&);
- void operator=(const Block&);
-
- class Iter;
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_TABLE_BLOCK_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/block_builder.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/block_builder.cc b/thirdparty/leveldb-1.18/table/block_builder.cc
deleted file mode 100755
index 3abd631..0000000
--- a/thirdparty/leveldb-1.18/table/block_builder.cc
+++ /dev/null
@@ -1,109 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// BlockBuilder generates blocks where keys are prefix-compressed:
-//
-// When we store a key, we drop the prefix shared with the previous
-// string. This helps reduce the space requirement significantly.
-// Furthermore, once every K keys, we do not apply the prefix
-// compression and store the entire key. We call this a "restart
-// point". The tail end of the block stores the offsets of all of the
-// restart points, and can be used to do a binary search when looking
-// for a particular key. Values are stored as-is (without compression)
-// immediately following the corresponding key.
-//
-// An entry for a particular key-value pair has the form:
-// shared_bytes: varint32
-// unshared_bytes: varint32
-// value_length: varint32
-// key_delta: char[unshared_bytes]
-// value: char[value_length]
-// shared_bytes == 0 for restart points.
-//
-// The trailer of the block has the form:
-// restarts: uint32[num_restarts]
-// num_restarts: uint32
-// restarts[i] contains the offset within the block of the ith restart point.
-
-#include "table/block_builder.h"
-
-#include <algorithm>
-#include <assert.h>
-#include "leveldb/comparator.h"
-#include "leveldb/table_builder.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-BlockBuilder::BlockBuilder(const Options* options)
- : options_(options),
- restarts_(),
- counter_(0),
- finished_(false) {
- assert(options->block_restart_interval >= 1);
- restarts_.push_back(0); // First restart point is at offset 0
-}
-
-void BlockBuilder::Reset() {
- buffer_.clear();
- restarts_.clear();
- restarts_.push_back(0); // First restart point is at offset 0
- counter_ = 0;
- finished_ = false;
- last_key_.clear();
-}
-
-size_t BlockBuilder::CurrentSizeEstimate() const {
- return (buffer_.size() + // Raw data buffer
- restarts_.size() * sizeof(uint32_t) + // Restart array
- sizeof(uint32_t)); // Restart array length
-}
-
-Slice BlockBuilder::Finish() {
- // Append restart array
- for (size_t i = 0; i < restarts_.size(); i++) {
- PutFixed32(&buffer_, restarts_[i]);
- }
- PutFixed32(&buffer_, static_cast<uint32_t>(restarts_.size()));
- finished_ = true;
- return Slice(buffer_);
-}
-
-void BlockBuilder::Add(const Slice& key, const Slice& value) {
- Slice last_key_piece(last_key_);
- assert(!finished_);
- assert(counter_ <= options_->block_restart_interval);
- assert(buffer_.empty() // No values yet?
- || options_->comparator->Compare(key, last_key_piece) > 0);
- size_t shared = 0;
- if (counter_ < options_->block_restart_interval) {
- // See how much sharing to do with previous string
- const size_t min_length = std::min(last_key_piece.size(), key.size());
- while ((shared < min_length) && (last_key_piece[shared] == key[shared])) {
- shared++;
- }
- } else {
- // Restart compression
- restarts_.push_back(static_cast<unsigned int>(buffer_.size()));
- counter_ = 0;
- }
- const size_t non_shared = key.size() - shared;
-
- // Add "<shared><non_shared><value_size>" to buffer_
- PutVarint32(&buffer_, static_cast<uint32_t>(shared));
- PutVarint32(&buffer_, static_cast<uint32_t>(non_shared));
- PutVarint32(&buffer_, static_cast<uint32_t>(value.size()));
-
- // Add string delta to buffer_ followed by value
- buffer_.append(key.data() + shared, non_shared);
- buffer_.append(value.data(), value.size());
-
- // Update state
- last_key_.resize(shared);
- last_key_.append(key.data() + shared, non_shared);
- assert(Slice(last_key_) == key);
- counter_++;
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/block_builder.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/block_builder.h b/thirdparty/leveldb-1.18/table/block_builder.h
deleted file mode 100755
index 4fbcb33..0000000
--- a/thirdparty/leveldb-1.18/table/block_builder.h
+++ /dev/null
@@ -1,57 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_
-#define STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_
-
-#include <vector>
-
-#include <stdint.h>
-#include "leveldb/slice.h"
-
-namespace leveldb {
-
-struct Options;
-
-class BlockBuilder {
- public:
- explicit BlockBuilder(const Options* options);
-
- // Reset the contents as if the BlockBuilder was just constructed.
- void Reset();
-
- // REQUIRES: Finish() has not been called since the last call to Reset().
- // REQUIRES: key is larger than any previously added key
- void Add(const Slice& key, const Slice& value);
-
- // Finish building the block and return a slice that refers to the
- // block contents. The returned slice will remain valid for the
- // lifetime of this builder or until Reset() is called.
- Slice Finish();
-
- // Returns an estimate of the current (uncompressed) size of the block
- // we are building.
- size_t CurrentSizeEstimate() const;
-
- // Return true iff no entries have been added since the last Reset()
- bool empty() const {
- return buffer_.empty();
- }
-
- private:
- const Options* options_;
- std::string buffer_; // Destination buffer
- std::vector<uint32_t> restarts_; // Restart points
- int counter_; // Number of entries emitted since restart
- bool finished_; // Has Finish() been called?
- std::string last_key_;
-
- // No copying allowed
- BlockBuilder(const BlockBuilder&);
- void operator=(const BlockBuilder&);
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_TABLE_BLOCK_BUILDER_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/filter_block.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/filter_block.cc b/thirdparty/leveldb-1.18/table/filter_block.cc
deleted file mode 100755
index 1981710..0000000
--- a/thirdparty/leveldb-1.18/table/filter_block.cc
+++ /dev/null
@@ -1,111 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "table/filter_block.h"
-
-#include "leveldb/filter_policy.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-// See doc/table_format.txt for an explanation of the filter block format.
-
-// Generate new filter every 2KB of data
-static const size_t kFilterBaseLg = 11;
-static const size_t kFilterBase = 1 << kFilterBaseLg;
-
-FilterBlockBuilder::FilterBlockBuilder(const FilterPolicy* policy)
- : policy_(policy) {
-}
-
-void FilterBlockBuilder::StartBlock(uint64_t block_offset) {
- uint64_t filter_index = (block_offset / kFilterBase);
- assert(filter_index >= filter_offsets_.size());
- while (filter_index > filter_offsets_.size()) {
- GenerateFilter();
- }
-}
-
-void FilterBlockBuilder::AddKey(const Slice& key) {
- Slice k = key;
- start_.push_back(keys_.size());
- keys_.append(k.data(), k.size());
-}
-
-Slice FilterBlockBuilder::Finish() {
- if (!start_.empty()) {
- GenerateFilter();
- }
-
- // Append array of per-filter offsets
- const uint32_t array_offset = static_cast<uint32_t>(result_.size());
- for (size_t i = 0; i < filter_offsets_.size(); i++) {
- PutFixed32(&result_, filter_offsets_[i]);
- }
-
- PutFixed32(&result_, array_offset);
- result_.push_back(kFilterBaseLg); // Save encoding parameter in result
- return Slice(result_);
-}
-
-void FilterBlockBuilder::GenerateFilter() {
- const size_t num_keys = start_.size();
- if (num_keys == 0) {
- // Fast path if there are no keys for this filter
- filter_offsets_.push_back(static_cast<unsigned int>(result_.size()));
- return;
- }
-
- // Make list of keys from flattened key structure
- start_.push_back(keys_.size()); // Simplify length computation
- tmp_keys_.resize(num_keys);
- for (size_t i = 0; i < num_keys; i++) {
- const char* base = keys_.data() + start_[i];
- size_t length = start_[i+1] - start_[i];
- tmp_keys_[i] = Slice(base, length);
- }
-
- // Generate filter for current set of keys and append to result_.
- filter_offsets_.push_back(static_cast<unsigned int>(result_.size()));
- policy_->CreateFilter(&tmp_keys_[0], static_cast<int>(num_keys), &result_);
-
- tmp_keys_.clear();
- keys_.clear();
- start_.clear();
-}
-
-FilterBlockReader::FilterBlockReader(const FilterPolicy* policy,
- const Slice& contents)
- : policy_(policy),
- data_(NULL),
- offset_(NULL),
- num_(0),
- base_lg_(0) {
- size_t n = contents.size();
- if (n < 5) return; // 1 byte for base_lg_ and 4 for start of offset array
- base_lg_ = contents[n-1];
- uint32_t last_word = DecodeFixed32(contents.data() + n - 5);
- if (last_word > n - 5) return;
- data_ = contents.data();
- offset_ = data_ + last_word;
- num_ = (n - 5 - last_word) / 4;
-}
-
-bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) {
- uint64_t index = block_offset >> base_lg_;
- if (index < num_) {
- uint32_t start = DecodeFixed32(offset_ + index*4);
- uint32_t limit = DecodeFixed32(offset_ + index*4 + 4);
- if (start <= limit && limit <= (offset_ - data_)) {
- Slice filter = Slice(data_ + start, limit - start);
- return policy_->KeyMayMatch(key, filter);
- } else if (start == limit) {
- // Empty filters do not match any keys
- return false;
- }
- }
- return true; // Errors are treated as potential matches
-}
-
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/filter_block.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/filter_block.h b/thirdparty/leveldb-1.18/table/filter_block.h
deleted file mode 100755
index c67d010..0000000
--- a/thirdparty/leveldb-1.18/table/filter_block.h
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A filter block is stored near the end of a Table file. It contains
-// filters (e.g., bloom filters) for all data blocks in the table combined
-// into a single filter block.
-
-#ifndef STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_
-#define STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_
-
-#include <stddef.h>
-#include <stdint.h>
-#include <string>
-#include <vector>
-#include "leveldb/slice.h"
-#include "util/hash.h"
-
-namespace leveldb {
-
-class FilterPolicy;
-
-// A FilterBlockBuilder is used to construct all of the filters for a
-// particular Table. It generates a single string which is stored as
-// a special block in the Table.
-//
-// The sequence of calls to FilterBlockBuilder must match the regexp:
-// (StartBlock AddKey*)* Finish
-class FilterBlockBuilder {
- public:
- explicit FilterBlockBuilder(const FilterPolicy*);
-
- void StartBlock(uint64_t block_offset);
- void AddKey(const Slice& key);
- Slice Finish();
-
- private:
- void GenerateFilter();
-
- const FilterPolicy* policy_;
- std::string keys_; // Flattened key contents
- std::vector<size_t> start_; // Starting index in keys_ of each key
- std::string result_; // Filter data computed so far
- std::vector<Slice> tmp_keys_; // policy_->CreateFilter() argument
- std::vector<uint32_t> filter_offsets_;
-
- // No copying allowed
- FilterBlockBuilder(const FilterBlockBuilder&);
- void operator=(const FilterBlockBuilder&);
-};
-
-class FilterBlockReader {
- public:
- // REQUIRES: "contents" and *policy must stay live while *this is live.
- FilterBlockReader(const FilterPolicy* policy, const Slice& contents);
- bool KeyMayMatch(uint64_t block_offset, const Slice& key);
-
- private:
- const FilterPolicy* policy_;
- const char* data_; // Pointer to filter data (at block-start)
- const char* offset_; // Pointer to beginning of offset array (at block-end)
- size_t num_; // Number of entries in offset array
- size_t base_lg_; // Encoding parameter (see kFilterBaseLg in .cc file)
-};
-
-}
-
-#endif // STORAGE_LEVELDB_TABLE_FILTER_BLOCK_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/filter_block_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/filter_block_test.cc b/thirdparty/leveldb-1.18/table/filter_block_test.cc
deleted file mode 100755
index 8c4a474..0000000
--- a/thirdparty/leveldb-1.18/table/filter_block_test.cc
+++ /dev/null
@@ -1,128 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "table/filter_block.h"
-
-#include "leveldb/filter_policy.h"
-#include "util/coding.h"
-#include "util/hash.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-// For testing: emit an array with one hash value per key
-class TestHashFilter : public FilterPolicy {
- public:
- virtual const char* Name() const {
- return "TestHashFilter";
- }
-
- virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {
- for (int i = 0; i < n; i++) {
- uint32_t h = Hash(keys[i].data(), keys[i].size(), 1);
- PutFixed32(dst, h);
- }
- }
-
- virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const {
- uint32_t h = Hash(key.data(), key.size(), 1);
- for (size_t i = 0; i + 4 <= filter.size(); i += 4) {
- if (h == DecodeFixed32(filter.data() + i)) {
- return true;
- }
- }
- return false;
- }
-};
-
-class FilterBlockTest {
- public:
- TestHashFilter policy_;
-};
-
-TEST(FilterBlockTest, EmptyBuilder) {
- FilterBlockBuilder builder(&policy_);
- Slice block = builder.Finish();
- ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block));
- FilterBlockReader reader(&policy_, block);
- ASSERT_TRUE(reader.KeyMayMatch(0, "foo"));
- ASSERT_TRUE(reader.KeyMayMatch(100000, "foo"));
-}
-
-TEST(FilterBlockTest, SingleChunk) {
- FilterBlockBuilder builder(&policy_);
- builder.StartBlock(100);
- builder.AddKey("foo");
- builder.AddKey("bar");
- builder.AddKey("box");
- builder.StartBlock(200);
- builder.AddKey("box");
- builder.StartBlock(300);
- builder.AddKey("hello");
- Slice block = builder.Finish();
- FilterBlockReader reader(&policy_, block);
- ASSERT_TRUE(reader.KeyMayMatch(100, "foo"));
- ASSERT_TRUE(reader.KeyMayMatch(100, "bar"));
- ASSERT_TRUE(reader.KeyMayMatch(100, "box"));
- ASSERT_TRUE(reader.KeyMayMatch(100, "hello"));
- ASSERT_TRUE(reader.KeyMayMatch(100, "foo"));
- ASSERT_TRUE(! reader.KeyMayMatch(100, "missing"));
- ASSERT_TRUE(! reader.KeyMayMatch(100, "other"));
-}
-
-TEST(FilterBlockTest, MultiChunk) {
- FilterBlockBuilder builder(&policy_);
-
- // First filter
- builder.StartBlock(0);
- builder.AddKey("foo");
- builder.StartBlock(2000);
- builder.AddKey("bar");
-
- // Second filter
- builder.StartBlock(3100);
- builder.AddKey("box");
-
- // Third filter is empty
-
- // Last filter
- builder.StartBlock(9000);
- builder.AddKey("box");
- builder.AddKey("hello");
-
- Slice block = builder.Finish();
- FilterBlockReader reader(&policy_, block);
-
- // Check first filter
- ASSERT_TRUE(reader.KeyMayMatch(0, "foo"));
- ASSERT_TRUE(reader.KeyMayMatch(2000, "bar"));
- ASSERT_TRUE(! reader.KeyMayMatch(0, "box"));
- ASSERT_TRUE(! reader.KeyMayMatch(0, "hello"));
-
- // Check second filter
- ASSERT_TRUE(reader.KeyMayMatch(3100, "box"));
- ASSERT_TRUE(! reader.KeyMayMatch(3100, "foo"));
- ASSERT_TRUE(! reader.KeyMayMatch(3100, "bar"));
- ASSERT_TRUE(! reader.KeyMayMatch(3100, "hello"));
-
- // Check third filter (empty)
- ASSERT_TRUE(! reader.KeyMayMatch(4100, "foo"));
- ASSERT_TRUE(! reader.KeyMayMatch(4100, "bar"));
- ASSERT_TRUE(! reader.KeyMayMatch(4100, "box"));
- ASSERT_TRUE(! reader.KeyMayMatch(4100, "hello"));
-
- // Check last filter
- ASSERT_TRUE(reader.KeyMayMatch(9000, "box"));
- ASSERT_TRUE(reader.KeyMayMatch(9000, "hello"));
- ASSERT_TRUE(! reader.KeyMayMatch(9000, "foo"));
- ASSERT_TRUE(! reader.KeyMayMatch(9000, "bar"));
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/format.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/format.cc b/thirdparty/leveldb-1.18/table/format.cc
deleted file mode 100755
index aa63144..0000000
--- a/thirdparty/leveldb-1.18/table/format.cc
+++ /dev/null
@@ -1,145 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "table/format.h"
-
-#include "leveldb/env.h"
-#include "port/port.h"
-#include "table/block.h"
-#include "util/coding.h"
-#include "util/crc32c.h"
-
-namespace leveldb {
-
-void BlockHandle::EncodeTo(std::string* dst) const {
- // Sanity check that all fields have been set
- assert(offset_ != ~static_cast<uint64_t>(0));
- assert(size_ != ~static_cast<uint64_t>(0));
- PutVarint64(dst, offset_);
- PutVarint64(dst, size_);
-}
-
-Status BlockHandle::DecodeFrom(Slice* input) {
- if (GetVarint64(input, &offset_) &&
- GetVarint64(input, &size_)) {
- return Status::OK();
- } else {
- return Status::Corruption("bad block handle");
- }
-}
-
-void Footer::EncodeTo(std::string* dst) const {
-#ifndef NDEBUG
- const size_t original_size = dst->size();
-#endif
- metaindex_handle_.EncodeTo(dst);
- index_handle_.EncodeTo(dst);
- dst->resize(2 * BlockHandle::kMaxEncodedLength); // Padding
- PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber & 0xffffffffu));
- PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber >> 32));
- assert(dst->size() == original_size + kEncodedLength);
-}
-
-Status Footer::DecodeFrom(Slice* input) {
- const char* magic_ptr = input->data() + kEncodedLength - 8;
- const uint32_t magic_lo = DecodeFixed32(magic_ptr);
- const uint32_t magic_hi = DecodeFixed32(magic_ptr + 4);
- const uint64_t magic = ((static_cast<uint64_t>(magic_hi) << 32) |
- (static_cast<uint64_t>(magic_lo)));
- if (magic != kTableMagicNumber) {
- return Status::Corruption("not an sstable (bad magic number)");
- }
-
- Status result = metaindex_handle_.DecodeFrom(input);
- if (result.ok()) {
- result = index_handle_.DecodeFrom(input);
- }
- if (result.ok()) {
- // We skip over any leftover data (just padding for now) in "input"
- const char* end = magic_ptr + 8;
- *input = Slice(end, input->data() + input->size() - end);
- }
- return result;
-}
-
-Status ReadBlock(RandomAccessFile* file,
- const ReadOptions& options,
- const BlockHandle& handle,
- BlockContents* result) {
- result->data = Slice();
- result->cachable = false;
- result->heap_allocated = false;
-
- // Read the block contents as well as the type/crc footer.
- // See table_builder.cc for the code that built this structure.
- size_t n = static_cast<size_t>(handle.size());
- char* buf = new char[n + kBlockTrailerSize];
- Slice contents;
- Status s = file->Read(handle.offset(), n + kBlockTrailerSize, &contents, buf);
- if (!s.ok()) {
- delete[] buf;
- return s;
- }
- if (contents.size() != n + kBlockTrailerSize) {
- delete[] buf;
- return Status::Corruption("truncated block read");
- }
-
- // Check the crc of the type and the block contents
- const char* data = contents.data(); // Pointer to where Read put the data
- if (options.verify_checksums) {
- const uint32_t crc = crc32c::Unmask(DecodeFixed32(data + n + 1));
- const uint32_t actual = crc32c::Value(data, n + 1);
- if (actual != crc) {
- delete[] buf;
- s = Status::Corruption("block checksum mismatch");
- return s;
- }
- }
-
- switch (data[n]) {
- case kNoCompression:
- if (data != buf) {
- // File implementation gave us pointer to some other data.
- // Use it directly under the assumption that it will be live
- // while the file is open.
- delete[] buf;
- result->data = Slice(data, n);
- result->heap_allocated = false;
- result->cachable = false; // Do not double-cache
- } else {
- result->data = Slice(buf, n);
- result->heap_allocated = true;
- result->cachable = true;
- }
-
- // Ok
- break;
- case kSnappyCompression: {
- size_t ulength = 0;
- if (!port::Snappy_GetUncompressedLength(data, n, &ulength)) {
- delete[] buf;
- return Status::Corruption("corrupted compressed block contents");
- }
- char* ubuf = new char[ulength];
- if (!port::Snappy_Uncompress(data, n, ubuf)) {
- delete[] buf;
- delete[] ubuf;
- return Status::Corruption("corrupted compressed block contents");
- }
- delete[] buf;
- result->data = Slice(ubuf, ulength);
- result->heap_allocated = true;
- result->cachable = true;
- break;
- }
- default:
- delete[] buf;
- return Status::Corruption("bad block type");
- }
-
- return Status::OK();
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/format.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/format.h b/thirdparty/leveldb-1.18/table/format.h
deleted file mode 100755
index 6c0b80c..0000000
--- a/thirdparty/leveldb-1.18/table/format.h
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_TABLE_FORMAT_H_
-#define STORAGE_LEVELDB_TABLE_FORMAT_H_
-
-#include <string>
-#include <stdint.h>
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-#include "leveldb/table_builder.h"
-
-namespace leveldb {
-
-class Block;
-class RandomAccessFile;
-struct ReadOptions;
-
-// BlockHandle is a pointer to the extent of a file that stores a data
-// block or a meta block.
-class BlockHandle {
- public:
- BlockHandle();
-
- // The offset of the block in the file.
- uint64_t offset() const { return offset_; }
- void set_offset(uint64_t offset) { offset_ = offset; }
-
- // The size of the stored block
- uint64_t size() const { return size_; }
- void set_size(uint64_t size) { size_ = size; }
-
- void EncodeTo(std::string* dst) const;
- Status DecodeFrom(Slice* input);
-
- // Maximum encoding length of a BlockHandle
- enum { kMaxEncodedLength = 10 + 10 };
-
- private:
- uint64_t offset_;
- uint64_t size_;
-};
-
-// Footer encapsulates the fixed information stored at the tail
-// end of every table file.
-class Footer {
- public:
- Footer() { }
-
- // The block handle for the metaindex block of the table
- const BlockHandle& metaindex_handle() const { return metaindex_handle_; }
- void set_metaindex_handle(const BlockHandle& h) { metaindex_handle_ = h; }
-
- // The block handle for the index block of the table
- const BlockHandle& index_handle() const {
- return index_handle_;
- }
- void set_index_handle(const BlockHandle& h) {
- index_handle_ = h;
- }
-
- void EncodeTo(std::string* dst) const;
- Status DecodeFrom(Slice* input);
-
- // Encoded length of a Footer. Note that the serialization of a
- // Footer will always occupy exactly this many bytes. It consists
- // of two block handles and a magic number.
- enum {
- kEncodedLength = 2*BlockHandle::kMaxEncodedLength + 8
- };
-
- private:
- BlockHandle metaindex_handle_;
- BlockHandle index_handle_;
-};
-
-// kTableMagicNumber was picked by running
-// echo http://code.google.com/p/leveldb/ | sha1sum
-// and taking the leading 64 bits.
-static const uint64_t kTableMagicNumber = 0xdb4775248b80fb57ull;
-
-// 1-byte type + 32-bit crc
-static const size_t kBlockTrailerSize = 5;
-
-struct BlockContents {
- Slice data; // Actual contents of data
- bool cachable; // True iff data can be cached
- bool heap_allocated; // True iff caller should delete[] data.data()
-};
-
-// Read the block identified by "handle" from "file". On failure
-// return non-OK. On success fill *result and return OK.
-extern Status ReadBlock(RandomAccessFile* file,
- const ReadOptions& options,
- const BlockHandle& handle,
- BlockContents* result);
-
-// Implementation details follow. Clients should ignore,
-
-inline BlockHandle::BlockHandle()
- : offset_(~static_cast<uint64_t>(0)),
- size_(~static_cast<uint64_t>(0)) {
-}
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_TABLE_FORMAT_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/iterator.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/iterator.cc b/thirdparty/leveldb-1.18/table/iterator.cc
deleted file mode 100755
index 3d1c87f..0000000
--- a/thirdparty/leveldb-1.18/table/iterator.cc
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/iterator.h"
-
-namespace leveldb {
-
-Iterator::Iterator() {
- cleanup_.function = NULL;
- cleanup_.next = NULL;
-}
-
-Iterator::~Iterator() {
- if (cleanup_.function != NULL) {
- (*cleanup_.function)(cleanup_.arg1, cleanup_.arg2);
- for (Cleanup* c = cleanup_.next; c != NULL; ) {
- (*c->function)(c->arg1, c->arg2);
- Cleanup* next = c->next;
- delete c;
- c = next;
- }
- }
-}
-
-void Iterator::RegisterCleanup(CleanupFunction func, void* arg1, void* arg2) {
- assert(func != NULL);
- Cleanup* c;
- if (cleanup_.function == NULL) {
- c = &cleanup_;
- } else {
- c = new Cleanup;
- c->next = cleanup_.next;
- cleanup_.next = c;
- }
- c->function = func;
- c->arg1 = arg1;
- c->arg2 = arg2;
-}
-
-namespace {
-class EmptyIterator : public Iterator {
- public:
- EmptyIterator(const Status& s) : status_(s) { }
- virtual bool Valid() const { return false; }
- virtual void Seek(const Slice& target) { }
- virtual void SeekToFirst() { }
- virtual void SeekToLast() { }
- virtual void Next() { assert(false); }
- virtual void Prev() { assert(false); }
- Slice key() const { assert(false); return Slice(); }
- Slice value() const { assert(false); return Slice(); }
- virtual Status status() const { return status_; }
- private:
- Status status_;
-};
-} // namespace
-
-Iterator* NewEmptyIterator() {
- return new EmptyIterator(Status::OK());
-}
-
-Iterator* NewErrorIterator(const Status& status) {
- return new EmptyIterator(status);
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/iterator_wrapper.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/iterator_wrapper.h b/thirdparty/leveldb-1.18/table/iterator_wrapper.h
deleted file mode 100755
index 9e16b3d..0000000
--- a/thirdparty/leveldb-1.18/table/iterator_wrapper.h
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_
-#define STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_
-
-namespace leveldb {
-
-// A internal wrapper class with an interface similar to Iterator that
-// caches the valid() and key() results for an underlying iterator.
-// This can help avoid virtual function calls and also gives better
-// cache locality.
-class IteratorWrapper {
- public:
- IteratorWrapper(): iter_(NULL), valid_(false) { }
- explicit IteratorWrapper(Iterator* iter): iter_(NULL) {
- Set(iter);
- }
- ~IteratorWrapper() { delete iter_; }
- Iterator* iter() const { return iter_; }
-
- // Takes ownership of "iter" and will delete it when destroyed, or
- // when Set() is invoked again.
- void Set(Iterator* iter) {
- delete iter_;
- iter_ = iter;
- if (iter_ == NULL) {
- valid_ = false;
- } else {
- Update();
- }
- }
-
-
- // Iterator interface methods
- bool Valid() const { return valid_; }
- Slice key() const { assert(Valid()); return key_; }
- Slice value() const { assert(Valid()); return iter_->value(); }
- // Methods below require iter() != NULL
- Status status() const { assert(iter_); return iter_->status(); }
- void Next() { assert(iter_); iter_->Next(); Update(); }
- void Prev() { assert(iter_); iter_->Prev(); Update(); }
- void Seek(const Slice& k) { assert(iter_); iter_->Seek(k); Update(); }
- void SeekToFirst() { assert(iter_); iter_->SeekToFirst(); Update(); }
- void SeekToLast() { assert(iter_); iter_->SeekToLast(); Update(); }
-
- private:
- void Update() {
- valid_ = iter_->Valid();
- if (valid_) {
- key_ = iter_->key();
- }
- }
-
- Iterator* iter_;
- bool valid_;
- Slice key_;
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_TABLE_ITERATOR_WRAPPER_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/merger.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/merger.cc b/thirdparty/leveldb-1.18/table/merger.cc
deleted file mode 100755
index 8ac844e..0000000
--- a/thirdparty/leveldb-1.18/table/merger.cc
+++ /dev/null
@@ -1,198 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "table/merger.h"
-
-#include "leveldb/comparator.h"
-#include "leveldb/iterator.h"
-#include "table/iterator_wrapper.h"
-
-namespace leveldb {
-
-namespace {
-class MergingIterator : public Iterator {
- public:
- MergingIterator(const Comparator* comparator, Iterator** children, int n)
- : comparator_(comparator),
- children_(new IteratorWrapper[n]),
- n_(n),
- current_(NULL),
- direction_(kForward) {
- for (int i = 0; i < n; i++) {
- children_[i].Set(children[i]);
- }
- }
-
- virtual ~MergingIterator() {
- delete[] children_;
- }
-
- virtual bool Valid() const {
- return (current_ != NULL);
- }
-
- virtual void SeekToFirst() {
- for (int i = 0; i < n_; i++) {
- children_[i].SeekToFirst();
- }
- FindSmallest();
- direction_ = kForward;
- }
-
- virtual void SeekToLast() {
- for (int i = 0; i < n_; i++) {
- children_[i].SeekToLast();
- }
- FindLargest();
- direction_ = kReverse;
- }
-
- virtual void Seek(const Slice& target) {
- for (int i = 0; i < n_; i++) {
- children_[i].Seek(target);
- }
- FindSmallest();
- direction_ = kForward;
- }
-
- virtual void Next() {
- assert(Valid());
-
- // Ensure that all children are positioned after key().
- // If we are moving in the forward direction, it is already
- // true for all of the non-current_ children since current_ is
- // the smallest child and key() == current_->key(). Otherwise,
- // we explicitly position the non-current_ children.
- if (direction_ != kForward) {
- for (int i = 0; i < n_; i++) {
- IteratorWrapper* child = &children_[i];
- if (child != current_) {
- child->Seek(key());
- if (child->Valid() &&
- comparator_->Compare(key(), child->key()) == 0) {
- child->Next();
- }
- }
- }
- direction_ = kForward;
- }
-
- current_->Next();
- FindSmallest();
- }
-
- virtual void Prev() {
- assert(Valid());
-
- // Ensure that all children are positioned before key().
- // If we are moving in the reverse direction, it is already
- // true for all of the non-current_ children since current_ is
- // the largest child and key() == current_->key(). Otherwise,
- // we explicitly position the non-current_ children.
- if (direction_ != kReverse) {
- for (int i = 0; i < n_; i++) {
- IteratorWrapper* child = &children_[i];
- if (child != current_) {
- child->Seek(key());
- if (child->Valid()) {
- // Child is at first entry >= key(). Step back one to be < key()
- child->Prev();
- } else {
- // Child has no entries >= key(). Position at last entry.
- child->SeekToLast();
- }
- }
- }
- direction_ = kReverse;
- }
-
- current_->Prev();
- FindLargest();
- }
-
- virtual Slice key() const {
- assert(Valid());
- return current_->key();
- }
-
- virtual Slice value() const {
- assert(Valid());
- return current_->value();
- }
-
- virtual Status status() const {
- Status status;
- for (int i = 0; i < n_; i++) {
- status = children_[i].status();
- if (!status.ok()) {
- break;
- }
- }
- return status;
- }
-
- private:
- void FindSmallest();
- void FindLargest();
-
- // We might want to use a heap in case there are lots of children.
- // For now we use a simple array since we expect a very small number
- // of children in leveldb.
- const Comparator* comparator_;
- IteratorWrapper* children_;
- int n_;
- IteratorWrapper* current_;
-
- // Which direction is the iterator moving?
- enum Direction {
- kForward,
- kReverse
- };
- Direction direction_;
-};
-
-void MergingIterator::FindSmallest() {
- IteratorWrapper* smallest = NULL;
- for (int i = 0; i < n_; i++) {
- IteratorWrapper* child = &children_[i];
- if (child->Valid()) {
- if (smallest == NULL) {
- smallest = child;
- } else if (comparator_->Compare(child->key(), smallest->key()) < 0) {
- smallest = child;
- }
- }
- }
- current_ = smallest;
-}
-
-void MergingIterator::FindLargest() {
- IteratorWrapper* largest = NULL;
- for (int i = n_-1; i >= 0; i--) {
- IteratorWrapper* child = &children_[i];
- if (child->Valid()) {
- if (largest == NULL) {
- largest = child;
- } else if (comparator_->Compare(child->key(), largest->key()) > 0) {
- largest = child;
- }
- }
- }
- current_ = largest;
-}
-} // namespace
-
-Iterator* NewMergingIterator(const Comparator* cmp, Iterator** list, size_t n) {
- // the assertion below has been removed has size_t cannot be negative
- //assert(n >= 0);
- if (n == 0) {
- return NewEmptyIterator();
- } else if (n == 1) {
- return list[0];
- } else {
- return new MergingIterator(cmp, list, static_cast<unsigned int>(n));
- }
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/merger.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/merger.h b/thirdparty/leveldb-1.18/table/merger.h
deleted file mode 100755
index d272a45..0000000
--- a/thirdparty/leveldb-1.18/table/merger.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_TABLE_MERGER_H_
-#define STORAGE_LEVELDB_TABLE_MERGER_H_
-
-#include <stddef.h>
-
-namespace leveldb {
-
-class Comparator;
-class Iterator;
-
-// Return an iterator that provided the union of the data in
-// children[0,n-1]. Takes ownership of the child iterators and
-// will delete them when the result iterator is deleted.
-//
-// The result does no duplicate suppression. I.e., if a particular
-// key is present in K child iterators, it will be yielded K times.
-//
-// REQUIRES: n >= 0
-extern Iterator* NewMergingIterator(
- const Comparator* comparator, Iterator** children, size_t n);
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_TABLE_MERGER_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/table.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/table.cc b/thirdparty/leveldb-1.18/table/table.cc
deleted file mode 100755
index dff8a82..0000000
--- a/thirdparty/leveldb-1.18/table/table.cc
+++ /dev/null
@@ -1,285 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/table.h"
-
-#include "leveldb/cache.h"
-#include "leveldb/comparator.h"
-#include "leveldb/env.h"
-#include "leveldb/filter_policy.h"
-#include "leveldb/options.h"
-#include "table/block.h"
-#include "table/filter_block.h"
-#include "table/format.h"
-#include "table/two_level_iterator.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-struct Table::Rep {
- ~Rep() {
- delete filter;
- delete [] filter_data;
- delete index_block;
- }
-
- Options options;
- Status status;
- RandomAccessFile* file;
- uint64_t cache_id;
- FilterBlockReader* filter;
- const char* filter_data;
-
- BlockHandle metaindex_handle; // Handle to metaindex_block: saved from footer
- Block* index_block;
-};
-
-Status Table::Open(const Options& options,
- RandomAccessFile* file,
- uint64_t size,
- Table** table) {
- *table = NULL;
- if (size < Footer::kEncodedLength) {
- return Status::Corruption("file is too short to be an sstable");
- }
-
- char footer_space[Footer::kEncodedLength];
- Slice footer_input;
- Status s = file->Read(size - Footer::kEncodedLength, Footer::kEncodedLength,
- &footer_input, footer_space);
- if (!s.ok()) return s;
-
- Footer footer;
- s = footer.DecodeFrom(&footer_input);
- if (!s.ok()) return s;
-
- // Read the index block
- BlockContents contents;
- Block* index_block = NULL;
- if (s.ok()) {
- ReadOptions opt;
- if (options.paranoid_checks) {
- opt.verify_checksums = true;
- }
- s = ReadBlock(file, opt, footer.index_handle(), &contents);
- if (s.ok()) {
- index_block = new Block(contents);
- }
- }
-
- if (s.ok()) {
- // We've successfully read the footer and the index block: we're
- // ready to serve requests.
- Rep* rep = new Table::Rep;
- rep->options = options;
- rep->file = file;
- rep->metaindex_handle = footer.metaindex_handle();
- rep->index_block = index_block;
- rep->cache_id = (options.block_cache ? options.block_cache->NewId() : 0);
- rep->filter_data = NULL;
- rep->filter = NULL;
- *table = new Table(rep);
- (*table)->ReadMeta(footer);
- } else {
- if (index_block) delete index_block;
- }
-
- return s;
-}
-
-void Table::ReadMeta(const Footer& footer) {
- if (rep_->options.filter_policy == NULL) {
- return; // Do not need any metadata
- }
-
- // TODO(sanjay): Skip this if footer.metaindex_handle() size indicates
- // it is an empty block.
- ReadOptions opt;
- if (rep_->options.paranoid_checks) {
- opt.verify_checksums = true;
- }
- BlockContents contents;
- if (!ReadBlock(rep_->file, opt, footer.metaindex_handle(), &contents).ok()) {
- // Do not propagate errors since meta info is not needed for operation
- return;
- }
- Block* meta = new Block(contents);
-
- Iterator* iter = meta->NewIterator(BytewiseComparator());
- std::string key = "filter.";
- key.append(rep_->options.filter_policy->Name());
- iter->Seek(key);
- if (iter->Valid() && iter->key() == Slice(key)) {
- ReadFilter(iter->value());
- }
- delete iter;
- delete meta;
-}
-
-void Table::ReadFilter(const Slice& filter_handle_value) {
- Slice v = filter_handle_value;
- BlockHandle filter_handle;
- if (!filter_handle.DecodeFrom(&v).ok()) {
- return;
- }
-
- // We might want to unify with ReadBlock() if we start
- // requiring checksum verification in Table::Open.
- ReadOptions opt;
- if (rep_->options.paranoid_checks) {
- opt.verify_checksums = true;
- }
- BlockContents block;
- if (!ReadBlock(rep_->file, opt, filter_handle, &block).ok()) {
- return;
- }
- if (block.heap_allocated) {
- rep_->filter_data = block.data.data(); // Will need to delete later
- }
- rep_->filter = new FilterBlockReader(rep_->options.filter_policy, block.data);
-}
-
-Table::~Table() {
- delete rep_;
-}
-
-static void DeleteBlock(void* arg, void* ignored) {
- delete reinterpret_cast<Block*>(arg);
-}
-
-static void DeleteCachedBlock(const Slice& key, void* value) {
- Block* block = reinterpret_cast<Block*>(value);
- delete block;
-}
-
-static void ReleaseBlock(void* arg, void* h) {
- Cache* cache = reinterpret_cast<Cache*>(arg);
- Cache::Handle* handle = reinterpret_cast<Cache::Handle*>(h);
- cache->Release(handle);
-}
-
-// Convert an index iterator value (i.e., an encoded BlockHandle)
-// into an iterator over the contents of the corresponding block.
-Iterator* Table::BlockReader(void* arg,
- const ReadOptions& options,
- const Slice& index_value) {
- Table* table = reinterpret_cast<Table*>(arg);
- Cache* block_cache = table->rep_->options.block_cache;
- Block* block = NULL;
- Cache::Handle* cache_handle = NULL;
-
- BlockHandle handle;
- Slice input = index_value;
- Status s = handle.DecodeFrom(&input);
- // We intentionally allow extra stuff in index_value so that we
- // can add more features in the future.
-
- if (s.ok()) {
- BlockContents contents;
- if (block_cache != NULL) {
- char cache_key_buffer[16];
- EncodeFixed64(cache_key_buffer, table->rep_->cache_id);
- EncodeFixed64(cache_key_buffer+8, handle.offset());
- Slice key(cache_key_buffer, sizeof(cache_key_buffer));
- cache_handle = block_cache->Lookup(key);
- if (cache_handle != NULL) {
- block = reinterpret_cast<Block*>(block_cache->Value(cache_handle));
- } else {
- s = ReadBlock(table->rep_->file, options, handle, &contents);
- if (s.ok()) {
- block = new Block(contents);
- if (contents.cachable && options.fill_cache) {
- cache_handle = block_cache->Insert(
- key, block, block->size(), &DeleteCachedBlock);
- }
- }
- }
- } else {
- s = ReadBlock(table->rep_->file, options, handle, &contents);
- if (s.ok()) {
- block = new Block(contents);
- }
- }
- }
-
- Iterator* iter;
- if (block != NULL) {
- iter = block->NewIterator(table->rep_->options.comparator);
- if (cache_handle == NULL) {
- iter->RegisterCleanup(&DeleteBlock, block, NULL);
- } else {
- iter->RegisterCleanup(&ReleaseBlock, block_cache, cache_handle);
- }
- } else {
- iter = NewErrorIterator(s);
- }
- return iter;
-}
-
-Iterator* Table::NewIterator(const ReadOptions& options) const {
- return NewTwoLevelIterator(
- rep_->index_block->NewIterator(rep_->options.comparator),
- &Table::BlockReader, const_cast<Table*>(this), options);
-}
-
-Status Table::InternalGet(const ReadOptions& options, const Slice& k,
- void* arg,
- void (*saver)(void*, const Slice&, const Slice&)) {
- Status s;
- Iterator* iiter = rep_->index_block->NewIterator(rep_->options.comparator);
- iiter->Seek(k);
- if (iiter->Valid()) {
- Slice handle_value = iiter->value();
- FilterBlockReader* filter = rep_->filter;
- BlockHandle handle;
- if (filter != NULL &&
- handle.DecodeFrom(&handle_value).ok() &&
- !filter->KeyMayMatch(handle.offset(), k)) {
- // Not found
- } else {
- Iterator* block_iter = BlockReader(this, options, iiter->value());
- block_iter->Seek(k);
- if (block_iter->Valid()) {
- (*saver)(arg, block_iter->key(), block_iter->value());
- }
- s = block_iter->status();
- delete block_iter;
- }
- }
- if (s.ok()) {
- s = iiter->status();
- }
- delete iiter;
- return s;
-}
-
-
-uint64_t Table::ApproximateOffsetOf(const Slice& key) const {
- Iterator* index_iter =
- rep_->index_block->NewIterator(rep_->options.comparator);
- index_iter->Seek(key);
- uint64_t result;
- if (index_iter->Valid()) {
- BlockHandle handle;
- Slice input = index_iter->value();
- Status s = handle.DecodeFrom(&input);
- if (s.ok()) {
- result = handle.offset();
- } else {
- // Strange: we can't decode the block handle in the index block.
- // We'll just return the offset of the metaindex block, which is
- // close to the whole file size for this case.
- result = rep_->metaindex_handle.offset();
- }
- } else {
- // key is past the last key in the file. Approximate the offset
- // by returning the offset of the metaindex block (which is
- // right near the end of the file).
- result = rep_->metaindex_handle.offset();
- }
- delete index_iter;
- return result;
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/table_builder.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/table_builder.cc b/thirdparty/leveldb-1.18/table/table_builder.cc
deleted file mode 100755
index 62002c8..0000000
--- a/thirdparty/leveldb-1.18/table/table_builder.cc
+++ /dev/null
@@ -1,270 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/table_builder.h"
-
-#include <assert.h>
-#include "leveldb/comparator.h"
-#include "leveldb/env.h"
-#include "leveldb/filter_policy.h"
-#include "leveldb/options.h"
-#include "table/block_builder.h"
-#include "table/filter_block.h"
-#include "table/format.h"
-#include "util/coding.h"
-#include "util/crc32c.h"
-
-namespace leveldb {
-
-struct TableBuilder::Rep {
- Options options;
- Options index_block_options;
- WritableFile* file;
- uint64_t offset;
- Status status;
- BlockBuilder data_block;
- BlockBuilder index_block;
- std::string last_key;
- int64_t num_entries;
- bool closed; // Either Finish() or Abandon() has been called.
- FilterBlockBuilder* filter_block;
-
- // We do not emit the index entry for a block until we have seen the
- // first key for the next data block. This allows us to use shorter
- // keys in the index block. For example, consider a block boundary
- // between the keys "the quick brown fox" and "the who". We can use
- // "the r" as the key for the index block entry since it is >= all
- // entries in the first block and < all entries in subsequent
- // blocks.
- //
- // Invariant: r->pending_index_entry is true only if data_block is empty.
- bool pending_index_entry;
- BlockHandle pending_handle; // Handle to add to index block
-
- std::string compressed_output;
-
- Rep(const Options& opt, WritableFile* f)
- : options(opt),
- index_block_options(opt),
- file(f),
- offset(0),
- data_block(&options),
- index_block(&index_block_options),
- num_entries(0),
- closed(false),
- filter_block(opt.filter_policy == NULL ? NULL
- : new FilterBlockBuilder(opt.filter_policy)),
- pending_index_entry(false) {
- index_block_options.block_restart_interval = 1;
- }
-};
-
-TableBuilder::TableBuilder(const Options& options, WritableFile* file)
- : rep_(new Rep(options, file)) {
- if (rep_->filter_block != NULL) {
- rep_->filter_block->StartBlock(0);
- }
-}
-
-TableBuilder::~TableBuilder() {
- assert(rep_->closed); // Catch errors where caller forgot to call Finish()
- delete rep_->filter_block;
- delete rep_;
-}
-
-Status TableBuilder::ChangeOptions(const Options& options) {
- // Note: if more fields are added to Options, update
- // this function to catch changes that should not be allowed to
- // change in the middle of building a Table.
- if (options.comparator != rep_->options.comparator) {
- return Status::InvalidArgument("changing comparator while building table");
- }
-
- // Note that any live BlockBuilders point to rep_->options and therefore
- // will automatically pick up the updated options.
- rep_->options = options;
- rep_->index_block_options = options;
- rep_->index_block_options.block_restart_interval = 1;
- return Status::OK();
-}
-
-void TableBuilder::Add(const Slice& key, const Slice& value) {
- Rep* r = rep_;
- assert(!r->closed);
- if (!ok()) return;
- if (r->num_entries > 0) {
- assert(r->options.comparator->Compare(key, Slice(r->last_key)) > 0);
- }
-
- if (r->pending_index_entry) {
- assert(r->data_block.empty());
- r->options.comparator->FindShortestSeparator(&r->last_key, key);
- std::string handle_encoding;
- r->pending_handle.EncodeTo(&handle_encoding);
- r->index_block.Add(r->last_key, Slice(handle_encoding));
- r->pending_index_entry = false;
- }
-
- if (r->filter_block != NULL) {
- r->filter_block->AddKey(key);
- }
-
- r->last_key.assign(key.data(), key.size());
- r->num_entries++;
- r->data_block.Add(key, value);
-
- const size_t estimated_block_size = r->data_block.CurrentSizeEstimate();
- if (estimated_block_size >= r->options.block_size) {
- Flush();
- }
-}
-
-void TableBuilder::Flush() {
- Rep* r = rep_;
- assert(!r->closed);
- if (!ok()) return;
- if (r->data_block.empty()) return;
- assert(!r->pending_index_entry);
- WriteBlock(&r->data_block, &r->pending_handle);
- if (ok()) {
- r->pending_index_entry = true;
- r->status = r->file->Flush();
- }
- if (r->filter_block != NULL) {
- r->filter_block->StartBlock(r->offset);
- }
-}
-
-void TableBuilder::WriteBlock(BlockBuilder* block, BlockHandle* handle) {
- // File format contains a sequence of blocks where each block has:
- // block_data: uint8[n]
- // type: uint8
- // crc: uint32
- assert(ok());
- Rep* r = rep_;
- Slice raw = block->Finish();
-
- Slice block_contents;
- CompressionType type = r->options.compression;
- // TODO(postrelease): Support more compression options: zlib?
- switch (type) {
- case kNoCompression:
- block_contents = raw;
- break;
-
- case kSnappyCompression: {
- std::string* compressed = &r->compressed_output;
- if (port::Snappy_Compress(raw.data(), raw.size(), compressed) &&
- compressed->size() < raw.size() - (raw.size() / 8u)) {
- block_contents = *compressed;
- } else {
- // Snappy not supported, or compressed less than 12.5%, so just
- // store uncompressed form
- block_contents = raw;
- type = kNoCompression;
- }
- break;
- }
- }
- WriteRawBlock(block_contents, type, handle);
- r->compressed_output.clear();
- block->Reset();
-}
-
-void TableBuilder::WriteRawBlock(const Slice& block_contents,
- CompressionType type,
- BlockHandle* handle) {
- Rep* r = rep_;
- handle->set_offset(r->offset);
- handle->set_size(block_contents.size());
- r->status = r->file->Append(block_contents);
- if (r->status.ok()) {
- char trailer[kBlockTrailerSize];
- trailer[0] = type;
- uint32_t crc = crc32c::Value(block_contents.data(), block_contents.size());
- crc = crc32c::Extend(crc, trailer, 1); // Extend crc to cover block type
- EncodeFixed32(trailer+1, crc32c::Mask(crc));
- r->status = r->file->Append(Slice(trailer, kBlockTrailerSize));
- if (r->status.ok()) {
- r->offset += block_contents.size() + kBlockTrailerSize;
- }
- }
-}
-
-Status TableBuilder::status() const {
- return rep_->status;
-}
-
-Status TableBuilder::Finish() {
- Rep* r = rep_;
- Flush();
- assert(!r->closed);
- r->closed = true;
-
- BlockHandle filter_block_handle, metaindex_block_handle, index_block_handle;
-
- // Write filter block
- if (ok() && r->filter_block != NULL) {
- WriteRawBlock(r->filter_block->Finish(), kNoCompression,
- &filter_block_handle);
- }
-
- // Write metaindex block
- if (ok()) {
- BlockBuilder meta_index_block(&r->options);
- if (r->filter_block != NULL) {
- // Add mapping from "filter.Name" to location of filter data
- std::string key = "filter.";
- key.append(r->options.filter_policy->Name());
- std::string handle_encoding;
- filter_block_handle.EncodeTo(&handle_encoding);
- meta_index_block.Add(key, handle_encoding);
- }
-
- // TODO(postrelease): Add stats and other meta blocks
- WriteBlock(&meta_index_block, &metaindex_block_handle);
- }
-
- // Write index block
- if (ok()) {
- if (r->pending_index_entry) {
- r->options.comparator->FindShortSuccessor(&r->last_key);
- std::string handle_encoding;
- r->pending_handle.EncodeTo(&handle_encoding);
- r->index_block.Add(r->last_key, Slice(handle_encoding));
- r->pending_index_entry = false;
- }
- WriteBlock(&r->index_block, &index_block_handle);
- }
-
- // Write footer
- if (ok()) {
- Footer footer;
- footer.set_metaindex_handle(metaindex_block_handle);
- footer.set_index_handle(index_block_handle);
- std::string footer_encoding;
- footer.EncodeTo(&footer_encoding);
- r->status = r->file->Append(footer_encoding);
- if (r->status.ok()) {
- r->offset += footer_encoding.size();
- }
- }
- return r->status;
-}
-
-void TableBuilder::Abandon() {
- Rep* r = rep_;
- assert(!r->closed);
- r->closed = true;
-}
-
-uint64_t TableBuilder::NumEntries() const {
- return rep_->num_entries;
-}
-
-uint64_t TableBuilder::FileSize() const {
- return rep_->offset;
-}
-
-} // namespace leveldb
[44/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/repair.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/repair.cc b/thirdparty/leveldb-1.18/db/repair.cc
deleted file mode 100755
index 4cd4bb0..0000000
--- a/thirdparty/leveldb-1.18/db/repair.cc
+++ /dev/null
@@ -1,461 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// We recover the contents of the descriptor from the other files we find.
-// (1) Any log files are first converted to tables
-// (2) We scan every table to compute
-// (a) smallest/largest for the table
-// (b) largest sequence number in the table
-// (3) We generate descriptor contents:
-// - log number is set to zero
-// - next-file-number is set to 1 + largest file number we found
-// - last-sequence-number is set to largest sequence# found across
-// all tables (see 2c)
-// - compaction pointers are cleared
-// - every table file is added at level 0
-//
-// Possible optimization 1:
-// (a) Compute total size and use to pick appropriate max-level M
-// (b) Sort tables by largest sequence# in the table
-// (c) For each table: if it overlaps earlier table, place in level-0,
-// else place in level-M.
-// Possible optimization 2:
-// Store per-table metadata (smallest, largest, largest-seq#, ...)
-// in the table's meta section to speed up ScanTable.
-
-#include "db/builder.h"
-#include "db/db_impl.h"
-#include "db/dbformat.h"
-#include "db/filename.h"
-#include "db/log_reader.h"
-#include "db/log_writer.h"
-#include "db/memtable.h"
-#include "db/table_cache.h"
-#include "db/version_edit.h"
-#include "db/write_batch_internal.h"
-#include "leveldb/comparator.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-
-namespace leveldb {
-
-namespace {
-
-class Repairer {
- public:
- Repairer(const std::string& dbname, const Options& options)
- : dbname_(dbname),
- env_(options.env),
- icmp_(options.comparator),
- ipolicy_(options.filter_policy),
- options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
- owns_info_log_(options_.info_log != options.info_log),
- owns_cache_(options_.block_cache != options.block_cache),
- next_file_number_(1) {
- // TableCache can be small since we expect each table to be opened once.
- table_cache_ = new TableCache(dbname_, &options_, 10);
- }
-
- ~Repairer() {
- delete table_cache_;
- if (owns_info_log_) {
- delete options_.info_log;
- }
- if (owns_cache_) {
- delete options_.block_cache;
- }
- }
-
- Status Run() {
- Status status = FindFiles();
- if (status.ok()) {
- ConvertLogFilesToTables();
- ExtractMetaData();
- status = WriteDescriptor();
- }
- if (status.ok()) {
- unsigned long long bytes = 0;
- for (size_t i = 0; i < tables_.size(); i++) {
- bytes += tables_[i].meta.file_size;
- }
- Log(options_.info_log,
- "**** Repaired leveldb %s; "
- "recovered %d files; %llu bytes. "
- "Some data may have been lost. "
- "****",
- dbname_.c_str(),
- static_cast<int>(tables_.size()),
- bytes);
- }
- return status;
- }
-
- private:
- struct TableInfo {
- FileMetaData meta;
- SequenceNumber max_sequence;
- };
-
- std::string const dbname_;
- Env* const env_;
- InternalKeyComparator const icmp_;
- InternalFilterPolicy const ipolicy_;
- Options const options_;
- bool owns_info_log_;
- bool owns_cache_;
- TableCache* table_cache_;
- VersionEdit edit_;
-
- std::vector<std::string> manifests_;
- std::vector<uint64_t> table_numbers_;
- std::vector<uint64_t> logs_;
- std::vector<TableInfo> tables_;
- uint64_t next_file_number_;
-
- Status FindFiles() {
- std::vector<std::string> filenames;
- Status status = env_->GetChildren(dbname_, &filenames);
- if (!status.ok()) {
- return status;
- }
- if (filenames.empty()) {
- return Status::IOError(dbname_, "repair found no files");
- }
-
- uint64_t number;
- FileType type;
- for (size_t i = 0; i < filenames.size(); i++) {
- if (ParseFileName(filenames[i], &number, &type)) {
- if (type == kDescriptorFile) {
- manifests_.push_back(filenames[i]);
- } else {
- if (number + 1 > next_file_number_) {
- next_file_number_ = number + 1;
- }
- if (type == kLogFile) {
- logs_.push_back(number);
- } else if (type == kTableFile) {
- table_numbers_.push_back(number);
- } else {
- // Ignore other files
- }
- }
- }
- }
- return status;
- }
-
- void ConvertLogFilesToTables() {
- for (size_t i = 0; i < logs_.size(); i++) {
- std::string logname = LogFileName(dbname_, logs_[i]);
- Status status = ConvertLogToTable(logs_[i]);
- if (!status.ok()) {
- Log(options_.info_log, "Log #%llu: ignoring conversion error: %s",
- (unsigned long long) logs_[i],
- status.ToString().c_str());
- }
- ArchiveFile(logname);
- }
- }
-
- Status ConvertLogToTable(uint64_t log) {
- struct LogReporter : public log::Reader::Reporter {
- Env* env;
- Logger* info_log;
- uint64_t lognum;
- virtual void Corruption(size_t bytes, const Status& s) {
- // We print error messages for corruption, but continue repairing.
- Log(info_log, "Log #%llu: dropping %d bytes; %s",
- (unsigned long long) lognum,
- static_cast<int>(bytes),
- s.ToString().c_str());
- }
- };
-
- // Open the log file
- std::string logname = LogFileName(dbname_, log);
- SequentialFile* lfile;
- Status status = env_->NewSequentialFile(logname, &lfile);
- if (!status.ok()) {
- return status;
- }
-
- // Create the log reader.
- LogReporter reporter;
- reporter.env = env_;
- reporter.info_log = options_.info_log;
- reporter.lognum = log;
- // We intentionally make log::Reader do checksumming so that
- // corruptions cause entire commits to be skipped instead of
- // propagating bad information (like overly large sequence
- // numbers).
- log::Reader reader(lfile, &reporter, false/*do not checksum*/,
- 0/*initial_offset*/);
-
- // Read all the records and add to a memtable
- std::string scratch;
- Slice record;
- WriteBatch batch;
- MemTable* mem = new MemTable(icmp_);
- mem->Ref();
- int counter = 0;
- while (reader.ReadRecord(&record, &scratch)) {
- if (record.size() < 12) {
- reporter.Corruption(
- record.size(), Status::Corruption("log record too small"));
- continue;
- }
- WriteBatchInternal::SetContents(&batch, record);
- status = WriteBatchInternal::InsertInto(&batch, mem);
- if (status.ok()) {
- counter += WriteBatchInternal::Count(&batch);
- } else {
- Log(options_.info_log, "Log #%llu: ignoring %s",
- (unsigned long long) log,
- status.ToString().c_str());
- status = Status::OK(); // Keep going with rest of file
- }
- }
- delete lfile;
-
- // Do not record a version edit for this conversion to a Table
- // since ExtractMetaData() will also generate edits.
- FileMetaData meta;
- meta.number = next_file_number_++;
- Iterator* iter = mem->NewIterator();
- status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
- delete iter;
- mem->Unref();
- mem = NULL;
- if (status.ok()) {
- if (meta.file_size > 0) {
- table_numbers_.push_back(meta.number);
- }
- }
- Log(options_.info_log, "Log #%llu: %d ops saved to Table #%llu %s",
- (unsigned long long) log,
- counter,
- (unsigned long long) meta.number,
- status.ToString().c_str());
- return status;
- }
-
- void ExtractMetaData() {
- for (size_t i = 0; i < table_numbers_.size(); i++) {
- ScanTable(table_numbers_[i]);
- }
- }
-
- Iterator* NewTableIterator(const FileMetaData& meta) {
- // Same as compaction iterators: if paranoid_checks are on, turn
- // on checksum verification.
- ReadOptions r;
- r.verify_checksums = options_.paranoid_checks;
- return table_cache_->NewIterator(r, meta.number, meta.file_size);
- }
-
- void ScanTable(uint64_t number) {
- TableInfo t;
- t.meta.number = number;
- std::string fname = TableFileName(dbname_, number);
- Status status = env_->GetFileSize(fname, &t.meta.file_size);
- if (!status.ok()) {
- // Try alternate file name.
- fname = SSTTableFileName(dbname_, number);
- Status s2 = env_->GetFileSize(fname, &t.meta.file_size);
- if (s2.ok()) {
- status = Status::OK();
- }
- }
- if (!status.ok()) {
- ArchiveFile(TableFileName(dbname_, number));
- ArchiveFile(SSTTableFileName(dbname_, number));
- Log(options_.info_log, "Table #%llu: dropped: %s",
- (unsigned long long) t.meta.number,
- status.ToString().c_str());
- return;
- }
-
- // Extract metadata by scanning through table.
- int counter = 0;
- Iterator* iter = NewTableIterator(t.meta);
- bool empty = true;
- ParsedInternalKey parsed;
- t.max_sequence = 0;
- for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
- Slice key = iter->key();
- if (!ParseInternalKey(key, &parsed)) {
- Log(options_.info_log, "Table #%llu: unparsable key %s",
- (unsigned long long) t.meta.number,
- EscapeString(key).c_str());
- continue;
- }
-
- counter++;
- if (empty) {
- empty = false;
- t.meta.smallest.DecodeFrom(key);
- }
- t.meta.largest.DecodeFrom(key);
- if (parsed.sequence > t.max_sequence) {
- t.max_sequence = parsed.sequence;
- }
- }
- if (!iter->status().ok()) {
- status = iter->status();
- }
- delete iter;
- Log(options_.info_log, "Table #%llu: %d entries %s",
- (unsigned long long) t.meta.number,
- counter,
- status.ToString().c_str());
-
- if (status.ok()) {
- tables_.push_back(t);
- } else {
- RepairTable(fname, t); // RepairTable archives input file.
- }
- }
-
- void RepairTable(const std::string& src, TableInfo t) {
- // We will copy src contents to a new table and then rename the
- // new table over the source.
-
- // Create builder.
- std::string copy = TableFileName(dbname_, next_file_number_++);
- WritableFile* file;
- Status s = env_->NewWritableFile(copy, &file);
- if (!s.ok()) {
- return;
- }
- TableBuilder* builder = new TableBuilder(options_, file);
-
- // Copy data.
- Iterator* iter = NewTableIterator(t.meta);
- int counter = 0;
- for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
- builder->Add(iter->key(), iter->value());
- counter++;
- }
- delete iter;
-
- ArchiveFile(src);
- if (counter == 0) {
- builder->Abandon(); // Nothing to save
- } else {
- s = builder->Finish();
- if (s.ok()) {
- t.meta.file_size = builder->FileSize();
- }
- }
- delete builder;
- builder = NULL;
-
- if (s.ok()) {
- s = file->Close();
- }
- delete file;
- file = NULL;
-
- if (counter > 0 && s.ok()) {
- std::string orig = TableFileName(dbname_, t.meta.number);
- s = env_->RenameFile(copy, orig);
- if (s.ok()) {
- Log(options_.info_log, "Table #%llu: %d entries repaired",
- (unsigned long long) t.meta.number, counter);
- tables_.push_back(t);
- }
- }
- if (!s.ok()) {
- env_->DeleteFile(copy);
- }
- }
-
- Status WriteDescriptor() {
- std::string tmp = TempFileName(dbname_, 1);
- WritableFile* file;
- Status status = env_->NewWritableFile(tmp, &file);
- if (!status.ok()) {
- return status;
- }
-
- SequenceNumber max_sequence = 0;
- for (size_t i = 0; i < tables_.size(); i++) {
- if (max_sequence < tables_[i].max_sequence) {
- max_sequence = tables_[i].max_sequence;
- }
- }
-
- edit_.SetComparatorName(icmp_.user_comparator()->Name());
- edit_.SetLogNumber(0);
- edit_.SetNextFile(next_file_number_);
- edit_.SetLastSequence(max_sequence);
-
- for (size_t i = 0; i < tables_.size(); i++) {
- // TODO(opt): separate out into multiple levels
- const TableInfo& t = tables_[i];
- edit_.AddFile(0, t.meta.number, t.meta.file_size,
- t.meta.smallest, t.meta.largest);
- }
-
- //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
- {
- log::Writer log(file);
- std::string record;
- edit_.EncodeTo(&record);
- status = log.AddRecord(record);
- }
- if (status.ok()) {
- status = file->Close();
- }
- delete file;
- file = NULL;
-
- if (!status.ok()) {
- env_->DeleteFile(tmp);
- } else {
- // Discard older manifests
- for (size_t i = 0; i < manifests_.size(); i++) {
- ArchiveFile(dbname_ + "/" + manifests_[i]);
- }
-
- // Install new manifest
- status = env_->RenameFile(tmp, DescriptorFileName(dbname_, 1));
- if (status.ok()) {
- status = SetCurrentFile(env_, dbname_, 1);
- } else {
- env_->DeleteFile(tmp);
- }
- }
- return status;
- }
-
- void ArchiveFile(const std::string& fname) {
- // Move into another directory. E.g., for
- // dir/foo
- // rename to
- // dir/lost/foo
- const char* slash = strrchr(fname.c_str(), '/');
- std::string new_dir;
- if (slash != NULL) {
- new_dir.assign(fname.data(), slash - fname.data());
- }
- new_dir.append("/lost");
- env_->CreateDir(new_dir); // Ignore error
- std::string new_file = new_dir;
- new_file.append("/");
- new_file.append((slash == NULL) ? fname.c_str() : slash + 1);
- Status s = env_->RenameFile(fname, new_file);
- Log(options_.info_log, "Archiving %s: %s\n",
- fname.c_str(), s.ToString().c_str());
- }
-};
-} // namespace
-
-Status RepairDB(const std::string& dbname, const Options& options) {
- Repairer repairer(dbname, options);
- return repairer.Run();
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/skiplist.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/skiplist.h b/thirdparty/leveldb-1.18/db/skiplist.h
deleted file mode 100755
index ed8b092..0000000
--- a/thirdparty/leveldb-1.18/db/skiplist.h
+++ /dev/null
@@ -1,384 +0,0 @@
-#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_
-#define STORAGE_LEVELDB_DB_SKIPLIST_H_
-
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Thread safety
-// -------------
-//
-// Writes require external synchronization, most likely a mutex.
-// Reads require a guarantee that the SkipList will not be destroyed
-// while the read is in progress. Apart from that, reads progress
-// without any internal locking or synchronization.
-//
-// Invariants:
-//
-// (1) Allocated nodes are never deleted until the SkipList is
-// destroyed. This is trivially guaranteed by the code since we
-// never delete any skip list nodes.
-//
-// (2) The contents of a Node except for the next/prev pointers are
-// immutable after the Node has been linked into the SkipList.
-// Only Insert() modifies the list, and it is careful to initialize
-// a node and use release-stores to publish the nodes in one or
-// more lists.
-//
-// ... prev vs. next pointer ordering ...
-
-#include <assert.h>
-#include <stdlib.h>
-#include "port/port.h"
-#include "util/arena.h"
-#include "util/random.h"
-
-namespace leveldb {
-
-class Arena;
-
-template<typename Key, class Comparator>
-class SkipList {
- private:
- struct Node;
-
- public:
- // Create a new SkipList object that will use "cmp" for comparing keys,
- // and will allocate memory using "*arena". Objects allocated in the arena
- // must remain allocated for the lifetime of the skiplist object.
- explicit SkipList(Comparator cmp, Arena* arena);
-
- // Insert key into the list.
- // REQUIRES: nothing that compares equal to key is currently in the list.
- void Insert(const Key& key);
-
- // Returns true iff an entry that compares equal to key is in the list.
- bool Contains(const Key& key) const;
-
- // Iteration over the contents of a skip list
- class Iterator {
- public:
- // Initialize an iterator over the specified list.
- // The returned iterator is not valid.
- explicit Iterator(const SkipList* list);
-
- // Returns true iff the iterator is positioned at a valid node.
- bool Valid() const;
-
- // Returns the key at the current position.
- // REQUIRES: Valid()
- const Key& key() const;
-
- // Advances to the next position.
- // REQUIRES: Valid()
- void Next();
-
- // Advances to the previous position.
- // REQUIRES: Valid()
- void Prev();
-
- // Advance to the first entry with a key >= target
- void Seek(const Key& target);
-
- // Position at the first entry in list.
- // Final state of iterator is Valid() iff list is not empty.
- void SeekToFirst();
-
- // Position at the last entry in list.
- // Final state of iterator is Valid() iff list is not empty.
- void SeekToLast();
-
- private:
- const SkipList* list_;
- Node* node_;
- // Intentionally copyable
- };
-
- private:
- enum { kMaxHeight = 12 };
-
- // Immutable after construction
- Comparator const compare_;
- Arena* const arena_; // Arena used for allocations of nodes
-
- Node* const head_;
-
- // Modified only by Insert(). Read racily by readers, but stale
- // values are ok.
- port::AtomicPointer max_height_; // Height of the entire list
-
- inline int GetMaxHeight() const {
- return static_cast<int>(
- reinterpret_cast<intptr_t>(max_height_.NoBarrier_Load()));
- }
-
- // Read/written only by Insert().
- Random rnd_;
-
- Node* NewNode(const Key& key, int height);
- int RandomHeight();
- bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); }
-
- // Return true if key is greater than the data stored in "n"
- bool KeyIsAfterNode(const Key& key, Node* n) const;
-
- // Return the earliest node that comes at or after key.
- // Return NULL if there is no such node.
- //
- // If prev is non-NULL, fills prev[level] with pointer to previous
- // node at "level" for every level in [0..max_height_-1].
- Node* FindGreaterOrEqual(const Key& key, Node** prev) const;
-
- // Return the latest node with a key < key.
- // Return head_ if there is no such node.
- Node* FindLessThan(const Key& key) const;
-
- // Return the last node in the list.
- // Return head_ if list is empty.
- Node* FindLast() const;
-
- // No copying allowed
- SkipList(const SkipList&);
- void operator=(const SkipList&);
-};
-
-// Implementation details follow
-template<typename Key, class Comparator>
-struct SkipList<Key,Comparator>::Node {
- explicit Node(const Key& k) : key(k) { }
-
- Key const key;
-
- // Accessors/mutators for links. Wrapped in methods so we can
- // add the appropriate barriers as necessary.
- Node* Next(int n) {
- assert(n >= 0);
- // Use an 'acquire load' so that we observe a fully initialized
- // version of the returned Node.
- return reinterpret_cast<Node*>(next_[n].Acquire_Load());
- }
- void SetNext(int n, Node* x) {
- assert(n >= 0);
- // Use a 'release store' so that anybody who reads through this
- // pointer observes a fully initialized version of the inserted node.
- next_[n].Release_Store(x);
- }
-
- // No-barrier variants that can be safely used in a few locations.
- Node* NoBarrier_Next(int n) {
- assert(n >= 0);
- return reinterpret_cast<Node*>(next_[n].NoBarrier_Load());
- }
- void NoBarrier_SetNext(int n, Node* x) {
- assert(n >= 0);
- next_[n].NoBarrier_Store(x);
- }
-
- private:
- // Array of length equal to the node height. next_[0] is lowest level link.
- port::AtomicPointer next_[1];
-};
-
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node*
-SkipList<Key,Comparator>::NewNode(const Key& key, int height) {
- char* mem = arena_->AllocateAligned(
- sizeof(Node) + sizeof(port::AtomicPointer) * (height - 1));
- return new (mem) Node(key);
-}
-
-template<typename Key, class Comparator>
-inline SkipList<Key,Comparator>::Iterator::Iterator(const SkipList* list) {
- list_ = list;
- node_ = NULL;
-}
-
-template<typename Key, class Comparator>
-inline bool SkipList<Key,Comparator>::Iterator::Valid() const {
- return node_ != NULL;
-}
-
-template<typename Key, class Comparator>
-inline const Key& SkipList<Key,Comparator>::Iterator::key() const {
- assert(Valid());
- return node_->key;
-}
-
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::Next() {
- assert(Valid());
- node_ = node_->Next(0);
-}
-
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::Prev() {
- // Instead of using explicit "prev" links, we just search for the
- // last node that falls before key.
- assert(Valid());
- node_ = list_->FindLessThan(node_->key);
- if (node_ == list_->head_) {
- node_ = NULL;
- }
-}
-
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::Seek(const Key& target) {
- node_ = list_->FindGreaterOrEqual(target, NULL);
-}
-
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::SeekToFirst() {
- node_ = list_->head_->Next(0);
-}
-
-template<typename Key, class Comparator>
-inline void SkipList<Key,Comparator>::Iterator::SeekToLast() {
- node_ = list_->FindLast();
- if (node_ == list_->head_) {
- node_ = NULL;
- }
-}
-
-template<typename Key, class Comparator>
-int SkipList<Key,Comparator>::RandomHeight() {
- // Increase height with probability 1 in kBranching
- static const unsigned int kBranching = 4;
- int height = 1;
- while (height < kMaxHeight && ((rnd_.Next() % kBranching) == 0)) {
- height++;
- }
- assert(height > 0);
- assert(height <= kMaxHeight);
- return height;
-}
-
-template<typename Key, class Comparator>
-bool SkipList<Key,Comparator>::KeyIsAfterNode(const Key& key, Node* n) const {
- // NULL n is considered infinite
- return (n != NULL) && (compare_(n->key, key) < 0);
-}
-
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOrEqual(const Key& key, Node** prev)
- const {
- Node* x = head_;
- int level = GetMaxHeight() - 1;
- while (true) {
- Node* next = x->Next(level);
- if (KeyIsAfterNode(key, next)) {
- // Keep searching in this list
- x = next;
- } else {
- if (prev != NULL) prev[level] = x;
- if (level == 0) {
- return next;
- } else {
- // Switch to next list
- level--;
- }
- }
- }
-}
-
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node*
-SkipList<Key,Comparator>::FindLessThan(const Key& key) const {
- Node* x = head_;
- int level = GetMaxHeight() - 1;
- while (true) {
- assert(x == head_ || compare_(x->key, key) < 0);
- Node* next = x->Next(level);
- if (next == NULL || compare_(next->key, key) >= 0) {
- if (level == 0) {
- return x;
- } else {
- // Switch to next list
- level--;
- }
- } else {
- x = next;
- }
- }
-}
-
-template<typename Key, class Comparator>
-typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindLast()
- const {
- Node* x = head_;
- int level = GetMaxHeight() - 1;
- while (true) {
- Node* next = x->Next(level);
- if (next == NULL) {
- if (level == 0) {
- return x;
- } else {
- // Switch to next list
- level--;
- }
- } else {
- x = next;
- }
- }
-}
-
-template<typename Key, class Comparator>
-SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena)
- : compare_(cmp),
- arena_(arena),
- head_(NewNode(0 /* any key will do */, kMaxHeight)),
- max_height_(reinterpret_cast<void*>(1)),
- rnd_(0xdeadbeef) {
- for (int i = 0; i < kMaxHeight; i++) {
- head_->SetNext(i, NULL);
- }
-}
-
-template<typename Key, class Comparator>
-void SkipList<Key,Comparator>::Insert(const Key& key) {
- // TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual()
- // here since Insert() is externally synchronized.
- Node* prev[kMaxHeight];
- Node* x = FindGreaterOrEqual(key, prev);
-
- // Our data structure does not allow duplicate insertion
- assert(x == NULL || !Equal(key, x->key));
-
- int height = RandomHeight();
- if (height > GetMaxHeight()) {
- for (int i = GetMaxHeight(); i < height; i++) {
- prev[i] = head_;
- }
- //fprintf(stderr, "Change height from %d to %d\n", max_height_, height);
-
- // It is ok to mutate max_height_ without any synchronization
- // with concurrent readers. A concurrent reader that observes
- // the new value of max_height_ will see either the old value of
- // new level pointers from head_ (NULL), or a new value set in
- // the loop below. In the former case the reader will
- // immediately drop to the next level since NULL sorts after all
- // keys. In the latter case the reader will use the new node.
- max_height_.NoBarrier_Store(reinterpret_cast<void*>(height));
- }
-
- x = NewNode(key, height);
- for (int i = 0; i < height; i++) {
- // NoBarrier_SetNext() suffices since we will add a barrier when
- // we publish a pointer to "x" in prev[i].
- x->NoBarrier_SetNext(i, prev[i]->NoBarrier_Next(i));
- prev[i]->SetNext(i, x);
- }
-}
-
-template<typename Key, class Comparator>
-bool SkipList<Key,Comparator>::Contains(const Key& key) const {
- Node* x = FindGreaterOrEqual(key, NULL);
- if (x != NULL && Equal(key, x->key)) {
- return true;
- } else {
- return false;
- }
-}
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/skiplist_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/skiplist_test.cc b/thirdparty/leveldb-1.18/db/skiplist_test.cc
deleted file mode 100755
index bb9f59b..0000000
--- a/thirdparty/leveldb-1.18/db/skiplist_test.cc
+++ /dev/null
@@ -1,378 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/skiplist.h"
-#include <set>
-#include "leveldb/env.h"
-#include "util/arena.h"
-#include "util/hash.h"
-#include "util/random.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-typedef uint64_t Key;
-
-struct Comparator {
- int operator()(const Key& a, const Key& b) const {
- if (a < b) {
- return -1;
- } else if (a > b) {
- return +1;
- } else {
- return 0;
- }
- }
-};
-
-class SkipTest { };
-
-TEST(SkipTest, Empty) {
- Arena arena;
- Comparator cmp;
- SkipList<Key, Comparator> list(cmp, &arena);
- ASSERT_TRUE(!list.Contains(10));
-
- SkipList<Key, Comparator>::Iterator iter(&list);
- ASSERT_TRUE(!iter.Valid());
- iter.SeekToFirst();
- ASSERT_TRUE(!iter.Valid());
- iter.Seek(100);
- ASSERT_TRUE(!iter.Valid());
- iter.SeekToLast();
- ASSERT_TRUE(!iter.Valid());
-}
-
-TEST(SkipTest, InsertAndLookup) {
- const int N = 2000;
- const int R = 5000;
- Random rnd(1000);
- std::set<Key> keys;
- Arena arena;
- Comparator cmp;
- SkipList<Key, Comparator> list(cmp, &arena);
- for (int i = 0; i < N; i++) {
- Key key = rnd.Next() % R;
- if (keys.insert(key).second) {
- list.Insert(key);
- }
- }
-
- for (int i = 0; i < R; i++) {
- if (list.Contains(i)) {
- ASSERT_EQ(keys.count(i), 1);
- } else {
- ASSERT_EQ(keys.count(i), 0);
- }
- }
-
- // Simple iterator tests
- {
- SkipList<Key, Comparator>::Iterator iter(&list);
- ASSERT_TRUE(!iter.Valid());
-
- iter.Seek(0);
- ASSERT_TRUE(iter.Valid());
- ASSERT_EQ(*(keys.begin()), iter.key());
-
- iter.SeekToFirst();
- ASSERT_TRUE(iter.Valid());
- ASSERT_EQ(*(keys.begin()), iter.key());
-
- iter.SeekToLast();
- ASSERT_TRUE(iter.Valid());
- ASSERT_EQ(*(keys.rbegin()), iter.key());
- }
-
- // Forward iteration test
- for (int i = 0; i < R; i++) {
- SkipList<Key, Comparator>::Iterator iter(&list);
- iter.Seek(i);
-
- // Compare against model iterator
- std::set<Key>::iterator model_iter = keys.lower_bound(i);
- for (int j = 0; j < 3; j++) {
- if (model_iter == keys.end()) {
- ASSERT_TRUE(!iter.Valid());
- break;
- } else {
- ASSERT_TRUE(iter.Valid());
- ASSERT_EQ(*model_iter, iter.key());
- ++model_iter;
- iter.Next();
- }
- }
- }
-
- // Backward iteration test
- {
- SkipList<Key, Comparator>::Iterator iter(&list);
- iter.SeekToLast();
-
- // Compare against model iterator
- for (std::set<Key>::reverse_iterator model_iter = keys.rbegin();
- model_iter != keys.rend();
- ++model_iter) {
- ASSERT_TRUE(iter.Valid());
- ASSERT_EQ(*model_iter, iter.key());
- iter.Prev();
- }
- ASSERT_TRUE(!iter.Valid());
- }
-}
-
-// We want to make sure that with a single writer and multiple
-// concurrent readers (with no synchronization other than when a
-// reader's iterator is created), the reader always observes all the
-// data that was present in the skip list when the iterator was
-// constructor. Because insertions are happening concurrently, we may
-// also observe new values that were inserted since the iterator was
-// constructed, but we should never miss any values that were present
-// at iterator construction time.
-//
-// We generate multi-part keys:
-// <key,gen,hash>
-// where:
-// key is in range [0..K-1]
-// gen is a generation number for key
-// hash is hash(key,gen)
-//
-// The insertion code picks a random key, sets gen to be 1 + the last
-// generation number inserted for that key, and sets hash to Hash(key,gen).
-//
-// At the beginning of a read, we snapshot the last inserted
-// generation number for each key. We then iterate, including random
-// calls to Next() and Seek(). For every key we encounter, we
-// check that it is either expected given the initial snapshot or has
-// been concurrently added since the iterator started.
-class ConcurrentTest {
- private:
- static const uint32_t K = 4;
-
- static uint64_t key(Key key) { return (key >> 40); }
- static uint64_t gen(Key key) { return (key >> 8) & 0xffffffffu; }
- static uint64_t hash(Key key) { return key & 0xff; }
-
- static uint64_t HashNumbers(uint64_t k, uint64_t g) {
- uint64_t data[2] = { k, g };
- return Hash(reinterpret_cast<char*>(data), sizeof(data), 0);
- }
-
- static Key MakeKey(uint64_t k, uint64_t g) {
- assert(sizeof(Key) == sizeof(uint64_t));
- assert(k <= K); // We sometimes pass K to seek to the end of the skiplist
- assert(g <= 0xffffffffu);
- return ((k << 40) | (g << 8) | (HashNumbers(k, g) & 0xff));
- }
-
- static bool IsValidKey(Key k) {
- return hash(k) == (HashNumbers(key(k), gen(k)) & 0xff);
- }
-
- static Key RandomTarget(Random* rnd) {
- switch (rnd->Next() % 10) {
- case 0:
- // Seek to beginning
- return MakeKey(0, 0);
- case 1:
- // Seek to end
- return MakeKey(K, 0);
- default:
- // Seek to middle
- return MakeKey(rnd->Next() % K, 0);
- }
- }
-
- // Per-key generation
- struct State {
- port::AtomicPointer generation[K];
- void Set(int k, intptr_t v) {
- generation[k].Release_Store(reinterpret_cast<void*>(v));
- }
- intptr_t Get(int k) {
- return reinterpret_cast<intptr_t>(generation[k].Acquire_Load());
- }
-
- State() {
- for (int k = 0; k < K; k++) {
- Set(k, 0);
- }
- }
- };
-
- // Current state of the test
- State current_;
-
- Arena arena_;
-
- // SkipList is not protected by mu_. We just use a single writer
- // thread to modify it.
- SkipList<Key, Comparator> list_;
-
- public:
- ConcurrentTest() : list_(Comparator(), &arena_) { }
-
- // REQUIRES: External synchronization
- void WriteStep(Random* rnd) {
- const uint32_t k = rnd->Next() % K;
- const intptr_t g = current_.Get(k) + 1;
- const Key key = MakeKey(k, g);
- list_.Insert(key);
- current_.Set(k, g);
- }
-
- void ReadStep(Random* rnd) {
- // Remember the initial committed state of the skiplist.
- State initial_state;
- for (int k = 0; k < K; k++) {
- initial_state.Set(k, current_.Get(k));
- }
-
- Key pos = RandomTarget(rnd);
- SkipList<Key, Comparator>::Iterator iter(&list_);
- iter.Seek(pos);
- while (true) {
- Key current;
- if (!iter.Valid()) {
- current = MakeKey(K, 0);
- } else {
- current = iter.key();
- ASSERT_TRUE(IsValidKey(current)) << current;
- }
- ASSERT_LE(pos, current) << "should not go backwards";
-
- // Verify that everything in [pos,current) was not present in
- // initial_state.
- while (pos < current) {
- ASSERT_LT(key(pos), K) << pos;
-
- // Note that generation 0 is never inserted, so it is ok if
- // <*,0,*> is missing.
- ASSERT_TRUE((gen(pos) == 0u) ||
- (gen(pos) > static_cast<uint64_t>(initial_state.Get(static_cast<int>(key(pos)))))
- ) << "key: " << key(pos)
- << "; gen: " << gen(pos)
- << "; initgen: "
- << initial_state.Get(static_cast<int>(key(pos)));
-
- // Advance to next key in the valid key space
- if (key(pos) < key(current)) {
- pos = MakeKey(key(pos) + 1, 0);
- } else {
- pos = MakeKey(key(pos), gen(pos) + 1);
- }
- }
-
- if (!iter.Valid()) {
- break;
- }
-
- if (rnd->Next() % 2) {
- iter.Next();
- pos = MakeKey(key(pos), gen(pos) + 1);
- } else {
- Key new_target = RandomTarget(rnd);
- if (new_target > pos) {
- pos = new_target;
- iter.Seek(new_target);
- }
- }
- }
- }
-};
-const uint32_t ConcurrentTest::K;
-
-// Simple test that does single-threaded testing of the ConcurrentTest
-// scaffolding.
-TEST(SkipTest, ConcurrentWithoutThreads) {
- ConcurrentTest test;
- Random rnd(test::RandomSeed());
- for (int i = 0; i < 10000; i++) {
- test.ReadStep(&rnd);
- test.WriteStep(&rnd);
- }
-}
-
-class TestState {
- public:
- ConcurrentTest t_;
- int seed_;
- port::AtomicPointer quit_flag_;
-
- enum ReaderState {
- STARTING,
- RUNNING,
- DONE
- };
-
- explicit TestState(int s)
- : seed_(s),
- quit_flag_(NULL),
- state_(STARTING),
- state_cv_(&mu_) {}
-
- void Wait(ReaderState s) {
- mu_.Lock();
- while (state_ != s) {
- state_cv_.Wait();
- }
- mu_.Unlock();
- }
-
- void Change(ReaderState s) {
- mu_.Lock();
- state_ = s;
- state_cv_.Signal();
- mu_.Unlock();
- }
-
- private:
- port::Mutex mu_;
- ReaderState state_;
- port::CondVar state_cv_;
-};
-
-static void ConcurrentReader(void* arg) {
- TestState* state = reinterpret_cast<TestState*>(arg);
- Random rnd(state->seed_);
- int64_t reads = 0;
- state->Change(TestState::RUNNING);
- while (!state->quit_flag_.Acquire_Load()) {
- state->t_.ReadStep(&rnd);
- ++reads;
- }
- state->Change(TestState::DONE);
-}
-
-static void RunConcurrent(int run) {
- const int seed = test::RandomSeed() + (run * 100);
- Random rnd(seed);
- const int N = 1000;
- const int kSize = 1000;
- for (int i = 0; i < N; i++) {
- if ((i % 100) == 0) {
- fprintf(stderr, "Run %d of %d\n", i, N);
- }
- TestState state(seed + 1);
- Env::Default()->Schedule(ConcurrentReader, &state);
- state.Wait(TestState::RUNNING);
- for (int i = 0; i < kSize; i++) {
- state.t_.WriteStep(&rnd);
- }
- state.quit_flag_.Release_Store(&state); // Any non-NULL arg will do
- state.Wait(TestState::DONE);
- }
-}
-
-TEST(SkipTest, Concurrent1) { RunConcurrent(1); }
-TEST(SkipTest, Concurrent2) { RunConcurrent(2); }
-TEST(SkipTest, Concurrent3) { RunConcurrent(3); }
-TEST(SkipTest, Concurrent4) { RunConcurrent(4); }
-TEST(SkipTest, Concurrent5) { RunConcurrent(5); }
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/snapshot.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/snapshot.h b/thirdparty/leveldb-1.18/db/snapshot.h
deleted file mode 100755
index e7f8fd2..0000000
--- a/thirdparty/leveldb-1.18/db/snapshot.h
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_
-#define STORAGE_LEVELDB_DB_SNAPSHOT_H_
-
-#include "leveldb/db.h"
-
-namespace leveldb {
-
-class SnapshotList;
-
-// Snapshots are kept in a doubly-linked list in the DB.
-// Each SnapshotImpl corresponds to a particular sequence number.
-class SnapshotImpl : public Snapshot {
- public:
- SequenceNumber number_; // const after creation
-
- private:
- friend class SnapshotList;
-
- // SnapshotImpl is kept in a doubly-linked circular list
- SnapshotImpl* prev_;
- SnapshotImpl* next_;
-
- SnapshotList* list_; // just for sanity checks
-};
-
-class SnapshotList {
- public:
- SnapshotList() {
- list_.prev_ = &list_;
- list_.next_ = &list_;
- }
-
- bool empty() const { return list_.next_ == &list_; }
- SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; }
- SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; }
-
- const SnapshotImpl* New(SequenceNumber seq) {
- SnapshotImpl* s = new SnapshotImpl;
- s->number_ = seq;
- s->list_ = this;
- s->next_ = &list_;
- s->prev_ = list_.prev_;
- s->prev_->next_ = s;
- s->next_->prev_ = s;
- return s;
- }
-
- void Delete(const SnapshotImpl* s) {
- assert(s->list_ == this);
- s->prev_->next_ = s->next_;
- s->next_->prev_ = s->prev_;
- delete s;
- }
-
- private:
- // Dummy head of doubly-linked list of snapshots
- SnapshotImpl list_;
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/table_cache.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/table_cache.cc b/thirdparty/leveldb-1.18/db/table_cache.cc
deleted file mode 100755
index e3d82cd..0000000
--- a/thirdparty/leveldb-1.18/db/table_cache.cc
+++ /dev/null
@@ -1,127 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/table_cache.h"
-
-#include "db/filename.h"
-#include "leveldb/env.h"
-#include "leveldb/table.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-struct TableAndFile {
- RandomAccessFile* file;
- Table* table;
-};
-
-static void DeleteEntry(const Slice& key, void* value) {
- TableAndFile* tf = reinterpret_cast<TableAndFile*>(value);
- delete tf->table;
- delete tf->file;
- delete tf;
-}
-
-static void UnrefEntry(void* arg1, void* arg2) {
- Cache* cache = reinterpret_cast<Cache*>(arg1);
- Cache::Handle* h = reinterpret_cast<Cache::Handle*>(arg2);
- cache->Release(h);
-}
-
-TableCache::TableCache(const std::string& dbname,
- const Options* options,
- int entries)
- : env_(options->env),
- dbname_(dbname),
- options_(options),
- cache_(NewLRUCache(entries)) {
-}
-
-TableCache::~TableCache() {
- delete cache_;
-}
-
-Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
- Cache::Handle** handle) {
- Status s;
- char buf[sizeof(file_number)];
- EncodeFixed64(buf, file_number);
- Slice key(buf, sizeof(buf));
- *handle = cache_->Lookup(key);
- if (*handle == NULL) {
- std::string fname = TableFileName(dbname_, file_number);
- RandomAccessFile* file = NULL;
- Table* table = NULL;
- s = env_->NewRandomAccessFile(fname, &file);
- if (!s.ok()) {
- std::string old_fname = SSTTableFileName(dbname_, file_number);
- if (env_->NewRandomAccessFile(old_fname, &file).ok()) {
- s = Status::OK();
- }
- }
- if (s.ok()) {
- s = Table::Open(*options_, file, file_size, &table);
- }
-
- if (!s.ok()) {
- assert(table == NULL);
- delete file;
- // We do not cache error results so that if the error is transient,
- // or somebody repairs the file, we recover automatically.
- } else {
- TableAndFile* tf = new TableAndFile;
- tf->file = file;
- tf->table = table;
- *handle = cache_->Insert(key, tf, 1, &DeleteEntry);
- }
- }
- return s;
-}
-
-Iterator* TableCache::NewIterator(const ReadOptions& options,
- uint64_t file_number,
- uint64_t file_size,
- Table** tableptr) {
- if (tableptr != NULL) {
- *tableptr = NULL;
- }
-
- Cache::Handle* handle = NULL;
- Status s = FindTable(file_number, file_size, &handle);
- if (!s.ok()) {
- return NewErrorIterator(s);
- }
-
- Table* table = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
- Iterator* result = table->NewIterator(options);
- result->RegisterCleanup(&UnrefEntry, cache_, handle);
- if (tableptr != NULL) {
- *tableptr = table;
- }
- return result;
-}
-
-Status TableCache::Get(const ReadOptions& options,
- uint64_t file_number,
- uint64_t file_size,
- const Slice& k,
- void* arg,
- void (*saver)(void*, const Slice&, const Slice&)) {
- Cache::Handle* handle = NULL;
- Status s = FindTable(file_number, file_size, &handle);
- if (s.ok()) {
- Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
- s = t->InternalGet(options, k, arg, saver);
- cache_->Release(handle);
- }
- return s;
-}
-
-void TableCache::Evict(uint64_t file_number) {
- char buf[sizeof(file_number)];
- EncodeFixed64(buf, file_number);
- cache_->Erase(Slice(buf, sizeof(buf)));
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/table_cache.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/table_cache.h b/thirdparty/leveldb-1.18/db/table_cache.h
deleted file mode 100755
index 8cf4aaf..0000000
--- a/thirdparty/leveldb-1.18/db/table_cache.h
+++ /dev/null
@@ -1,61 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Thread-safe (provides internal synchronization)
-
-#ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_
-#define STORAGE_LEVELDB_DB_TABLE_CACHE_H_
-
-#include <string>
-#include <stdint.h>
-#include "db/dbformat.h"
-#include "leveldb/cache.h"
-#include "leveldb/table.h"
-#include "port/port.h"
-
-namespace leveldb {
-
-class Env;
-
-class TableCache {
- public:
- TableCache(const std::string& dbname, const Options* options, int entries);
- ~TableCache();
-
- // Return an iterator for the specified file number (the corresponding
- // file length must be exactly "file_size" bytes). If "tableptr" is
- // non-NULL, also sets "*tableptr" to point to the Table object
- // underlying the returned iterator, or NULL if no Table object underlies
- // the returned iterator. The returned "*tableptr" object is owned by
- // the cache and should not be deleted, and is valid for as long as the
- // returned iterator is live.
- Iterator* NewIterator(const ReadOptions& options,
- uint64_t file_number,
- uint64_t file_size,
- Table** tableptr = NULL);
-
- // If a seek to internal key "k" in specified file finds an entry,
- // call (*handle_result)(arg, found_key, found_value).
- Status Get(const ReadOptions& options,
- uint64_t file_number,
- uint64_t file_size,
- const Slice& k,
- void* arg,
- void (*handle_result)(void*, const Slice&, const Slice&));
-
- // Evict any entry for the specified file number
- void Evict(uint64_t file_number);
-
- private:
- Env* const env_;
- const std::string dbname_;
- const Options* options_;
- Cache* cache_;
-
- Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**);
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/version_edit.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/version_edit.cc b/thirdparty/leveldb-1.18/db/version_edit.cc
deleted file mode 100755
index f10a2d5..0000000
--- a/thirdparty/leveldb-1.18/db/version_edit.cc
+++ /dev/null
@@ -1,266 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/version_edit.h"
-
-#include "db/version_set.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-// Tag numbers for serialized VersionEdit. These numbers are written to
-// disk and should not be changed.
-enum Tag {
- kComparator = 1,
- kLogNumber = 2,
- kNextFileNumber = 3,
- kLastSequence = 4,
- kCompactPointer = 5,
- kDeletedFile = 6,
- kNewFile = 7,
- // 8 was used for large value refs
- kPrevLogNumber = 9
-};
-
-void VersionEdit::Clear() {
- comparator_.clear();
- log_number_ = 0;
- prev_log_number_ = 0;
- last_sequence_ = 0;
- next_file_number_ = 0;
- has_comparator_ = false;
- has_log_number_ = false;
- has_prev_log_number_ = false;
- has_next_file_number_ = false;
- has_last_sequence_ = false;
- deleted_files_.clear();
- new_files_.clear();
-}
-
-void VersionEdit::EncodeTo(std::string* dst) const {
- if (has_comparator_) {
- PutVarint32(dst, kComparator);
- PutLengthPrefixedSlice(dst, comparator_);
- }
- if (has_log_number_) {
- PutVarint32(dst, kLogNumber);
- PutVarint64(dst, log_number_);
- }
- if (has_prev_log_number_) {
- PutVarint32(dst, kPrevLogNumber);
- PutVarint64(dst, prev_log_number_);
- }
- if (has_next_file_number_) {
- PutVarint32(dst, kNextFileNumber);
- PutVarint64(dst, next_file_number_);
- }
- if (has_last_sequence_) {
- PutVarint32(dst, kLastSequence);
- PutVarint64(dst, last_sequence_);
- }
-
- for (size_t i = 0; i < compact_pointers_.size(); i++) {
- PutVarint32(dst, kCompactPointer);
- PutVarint32(dst, compact_pointers_[i].first); // level
- PutLengthPrefixedSlice(dst, compact_pointers_[i].second.Encode());
- }
-
- for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
- iter != deleted_files_.end();
- ++iter) {
- PutVarint32(dst, kDeletedFile);
- PutVarint32(dst, iter->first); // level
- PutVarint64(dst, iter->second); // file number
- }
-
- for (size_t i = 0; i < new_files_.size(); i++) {
- const FileMetaData& f = new_files_[i].second;
- PutVarint32(dst, kNewFile);
- PutVarint32(dst, new_files_[i].first); // level
- PutVarint64(dst, f.number);
- PutVarint64(dst, f.file_size);
- PutLengthPrefixedSlice(dst, f.smallest.Encode());
- PutLengthPrefixedSlice(dst, f.largest.Encode());
- }
-}
-
-static bool GetInternalKey(Slice* input, InternalKey* dst) {
- Slice str;
- if (GetLengthPrefixedSlice(input, &str)) {
- dst->DecodeFrom(str);
- return true;
- } else {
- return false;
- }
-}
-
-static bool GetLevel(Slice* input, int* level) {
- uint32_t v;
- if (GetVarint32(input, &v) &&
- v < config::kNumLevels) {
- *level = v;
- return true;
- } else {
- return false;
- }
-}
-
-Status VersionEdit::DecodeFrom(const Slice& src) {
- Clear();
- Slice input = src;
- const char* msg = NULL;
- uint32_t tag;
-
- // Temporary storage for parsing
- int level;
- uint64_t number;
- FileMetaData f;
- Slice str;
- InternalKey key;
-
- while (msg == NULL && GetVarint32(&input, &tag)) {
- switch (tag) {
- case kComparator:
- if (GetLengthPrefixedSlice(&input, &str)) {
- comparator_ = str.ToString();
- has_comparator_ = true;
- } else {
- msg = "comparator name";
- }
- break;
-
- case kLogNumber:
- if (GetVarint64(&input, &log_number_)) {
- has_log_number_ = true;
- } else {
- msg = "log number";
- }
- break;
-
- case kPrevLogNumber:
- if (GetVarint64(&input, &prev_log_number_)) {
- has_prev_log_number_ = true;
- } else {
- msg = "previous log number";
- }
- break;
-
- case kNextFileNumber:
- if (GetVarint64(&input, &next_file_number_)) {
- has_next_file_number_ = true;
- } else {
- msg = "next file number";
- }
- break;
-
- case kLastSequence:
- if (GetVarint64(&input, &last_sequence_)) {
- has_last_sequence_ = true;
- } else {
- msg = "last sequence number";
- }
- break;
-
- case kCompactPointer:
- if (GetLevel(&input, &level) &&
- GetInternalKey(&input, &key)) {
- compact_pointers_.push_back(std::make_pair(level, key));
- } else {
- msg = "compaction pointer";
- }
- break;
-
- case kDeletedFile:
- if (GetLevel(&input, &level) &&
- GetVarint64(&input, &number)) {
- deleted_files_.insert(std::make_pair(level, number));
- } else {
- msg = "deleted file";
- }
- break;
-
- case kNewFile:
- if (GetLevel(&input, &level) &&
- GetVarint64(&input, &f.number) &&
- GetVarint64(&input, &f.file_size) &&
- GetInternalKey(&input, &f.smallest) &&
- GetInternalKey(&input, &f.largest)) {
- new_files_.push_back(std::make_pair(level, f));
- } else {
- msg = "new-file entry";
- }
- break;
-
- default:
- msg = "unknown tag";
- break;
- }
- }
-
- if (msg == NULL && !input.empty()) {
- msg = "invalid tag";
- }
-
- Status result;
- if (msg != NULL) {
- result = Status::Corruption("VersionEdit", msg);
- }
- return result;
-}
-
-std::string VersionEdit::DebugString() const {
- std::string r;
- r.append("VersionEdit {");
- if (has_comparator_) {
- r.append("\n Comparator: ");
- r.append(comparator_);
- }
- if (has_log_number_) {
- r.append("\n LogNumber: ");
- AppendNumberTo(&r, log_number_);
- }
- if (has_prev_log_number_) {
- r.append("\n PrevLogNumber: ");
- AppendNumberTo(&r, prev_log_number_);
- }
- if (has_next_file_number_) {
- r.append("\n NextFile: ");
- AppendNumberTo(&r, next_file_number_);
- }
- if (has_last_sequence_) {
- r.append("\n LastSeq: ");
- AppendNumberTo(&r, last_sequence_);
- }
- for (size_t i = 0; i < compact_pointers_.size(); i++) {
- r.append("\n CompactPointer: ");
- AppendNumberTo(&r, compact_pointers_[i].first);
- r.append(" ");
- r.append(compact_pointers_[i].second.DebugString());
- }
- for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
- iter != deleted_files_.end();
- ++iter) {
- r.append("\n DeleteFile: ");
- AppendNumberTo(&r, iter->first);
- r.append(" ");
- AppendNumberTo(&r, iter->second);
- }
- for (size_t i = 0; i < new_files_.size(); i++) {
- const FileMetaData& f = new_files_[i].second;
- r.append("\n AddFile: ");
- AppendNumberTo(&r, new_files_[i].first);
- r.append(" ");
- AppendNumberTo(&r, f.number);
- r.append(" ");
- AppendNumberTo(&r, f.file_size);
- r.append(" ");
- r.append(f.smallest.DebugString());
- r.append(" .. ");
- r.append(f.largest.DebugString());
- }
- r.append("\n}\n");
- return r;
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/version_edit.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/version_edit.h b/thirdparty/leveldb-1.18/db/version_edit.h
deleted file mode 100755
index 62b02d4..0000000
--- a/thirdparty/leveldb-1.18/db/version_edit.h
+++ /dev/null
@@ -1,107 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_VERSION_EDIT_H_
-#define STORAGE_LEVELDB_DB_VERSION_EDIT_H_
-
-#include <set>
-#include <utility>
-#include <vector>
-#include "db/dbformat.h"
-
-namespace leveldb {
-
-class VersionSet;
-
-struct FileMetaData {
- int refs;
- size_t allowed_seeks; // Seeks allowed until compaction
- uint64_t number;
- uint64_t file_size; // File size in bytes
- InternalKey smallest; // Smallest internal key served by table
- InternalKey largest; // Largest internal key served by table
-
- FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0) { }
-};
-
-class VersionEdit {
- public:
- VersionEdit() { Clear(); }
- ~VersionEdit() { }
-
- void Clear();
-
- void SetComparatorName(const Slice& name) {
- has_comparator_ = true;
- comparator_ = name.ToString();
- }
- void SetLogNumber(uint64_t num) {
- has_log_number_ = true;
- log_number_ = num;
- }
- void SetPrevLogNumber(uint64_t num) {
- has_prev_log_number_ = true;
- prev_log_number_ = num;
- }
- void SetNextFile(uint64_t num) {
- has_next_file_number_ = true;
- next_file_number_ = num;
- }
- void SetLastSequence(SequenceNumber seq) {
- has_last_sequence_ = true;
- last_sequence_ = seq;
- }
- void SetCompactPointer(int level, const InternalKey& key) {
- compact_pointers_.push_back(std::make_pair(level, key));
- }
-
- // Add the specified file at the specified number.
- // REQUIRES: This version has not been saved (see VersionSet::SaveTo)
- // REQUIRES: "smallest" and "largest" are smallest and largest keys in file
- void AddFile(int level, uint64_t file,
- uint64_t file_size,
- const InternalKey& smallest,
- const InternalKey& largest) {
- FileMetaData f;
- f.number = file;
- f.file_size = file_size;
- f.smallest = smallest;
- f.largest = largest;
- new_files_.push_back(std::make_pair(level, f));
- }
-
- // Delete the specified "file" from the specified "level".
- void DeleteFile(int level, uint64_t file) {
- deleted_files_.insert(std::make_pair(level, file));
- }
-
- void EncodeTo(std::string* dst) const;
- Status DecodeFrom(const Slice& src);
-
- std::string DebugString() const;
-
- private:
- friend class VersionSet;
-
- typedef std::set< std::pair<int, uint64_t> > DeletedFileSet;
-
- std::string comparator_;
- uint64_t log_number_;
- uint64_t prev_log_number_;
- uint64_t next_file_number_;
- SequenceNumber last_sequence_;
- bool has_comparator_;
- bool has_log_number_;
- bool has_prev_log_number_;
- bool has_next_file_number_;
- bool has_last_sequence_;
-
- std::vector< std::pair<int, InternalKey> > compact_pointers_;
- DeletedFileSet deleted_files_;
- std::vector< std::pair<int, FileMetaData> > new_files_;
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_VERSION_EDIT_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/version_edit_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/version_edit_test.cc b/thirdparty/leveldb-1.18/db/version_edit_test.cc
deleted file mode 100755
index 280310b..0000000
--- a/thirdparty/leveldb-1.18/db/version_edit_test.cc
+++ /dev/null
@@ -1,46 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/version_edit.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-static void TestEncodeDecode(const VersionEdit& edit) {
- std::string encoded, encoded2;
- edit.EncodeTo(&encoded);
- VersionEdit parsed;
- Status s = parsed.DecodeFrom(encoded);
- ASSERT_TRUE(s.ok()) << s.ToString();
- parsed.EncodeTo(&encoded2);
- ASSERT_EQ(encoded, encoded2);
-}
-
-class VersionEditTest { };
-
-TEST(VersionEditTest, EncodeDecode) {
- static const uint64_t kBig = 1ull << 50;
-
- VersionEdit edit;
- for (int i = 0; i < 4; i++) {
- TestEncodeDecode(edit);
- edit.AddFile(3, kBig + 300 + i, kBig + 400 + i,
- InternalKey("foo", kBig + 500 + i, kTypeValue),
- InternalKey("zoo", kBig + 600 + i, kTypeDeletion));
- edit.DeleteFile(4, kBig + 700 + i);
- edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue));
- }
-
- edit.SetComparatorName("foo");
- edit.SetLogNumber(kBig + 100);
- edit.SetNextFile(kBig + 200);
- edit.SetLastSequence(kBig + 1000);
- TestEncodeDecode(edit);
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
[18/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_iter.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_iter.cc b/thirdparty/rocksdb/db/db_iter.cc
new file mode 100644
index 0000000..801b110
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_iter.cc
@@ -0,0 +1,1256 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/db_iter.h"
+#include <stdexcept>
+#include <deque>
+#include <string>
+#include <limits>
+
+#include "db/dbformat.h"
+#include "db/merge_context.h"
+#include "db/merge_helper.h"
+#include "db/pinned_iterators_manager.h"
+#include "monitoring/perf_context_imp.h"
+#include "port/port.h"
+#include "rocksdb/env.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/options.h"
+#include "table/internal_iterator.h"
+#include "util/arena.h"
+#include "util/filename.h"
+#include "util/logging.h"
+#include "util/mutexlock.h"
+#include "util/string_util.h"
+
+namespace rocksdb {
+
+#if 0
+static void DumpInternalIter(Iterator* iter) {
+ for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+ ParsedInternalKey k;
+ if (!ParseInternalKey(iter->key(), &k)) {
+ fprintf(stderr, "Corrupt '%s'\n", EscapeString(iter->key()).c_str());
+ } else {
+ fprintf(stderr, "@ '%s'\n", k.DebugString().c_str());
+ }
+ }
+}
+#endif
+
+// Memtables and sstables that make the DB representation contain
+// (userkey,seq,type) => uservalue entries. DBIter
+// combines multiple entries for the same userkey found in the DB
+// representation into a single entry while accounting for sequence
+// numbers, deletion markers, overwrites, etc.
+class DBIter: public Iterator {
+ public:
+ // The following is grossly complicated. TODO: clean it up
+ // Which direction is the iterator currently moving?
+ // (1) When moving forward, the internal iterator is positioned at
+ // the exact entry that yields this->key(), this->value()
+ // (2) When moving backwards, the internal iterator is positioned
+ // just before all entries whose user key == this->key().
+ enum Direction {
+ kForward,
+ kReverse
+ };
+
+ // LocalStatistics contain Statistics counters that will be aggregated per
+ // each iterator instance and then will be sent to the global statistics when
+ // the iterator is destroyed.
+ //
+ // The purpose of this approach is to avoid perf regression happening
+ // when multiple threads bump the atomic counters from a DBIter::Next().
+ struct LocalStatistics {
+ explicit LocalStatistics() { ResetCounters(); }
+
+ void ResetCounters() {
+ next_count_ = 0;
+ next_found_count_ = 0;
+ prev_count_ = 0;
+ prev_found_count_ = 0;
+ bytes_read_ = 0;
+ }
+
+ void BumpGlobalStatistics(Statistics* global_statistics) {
+ RecordTick(global_statistics, NUMBER_DB_NEXT, next_count_);
+ RecordTick(global_statistics, NUMBER_DB_NEXT_FOUND, next_found_count_);
+ RecordTick(global_statistics, NUMBER_DB_PREV, prev_count_);
+ RecordTick(global_statistics, NUMBER_DB_PREV_FOUND, prev_found_count_);
+ RecordTick(global_statistics, ITER_BYTES_READ, bytes_read_);
+ ResetCounters();
+ }
+
+ // Map to Tickers::NUMBER_DB_NEXT
+ uint64_t next_count_;
+ // Map to Tickers::NUMBER_DB_NEXT_FOUND
+ uint64_t next_found_count_;
+ // Map to Tickers::NUMBER_DB_PREV
+ uint64_t prev_count_;
+ // Map to Tickers::NUMBER_DB_PREV_FOUND
+ uint64_t prev_found_count_;
+ // Map to Tickers::ITER_BYTES_READ
+ uint64_t bytes_read_;
+ };
+
+ DBIter(Env* _env, const ReadOptions& read_options,
+ const ImmutableCFOptions& cf_options, const Comparator* cmp,
+ InternalIterator* iter, SequenceNumber s, bool arena_mode,
+ uint64_t max_sequential_skip_in_iterations)
+ : arena_mode_(arena_mode),
+ env_(_env),
+ logger_(cf_options.info_log),
+ user_comparator_(cmp),
+ merge_operator_(cf_options.merge_operator),
+ iter_(iter),
+ sequence_(s),
+ direction_(kForward),
+ valid_(false),
+ current_entry_is_merged_(false),
+ statistics_(cf_options.statistics),
+ iterate_upper_bound_(read_options.iterate_upper_bound),
+ prefix_same_as_start_(read_options.prefix_same_as_start),
+ pin_thru_lifetime_(read_options.pin_data),
+ total_order_seek_(read_options.total_order_seek),
+ range_del_agg_(cf_options.internal_comparator, s,
+ true /* collapse_deletions */) {
+ RecordTick(statistics_, NO_ITERATORS);
+ prefix_extractor_ = cf_options.prefix_extractor;
+ max_skip_ = max_sequential_skip_in_iterations;
+ max_skippable_internal_keys_ = read_options.max_skippable_internal_keys;
+ if (pin_thru_lifetime_) {
+ pinned_iters_mgr_.StartPinning();
+ }
+ if (iter_) {
+ iter_->SetPinnedItersMgr(&pinned_iters_mgr_);
+ }
+ }
+ virtual ~DBIter() {
+ // Release pinned data if any
+ if (pinned_iters_mgr_.PinningEnabled()) {
+ pinned_iters_mgr_.ReleasePinnedData();
+ }
+ RecordTick(statistics_, NO_ITERATORS, -1);
+ local_stats_.BumpGlobalStatistics(statistics_);
+ if (!arena_mode_) {
+ delete iter_;
+ } else {
+ iter_->~InternalIterator();
+ }
+ }
+ virtual void SetIter(InternalIterator* iter) {
+ assert(iter_ == nullptr);
+ iter_ = iter;
+ iter_->SetPinnedItersMgr(&pinned_iters_mgr_);
+ }
+ virtual RangeDelAggregator* GetRangeDelAggregator() {
+ return &range_del_agg_;
+ }
+
+ virtual bool Valid() const override { return valid_; }
+ virtual Slice key() const override {
+ assert(valid_);
+ return saved_key_.GetUserKey();
+ }
+ virtual Slice value() const override {
+ assert(valid_);
+ if (current_entry_is_merged_) {
+ // If pinned_value_ is set then the result of merge operator is one of
+ // the merge operands and we should return it.
+ return pinned_value_.data() ? pinned_value_ : saved_value_;
+ } else if (direction_ == kReverse) {
+ return pinned_value_;
+ } else {
+ return iter_->value();
+ }
+ }
+ virtual Status status() const override {
+ if (status_.ok()) {
+ return iter_->status();
+ } else {
+ return status_;
+ }
+ }
+
+ virtual Status GetProperty(std::string prop_name,
+ std::string* prop) override {
+ if (prop == nullptr) {
+ return Status::InvalidArgument("prop is nullptr");
+ }
+ if (prop_name == "rocksdb.iterator.super-version-number") {
+ // First try to pass the value returned from inner iterator.
+ return iter_->GetProperty(prop_name, prop);
+ } else if (prop_name == "rocksdb.iterator.is-key-pinned") {
+ if (valid_) {
+ *prop = (pin_thru_lifetime_ && saved_key_.IsKeyPinned()) ? "1" : "0";
+ } else {
+ *prop = "Iterator is not valid.";
+ }
+ return Status::OK();
+ }
+ return Status::InvalidArgument("Undentified property.");
+ }
+
+ virtual void Next() override;
+ virtual void Prev() override;
+ virtual void Seek(const Slice& target) override;
+ virtual void SeekForPrev(const Slice& target) override;
+ virtual void SeekToFirst() override;
+ virtual void SeekToLast() override;
+ Env* env() { return env_; }
+ void set_sequence(uint64_t s) { sequence_ = s; }
+ void set_valid(bool v) { valid_ = v; }
+
+ private:
+ void ReverseToForward();
+ void ReverseToBackward();
+ void PrevInternal();
+ void FindParseableKey(ParsedInternalKey* ikey, Direction direction);
+ bool FindValueForCurrentKey();
+ bool FindValueForCurrentKeyUsingSeek();
+ void FindPrevUserKey();
+ void FindNextUserKey();
+ inline void FindNextUserEntry(bool skipping, bool prefix_check);
+ void FindNextUserEntryInternal(bool skipping, bool prefix_check);
+ bool ParseKey(ParsedInternalKey* key);
+ void MergeValuesNewToOld();
+ bool TooManyInternalKeysSkipped(bool increment = true);
+
+ // Temporarily pin the blocks that we encounter until ReleaseTempPinnedData()
+ // is called
+ void TempPinData() {
+ if (!pin_thru_lifetime_) {
+ pinned_iters_mgr_.StartPinning();
+ }
+ }
+
+ // Release blocks pinned by TempPinData()
+ void ReleaseTempPinnedData() {
+ if (!pin_thru_lifetime_ && pinned_iters_mgr_.PinningEnabled()) {
+ pinned_iters_mgr_.ReleasePinnedData();
+ }
+ }
+
+ inline void ClearSavedValue() {
+ if (saved_value_.capacity() > 1048576) {
+ std::string empty;
+ swap(empty, saved_value_);
+ } else {
+ saved_value_.clear();
+ }
+ }
+
+ inline void ResetInternalKeysSkippedCounter() {
+ num_internal_keys_skipped_ = 0;
+ }
+
+ const SliceTransform* prefix_extractor_;
+ bool arena_mode_;
+ Env* const env_;
+ Logger* logger_;
+ const Comparator* const user_comparator_;
+ const MergeOperator* const merge_operator_;
+ InternalIterator* iter_;
+ SequenceNumber sequence_;
+
+ Status status_;
+ IterKey saved_key_;
+ std::string saved_value_;
+ Slice pinned_value_;
+ Direction direction_;
+ bool valid_;
+ bool current_entry_is_merged_;
+ // for prefix seek mode to support prev()
+ Statistics* statistics_;
+ uint64_t max_skip_;
+ uint64_t max_skippable_internal_keys_;
+ uint64_t num_internal_keys_skipped_;
+ const Slice* iterate_upper_bound_;
+ IterKey prefix_start_buf_;
+ Slice prefix_start_key_;
+ const bool prefix_same_as_start_;
+ // Means that we will pin all data blocks we read as long the Iterator
+ // is not deleted, will be true if ReadOptions::pin_data is true
+ const bool pin_thru_lifetime_;
+ const bool total_order_seek_;
+ // List of operands for merge operator.
+ MergeContext merge_context_;
+ RangeDelAggregator range_del_agg_;
+ LocalStatistics local_stats_;
+ PinnedIteratorsManager pinned_iters_mgr_;
+
+ // No copying allowed
+ DBIter(const DBIter&);
+ void operator=(const DBIter&);
+};
+
+inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
+ if (!ParseInternalKey(iter_->key(), ikey)) {
+ status_ = Status::Corruption("corrupted internal key in DBIter");
+ ROCKS_LOG_ERROR(logger_, "corrupted internal key in DBIter: %s",
+ iter_->key().ToString(true).c_str());
+ return false;
+ } else {
+ return true;
+ }
+}
+
+void DBIter::Next() {
+ assert(valid_);
+
+ // Release temporarily pinned blocks from last operation
+ ReleaseTempPinnedData();
+ ResetInternalKeysSkippedCounter();
+ if (direction_ == kReverse) {
+ ReverseToForward();
+ } else if (iter_->Valid() && !current_entry_is_merged_) {
+ // If the current value is not a merge, the iter position is the
+ // current key, which is already returned. We can safely issue a
+ // Next() without checking the current key.
+ // If the current key is a merge, very likely iter already points
+ // to the next internal position.
+ iter_->Next();
+ PERF_COUNTER_ADD(internal_key_skipped_count, 1);
+ }
+
+ if (statistics_ != nullptr) {
+ local_stats_.next_count_++;
+ }
+ // Now we point to the next internal position, for both of merge and
+ // not merge cases.
+ if (!iter_->Valid()) {
+ valid_ = false;
+ return;
+ }
+ FindNextUserEntry(true /* skipping the current user key */, prefix_same_as_start_);
+ if (statistics_ != nullptr && valid_) {
+ local_stats_.next_found_count_++;
+ local_stats_.bytes_read_ += (key().size() + value().size());
+ }
+}
+
+// PRE: saved_key_ has the current user key if skipping
+// POST: saved_key_ should have the next user key if valid_,
+// if the current entry is a result of merge
+// current_entry_is_merged_ => true
+// saved_value_ => the merged value
+//
+// NOTE: In between, saved_key_ can point to a user key that has
+// a delete marker or a sequence number higher than sequence_
+// saved_key_ MUST have a proper user_key before calling this function
+//
+// The prefix_check parameter controls whether we check the iterated
+// keys against the prefix of the seeked key. Set to false when
+// performing a seek without a key (e.g. SeekToFirst). Set to
+// prefix_same_as_start_ for other iterations.
+inline void DBIter::FindNextUserEntry(bool skipping, bool prefix_check) {
+ PERF_TIMER_GUARD(find_next_user_entry_time);
+ FindNextUserEntryInternal(skipping, prefix_check);
+}
+
+// Actual implementation of DBIter::FindNextUserEntry()
+void DBIter::FindNextUserEntryInternal(bool skipping, bool prefix_check) {
+ // Loop until we hit an acceptable entry to yield
+ assert(iter_->Valid());
+ assert(direction_ == kForward);
+ current_entry_is_merged_ = false;
+
+ // How many times in a row we have skipped an entry with user key less than
+ // or equal to saved_key_. We could skip these entries either because
+ // sequence numbers were too high or because skipping = true.
+ // What saved_key_ contains throughout this method:
+ // - if skipping : saved_key_ contains the key that we need to skip,
+ // and we haven't seen any keys greater than that,
+ // - if num_skipped > 0 : saved_key_ contains the key that we have skipped
+ // num_skipped times, and we haven't seen any keys
+ // greater than that,
+ // - none of the above : saved_key_ can contain anything, it doesn't matter.
+ uint64_t num_skipped = 0;
+
+ do {
+ ParsedInternalKey ikey;
+
+ if (!ParseKey(&ikey)) {
+ // Skip corrupted keys.
+ iter_->Next();
+ continue;
+ }
+
+ if (iterate_upper_bound_ != nullptr &&
+ user_comparator_->Compare(ikey.user_key, *iterate_upper_bound_) >= 0) {
+ break;
+ }
+
+ if (prefix_extractor_ && prefix_check &&
+ prefix_extractor_->Transform(ikey.user_key)
+ .compare(prefix_start_key_) != 0) {
+ break;
+ }
+
+ if (TooManyInternalKeysSkipped()) {
+ return;
+ }
+
+ if (ikey.sequence <= sequence_) {
+ if (skipping &&
+ user_comparator_->Compare(ikey.user_key, saved_key_.GetUserKey()) <=
+ 0) {
+ num_skipped++; // skip this entry
+ PERF_COUNTER_ADD(internal_key_skipped_count, 1);
+ } else {
+ num_skipped = 0;
+ switch (ikey.type) {
+ case kTypeDeletion:
+ case kTypeSingleDeletion:
+ // Arrange to skip all upcoming entries for this key since
+ // they are hidden by this deletion.
+ saved_key_.SetUserKey(
+ ikey.user_key,
+ !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
+ skipping = true;
+ PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
+ break;
+ case kTypeValue:
+ saved_key_.SetUserKey(
+ ikey.user_key,
+ !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
+ if (range_del_agg_.ShouldDelete(
+ ikey, RangeDelAggregator::RangePositioningMode::
+ kForwardTraversal)) {
+ // Arrange to skip all upcoming entries for this key since
+ // they are hidden by this deletion.
+ skipping = true;
+ num_skipped = 0;
+ PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
+ } else {
+ valid_ = true;
+ return;
+ }
+ break;
+ case kTypeMerge:
+ saved_key_.SetUserKey(
+ ikey.user_key,
+ !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
+ if (range_del_agg_.ShouldDelete(
+ ikey, RangeDelAggregator::RangePositioningMode::
+ kForwardTraversal)) {
+ // Arrange to skip all upcoming entries for this key since
+ // they are hidden by this deletion.
+ skipping = true;
+ num_skipped = 0;
+ PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
+ } else {
+ // By now, we are sure the current ikey is going to yield a
+ // value
+ current_entry_is_merged_ = true;
+ valid_ = true;
+ MergeValuesNewToOld(); // Go to a different state machine
+ return;
+ }
+ break;
+ default:
+ assert(false);
+ break;
+ }
+ }
+ } else {
+ // This key was inserted after our snapshot was taken.
+ PERF_COUNTER_ADD(internal_recent_skipped_count, 1);
+
+ // Here saved_key_ may contain some old key, or the default empty key, or
+ // key assigned by some random other method. We don't care.
+ if (user_comparator_->Compare(ikey.user_key, saved_key_.GetUserKey()) <=
+ 0) {
+ num_skipped++;
+ } else {
+ saved_key_.SetUserKey(
+ ikey.user_key,
+ !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
+ skipping = false;
+ num_skipped = 0;
+ }
+ }
+
+ // If we have sequentially iterated via numerous equal keys, then it's
+ // better to seek so that we can avoid too many key comparisons.
+ if (num_skipped > max_skip_) {
+ num_skipped = 0;
+ std::string last_key;
+ if (skipping) {
+ // We're looking for the next user-key but all we see are the same
+ // user-key with decreasing sequence numbers. Fast forward to
+ // sequence number 0 and type deletion (the smallest type).
+ AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetUserKey(),
+ 0, kTypeDeletion));
+ // Don't set skipping = false because we may still see more user-keys
+ // equal to saved_key_.
+ } else {
+ // We saw multiple entries with this user key and sequence numbers
+ // higher than sequence_. Fast forward to sequence_.
+ // Note that this only covers a case when a higher key was overwritten
+ // many times since our snapshot was taken, not the case when a lot of
+ // different keys were inserted after our snapshot was taken.
+ AppendInternalKey(&last_key,
+ ParsedInternalKey(saved_key_.GetUserKey(), sequence_,
+ kValueTypeForSeek));
+ }
+ iter_->Seek(last_key);
+ RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
+ } else {
+ iter_->Next();
+ }
+ } while (iter_->Valid());
+ valid_ = false;
+}
+
+// Merge values of the same user key starting from the current iter_ position
+// Scan from the newer entries to older entries.
+// PRE: iter_->key() points to the first merge type entry
+// saved_key_ stores the user key
+// POST: saved_value_ has the merged value for the user key
+// iter_ points to the next entry (or invalid)
+void DBIter::MergeValuesNewToOld() {
+ if (!merge_operator_) {
+ ROCKS_LOG_ERROR(logger_, "Options::merge_operator is null.");
+ status_ = Status::InvalidArgument("merge_operator_ must be set.");
+ valid_ = false;
+ return;
+ }
+
+ // Temporarily pin the blocks that hold merge operands
+ TempPinData();
+ merge_context_.Clear();
+ // Start the merge process by pushing the first operand
+ merge_context_.PushOperand(iter_->value(),
+ iter_->IsValuePinned() /* operand_pinned */);
+
+ ParsedInternalKey ikey;
+ Status s;
+ for (iter_->Next(); iter_->Valid(); iter_->Next()) {
+ if (!ParseKey(&ikey)) {
+ // skip corrupted key
+ continue;
+ }
+
+ if (!user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
+ // hit the next user key, stop right here
+ break;
+ } else if (kTypeDeletion == ikey.type || kTypeSingleDeletion == ikey.type ||
+ range_del_agg_.ShouldDelete(
+ ikey, RangeDelAggregator::RangePositioningMode::
+ kForwardTraversal)) {
+ // hit a delete with the same user key, stop right here
+ // iter_ is positioned after delete
+ iter_->Next();
+ break;
+ } else if (kTypeValue == ikey.type) {
+ // hit a put, merge the put value with operands and store the
+ // final result in saved_value_. We are done!
+ // ignore corruption if there is any.
+ const Slice val = iter_->value();
+ s = MergeHelper::TimedFullMerge(
+ merge_operator_, ikey.user_key, &val, merge_context_.GetOperands(),
+ &saved_value_, logger_, statistics_, env_, &pinned_value_, true);
+ if (!s.ok()) {
+ status_ = s;
+ }
+ // iter_ is positioned after put
+ iter_->Next();
+ return;
+ } else if (kTypeMerge == ikey.type) {
+ // hit a merge, add the value as an operand and run associative merge.
+ // when complete, add result to operands and continue.
+ merge_context_.PushOperand(iter_->value(),
+ iter_->IsValuePinned() /* operand_pinned */);
+ PERF_COUNTER_ADD(internal_merge_count, 1);
+ } else {
+ assert(false);
+ }
+ }
+
+ // we either exhausted all internal keys under this user key, or hit
+ // a deletion marker.
+ // feed null as the existing value to the merge operator, such that
+ // client can differentiate this scenario and do things accordingly.
+ s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
+ nullptr, merge_context_.GetOperands(),
+ &saved_value_, logger_, statistics_, env_,
+ &pinned_value_, true);
+ if (!s.ok()) {
+ status_ = s;
+ }
+}
+
+void DBIter::Prev() {
+ assert(valid_);
+ ReleaseTempPinnedData();
+ ResetInternalKeysSkippedCounter();
+ if (direction_ == kForward) {
+ ReverseToBackward();
+ }
+ PrevInternal();
+ if (statistics_ != nullptr) {
+ local_stats_.prev_count_++;
+ if (valid_) {
+ local_stats_.prev_found_count_++;
+ local_stats_.bytes_read_ += (key().size() + value().size());
+ }
+ }
+}
+
+void DBIter::ReverseToForward() {
+ if (prefix_extractor_ != nullptr && !total_order_seek_) {
+ IterKey last_key;
+ last_key.SetInternalKey(ParsedInternalKey(
+ saved_key_.GetUserKey(), kMaxSequenceNumber, kValueTypeForSeek));
+ iter_->Seek(last_key.GetInternalKey());
+ }
+ FindNextUserKey();
+ direction_ = kForward;
+ if (!iter_->Valid()) {
+ iter_->SeekToFirst();
+ range_del_agg_.InvalidateTombstoneMapPositions();
+ }
+}
+
+void DBIter::ReverseToBackward() {
+ if (prefix_extractor_ != nullptr && !total_order_seek_) {
+ IterKey last_key;
+ last_key.SetInternalKey(ParsedInternalKey(saved_key_.GetUserKey(), 0,
+ kValueTypeForSeekForPrev));
+ iter_->SeekForPrev(last_key.GetInternalKey());
+ }
+ if (current_entry_is_merged_) {
+ // Not placed in the same key. Need to call Prev() until finding the
+ // previous key.
+ if (!iter_->Valid()) {
+ iter_->SeekToLast();
+ range_del_agg_.InvalidateTombstoneMapPositions();
+ }
+ ParsedInternalKey ikey;
+ FindParseableKey(&ikey, kReverse);
+ while (iter_->Valid() &&
+ user_comparator_->Compare(ikey.user_key, saved_key_.GetUserKey()) >
+ 0) {
+ assert(ikey.sequence != kMaxSequenceNumber);
+ if (ikey.sequence > sequence_) {
+ PERF_COUNTER_ADD(internal_recent_skipped_count, 1);
+ } else {
+ PERF_COUNTER_ADD(internal_key_skipped_count, 1);
+ }
+ iter_->Prev();
+ FindParseableKey(&ikey, kReverse);
+ }
+ }
+#ifndef NDEBUG
+ if (iter_->Valid()) {
+ ParsedInternalKey ikey;
+ assert(ParseKey(&ikey));
+ assert(user_comparator_->Compare(ikey.user_key, saved_key_.GetUserKey()) <=
+ 0);
+ }
+#endif
+
+ FindPrevUserKey();
+ direction_ = kReverse;
+}
+
+void DBIter::PrevInternal() {
+ if (!iter_->Valid()) {
+ valid_ = false;
+ return;
+ }
+
+ ParsedInternalKey ikey;
+
+ while (iter_->Valid()) {
+ saved_key_.SetUserKey(
+ ExtractUserKey(iter_->key()),
+ !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
+
+ if (FindValueForCurrentKey()) {
+ valid_ = true;
+ if (!iter_->Valid()) {
+ return;
+ }
+ FindParseableKey(&ikey, kReverse);
+ if (user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
+ FindPrevUserKey();
+ }
+ if (valid_ && prefix_extractor_ && prefix_same_as_start_ &&
+ prefix_extractor_->Transform(saved_key_.GetUserKey())
+ .compare(prefix_start_key_) != 0) {
+ valid_ = false;
+ }
+ return;
+ }
+
+ if (TooManyInternalKeysSkipped(false)) {
+ return;
+ }
+
+ if (!iter_->Valid()) {
+ break;
+ }
+ FindParseableKey(&ikey, kReverse);
+ if (user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
+ FindPrevUserKey();
+ }
+ }
+ // We haven't found any key - iterator is not valid
+ // Or the prefix is different than start prefix
+ assert(!iter_->Valid());
+ valid_ = false;
+}
+
+// This function checks, if the entry with biggest sequence_number <= sequence_
+// is non kTypeDeletion or kTypeSingleDeletion. If it's not, we save value in
+// saved_value_
+bool DBIter::FindValueForCurrentKey() {
+ assert(iter_->Valid());
+ merge_context_.Clear();
+ current_entry_is_merged_ = false;
+ // last entry before merge (could be kTypeDeletion, kTypeSingleDeletion or
+ // kTypeValue)
+ ValueType last_not_merge_type = kTypeDeletion;
+ ValueType last_key_entry_type = kTypeDeletion;
+
+ ParsedInternalKey ikey;
+ FindParseableKey(&ikey, kReverse);
+
+ // Temporarily pin blocks that hold (merge operands / the value)
+ ReleaseTempPinnedData();
+ TempPinData();
+ size_t num_skipped = 0;
+ while (iter_->Valid() && ikey.sequence <= sequence_ &&
+ user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
+ if (TooManyInternalKeysSkipped()) {
+ return false;
+ }
+
+ // We iterate too much: let's use Seek() to avoid too much key comparisons
+ if (num_skipped >= max_skip_) {
+ return FindValueForCurrentKeyUsingSeek();
+ }
+
+ last_key_entry_type = ikey.type;
+ switch (last_key_entry_type) {
+ case kTypeValue:
+ if (range_del_agg_.ShouldDelete(
+ ikey,
+ RangeDelAggregator::RangePositioningMode::kBackwardTraversal)) {
+ last_key_entry_type = kTypeRangeDeletion;
+ PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
+ } else {
+ assert(iter_->IsValuePinned());
+ pinned_value_ = iter_->value();
+ }
+ merge_context_.Clear();
+ last_not_merge_type = last_key_entry_type;
+ break;
+ case kTypeDeletion:
+ case kTypeSingleDeletion:
+ merge_context_.Clear();
+ last_not_merge_type = last_key_entry_type;
+ PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
+ break;
+ case kTypeMerge:
+ if (range_del_agg_.ShouldDelete(
+ ikey,
+ RangeDelAggregator::RangePositioningMode::kBackwardTraversal)) {
+ merge_context_.Clear();
+ last_key_entry_type = kTypeRangeDeletion;
+ last_not_merge_type = last_key_entry_type;
+ PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
+ } else {
+ assert(merge_operator_ != nullptr);
+ merge_context_.PushOperandBack(
+ iter_->value(), iter_->IsValuePinned() /* operand_pinned */);
+ PERF_COUNTER_ADD(internal_merge_count, 1);
+ }
+ break;
+ default:
+ assert(false);
+ }
+
+ PERF_COUNTER_ADD(internal_key_skipped_count, 1);
+ assert(user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey()));
+ iter_->Prev();
+ ++num_skipped;
+ FindParseableKey(&ikey, kReverse);
+ }
+
+ Status s;
+ switch (last_key_entry_type) {
+ case kTypeDeletion:
+ case kTypeSingleDeletion:
+ case kTypeRangeDeletion:
+ valid_ = false;
+ return false;
+ case kTypeMerge:
+ current_entry_is_merged_ = true;
+ if (last_not_merge_type == kTypeDeletion ||
+ last_not_merge_type == kTypeSingleDeletion ||
+ last_not_merge_type == kTypeRangeDeletion) {
+ s = MergeHelper::TimedFullMerge(
+ merge_operator_, saved_key_.GetUserKey(), nullptr,
+ merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
+ env_, &pinned_value_, true);
+ } else {
+ assert(last_not_merge_type == kTypeValue);
+ s = MergeHelper::TimedFullMerge(
+ merge_operator_, saved_key_.GetUserKey(), &pinned_value_,
+ merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
+ env_, &pinned_value_, true);
+ }
+ break;
+ case kTypeValue:
+ // do nothing - we've already has value in saved_value_
+ break;
+ default:
+ assert(false);
+ break;
+ }
+ valid_ = true;
+ if (!s.ok()) {
+ status_ = s;
+ }
+ return true;
+}
+
+// This function is used in FindValueForCurrentKey.
+// We use Seek() function instead of Prev() to find necessary value
+bool DBIter::FindValueForCurrentKeyUsingSeek() {
+ // FindValueForCurrentKey will enable pinning before calling
+ // FindValueForCurrentKeyUsingSeek()
+ assert(pinned_iters_mgr_.PinningEnabled());
+ std::string last_key;
+ AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetUserKey(),
+ sequence_, kValueTypeForSeek));
+ iter_->Seek(last_key);
+ RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
+
+ // assume there is at least one parseable key for this user key
+ ParsedInternalKey ikey;
+ FindParseableKey(&ikey, kForward);
+
+ if (ikey.type == kTypeDeletion || ikey.type == kTypeSingleDeletion ||
+ range_del_agg_.ShouldDelete(
+ ikey, RangeDelAggregator::RangePositioningMode::kBackwardTraversal)) {
+ valid_ = false;
+ return false;
+ }
+ if (ikey.type == kTypeValue) {
+ assert(iter_->IsValuePinned());
+ pinned_value_ = iter_->value();
+ valid_ = true;
+ return true;
+ }
+
+ // kTypeMerge. We need to collect all kTypeMerge values and save them
+ // in operands
+ current_entry_is_merged_ = true;
+ merge_context_.Clear();
+ while (
+ iter_->Valid() &&
+ user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey()) &&
+ ikey.type == kTypeMerge &&
+ !range_del_agg_.ShouldDelete(
+ ikey, RangeDelAggregator::RangePositioningMode::kBackwardTraversal)) {
+ merge_context_.PushOperand(iter_->value(),
+ iter_->IsValuePinned() /* operand_pinned */);
+ PERF_COUNTER_ADD(internal_merge_count, 1);
+ iter_->Next();
+ FindParseableKey(&ikey, kForward);
+ }
+
+ Status s;
+ if (!iter_->Valid() ||
+ !user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey()) ||
+ ikey.type == kTypeDeletion || ikey.type == kTypeSingleDeletion ||
+ range_del_agg_.ShouldDelete(
+ ikey, RangeDelAggregator::RangePositioningMode::kBackwardTraversal)) {
+ s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
+ nullptr, merge_context_.GetOperands(),
+ &saved_value_, logger_, statistics_, env_,
+ &pinned_value_, true);
+ // Make iter_ valid and point to saved_key_
+ if (!iter_->Valid() ||
+ !user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
+ iter_->Seek(last_key);
+ RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
+ }
+ valid_ = true;
+ if (!s.ok()) {
+ status_ = s;
+ }
+ return true;
+ }
+
+ const Slice& val = iter_->value();
+ s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
+ &val, merge_context_.GetOperands(),
+ &saved_value_, logger_, statistics_, env_,
+ &pinned_value_, true);
+ valid_ = true;
+ if (!s.ok()) {
+ status_ = s;
+ }
+ return true;
+}
+
+// Used in Next to change directions
+// Go to next user key
+// Don't use Seek(),
+// because next user key will be very close
+void DBIter::FindNextUserKey() {
+ if (!iter_->Valid()) {
+ return;
+ }
+ ParsedInternalKey ikey;
+ FindParseableKey(&ikey, kForward);
+ while (iter_->Valid() &&
+ !user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
+ iter_->Next();
+ FindParseableKey(&ikey, kForward);
+ }
+}
+
+// Go to previous user_key
+void DBIter::FindPrevUserKey() {
+ if (!iter_->Valid()) {
+ return;
+ }
+ size_t num_skipped = 0;
+ ParsedInternalKey ikey;
+ FindParseableKey(&ikey, kReverse);
+ int cmp;
+ while (iter_->Valid() &&
+ ((cmp = user_comparator_->Compare(ikey.user_key,
+ saved_key_.GetUserKey())) == 0 ||
+ (cmp > 0 && ikey.sequence > sequence_))) {
+ if (TooManyInternalKeysSkipped()) {
+ return;
+ }
+
+ if (cmp == 0) {
+ if (num_skipped >= max_skip_) {
+ num_skipped = 0;
+ IterKey last_key;
+ last_key.SetInternalKey(ParsedInternalKey(
+ saved_key_.GetUserKey(), kMaxSequenceNumber, kValueTypeForSeek));
+ iter_->Seek(last_key.GetInternalKey());
+ RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
+ } else {
+ ++num_skipped;
+ }
+ }
+ assert(ikey.sequence != kMaxSequenceNumber);
+ if (ikey.sequence > sequence_) {
+ PERF_COUNTER_ADD(internal_recent_skipped_count, 1);
+ } else {
+ PERF_COUNTER_ADD(internal_key_skipped_count, 1);
+ }
+ iter_->Prev();
+ FindParseableKey(&ikey, kReverse);
+ }
+}
+
+bool DBIter::TooManyInternalKeysSkipped(bool increment) {
+ if ((max_skippable_internal_keys_ > 0) &&
+ (num_internal_keys_skipped_ > max_skippable_internal_keys_)) {
+ valid_ = false;
+ status_ = Status::Incomplete("Too many internal keys skipped.");
+ return true;
+ } else if (increment) {
+ num_internal_keys_skipped_++;
+ }
+ return false;
+}
+
+// Skip all unparseable keys
+void DBIter::FindParseableKey(ParsedInternalKey* ikey, Direction direction) {
+ while (iter_->Valid() && !ParseKey(ikey)) {
+ if (direction == kReverse) {
+ iter_->Prev();
+ } else {
+ iter_->Next();
+ }
+ }
+}
+
+void DBIter::Seek(const Slice& target) {
+ StopWatch sw(env_, statistics_, DB_SEEK);
+ ReleaseTempPinnedData();
+ ResetInternalKeysSkippedCounter();
+ saved_key_.Clear();
+ saved_key_.SetInternalKey(target, sequence_);
+
+ {
+ PERF_TIMER_GUARD(seek_internal_seek_time);
+ iter_->Seek(saved_key_.GetInternalKey());
+ range_del_agg_.InvalidateTombstoneMapPositions();
+ }
+ RecordTick(statistics_, NUMBER_DB_SEEK);
+ if (iter_->Valid()) {
+ if (prefix_extractor_ && prefix_same_as_start_) {
+ prefix_start_key_ = prefix_extractor_->Transform(target);
+ }
+ direction_ = kForward;
+ ClearSavedValue();
+ FindNextUserEntry(false /* not skipping */, prefix_same_as_start_);
+ if (!valid_) {
+ prefix_start_key_.clear();
+ }
+ if (statistics_ != nullptr) {
+ if (valid_) {
+ RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
+ RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
+ }
+ }
+ } else {
+ valid_ = false;
+ }
+
+ if (valid_ && prefix_extractor_ && prefix_same_as_start_) {
+ prefix_start_buf_.SetUserKey(prefix_start_key_);
+ prefix_start_key_ = prefix_start_buf_.GetUserKey();
+ }
+}
+
+void DBIter::SeekForPrev(const Slice& target) {
+ StopWatch sw(env_, statistics_, DB_SEEK);
+ ReleaseTempPinnedData();
+ ResetInternalKeysSkippedCounter();
+ saved_key_.Clear();
+ // now saved_key is used to store internal key.
+ saved_key_.SetInternalKey(target, 0 /* sequence_number */,
+ kValueTypeForSeekForPrev);
+
+ {
+ PERF_TIMER_GUARD(seek_internal_seek_time);
+ iter_->SeekForPrev(saved_key_.GetInternalKey());
+ range_del_agg_.InvalidateTombstoneMapPositions();
+ }
+
+ RecordTick(statistics_, NUMBER_DB_SEEK);
+ if (iter_->Valid()) {
+ if (prefix_extractor_ && prefix_same_as_start_) {
+ prefix_start_key_ = prefix_extractor_->Transform(target);
+ }
+ direction_ = kReverse;
+ ClearSavedValue();
+ PrevInternal();
+ if (!valid_) {
+ prefix_start_key_.clear();
+ }
+ if (statistics_ != nullptr) {
+ if (valid_) {
+ RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
+ RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
+ }
+ }
+ } else {
+ valid_ = false;
+ }
+ if (valid_ && prefix_extractor_ && prefix_same_as_start_) {
+ prefix_start_buf_.SetUserKey(prefix_start_key_);
+ prefix_start_key_ = prefix_start_buf_.GetUserKey();
+ }
+}
+
+void DBIter::SeekToFirst() {
+ // Don't use iter_::Seek() if we set a prefix extractor
+ // because prefix seek will be used.
+ if (prefix_extractor_ != nullptr) {
+ max_skip_ = std::numeric_limits<uint64_t>::max();
+ }
+ direction_ = kForward;
+ ReleaseTempPinnedData();
+ ResetInternalKeysSkippedCounter();
+ ClearSavedValue();
+
+ {
+ PERF_TIMER_GUARD(seek_internal_seek_time);
+ iter_->SeekToFirst();
+ range_del_agg_.InvalidateTombstoneMapPositions();
+ }
+
+ RecordTick(statistics_, NUMBER_DB_SEEK);
+ if (iter_->Valid()) {
+ saved_key_.SetUserKey(
+ ExtractUserKey(iter_->key()),
+ !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
+ FindNextUserEntry(false /* not skipping */, false /* no prefix check */);
+ if (statistics_ != nullptr) {
+ if (valid_) {
+ RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
+ RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
+ }
+ }
+ } else {
+ valid_ = false;
+ }
+ if (valid_ && prefix_extractor_ && prefix_same_as_start_) {
+ prefix_start_buf_.SetUserKey(
+ prefix_extractor_->Transform(saved_key_.GetUserKey()));
+ prefix_start_key_ = prefix_start_buf_.GetUserKey();
+ }
+}
+
+void DBIter::SeekToLast() {
+ // Don't use iter_::Seek() if we set a prefix extractor
+ // because prefix seek will be used.
+ if (prefix_extractor_ != nullptr) {
+ max_skip_ = std::numeric_limits<uint64_t>::max();
+ }
+ direction_ = kReverse;
+ ReleaseTempPinnedData();
+ ResetInternalKeysSkippedCounter();
+ ClearSavedValue();
+
+ {
+ PERF_TIMER_GUARD(seek_internal_seek_time);
+ iter_->SeekToLast();
+ range_del_agg_.InvalidateTombstoneMapPositions();
+ }
+ // When the iterate_upper_bound is set to a value,
+ // it will seek to the last key before the
+ // ReadOptions.iterate_upper_bound
+ if (iter_->Valid() && iterate_upper_bound_ != nullptr) {
+ SeekForPrev(*iterate_upper_bound_);
+ range_del_agg_.InvalidateTombstoneMapPositions();
+ if (!Valid()) {
+ return;
+ } else if (user_comparator_->Equal(*iterate_upper_bound_, key())) {
+ Prev();
+ }
+ } else {
+ PrevInternal();
+ }
+ if (statistics_ != nullptr) {
+ RecordTick(statistics_, NUMBER_DB_SEEK);
+ if (valid_) {
+ RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
+ RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
+ }
+ }
+ if (valid_ && prefix_extractor_ && prefix_same_as_start_) {
+ prefix_start_buf_.SetUserKey(
+ prefix_extractor_->Transform(saved_key_.GetUserKey()));
+ prefix_start_key_ = prefix_start_buf_.GetUserKey();
+ }
+}
+
+Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
+ const ImmutableCFOptions& cf_options,
+ const Comparator* user_key_comparator,
+ InternalIterator* internal_iter,
+ const SequenceNumber& sequence,
+ uint64_t max_sequential_skip_in_iterations) {
+ DBIter* db_iter = new DBIter(env, read_options, cf_options,
+ user_key_comparator, internal_iter, sequence,
+ false, max_sequential_skip_in_iterations);
+ return db_iter;
+}
+
+ArenaWrappedDBIter::~ArenaWrappedDBIter() { db_iter_->~DBIter(); }
+
+RangeDelAggregator* ArenaWrappedDBIter::GetRangeDelAggregator() {
+ return db_iter_->GetRangeDelAggregator();
+}
+
+void ArenaWrappedDBIter::SetIterUnderDBIter(InternalIterator* iter) {
+ static_cast<DBIter*>(db_iter_)->SetIter(iter);
+}
+
+inline bool ArenaWrappedDBIter::Valid() const { return db_iter_->Valid(); }
+inline void ArenaWrappedDBIter::SeekToFirst() { db_iter_->SeekToFirst(); }
+inline void ArenaWrappedDBIter::SeekToLast() { db_iter_->SeekToLast(); }
+inline void ArenaWrappedDBIter::Seek(const Slice& target) {
+ db_iter_->Seek(target);
+}
+inline void ArenaWrappedDBIter::SeekForPrev(const Slice& target) {
+ db_iter_->SeekForPrev(target);
+}
+inline void ArenaWrappedDBIter::Next() { db_iter_->Next(); }
+inline void ArenaWrappedDBIter::Prev() { db_iter_->Prev(); }
+inline Slice ArenaWrappedDBIter::key() const { return db_iter_->key(); }
+inline Slice ArenaWrappedDBIter::value() const { return db_iter_->value(); }
+inline Status ArenaWrappedDBIter::status() const { return db_iter_->status(); }
+inline Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
+ std::string* prop) {
+ if (prop_name == "rocksdb.iterator.super-version-number") {
+ // First try to pass the value returned from inner iterator.
+ if (!db_iter_->GetProperty(prop_name, prop).ok()) {
+ *prop = ToString(sv_number_);
+ }
+ return Status::OK();
+ }
+ return db_iter_->GetProperty(prop_name, prop);
+}
+
+void ArenaWrappedDBIter::Init(Env* env, const ReadOptions& read_options,
+ const ImmutableCFOptions& cf_options,
+ const SequenceNumber& sequence,
+ uint64_t max_sequential_skip_in_iteration,
+ uint64_t version_number) {
+ auto mem = arena_.AllocateAligned(sizeof(DBIter));
+ db_iter_ = new (mem)
+ DBIter(env, read_options, cf_options, cf_options.user_comparator, nullptr,
+ sequence, true, max_sequential_skip_in_iteration);
+ sv_number_ = version_number;
+}
+
+Status ArenaWrappedDBIter::Refresh() {
+ if (cfd_ == nullptr || db_impl_ == nullptr) {
+ return Status::NotSupported("Creating renew iterator is not allowed.");
+ }
+ assert(db_iter_ != nullptr);
+ SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
+ uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
+ if (sv_number_ != cur_sv_number) {
+ Env* env = db_iter_->env();
+ db_iter_->~DBIter();
+ arena_.~Arena();
+ new (&arena_) Arena();
+
+ SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_->mutex());
+ Init(env, read_options_, *(cfd_->ioptions()), latest_seq,
+ sv->mutable_cf_options.max_sequential_skip_in_iterations,
+ cur_sv_number);
+
+ InternalIterator* internal_iter = db_impl_->NewInternalIterator(
+ read_options_, cfd_, sv, &arena_, db_iter_->GetRangeDelAggregator());
+ SetIterUnderDBIter(internal_iter);
+ } else {
+ db_iter_->set_sequence(latest_seq);
+ db_iter_->set_valid(false);
+ }
+ return Status::OK();
+}
+
+ArenaWrappedDBIter* NewArenaWrappedDbIterator(
+ Env* env, const ReadOptions& read_options,
+ const ImmutableCFOptions& cf_options, const SequenceNumber& sequence,
+ uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+ DBImpl* db_impl, ColumnFamilyData* cfd) {
+ ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
+ iter->Init(env, read_options, cf_options, sequence,
+ max_sequential_skip_in_iterations, version_number);
+ if (db_impl != nullptr && cfd != nullptr) {
+ iter->StoreRefreshInfo(read_options, db_impl, cfd);
+ }
+
+ return iter;
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/db_iter.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/db_iter.h b/thirdparty/rocksdb/db/db_iter.h
new file mode 100644
index 0000000..ea98ff4
--- /dev/null
+++ b/thirdparty/rocksdb/db/db_iter.h
@@ -0,0 +1,100 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <stdint.h>
+#include <string>
+#include "db/db_impl.h"
+#include "db/dbformat.h"
+#include "db/range_del_aggregator.h"
+#include "options/cf_options.h"
+#include "rocksdb/db.h"
+#include "rocksdb/iterator.h"
+#include "util/arena.h"
+#include "util/autovector.h"
+
+namespace rocksdb {
+
+class Arena;
+class DBIter;
+class InternalIterator;
+
+// Return a new iterator that converts internal keys (yielded by
+// "*internal_iter") that were live at the specified "sequence" number
+// into appropriate user keys.
+extern Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
+ const ImmutableCFOptions& cf_options,
+ const Comparator* user_key_comparator,
+ InternalIterator* internal_iter,
+ const SequenceNumber& sequence,
+ uint64_t max_sequential_skip_in_iterations);
+
+// A wrapper iterator which wraps DB Iterator and the arena, with which the DB
+// iterator is supposed be allocated. This class is used as an entry point of
+// a iterator hierarchy whose memory can be allocated inline. In that way,
+// accessing the iterator tree can be more cache friendly. It is also faster
+// to allocate.
+class ArenaWrappedDBIter : public Iterator {
+ public:
+ virtual ~ArenaWrappedDBIter();
+
+ // Get the arena to be used to allocate memory for DBIter to be wrapped,
+ // as well as child iterators in it.
+ virtual Arena* GetArena() { return &arena_; }
+ virtual RangeDelAggregator* GetRangeDelAggregator();
+
+ // Set the internal iterator wrapped inside the DB Iterator. Usually it is
+ // a merging iterator.
+ virtual void SetIterUnderDBIter(InternalIterator* iter);
+ virtual bool Valid() const override;
+ virtual void SeekToFirst() override;
+ virtual void SeekToLast() override;
+ virtual void Seek(const Slice& target) override;
+ virtual void SeekForPrev(const Slice& target) override;
+ virtual void Next() override;
+ virtual void Prev() override;
+ virtual Slice key() const override;
+ virtual Slice value() const override;
+ virtual Status status() const override;
+ virtual Status Refresh() override;
+
+ virtual Status GetProperty(std::string prop_name, std::string* prop) override;
+
+ void Init(Env* env, const ReadOptions& read_options,
+ const ImmutableCFOptions& cf_options,
+ const SequenceNumber& sequence,
+ uint64_t max_sequential_skip_in_iterations,
+ uint64_t version_number);
+
+ void StoreRefreshInfo(const ReadOptions& read_options, DBImpl* db_impl,
+ ColumnFamilyData* cfd) {
+ read_options_ = read_options;
+ db_impl_ = db_impl;
+ cfd_ = cfd;
+ }
+
+ private:
+ DBIter* db_iter_;
+ Arena arena_;
+ uint64_t sv_number_;
+ ColumnFamilyData* cfd_ = nullptr;
+ DBImpl* db_impl_ = nullptr;
+ ReadOptions read_options_;
+};
+
+// Generate the arena wrapped iterator class.
+// `db_impl` and `cfd` are used for reneweal. If left null, renewal will not
+// be supported.
+extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
+ Env* env, const ReadOptions& read_options,
+ const ImmutableCFOptions& cf_options, const SequenceNumber& sequence,
+ uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+ DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr);
+
+} // namespace rocksdb
[45/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/dbformat.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/dbformat.h b/thirdparty/leveldb-1.18/db/dbformat.h
deleted file mode 100755
index ea897b1..0000000
--- a/thirdparty/leveldb-1.18/db/dbformat.h
+++ /dev/null
@@ -1,230 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_
-#define STORAGE_LEVELDB_DB_DBFORMAT_H_
-
-#include <stdio.h>
-#include "leveldb/comparator.h"
-#include "leveldb/db.h"
-#include "leveldb/filter_policy.h"
-#include "leveldb/slice.h"
-#include "leveldb/table_builder.h"
-#include "util/coding.h"
-#include "util/logging.h"
-
-namespace leveldb {
-
-// Grouping of constants. We may want to make some of these
-// parameters set via options.
-namespace config {
-static const int kNumLevels = 7;
-
-// Level-0 compaction is started when we hit this many files.
-static const int kL0_CompactionTrigger = 4;
-
-// Soft limit on number of level-0 files. We slow down writes at this point.
-static const int kL0_SlowdownWritesTrigger = 8;
-
-// Maximum number of level-0 files. We stop writes at this point.
-static const int kL0_StopWritesTrigger = 12;
-
-// Maximum level to which a new compacted memtable is pushed if it
-// does not create overlap. We try to push to level 2 to avoid the
-// relatively expensive level 0=>1 compactions and to avoid some
-// expensive manifest file operations. We do not push all the way to
-// the largest level since that can generate a lot of wasted disk
-// space if the same key space is being repeatedly overwritten.
-static const int kMaxMemCompactLevel = 2;
-
-// Approximate gap in bytes between samples of data read during iteration.
-static const int kReadBytesPeriod = 1048576;
-
-} // namespace config
-
-class InternalKey;
-
-// Value types encoded as the last component of internal keys.
-// DO NOT CHANGE THESE ENUM VALUES: they are embedded in the on-disk
-// data structures.
-enum ValueType {
- kTypeDeletion = 0x0,
- kTypeValue = 0x1
-};
-// kValueTypeForSeek defines the ValueType that should be passed when
-// constructing a ParsedInternalKey object for seeking to a particular
-// sequence number (since we sort sequence numbers in decreasing order
-// and the value type is embedded as the low 8 bits in the sequence
-// number in internal keys, we need to use the highest-numbered
-// ValueType, not the lowest).
-static const ValueType kValueTypeForSeek = kTypeValue;
-
-typedef uint64_t SequenceNumber;
-
-// We leave eight bits empty at the bottom so a type and sequence#
-// can be packed together into 64-bits.
-static const SequenceNumber kMaxSequenceNumber =
- ((0x1ull << 56) - 1);
-
-struct ParsedInternalKey {
- Slice user_key;
- SequenceNumber sequence;
- ValueType type;
-
- ParsedInternalKey() { } // Intentionally left uninitialized (for speed)
- ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
- : user_key(u), sequence(seq), type(t) { }
- std::string DebugString() const;
-};
-
-// Return the length of the encoding of "key".
-inline size_t InternalKeyEncodingLength(const ParsedInternalKey& key) {
- return key.user_key.size() + 8;
-}
-
-// Append the serialization of "key" to *result.
-extern void AppendInternalKey(std::string* result,
- const ParsedInternalKey& key);
-
-// Attempt to parse an internal key from "internal_key". On success,
-// stores the parsed data in "*result", and returns true.
-//
-// On error, returns false, leaves "*result" in an undefined state.
-extern bool ParseInternalKey(const Slice& internal_key,
- ParsedInternalKey* result);
-
-// Returns the user key portion of an internal key.
-inline Slice ExtractUserKey(const Slice& internal_key) {
- assert(internal_key.size() >= 8);
- return Slice(internal_key.data(), internal_key.size() - 8);
-}
-
-inline ValueType ExtractValueType(const Slice& internal_key) {
- assert(internal_key.size() >= 8);
- const size_t n = internal_key.size();
- uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
- unsigned char c = num & 0xff;
- return static_cast<ValueType>(c);
-}
-
-// A comparator for internal keys that uses a specified comparator for
-// the user key portion and breaks ties by decreasing sequence number.
-class InternalKeyComparator : public Comparator {
- private:
- const Comparator* user_comparator_;
- public:
- explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) { }
- virtual const char* Name() const;
- virtual int Compare(const Slice& a, const Slice& b) const;
- virtual void FindShortestSeparator(
- std::string* start,
- const Slice& limit) const;
- virtual void FindShortSuccessor(std::string* key) const;
-
- const Comparator* user_comparator() const { return user_comparator_; }
-
- int Compare(const InternalKey& a, const InternalKey& b) const;
-};
-
-// Filter policy wrapper that converts from internal keys to user keys
-class InternalFilterPolicy : public FilterPolicy {
- private:
- const FilterPolicy* const user_policy_;
- public:
- explicit InternalFilterPolicy(const FilterPolicy* p) : user_policy_(p) { }
- virtual const char* Name() const;
- virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const;
- virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const;
-};
-
-// Modules in this directory should keep internal keys wrapped inside
-// the following class instead of plain strings so that we do not
-// incorrectly use string comparisons instead of an InternalKeyComparator.
-class InternalKey {
- private:
- std::string rep_;
- public:
- InternalKey() { } // Leave rep_ as empty to indicate it is invalid
- InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) {
- AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t));
- }
-
- void DecodeFrom(const Slice& s) { rep_.assign(s.data(), s.size()); }
- Slice Encode() const {
- assert(!rep_.empty());
- return rep_;
- }
-
- Slice user_key() const { return ExtractUserKey(rep_); }
-
- void SetFrom(const ParsedInternalKey& p) {
- rep_.clear();
- AppendInternalKey(&rep_, p);
- }
-
- void Clear() { rep_.clear(); }
-
- std::string DebugString() const;
-};
-
-inline int InternalKeyComparator::Compare(
- const InternalKey& a, const InternalKey& b) const {
- return Compare(a.Encode(), b.Encode());
-}
-
-inline bool ParseInternalKey(const Slice& internal_key,
- ParsedInternalKey* result) {
- const size_t n = internal_key.size();
- if (n < 8) return false;
- uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
- unsigned char c = num & 0xff;
- result->sequence = num >> 8;
- result->type = static_cast<ValueType>(c);
- result->user_key = Slice(internal_key.data(), n - 8);
- return (c <= static_cast<unsigned char>(kTypeValue));
-}
-
-// A helper class useful for DBImpl::Get()
-class LookupKey {
- public:
- // Initialize *this for looking up user_key at a snapshot with
- // the specified sequence number.
- LookupKey(const Slice& user_key, SequenceNumber sequence);
-
- ~LookupKey();
-
- // Return a key suitable for lookup in a MemTable.
- Slice memtable_key() const { return Slice(start_, end_ - start_); }
-
- // Return an internal key (suitable for passing to an internal iterator)
- Slice internal_key() const { return Slice(kstart_, end_ - kstart_); }
-
- // Return the user key
- Slice user_key() const { return Slice(kstart_, end_ - kstart_ - 8); }
-
- private:
- // We construct a char array of the form:
- // klength varint32 <-- start_
- // userkey char[klength] <-- kstart_
- // tag uint64
- // <-- end_
- // The array is a suitable MemTable key.
- // The suffix starting with "userkey" can be used as an InternalKey.
- const char* start_;
- const char* kstart_;
- const char* end_;
- char space_[200]; // Avoid allocation for short keys
-
- // No copying allowed
- LookupKey(const LookupKey&);
- void operator=(const LookupKey&);
-};
-
-inline LookupKey::~LookupKey() {
- if (start_ != space_) delete[] start_;
-}
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_DBFORMAT_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/dbformat_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/dbformat_test.cc b/thirdparty/leveldb-1.18/db/dbformat_test.cc
deleted file mode 100755
index 5d82f5d..0000000
--- a/thirdparty/leveldb-1.18/db/dbformat_test.cc
+++ /dev/null
@@ -1,112 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/dbformat.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-static std::string IKey(const std::string& user_key,
- uint64_t seq,
- ValueType vt) {
- std::string encoded;
- AppendInternalKey(&encoded, ParsedInternalKey(user_key, seq, vt));
- return encoded;
-}
-
-static std::string Shorten(const std::string& s, const std::string& l) {
- std::string result = s;
- InternalKeyComparator(BytewiseComparator()).FindShortestSeparator(&result, l);
- return result;
-}
-
-static std::string ShortSuccessor(const std::string& s) {
- std::string result = s;
- InternalKeyComparator(BytewiseComparator()).FindShortSuccessor(&result);
- return result;
-}
-
-static void TestKey(const std::string& key,
- uint64_t seq,
- ValueType vt) {
- std::string encoded = IKey(key, seq, vt);
-
- Slice in(encoded);
- ParsedInternalKey decoded("", 0, kTypeValue);
-
- ASSERT_TRUE(ParseInternalKey(in, &decoded));
- ASSERT_EQ(key, decoded.user_key.ToString());
- ASSERT_EQ(seq, decoded.sequence);
- ASSERT_EQ(vt, decoded.type);
-
- ASSERT_TRUE(!ParseInternalKey(Slice("bar"), &decoded));
-}
-
-class FormatTest { };
-
-TEST(FormatTest, InternalKey_EncodeDecode) {
- const char* keys[] = { "", "k", "hello", "longggggggggggggggggggggg" };
- const uint64_t seq[] = {
- 1, 2, 3,
- (1ull << 8) - 1, 1ull << 8, (1ull << 8) + 1,
- (1ull << 16) - 1, 1ull << 16, (1ull << 16) + 1,
- (1ull << 32) - 1, 1ull << 32, (1ull << 32) + 1
- };
- for (int k = 0; k < sizeof(keys) / sizeof(keys[0]); k++) {
- for (int s = 0; s < sizeof(seq) / sizeof(seq[0]); s++) {
- TestKey(keys[k], seq[s], kTypeValue);
- TestKey("hello", 1, kTypeDeletion);
- }
- }
-}
-
-TEST(FormatTest, InternalKeyShortSeparator) {
- // When user keys are same
- ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("foo", 99, kTypeValue)));
- ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("foo", 101, kTypeValue)));
- ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("foo", 100, kTypeValue)));
- ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("foo", 100, kTypeDeletion)));
-
- // When user keys are misordered
- ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("bar", 99, kTypeValue)));
-
- // When user keys are different, but correctly ordered
- ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("hello", 200, kTypeValue)));
-
- // When start user key is prefix of limit user key
- ASSERT_EQ(IKey("foo", 100, kTypeValue),
- Shorten(IKey("foo", 100, kTypeValue),
- IKey("foobar", 200, kTypeValue)));
-
- // When limit user key is prefix of start user key
- ASSERT_EQ(IKey("foobar", 100, kTypeValue),
- Shorten(IKey("foobar", 100, kTypeValue),
- IKey("foo", 200, kTypeValue)));
-}
-
-TEST(FormatTest, InternalKeyShortestSuccessor) {
- ASSERT_EQ(IKey("g", kMaxSequenceNumber, kValueTypeForSeek),
- ShortSuccessor(IKey("foo", 100, kTypeValue)));
- ASSERT_EQ(IKey("\xff\xff", 100, kTypeValue),
- ShortSuccessor(IKey("\xff\xff", 100, kTypeValue)));
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/dumpfile.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/dumpfile.cc b/thirdparty/leveldb-1.18/db/dumpfile.cc
deleted file mode 100755
index 61c47c2..0000000
--- a/thirdparty/leveldb-1.18/db/dumpfile.cc
+++ /dev/null
@@ -1,225 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <stdio.h>
-#include "db/dbformat.h"
-#include "db/filename.h"
-#include "db/log_reader.h"
-#include "db/version_edit.h"
-#include "db/write_batch_internal.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
-#include "leveldb/options.h"
-#include "leveldb/status.h"
-#include "leveldb/table.h"
-#include "leveldb/write_batch.h"
-#include "util/logging.h"
-
-namespace leveldb {
-
-namespace {
-
-bool GuessType(const std::string& fname, FileType* type) {
- size_t pos = fname.rfind('/');
- std::string basename;
- if (pos == std::string::npos) {
- basename = fname;
- } else {
- basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1);
- }
- uint64_t ignored;
- return ParseFileName(basename, &ignored, type);
-}
-
-// Notified when log reader encounters corruption.
-class CorruptionReporter : public log::Reader::Reporter {
- public:
- WritableFile* dst_;
- virtual void Corruption(size_t bytes, const Status& status) {
- std::string r = "corruption: ";
- AppendNumberTo(&r, bytes);
- r += " bytes; ";
- r += status.ToString();
- r.push_back('\n');
- dst_->Append(r);
- }
-};
-
-// Print contents of a log file. (*func)() is called on every record.
-Status PrintLogContents(Env* env, const std::string& fname,
- void (*func)(uint64_t, Slice, WritableFile*),
- WritableFile* dst) {
- SequentialFile* file;
- Status s = env->NewSequentialFile(fname, &file);
- if (!s.ok()) {
- return s;
- }
- CorruptionReporter reporter;
- reporter.dst_ = dst;
- log::Reader reader(file, &reporter, true, 0);
- Slice record;
- std::string scratch;
- while (reader.ReadRecord(&record, &scratch)) {
- (*func)(reader.LastRecordOffset(), record, dst);
- }
- delete file;
- return Status::OK();
-}
-
-// Called on every item found in a WriteBatch.
-class WriteBatchItemPrinter : public WriteBatch::Handler {
- public:
- WritableFile* dst_;
- virtual void Put(const Slice& key, const Slice& value) {
- std::string r = " put '";
- AppendEscapedStringTo(&r, key);
- r += "' '";
- AppendEscapedStringTo(&r, value);
- r += "'\n";
- dst_->Append(r);
- }
- virtual void Delete(const Slice& key) {
- std::string r = " del '";
- AppendEscapedStringTo(&r, key);
- r += "'\n";
- dst_->Append(r);
- }
-};
-
-
-// Called on every log record (each one of which is a WriteBatch)
-// found in a kLogFile.
-static void WriteBatchPrinter(uint64_t pos, Slice record, WritableFile* dst) {
- std::string r = "--- offset ";
- AppendNumberTo(&r, pos);
- r += "; ";
- if (record.size() < 12) {
- r += "log record length ";
- AppendNumberTo(&r, record.size());
- r += " is too small\n";
- dst->Append(r);
- return;
- }
- WriteBatch batch;
- WriteBatchInternal::SetContents(&batch, record);
- r += "sequence ";
- AppendNumberTo(&r, WriteBatchInternal::Sequence(&batch));
- r.push_back('\n');
- dst->Append(r);
- WriteBatchItemPrinter batch_item_printer;
- batch_item_printer.dst_ = dst;
- Status s = batch.Iterate(&batch_item_printer);
- if (!s.ok()) {
- dst->Append(" error: " + s.ToString() + "\n");
- }
-}
-
-Status DumpLog(Env* env, const std::string& fname, WritableFile* dst) {
- return PrintLogContents(env, fname, WriteBatchPrinter, dst);
-}
-
-// Called on every log record (each one of which is a WriteBatch)
-// found in a kDescriptorFile.
-static void VersionEditPrinter(uint64_t pos, Slice record, WritableFile* dst) {
- std::string r = "--- offset ";
- AppendNumberTo(&r, pos);
- r += "; ";
- VersionEdit edit;
- Status s = edit.DecodeFrom(record);
- if (!s.ok()) {
- r += s.ToString();
- r.push_back('\n');
- } else {
- r += edit.DebugString();
- }
- dst->Append(r);
-}
-
-Status DumpDescriptor(Env* env, const std::string& fname, WritableFile* dst) {
- return PrintLogContents(env, fname, VersionEditPrinter, dst);
-}
-
-Status DumpTable(Env* env, const std::string& fname, WritableFile* dst) {
- uint64_t file_size;
- RandomAccessFile* file = NULL;
- Table* table = NULL;
- Status s = env->GetFileSize(fname, &file_size);
- if (s.ok()) {
- s = env->NewRandomAccessFile(fname, &file);
- }
- if (s.ok()) {
- // We use the default comparator, which may or may not match the
- // comparator used in this database. However this should not cause
- // problems since we only use Table operations that do not require
- // any comparisons. In particular, we do not call Seek or Prev.
- s = Table::Open(Options(), file, file_size, &table);
- }
- if (!s.ok()) {
- delete table;
- delete file;
- return s;
- }
-
- ReadOptions ro;
- ro.fill_cache = false;
- Iterator* iter = table->NewIterator(ro);
- std::string r;
- for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
- r.clear();
- ParsedInternalKey key;
- if (!ParseInternalKey(iter->key(), &key)) {
- r = "badkey '";
- AppendEscapedStringTo(&r, iter->key());
- r += "' => '";
- AppendEscapedStringTo(&r, iter->value());
- r += "'\n";
- dst->Append(r);
- } else {
- r = "'";
- AppendEscapedStringTo(&r, key.user_key);
- r += "' @ ";
- AppendNumberTo(&r, key.sequence);
- r += " : ";
- if (key.type == kTypeDeletion) {
- r += "del";
- } else if (key.type == kTypeValue) {
- r += "val";
- } else {
- AppendNumberTo(&r, key.type);
- }
- r += " => '";
- AppendEscapedStringTo(&r, iter->value());
- r += "'\n";
- dst->Append(r);
- }
- }
- s = iter->status();
- if (!s.ok()) {
- dst->Append("iterator error: " + s.ToString() + "\n");
- }
-
- delete iter;
- delete table;
- delete file;
- return Status::OK();
-}
-
-} // namespace
-
-Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) {
- FileType ftype;
- if (!GuessType(fname, &ftype)) {
- return Status::InvalidArgument(fname + ": unknown file type");
- }
- switch (ftype) {
- case kLogFile: return DumpLog(env, fname, dst);
- case kDescriptorFile: return DumpDescriptor(env, fname, dst);
- case kTableFile: return DumpTable(env, fname, dst);
- default:
- break;
- }
- return Status::InvalidArgument(fname + ": not a dump-able file type");
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/filename.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/filename.cc b/thirdparty/leveldb-1.18/db/filename.cc
deleted file mode 100755
index da32946..0000000
--- a/thirdparty/leveldb-1.18/db/filename.cc
+++ /dev/null
@@ -1,144 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <ctype.h>
-#include <stdio.h>
-#include "db/filename.h"
-#include "db/dbformat.h"
-#include "leveldb/env.h"
-#include "util/logging.h"
-
-namespace leveldb {
-
-// A utility routine: write "data" to the named file and Sync() it.
-extern Status WriteStringToFileSync(Env* env, const Slice& data,
- const std::string& fname);
-
-static std::string MakeFileName(const std::string& name, uint64_t number,
- const char* suffix) {
- char buf[100];
- snprintf(buf, sizeof(buf), "/%06llu.%s",
- static_cast<unsigned long long>(number),
- suffix);
- return name + buf;
-}
-
-std::string LogFileName(const std::string& name, uint64_t number) {
- assert(number > 0);
- return MakeFileName(name, number, "log");
-}
-
-std::string TableFileName(const std::string& name, uint64_t number) {
- assert(number > 0);
- return MakeFileName(name, number, "ldb");
-}
-
-std::string SSTTableFileName(const std::string& name, uint64_t number) {
- assert(number > 0);
- return MakeFileName(name, number, "sst");
-}
-
-std::string DescriptorFileName(const std::string& dbname, uint64_t number) {
- assert(number > 0);
- char buf[100];
- snprintf(buf, sizeof(buf), "/MANIFEST-%06llu",
- static_cast<unsigned long long>(number));
- return dbname + buf;
-}
-
-std::string CurrentFileName(const std::string& dbname) {
- return dbname + "/CURRENT";
-}
-
-std::string LockFileName(const std::string& dbname) {
- return dbname + "/LOCK";
-}
-
-std::string TempFileName(const std::string& dbname, uint64_t number) {
- assert(number > 0);
- return MakeFileName(dbname, number, "dbtmp");
-}
-
-std::string InfoLogFileName(const std::string& dbname) {
- return dbname + "/LOG";
-}
-
-// Return the name of the old info log file for "dbname".
-std::string OldInfoLogFileName(const std::string& dbname) {
- return dbname + "/LOG.old";
-}
-
-
-// Owned filenames have the form:
-// dbname/CURRENT
-// dbname/LOCK
-// dbname/LOG
-// dbname/LOG.old
-// dbname/MANIFEST-[0-9]+
-// dbname/[0-9]+.(log|sst|ldb)
-bool ParseFileName(const std::string& fname,
- uint64_t* number,
- FileType* type) {
- Slice rest(fname);
- if (rest == "CURRENT") {
- *number = 0;
- *type = kCurrentFile;
- } else if (rest == "LOCK") {
- *number = 0;
- *type = kDBLockFile;
- } else if (rest == "LOG" || rest == "LOG.old") {
- *number = 0;
- *type = kInfoLogFile;
- } else if (rest.starts_with("MANIFEST-")) {
- rest.remove_prefix(strlen("MANIFEST-"));
- uint64_t num;
- if (!ConsumeDecimalNumber(&rest, &num)) {
- return false;
- }
- if (!rest.empty()) {
- return false;
- }
- *type = kDescriptorFile;
- *number = num;
- } else {
- // Avoid strtoull() to keep filename format independent of the
- // current locale
- uint64_t num;
- if (!ConsumeDecimalNumber(&rest, &num)) {
- return false;
- }
- Slice suffix = rest;
- if (suffix == Slice(".log")) {
- *type = kLogFile;
- } else if (suffix == Slice(".sst") || suffix == Slice(".ldb")) {
- *type = kTableFile;
- } else if (suffix == Slice(".dbtmp")) {
- *type = kTempFile;
- } else {
- return false;
- }
- *number = num;
- }
- return true;
-}
-
-Status SetCurrentFile(Env* env, const std::string& dbname,
- uint64_t descriptor_number) {
- // Remove leading "dbname/" and add newline to manifest file name
- std::string manifest = DescriptorFileName(dbname, descriptor_number);
- Slice contents = manifest;
- assert(contents.starts_with(dbname + "/"));
- contents.remove_prefix(dbname.size() + 1);
- std::string tmp = TempFileName(dbname, descriptor_number);
- Status s = WriteStringToFileSync(env, contents.ToString() + "\n", tmp);
- if (s.ok()) {
- s = env->RenameFile(tmp, CurrentFileName(dbname));
- }
- if (!s.ok()) {
- env->DeleteFile(tmp);
- }
- return s;
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/filename.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/filename.h b/thirdparty/leveldb-1.18/db/filename.h
deleted file mode 100755
index 87a7526..0000000
--- a/thirdparty/leveldb-1.18/db/filename.h
+++ /dev/null
@@ -1,85 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// File names used by DB code
-
-#ifndef STORAGE_LEVELDB_DB_FILENAME_H_
-#define STORAGE_LEVELDB_DB_FILENAME_H_
-
-#include <stdint.h>
-#include <string>
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-#include "port/port.h"
-
-namespace leveldb {
-
-class Env;
-
-enum FileType {
- kLogFile,
- kDBLockFile,
- kTableFile,
- kDescriptorFile,
- kCurrentFile,
- kTempFile,
- kInfoLogFile // Either the current one, or an old one
-};
-
-// Return the name of the log file with the specified number
-// in the db named by "dbname". The result will be prefixed with
-// "dbname".
-extern std::string LogFileName(const std::string& dbname, uint64_t number);
-
-// Return the name of the sstable with the specified number
-// in the db named by "dbname". The result will be prefixed with
-// "dbname".
-extern std::string TableFileName(const std::string& dbname, uint64_t number);
-
-// Return the legacy file name for an sstable with the specified number
-// in the db named by "dbname". The result will be prefixed with
-// "dbname".
-extern std::string SSTTableFileName(const std::string& dbname, uint64_t number);
-
-// Return the name of the descriptor file for the db named by
-// "dbname" and the specified incarnation number. The result will be
-// prefixed with "dbname".
-extern std::string DescriptorFileName(const std::string& dbname,
- uint64_t number);
-
-// Return the name of the current file. This file contains the name
-// of the current manifest file. The result will be prefixed with
-// "dbname".
-extern std::string CurrentFileName(const std::string& dbname);
-
-// Return the name of the lock file for the db named by
-// "dbname". The result will be prefixed with "dbname".
-extern std::string LockFileName(const std::string& dbname);
-
-// Return the name of a temporary file owned by the db named "dbname".
-// The result will be prefixed with "dbname".
-extern std::string TempFileName(const std::string& dbname, uint64_t number);
-
-// Return the name of the info log file for "dbname".
-extern std::string InfoLogFileName(const std::string& dbname);
-
-// Return the name of the old info log file for "dbname".
-extern std::string OldInfoLogFileName(const std::string& dbname);
-
-// If filename is a leveldb file, store the type of the file in *type.
-// The number encoded in the filename is stored in *number. If the
-// filename was successfully parsed, returns true. Else return false.
-extern bool ParseFileName(const std::string& filename,
- uint64_t* number,
- FileType* type);
-
-// Make the CURRENT file point to the descriptor file with the
-// specified number.
-extern Status SetCurrentFile(Env* env, const std::string& dbname,
- uint64_t descriptor_number);
-
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_FILENAME_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/filename_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/filename_test.cc b/thirdparty/leveldb-1.18/db/filename_test.cc
deleted file mode 100755
index a32556d..0000000
--- a/thirdparty/leveldb-1.18/db/filename_test.cc
+++ /dev/null
@@ -1,123 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/filename.h"
-
-#include "db/dbformat.h"
-#include "port/port.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-class FileNameTest { };
-
-TEST(FileNameTest, Parse) {
- Slice db;
- FileType type;
- uint64_t number;
-
- // Successful parses
- static struct {
- const char* fname;
- uint64_t number;
- FileType type;
- } cases[] = {
- { "100.log", 100, kLogFile },
- { "0.log", 0, kLogFile },
- { "0.sst", 0, kTableFile },
- { "0.ldb", 0, kTableFile },
- { "CURRENT", 0, kCurrentFile },
- { "LOCK", 0, kDBLockFile },
- { "MANIFEST-2", 2, kDescriptorFile },
- { "MANIFEST-7", 7, kDescriptorFile },
- { "LOG", 0, kInfoLogFile },
- { "LOG.old", 0, kInfoLogFile },
- { "18446744073709551615.log", 18446744073709551615ull, kLogFile },
- };
- for (int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
- std::string f = cases[i].fname;
- ASSERT_TRUE(ParseFileName(f, &number, &type)) << f;
- ASSERT_EQ(cases[i].type, type) << f;
- ASSERT_EQ(cases[i].number, number) << f;
- }
-
- // Errors
- static const char* errors[] = {
- "",
- "foo",
- "foo-dx-100.log",
- ".log",
- "",
- "manifest",
- "CURREN",
- "CURRENTX",
- "MANIFES",
- "MANIFEST",
- "MANIFEST-",
- "XMANIFEST-3",
- "MANIFEST-3x",
- "LOC",
- "LOCKx",
- "LO",
- "LOGx",
- "18446744073709551616.log",
- "184467440737095516150.log",
- "100",
- "100.",
- "100.lop"
- };
- for (int i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) {
- std::string f = errors[i];
- ASSERT_TRUE(!ParseFileName(f, &number, &type)) << f;
- }
-}
-
-TEST(FileNameTest, Construction) {
- uint64_t number;
- FileType type;
- std::string fname;
-
- fname = CurrentFileName("foo");
- ASSERT_EQ("foo/", std::string(fname.data(), 4));
- ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
- ASSERT_EQ(0, number);
- ASSERT_EQ(kCurrentFile, type);
-
- fname = LockFileName("foo");
- ASSERT_EQ("foo/", std::string(fname.data(), 4));
- ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
- ASSERT_EQ(0, number);
- ASSERT_EQ(kDBLockFile, type);
-
- fname = LogFileName("foo", 192);
- ASSERT_EQ("foo/", std::string(fname.data(), 4));
- ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
- ASSERT_EQ(192, number);
- ASSERT_EQ(kLogFile, type);
-
- fname = TableFileName("bar", 200);
- ASSERT_EQ("bar/", std::string(fname.data(), 4));
- ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
- ASSERT_EQ(200, number);
- ASSERT_EQ(kTableFile, type);
-
- fname = DescriptorFileName("bar", 100);
- ASSERT_EQ("bar/", std::string(fname.data(), 4));
- ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
- ASSERT_EQ(100, number);
- ASSERT_EQ(kDescriptorFile, type);
-
- fname = TempFileName("tmp", 999);
- ASSERT_EQ("tmp/", std::string(fname.data(), 4));
- ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
- ASSERT_EQ(999, number);
- ASSERT_EQ(kTempFile, type);
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/leveldb_main.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/leveldb_main.cc b/thirdparty/leveldb-1.18/db/leveldb_main.cc
deleted file mode 100755
index 9f4b7dd..0000000
--- a/thirdparty/leveldb-1.18/db/leveldb_main.cc
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <stdio.h>
-#include "leveldb/dumpfile.h"
-#include "leveldb/env.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-namespace {
-
-class StdoutPrinter : public WritableFile {
- public:
- virtual Status Append(const Slice& data) {
- fwrite(data.data(), 1, data.size(), stdout);
- return Status::OK();
- }
- virtual Status Close() { return Status::OK(); }
- virtual Status Flush() { return Status::OK(); }
- virtual Status Sync() { return Status::OK(); }
-};
-
-bool HandleDumpCommand(Env* env, char** files, int num) {
- StdoutPrinter printer;
- bool ok = true;
- for (int i = 0; i < num; i++) {
- Status s = DumpFile(env, files[i], &printer);
- if (!s.ok()) {
- fprintf(stderr, "%s\n", s.ToString().c_str());
- ok = false;
- }
- }
- return ok;
-}
-
-} // namespace
-} // namespace leveldb
-
-static void Usage() {
- fprintf(
- stderr,
- "Usage: leveldbutil command...\n"
- " dump files... -- dump contents of specified files\n"
- );
-}
-
-int main(int argc, char** argv) {
- leveldb::Env* env = leveldb::Env::Default();
- bool ok = true;
- if (argc < 2) {
- Usage();
- ok = false;
- } else {
- std::string command = argv[1];
- if (command == "dump") {
- ok = leveldb::HandleDumpCommand(env, argv+2, argc-2);
- } else {
- Usage();
- ok = false;
- }
- }
- return (ok ? 0 : 1);
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/log_format.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/log_format.h b/thirdparty/leveldb-1.18/db/log_format.h
deleted file mode 100755
index a8c06ef..0000000
--- a/thirdparty/leveldb-1.18/db/log_format.h
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Log format information shared by reader and writer.
-// See ../doc/log_format.txt for more detail.
-
-#ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_
-#define STORAGE_LEVELDB_DB_LOG_FORMAT_H_
-
-namespace leveldb {
-namespace log {
-
-enum RecordType {
- // Zero is reserved for preallocated files
- kZeroType = 0,
-
- kFullType = 1,
-
- // For fragments
- kFirstType = 2,
- kMiddleType = 3,
- kLastType = 4
-};
-static const int kMaxRecordType = kLastType;
-
-static const int kBlockSize = 32768;
-
-// Header is checksum (4 bytes), length (2 bytes), type (1 byte).
-static const int kHeaderSize = 4 + 2 + 1;
-
-} // namespace log
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/log_reader.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/log_reader.cc b/thirdparty/leveldb-1.18/db/log_reader.cc
deleted file mode 100755
index e44b66c..0000000
--- a/thirdparty/leveldb-1.18/db/log_reader.cc
+++ /dev/null
@@ -1,266 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/log_reader.h"
-
-#include <stdio.h>
-#include "leveldb/env.h"
-#include "util/coding.h"
-#include "util/crc32c.h"
-
-namespace leveldb {
-namespace log {
-
-Reader::Reporter::~Reporter() {
-}
-
-Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
- uint64_t initial_offset)
- : file_(file),
- reporter_(reporter),
- checksum_(checksum),
- backing_store_(new char[kBlockSize]),
- buffer_(),
- eof_(false),
- last_record_offset_(0),
- end_of_buffer_offset_(0),
- initial_offset_(initial_offset) {
-}
-
-Reader::~Reader() {
- delete[] backing_store_;
-}
-
-bool Reader::SkipToInitialBlock() {
- size_t offset_in_block = initial_offset_ % kBlockSize;
- uint64_t block_start_location = initial_offset_ - offset_in_block;
-
- // Don't search a block if we'd be in the trailer
- if (offset_in_block > kBlockSize - 6) {
- offset_in_block = 0;
- block_start_location += kBlockSize;
- }
-
- end_of_buffer_offset_ = block_start_location;
-
- // Skip to start of first block that can contain the initial record
- if (block_start_location > 0) {
- Status skip_status = file_->Skip(block_start_location);
- if (!skip_status.ok()) {
- ReportDrop(block_start_location, skip_status);
- return false;
- }
- }
-
- return true;
-}
-
-bool Reader::ReadRecord(Slice* record, std::string* scratch) {
- if (last_record_offset_ < initial_offset_) {
- if (!SkipToInitialBlock()) {
- return false;
- }
- }
-
- scratch->clear();
- record->clear();
- bool in_fragmented_record = false;
- // Record offset of the logical record that we're reading
- // 0 is a dummy value to make compilers happy
- uint64_t prospective_record_offset = 0;
-
- Slice fragment;
- while (true) {
- uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
- const unsigned int record_type = ReadPhysicalRecord(&fragment);
- switch (record_type) {
- case kFullType:
- if (in_fragmented_record) {
- // Handle bug in earlier versions of log::Writer where
- // it could emit an empty kFirstType record at the tail end
- // of a block followed by a kFullType or kFirstType record
- // at the beginning of the next block.
- if (scratch->empty()) {
- in_fragmented_record = false;
- } else {
- ReportCorruption(scratch->size(), "partial record without end(1)");
- }
- }
- prospective_record_offset = physical_record_offset;
- scratch->clear();
- *record = fragment;
- last_record_offset_ = prospective_record_offset;
- return true;
-
- case kFirstType:
- if (in_fragmented_record) {
- // Handle bug in earlier versions of log::Writer where
- // it could emit an empty kFirstType record at the tail end
- // of a block followed by a kFullType or kFirstType record
- // at the beginning of the next block.
- if (scratch->empty()) {
- in_fragmented_record = false;
- } else {
- ReportCorruption(scratch->size(), "partial record without end(2)");
- }
- }
- prospective_record_offset = physical_record_offset;
- scratch->assign(fragment.data(), fragment.size());
- in_fragmented_record = true;
- break;
-
- case kMiddleType:
- if (!in_fragmented_record) {
- ReportCorruption(fragment.size(),
- "missing start of fragmented record(1)");
- } else {
- scratch->append(fragment.data(), fragment.size());
- }
- break;
-
- case kLastType:
- if (!in_fragmented_record) {
- ReportCorruption(fragment.size(),
- "missing start of fragmented record(2)");
- } else {
- scratch->append(fragment.data(), fragment.size());
- *record = Slice(*scratch);
- last_record_offset_ = prospective_record_offset;
- return true;
- }
- break;
-
- case kEof:
- if (in_fragmented_record) {
- // This can be caused by the writer dying immediately after
- // writing a physical record but before completing the next; don't
- // treat it as a corruption, just ignore the entire logical record.
- scratch->clear();
- }
- return false;
-
- case kBadRecord:
- if (in_fragmented_record) {
- ReportCorruption(scratch->size(), "error in middle of record");
- in_fragmented_record = false;
- scratch->clear();
- }
- break;
-
- default: {
- char buf[40];
- snprintf(buf, sizeof(buf), "unknown record type %u", record_type);
- ReportCorruption(
- (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
- buf);
- in_fragmented_record = false;
- scratch->clear();
- break;
- }
- }
- }
- return false;
-}
-
-uint64_t Reader::LastRecordOffset() {
- return last_record_offset_;
-}
-
-void Reader::ReportCorruption(uint64_t bytes, const char* reason) {
- ReportDrop(bytes, Status::Corruption(reason));
-}
-
-void Reader::ReportDrop(uint64_t bytes, const Status& reason) {
- if (reporter_ != NULL &&
- end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) {
- reporter_->Corruption(static_cast<size_t>(bytes), reason);
- }
-}
-
-unsigned int Reader::ReadPhysicalRecord(Slice* result) {
- while (true) {
- if (buffer_.size() < kHeaderSize) {
- if (!eof_) {
- // Last read was a full read, so this is a trailer to skip
- buffer_.clear();
- Status status = file_->Read(kBlockSize, &buffer_, backing_store_);
- end_of_buffer_offset_ += buffer_.size();
- if (!status.ok()) {
- buffer_.clear();
- ReportDrop(kBlockSize, status);
- eof_ = true;
- return kEof;
- } else if (buffer_.size() < kBlockSize) {
- eof_ = true;
- }
- continue;
- } else {
- // Note that if buffer_ is non-empty, we have a truncated header at the
- // end of the file, which can be caused by the writer crashing in the
- // middle of writing the header. Instead of considering this an error,
- // just report EOF.
- buffer_.clear();
- return kEof;
- }
- }
-
- // Parse the header
- const char* header = buffer_.data();
- const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
- const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
- const unsigned int type = header[6];
- const uint32_t length = a | (b << 8);
- if (kHeaderSize + length > buffer_.size()) {
- size_t drop_size = buffer_.size();
- buffer_.clear();
- if (!eof_) {
- ReportCorruption(drop_size, "bad record length");
- return kBadRecord;
- }
- // If the end of the file has been reached without reading |length| bytes
- // of payload, assume the writer died in the middle of writing the record.
- // Don't report a corruption.
- return kEof;
- }
-
- if (type == kZeroType && length == 0) {
- // Skip zero length record without reporting any drops since
- // such records are produced by the mmap based writing code in
- // env_posix.cc that preallocates file regions.
- buffer_.clear();
- return kBadRecord;
- }
-
- // Check crc
- if (checksum_) {
- uint32_t expected_crc = crc32c::Unmask(DecodeFixed32(header));
- uint32_t actual_crc = crc32c::Value(header + 6, 1 + length);
- if (actual_crc != expected_crc) {
- // Drop the rest of the buffer since "length" itself may have
- // been corrupted and if we trust it, we could find some
- // fragment of a real log record that just happens to look
- // like a valid log record.
- size_t drop_size = buffer_.size();
- buffer_.clear();
- ReportCorruption(drop_size, "checksum mismatch");
- return kBadRecord;
- }
- }
-
- buffer_.remove_prefix(kHeaderSize + length);
-
- // Skip physical record that started before initial_offset_
- if (end_of_buffer_offset_ - buffer_.size() - kHeaderSize - length <
- initial_offset_) {
- result->clear();
- return kBadRecord;
- }
-
- *result = Slice(header + kHeaderSize, length);
- return type;
- }
-}
-
-} // namespace log
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/log_reader.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/log_reader.h b/thirdparty/leveldb-1.18/db/log_reader.h
deleted file mode 100755
index 6aff791..0000000
--- a/thirdparty/leveldb-1.18/db/log_reader.h
+++ /dev/null
@@ -1,108 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_LOG_READER_H_
-#define STORAGE_LEVELDB_DB_LOG_READER_H_
-
-#include <stdint.h>
-
-#include "db/log_format.h"
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class SequentialFile;
-
-namespace log {
-
-class Reader {
- public:
- // Interface for reporting errors.
- class Reporter {
- public:
- virtual ~Reporter();
-
- // Some corruption was detected. "size" is the approximate number
- // of bytes dropped due to the corruption.
- virtual void Corruption(size_t bytes, const Status& status) = 0;
- };
-
- // Create a reader that will return log records from "*file".
- // "*file" must remain live while this Reader is in use.
- //
- // If "reporter" is non-NULL, it is notified whenever some data is
- // dropped due to a detected corruption. "*reporter" must remain
- // live while this Reader is in use.
- //
- // If "checksum" is true, verify checksums if available.
- //
- // The Reader will start reading at the first record located at physical
- // position >= initial_offset within the file.
- Reader(SequentialFile* file, Reporter* reporter, bool checksum,
- uint64_t initial_offset);
-
- ~Reader();
-
- // Read the next record into *record. Returns true if read
- // successfully, false if we hit end of the input. May use
- // "*scratch" as temporary storage. The contents filled in *record
- // will only be valid until the next mutating operation on this
- // reader or the next mutation to *scratch.
- bool ReadRecord(Slice* record, std::string* scratch);
-
- // Returns the physical offset of the last record returned by ReadRecord.
- //
- // Undefined before the first call to ReadRecord.
- uint64_t LastRecordOffset();
-
- private:
- SequentialFile* const file_;
- Reporter* const reporter_;
- bool const checksum_;
- char* const backing_store_;
- Slice buffer_;
- bool eof_; // Last Read() indicated EOF by returning < kBlockSize
-
- // Offset of the last record returned by ReadRecord.
- uint64_t last_record_offset_;
- // Offset of the first location past the end of buffer_.
- uint64_t end_of_buffer_offset_;
-
- // Offset at which to start looking for the first record to return
- uint64_t const initial_offset_;
-
- // Extend record types with the following special values
- enum {
- kEof = kMaxRecordType + 1,
- // Returned whenever we find an invalid physical record.
- // Currently there are three situations in which this happens:
- // * The record has an invalid CRC (ReadPhysicalRecord reports a drop)
- // * The record is a 0-length record (No drop is reported)
- // * The record is below constructor's initial_offset (No drop is reported)
- kBadRecord = kMaxRecordType + 2
- };
-
- // Skips all blocks that are completely before "initial_offset_".
- //
- // Returns true on success. Handles reporting.
- bool SkipToInitialBlock();
-
- // Return type, or one of the preceding special values
- unsigned int ReadPhysicalRecord(Slice* result);
-
- // Reports dropped bytes to the reporter.
- // buffer_ must be updated to remove the dropped bytes prior to invocation.
- void ReportCorruption(uint64_t bytes, const char* reason);
- void ReportDrop(uint64_t bytes, const Status& reason);
-
- // No copying allowed
- Reader(const Reader&);
- void operator=(const Reader&);
-};
-
-} // namespace log
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_LOG_READER_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/log_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/log_test.cc b/thirdparty/leveldb-1.18/db/log_test.cc
deleted file mode 100755
index dcf0562..0000000
--- a/thirdparty/leveldb-1.18/db/log_test.cc
+++ /dev/null
@@ -1,530 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/log_reader.h"
-#include "db/log_writer.h"
-#include "leveldb/env.h"
-#include "util/coding.h"
-#include "util/crc32c.h"
-#include "util/random.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-namespace log {
-
-// Construct a string of the specified length made out of the supplied
-// partial string.
-static std::string BigString(const std::string& partial_string, size_t n) {
- std::string result;
- while (result.size() < n) {
- result.append(partial_string);
- }
- result.resize(n);
- return result;
-}
-
-// Construct a string from a number
-static std::string NumberString(int n) {
- char buf[50];
- snprintf(buf, sizeof(buf), "%d.", n);
- return std::string(buf);
-}
-
-// Return a skewed potentially long string
-static std::string RandomSkewedString(int i, Random* rnd) {
- return BigString(NumberString(i), rnd->Skewed(17));
-}
-
-class LogTest {
- private:
- class StringDest : public WritableFile {
- public:
- std::string contents_;
-
- virtual Status Close() { return Status::OK(); }
- virtual Status Flush() { return Status::OK(); }
- virtual Status Sync() { return Status::OK(); }
- virtual Status Append(const Slice& slice) {
- contents_.append(slice.data(), slice.size());
- return Status::OK();
- }
- };
-
- class StringSource : public SequentialFile {
- public:
- Slice contents_;
- bool force_error_;
- bool returned_partial_;
- StringSource() : force_error_(false), returned_partial_(false) { }
-
- virtual Status Read(size_t n, Slice* result, char* scratch) {
- ASSERT_TRUE(!returned_partial_) << "must not Read() after eof/error";
-
- if (force_error_) {
- force_error_ = false;
- returned_partial_ = true;
- return Status::Corruption("read error");
- }
-
- if (contents_.size() < n) {
- n = contents_.size();
- returned_partial_ = true;
- }
- *result = Slice(contents_.data(), n);
- contents_.remove_prefix(n);
- return Status::OK();
- }
-
- virtual Status Skip(uint64_t n) {
- if (n > contents_.size()) {
- contents_.clear();
- return Status::NotFound("in-memory file skipepd past end");
- }
-
- contents_.remove_prefix(n);
-
- return Status::OK();
- }
- };
-
- class ReportCollector : public Reader::Reporter {
- public:
- size_t dropped_bytes_;
- std::string message_;
-
- ReportCollector() : dropped_bytes_(0) { }
- virtual void Corruption(size_t bytes, const Status& status) {
- dropped_bytes_ += bytes;
- message_.append(status.ToString());
- }
- };
-
- StringDest dest_;
- StringSource source_;
- ReportCollector report_;
- bool reading_;
- Writer writer_;
- Reader reader_;
-
- // Record metadata for testing initial offset functionality
- static size_t initial_offset_record_sizes_[];
- static uint64_t initial_offset_last_record_offsets_[];
-
- public:
- LogTest() : reading_(false),
- writer_(&dest_),
- reader_(&source_, &report_, true/*checksum*/,
- 0/*initial_offset*/) {
- }
-
- void Write(const std::string& msg) {
- ASSERT_TRUE(!reading_) << "Write() after starting to read";
- writer_.AddRecord(Slice(msg));
- }
-
- size_t WrittenBytes() const {
- return dest_.contents_.size();
- }
-
- std::string Read() {
- if (!reading_) {
- reading_ = true;
- source_.contents_ = Slice(dest_.contents_);
- }
- std::string scratch;
- Slice record;
- if (reader_.ReadRecord(&record, &scratch)) {
- return record.ToString();
- } else {
- return "EOF";
- }
- }
-
- void IncrementByte(int offset, int delta) {
- dest_.contents_[offset] += delta;
- }
-
- void SetByte(int offset, char new_byte) {
- dest_.contents_[offset] = new_byte;
- }
-
- void ShrinkSize(int bytes) {
- dest_.contents_.resize(dest_.contents_.size() - bytes);
- }
-
- void FixChecksum(int header_offset, int len) {
- // Compute crc of type/len/data
- uint32_t crc = crc32c::Value(&dest_.contents_[header_offset+6], 1 + len);
- crc = crc32c::Mask(crc);
- EncodeFixed32(&dest_.contents_[header_offset], crc);
- }
-
- void ForceError() {
- source_.force_error_ = true;
- }
-
- size_t DroppedBytes() const {
- return report_.dropped_bytes_;
- }
-
- std::string ReportMessage() const {
- return report_.message_;
- }
-
- // Returns OK iff recorded error message contains "msg"
- std::string MatchError(const std::string& msg) const {
- if (report_.message_.find(msg) == std::string::npos) {
- return report_.message_;
- } else {
- return "OK";
- }
- }
-
- void WriteInitialOffsetLog() {
- for (int i = 0; i < 4; i++) {
- std::string record(initial_offset_record_sizes_[i],
- static_cast<char>('a' + i));
- Write(record);
- }
- }
-
- void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) {
- WriteInitialOffsetLog();
- reading_ = true;
- source_.contents_ = Slice(dest_.contents_);
- Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
- WrittenBytes() + offset_past_end);
- Slice record;
- std::string scratch;
- ASSERT_TRUE(!offset_reader->ReadRecord(&record, &scratch));
- delete offset_reader;
- }
-
- void CheckInitialOffsetRecord(uint64_t initial_offset,
- int expected_record_offset) {
- WriteInitialOffsetLog();
- reading_ = true;
- source_.contents_ = Slice(dest_.contents_);
- Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
- initial_offset);
- Slice record;
- std::string scratch;
- ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
- ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset],
- record.size());
- ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
- offset_reader->LastRecordOffset());
- ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
- delete offset_reader;
- }
-
-};
-
-size_t LogTest::initial_offset_record_sizes_[] =
- {10000, // Two sizable records in first block
- 10000,
- 2 * log::kBlockSize - 1000, // Span three blocks
- 1};
-
-uint64_t LogTest::initial_offset_last_record_offsets_[] =
- {0,
- kHeaderSize + 10000,
- 2 * (kHeaderSize + 10000),
- 2 * (kHeaderSize + 10000) +
- (2 * log::kBlockSize - 1000) + 3 * kHeaderSize};
-
-
-TEST(LogTest, Empty) {
- ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, ReadWrite) {
- Write("foo");
- Write("bar");
- Write("");
- Write("xxxx");
- ASSERT_EQ("foo", Read());
- ASSERT_EQ("bar", Read());
- ASSERT_EQ("", Read());
- ASSERT_EQ("xxxx", Read());
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ("EOF", Read()); // Make sure reads at eof work
-}
-
-TEST(LogTest, ManyBlocks) {
- for (int i = 0; i < 100000; i++) {
- Write(NumberString(i));
- }
- for (int i = 0; i < 100000; i++) {
- ASSERT_EQ(NumberString(i), Read());
- }
- ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, Fragmentation) {
- Write("small");
- Write(BigString("medium", 50000));
- Write(BigString("large", 100000));
- ASSERT_EQ("small", Read());
- ASSERT_EQ(BigString("medium", 50000), Read());
- ASSERT_EQ(BigString("large", 100000), Read());
- ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, MarginalTrailer) {
- // Make a trailer that is exactly the same length as an empty record.
- const int n = kBlockSize - 2*kHeaderSize;
- Write(BigString("foo", n));
- ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes());
- Write("");
- Write("bar");
- ASSERT_EQ(BigString("foo", n), Read());
- ASSERT_EQ("", Read());
- ASSERT_EQ("bar", Read());
- ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, MarginalTrailer2) {
- // Make a trailer that is exactly the same length as an empty record.
- const int n = kBlockSize - 2*kHeaderSize;
- Write(BigString("foo", n));
- ASSERT_EQ(kBlockSize - kHeaderSize, WrittenBytes());
- Write("bar");
- ASSERT_EQ(BigString("foo", n), Read());
- ASSERT_EQ("bar", Read());
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ(0, DroppedBytes());
- ASSERT_EQ("", ReportMessage());
-}
-
-TEST(LogTest, ShortTrailer) {
- const int n = kBlockSize - 2*kHeaderSize + 4;
- Write(BigString("foo", n));
- ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
- Write("");
- Write("bar");
- ASSERT_EQ(BigString("foo", n), Read());
- ASSERT_EQ("", Read());
- ASSERT_EQ("bar", Read());
- ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, AlignedEof) {
- const int n = kBlockSize - 2*kHeaderSize + 4;
- Write(BigString("foo", n));
- ASSERT_EQ(kBlockSize - kHeaderSize + 4, WrittenBytes());
- ASSERT_EQ(BigString("foo", n), Read());
- ASSERT_EQ("EOF", Read());
-}
-
-TEST(LogTest, RandomRead) {
- const int N = 500;
- Random write_rnd(301);
- for (int i = 0; i < N; i++) {
- Write(RandomSkewedString(i, &write_rnd));
- }
- Random read_rnd(301);
- for (int i = 0; i < N; i++) {
- ASSERT_EQ(RandomSkewedString(i, &read_rnd), Read());
- }
- ASSERT_EQ("EOF", Read());
-}
-
-// Tests of all the error paths in log_reader.cc follow:
-
-TEST(LogTest, ReadError) {
- Write("foo");
- ForceError();
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ(kBlockSize, DroppedBytes());
- ASSERT_EQ("OK", MatchError("read error"));
-}
-
-TEST(LogTest, BadRecordType) {
- Write("foo");
- // Type is stored in header[6]
- IncrementByte(6, 100);
- FixChecksum(0, 3);
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ(3, DroppedBytes());
- ASSERT_EQ("OK", MatchError("unknown record type"));
-}
-
-TEST(LogTest, TruncatedTrailingRecordIsIgnored) {
- Write("foo");
- ShrinkSize(4); // Drop all payload as well as a header byte
- ASSERT_EQ("EOF", Read());
- // Truncated last record is ignored, not treated as an error.
- ASSERT_EQ(0, DroppedBytes());
- ASSERT_EQ("", ReportMessage());
-}
-
-TEST(LogTest, BadLength) {
- const int kPayloadSize = kBlockSize - kHeaderSize;
- Write(BigString("bar", kPayloadSize));
- Write("foo");
- // Least significant size byte is stored in header[4].
- IncrementByte(4, 1);
- ASSERT_EQ("foo", Read());
- ASSERT_EQ(kBlockSize, DroppedBytes());
- ASSERT_EQ("OK", MatchError("bad record length"));
-}
-
-TEST(LogTest, BadLengthAtEndIsIgnored) {
- Write("foo");
- ShrinkSize(1);
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ(0, DroppedBytes());
- ASSERT_EQ("", ReportMessage());
-}
-
-TEST(LogTest, ChecksumMismatch) {
- Write("foo");
- IncrementByte(0, 10);
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ(10, DroppedBytes());
- ASSERT_EQ("OK", MatchError("checksum mismatch"));
-}
-
-TEST(LogTest, UnexpectedMiddleType) {
- Write("foo");
- SetByte(6, kMiddleType);
- FixChecksum(0, 3);
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ(3, DroppedBytes());
- ASSERT_EQ("OK", MatchError("missing start"));
-}
-
-TEST(LogTest, UnexpectedLastType) {
- Write("foo");
- SetByte(6, kLastType);
- FixChecksum(0, 3);
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ(3, DroppedBytes());
- ASSERT_EQ("OK", MatchError("missing start"));
-}
-
-TEST(LogTest, UnexpectedFullType) {
- Write("foo");
- Write("bar");
- SetByte(6, kFirstType);
- FixChecksum(0, 3);
- ASSERT_EQ("bar", Read());
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ(3, DroppedBytes());
- ASSERT_EQ("OK", MatchError("partial record without end"));
-}
-
-TEST(LogTest, UnexpectedFirstType) {
- Write("foo");
- Write(BigString("bar", 100000));
- SetByte(6, kFirstType);
- FixChecksum(0, 3);
- ASSERT_EQ(BigString("bar", 100000), Read());
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ(3, DroppedBytes());
- ASSERT_EQ("OK", MatchError("partial record without end"));
-}
-
-TEST(LogTest, MissingLastIsIgnored) {
- Write(BigString("bar", kBlockSize));
- // Remove the LAST block, including header.
- ShrinkSize(14);
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ("", ReportMessage());
- ASSERT_EQ(0, DroppedBytes());
-}
-
-TEST(LogTest, PartialLastIsIgnored) {
- Write(BigString("bar", kBlockSize));
- // Cause a bad record length in the LAST block.
- ShrinkSize(1);
- ASSERT_EQ("EOF", Read());
- ASSERT_EQ("", ReportMessage());
- ASSERT_EQ(0, DroppedBytes());
-}
-
-TEST(LogTest, ErrorJoinsRecords) {
- // Consider two fragmented records:
- // first(R1) last(R1) first(R2) last(R2)
- // where the middle two fragments disappear. We do not want
- // first(R1),last(R2) to get joined and returned as a valid record.
-
- // Write records that span two blocks
- Write(BigString("foo", kBlockSize));
- Write(BigString("bar", kBlockSize));
- Write("correct");
-
- // Wipe the middle block
- for (int offset = kBlockSize; offset < 2*kBlockSize; offset++) {
- SetByte(offset, 'x');
- }
-
- ASSERT_EQ("correct", Read());
- ASSERT_EQ("EOF", Read());
- const size_t dropped = DroppedBytes();
- ASSERT_LE(dropped, 2*kBlockSize + 100);
- ASSERT_GE(dropped, 2*kBlockSize);
-}
-
-TEST(LogTest, ReadStart) {
- CheckInitialOffsetRecord(0, 0);
-}
-
-TEST(LogTest, ReadSecondOneOff) {
- CheckInitialOffsetRecord(1, 1);
-}
-
-TEST(LogTest, ReadSecondTenThousand) {
- CheckInitialOffsetRecord(10000, 1);
-}
-
-TEST(LogTest, ReadSecondStart) {
- CheckInitialOffsetRecord(10007, 1);
-}
-
-TEST(LogTest, ReadThirdOneOff) {
- CheckInitialOffsetRecord(10008, 2);
-}
-
-TEST(LogTest, ReadThirdStart) {
- CheckInitialOffsetRecord(20014, 2);
-}
-
-TEST(LogTest, ReadFourthOneOff) {
- CheckInitialOffsetRecord(20015, 3);
-}
-
-TEST(LogTest, ReadFourthFirstBlockTrailer) {
- CheckInitialOffsetRecord(log::kBlockSize - 4, 3);
-}
-
-TEST(LogTest, ReadFourthMiddleBlock) {
- CheckInitialOffsetRecord(log::kBlockSize + 1, 3);
-}
-
-TEST(LogTest, ReadFourthLastBlock) {
- CheckInitialOffsetRecord(2 * log::kBlockSize + 1, 3);
-}
-
-TEST(LogTest, ReadFourthStart) {
- CheckInitialOffsetRecord(
- 2 * (kHeaderSize + 1000) + (2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
- 3);
-}
-
-TEST(LogTest, ReadEnd) {
- CheckOffsetPastEndReturnsNoRecords(0);
-}
-
-TEST(LogTest, ReadPastEnd) {
- CheckOffsetPastEndReturnsNoRecords(5);
-}
-
-} // namespace log
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/log_writer.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/log_writer.cc b/thirdparty/leveldb-1.18/db/log_writer.cc
deleted file mode 100755
index 9a39d28..0000000
--- a/thirdparty/leveldb-1.18/db/log_writer.cc
+++ /dev/null
@@ -1,103 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/log_writer.h"
-
-#include <stdint.h>
-#include "leveldb/env.h"
-#include "util/coding.h"
-#include "util/crc32c.h"
-
-namespace leveldb {
-namespace log {
-
-Writer::Writer(WritableFile* dest)
- : dest_(dest),
- block_offset_(0) {
- for (int i = 0; i <= kMaxRecordType; i++) {
- char t = static_cast<char>(i);
- type_crc_[i] = crc32c::Value(&t, 1);
- }
-}
-
-Writer::~Writer() {
-}
-
-Status Writer::AddRecord(const Slice& slice) {
- const char* ptr = slice.data();
- size_t left = slice.size();
-
- // Fragment the record if necessary and emit it. Note that if slice
- // is empty, we still want to iterate once to emit a single
- // zero-length record
- Status s;
- bool begin = true;
- do {
- const int leftover = kBlockSize - block_offset_;
- assert(leftover >= 0);
- if (leftover < kHeaderSize) {
- // Switch to a new block
- if (leftover > 0) {
- // Fill the trailer (literal below relies on kHeaderSize being 7)
- assert(kHeaderSize == 7);
- dest_->Append(Slice("\x00\x00\x00\x00\x00\x00", leftover));
- }
- block_offset_ = 0;
- }
-
- // Invariant: we never leave < kHeaderSize bytes in a block.
- assert(kBlockSize - block_offset_ - kHeaderSize >= 0);
-
- const size_t avail = kBlockSize - block_offset_ - kHeaderSize;
- const size_t fragment_length = (left < avail) ? left : avail;
-
- RecordType type;
- const bool end = (left == fragment_length);
- if (begin && end) {
- type = kFullType;
- } else if (begin) {
- type = kFirstType;
- } else if (end) {
- type = kLastType;
- } else {
- type = kMiddleType;
- }
-
- s = EmitPhysicalRecord(type, ptr, fragment_length);
- ptr += fragment_length;
- left -= fragment_length;
- begin = false;
- } while (s.ok() && left > 0);
- return s;
-}
-
-Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
- assert(n <= 0xffff); // Must fit in two bytes
- assert(block_offset_ + kHeaderSize + n <= kBlockSize);
-
- // Format the header
- char buf[kHeaderSize];
- buf[4] = static_cast<char>(n & 0xff);
- buf[5] = static_cast<char>(n >> 8);
- buf[6] = static_cast<char>(t);
-
- // Compute the crc of the record type and the payload.
- uint32_t crc = crc32c::Extend(type_crc_[t], ptr, n);
- crc = crc32c::Mask(crc); // Adjust for storage
- EncodeFixed32(buf, crc);
-
- // Write the header and the payload
- Status s = dest_->Append(Slice(buf, kHeaderSize));
- if (s.ok()) {
- s = dest_->Append(Slice(ptr, n));
- if (s.ok()) {
- s = dest_->Flush();
- }
- }
- block_offset_ += kHeaderSize + static_cast<int>(n);
- return s;
-}
-
-} // namespace log
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/log_writer.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/log_writer.h b/thirdparty/leveldb-1.18/db/log_writer.h
deleted file mode 100755
index a3a954d..0000000
--- a/thirdparty/leveldb-1.18/db/log_writer.h
+++ /dev/null
@@ -1,48 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_
-#define STORAGE_LEVELDB_DB_LOG_WRITER_H_
-
-#include <stdint.h>
-#include "db/log_format.h"
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class WritableFile;
-
-namespace log {
-
-class Writer {
- public:
- // Create a writer that will append data to "*dest".
- // "*dest" must be initially empty.
- // "*dest" must remain live while this Writer is in use.
- explicit Writer(WritableFile* dest);
- ~Writer();
-
- Status AddRecord(const Slice& slice);
-
- private:
- WritableFile* dest_;
- int block_offset_; // Current offset in block
-
- // crc32c values for all supported record types. These are
- // pre-computed to reduce the overhead of computing the crc of the
- // record type stored in the header.
- uint32_t type_crc_[kMaxRecordType + 1];
-
- Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
-
- // No copying allowed
- Writer(const Writer&);
- void operator=(const Writer&);
-};
-
-} // namespace log
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/memtable.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/memtable.cc b/thirdparty/leveldb-1.18/db/memtable.cc
deleted file mode 100755
index c190838..0000000
--- a/thirdparty/leveldb-1.18/db/memtable.cc
+++ /dev/null
@@ -1,145 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/memtable.h"
-#include "db/dbformat.h"
-#include "leveldb/comparator.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-static Slice GetLengthPrefixedSlice(const char* data) {
- uint32_t len;
- const char* p = data;
- p = GetVarint32Ptr(p, p + 5, &len); // +5: we assume "p" is not corrupted
- return Slice(p, len);
-}
-
-MemTable::MemTable(const InternalKeyComparator& cmp)
- : comparator_(cmp),
- refs_(0),
- table_(comparator_, &arena_) {
-}
-
-MemTable::~MemTable() {
- assert(refs_ == 0);
-}
-
-size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); }
-
-int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr)
- const {
- // Internal keys are encoded as length-prefixed strings.
- Slice a = GetLengthPrefixedSlice(aptr);
- Slice b = GetLengthPrefixedSlice(bptr);
- return comparator.Compare(a, b);
-}
-
-// Encode a suitable internal key target for "target" and return it.
-// Uses *scratch as scratch space, and the returned pointer will point
-// into this scratch space.
-static const char* EncodeKey(std::string* scratch, const Slice& target) {
- scratch->clear();
- PutVarint32(scratch, static_cast<uint32_t>(target.size()));
- scratch->append(target.data(), target.size());
- return scratch->data();
-}
-
-class MemTableIterator: public Iterator {
- public:
- explicit MemTableIterator(MemTable::Table* table) : iter_(table) { }
-
- virtual bool Valid() const { return iter_.Valid(); }
- virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); }
- virtual void SeekToFirst() { iter_.SeekToFirst(); }
- virtual void SeekToLast() { iter_.SeekToLast(); }
- virtual void Next() { iter_.Next(); }
- virtual void Prev() { iter_.Prev(); }
- virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); }
- virtual Slice value() const {
- Slice key_slice = GetLengthPrefixedSlice(iter_.key());
- return GetLengthPrefixedSlice(key_slice.data() + key_slice.size());
- }
-
- virtual Status status() const { return Status::OK(); }
-
- private:
- MemTable::Table::Iterator iter_;
- std::string tmp_; // For passing to EncodeKey
-
- // No copying allowed
- MemTableIterator(const MemTableIterator&);
- void operator=(const MemTableIterator&);
-};
-
-Iterator* MemTable::NewIterator() {
- return new MemTableIterator(&table_);
-}
-
-void MemTable::Add(SequenceNumber s, ValueType type,
- const Slice& key,
- const Slice& value) {
- // Format of an entry is concatenation of:
- // key_size : varint32 of internal_key.size()
- // key bytes : char[internal_key.size()]
- // value_size : varint32 of value.size()
- // value bytes : char[value.size()]
- uint32_t key_size = static_cast<uint32_t>(key.size());
- uint32_t val_size = static_cast<uint32_t>(value.size());
- uint32_t internal_key_size = key_size + 8;
- const size_t encoded_len =
- VarintLength(internal_key_size) + internal_key_size +
- VarintLength(val_size) + val_size;
- char* buf = arena_.Allocate(encoded_len);
- char* p = EncodeVarint32(buf, internal_key_size);
- memcpy(p, key.data(), key_size);
- p += key_size;
- EncodeFixed64(p, (s << 8) | type);
- p += 8;
- p = EncodeVarint32(p, val_size);
- memcpy(p, value.data(), val_size);
- assert(static_cast<size_t>((p + val_size) - buf) == encoded_len);
- table_.Insert(buf);
-}
-
-bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
- Slice memkey = key.memtable_key();
- Table::Iterator iter(&table_);
- iter.Seek(memkey.data());
- if (iter.Valid()) {
- // entry format is:
- // klength varint32
- // userkey char[klength]
- // tag uint64
- // vlength varint32
- // value char[vlength]
- // Check that it belongs to same user key. We do not check the
- // sequence number since the Seek() call above should have skipped
- // all entries with overly large sequence numbers.
- const char* entry = iter.key();
- uint32_t key_length;
- const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length);
- if (comparator_.comparator.user_comparator()->Compare(
- Slice(key_ptr, key_length - 8),
- key.user_key()) == 0) {
- // Correct user key
- const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
- switch (static_cast<ValueType>(tag & 0xff)) {
- case kTypeValue: {
- Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
- value->assign(v.data(), v.size());
- return true;
- }
- case kTypeDeletion:
- *s = Status::NotFound(Slice());
- return true;
- }
- }
- }
- return false;
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/memtable.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/memtable.h b/thirdparty/leveldb-1.18/db/memtable.h
deleted file mode 100755
index 92e90bb..0000000
--- a/thirdparty/leveldb-1.18/db/memtable.h
+++ /dev/null
@@ -1,91 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_MEMTABLE_H_
-#define STORAGE_LEVELDB_DB_MEMTABLE_H_
-
-#include <string>
-#include "leveldb/db.h"
-#include "db/dbformat.h"
-#include "db/skiplist.h"
-#include "util/arena.h"
-
-namespace leveldb {
-
-class InternalKeyComparator;
-class Mutex;
-class MemTableIterator;
-
-class MemTable {
- public:
- // MemTables are reference counted. The initial reference count
- // is zero and the caller must call Ref() at least once.
- explicit MemTable(const InternalKeyComparator& comparator);
-
- // Increase reference count.
- void Ref() { ++refs_; }
-
- // Drop reference count. Delete if no more references exist.
- void Unref() {
- --refs_;
- assert(refs_ >= 0);
- if (refs_ <= 0) {
- delete this;
- }
- }
-
- // Returns an estimate of the number of bytes of data in use by this
- // data structure.
- //
- // REQUIRES: external synchronization to prevent simultaneous
- // operations on the same MemTable.
- size_t ApproximateMemoryUsage();
-
- // Return an iterator that yields the contents of the memtable.
- //
- // The caller must ensure that the underlying MemTable remains live
- // while the returned iterator is live. The keys returned by this
- // iterator are internal keys encoded by AppendInternalKey in the
- // db/format.{h,cc} module.
- Iterator* NewIterator();
-
- // Add an entry into memtable that maps key to value at the
- // specified sequence number and with the specified type.
- // Typically value will be empty if type==kTypeDeletion.
- void Add(SequenceNumber seq, ValueType type,
- const Slice& key,
- const Slice& value);
-
- // If memtable contains a value for key, store it in *value and return true.
- // If memtable contains a deletion for key, store a NotFound() error
- // in *status and return true.
- // Else, return false.
- bool Get(const LookupKey& key, std::string* value, Status* s);
-
- private:
- ~MemTable(); // Private since only Unref() should be used to delete it
-
- struct KeyComparator {
- const InternalKeyComparator comparator;
- explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { }
- int operator()(const char* a, const char* b) const;
- };
- friend class MemTableIterator;
- friend class MemTableBackwardIterator;
-
- typedef SkipList<const char*, KeyComparator> Table;
-
- KeyComparator comparator_;
- int refs_;
- Arena arena_;
- Table table_;
-
- // No copying allowed
- MemTable(const MemTable&);
- void operator=(const MemTable&);
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_MEMTABLE_H_
[40/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/filter_policy.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/filter_policy.h b/thirdparty/leveldb-1.18/include/leveldb/filter_policy.h
deleted file mode 100755
index 1fba080..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/filter_policy.h
+++ /dev/null
@@ -1,70 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A database can be configured with a custom FilterPolicy object.
-// This object is responsible for creating a small filter from a set
-// of keys. These filters are stored in leveldb and are consulted
-// automatically by leveldb to decide whether or not to read some
-// information from disk. In many cases, a filter can cut down the
-// number of disk seeks form a handful to a single disk seek per
-// DB::Get() call.
-//
-// Most people will want to use the builtin bloom filter support (see
-// NewBloomFilterPolicy() below).
-
-#ifndef STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
-#define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
-
-#include <string>
-
-namespace leveldb {
-
-class Slice;
-
-class FilterPolicy {
- public:
- virtual ~FilterPolicy();
-
- // Return the name of this policy. Note that if the filter encoding
- // changes in an incompatible way, the name returned by this method
- // must be changed. Otherwise, old incompatible filters may be
- // passed to methods of this type.
- virtual const char* Name() const = 0;
-
- // keys[0,n-1] contains a list of keys (potentially with duplicates)
- // that are ordered according to the user supplied comparator.
- // Append a filter that summarizes keys[0,n-1] to *dst.
- //
- // Warning: do not change the initial contents of *dst. Instead,
- // append the newly constructed filter to *dst.
- virtual void CreateFilter(const Slice* keys, int n, std::string* dst)
- const = 0;
-
- // "filter" contains the data appended by a preceding call to
- // CreateFilter() on this class. This method must return true if
- // the key was in the list of keys passed to CreateFilter().
- // This method may return true or false if the key was not on the
- // list, but it should aim to return false with a high probability.
- virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const = 0;
-};
-
-// Return a new filter policy that uses a bloom filter with approximately
-// the specified number of bits per key. A good value for bits_per_key
-// is 10, which yields a filter with ~ 1% false positive rate.
-//
-// Callers must delete the result after any database that is using the
-// result has been closed.
-//
-// Note: if you are using a custom comparator that ignores some parts
-// of the keys being compared, you must not use NewBloomFilterPolicy()
-// and must provide your own FilterPolicy that also ignores the
-// corresponding parts of the keys. For example, if the comparator
-// ignores trailing spaces, it would be incorrect to use a
-// FilterPolicy (like NewBloomFilterPolicy) that does not ignore
-// trailing spaces in keys.
-extern const FilterPolicy* NewBloomFilterPolicy(int bits_per_key);
-
-}
-
-#endif // STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/iterator.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/iterator.h b/thirdparty/leveldb-1.18/include/leveldb/iterator.h
deleted file mode 100755
index 76aced0..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/iterator.h
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// An iterator yields a sequence of key/value pairs from a source.
-// The following class defines the interface. Multiple implementations
-// are provided by this library. In particular, iterators are provided
-// to access the contents of a Table or a DB.
-//
-// Multiple threads can invoke const methods on an Iterator without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Iterator must use
-// external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
-#define STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
-
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class Iterator {
- public:
- Iterator();
- virtual ~Iterator();
-
- // An iterator is either positioned at a key/value pair, or
- // not valid. This method returns true iff the iterator is valid.
- virtual bool Valid() const = 0;
-
- // Position at the first key in the source. The iterator is Valid()
- // after this call iff the source is not empty.
- virtual void SeekToFirst() = 0;
-
- // Position at the last key in the source. The iterator is
- // Valid() after this call iff the source is not empty.
- virtual void SeekToLast() = 0;
-
- // Position at the first key in the source that at or past target
- // The iterator is Valid() after this call iff the source contains
- // an entry that comes at or past target.
- virtual void Seek(const Slice& target) = 0;
-
- // Moves to the next entry in the source. After this call, Valid() is
- // true iff the iterator was not positioned at the last entry in the source.
- // REQUIRES: Valid()
- virtual void Next() = 0;
-
- // Moves to the previous entry in the source. After this call, Valid() is
- // true iff the iterator was not positioned at the first entry in source.
- // REQUIRES: Valid()
- virtual void Prev() = 0;
-
- // Return the key for the current entry. The underlying storage for
- // the returned slice is valid only until the next modification of
- // the iterator.
- // REQUIRES: Valid()
- virtual Slice key() const = 0;
-
- // Return the value for the current entry. The underlying storage for
- // the returned slice is valid only until the next modification of
- // the iterator.
- // REQUIRES: Valid()
- virtual Slice value() const = 0;
-
- // If an error has occurred, return it. Else return an ok status.
- virtual Status status() const = 0;
-
- // Clients are allowed to register function/arg1/arg2 triples that
- // will be invoked when this iterator is destroyed.
- //
- // Note that unlike all of the preceding methods, this method is
- // not abstract and therefore clients should not override it.
- typedef void (*CleanupFunction)(void* arg1, void* arg2);
- void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2);
-
- private:
- struct Cleanup {
- CleanupFunction function;
- void* arg1;
- void* arg2;
- Cleanup* next;
- };
- Cleanup cleanup_;
-
- // No copying allowed
- Iterator(const Iterator&);
- void operator=(const Iterator&);
-};
-
-// Return an empty iterator (yields nothing).
-extern Iterator* NewEmptyIterator();
-
-// Return an empty iterator with the specified status.
-extern Iterator* NewErrorIterator(const Status& status);
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/options.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/options.h b/thirdparty/leveldb-1.18/include/leveldb/options.h
deleted file mode 100755
index 7c9b973..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/options.h
+++ /dev/null
@@ -1,195 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
-#define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
-
-#include <stddef.h>
-
-namespace leveldb {
-
-class Cache;
-class Comparator;
-class Env;
-class FilterPolicy;
-class Logger;
-class Snapshot;
-
-// DB contents are stored in a set of blocks, each of which holds a
-// sequence of key,value pairs. Each block may be compressed before
-// being stored in a file. The following enum describes which
-// compression method (if any) is used to compress a block.
-enum CompressionType {
- // NOTE: do not change the values of existing entries, as these are
- // part of the persistent format on disk.
- kNoCompression = 0x0,
- kSnappyCompression = 0x1
-};
-
-// Options to control the behavior of a database (passed to DB::Open)
-struct Options {
- // -------------------
- // Parameters that affect behavior
-
- // Comparator used to define the order of keys in the table.
- // Default: a comparator that uses lexicographic byte-wise ordering
- //
- // REQUIRES: The client must ensure that the comparator supplied
- // here has the same name and orders keys *exactly* the same as the
- // comparator provided to previous open calls on the same DB.
- const Comparator* comparator;
-
- // If true, the database will be created if it is missing.
- // Default: false
- bool create_if_missing;
-
- // If true, an error is raised if the database already exists.
- // Default: false
- bool error_if_exists;
-
- // If true, the implementation will do aggressive checking of the
- // data it is processing and will stop early if it detects any
- // errors. This may have unforeseen ramifications: for example, a
- // corruption of one DB entry may cause a large number of entries to
- // become unreadable or for the entire DB to become unopenable.
- // Default: false
- bool paranoid_checks;
-
- // Use the specified object to interact with the environment,
- // e.g. to read/write files, schedule background work, etc.
- // Default: Env::Default()
- Env* env;
-
- // Any internal progress/error information generated by the db will
- // be written to info_log if it is non-NULL, or to a file stored
- // in the same directory as the DB contents if info_log is NULL.
- // Default: NULL
- Logger* info_log;
-
- // -------------------
- // Parameters that affect performance
-
- // Amount of data to build up in memory (backed by an unsorted log
- // on disk) before converting to a sorted on-disk file.
- //
- // Larger values increase performance, especially during bulk loads.
- // Up to two write buffers may be held in memory at the same time,
- // so you may wish to adjust this parameter to control memory usage.
- // Also, a larger write buffer will result in a longer recovery time
- // the next time the database is opened.
- //
- // Default: 4MB
- size_t write_buffer_size;
-
- // Number of open files that can be used by the DB. You may need to
- // increase this if your database has a large working set (budget
- // one open file per 2MB of working set).
- //
- // Default: 1000
- int max_open_files;
-
- // Control over blocks (user data is stored in a set of blocks, and
- // a block is the unit of reading from disk).
-
- // If non-NULL, use the specified cache for blocks.
- // If NULL, leveldb will automatically create and use an 8MB internal cache.
- // Default: NULL
- Cache* block_cache;
-
- // Approximate size of user data packed per block. Note that the
- // block size specified here corresponds to uncompressed data. The
- // actual size of the unit read from disk may be smaller if
- // compression is enabled. This parameter can be changed dynamically.
- //
- // Default: 4K
- size_t block_size;
-
- // Number of keys between restart points for delta encoding of keys.
- // This parameter can be changed dynamically. Most clients should
- // leave this parameter alone.
- //
- // Default: 16
- int block_restart_interval;
-
- // Compress blocks using the specified compression algorithm. This
- // parameter can be changed dynamically.
- //
- // Default: kSnappyCompression, which gives lightweight but fast
- // compression.
- //
- // Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz:
- // ~200-500MB/s compression
- // ~400-800MB/s decompression
- // Note that these speeds are significantly faster than most
- // persistent storage speeds, and therefore it is typically never
- // worth switching to kNoCompression. Even if the input data is
- // incompressible, the kSnappyCompression implementation will
- // efficiently detect that and will switch to uncompressed mode.
- CompressionType compression;
-
- // If non-NULL, use the specified filter policy to reduce disk reads.
- // Many applications will benefit from passing the result of
- // NewBloomFilterPolicy() here.
- //
- // Default: NULL
- const FilterPolicy* filter_policy;
-
- // Create an Options object with default values for all fields.
- Options();
-};
-
-// Options that control read operations
-struct ReadOptions {
- // If true, all data read from underlying storage will be
- // verified against corresponding checksums.
- // Default: false
- bool verify_checksums;
-
- // Should the data read for this iteration be cached in memory?
- // Callers may wish to set this field to false for bulk scans.
- // Default: true
- bool fill_cache;
-
- // If "snapshot" is non-NULL, read as of the supplied snapshot
- // (which must belong to the DB that is being read and which must
- // not have been released). If "snapshot" is NULL, use an implicit
- // snapshot of the state at the beginning of this read operation.
- // Default: NULL
- const Snapshot* snapshot;
-
- ReadOptions()
- : verify_checksums(false),
- fill_cache(true),
- snapshot(NULL) {
- }
-};
-
-// Options that control write operations
-struct WriteOptions {
- // If true, the write will be flushed from the operating system
- // buffer cache (by calling WritableFile::Sync()) before the write
- // is considered complete. If this flag is true, writes will be
- // slower.
- //
- // If this flag is false, and the machine crashes, some recent
- // writes may be lost. Note that if it is just the process that
- // crashes (i.e., the machine does not reboot), no writes will be
- // lost even if sync==false.
- //
- // In other words, a DB write with sync==false has similar
- // crash semantics as the "write()" system call. A DB write
- // with sync==true has similar crash semantics to a "write()"
- // system call followed by "fsync()".
- //
- // Default: false
- bool sync;
-
- WriteOptions()
- : sync(false) {
- }
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/slice.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/slice.h b/thirdparty/leveldb-1.18/include/leveldb/slice.h
deleted file mode 100755
index bc36798..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/slice.h
+++ /dev/null
@@ -1,109 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Slice is a simple structure containing a pointer into some external
-// storage and a size. The user of a Slice must ensure that the slice
-// is not used after the corresponding external storage has been
-// deallocated.
-//
-// Multiple threads can invoke const methods on a Slice without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Slice must use
-// external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_
-#define STORAGE_LEVELDB_INCLUDE_SLICE_H_
-
-#include <assert.h>
-#include <stddef.h>
-#include <string.h>
-#include <string>
-
-namespace leveldb {
-
-class Slice {
- public:
- // Create an empty slice.
- Slice() : data_(""), size_(0) { }
-
- // Create a slice that refers to d[0,n-1].
- Slice(const char* d, size_t n) : data_(d), size_(n) { }
-
- // Create a slice that refers to the contents of "s"
- Slice(const std::string& s) : data_(s.data()), size_(s.size()) { }
-
- // Create a slice that refers to s[0,strlen(s)-1]
- Slice(const char* s) : data_(s), size_(strlen(s)) { }
-
- // Return a pointer to the beginning of the referenced data
- const char* data() const { return data_; }
-
- // Return the length (in bytes) of the referenced data
- size_t size() const { return size_; }
-
- // Return true iff the length of the referenced data is zero
- bool empty() const { return size_ == 0; }
-
- // Return the ith byte in the referenced data.
- // REQUIRES: n < size()
- char operator[](size_t n) const {
- assert(n < size());
- return data_[n];
- }
-
- // Change this slice to refer to an empty array
- void clear() { data_ = ""; size_ = 0; }
-
- // Drop the first "n" bytes from this slice.
- void remove_prefix(size_t n) {
- assert(n <= size());
- data_ += n;
- size_ -= n;
- }
-
- // Return a string that contains the copy of the referenced data.
- std::string ToString() const { return std::string(data_, size_); }
-
- // Three-way comparison. Returns value:
- // < 0 iff "*this" < "b",
- // == 0 iff "*this" == "b",
- // > 0 iff "*this" > "b"
- int compare(const Slice& b) const;
-
- // Return true iff "x" is a prefix of "*this"
- bool starts_with(const Slice& x) const {
- return ((size_ >= x.size_) &&
- (memcmp(data_, x.data_, x.size_) == 0));
- }
-
- private:
- const char* data_;
- size_t size_;
-
- // Intentionally copyable
-};
-
-inline bool operator==(const Slice& x, const Slice& y) {
- return ((x.size() == y.size()) &&
- (memcmp(x.data(), y.data(), x.size()) == 0));
-}
-
-inline bool operator!=(const Slice& x, const Slice& y) {
- return !(x == y);
-}
-
-inline int Slice::compare(const Slice& b) const {
- const size_t min_len = (size_ < b.size_) ? size_ : b.size_;
- int r = memcmp(data_, b.data_, min_len);
- if (r == 0) {
- if (size_ < b.size_) r = -1;
- else if (size_ > b.size_) r = +1;
- }
- return r;
-}
-
-} // namespace leveldb
-
-
-#endif // STORAGE_LEVELDB_INCLUDE_SLICE_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/status.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/status.h b/thirdparty/leveldb-1.18/include/leveldb/status.h
deleted file mode 100755
index 089d2ab..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/status.h
+++ /dev/null
@@ -1,107 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A Status encapsulates the result of an operation. It may indicate success,
-// or it may indicate an error with an associated error message.
-//
-// Multiple threads can invoke const methods on a Status without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Status must use
-// external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_STATUS_H_
-#define STORAGE_LEVELDB_INCLUDE_STATUS_H_
-
-#include <string>
-#include "leveldb/slice.h"
-
-namespace leveldb {
-
-class Status {
- public:
- // Create a success status.
- Status() : state_(NULL) { }
- ~Status() { delete[] state_; }
-
- // Copy the specified status.
- Status(const Status& s);
- void operator=(const Status& s);
-
- // Return a success status.
- static Status OK() { return Status(); }
-
- // Return error status of an appropriate type.
- static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) {
- return Status(kNotFound, msg, msg2);
- }
- static Status Corruption(const Slice& msg, const Slice& msg2 = Slice()) {
- return Status(kCorruption, msg, msg2);
- }
- static Status NotSupported(const Slice& msg, const Slice& msg2 = Slice()) {
- return Status(kNotSupported, msg, msg2);
- }
- static Status InvalidArgument(const Slice& msg, const Slice& msg2 = Slice()) {
- return Status(kInvalidArgument, msg, msg2);
- }
- static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) {
- return Status(kIOError, msg, msg2);
- }
-
- // Returns true iff the status indicates success.
- bool ok() const { return (state_ == NULL); }
-
- // Returns true iff the status indicates a NotFound error.
- bool IsNotFound() const { return code() == kNotFound; }
-
- // Returns true iff the status indicates a Corruption error.
- bool IsCorruption() const { return code() == kCorruption; }
-
- // Returns true iff the status indicates an IOError.
- bool IsIOError() const { return code() == kIOError; }
-
- // Return a string representation of this status suitable for printing.
- // Returns the string "OK" for success.
- std::string ToString() const;
-
- // b14 hack: expose internal code to allow error code translation
- public:
- // OK status has a NULL state_. Otherwise, state_ is a new[] array
- // of the following form:
- // state_[0..3] == length of message
- // state_[4] == code
- // state_[5..] == message
- const char* state_;
-
- enum Code {
- kOk = 0,
- kNotFound = 1,
- kCorruption = 2,
- kNotSupported = 3,
- kInvalidArgument = 4,
- kIOError = 5
- };
-
- Code code() const {
- return (state_ == NULL) ? kOk : static_cast<Code>(state_[4]);
- }
-
- Status(Code code, const Slice& msg, const Slice& msg2);
- static const char* CopyState(const char* s);
-};
-
-inline Status::Status(const Status& s) {
- state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
-}
-inline void Status::operator=(const Status& s) {
- // The following condition catches both aliasing (when this == &s),
- // and the common case where both s and *this are ok.
- if (state_ != s.state_) {
- delete[] state_;
- state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
- }
-}
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_INCLUDE_STATUS_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/table.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/table.h b/thirdparty/leveldb-1.18/include/leveldb/table.h
deleted file mode 100755
index a9746c3..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/table.h
+++ /dev/null
@@ -1,85 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_TABLE_H_
-#define STORAGE_LEVELDB_INCLUDE_TABLE_H_
-
-#include <stdint.h>
-#include "leveldb/iterator.h"
-
-namespace leveldb {
-
-class Block;
-class BlockHandle;
-class Footer;
-struct Options;
-class RandomAccessFile;
-struct ReadOptions;
-class TableCache;
-
-// A Table is a sorted map from strings to strings. Tables are
-// immutable and persistent. A Table may be safely accessed from
-// multiple threads without external synchronization.
-class Table {
- public:
- // Attempt to open the table that is stored in bytes [0..file_size)
- // of "file", and read the metadata entries necessary to allow
- // retrieving data from the table.
- //
- // If successful, returns ok and sets "*table" to the newly opened
- // table. The client should delete "*table" when no longer needed.
- // If there was an error while initializing the table, sets "*table"
- // to NULL and returns a non-ok status. Does not take ownership of
- // "*source", but the client must ensure that "source" remains live
- // for the duration of the returned table's lifetime.
- //
- // *file must remain live while this Table is in use.
- static Status Open(const Options& options,
- RandomAccessFile* file,
- uint64_t file_size,
- Table** table);
-
- ~Table();
-
- // Returns a new iterator over the table contents.
- // The result of NewIterator() is initially invalid (caller must
- // call one of the Seek methods on the iterator before using it).
- Iterator* NewIterator(const ReadOptions&) const;
-
- // Given a key, return an approximate byte offset in the file where
- // the data for that key begins (or would begin if the key were
- // present in the file). The returned value is in terms of file
- // bytes, and so includes effects like compression of the underlying data.
- // E.g., the approximate offset of the last key in the table will
- // be close to the file length.
- uint64_t ApproximateOffsetOf(const Slice& key) const;
-
- private:
- struct Rep;
- Rep* rep_;
-
- explicit Table(Rep* rep) { rep_ = rep; }
- static Iterator* BlockReader(void*, const ReadOptions&, const Slice&);
-
- // Calls (*handle_result)(arg, ...) with the entry found after a call
- // to Seek(key). May not make such a call if filter policy says
- // that key is not present.
- friend class TableCache;
- Status InternalGet(
- const ReadOptions&, const Slice& key,
- void* arg,
- void (*handle_result)(void* arg, const Slice& k, const Slice& v));
-
-
- void ReadMeta(const Footer& footer);
- void ReadFilter(const Slice& filter_handle_value);
-
- // No copying allowed
- Table(const Table&);
- void operator=(const Table&);
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_INCLUDE_TABLE_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/table_builder.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/table_builder.h b/thirdparty/leveldb-1.18/include/leveldb/table_builder.h
deleted file mode 100755
index 5fd1dc7..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/table_builder.h
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// TableBuilder provides the interface used to build a Table
-// (an immutable and sorted map from keys to values).
-//
-// Multiple threads can invoke const methods on a TableBuilder without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same TableBuilder must use
-// external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
-#define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
-
-#include <stdint.h>
-#include "leveldb/options.h"
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class BlockBuilder;
-class BlockHandle;
-class WritableFile;
-
-class TableBuilder {
- public:
- // Create a builder that will store the contents of the table it is
- // building in *file. Does not close the file. It is up to the
- // caller to close the file after calling Finish().
- TableBuilder(const Options& options, WritableFile* file);
-
- // REQUIRES: Either Finish() or Abandon() has been called.
- ~TableBuilder();
-
- // Change the options used by this builder. Note: only some of the
- // option fields can be changed after construction. If a field is
- // not allowed to change dynamically and its value in the structure
- // passed to the constructor is different from its value in the
- // structure passed to this method, this method will return an error
- // without changing any fields.
- Status ChangeOptions(const Options& options);
-
- // Add key,value to the table being constructed.
- // REQUIRES: key is after any previously added key according to comparator.
- // REQUIRES: Finish(), Abandon() have not been called
- void Add(const Slice& key, const Slice& value);
-
- // Advanced operation: flush any buffered key/value pairs to file.
- // Can be used to ensure that two adjacent entries never live in
- // the same data block. Most clients should not need to use this method.
- // REQUIRES: Finish(), Abandon() have not been called
- void Flush();
-
- // Return non-ok iff some error has been detected.
- Status status() const;
-
- // Finish building the table. Stops using the file passed to the
- // constructor after this function returns.
- // REQUIRES: Finish(), Abandon() have not been called
- Status Finish();
-
- // Indicate that the contents of this builder should be abandoned. Stops
- // using the file passed to the constructor after this function returns.
- // If the caller is not going to call Finish(), it must call Abandon()
- // before destroying this builder.
- // REQUIRES: Finish(), Abandon() have not been called
- void Abandon();
-
- // Number of calls to Add() so far.
- uint64_t NumEntries() const;
-
- // Size of the file generated so far. If invoked after a successful
- // Finish() call, returns the size of the final generated file.
- uint64_t FileSize() const;
-
- private:
- bool ok() const { return status().ok(); }
- void WriteBlock(BlockBuilder* block, BlockHandle* handle);
- void WriteRawBlock(const Slice& data, CompressionType, BlockHandle* handle);
-
- struct Rep;
- Rep* rep_;
-
- // No copying allowed
- TableBuilder(const TableBuilder&);
- void operator=(const TableBuilder&);
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/include/leveldb/write_batch.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/include/leveldb/write_batch.h b/thirdparty/leveldb-1.18/include/leveldb/write_batch.h
deleted file mode 100755
index ee9aab6..0000000
--- a/thirdparty/leveldb-1.18/include/leveldb/write_batch.h
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// WriteBatch holds a collection of updates to apply atomically to a DB.
-//
-// The updates are applied in the order in which they are added
-// to the WriteBatch. For example, the value of "key" will be "v3"
-// after the following batch is written:
-//
-// batch.Put("key", "v1");
-// batch.Delete("key");
-// batch.Put("key", "v2");
-// batch.Put("key", "v3");
-//
-// Multiple threads can invoke const methods on a WriteBatch without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same WriteBatch must use
-// external synchronization.
-
-#ifndef STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
-#define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
-
-#include <string>
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-class Slice;
-
-class WriteBatch {
- public:
- WriteBatch();
- ~WriteBatch();
-
- // Store the mapping "key->value" in the database.
- void Put(const Slice& key, const Slice& value);
-
- // If the database contains a mapping for "key", erase it. Else do nothing.
- void Delete(const Slice& key);
-
- // Clear all updates buffered in this batch.
- void Clear();
-
- // Support for iterating over the contents of a batch.
- class Handler {
- public:
- virtual ~Handler();
- virtual void Put(const Slice& key, const Slice& value) = 0;
- virtual void Delete(const Slice& key) = 0;
- };
- Status Iterate(Handler* handler) const;
-
- private:
- friend class WriteBatchInternal;
-
- std::string rep_; // See comment in write_batch.cc for the format of rep_
-
- // Intentionally copyable
-};
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/issues/issue178_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/issues/issue178_test.cc b/thirdparty/leveldb-1.18/issues/issue178_test.cc
deleted file mode 100755
index 1b1cf8b..0000000
--- a/thirdparty/leveldb-1.18/issues/issue178_test.cc
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright (c) 2013 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-// Test for issue 178: a manual compaction causes deleted data to reappear.
-#include <iostream>
-#include <sstream>
-#include <cstdlib>
-
-#include "leveldb/db.h"
-#include "leveldb/write_batch.h"
-#include "util/testharness.h"
-
-namespace {
-
-const int kNumKeys = 1100000;
-
-std::string Key1(int i) {
- char buf[100];
- snprintf(buf, sizeof(buf), "my_key_%d", i);
- return buf;
-}
-
-std::string Key2(int i) {
- return Key1(i) + "_xxx";
-}
-
-class Issue178 { };
-
-TEST(Issue178, Test) {
- // Get rid of any state from an old run.
- std::string dbpath = leveldb::test::TmpDir() + "/leveldb_cbug_test";
- DestroyDB(dbpath, leveldb::Options());
-
- // Open database. Disable compression since it affects the creation
- // of layers and the code below is trying to test against a very
- // specific scenario.
- leveldb::DB* db;
- leveldb::Options db_options;
- db_options.create_if_missing = true;
- db_options.compression = leveldb::kNoCompression;
- ASSERT_OK(leveldb::DB::Open(db_options, dbpath, &db));
-
- // create first key range
- leveldb::WriteBatch batch;
- for (size_t i = 0; i < kNumKeys; i++) {
- batch.Put(Key1(i), "value for range 1 key");
- }
- ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));
-
- // create second key range
- batch.Clear();
- for (size_t i = 0; i < kNumKeys; i++) {
- batch.Put(Key2(i), "value for range 2 key");
- }
- ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));
-
- // delete second key range
- batch.Clear();
- for (size_t i = 0; i < kNumKeys; i++) {
- batch.Delete(Key2(i));
- }
- ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch));
-
- // compact database
- std::string start_key = Key1(0);
- std::string end_key = Key1(kNumKeys - 1);
- leveldb::Slice least(start_key.data(), start_key.size());
- leveldb::Slice greatest(end_key.data(), end_key.size());
-
- // commenting out the line below causes the example to work correctly
- db->CompactRange(&least, &greatest);
-
- // count the keys
- leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions());
- size_t num_keys = 0;
- for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
- num_keys++;
- }
- delete iter;
- ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys";
-
- // close database
- delete db;
- DestroyDB(dbpath, leveldb::Options());
-}
-
-} // anonymous namespace
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/issues/issue200_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/issues/issue200_test.cc b/thirdparty/leveldb-1.18/issues/issue200_test.cc
deleted file mode 100755
index 1cec79f..0000000
--- a/thirdparty/leveldb-1.18/issues/issue200_test.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-// Copyright (c) 2013 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-// Test for issue 200: when iterator switches direction from backward
-// to forward, the current key can be yielded unexpectedly if a new
-// mutation has been added just before the current key.
-
-#include "leveldb/db.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-class Issue200 { };
-
-TEST(Issue200, Test) {
- // Get rid of any state from an old run.
- std::string dbpath = test::TmpDir() + "/leveldb_issue200_test";
- DestroyDB(dbpath, Options());
-
- DB *db;
- Options options;
- options.create_if_missing = true;
- ASSERT_OK(DB::Open(options, dbpath, &db));
-
- WriteOptions write_options;
- ASSERT_OK(db->Put(write_options, "1", "b"));
- ASSERT_OK(db->Put(write_options, "2", "c"));
- ASSERT_OK(db->Put(write_options, "3", "d"));
- ASSERT_OK(db->Put(write_options, "4", "e"));
- ASSERT_OK(db->Put(write_options, "5", "f"));
-
- ReadOptions read_options;
- Iterator *iter = db->NewIterator(read_options);
-
- // Add an element that should not be reflected in the iterator.
- ASSERT_OK(db->Put(write_options, "25", "cd"));
-
- iter->Seek("5");
- ASSERT_EQ(iter->key().ToString(), "5");
- iter->Prev();
- ASSERT_EQ(iter->key().ToString(), "4");
- iter->Prev();
- ASSERT_EQ(iter->key().ToString(), "3");
- iter->Next();
- ASSERT_EQ(iter->key().ToString(), "4");
- iter->Next();
- ASSERT_EQ(iter->key().ToString(), "5");
-
- delete iter;
- delete db;
- DestroyDB(dbpath, options);
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/README
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/README b/thirdparty/leveldb-1.18/port/README
deleted file mode 100755
index 422563e..0000000
--- a/thirdparty/leveldb-1.18/port/README
+++ /dev/null
@@ -1,10 +0,0 @@
-This directory contains interfaces and implementations that isolate the
-rest of the package from platform details.
-
-Code in the rest of the package includes "port.h" from this directory.
-"port.h" in turn includes a platform specific "port_<platform>.h" file
-that provides the platform specific implementation.
-
-See port_posix.h for an example of what must be provided in a platform
-specific header file.
-
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/atomic_pointer.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/atomic_pointer.h b/thirdparty/leveldb-1.18/port/atomic_pointer.h
deleted file mode 100755
index 9bf091f..0000000
--- a/thirdparty/leveldb-1.18/port/atomic_pointer.h
+++ /dev/null
@@ -1,223 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-// AtomicPointer provides storage for a lock-free pointer.
-// Platform-dependent implementation of AtomicPointer:
-// - If the platform provides a cheap barrier, we use it with raw pointers
-// - If <atomic> is present (on newer versions of gcc, it is), we use
-// a <atomic>-based AtomicPointer. However we prefer the memory
-// barrier based version, because at least on a gcc 4.4 32-bit build
-// on linux, we have encountered a buggy <atomic> implementation.
-// Also, some <atomic> implementations are much slower than a memory-barrier
-// based implementation (~16ns for <atomic> based acquire-load vs. ~1ns for
-// a barrier based acquire-load).
-// This code is based on atomicops-internals-* in Google's perftools:
-// http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase
-
-#ifndef PORT_ATOMIC_POINTER_H_
-#define PORT_ATOMIC_POINTER_H_
-
-#include <stdint.h>
-#ifdef LEVELDB_ATOMIC_PRESENT
-#include <atomic>
-#endif
-#ifdef OS_WIN
-#include <windows.h>
-#endif
-#ifdef OS_MACOSX
-#include <libkern/OSAtomic.h>
-#endif
-
-#if defined(_M_X64) || defined(__x86_64__)
-#define ARCH_CPU_X86_FAMILY 1
-#elif defined(_M_IX86) || defined(__i386__) || defined(__i386)
-#define ARCH_CPU_X86_FAMILY 1
-#elif defined(__ARMEL__)
-#define ARCH_CPU_ARM_FAMILY 1
-#elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__)
-#define ARCH_CPU_PPC_FAMILY 1
-#endif
-
-namespace leveldb {
-namespace port {
-
-// Define MemoryBarrier() if available
-// Windows on x86
-#if defined(OS_WIN) && defined(COMPILER_MSVC) && defined(ARCH_CPU_X86_FAMILY)
-// windows.h already provides a MemoryBarrier(void) macro
-// http://msdn.microsoft.com/en-us/library/ms684208(v=vs.85).aspx
-#define LEVELDB_HAVE_MEMORY_BARRIER
-
-// Mac OS
-#elif defined(OS_MACOSX)
-inline void MemoryBarrier() {
- OSMemoryBarrier();
-}
-#define LEVELDB_HAVE_MEMORY_BARRIER
-
-// Gcc on x86
-#elif defined(ARCH_CPU_X86_FAMILY) && defined(__GNUC__)
-inline void MemoryBarrier() {
- // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on
- // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering.
- __asm__ __volatile__("" : : : "memory");
-}
-#define LEVELDB_HAVE_MEMORY_BARRIER
-
-// Sun Studio
-#elif defined(ARCH_CPU_X86_FAMILY) && defined(__SUNPRO_CC)
-inline void MemoryBarrier() {
- // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on
- // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering.
- asm volatile("" : : : "memory");
-}
-#define LEVELDB_HAVE_MEMORY_BARRIER
-
-// ARM Linux
-#elif defined(ARCH_CPU_ARM_FAMILY) && defined(__linux__)
-typedef void (*LinuxKernelMemoryBarrierFunc)(void);
-// The Linux ARM kernel provides a highly optimized device-specific memory
-// barrier function at a fixed memory address that is mapped in every
-// user-level process.
-//
-// This beats using CPU-specific instructions which are, on single-core
-// devices, un-necessary and very costly (e.g. ARMv7-A "dmb" takes more
-// than 180ns on a Cortex-A8 like the one on a Nexus One). Benchmarking
-// shows that the extra function call cost is completely negligible on
-// multi-core devices.
-//
-inline void MemoryBarrier() {
- (*(LinuxKernelMemoryBarrierFunc)0xffff0fa0)();
-}
-#define LEVELDB_HAVE_MEMORY_BARRIER
-
-// PPC
-#elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__)
-inline void MemoryBarrier() {
- // TODO for some powerpc expert: is there a cheaper suitable variant?
- // Perhaps by having separate barriers for acquire and release ops.
- asm volatile("sync" : : : "memory");
-}
-#define LEVELDB_HAVE_MEMORY_BARRIER
-
-#endif
-
-// AtomicPointer built using platform-specific MemoryBarrier()
-#if defined(LEVELDB_HAVE_MEMORY_BARRIER)
-class AtomicPointer {
- private:
- void* rep_;
- public:
- AtomicPointer() { }
- explicit AtomicPointer(void* p) : rep_(p) {}
- inline void* NoBarrier_Load() const { return rep_; }
- inline void NoBarrier_Store(void* v) { rep_ = v; }
- inline void* Acquire_Load() const {
- void* result = rep_;
- MemoryBarrier();
- return result;
- }
- inline void Release_Store(void* v) {
- MemoryBarrier();
- rep_ = v;
- }
-};
-
-// AtomicPointer based on <cstdatomic>
-#elif defined(LEVELDB_ATOMIC_PRESENT)
-class AtomicPointer {
- private:
- std::atomic<void*> rep_;
- public:
- AtomicPointer() { }
- explicit AtomicPointer(void* v) : rep_(v) { }
- inline void* Acquire_Load() const {
- return rep_.load(std::memory_order_acquire);
- }
- inline void Release_Store(void* v) {
- rep_.store(v, std::memory_order_release);
- }
- inline void* NoBarrier_Load() const {
- return rep_.load(std::memory_order_relaxed);
- }
- inline void NoBarrier_Store(void* v) {
- rep_.store(v, std::memory_order_relaxed);
- }
-};
-
-// Atomic pointer based on sparc memory barriers
-#elif defined(__sparcv9) && defined(__GNUC__)
-class AtomicPointer {
- private:
- void* rep_;
- public:
- AtomicPointer() { }
- explicit AtomicPointer(void* v) : rep_(v) { }
- inline void* Acquire_Load() const {
- void* val;
- __asm__ __volatile__ (
- "ldx [%[rep_]], %[val] \n\t"
- "membar #LoadLoad|#LoadStore \n\t"
- : [val] "=r" (val)
- : [rep_] "r" (&rep_)
- : "memory");
- return val;
- }
- inline void Release_Store(void* v) {
- __asm__ __volatile__ (
- "membar #LoadStore|#StoreStore \n\t"
- "stx %[v], [%[rep_]] \n\t"
- :
- : [rep_] "r" (&rep_), [v] "r" (v)
- : "memory");
- }
- inline void* NoBarrier_Load() const { return rep_; }
- inline void NoBarrier_Store(void* v) { rep_ = v; }
-};
-
-// Atomic pointer based on ia64 acq/rel
-#elif defined(__ia64) && defined(__GNUC__)
-class AtomicPointer {
- private:
- void* rep_;
- public:
- AtomicPointer() { }
- explicit AtomicPointer(void* v) : rep_(v) { }
- inline void* Acquire_Load() const {
- void* val ;
- __asm__ __volatile__ (
- "ld8.acq %[val] = [%[rep_]] \n\t"
- : [val] "=r" (val)
- : [rep_] "r" (&rep_)
- : "memory"
- );
- return val;
- }
- inline void Release_Store(void* v) {
- __asm__ __volatile__ (
- "st8.rel [%[rep_]] = %[v] \n\t"
- :
- : [rep_] "r" (&rep_), [v] "r" (v)
- : "memory"
- );
- }
- inline void* NoBarrier_Load() const { return rep_; }
- inline void NoBarrier_Store(void* v) { rep_ = v; }
-};
-
-// We have neither MemoryBarrier(), nor <atomic>
-#else
-#error Please implement AtomicPointer for this platform.
-
-#endif
-
-#undef LEVELDB_HAVE_MEMORY_BARRIER
-#undef ARCH_CPU_X86_FAMILY
-#undef ARCH_CPU_ARM_FAMILY
-#undef ARCH_CPU_PPC_FAMILY
-
-} // namespace port
-} // namespace leveldb
-
-#endif // PORT_ATOMIC_POINTER_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/port.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/port.h b/thirdparty/leveldb-1.18/port/port.h
deleted file mode 100755
index 26f0660..0000000
--- a/thirdparty/leveldb-1.18/port/port.h
+++ /dev/null
@@ -1,16 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_PORT_PORT_H_
-#define STORAGE_LEVELDB_PORT_PORT_H_
-
-#include <string.h>
-
-#ifdef WIN32
-#include "port/port_win.h"
-#else
-#include "port/port_posix.h"
-#endif
-
-#endif // STORAGE_LEVELDB_PORT_PORT_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/port_android.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/port_android.cc b/thirdparty/leveldb-1.18/port/port_android.cc
deleted file mode 100755
index 815abf2..0000000
--- a/thirdparty/leveldb-1.18/port/port_android.cc
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "port/port_android.h"
-
-#include <cstdlib>
-
-extern "C" {
-size_t fread_unlocked(void *a, size_t b, size_t c, FILE *d) {
- return fread(a, b, c, d);
-}
-
-size_t fwrite_unlocked(const void *a, size_t b, size_t c, FILE *d) {
- return fwrite(a, b, c, d);
-}
-
-int fflush_unlocked(FILE *f) {
- return fflush(f);
-}
-
-int fdatasync(int fd) {
- return fsync(fd);
-}
-}
-
-namespace leveldb {
-namespace port {
-
-static void PthreadCall(const char* label, int result) {
- if (result != 0) {
- fprintf(stderr, "pthread %s: %s\n", label, strerror(result));
- abort();
- }
-}
-
-Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); }
-Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); }
-void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); }
-void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); }
-
-CondVar::CondVar(Mutex* mu)
- : mu_(mu) {
- PthreadCall("init cv", pthread_cond_init(&cv_, NULL));
-}
-
-CondVar::~CondVar() {
- PthreadCall("destroy cv", pthread_cond_destroy(&cv_));
-}
-
-void CondVar::Wait() {
- PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_));
-}
-
-void CondVar::Signal(){
- PthreadCall("signal", pthread_cond_signal(&cv_));
-}
-
-void CondVar::SignalAll() {
- PthreadCall("broadcast", pthread_cond_broadcast(&cv_));
-}
-
-} // namespace port
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/port_android.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/port_android.h b/thirdparty/leveldb-1.18/port/port_android.h
deleted file mode 100755
index b733388..0000000
--- a/thirdparty/leveldb-1.18/port/port_android.h
+++ /dev/null
@@ -1,159 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// See port_example.h for documentation for the following types/functions.
-
-#ifndef STORAGE_LEVELDB_PORT_PORT_ANDROID_H_
-#define STORAGE_LEVELDB_PORT_PORT_ANDROID_H_
-
-#include <endian.h>
-#include <pthread.h>
-#include <stdint.h>
-#include <unistd.h>
-#include <string>
-#include <cctype>
-
-// Collapse the plethora of ARM flavors available to an easier to manage set
-// Defs reference is at https://wiki.edubuntu.org/ARM/Thumb2PortingHowto
-#if defined(__ARM_ARCH_6__) || \
- defined(__ARM_ARCH_6J__) || \
- defined(__ARM_ARCH_6K__) || \
- defined(__ARM_ARCH_6Z__) || \
- defined(__ARM_ARCH_6T2__) || \
- defined(__ARM_ARCH_6ZK__) || \
- defined(__ARM_ARCH_7__) || \
- defined(__ARM_ARCH_7R__) || \
- defined(__ARM_ARCH_7A__)
-#define ARMV6_OR_7 1
-#endif
-
-extern "C" {
- size_t fread_unlocked(void *a, size_t b, size_t c, FILE *d);
- size_t fwrite_unlocked(const void *a, size_t b, size_t c, FILE *d);
- int fflush_unlocked(FILE *f);
- int fdatasync (int fd);
-}
-
-namespace leveldb {
-namespace port {
-
-static const bool kLittleEndian = __BYTE_ORDER == __LITTLE_ENDIAN;
-
-class CondVar;
-
-class Mutex {
- public:
- Mutex();
- ~Mutex();
-
- void Lock();
- void Unlock();
- void AssertHeld() {
- //TODO(gabor): How can I implement this?
- }
-
- private:
- friend class CondVar;
- pthread_mutex_t mu_;
-
- // No copying
- Mutex(const Mutex&);
- void operator=(const Mutex&);
-};
-
-class CondVar {
- public:
- explicit CondVar(Mutex* mu);
- ~CondVar();
- void Wait();
- void Signal();
- void SignalAll();
- private:
- Mutex* mu_;
- pthread_cond_t cv_;
-};
-
-#ifndef ARMV6_OR_7
-// On ARM chipsets <V6, 0xffff0fa0 is the hard coded address of a
-// memory barrier function provided by the kernel.
-typedef void (*LinuxKernelMemoryBarrierFunc)(void);
-// TODO(user): ATTRIBUTE_WEAK is undefined, so this fails to build on
-// non-ARMV6_OR_7. We may be able to replace it with __attribute__((weak)) for
-// older ARM builds, but x86 builds will require a different memory barrier.
-LinuxKernelMemoryBarrierFunc pLinuxKernelMemoryBarrier ATTRIBUTE_WEAK =
- (LinuxKernelMemoryBarrierFunc) 0xffff0fa0;
-#endif
-
-// Storage for a lock-free pointer
-class AtomicPointer {
- private:
- void* rep_;
-
- inline void MemoryBarrier() const {
- // TODO(gabor): This only works on Android instruction sets >= V6
-#ifdef ARMV6_OR_7
- __asm__ __volatile__("dmb" : : : "memory");
-#else
- pLinuxKernelMemoryBarrier();
-#endif
- }
-
- public:
- AtomicPointer() { }
- explicit AtomicPointer(void* v) : rep_(v) { }
- inline void* Acquire_Load() const {
- void* r = rep_;
- MemoryBarrier();
- return r;
- }
- inline void Release_Store(void* v) {
- MemoryBarrier();
- rep_ = v;
- }
- inline void* NoBarrier_Load() const {
- void* r = rep_;
- return r;
- }
- inline void NoBarrier_Store(void* v) {
- rep_ = v;
- }
-};
-
-// TODO(gabor): Implement compress
-inline bool Snappy_Compress(
- const char* input,
- size_t input_length,
- std::string* output) {
- return false;
-}
-
-// TODO(gabor): Implement uncompress
-inline bool Snappy_GetUncompressedLength(const char* input, size_t length,
- size_t* result) {
- return false;
-}
-
-// TODO(gabor): Implement uncompress
-inline bool Snappy_Uncompress(
- const char* input_data,
- size_t input_length,
- char* output) {
- return false;
-}
-
-inline uint64_t ThreadIdentifier() {
- pthread_t tid = pthread_self();
- uint64_t r = 0;
- memcpy(&r, &tid, sizeof(r) < sizeof(tid) ? sizeof(r) : sizeof(tid));
- return r;
-}
-
-inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) {
- return false;
-}
-
-} // namespace port
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_PORT_PORT_ANDROID_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/port_example.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/port_example.h b/thirdparty/leveldb-1.18/port/port_example.h
deleted file mode 100755
index ab9e489..0000000
--- a/thirdparty/leveldb-1.18/port/port_example.h
+++ /dev/null
@@ -1,135 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// This file contains the specification, but not the implementations,
-// of the types/operations/etc. that should be defined by a platform
-// specific port_<platform>.h file. Use this file as a reference for
-// how to port this package to a new platform.
-
-#ifndef STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_
-#define STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_
-
-namespace leveldb {
-namespace port {
-
-// TODO(jorlow): Many of these belong more in the environment class rather than
-// here. We should try moving them and see if it affects perf.
-
-// The following boolean constant must be true on a little-endian machine
-// and false otherwise.
-static const bool kLittleEndian = true /* or some other expression */;
-
-// ------------------ Threading -------------------
-
-// A Mutex represents an exclusive lock.
-class Mutex {
- public:
- Mutex();
- ~Mutex();
-
- // Lock the mutex. Waits until other lockers have exited.
- // Will deadlock if the mutex is already locked by this thread.
- void Lock();
-
- // Unlock the mutex.
- // REQUIRES: This mutex was locked by this thread.
- void Unlock();
-
- // Optionally crash if this thread does not hold this mutex.
- // The implementation must be fast, especially if NDEBUG is
- // defined. The implementation is allowed to skip all checks.
- void AssertHeld();
-};
-
-class CondVar {
- public:
- explicit CondVar(Mutex* mu);
- ~CondVar();
-
- // Atomically release *mu and block on this condition variable until
- // either a call to SignalAll(), or a call to Signal() that picks
- // this thread to wakeup.
- // REQUIRES: this thread holds *mu
- void Wait();
-
- // If there are some threads waiting, wake up at least one of them.
- void Signal();
-
- // Wake up all waiting threads.
- void SignallAll();
-};
-
-// Thread-safe initialization.
-// Used as follows:
-// static port::OnceType init_control = LEVELDB_ONCE_INIT;
-// static void Initializer() { ... do something ...; }
-// ...
-// port::InitOnce(&init_control, &Initializer);
-typedef intptr_t OnceType;
-#define LEVELDB_ONCE_INIT 0
-extern void InitOnce(port::OnceType*, void (*initializer)());
-
-// A type that holds a pointer that can be read or written atomically
-// (i.e., without word-tearing.)
-class AtomicPointer {
- private:
- intptr_t rep_;
- public:
- // Initialize to arbitrary value
- AtomicPointer();
-
- // Initialize to hold v
- explicit AtomicPointer(void* v) : rep_(v) { }
-
- // Read and return the stored pointer with the guarantee that no
- // later memory access (read or write) by this thread can be
- // reordered ahead of this read.
- void* Acquire_Load() const;
-
- // Set v as the stored pointer with the guarantee that no earlier
- // memory access (read or write) by this thread can be reordered
- // after this store.
- void Release_Store(void* v);
-
- // Read the stored pointer with no ordering guarantees.
- void* NoBarrier_Load() const;
-
- // Set va as the stored pointer with no ordering guarantees.
- void NoBarrier_Store(void* v);
-};
-
-// ------------------ Compression -------------------
-
-// Store the snappy compression of "input[0,input_length-1]" in *output.
-// Returns false if snappy is not supported by this port.
-extern bool Snappy_Compress(const char* input, size_t input_length,
- std::string* output);
-
-// If input[0,input_length-1] looks like a valid snappy compressed
-// buffer, store the size of the uncompressed data in *result and
-// return true. Else return false.
-extern bool Snappy_GetUncompressedLength(const char* input, size_t length,
- size_t* result);
-
-// Attempt to snappy uncompress input[0,input_length-1] into *output.
-// Returns true if successful, false if the input is invalid lightweight
-// compressed data.
-//
-// REQUIRES: at least the first "n" bytes of output[] must be writable
-// where "n" is the result of a successful call to
-// Snappy_GetUncompressedLength.
-extern bool Snappy_Uncompress(const char* input_data, size_t input_length,
- char* output);
-
-// ------------------ Miscellaneous -------------------
-
-// If heap profiling is not supported, returns false.
-// Else repeatedly calls (*func)(arg, data, n) and then returns true.
-// The concatenation of all "data[0,n-1]" fragments is the heap profile.
-extern bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg);
-
-} // namespace port
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/port_posix.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/port_posix.cc b/thirdparty/leveldb-1.18/port/port_posix.cc
deleted file mode 100755
index 5ba127a..0000000
--- a/thirdparty/leveldb-1.18/port/port_posix.cc
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "port/port_posix.h"
-
-#include <cstdlib>
-#include <stdio.h>
-#include <string.h>
-#include "util/logging.h"
-
-namespace leveldb {
-namespace port {
-
-static void PthreadCall(const char* label, int result) {
- if (result != 0) {
- fprintf(stderr, "pthread %s: %s\n", label, strerror(result));
- abort();
- }
-}
-
-Mutex::Mutex() { PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); }
-
-Mutex::~Mutex() { PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); }
-
-void Mutex::Lock() { PthreadCall("lock", pthread_mutex_lock(&mu_)); }
-
-void Mutex::Unlock() { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); }
-
-CondVar::CondVar(Mutex* mu)
- : mu_(mu) {
- PthreadCall("init cv", pthread_cond_init(&cv_, NULL));
-}
-
-CondVar::~CondVar() { PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); }
-
-void CondVar::Wait() {
- PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_));
-}
-
-void CondVar::Signal() {
- PthreadCall("signal", pthread_cond_signal(&cv_));
-}
-
-void CondVar::SignalAll() {
- PthreadCall("broadcast", pthread_cond_broadcast(&cv_));
-}
-
-void InitOnce(OnceType* once, void (*initializer)()) {
- PthreadCall("once", pthread_once(once, initializer));
-}
-
-} // namespace port
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/port_posix.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/port_posix.h b/thirdparty/leveldb-1.18/port/port_posix.h
deleted file mode 100755
index 89fc222..0000000
--- a/thirdparty/leveldb-1.18/port/port_posix.h
+++ /dev/null
@@ -1,154 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// See port_example.h for documentation for the following types/functions.
-
-#ifndef STORAGE_LEVELDB_PORT_PORT_POSIX_H_
-#define STORAGE_LEVELDB_PORT_PORT_POSIX_H_
-
-#undef PLATFORM_IS_LITTLE_ENDIAN
-#if defined(OS_MACOSX)
- #include <machine/endian.h>
- #if defined(__DARWIN_LITTLE_ENDIAN) && defined(__DARWIN_BYTE_ORDER)
- #define PLATFORM_IS_LITTLE_ENDIAN \
- (__DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN)
- #endif
-#elif defined(OS_SOLARIS)
- #include <sys/isa_defs.h>
- #ifdef _LITTLE_ENDIAN
- #define PLATFORM_IS_LITTLE_ENDIAN true
- #else
- #define PLATFORM_IS_LITTLE_ENDIAN false
- #endif
-#elif defined(OS_FREEBSD) || defined(OS_OPENBSD) ||\
- defined(OS_NETBSD) || defined(OS_DRAGONFLYBSD)
- #include <sys/types.h>
- #include <sys/endian.h>
- #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN)
-#elif defined(OS_HPUX)
- #define PLATFORM_IS_LITTLE_ENDIAN false
-#elif defined(OS_ANDROID)
- // Due to a bug in the NDK x86 <sys/endian.h> definition,
- // _BYTE_ORDER must be used instead of __BYTE_ORDER on Android.
- // See http://code.google.com/p/android/issues/detail?id=39824
- #include <endian.h>
- #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN)
-#else
- #include <endian.h>
-#endif
-
-#include <pthread.h>
-#ifdef SNAPPY
-#include <snappy.h>
-#endif
-#include <stdint.h>
-#include <string>
-#include "port/atomic_pointer.h"
-
-#ifndef PLATFORM_IS_LITTLE_ENDIAN
-#define PLATFORM_IS_LITTLE_ENDIAN (__BYTE_ORDER == __LITTLE_ENDIAN)
-#endif
-
-#if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\
- defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\
- defined(OS_ANDROID) || defined(OS_HPUX) || defined(CYGWIN)
-// Use fread/fwrite/fflush on platforms without _unlocked variants
-#define fread_unlocked fread
-#define fwrite_unlocked fwrite
-#define fflush_unlocked fflush
-#endif
-
-#if defined(OS_MACOSX) || defined(OS_FREEBSD) ||\
- defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD)
-// Use fsync() on platforms without fdatasync()
-#define fdatasync fsync
-#endif
-
-#if defined(OS_ANDROID) && __ANDROID_API__ < 9
-// fdatasync() was only introduced in API level 9 on Android. Use fsync()
-// when targetting older platforms.
-#define fdatasync fsync
-#endif
-
-namespace leveldb {
-namespace port {
-
-static const bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN;
-#undef PLATFORM_IS_LITTLE_ENDIAN
-
-class CondVar;
-
-class Mutex {
- public:
- Mutex();
- ~Mutex();
-
- void Lock();
- void Unlock();
- void AssertHeld() { }
-
- private:
- friend class CondVar;
- pthread_mutex_t mu_;
-
- // No copying
- Mutex(const Mutex&);
- void operator=(const Mutex&);
-};
-
-class CondVar {
- public:
- explicit CondVar(Mutex* mu);
- ~CondVar();
- void Wait();
- void Signal();
- void SignalAll();
- private:
- pthread_cond_t cv_;
- Mutex* mu_;
-};
-
-typedef pthread_once_t OnceType;
-#define LEVELDB_ONCE_INIT PTHREAD_ONCE_INIT
-extern void InitOnce(OnceType* once, void (*initializer)());
-
-inline bool Snappy_Compress(const char* input, size_t length,
- ::std::string* output) {
-#ifdef SNAPPY
- output->resize(snappy::MaxCompressedLength(length));
- size_t outlen;
- snappy::RawCompress(input, length, &(*output)[0], &outlen);
- output->resize(outlen);
- return true;
-#endif
-
- return false;
-}
-
-inline bool Snappy_GetUncompressedLength(const char* input, size_t length,
- size_t* result) {
-#ifdef SNAPPY
- return snappy::GetUncompressedLength(input, length, result);
-#else
- return false;
-#endif
-}
-
-inline bool Snappy_Uncompress(const char* input, size_t length,
- char* output) {
-#ifdef SNAPPY
- return snappy::RawUncompress(input, length, output);
-#else
- return false;
-#endif
-}
-
-inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) {
- return false;
-}
-
-} // namespace port
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_PORT_PORT_POSIX_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/port_win.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/port_win.cc b/thirdparty/leveldb-1.18/port/port_win.cc
deleted file mode 100755
index 4ca64ac..0000000
--- a/thirdparty/leveldb-1.18/port/port_win.cc
+++ /dev/null
@@ -1,182 +0,0 @@
-// LevelDB Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// See port_example.h for documentation for the following types/functions.
-
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above copyright
-// notice, this list of conditions and the following disclaimer in the
-// documentation and/or other materials provided with the distribution.
-// * Neither the name of the University of California, Berkeley nor the
-// names of its contributors may be used to endorse or promote products
-// derived from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
-// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
-// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-
-#include "port/port_win.h"
-
-#include <windows.h>
-#include <cassert>
-
-namespace leveldb {
-namespace port {
-
-Mutex::Mutex() :
- mutex_(::CreateMutex(NULL, FALSE, NULL)) {
- assert(mutex_);
-}
-
-Mutex::~Mutex() {
- assert(mutex_);
- ::CloseHandle(mutex_);
-}
-
-void Mutex::Lock() {
- assert(mutex_);
- ::WaitForSingleObject(mutex_, INFINITE);
-}
-
-void Mutex::Unlock() {
- assert(mutex_);
- ::ReleaseMutex(mutex_);
-}
-
-void Mutex::AssertHeld() {
- assert(mutex_);
- assert(1);
-}
-
-CondVar::CondVar(Mutex* mu) :
- waiting_(0),
- mu_(mu),
- sema_(::CreateSemaphore(NULL, 0, 0x7fffffff, NULL)),
- event_(::CreateEvent(NULL, FALSE, FALSE, NULL)),
- broadcasted_(false){
- assert(mu_);
-}
-
-CondVar::~CondVar() {
- ::CloseHandle(sema_);
- ::CloseHandle(event_);
-}
-
-void CondVar::Wait() {
- wait_mtx_.Lock();
- ++waiting_;
- assert(waiting_ > 0);
- wait_mtx_.Unlock();
-
- ::SignalObjectAndWait(mu_->mutex_, sema_, INFINITE, FALSE);
-
- wait_mtx_.Lock();
- bool last = broadcasted_ && (--waiting_ == 0);
- assert(waiting_ >= 0);
- wait_mtx_.Unlock();
-
- // we leave this function with the mutex held
- if (last)
- {
- ::SignalObjectAndWait(event_, mu_->mutex_, INFINITE, FALSE);
- }
- else
- {
- ::WaitForSingleObject(mu_->mutex_, INFINITE);
- }
-}
-
-void CondVar::Signal() {
- wait_mtx_.Lock();
- bool waiters = waiting_ > 0;
- wait_mtx_.Unlock();
-
- if (waiters)
- {
- ::ReleaseSemaphore(sema_, 1, 0);
- }
-}
-
-void CondVar::SignalAll() {
- wait_mtx_.Lock();
-
- broadcasted_ = (waiting_ > 0);
-
- if (broadcasted_)
- {
- // release all
- ::ReleaseSemaphore(sema_, waiting_, 0);
- wait_mtx_.Unlock();
- ::WaitForSingleObject(event_, INFINITE);
- broadcasted_ = false;
- }
- else
- {
- wait_mtx_.Unlock();
- }
-}
-
-AtomicPointer::AtomicPointer(void* v) {
- Release_Store(v);
-}
-
-void* AtomicPointer::Acquire_Load() const {
- void * p = nullptr;
- InterlockedExchangePointer(&p, rep_);
- return p;
-}
-
-void AtomicPointer::Release_Store(void* v) {
- InterlockedExchangePointer(&rep_, v);
-}
-
-void* AtomicPointer::NoBarrier_Load() const {
- return rep_;
-}
-
-void AtomicPointer::NoBarrier_Store(void* v) {
- rep_ = v;
-}
-
-enum InitializationState
-{
- Uninitialized = 0,
- Running = 1,
- Initialized = 2
-};
-
-void InitOnce(OnceType* once, void (*initializer)()) {
-
- static_assert(Uninitialized == LEVELDB_ONCE_INIT, "Invalid uninitialized state value");
-
- InitializationState state = static_cast<InitializationState>(InterlockedCompareExchange(once, Running, Uninitialized));
-
- if (state == Uninitialized) {
- initializer();
- *once = Initialized;
- }
-
- if (state == Running) {
- while(*once != Initialized) {
- Sleep(0); // yield
- }
- }
-
- assert(*once == Initialized);
-}
-
-}
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/port_win.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/port_win.h b/thirdparty/leveldb-1.18/port/port_win.h
deleted file mode 100755
index dbc37ad..0000000
--- a/thirdparty/leveldb-1.18/port/port_win.h
+++ /dev/null
@@ -1,164 +0,0 @@
-// LevelDB Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// See port_example.h for documentation for the following types/functions.
-
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-// * Redistributions in binary form must reproduce the above copyright
-// notice, this list of conditions and the following disclaimer in the
-// documentation and/or other materials provided with the distribution.
-// * Neither the name of the University of California, Berkeley nor the
-// names of its contributors may be used to endorse or promote products
-// derived from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
-// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
-// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-
-#ifndef STORAGE_LEVELDB_PORT_PORT_WIN_H_
-#define STORAGE_LEVELDB_PORT_PORT_WIN_H_
-
-#if _MSC_VER < 1900
-#define snprintf _snprintf
-#endif
-#define close _close
-#define fread_unlocked _fread_nolock
-
-#ifdef SNAPPY
-#include <snappy.h>
-#endif
-
-#include <string>
-
-#include <stdint.h>
-
-namespace leveldb {
-namespace port {
-
-// Windows is little endian (for now :p)
-static const bool kLittleEndian = true;
-
-class CondVar;
-
-class Mutex {
- public:
- Mutex();
- ~Mutex();
-
- void Lock();
- void Unlock();
- void AssertHeld();
-
- private:
- friend class CondVar;
- // critical sections are more efficient than mutexes
- // but they are not recursive and can only be used to synchronize threads
- // within the same process
- // additionnaly they cannot be used with SignalObjectAndWait that we use for
- // CondVar
- // we use opaque void * to avoid including windows.h in port_win.h
- void* mutex_;
-
- // No copying
- Mutex(const Mutex&);
- void operator=(const Mutex&);
-};
-
-// the Win32 API offers a dependable condition variable mechanism, but only
-// starting with
-// Windows 2008 and Vista
-// no matter what we will implement our own condition variable with a semaphore
-// implementation as described in a paper written by Douglas C. Schmidt and
-// Irfan Pyarali
-class CondVar {
- public:
- explicit CondVar(Mutex* mu);
- ~CondVar();
- void Wait();
- void Signal();
- void SignalAll();
-
- private:
- Mutex* mu_;
-
- Mutex wait_mtx_;
- long waiting_;
-
- void* sema_;
- void* event_;
-
- bool broadcasted_;
-};
-
-// Storage for a lock-free pointer
-class AtomicPointer {
- private:
- void* rep_;
-
- public:
- AtomicPointer() : rep_(nullptr) {}
- explicit AtomicPointer(void* v);
- void* Acquire_Load() const;
-
- void Release_Store(void* v);
-
- void* NoBarrier_Load() const;
-
- void NoBarrier_Store(void* v);
-};
-
-typedef volatile long OnceType;
-#define LEVELDB_ONCE_INIT (0)
-
-extern void InitOnce(OnceType* once, void (*initializer)());
-
-inline bool Snappy_Compress(const char* input, size_t length,
- ::std::string* output) {
-#ifdef SNAPPY
- output->resize(snappy::MaxCompressedLength(length));
- size_t outlen;
- snappy::RawCompress(input, length, &(*output)[0], &outlen);
- output->resize(outlen);
- return true;
-#endif
-
- return false;
-}
-
-inline bool Snappy_GetUncompressedLength(const char* input, size_t length,
- size_t* result) {
-#ifdef SNAPPY
- return snappy::GetUncompressedLength(input, length, result);
-#else
- return false;
-#endif
-}
-
-inline bool Snappy_Uncompress(const char* input, size_t length, char* output) {
-#ifdef SNAPPY
- return snappy::RawUncompress(input, length, output);
-#else
- return false;
-#endif
-}
-
-inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) {
- return false;
-}
-}
-}
-
-#endif // STORAGE_LEVELDB_PORT_PORT_WIN_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/sha1_portable.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/sha1_portable.cc b/thirdparty/leveldb-1.18/port/sha1_portable.cc
deleted file mode 100755
index 8fa7277..0000000
--- a/thirdparty/leveldb-1.18/port/sha1_portable.cc
+++ /dev/null
@@ -1,298 +0,0 @@
-// Portions copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// This module provides a slow but portable implementation of
-// the SHA1 hash function.
-//
-// It is adapted from free code written by Paul E. Jones
-// <pa...@packetizer.com>. See http://www.packetizer.com/security/sha1/
-//
-// The license for the original code is:
-/*
- Copyright (C) 1998, 2009
- Paul E. Jones <pa...@packetizer.com>
-
- Freeware Public License (FPL)
-
- This software is licensed as "freeware." Permission to distribute
- this software in source and binary forms, including incorporation
- into other products, is hereby granted without a fee. THIS SOFTWARE
- IS PROVIDED 'AS IS' AND WITHOUT ANY EXPRESSED OR IMPLIED WARRANTIES,
- INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
- AND FITNESS FOR A PARTICULAR PURPOSE. THE AUTHOR SHALL NOT BE HELD
- LIABLE FOR ANY DAMAGES RESULTING FROM THE USE OF THIS SOFTWARE, EITHER
- DIRECTLY OR INDIRECTLY, INCLUDING, BUT NOT LIMITED TO, LOSS OF DATA
- OR DATA BEING RENDERED INACCURATE.
-*/
-
-#include "port/sha1_portable.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-
-namespace leveldb {
-namespace port {
-
-/*
- * Description:
- * This class implements the Secure Hashing Standard as defined
- * in FIPS PUB 180-1 published April 17, 1995.
- */
-
-/*
- * This structure will hold context information for the hashing
- * operation
- */
-typedef struct SHA1Context {
- unsigned Message_Digest[5]; /* Message Digest (output) */
-
- unsigned Length_Low; /* Message length in bits */
- unsigned Length_High; /* Message length in bits */
-
- unsigned char Message_Block[64]; /* 512-bit message blocks */
- int Message_Block_Index; /* Index into message block array */
-
- bool Computed; /* Is the digest computed? */
- bool Corrupted; /* Is the message digest corruped? */
-} SHA1Context;
-
-/*
- * Portability Issues:
- * SHA-1 is defined in terms of 32-bit "words". This code was
- * written with the expectation that the processor has at least
- * a 32-bit machine word size. If the machine word size is larger,
- * the code should still function properly. One caveat to that
- * is that the input functions taking characters and character
- * arrays assume that only 8 bits of information are stored in each
- * character.
- */
-
-/*
- * Define the circular shift macro
- */
-#define SHA1CircularShift(bits,word) \
- ((((word) << (bits)) & 0xFFFFFFFF) | \
- ((word) >> (32-(bits))))
-
-/* Function prototypes */
-static void SHA1ProcessMessageBlock(SHA1Context *);
-static void SHA1PadMessage(SHA1Context *);
-
-// Initialize the SHA1Context in preparation for computing a new
-// message digest.
-static void SHA1Reset(SHA1Context* context) {
- context->Length_Low = 0;
- context->Length_High = 0;
- context->Message_Block_Index = 0;
-
- context->Message_Digest[0] = 0x67452301;
- context->Message_Digest[1] = 0xEFCDAB89;
- context->Message_Digest[2] = 0x98BADCFE;
- context->Message_Digest[3] = 0x10325476;
- context->Message_Digest[4] = 0xC3D2E1F0;
-
- context->Computed = false;
- context->Corrupted = false;
-}
-
-// This function will return the 160-bit message digest into the
-// Message_Digest array within the SHA1Context provided
-static bool SHA1Result(SHA1Context *context) {
- if (context->Corrupted) {
- return false;
- }
-
- if (!context->Computed) {
- SHA1PadMessage(context);
- context->Computed = true;
- }
- return true;
-}
-
-// This function accepts an array of bytes as the next portion of
-// the message.
-static void SHA1Input(SHA1Context *context,
- const unsigned char *message_array,
- unsigned length) {
- if (!length) return;
-
- if (context->Computed || context->Corrupted) {
- context->Corrupted = true;
- return;
- }
-
- while(length-- && !context->Corrupted) {
- context->Message_Block[context->Message_Block_Index++] =
- (*message_array & 0xFF);
-
- context->Length_Low += 8;
- /* Force it to 32 bits */
- context->Length_Low &= 0xFFFFFFFF;
- if (context->Length_Low == 0) {
- context->Length_High++;
- /* Force it to 32 bits */
- context->Length_High &= 0xFFFFFFFF;
- if (context->Length_High == 0)
- {
- /* Message is too long */
- context->Corrupted = true;
- }
- }
-
- if (context->Message_Block_Index == 64)
- {
- SHA1ProcessMessageBlock(context);
- }
-
- message_array++;
- }
-}
-
-// This function will process the next 512 bits of the message stored
-// in the Message_Block array.
-static void SHA1ProcessMessageBlock(SHA1Context *context) {
- const unsigned K[] = // Constants defined in SHA-1
- {
- 0x5A827999,
- 0x6ED9EBA1,
- 0x8F1BBCDC,
- 0xCA62C1D6
- };
- int t; // Loop counter
- unsigned temp; // Temporary word value
- unsigned W[80]; // Word sequence
- unsigned A, B, C, D, E; // Word buffers
-
- // Initialize the first 16 words in the array W
- for(t = 0; t < 16; t++) {
- W[t] = ((unsigned) context->Message_Block[t * 4]) << 24;
- W[t] |= ((unsigned) context->Message_Block[t * 4 + 1]) << 16;
- W[t] |= ((unsigned) context->Message_Block[t * 4 + 2]) << 8;
- W[t] |= ((unsigned) context->Message_Block[t * 4 + 3]);
- }
-
- for(t = 16; t < 80; t++) {
- W[t] = SHA1CircularShift(1,W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16]);
- }
-
- A = context->Message_Digest[0];
- B = context->Message_Digest[1];
- C = context->Message_Digest[2];
- D = context->Message_Digest[3];
- E = context->Message_Digest[4];
-
- for(t = 0; t < 20; t++) {
- temp = SHA1CircularShift(5,A) +
- ((B & C) | ((~B) & D)) + E + W[t] + K[0];
- temp &= 0xFFFFFFFF;
- E = D;
- D = C;
- C = SHA1CircularShift(30,B);
- B = A;
- A = temp;
- }
-
- for(t = 20; t < 40; t++) {
- temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[1];
- temp &= 0xFFFFFFFF;
- E = D;
- D = C;
- C = SHA1CircularShift(30,B);
- B = A;
- A = temp;
- }
-
- for(t = 40; t < 60; t++) {
- temp = SHA1CircularShift(5,A) +
- ((B & C) | (B & D) | (C & D)) + E + W[t] + K[2];
- temp &= 0xFFFFFFFF;
- E = D;
- D = C;
- C = SHA1CircularShift(30,B);
- B = A;
- A = temp;
- }
-
- for(t = 60; t < 80; t++) {
- temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[3];
- temp &= 0xFFFFFFFF;
- E = D;
- D = C;
- C = SHA1CircularShift(30,B);
- B = A;
- A = temp;
- }
-
- context->Message_Digest[0] = (context->Message_Digest[0] + A) & 0xFFFFFFFF;
- context->Message_Digest[1] = (context->Message_Digest[1] + B) & 0xFFFFFFFF;
- context->Message_Digest[2] = (context->Message_Digest[2] + C) & 0xFFFFFFFF;
- context->Message_Digest[3] = (context->Message_Digest[3] + D) & 0xFFFFFFFF;
- context->Message_Digest[4] = (context->Message_Digest[4] + E) & 0xFFFFFFFF;
-
- context->Message_Block_Index = 0;
-}
-
-// According to the standard, the message must be padded to an even
-// 512 bits. The first padding bit must be a '1'. The last 64 bits
-// represent the length of the original message. All bits in between
-// should be 0. This function will pad the message according to those
-// rules by filling the Message_Block array accordingly. It will also
-// call SHA1ProcessMessageBlock() appropriately. When it returns, it
-// can be assumed that the message digest has been computed.
-static void SHA1PadMessage(SHA1Context *context) {
- // Check to see if the current message block is too small to hold
- // the initial padding bits and length. If so, we will pad the
- // block, process it, and then continue padding into a second block.
- if (context->Message_Block_Index > 55) {
- context->Message_Block[context->Message_Block_Index++] = 0x80;
- while(context->Message_Block_Index < 64) {
- context->Message_Block[context->Message_Block_Index++] = 0;
- }
-
- SHA1ProcessMessageBlock(context);
-
- while(context->Message_Block_Index < 56) {
- context->Message_Block[context->Message_Block_Index++] = 0;
- }
- } else {
- context->Message_Block[context->Message_Block_Index++] = 0x80;
- while(context->Message_Block_Index < 56) {
- context->Message_Block[context->Message_Block_Index++] = 0;
- }
- }
-
- // Store the message length as the last 8 octets
- context->Message_Block[56] = (context->Length_High >> 24) & 0xFF;
- context->Message_Block[57] = (context->Length_High >> 16) & 0xFF;
- context->Message_Block[58] = (context->Length_High >> 8) & 0xFF;
- context->Message_Block[59] = (context->Length_High) & 0xFF;
- context->Message_Block[60] = (context->Length_Low >> 24) & 0xFF;
- context->Message_Block[61] = (context->Length_Low >> 16) & 0xFF;
- context->Message_Block[62] = (context->Length_Low >> 8) & 0xFF;
- context->Message_Block[63] = (context->Length_Low) & 0xFF;
-
- SHA1ProcessMessageBlock(context);
-}
-
-
-void SHA1_Hash_Portable(const char* data, size_t len, char* hash_array) {
- SHA1Context context;
- SHA1Reset(&context);
- SHA1Input(&context, reinterpret_cast<const unsigned char*>(data), len);
- bool ok = SHA1Result(&context);
- if (!ok) {
- fprintf(stderr, "Unexpected error in SHA1_Hash_Portable code\n");
- exit(1);
- }
- for (int i = 0; i < 5; i++) {
- uint32_t value = context.Message_Digest[i];
- hash_array[i*4 + 0] = (value >> 24) & 0xff;
- hash_array[i*4 + 1] = (value >> 16) & 0xff;
- hash_array[i*4 + 2] = (value >> 8) & 0xff;
- hash_array[i*4 + 3] = value & 0xff;
- }
-}
-
-}
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/sha1_portable.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/sha1_portable.h b/thirdparty/leveldb-1.18/port/sha1_portable.h
deleted file mode 100755
index 31db305..0000000
--- a/thirdparty/leveldb-1.18/port/sha1_portable.h
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_
-#define STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_
-
-#include <stddef.h>
-
-namespace leveldb {
-namespace port {
-
-// Compute the SHA1 hash value of "data[0..len-1]" and store it in
-// "hash_array[0..19]". hash_array must have 20 bytes of space available.
-//
-// This function is portable but may not be as fast as a version
-// optimized for your platform. It is provided as a default method
-// that can be used when porting leveldb to a new platform if no
-// better SHA1 hash implementation is available.
-void SHA1_Hash_Portable(const char* data, size_t len, char* hash_array);
-
-}
-}
-
-#endif // STORAGE_LEVELDB_PORT_SHA1_PORTABLE_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/port/sha1_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/port/sha1_test.cc b/thirdparty/leveldb-1.18/port/sha1_test.cc
deleted file mode 100755
index b182e67..0000000
--- a/thirdparty/leveldb-1.18/port/sha1_test.cc
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "port/port.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-namespace port {
-
-class SHA1 { };
-
-static std::string TestSHA1(const char* data, size_t len) {
- char hash_val[20];
- SHA1_Hash(data, len, hash_val);
- char buf[41];
- for (int i = 0; i < 20; i++) {
- snprintf(buf + i * 2, 41 - i * 2,
- "%02x",
- static_cast<unsigned int>(static_cast<unsigned char>(
- hash_val[i])));
- }
- return std::string(buf, 40);
-}
-
-TEST(SHA1, Simple) {
- ASSERT_EQ("da39a3ee5e6b4b0d3255bfef95601890afd80709", TestSHA1("", 0));
- ASSERT_EQ("aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d", TestSHA1("hello", 5));
- std::string x(10000, 'x');
- ASSERT_EQ("f8c5cde791c5056cf515881e701c8a9ecb439a75",
- TestSHA1(x.data(), x.size()));
-}
-
-}
-}
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
[31/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/make_package.sh
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/make_package.sh b/thirdparty/rocksdb/build_tools/make_package.sh
new file mode 100755
index 0000000..58bac44
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/make_package.sh
@@ -0,0 +1,128 @@
+#/usr/bin/env bash
+
+set -e
+
+function log() {
+ echo "[+] $1"
+}
+
+function fatal() {
+ echo "[!] $1"
+ exit 1
+}
+
+function platform() {
+ local __resultvar=$1
+ if [[ -f "/etc/yum.conf" ]]; then
+ eval $__resultvar="centos"
+ elif [[ -f "/etc/dpkg/dpkg.cfg" ]]; then
+ eval $__resultvar="ubuntu"
+ else
+ fatal "Unknwon operating system"
+ fi
+}
+platform OS
+
+function package() {
+ if [[ $OS = "ubuntu" ]]; then
+ if dpkg --get-selections | grep --quiet $1; then
+ log "$1 is already installed. skipping."
+ else
+ apt-get install $@ -y
+ fi
+ elif [[ $OS = "centos" ]]; then
+ if rpm -qa | grep --quiet $1; then
+ log "$1 is already installed. skipping."
+ else
+ yum install $@ -y
+ fi
+ fi
+}
+
+function detect_fpm_output() {
+ if [[ $OS = "ubuntu" ]]; then
+ export FPM_OUTPUT=deb
+ elif [[ $OS = "centos" ]]; then
+ export FPM_OUTPUT=rpm
+ fi
+}
+detect_fpm_output
+
+function gem_install() {
+ if gem list | grep --quiet $1; then
+ log "$1 is already installed. skipping."
+ else
+ gem install $@
+ fi
+}
+
+function main() {
+ if [[ $# -ne 1 ]]; then
+ fatal "Usage: $0 <rocksdb_version>"
+ else
+ log "using rocksdb version: $1"
+ fi
+
+ if [[ -d /vagrant ]]; then
+ if [[ $OS = "ubuntu" ]]; then
+ package g++-4.8
+ export CXX=g++-4.8
+
+ # the deb would depend on libgflags2, but the static lib is the only thing
+ # installed by make install
+ package libgflags-dev
+
+ package ruby-all-dev
+ elif [[ $OS = "centos" ]]; then
+ pushd /etc/yum.repos.d
+ if [[ ! -f /etc/yum.repos.d/devtools-1.1.repo ]]; then
+ wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo
+ fi
+ package devtoolset-1.1-gcc --enablerepo=testing-1.1-devtools-6
+ package devtoolset-1.1-gcc-c++ --enablerepo=testing-1.1-devtools-6
+ export CC=/opt/centos/devtoolset-1.1/root/usr/bin/gcc
+ export CPP=/opt/centos/devtoolset-1.1/root/usr/bin/cpp
+ export CXX=/opt/centos/devtoolset-1.1/root/usr/bin/c++
+ export PATH=$PATH:/opt/centos/devtoolset-1.1/root/usr/bin
+ popd
+ if ! rpm -qa | grep --quiet gflags; then
+ rpm -i https://github.com/schuhschuh/gflags/releases/download/v2.1.0/gflags-devel-2.1.0-1.amd64.rpm
+ fi
+
+ package ruby
+ package ruby-devel
+ package rubygems
+ package rpm-build
+ fi
+ fi
+ gem_install fpm
+
+ make static_lib
+ make install INSTALL_PATH=package
+
+ cd package
+
+ LIB_DIR=lib
+ if [[ -z "$ARCH" ]]; then
+ ARCH=$(getconf LONG_BIT)
+ fi
+ if [[ ("$FPM_OUTPUT" = "rpm") && ($ARCH -eq 64) ]]; then
+ mv lib lib64
+ LIB_DIR=lib64
+ fi
+
+ fpm \
+ -s dir \
+ -t $FPM_OUTPUT \
+ -n rocksdb \
+ -v $1 \
+ --prefix /usr \
+ --url http://rocksdb.org/ \
+ -m rocksdb@fb.com \
+ --license BSD \
+ --vendor Facebook \
+ --description "RocksDB is an embeddable persistent key-value store for fast storage." \
+ include $LIB_DIR
+}
+
+main $@
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/precommit_checker.py
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/precommit_checker.py b/thirdparty/rocksdb/build_tools/precommit_checker.py
new file mode 100755
index 0000000..0f8884d
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/precommit_checker.py
@@ -0,0 +1,208 @@
+#!/usr/local/fbcode/gcc-4.9-glibc-2.20-fb/bin/python2.7
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+import argparse
+import commands
+import subprocess
+import sys
+import re
+import os
+import time
+
+
+#
+# Simple logger
+#
+
+class Log:
+
+ def __init__(self, filename):
+ self.filename = filename
+ self.f = open(self.filename, 'w+', 0)
+
+ def caption(self, str):
+ line = "\n##### %s #####\n" % str
+ if self.f:
+ self.f.write("%s \n" % line)
+ else:
+ print(line)
+
+ def error(self, str):
+ data = "\n\n##### ERROR ##### %s" % str
+ if self.f:
+ self.f.write("%s \n" % data)
+ else:
+ print(data)
+
+ def log(self, str):
+ if self.f:
+ self.f.write("%s \n" % str)
+ else:
+ print(str)
+
+#
+# Shell Environment
+#
+
+
+class Env(object):
+
+ def __init__(self, logfile, tests):
+ self.tests = tests
+ self.log = Log(logfile)
+
+ def shell(self, cmd, path=os.getcwd()):
+ if path:
+ os.chdir(path)
+
+ self.log.log("==== shell session ===========================")
+ self.log.log("%s> %s" % (path, cmd))
+ status = subprocess.call("cd %s; %s" % (path, cmd), shell=True,
+ stdout=self.log.f, stderr=self.log.f)
+ self.log.log("status = %s" % status)
+ self.log.log("============================================== \n\n")
+ return status
+
+ def GetOutput(self, cmd, path=os.getcwd()):
+ if path:
+ os.chdir(path)
+
+ self.log.log("==== shell session ===========================")
+ self.log.log("%s> %s" % (path, cmd))
+ status, out = commands.getstatusoutput(cmd)
+ self.log.log("status = %s" % status)
+ self.log.log("out = %s" % out)
+ self.log.log("============================================== \n\n")
+ return status, out
+
+#
+# Pre-commit checker
+#
+
+
+class PreCommitChecker(Env):
+
+ def __init__(self, args):
+ Env.__init__(self, args.logfile, args.tests)
+ self.ignore_failure = args.ignore_failure
+
+ #
+ # Get commands for a given job from the determinator file
+ #
+ def get_commands(self, test):
+ status, out = self.GetOutput(
+ "RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".")
+ return status, out
+
+ #
+ # Run a specific CI job
+ #
+ def run_test(self, test):
+ self.log.caption("Running test %s locally" % test)
+
+ # get commands for the CI job determinator
+ status, cmds = self.get_commands(test)
+ if status != 0:
+ self.log.error("Error getting commands for test %s" % test)
+ return False
+
+ # Parse the JSON to extract the commands to run
+ cmds = re.findall("'shell':'([^\']*)'", cmds)
+
+ if len(cmds) == 0:
+ self.log.log("No commands found")
+ return False
+
+ # Run commands
+ for cmd in cmds:
+ # Replace J=<..> with the local environment variable
+ if "J" in os.environ:
+ cmd = cmd.replace("J=1", "J=%s" % os.environ["J"])
+ cmd = cmd.replace("make ", "make -j%s " % os.environ["J"])
+ # Run the command
+ status = self.shell(cmd, ".")
+ if status != 0:
+ self.log.error("Error running command %s for test %s"
+ % (cmd, test))
+ return False
+
+ return True
+
+ #
+ # Run specified CI jobs
+ #
+ def run_tests(self):
+ if not self.tests:
+ self.log.error("Invalid args. Please provide tests")
+ return False
+
+ self.print_separator()
+ self.print_row("TEST", "RESULT")
+ self.print_separator()
+
+ result = True
+ for test in self.tests:
+ start_time = time.time()
+ self.print_test(test)
+ result = self.run_test(test)
+ elapsed_min = (time.time() - start_time) / 60
+ if not result:
+ self.log.error("Error running test %s" % test)
+ self.print_result("FAIL (%dm)" % elapsed_min)
+ if not self.ignore_failure:
+ return False
+ result = False
+ else:
+ self.print_result("PASS (%dm)" % elapsed_min)
+
+ self.print_separator()
+ return result
+
+ #
+ # Print a line
+ #
+ def print_separator(self):
+ print("".ljust(60, "-"))
+
+ #
+ # Print two colums
+ #
+ def print_row(self, c0, c1):
+ print("%s%s" % (c0.ljust(40), c1.ljust(20)))
+
+ def print_test(self, test):
+ print(test.ljust(40), end="")
+ sys.stdout.flush()
+
+ def print_result(self, result):
+ print(result.ljust(20))
+
+#
+# Main
+#
+parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.')
+
+# --log <logfile>
+parser.add_argument('--logfile', default='/tmp/precommit-check.log',
+ help='Log file. Default is /tmp/precommit-check.log')
+# --ignore_failure
+parser.add_argument('--ignore_failure', action='store_true', default=False,
+ help='Stop when an error occurs')
+# <test ....>
+parser.add_argument('tests', nargs='+',
+ help='CI test(s) to run. e.g: unit punit asan tsan ubsan')
+
+args = parser.parse_args()
+checker = PreCommitChecker(args)
+
+print("Please follow log %s" % checker.log.filename)
+
+if not checker.run_tests():
+ print("Error running tests. Please check log file %s"
+ % checker.log.filename)
+ sys.exit(1)
+
+sys.exit(0)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/rocksdb-lego-determinator
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/rocksdb-lego-determinator b/thirdparty/rocksdb/build_tools/rocksdb-lego-determinator
new file mode 100755
index 0000000..a40b306
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/rocksdb-lego-determinator
@@ -0,0 +1,805 @@
+#!/usr/bin/env bash
+# This script is executed by Sandcastle
+# to determine next steps to run
+
+# Usage:
+# EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> rocks_ci.py <test-name>
+#
+# Input Value
+# -------------------------------------------------------------------------
+# EMAIL Email address to report on trigger conditions
+# ONCALL Email address to raise a task on failure
+# TRIGGER Trigger conditions for email. Valid values are fail, warn, all
+# SUBSCRIBER Email addresss to add as subscriber for task
+#
+
+#
+# Report configuration
+#
+REPORT_EMAIL=
+if [ ! -z $EMAIL ]; then
+ if [ -z $TRIGGER ]; then
+ TRIGGER="fail"
+ fi
+
+ REPORT_EMAIL="
+ {
+ 'type':'email',
+ 'triggers': [ '$TRIGGER' ],
+ 'emails':['$EMAIL']
+ },"
+fi
+
+CREATE_TASK=
+if [ ! -z $ONCALL ]; then
+ CREATE_TASK="
+ {
+ 'type':'task',
+ 'triggers':[ 'fail' ],
+ 'priority':0,
+ 'subscribers':[ '$SUBSCRIBER' ],
+ 'tags':[ 'rocksdb', 'ci' ],
+ },"
+fi
+
+# For now, create the tasks using only the dedicated task creation tool.
+CREATE_TASK=
+
+REPORT=
+if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then
+ REPORT="'report': [
+ $REPORT_EMAIL
+ $CREATE_TASK
+ ]"
+fi
+
+#
+# Helper variables
+#
+CLEANUP_ENV="
+{
+ 'name':'Cleanup environment',
+ 'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true) && make clean',
+ 'user':'root'
+}"
+
+# We will eventually set the RATIO to 1, but we want do this
+# in steps. RATIO=$(nproc) will make it work as J=1
+if [ -z $RATIO ]; then
+ RATIO=$(nproc)
+fi
+
+if [ -z $PARALLEL_J ]; then
+ PARALLEL_J="J=$(expr $(nproc) / ${RATIO})"
+fi
+
+if [ -z $PARALLEL_j ]; then
+ PARALLEL_j="-j$(expr $(nproc) / ${RATIO})"
+fi
+
+PARALLELISM="$PARALLEL_J $PARALLEL_j"
+
+DEBUG="OPT=-g"
+SHM="TEST_TMPDIR=/dev/shm/rocksdb"
+NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
+GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
+ASAN="COMPILE_WITH_ASAN=1"
+CLANG="USE_CLANG=1"
+LITE="OPT=\"-DROCKSDB_LITE -g\""
+TSAN="COMPILE_WITH_TSAN=1"
+UBSAN="COMPILE_WITH_UBSAN=1"
+DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
+HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
+SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH"
+PARSER="'parser':'python build_tools/error_filter.py $1'"
+
+CONTRUN_NAME="ROCKSDB_CONTRUN_NAME"
+
+# This code is getting called under various scenarios. What we care about is to
+# understand when it's called from nightly contruns because in that case we'll
+# create tasks for any failures. To follow the existing pattern, we'll check
+# the value of $ONCALL. If it's a diff then just call `false` to make sure
+# that errors will be properly propagated to the caller.
+if [ ! -z $ONCALL ]; then
+ TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL"
+else
+ TASK_CREATION_TOOL="false"
+fi
+
+ARTIFACTS=" 'artifacts': [
+ {
+ 'name':'database',
+ 'paths':[ '/dev/shm/rocksdb' ],
+ }
+]"
+
+#
+# A mechanism to disable tests temporarily
+#
+DISABLE_COMMANDS="[
+ {
+ 'name':'Disable test',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ {
+ 'name':'Job disabled. Please contact test owner',
+ 'shell':'exit 1',
+ 'user':'root'
+ },
+ ],
+ }
+]"
+
+#
+# RocksDB unit test
+#
+UNIT_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb Unit Test',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build and test RocksDB debug version',
+ 'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB unit test not under /dev/shm
+#
+UNIT_TEST_NON_SHM_COMMANDS="[
+ {
+ 'name':'Rocksdb Unit Test',
+ 'oncall':'$ONCALL',
+ 'timeout': 86400,
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build and test RocksDB debug version',
+ 'timeout': 86400,
+ 'shell':'$NON_SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB release build and unit tests
+#
+RELEASE_BUILD_COMMANDS="[
+ {
+ 'name':'Rocksdb Release Build',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build RocksDB release',
+ 'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB unit test on gcc-4.8.1
+#
+UNIT_TEST_COMMANDS_481="[
+ {
+ 'name':'Rocksdb Unit Test on GCC 4.8.1',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build and test RocksDB debug version',
+ 'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB release build and unit tests
+#
+RELEASE_BUILD_COMMANDS_481="[
+ {
+ 'name':'Rocksdb Release on GCC 4.8.1',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build RocksDB release on GCC 4.8.1',
+ 'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB unit test with CLANG
+#
+CLANG_UNIT_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb Unit Test',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build and test RocksDB debug',
+ 'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB release build with CLANG
+#
+CLANG_RELEASE_BUILD_COMMANDS="[
+ {
+ 'name':'Rocksdb CLANG Release Build',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build RocksDB release',
+ 'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB analyze
+#
+CLANG_ANALYZE_COMMANDS="[
+ {
+ 'name':'Rocksdb analyze',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'RocksDB build and analyze',
+ 'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB code coverage
+#
+CODE_COV_COMMANDS="[
+ {
+ 'name':'Rocksdb Unit Test Code Coverage',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build, test and collect code coverage info',
+ 'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB unity
+#
+UNITY_COMMANDS="[
+ {
+ 'name':'Rocksdb Unity',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build, test unity test',
+ 'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# Build RocksDB lite
+#
+LITE_BUILD_COMMANDS="[
+ {
+ 'name':'Rocksdb Lite build',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build RocksDB debug version',
+ 'shell':'$LITE make J=1 static_lib || $CONTRUN_NAME=lite_static_lib $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB lite tests
+#
+LITE_UNIT_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb Lite Unit Test',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build RocksDB debug version',
+ 'shell':'$SHM $LITE make J=1 check || $CONTRUN_NAME=lite_check $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB stress/crash test
+#
+STRESS_CRASH_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb Stress/Crash Test',
+ 'oncall':'$ONCALL',
+ 'timeout': 86400,
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build and run RocksDB debug stress tests',
+ 'shell':'$SHM $DEBUG make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ {
+ 'name':'Build and run RocksDB debug crash tests',
+ 'timeout': 86400,
+ 'shell':'$SHM $DEBUG make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ }
+ ],
+ $ARTIFACTS,
+ $REPORT
+ }
+]"
+
+# RocksDB write stress test.
+# We run on disk device on purpose (i.e. no $SHM)
+# because we want to add some randomness to fsync commands
+WRITE_STRESS_COMMANDS="[
+ {
+ 'name':'Rocksdb Write Stress Test',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build and run RocksDB write stress tests',
+ 'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ }
+ ],
+ 'artifacts': [{'name': 'database', 'paths': ['/tmp/rocksdb_write_stress']}],
+ $REPORT
+ }
+]"
+
+
+#
+# RocksDB test under address sanitizer
+#
+ASAN_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb Unit Test under ASAN',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Test RocksDB debug under ASAN',
+'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
+ 'user':'root',
+ $PARSER
+ }
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB crash testing under address sanitizer
+#
+ASAN_CRASH_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb crash test under ASAN',
+ 'oncall':'$ONCALL',
+ 'timeout': 86400,
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build and run RocksDB debug asan_crash_test',
+ 'timeout': 86400,
+ 'shell':'$SHM $DEBUG make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB test under undefined behavior sanitizer
+#
+UBSAN_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb Unit Test under UBSAN',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Test RocksDB debug under UBSAN',
+ 'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ }
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB crash testing under udnefined behavior sanitizer
+#
+UBSAN_CRASH_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb crash test under UBSAN',
+ 'oncall':'$ONCALL',
+ 'timeout': 86400,
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build and run RocksDB debug ubsan_crash_test',
+ 'timeout': 86400,
+ 'shell':'$SHM $DEBUG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB unit test under valgrind
+#
+VALGRIND_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb Unit Test under valgrind',
+ 'oncall':'$ONCALL',
+ 'timeout': 86400,
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Run RocksDB debug unit tests',
+ 'timeout': 86400,
+ 'shell':'$SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB test under TSAN
+#
+TSAN_UNIT_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb Unit Test under TSAN',
+ 'oncall':'$ONCALL',
+ 'timeout': 86400,
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Run RocksDB debug unit test',
+ 'timeout': 86400,
+ 'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB crash test under TSAN
+#
+TSAN_CRASH_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb Crash Test under TSAN',
+ 'oncall':'$ONCALL',
+ 'timeout': 86400,
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Compile and run',
+ 'timeout': 86400,
+ 'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 CRASH_TEST_EXT_ARGS=--log2_keys_per_lock=22 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB format compatible
+#
+
+run_format_compatible()
+{
+ export TEST_TMPDIR=/dev/shm/rocksdb
+ rm -rf /dev/shm/rocksdb
+ mkdir /dev/shm/rocksdb
+
+ tools/check_format_compatible.sh
+}
+
+FORMAT_COMPATIBLE_COMMANDS="[
+ {
+ 'name':'Rocksdb Format Compatible tests',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Run RocksDB debug unit test',
+ 'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB no compression
+#
+run_no_compression()
+{
+ export TEST_TMPDIR=/dev/shm/rocksdb
+ rm -rf /dev/shm/rocksdb
+ mkdir /dev/shm/rocksdb
+ make clean
+ cat build_tools/fbcode_config.sh | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh
+ mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh
+ cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile
+ mv .tmp.Makefile Makefile
+ make $DEBUG J=1 check
+}
+
+NO_COMPRESSION_COMMANDS="[
+ {
+ 'name':'Rocksdb No Compression tests',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Run RocksDB debug unit test',
+ 'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB regression
+#
+run_regression()
+{
+ time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX)
+
+ # ======= report size to ODS ========
+
+ # parameters: $1 -- key, $2 -- value
+ function send_size_to_ods {
+ curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \
+ --connect-timeout 60
+ }
+
+ # === normal build ===
+ make clean
+ make -j$(nproc) static_lib
+ send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a)
+ strip librocksdb.a
+ send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a)
+
+ make -j$(nproc) shared_lib
+ send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`)
+ strip `readlink -f librocksdb.so`
+ send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
+
+ # === lite build ===
+ make clean
+ OPT=-DROCKSDB_LITE make -j$(nproc) static_lib
+ send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a)
+ strip librocksdb.a
+ send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a)
+
+ OPT=-DROCKSDB_LITE make -j$(nproc) shared_lib
+ send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`)
+ strip `readlink -f librocksdb.so`
+ send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
+}
+
+REGRESSION_COMMANDS="[
+ {
+ 'name':'Rocksdb regression commands',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Make and run script',
+ 'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+#
+# RocksDB Java build
+#
+JAVA_BUILD_TEST_COMMANDS="[
+ {
+ 'name':'Rocksdb Java Build',
+ 'oncall':'$ONCALL',
+ 'steps': [
+ $CLEANUP_ENV,
+ {
+ 'name':'Build RocksDB for Java',
+ 'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL',
+ 'user':'root',
+ $PARSER
+ },
+ ],
+ $REPORT
+ }
+]"
+
+
+case $1 in
+ unit)
+ echo $UNIT_TEST_COMMANDS
+ ;;
+ unit_non_shm)
+ echo $UNIT_TEST_NON_SHM_COMMANDS
+ ;;
+ release)
+ echo $RELEASE_BUILD_COMMANDS
+ ;;
+ unit_481)
+ echo $UNIT_TEST_COMMANDS_481
+ ;;
+ release_481)
+ echo $RELEASE_BUILD_COMMANDS_481
+ ;;
+ clang_unit)
+ echo $CLANG_UNIT_TEST_COMMANDS
+ ;;
+ clang_release)
+ echo $CLANG_RELEASE_BUILD_COMMANDS
+ ;;
+ clang_analyze)
+ echo $CLANG_ANALYZE_COMMANDS
+ ;;
+ code_cov)
+ echo $CODE_COV_COMMANDS
+ ;;
+ unity)
+ echo $UNITY_COMMANDS
+ ;;
+ lite)
+ echo $LITE_BUILD_COMMANDS
+ ;;
+ lite_test)
+ echo $LITE_UNIT_TEST_COMMANDS
+ ;;
+ stress_crash)
+ echo $STRESS_CRASH_TEST_COMMANDS
+ ;;
+ write_stress)
+ echo $WRITE_STRESS_COMMANDS
+ ;;
+ asan)
+ echo $ASAN_TEST_COMMANDS
+ ;;
+ asan_crash)
+ echo $ASAN_CRASH_TEST_COMMANDS
+ ;;
+ ubsan)
+ echo $UBSAN_TEST_COMMANDS
+ ;;
+ ubsan_crash)
+ echo $UBSAN_CRASH_TEST_COMMANDS
+ ;;
+ valgrind)
+ echo $VALGRIND_TEST_COMMANDS
+ ;;
+ tsan)
+ echo $TSAN_UNIT_TEST_COMMANDS
+ ;;
+ tsan_crash)
+ echo $TSAN_CRASH_TEST_COMMANDS
+ ;;
+ format_compatible)
+ echo $FORMAT_COMPATIBLE_COMMANDS
+ ;;
+ run_format_compatible)
+ run_format_compatible
+ ;;
+ no_compression)
+ echo $NO_COMPRESSION_COMMANDS
+ ;;
+ run_no_compression)
+ run_no_compression
+ ;;
+ regression)
+ echo $REGRESSION_COMMANDS
+ ;;
+ run_regression)
+ run_regression
+ ;;
+ java_build)
+ echo $JAVA_BUILD_TEST_COMMANDS
+ ;;
+ *)
+ echo "Invalid determinator command"
+ ;;
+esac
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/update_dependencies.sh
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/update_dependencies.sh b/thirdparty/rocksdb/build_tools/update_dependencies.sh
new file mode 100755
index 0000000..c7b9932
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/update_dependencies.sh
@@ -0,0 +1,131 @@
+#!/bin/sh
+#
+# Update dependencies.sh file with the latest avaliable versions
+
+BASEDIR=$(dirname $0)
+OUTPUT=""
+
+function log_variable()
+{
+ echo "$1=${!1}" >> "$OUTPUT"
+}
+
+
+TP2_LATEST="/mnt/vol/engshare/fbcode/third-party2"
+## $1 => lib name
+## $2 => lib version (if not provided, will try to pick latest)
+## $3 => platform (if not provided, will try to pick latest gcc)
+##
+## get_lib_base will set a variable named ${LIB_NAME}_BASE to the lib location
+function get_lib_base()
+{
+ local lib_name=$1
+ local lib_version=$2
+ local lib_platform=$3
+
+ local result="$TP2_LATEST/$lib_name/"
+
+ # Lib Version
+ if [ -z "$lib_version" ] || [ "$lib_version" = "LATEST" ]; then
+ # version is not provided, use latest
+ result=`ls -dr1v $result/*/ | head -n1`
+ else
+ result="$result/$lib_version/"
+ fi
+
+ # Lib Platform
+ if [ -z "$lib_platform" ]; then
+ # platform is not provided, use latest gcc
+ result=`ls -dr1v $result/gcc-*[^fb]/ | head -n1`
+ else
+ result="$result/$lib_platform/"
+ fi
+
+ result=`ls -1d $result/*/ | head -n1`
+
+ # lib_name => LIB_NAME_BASE
+ local __res_var=${lib_name^^}"_BASE"
+ __res_var=`echo $__res_var | tr - _`
+ # LIB_NAME_BASE=$result
+ eval $__res_var=`readlink -f $result`
+
+ log_variable $__res_var
+}
+
+###########################################################
+# 5.x dependencies #
+###########################################################
+
+OUTPUT="$BASEDIR/dependencies.sh"
+
+rm -f "$OUTPUT"
+touch "$OUTPUT"
+
+echo "Writing dependencies to $OUTPUT"
+
+# Compilers locations
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/5.x/centos6-native/*/`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
+
+log_variable GCC_BASE
+log_variable CLANG_BASE
+
+# Libraries locations
+get_lib_base libgcc 5.x
+get_lib_base glibc 2.23
+get_lib_base snappy LATEST gcc-5-glibc-2.23
+get_lib_base zlib LATEST
+get_lib_base bzip2 LATEST
+get_lib_base lz4 LATEST
+get_lib_base zstd LATEST
+get_lib_base gflags LATEST
+get_lib_base jemalloc LATEST
+get_lib_base numa LATEST
+get_lib_base libunwind LATEST
+get_lib_base tbb 4.0_update2 gcc-5-glibc-2.23
+
+get_lib_base kernel-headers LATEST
+get_lib_base binutils LATEST centos6-native
+get_lib_base valgrind 3.10.0 gcc-5-glibc-2.23
+get_lib_base lua 5.2.3 gcc-5-glibc-2.23
+
+git diff $OUTPUT
+
+###########################################################
+# 4.8.1 dependencies #
+###########################################################
+
+OUTPUT="$BASEDIR/dependencies_4.8.1.sh"
+
+rm -f "$OUTPUT"
+touch "$OUTPUT"
+
+echo "Writing 4.8.1 dependencies to $OUTPUT"
+
+# Compilers locations
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
+
+log_variable GCC_BASE
+log_variable CLANG_BASE
+
+# Libraries locations
+get_lib_base libgcc 4.8.1 gcc-4.8.1-glibc-2.17
+get_lib_base glibc 2.17 gcc-4.8.1-glibc-2.17
+get_lib_base snappy LATEST gcc-4.8.1-glibc-2.17
+get_lib_base zlib LATEST gcc-4.8.1-glibc-2.17
+get_lib_base bzip2 LATEST gcc-4.8.1-glibc-2.17
+get_lib_base lz4 LATEST gcc-4.8.1-glibc-2.17
+get_lib_base zstd LATEST gcc-4.8.1-glibc-2.17
+get_lib_base gflags LATEST gcc-4.8.1-glibc-2.17
+get_lib_base jemalloc LATEST gcc-4.8.1-glibc-2.17
+get_lib_base numa LATEST gcc-4.8.1-glibc-2.17
+get_lib_base libunwind LATEST gcc-4.8.1-glibc-2.17
+get_lib_base tbb 4.0_update2 gcc-4.8.1-glibc-2.17
+
+get_lib_base kernel-headers LATEST gcc-4.8.1-glibc-2.17
+get_lib_base binutils LATEST centos6-native
+get_lib_base valgrind 3.8.1 gcc-4.8.1-glibc-2.17
+get_lib_base lua 5.2.3 centos6-native
+
+git diff $OUTPUT
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/version.sh
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/version.sh b/thirdparty/rocksdb/build_tools/version.sh
new file mode 100755
index 0000000..f3ca98c
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/version.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+if [ "$#" = "0" ]; then
+ echo "Usage: $0 major|minor|patch|full"
+ exit 1
+fi
+
+if [ "$1" = "major" ]; then
+ cat include/rocksdb/version.h | grep MAJOR | head -n1 | awk '{print $3}'
+fi
+if [ "$1" = "minor" ]; then
+ cat include/rocksdb/version.h | grep MINOR | head -n1 | awk '{print $3}'
+fi
+if [ "$1" = "patch" ]; then
+ cat include/rocksdb/version.h | grep PATCH | head -n1 | awk '{print $3}'
+fi
+if [ "$1" = "full" ]; then
+ awk '/#define ROCKSDB/ { env[$2] = $3 }
+ END { printf "%s.%s.%s\n", env["ROCKSDB_MAJOR"],
+ env["ROCKSDB_MINOR"],
+ env["ROCKSDB_PATCH"] }' \
+ include/rocksdb/version.h
+fi
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/cache/cache_bench.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/cache/cache_bench.cc b/thirdparty/rocksdb/cache/cache_bench.cc
new file mode 100644
index 0000000..16c2ced
--- /dev/null
+++ b/thirdparty/rocksdb/cache/cache_bench.cc
@@ -0,0 +1,284 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+#ifndef GFLAGS
+#include <cstdio>
+int main() {
+ fprintf(stderr, "Please install gflags to run rocksdb tools\n");
+ return 1;
+}
+#else
+
+#include <inttypes.h>
+#include <sys/types.h>
+#include <stdio.h>
+#include <gflags/gflags.h>
+
+#include "rocksdb/db.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/env.h"
+#include "port/port.h"
+#include "util/mutexlock.h"
+#include "util/random.h"
+
+using GFLAGS::ParseCommandLineFlags;
+
+static const uint32_t KB = 1024;
+
+DEFINE_int32(threads, 16, "Number of concurrent threads to run.");
+DEFINE_int64(cache_size, 8 * KB * KB,
+ "Number of bytes to use as a cache of uncompressed data.");
+DEFINE_int32(num_shard_bits, 4, "shard_bits.");
+
+DEFINE_int64(max_key, 1 * KB * KB * KB, "Max number of key to place in cache");
+DEFINE_uint64(ops_per_thread, 1200000, "Number of operations per thread.");
+
+DEFINE_bool(populate_cache, false, "Populate cache before operations");
+DEFINE_int32(insert_percent, 40,
+ "Ratio of insert to total workload (expressed as a percentage)");
+DEFINE_int32(lookup_percent, 50,
+ "Ratio of lookup to total workload (expressed as a percentage)");
+DEFINE_int32(erase_percent, 10,
+ "Ratio of erase to total workload (expressed as a percentage)");
+
+DEFINE_bool(use_clock_cache, false, "");
+
+namespace rocksdb {
+
+class CacheBench;
+namespace {
+void deleter(const Slice& key, void* value) {
+ delete reinterpret_cast<char *>(value);
+}
+
+// State shared by all concurrent executions of the same benchmark.
+class SharedState {
+ public:
+ explicit SharedState(CacheBench* cache_bench)
+ : cv_(&mu_),
+ num_threads_(FLAGS_threads),
+ num_initialized_(0),
+ start_(false),
+ num_done_(0),
+ cache_bench_(cache_bench) {
+ }
+
+ ~SharedState() {}
+
+ port::Mutex* GetMutex() {
+ return &mu_;
+ }
+
+ port::CondVar* GetCondVar() {
+ return &cv_;
+ }
+
+ CacheBench* GetCacheBench() const {
+ return cache_bench_;
+ }
+
+ void IncInitialized() {
+ num_initialized_++;
+ }
+
+ void IncDone() {
+ num_done_++;
+ }
+
+ bool AllInitialized() const {
+ return num_initialized_ >= num_threads_;
+ }
+
+ bool AllDone() const {
+ return num_done_ >= num_threads_;
+ }
+
+ void SetStart() {
+ start_ = true;
+ }
+
+ bool Started() const {
+ return start_;
+ }
+
+ private:
+ port::Mutex mu_;
+ port::CondVar cv_;
+
+ const uint64_t num_threads_;
+ uint64_t num_initialized_;
+ bool start_;
+ uint64_t num_done_;
+
+ CacheBench* cache_bench_;
+};
+
+// Per-thread state for concurrent executions of the same benchmark.
+struct ThreadState {
+ uint32_t tid;
+ Random rnd;
+ SharedState* shared;
+
+ ThreadState(uint32_t index, SharedState* _shared)
+ : tid(index), rnd(1000 + index), shared(_shared) {}
+};
+} // namespace
+
+class CacheBench {
+ public:
+ CacheBench() : num_threads_(FLAGS_threads) {
+ if (FLAGS_use_clock_cache) {
+ cache_ = NewClockCache(FLAGS_cache_size, FLAGS_num_shard_bits);
+ if (!cache_) {
+ fprintf(stderr, "Clock cache not supported.\n");
+ exit(1);
+ }
+ } else {
+ cache_ = NewLRUCache(FLAGS_cache_size, FLAGS_num_shard_bits);
+ }
+ }
+
+ ~CacheBench() {}
+
+ void PopulateCache() {
+ Random rnd(1);
+ for (int64_t i = 0; i < FLAGS_cache_size; i++) {
+ uint64_t rand_key = rnd.Next() % FLAGS_max_key;
+ // Cast uint64* to be char*, data would be copied to cache
+ Slice key(reinterpret_cast<char*>(&rand_key), 8);
+ // do insert
+ cache_->Insert(key, new char[10], 1, &deleter);
+ }
+ }
+
+ bool Run() {
+ rocksdb::Env* env = rocksdb::Env::Default();
+
+ PrintEnv();
+ SharedState shared(this);
+ std::vector<ThreadState*> threads(num_threads_);
+ for (uint32_t i = 0; i < num_threads_; i++) {
+ threads[i] = new ThreadState(i, &shared);
+ env->StartThread(ThreadBody, threads[i]);
+ }
+ {
+ MutexLock l(shared.GetMutex());
+ while (!shared.AllInitialized()) {
+ shared.GetCondVar()->Wait();
+ }
+ // Record start time
+ uint64_t start_time = env->NowMicros();
+
+ // Start all threads
+ shared.SetStart();
+ shared.GetCondVar()->SignalAll();
+
+ // Wait threads to complete
+ while (!shared.AllDone()) {
+ shared.GetCondVar()->Wait();
+ }
+
+ // Record end time
+ uint64_t end_time = env->NowMicros();
+ double elapsed = static_cast<double>(end_time - start_time) * 1e-6;
+ uint32_t qps = static_cast<uint32_t>(
+ static_cast<double>(FLAGS_threads * FLAGS_ops_per_thread) / elapsed);
+ fprintf(stdout, "Complete in %.3f s; QPS = %u\n", elapsed, qps);
+ }
+ return true;
+ }
+
+ private:
+ std::shared_ptr<Cache> cache_;
+ uint32_t num_threads_;
+
+ static void ThreadBody(void* v) {
+ ThreadState* thread = reinterpret_cast<ThreadState*>(v);
+ SharedState* shared = thread->shared;
+
+ {
+ MutexLock l(shared->GetMutex());
+ shared->IncInitialized();
+ if (shared->AllInitialized()) {
+ shared->GetCondVar()->SignalAll();
+ }
+ while (!shared->Started()) {
+ shared->GetCondVar()->Wait();
+ }
+ }
+ thread->shared->GetCacheBench()->OperateCache(thread);
+
+ {
+ MutexLock l(shared->GetMutex());
+ shared->IncDone();
+ if (shared->AllDone()) {
+ shared->GetCondVar()->SignalAll();
+ }
+ }
+ }
+
+ void OperateCache(ThreadState* thread) {
+ for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
+ uint64_t rand_key = thread->rnd.Next() % FLAGS_max_key;
+ // Cast uint64* to be char*, data would be copied to cache
+ Slice key(reinterpret_cast<char*>(&rand_key), 8);
+ int32_t prob_op = thread->rnd.Uniform(100);
+ if (prob_op >= 0 && prob_op < FLAGS_insert_percent) {
+ // do insert
+ cache_->Insert(key, new char[10], 1, &deleter);
+ } else if (prob_op -= FLAGS_insert_percent &&
+ prob_op < FLAGS_lookup_percent) {
+ // do lookup
+ auto handle = cache_->Lookup(key);
+ if (handle) {
+ cache_->Release(handle);
+ }
+ } else if (prob_op -= FLAGS_lookup_percent &&
+ prob_op < FLAGS_erase_percent) {
+ // do erase
+ cache_->Erase(key);
+ }
+ }
+ }
+
+ void PrintEnv() const {
+ printf("RocksDB version : %d.%d\n", kMajorVersion, kMinorVersion);
+ printf("Number of threads : %d\n", FLAGS_threads);
+ printf("Ops per thread : %" PRIu64 "\n", FLAGS_ops_per_thread);
+ printf("Cache size : %" PRIu64 "\n", FLAGS_cache_size);
+ printf("Num shard bits : %d\n", FLAGS_num_shard_bits);
+ printf("Max key : %" PRIu64 "\n", FLAGS_max_key);
+ printf("Populate cache : %d\n", FLAGS_populate_cache);
+ printf("Insert percentage : %d%%\n", FLAGS_insert_percent);
+ printf("Lookup percentage : %d%%\n", FLAGS_lookup_percent);
+ printf("Erase percentage : %d%%\n", FLAGS_erase_percent);
+ printf("----------------------------\n");
+ }
+};
+} // namespace rocksdb
+
+int main(int argc, char** argv) {
+ ParseCommandLineFlags(&argc, &argv, true);
+
+ if (FLAGS_threads <= 0) {
+ fprintf(stderr, "threads number <= 0\n");
+ exit(1);
+ }
+
+ rocksdb::CacheBench bench;
+ if (FLAGS_populate_cache) {
+ bench.PopulateCache();
+ }
+ if (bench.Run()) {
+ return 0;
+ } else {
+ return 1;
+ }
+}
+
+#endif // GFLAGS
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/cache/clock_cache.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/cache/clock_cache.cc b/thirdparty/rocksdb/cache/clock_cache.cc
new file mode 100644
index 0000000..7e42714
--- /dev/null
+++ b/thirdparty/rocksdb/cache/clock_cache.cc
@@ -0,0 +1,729 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "cache/clock_cache.h"
+
+#ifndef SUPPORT_CLOCK_CACHE
+
+namespace rocksdb {
+
+std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
+ bool strict_capacity_limit) {
+ // Clock cache not supported.
+ return nullptr;
+}
+
+} // namespace rocksdb
+
+#else
+
+#include <assert.h>
+#include <atomic>
+#include <deque>
+
+// "tbb/concurrent_hash_map.h" requires RTTI if exception is enabled.
+// Disable it so users can chooose to disable RTTI.
+#ifndef ROCKSDB_USE_RTTI
+#define TBB_USE_EXCEPTIONS 0
+#endif
+#include "tbb/concurrent_hash_map.h"
+
+#include "cache/sharded_cache.h"
+#include "port/port.h"
+#include "util/autovector.h"
+#include "util/mutexlock.h"
+
+namespace rocksdb {
+
+namespace {
+
+// An implementation of the Cache interface based on CLOCK algorithm, with
+// better concurrent performance than LRUCache. The idea of CLOCK algorithm
+// is to maintain all cache entries in a circular list, and an iterator
+// (the "head") pointing to the last examined entry. Eviction starts from the
+// current head. Each entry is given a second chance before eviction, if it
+// has been access since last examine. In contrast to LRU, no modification
+// to the internal data-structure (except for flipping the usage bit) needs
+// to be done upon lookup. This gives us oppertunity to implement a cache
+// with better concurrency.
+//
+// Each cache entry is represented by a cache handle, and all the handles
+// are arranged in a circular list, as describe above. Upon erase of an entry,
+// we never remove the handle. Instead, the handle is put into a recycle bin
+// to be re-use. This is to avoid memory dealocation, which is hard to deal
+// with in concurrent environment.
+//
+// The cache also maintains a concurrent hash map for lookup. Any concurrent
+// hash map implementation should do the work. We currently use
+// tbb::concurrent_hash_map because it supports concurrent erase.
+//
+// Each cache handle has the following flags and counters, which are squeeze
+// in an atomic interger, to make sure the handle always be in a consistent
+// state:
+//
+// * In-cache bit: whether the entry is reference by the cache itself. If
+// an entry is in cache, its key would also be available in the hash map.
+// * Usage bit: whether the entry has been access by user since last
+// examine for eviction. Can be reset by eviction.
+// * Reference count: reference count by user.
+//
+// An entry can be reference only when it's in cache. An entry can be evicted
+// only when it is in cache, has no usage since last examine, and reference
+// count is zero.
+//
+// The follow figure shows a possible layout of the cache. Boxes represents
+// cache handles and numbers in each box being in-cache bit, usage bit and
+// reference count respectively.
+//
+// hash map:
+// +-------+--------+
+// | key | handle |
+// +-------+--------+
+// | "foo" | 5 |-------------------------------------+
+// +-------+--------+ |
+// | "bar" | 2 |--+ |
+// +-------+--------+ | |
+// | |
+// head | |
+// | | |
+// circular list: | | |
+// +-------+ +-------+ +-------+ +-------+ +-------+ +-------
+// |(0,0,0)|---|(1,1,0)|---|(0,0,0)|---|(0,1,3)|---|(1,0,0)|---| ...
+// +-------+ +-------+ +-------+ +-------+ +-------+ +-------
+// | |
+// +-------+ +-----------+
+// | |
+// +---+---+
+// recycle bin: | 1 | 3 |
+// +---+---+
+//
+// Suppose we try to insert "baz" into the cache at this point and the cache is
+// full. The cache will first look for entries to evict, starting from where
+// head points to (the second entry). It resets usage bit of the second entry,
+// skips the third and fourth entry since they are not in cache, and finally
+// evict the fifth entry ("foo"). It looks at recycle bin for available handle,
+// grabs handle 3, and insert the key into the handle. The following figure
+// shows the resulting layout.
+//
+// hash map:
+// +-------+--------+
+// | key | handle |
+// +-------+--------+
+// | "baz" | 3 |-------------+
+// +-------+--------+ |
+// | "bar" | 2 |--+ |
+// +-------+--------+ | |
+// | |
+// | | head
+// | | |
+// circular list: | | |
+// +-------+ +-------+ +-------+ +-------+ +-------+ +-------
+// |(0,0,0)|---|(1,0,0)|---|(1,0,0)|---|(0,1,3)|---|(0,0,0)|---| ...
+// +-------+ +-------+ +-------+ +-------+ +-------+ +-------
+// | |
+// +-------+ +-----------------------------------+
+// | |
+// +---+---+
+// recycle bin: | 1 | 5 |
+// +---+---+
+//
+// A global mutex guards the circular list, the head, and the recycle bin.
+// We additionally require that modifying the hash map needs to hold the mutex.
+// As such, Modifying the cache (such as Insert() and Erase()) require to
+// hold the mutex. Lookup() only access the hash map and the flags associated
+// with each handle, and don't require explicit locking. Release() has to
+// acquire the mutex only when it releases the last reference to the entry and
+// the entry has been erased from cache explicitly. A future improvement could
+// be to remove the mutex completely.
+//
+// Benchmark:
+// We run readrandom db_bench on a test DB of size 13GB, with size of each
+// level:
+//
+// Level Files Size(MB)
+// -------------------------
+// L0 1 0.01
+// L1 18 17.32
+// L2 230 182.94
+// L3 1186 1833.63
+// L4 4602 8140.30
+//
+// We test with both 32 and 16 read threads, with 2GB cache size (the whole DB
+// doesn't fits in) and 64GB cache size (the whole DB can fit in cache), and
+// whether to put index and filter blocks in block cache. The benchmark runs
+// with
+// with RocksDB 4.10. We got the following result:
+//
+// Threads Cache Cache ClockCache LRUCache
+// Size Index/Filter Throughput(MB/s) Hit Throughput(MB/s) Hit
+// 32 2GB yes 466.7 85.9% 433.7 86.5%
+// 32 2GB no 529.9 72.7% 532.7 73.9%
+// 32 64GB yes 649.9 99.9% 507.9 99.9%
+// 32 64GB no 740.4 99.9% 662.8 99.9%
+// 16 2GB yes 278.4 85.9% 283.4 86.5%
+// 16 2GB no 318.6 72.7% 335.8 73.9%
+// 16 64GB yes 391.9 99.9% 353.3 99.9%
+// 16 64GB no 433.8 99.8% 419.4 99.8%
+
+// Cache entry meta data.
+struct CacheHandle {
+ Slice key;
+ uint32_t hash;
+ void* value;
+ size_t charge;
+ void (*deleter)(const Slice&, void* value);
+
+ // Flags and counters associated with the cache handle:
+ // lowest bit: n-cache bit
+ // second lowest bit: usage bit
+ // the rest bits: reference count
+ // The handle is unused when flags equals to 0. The thread decreases the count
+ // to 0 is responsible to put the handle back to recycle_ and cleanup memory.
+ std::atomic<uint32_t> flags;
+
+ CacheHandle() = default;
+
+ CacheHandle(const CacheHandle& a) { *this = a; }
+
+ CacheHandle(const Slice& k, void* v,
+ void (*del)(const Slice& key, void* value))
+ : key(k), value(v), deleter(del) {}
+
+ CacheHandle& operator=(const CacheHandle& a) {
+ // Only copy members needed for deletion.
+ key = a.key;
+ value = a.value;
+ deleter = a.deleter;
+ return *this;
+ }
+};
+
+// Key of hash map. We store hash value with the key for convenience.
+struct CacheKey {
+ Slice key;
+ uint32_t hash_value;
+
+ CacheKey() = default;
+
+ CacheKey(const Slice& k, uint32_t h) {
+ key = k;
+ hash_value = h;
+ }
+
+ static bool equal(const CacheKey& a, const CacheKey& b) {
+ return a.hash_value == b.hash_value && a.key == b.key;
+ }
+
+ static size_t hash(const CacheKey& a) {
+ return static_cast<size_t>(a.hash_value);
+ }
+};
+
+struct CleanupContext {
+ // List of values to be deleted, along with the key and deleter.
+ autovector<CacheHandle> to_delete_value;
+
+ // List of keys to be deleted.
+ autovector<const char*> to_delete_key;
+};
+
+// A cache shard which maintains its own CLOCK cache.
+class ClockCacheShard : public CacheShard {
+ public:
+ // Hash map type.
+ typedef tbb::concurrent_hash_map<CacheKey, CacheHandle*, CacheKey> HashTable;
+
+ ClockCacheShard();
+ ~ClockCacheShard();
+
+ // Interfaces
+ virtual void SetCapacity(size_t capacity) override;
+ virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
+ virtual Status Insert(const Slice& key, uint32_t hash, void* value,
+ size_t charge,
+ void (*deleter)(const Slice& key, void* value),
+ Cache::Handle** handle,
+ Cache::Priority priority) override;
+ virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
+ // If the entry in in cache, increase reference count and return true.
+ // Return false otherwise.
+ //
+ // Not necessary to hold mutex_ before being called.
+ virtual bool Ref(Cache::Handle* handle) override;
+ virtual bool Release(Cache::Handle* handle,
+ bool force_erase = false) override;
+ virtual void Erase(const Slice& key, uint32_t hash) override;
+ bool EraseAndConfirm(const Slice& key, uint32_t hash,
+ CleanupContext* context);
+ virtual size_t GetUsage() const override;
+ virtual size_t GetPinnedUsage() const override;
+ virtual void EraseUnRefEntries() override;
+ virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+ bool thread_safe) override;
+
+ private:
+ static const uint32_t kInCacheBit = 1;
+ static const uint32_t kUsageBit = 2;
+ static const uint32_t kRefsOffset = 2;
+ static const uint32_t kOneRef = 1 << kRefsOffset;
+
+ // Helper functions to extract cache handle flags and counters.
+ static bool InCache(uint32_t flags) { return flags & kInCacheBit; }
+ static bool HasUsage(uint32_t flags) { return flags & kUsageBit; }
+ static uint32_t CountRefs(uint32_t flags) { return flags >> kRefsOffset; }
+
+ // Decrease reference count of the entry. If this decreases the count to 0,
+ // recycle the entry. If set_usage is true, also set the usage bit.
+ //
+ // returns true if a value is erased.
+ //
+ // Not necessary to hold mutex_ before being called.
+ bool Unref(CacheHandle* handle, bool set_usage, CleanupContext* context);
+
+ // Unset in-cache bit of the entry. Recycle the handle if necessary.
+ //
+ // returns true if a value is erased.
+ //
+ // Has to hold mutex_ before being called.
+ bool UnsetInCache(CacheHandle* handle, CleanupContext* context);
+
+ // Put the handle back to recycle_ list, and put the value associated with
+ // it into to-be-deleted list. It doesn't cleanup the key as it might be
+ // reused by another handle.
+ //
+ // Has to hold mutex_ before being called.
+ void RecycleHandle(CacheHandle* handle, CleanupContext* context);
+
+ // Delete keys and values in to-be-deleted list. Call the method without
+ // holding mutex, as destructors can be expensive.
+ void Cleanup(const CleanupContext& context);
+
+ // Examine the handle for eviction. If the handle is in cache, usage bit is
+ // not set, and referece count is 0, evict it from cache. Otherwise unset
+ // the usage bit.
+ //
+ // Has to hold mutex_ before being called.
+ bool TryEvict(CacheHandle* value, CleanupContext* context);
+
+ // Scan through the circular list, evict entries until we get enough capacity
+ // for new cache entry of specific size. Return true if success, false
+ // otherwise.
+ //
+ // Has to hold mutex_ before being called.
+ bool EvictFromCache(size_t charge, CleanupContext* context);
+
+ CacheHandle* Insert(const Slice& key, uint32_t hash, void* value,
+ size_t change,
+ void (*deleter)(const Slice& key, void* value),
+ bool hold_reference, CleanupContext* context);
+
+ // Guards list_, head_, and recycle_. In addition, updating table_ also has
+ // to hold the mutex, to avoid the cache being in inconsistent state.
+ mutable port::Mutex mutex_;
+
+ // The circular list of cache handles. Initially the list is empty. Once a
+ // handle is needed by insertion, and no more handles are available in
+ // recycle bin, one more handle is appended to the end.
+ //
+ // We use std::deque for the circular list because we want to make sure
+ // pointers to handles are valid through out the life-cycle of the cache
+ // (in contrast to std::vector), and be able to grow the list (in contrast
+ // to statically allocated arrays).
+ std::deque<CacheHandle> list_;
+
+ // Pointer to the next handle in the circular list to be examine for
+ // eviction.
+ size_t head_;
+
+ // Recycle bin of cache handles.
+ autovector<CacheHandle*> recycle_;
+
+ // Maximum cache size.
+ std::atomic<size_t> capacity_;
+
+ // Current total size of the cache.
+ std::atomic<size_t> usage_;
+
+ // Total un-released cache size.
+ std::atomic<size_t> pinned_usage_;
+
+ // Whether allow insert into cache if cache is full.
+ std::atomic<bool> strict_capacity_limit_;
+
+ // Hash table (tbb::concurrent_hash_map) for lookup.
+ HashTable table_;
+};
+
+ClockCacheShard::ClockCacheShard()
+ : head_(0), usage_(0), pinned_usage_(0), strict_capacity_limit_(false) {}
+
+ClockCacheShard::~ClockCacheShard() {
+ for (auto& handle : list_) {
+ uint32_t flags = handle.flags.load(std::memory_order_relaxed);
+ if (InCache(flags) || CountRefs(flags) > 0) {
+ (*handle.deleter)(handle.key, handle.value);
+ delete[] handle.key.data();
+ }
+ }
+}
+
+size_t ClockCacheShard::GetUsage() const {
+ return usage_.load(std::memory_order_relaxed);
+}
+
+size_t ClockCacheShard::GetPinnedUsage() const {
+ return pinned_usage_.load(std::memory_order_relaxed);
+}
+
+void ClockCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+ bool thread_safe) {
+ if (thread_safe) {
+ mutex_.Lock();
+ }
+ for (auto& handle : list_) {
+ // Use relaxed semantics instead of acquire semantics since we are either
+ // holding mutex, or don't have thread safe requirement.
+ uint32_t flags = handle.flags.load(std::memory_order_relaxed);
+ if (InCache(flags)) {
+ callback(handle.value, handle.charge);
+ }
+ }
+ if (thread_safe) {
+ mutex_.Unlock();
+ }
+}
+
+void ClockCacheShard::RecycleHandle(CacheHandle* handle,
+ CleanupContext* context) {
+ mutex_.AssertHeld();
+ assert(!InCache(handle->flags) && CountRefs(handle->flags) == 0);
+ context->to_delete_key.push_back(handle->key.data());
+ context->to_delete_value.emplace_back(*handle);
+ handle->key.clear();
+ handle->value = nullptr;
+ handle->deleter = nullptr;
+ recycle_.push_back(handle);
+ usage_.fetch_sub(handle->charge, std::memory_order_relaxed);
+}
+
+void ClockCacheShard::Cleanup(const CleanupContext& context) {
+ for (const CacheHandle& handle : context.to_delete_value) {
+ if (handle.deleter) {
+ (*handle.deleter)(handle.key, handle.value);
+ }
+ }
+ for (const char* key : context.to_delete_key) {
+ delete[] key;
+ }
+}
+
+bool ClockCacheShard::Ref(Cache::Handle* h) {
+ auto handle = reinterpret_cast<CacheHandle*>(h);
+ // CAS loop to increase reference count.
+ uint32_t flags = handle->flags.load(std::memory_order_relaxed);
+ while (InCache(flags)) {
+ // Use acquire semantics on success, as further operations on the cache
+ // entry has to be order after reference count is increased.
+ if (handle->flags.compare_exchange_weak(flags, flags + kOneRef,
+ std::memory_order_acquire,
+ std::memory_order_relaxed)) {
+ if (CountRefs(flags) == 0) {
+ // No reference count before the operation.
+ pinned_usage_.fetch_add(handle->charge, std::memory_order_relaxed);
+ }
+ return true;
+ }
+ }
+ return false;
+}
+
+bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
+ CleanupContext* context) {
+ if (set_usage) {
+ handle->flags.fetch_or(kUsageBit, std::memory_order_relaxed);
+ }
+ // Use acquire-release semantics as previous operations on the cache entry
+ // has to be order before reference count is decreased, and potential cleanup
+ // of the entry has to be order after.
+ uint32_t flags = handle->flags.fetch_sub(kOneRef, std::memory_order_acq_rel);
+ assert(CountRefs(flags) > 0);
+ if (CountRefs(flags) == 1) {
+ // this is the last reference.
+ pinned_usage_.fetch_sub(handle->charge, std::memory_order_relaxed);
+ // Cleanup if it is the last reference.
+ if (!InCache(flags)) {
+ MutexLock l(&mutex_);
+ RecycleHandle(handle, context);
+ }
+ }
+ return context->to_delete_value.size();
+}
+
+bool ClockCacheShard::UnsetInCache(CacheHandle* handle,
+ CleanupContext* context) {
+ mutex_.AssertHeld();
+ // Use acquire-release semantics as previous operations on the cache entry
+ // has to be order before reference count is decreased, and potential cleanup
+ // of the entry has to be order after.
+ uint32_t flags =
+ handle->flags.fetch_and(~kInCacheBit, std::memory_order_acq_rel);
+ // Cleanup if it is the last reference.
+ if (InCache(flags) && CountRefs(flags) == 0) {
+ RecycleHandle(handle, context);
+ }
+ return context->to_delete_value.size();
+}
+
+bool ClockCacheShard::TryEvict(CacheHandle* handle, CleanupContext* context) {
+ mutex_.AssertHeld();
+ uint32_t flags = kInCacheBit;
+ if (handle->flags.compare_exchange_strong(flags, 0, std::memory_order_acquire,
+ std::memory_order_relaxed)) {
+ bool erased __attribute__((__unused__)) =
+ table_.erase(CacheKey(handle->key, handle->hash));
+ assert(erased);
+ RecycleHandle(handle, context);
+ return true;
+ }
+ handle->flags.fetch_and(~kUsageBit, std::memory_order_relaxed);
+ return false;
+}
+
+bool ClockCacheShard::EvictFromCache(size_t charge, CleanupContext* context) {
+ size_t usage = usage_.load(std::memory_order_relaxed);
+ size_t capacity = capacity_.load(std::memory_order_relaxed);
+ if (usage == 0) {
+ return charge <= capacity;
+ }
+ size_t new_head = head_;
+ bool second_iteration = false;
+ while (usage + charge > capacity) {
+ assert(new_head < list_.size());
+ if (TryEvict(&list_[new_head], context)) {
+ usage = usage_.load(std::memory_order_relaxed);
+ }
+ new_head = (new_head + 1 >= list_.size()) ? 0 : new_head + 1;
+ if (new_head == head_) {
+ if (second_iteration) {
+ return false;
+ } else {
+ second_iteration = true;
+ }
+ }
+ }
+ head_ = new_head;
+ return true;
+}
+
+void ClockCacheShard::SetCapacity(size_t capacity) {
+ CleanupContext context;
+ {
+ MutexLock l(&mutex_);
+ capacity_.store(capacity, std::memory_order_relaxed);
+ EvictFromCache(0, &context);
+ }
+ Cleanup(context);
+}
+
+void ClockCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
+ strict_capacity_limit_.store(strict_capacity_limit,
+ std::memory_order_relaxed);
+}
+
+CacheHandle* ClockCacheShard::Insert(
+ const Slice& key, uint32_t hash, void* value, size_t charge,
+ void (*deleter)(const Slice& key, void* value), bool hold_reference,
+ CleanupContext* context) {
+ MutexLock l(&mutex_);
+ bool success = EvictFromCache(charge, context);
+ bool strict = strict_capacity_limit_.load(std::memory_order_relaxed);
+ if (!success && (strict || !hold_reference)) {
+ context->to_delete_key.push_back(key.data());
+ if (!hold_reference) {
+ context->to_delete_value.emplace_back(key, value, deleter);
+ }
+ return nullptr;
+ }
+ // Grab available handle from recycle bin. If recycle bin is empty, create
+ // and append new handle to end of circular list.
+ CacheHandle* handle = nullptr;
+ if (!recycle_.empty()) {
+ handle = recycle_.back();
+ recycle_.pop_back();
+ } else {
+ list_.emplace_back();
+ handle = &list_.back();
+ }
+ // Fill handle.
+ handle->key = key;
+ handle->hash = hash;
+ handle->value = value;
+ handle->charge = charge;
+ handle->deleter = deleter;
+ uint32_t flags = hold_reference ? kInCacheBit + kOneRef : kInCacheBit;
+ handle->flags.store(flags, std::memory_order_relaxed);
+ HashTable::accessor accessor;
+ if (table_.find(accessor, CacheKey(key, hash))) {
+ CacheHandle* existing_handle = accessor->second;
+ table_.erase(accessor);
+ UnsetInCache(existing_handle, context);
+ }
+ table_.insert(HashTable::value_type(CacheKey(key, hash), handle));
+ if (hold_reference) {
+ pinned_usage_.fetch_add(charge, std::memory_order_relaxed);
+ }
+ usage_.fetch_add(charge, std::memory_order_relaxed);
+ return handle;
+}
+
+Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
+ size_t charge,
+ void (*deleter)(const Slice& key, void* value),
+ Cache::Handle** out_handle,
+ Cache::Priority priority) {
+ CleanupContext context;
+ HashTable::accessor accessor;
+ char* key_data = new char[key.size()];
+ memcpy(key_data, key.data(), key.size());
+ Slice key_copy(key_data, key.size());
+ CacheHandle* handle = Insert(key_copy, hash, value, charge, deleter,
+ out_handle != nullptr, &context);
+ Status s;
+ if (out_handle != nullptr) {
+ if (handle == nullptr) {
+ s = Status::Incomplete("Insert failed due to LRU cache being full.");
+ } else {
+ *out_handle = reinterpret_cast<Cache::Handle*>(handle);
+ }
+ }
+ Cleanup(context);
+ return s;
+}
+
+Cache::Handle* ClockCacheShard::Lookup(const Slice& key, uint32_t hash) {
+ HashTable::const_accessor accessor;
+ if (!table_.find(accessor, CacheKey(key, hash))) {
+ return nullptr;
+ }
+ CacheHandle* handle = accessor->second;
+ accessor.release();
+ // Ref() could fail if another thread sneak in and evict/erase the cache
+ // entry before we are able to hold reference.
+ if (!Ref(reinterpret_cast<Cache::Handle*>(handle))) {
+ return nullptr;
+ }
+ // Double check the key since the handle may now representing another key
+ // if other threads sneak in, evict/erase the entry and re-used the handle
+ // for another cache entry.
+ if (hash != handle->hash || key != handle->key) {
+ CleanupContext context;
+ Unref(handle, false, &context);
+ // It is possible Unref() delete the entry, so we need to cleanup.
+ Cleanup(context);
+ return nullptr;
+ }
+ return reinterpret_cast<Cache::Handle*>(handle);
+}
+
+bool ClockCacheShard::Release(Cache::Handle* h, bool force_erase) {
+ CleanupContext context;
+ CacheHandle* handle = reinterpret_cast<CacheHandle*>(h);
+ bool erased = Unref(handle, true, &context);
+ if (force_erase && !erased) {
+ erased = EraseAndConfirm(handle->key, handle->hash, &context);
+ }
+ Cleanup(context);
+ return erased;
+}
+
+void ClockCacheShard::Erase(const Slice& key, uint32_t hash) {
+ CleanupContext context;
+ EraseAndConfirm(key, hash, &context);
+ Cleanup(context);
+}
+
+bool ClockCacheShard::EraseAndConfirm(const Slice& key, uint32_t hash,
+ CleanupContext* context) {
+ MutexLock l(&mutex_);
+ HashTable::accessor accessor;
+ bool erased = false;
+ if (table_.find(accessor, CacheKey(key, hash))) {
+ CacheHandle* handle = accessor->second;
+ table_.erase(accessor);
+ erased = UnsetInCache(handle, context);
+ }
+ return erased;
+}
+
+void ClockCacheShard::EraseUnRefEntries() {
+ CleanupContext context;
+ {
+ MutexLock l(&mutex_);
+ table_.clear();
+ for (auto& handle : list_) {
+ UnsetInCache(&handle, &context);
+ }
+ }
+ Cleanup(context);
+}
+
+class ClockCache : public ShardedCache {
+ public:
+ ClockCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit)
+ : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
+ int num_shards = 1 << num_shard_bits;
+ shards_ = new ClockCacheShard[num_shards];
+ SetCapacity(capacity);
+ SetStrictCapacityLimit(strict_capacity_limit);
+ }
+
+ virtual ~ClockCache() { delete[] shards_; }
+
+ virtual const char* Name() const override { return "ClockCache"; }
+
+ virtual CacheShard* GetShard(int shard) override {
+ return reinterpret_cast<CacheShard*>(&shards_[shard]);
+ }
+
+ virtual const CacheShard* GetShard(int shard) const override {
+ return reinterpret_cast<CacheShard*>(&shards_[shard]);
+ }
+
+ virtual void* Value(Handle* handle) override {
+ return reinterpret_cast<const CacheHandle*>(handle)->value;
+ }
+
+ virtual size_t GetCharge(Handle* handle) const override {
+ return reinterpret_cast<const CacheHandle*>(handle)->charge;
+ }
+
+ virtual uint32_t GetHash(Handle* handle) const override {
+ return reinterpret_cast<const CacheHandle*>(handle)->hash;
+ }
+
+ virtual void DisownData() override { shards_ = nullptr; }
+
+ private:
+ ClockCacheShard* shards_;
+};
+
+} // end anonymous namespace
+
+std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
+ bool strict_capacity_limit) {
+ if (num_shard_bits < 0) {
+ num_shard_bits = GetDefaultCacheShardBits(capacity);
+ }
+ return std::make_shared<ClockCache>(capacity, num_shard_bits,
+ strict_capacity_limit);
+}
+
+} // namespace rocksdb
+
+#endif // SUPPORT_CLOCK_CACHE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/cache/clock_cache.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/cache/clock_cache.h b/thirdparty/rocksdb/cache/clock_cache.h
new file mode 100644
index 0000000..1614c0e
--- /dev/null
+++ b/thirdparty/rocksdb/cache/clock_cache.h
@@ -0,0 +1,16 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#include "rocksdb/cache.h"
+
+#if defined(TBB) && !defined(ROCKSDB_LITE)
+#define SUPPORT_CLOCK_CACHE
+#endif
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/cache/lru_cache.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/cache/lru_cache.cc b/thirdparty/rocksdb/cache/lru_cache.cc
new file mode 100644
index 0000000..f833374
--- /dev/null
+++ b/thirdparty/rocksdb/cache/lru_cache.cc
@@ -0,0 +1,524 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include "cache/lru_cache.h"
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+
+#include "util/mutexlock.h"
+
+namespace rocksdb {
+
+LRUHandleTable::LRUHandleTable() : list_(nullptr), length_(0), elems_(0) {
+ Resize();
+}
+
+LRUHandleTable::~LRUHandleTable() {
+ ApplyToAllCacheEntries([](LRUHandle* h) {
+ if (h->refs == 1) {
+ h->Free();
+ }
+ });
+ delete[] list_;
+}
+
+LRUHandle* LRUHandleTable::Lookup(const Slice& key, uint32_t hash) {
+ return *FindPointer(key, hash);
+}
+
+LRUHandle* LRUHandleTable::Insert(LRUHandle* h) {
+ LRUHandle** ptr = FindPointer(h->key(), h->hash);
+ LRUHandle* old = *ptr;
+ h->next_hash = (old == nullptr ? nullptr : old->next_hash);
+ *ptr = h;
+ if (old == nullptr) {
+ ++elems_;
+ if (elems_ > length_) {
+ // Since each cache entry is fairly large, we aim for a small
+ // average linked list length (<= 1).
+ Resize();
+ }
+ }
+ return old;
+}
+
+LRUHandle* LRUHandleTable::Remove(const Slice& key, uint32_t hash) {
+ LRUHandle** ptr = FindPointer(key, hash);
+ LRUHandle* result = *ptr;
+ if (result != nullptr) {
+ *ptr = result->next_hash;
+ --elems_;
+ }
+ return result;
+}
+
+LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
+ LRUHandle** ptr = &list_[hash & (length_ - 1)];
+ while (*ptr != nullptr && ((*ptr)->hash != hash || key != (*ptr)->key())) {
+ ptr = &(*ptr)->next_hash;
+ }
+ return ptr;
+}
+
+void LRUHandleTable::Resize() {
+ uint32_t new_length = 16;
+ while (new_length < elems_ * 1.5) {
+ new_length *= 2;
+ }
+ LRUHandle** new_list = new LRUHandle*[new_length];
+ memset(new_list, 0, sizeof(new_list[0]) * new_length);
+ uint32_t count = 0;
+ for (uint32_t i = 0; i < length_; i++) {
+ LRUHandle* h = list_[i];
+ while (h != nullptr) {
+ LRUHandle* next = h->next_hash;
+ uint32_t hash = h->hash;
+ LRUHandle** ptr = &new_list[hash & (new_length - 1)];
+ h->next_hash = *ptr;
+ *ptr = h;
+ h = next;
+ count++;
+ }
+ }
+ assert(elems_ == count);
+ delete[] list_;
+ list_ = new_list;
+ length_ = new_length;
+}
+
+LRUCacheShard::LRUCacheShard()
+ : high_pri_pool_usage_(0), usage_(0), lru_usage_(0) {
+ // Make empty circular linked list
+ lru_.next = &lru_;
+ lru_.prev = &lru_;
+ lru_low_pri_ = &lru_;
+}
+
+LRUCacheShard::~LRUCacheShard() {}
+
+bool LRUCacheShard::Unref(LRUHandle* e) {
+ assert(e->refs > 0);
+ e->refs--;
+ return e->refs == 0;
+}
+
+// Call deleter and free
+
+void LRUCacheShard::EraseUnRefEntries() {
+ autovector<LRUHandle*> last_reference_list;
+ {
+ MutexLock l(&mutex_);
+ while (lru_.next != &lru_) {
+ LRUHandle* old = lru_.next;
+ assert(old->InCache());
+ assert(old->refs ==
+ 1); // LRU list contains elements which may be evicted
+ LRU_Remove(old);
+ table_.Remove(old->key(), old->hash);
+ old->SetInCache(false);
+ Unref(old);
+ usage_ -= old->charge;
+ last_reference_list.push_back(old);
+ }
+ }
+
+ for (auto entry : last_reference_list) {
+ entry->Free();
+ }
+}
+
+void LRUCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+ bool thread_safe) {
+ if (thread_safe) {
+ mutex_.Lock();
+ }
+ table_.ApplyToAllCacheEntries(
+ [callback](LRUHandle* h) { callback(h->value, h->charge); });
+ if (thread_safe) {
+ mutex_.Unlock();
+ }
+}
+
+void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
+ *lru = &lru_;
+ *lru_low_pri = lru_low_pri_;
+}
+
+size_t LRUCacheShard::TEST_GetLRUSize() {
+ LRUHandle* lru_handle = lru_.next;
+ size_t lru_size = 0;
+ while (lru_handle != &lru_) {
+ lru_size++;
+ lru_handle = lru_handle->next;
+ }
+ return lru_size;
+}
+
+void LRUCacheShard::LRU_Remove(LRUHandle* e) {
+ assert(e->next != nullptr);
+ assert(e->prev != nullptr);
+ if (lru_low_pri_ == e) {
+ lru_low_pri_ = e->prev;
+ }
+ e->next->prev = e->prev;
+ e->prev->next = e->next;
+ e->prev = e->next = nullptr;
+ lru_usage_ -= e->charge;
+ if (e->InHighPriPool()) {
+ assert(high_pri_pool_usage_ >= e->charge);
+ high_pri_pool_usage_ -= e->charge;
+ }
+}
+
+void LRUCacheShard::LRU_Insert(LRUHandle* e) {
+ assert(e->next == nullptr);
+ assert(e->prev == nullptr);
+ if (high_pri_pool_ratio_ > 0 && e->IsHighPri()) {
+ // Inset "e" to head of LRU list.
+ e->next = &lru_;
+ e->prev = lru_.prev;
+ e->prev->next = e;
+ e->next->prev = e;
+ e->SetInHighPriPool(true);
+ high_pri_pool_usage_ += e->charge;
+ MaintainPoolSize();
+ } else {
+ // Insert "e" to the head of low-pri pool. Note that when
+ // high_pri_pool_ratio is 0, head of low-pri pool is also head of LRU list.
+ e->next = lru_low_pri_->next;
+ e->prev = lru_low_pri_;
+ e->prev->next = e;
+ e->next->prev = e;
+ e->SetInHighPriPool(false);
+ lru_low_pri_ = e;
+ }
+ lru_usage_ += e->charge;
+}
+
+void LRUCacheShard::MaintainPoolSize() {
+ while (high_pri_pool_usage_ > high_pri_pool_capacity_) {
+ // Overflow last entry in high-pri pool to low-pri pool.
+ lru_low_pri_ = lru_low_pri_->next;
+ assert(lru_low_pri_ != &lru_);
+ lru_low_pri_->SetInHighPriPool(false);
+ high_pri_pool_usage_ -= lru_low_pri_->charge;
+ }
+}
+
+void LRUCacheShard::EvictFromLRU(size_t charge,
+ autovector<LRUHandle*>* deleted) {
+ while (usage_ + charge > capacity_ && lru_.next != &lru_) {
+ LRUHandle* old = lru_.next;
+ assert(old->InCache());
+ assert(old->refs == 1); // LRU list contains elements which may be evicted
+ LRU_Remove(old);
+ table_.Remove(old->key(), old->hash);
+ old->SetInCache(false);
+ Unref(old);
+ usage_ -= old->charge;
+ deleted->push_back(old);
+ }
+}
+
+void* LRUCacheShard::operator new(size_t size) {
+ return rocksdb::port::cacheline_aligned_alloc(size);
+}
+
+void LRUCacheShard::operator delete(void *memblock) {
+ rocksdb::port::cacheline_aligned_free(memblock);
+}
+
+void LRUCacheShard::SetCapacity(size_t capacity) {
+ autovector<LRUHandle*> last_reference_list;
+ {
+ MutexLock l(&mutex_);
+ capacity_ = capacity;
+ high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
+ EvictFromLRU(0, &last_reference_list);
+ }
+ // we free the entries here outside of mutex for
+ // performance reasons
+ for (auto entry : last_reference_list) {
+ entry->Free();
+ }
+}
+
+void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
+ MutexLock l(&mutex_);
+ strict_capacity_limit_ = strict_capacity_limit;
+}
+
+Cache::Handle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
+ MutexLock l(&mutex_);
+ LRUHandle* e = table_.Lookup(key, hash);
+ if (e != nullptr) {
+ assert(e->InCache());
+ if (e->refs == 1) {
+ LRU_Remove(e);
+ }
+ e->refs++;
+ }
+ return reinterpret_cast<Cache::Handle*>(e);
+}
+
+bool LRUCacheShard::Ref(Cache::Handle* h) {
+ LRUHandle* handle = reinterpret_cast<LRUHandle*>(h);
+ MutexLock l(&mutex_);
+ if (handle->InCache() && handle->refs == 1) {
+ LRU_Remove(handle);
+ }
+ handle->refs++;
+ return true;
+}
+
+void LRUCacheShard::SetHighPriorityPoolRatio(double high_pri_pool_ratio) {
+ MutexLock l(&mutex_);
+ high_pri_pool_ratio_ = high_pri_pool_ratio;
+ high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
+ MaintainPoolSize();
+}
+
+bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
+ if (handle == nullptr) {
+ return false;
+ }
+ LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
+ bool last_reference = false;
+ {
+ MutexLock l(&mutex_);
+ last_reference = Unref(e);
+ if (last_reference) {
+ usage_ -= e->charge;
+ }
+ if (e->refs == 1 && e->InCache()) {
+ // The item is still in cache, and nobody else holds a reference to it
+ if (usage_ > capacity_ || force_erase) {
+ // the cache is full
+ // The LRU list must be empty since the cache is full
+ assert(!(usage_ > capacity_) || lru_.next == &lru_);
+ // take this opportunity and remove the item
+ table_.Remove(e->key(), e->hash);
+ e->SetInCache(false);
+ Unref(e);
+ usage_ -= e->charge;
+ last_reference = true;
+ } else {
+ // put the item on the list to be potentially freed
+ LRU_Insert(e);
+ }
+ }
+ }
+
+ // free outside of mutex
+ if (last_reference) {
+ e->Free();
+ }
+ return last_reference;
+}
+
+Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
+ size_t charge,
+ void (*deleter)(const Slice& key, void* value),
+ Cache::Handle** handle, Cache::Priority priority) {
+ // Allocate the memory here outside of the mutex
+ // If the cache is full, we'll have to release it
+ // It shouldn't happen very often though.
+ LRUHandle* e = reinterpret_cast<LRUHandle*>(
+ new char[sizeof(LRUHandle) - 1 + key.size()]);
+ Status s;
+ autovector<LRUHandle*> last_reference_list;
+
+ e->value = value;
+ e->deleter = deleter;
+ e->charge = charge;
+ e->key_length = key.size();
+ e->hash = hash;
+ e->refs = (handle == nullptr
+ ? 1
+ : 2); // One from LRUCache, one for the returned handle
+ e->next = e->prev = nullptr;
+ e->SetInCache(true);
+ e->SetPriority(priority);
+ memcpy(e->key_data, key.data(), key.size());
+
+ {
+ MutexLock l(&mutex_);
+
+ // Free the space following strict LRU policy until enough space
+ // is freed or the lru list is empty
+ EvictFromLRU(charge, &last_reference_list);
+
+ if (usage_ - lru_usage_ + charge > capacity_ &&
+ (strict_capacity_limit_ || handle == nullptr)) {
+ if (handle == nullptr) {
+ // Don't insert the entry but still return ok, as if the entry inserted
+ // into cache and get evicted immediately.
+ last_reference_list.push_back(e);
+ } else {
+ delete[] reinterpret_cast<char*>(e);
+ *handle = nullptr;
+ s = Status::Incomplete("Insert failed due to LRU cache being full.");
+ }
+ } else {
+ // insert into the cache
+ // note that the cache might get larger than its capacity if not enough
+ // space was freed
+ LRUHandle* old = table_.Insert(e);
+ usage_ += e->charge;
+ if (old != nullptr) {
+ old->SetInCache(false);
+ if (Unref(old)) {
+ usage_ -= old->charge;
+ // old is on LRU because it's in cache and its reference count
+ // was just 1 (Unref returned 0)
+ LRU_Remove(old);
+ last_reference_list.push_back(old);
+ }
+ }
+ if (handle == nullptr) {
+ LRU_Insert(e);
+ } else {
+ *handle = reinterpret_cast<Cache::Handle*>(e);
+ }
+ s = Status::OK();
+ }
+ }
+
+ // we free the entries here outside of mutex for
+ // performance reasons
+ for (auto entry : last_reference_list) {
+ entry->Free();
+ }
+
+ return s;
+}
+
+void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
+ LRUHandle* e;
+ bool last_reference = false;
+ {
+ MutexLock l(&mutex_);
+ e = table_.Remove(key, hash);
+ if (e != nullptr) {
+ last_reference = Unref(e);
+ if (last_reference) {
+ usage_ -= e->charge;
+ }
+ if (last_reference && e->InCache()) {
+ LRU_Remove(e);
+ }
+ e->SetInCache(false);
+ }
+ }
+
+ // mutex not held here
+ // last_reference will only be true if e != nullptr
+ if (last_reference) {
+ e->Free();
+ }
+}
+
+size_t LRUCacheShard::GetUsage() const {
+ MutexLock l(&mutex_);
+ return usage_;
+}
+
+size_t LRUCacheShard::GetPinnedUsage() const {
+ MutexLock l(&mutex_);
+ assert(usage_ >= lru_usage_);
+ return usage_ - lru_usage_;
+}
+
+std::string LRUCacheShard::GetPrintableOptions() const {
+ const int kBufferSize = 200;
+ char buffer[kBufferSize];
+ {
+ MutexLock l(&mutex_);
+ snprintf(buffer, kBufferSize, " high_pri_pool_ratio: %.3lf\n",
+ high_pri_pool_ratio_);
+ }
+ return std::string(buffer);
+}
+
+LRUCache::LRUCache(size_t capacity, int num_shard_bits,
+ bool strict_capacity_limit, double high_pri_pool_ratio)
+ : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
+ num_shards_ = 1 << num_shard_bits;
+#if defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable: 4316) // We've validated the alignment with the new operators
+#endif
+ shards_ = new LRUCacheShard[num_shards_];
+#if defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+ SetCapacity(capacity);
+ SetStrictCapacityLimit(strict_capacity_limit);
+ for (int i = 0; i < num_shards_; i++) {
+ shards_[i].SetHighPriorityPoolRatio(high_pri_pool_ratio);
+ }
+}
+
+LRUCache::~LRUCache() { delete[] shards_; }
+
+CacheShard* LRUCache::GetShard(int shard) {
+ return reinterpret_cast<CacheShard*>(&shards_[shard]);
+}
+
+const CacheShard* LRUCache::GetShard(int shard) const {
+ return reinterpret_cast<CacheShard*>(&shards_[shard]);
+}
+
+void* LRUCache::Value(Handle* handle) {
+ return reinterpret_cast<const LRUHandle*>(handle)->value;
+}
+
+size_t LRUCache::GetCharge(Handle* handle) const {
+ return reinterpret_cast<const LRUHandle*>(handle)->charge;
+}
+
+uint32_t LRUCache::GetHash(Handle* handle) const {
+ return reinterpret_cast<const LRUHandle*>(handle)->hash;
+}
+
+void LRUCache::DisownData() { shards_ = nullptr; }
+
+size_t LRUCache::TEST_GetLRUSize() {
+ size_t lru_size_of_all_shards = 0;
+ for (int i = 0; i < num_shards_; i++) {
+ lru_size_of_all_shards += shards_[i].TEST_GetLRUSize();
+ }
+ return lru_size_of_all_shards;
+}
+
+std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
+ bool strict_capacity_limit,
+ double high_pri_pool_ratio) {
+ if (num_shard_bits >= 20) {
+ return nullptr; // the cache cannot be sharded into too many fine pieces
+ }
+ if (high_pri_pool_ratio < 0.0 || high_pri_pool_ratio > 1.0) {
+ // invalid high_pri_pool_ratio
+ return nullptr;
+ }
+ if (num_shard_bits < 0) {
+ num_shard_bits = GetDefaultCacheShardBits(capacity);
+ }
+ return std::make_shared<LRUCache>(capacity, num_shard_bits,
+ strict_capacity_limit, high_pri_pool_ratio);
+}
+
+} // namespace rocksdb
[48/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/test/unit/ProvenanceTests.cpp
----------------------------------------------------------------------
diff --git a/libminifi/test/unit/ProvenanceTests.cpp b/libminifi/test/unit/ProvenanceTests.cpp
deleted file mode 100644
index 97cb646..0000000
--- a/libminifi/test/unit/ProvenanceTests.cpp
+++ /dev/null
@@ -1,164 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#define CATCH_CONFIG_MAIN // This tells Catch to provide a main() - only do this in one cpp file
-#include "../TestBase.h"
-#include <utility>
-#include <memory>
-#include <string>
-#include <map>
-#include "ProvenanceTestHelper.h"
-#include "provenance/Provenance.h"
-#include "FlowFileRecord.h"
-#include "core/Core.h"
-#include "core/repository/AtomicRepoEntries.h"
-#include "core/repository/VolatileProvenanceRepository.h"
-
-TEST_CASE("Test Provenance record create", "[Testprovenance::ProvenanceEventRecord]") {
- provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CREATE, "blah", "blahblah");
- REQUIRE(record1.getAttributes().size() == 0);
- REQUIRE(record1.getAlternateIdentifierUri().length() == 0);
-}
-
-TEST_CASE("Test Provenance record serialization", "[Testprovenance::ProvenanceEventRecordSerializeDeser]") {
- provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CREATE, "componentid", "componenttype");
-
- std::string eventId = record1.getEventId();
-
- std::string smileyface = ":)";
- record1.setDetails(smileyface);
-
- uint64_t sample = 65555;
- std::shared_ptr<core::Repository> testRepository = std::make_shared<TestRepository>();
- record1.setEventDuration(sample);
-
- record1.Serialize(testRepository);
- provenance::ProvenanceEventRecord record2;
- record2.setEventId(eventId);
- REQUIRE(record2.DeSerialize(testRepository) == true);
- REQUIRE(record2.getEventId() == record1.getEventId());
- REQUIRE(record2.getComponentId() == record1.getComponentId());
- REQUIRE(record2.getComponentType() == record1.getComponentType());
- REQUIRE(record2.getDetails() == record1.getDetails());
- REQUIRE(record2.getDetails() == smileyface);
- REQUIRE(record2.getEventDuration() == sample);
-}
-
-TEST_CASE("Test Flowfile record added to provenance", "[TestFlowAndProv1]") {
- provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CLONE, "componentid", "componenttype");
- std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>();
- std::string eventId = record1.getEventId();
- std::map<std::string, std::string> attributes;
- attributes.insert(std::pair<std::string, std::string>("potato", "potatoe"));
- attributes.insert(std::pair<std::string, std::string>("tomato", "tomatoe"));
- std::shared_ptr<core::repository::FlowFileRepository> frepo = std::make_shared<core::repository::FlowFileRepository>("ff", "./content_repository", 0, 0, 0);
- std::shared_ptr<minifi::FlowFileRecord> ffr1 = std::make_shared<minifi::FlowFileRecord>(frepo, content_repo, attributes);
-
- record1.addChildFlowFile(ffr1);
-
- uint64_t sample = 65555;
- std::shared_ptr<core::Repository> testRepository = std::make_shared<TestRepository>();
- record1.setEventDuration(sample);
-
- record1.Serialize(testRepository);
- provenance::ProvenanceEventRecord record2;
- record2.setEventId(eventId);
- REQUIRE(record2.DeSerialize(testRepository) == true);
- REQUIRE(record1.getChildrenUuids().size() == 1);
- REQUIRE(record2.getChildrenUuids().size() == 1);
- std::string childId = record2.getChildrenUuids().at(0);
- REQUIRE(childId == ffr1->getUUIDStr());
- record2.removeChildUuid(childId);
- REQUIRE(record2.getChildrenUuids().size() == 0);
-}
-
-TEST_CASE("Test Provenance record serialization Volatile", "[Testprovenance::ProvenanceEventRecordSerializeDeser]") {
- provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CREATE, "componentid", "componenttype");
-
- std::string eventId = record1.getEventId();
-
- std::string smileyface = ":)";
- record1.setDetails(smileyface);
-
- uint64_t sample = 65555;
-
- std::shared_ptr<core::Repository> testRepository = std::make_shared<core::repository::VolatileProvenanceRepository>();
- testRepository->initialize(0);
- record1.setEventDuration(sample);
-
- record1.Serialize(testRepository);
- provenance::ProvenanceEventRecord record2;
- record2.setEventId(eventId);
- REQUIRE(record2.DeSerialize(testRepository) == true);
- REQUIRE(record2.getEventId() == record1.getEventId());
- REQUIRE(record2.getComponentId() == record1.getComponentId());
- REQUIRE(record2.getComponentType() == record1.getComponentType());
- REQUIRE(record2.getDetails() == record1.getDetails());
- REQUIRE(record2.getDetails() == smileyface);
- REQUIRE(record2.getEventDuration() == sample);
-}
-
-TEST_CASE("Test Flowfile record added to provenance using Volatile Repo", "[TestFlowAndProv1]") {
- provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CLONE, "componentid", "componenttype");
- std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>();
- std::string eventId = record1.getEventId();
- std::map<std::string, std::string> attributes;
- attributes.insert(std::pair<std::string, std::string>("potato", "potatoe"));
- attributes.insert(std::pair<std::string, std::string>("tomato", "tomatoe"));
- std::shared_ptr<core::Repository> frepo = std::make_shared<core::repository::VolatileProvenanceRepository>();
- frepo->initialize(0);
- std::shared_ptr<minifi::FlowFileRecord> ffr1 = std::make_shared<minifi::FlowFileRecord>(frepo, content_repo, attributes);
-
- record1.addChildFlowFile(ffr1);
-
- uint64_t sample = 65555;
- std::shared_ptr<core::Repository> testRepository = std::make_shared<core::repository::VolatileProvenanceRepository>();
- testRepository->initialize(0);
- record1.setEventDuration(sample);
-
- record1.Serialize(testRepository);
- provenance::ProvenanceEventRecord record2;
- record2.setEventId(eventId);
- REQUIRE(record2.DeSerialize(testRepository) == true);
- REQUIRE(record1.getChildrenUuids().size() == 1);
- REQUIRE(record2.getChildrenUuids().size() == 1);
- std::string childId = record2.getChildrenUuids().at(0);
- REQUIRE(childId == ffr1->getUUIDStr());
- record2.removeChildUuid(childId);
- REQUIRE(record2.getChildrenUuids().size() == 0);
-}
-
-TEST_CASE("Test Provenance record serialization NoOp", "[Testprovenance::ProvenanceEventRecordSerializeDeser]") {
- provenance::ProvenanceEventRecord record1(provenance::ProvenanceEventRecord::ProvenanceEventType::CREATE, "componentid", "componenttype");
-
- std::string eventId = record1.getEventId();
-
- std::string smileyface = ":)";
- record1.setDetails(smileyface);
-
- uint64_t sample = 65555;
-
- std::shared_ptr<core::Repository> testRepository = std::make_shared<core::Repository>();
- testRepository->initialize(0);
- record1.setEventDuration(sample);
-
- REQUIRE(record1.Serialize(testRepository) == true);
- provenance::ProvenanceEventRecord record2;
- record2.setEventId(eventId);
- REQUIRE(record2.DeSerialize(testRepository) == false);
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/test/unit/RepoTests.cpp
----------------------------------------------------------------------
diff --git a/libminifi/test/unit/RepoTests.cpp b/libminifi/test/unit/RepoTests.cpp
deleted file mode 100644
index 6ee690b..0000000
--- a/libminifi/test/unit/RepoTests.cpp
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#define CATCH_CONFIG_MAIN // This tells Catch to provide a main() - only do this in one cpp file
-#include "../TestBase.h"
-#include <memory>
-#include <string>
-#include <map>
-#include "ProvenanceTestHelper.h"
-#include "provenance/Provenance.h"
-#include "FlowFileRecord.h"
-#include "core/Core.h"
-#include "core/repository/FlowFileRepository.h"
-#include "../../include/core/repository/AtomicRepoEntries.h"
-#include "properties/Configure.h"
-
-TEST_CASE("Test Repo Empty Value Attribute", "[TestFFR1]") {
- TestController testController;
- char format[] = "/tmp/testRepo.XXXXXX";
- char *dir = testController.createTempDirectory(format);
- std::shared_ptr<core::repository::FlowFileRepository> repository = std::make_shared<core::repository::FlowFileRepository>("ff", dir, 0, 0, 1);
-
- repository->initialize(std::make_shared<minifi::Configure>());
-
- std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>();
- minifi::FlowFileRecord record(repository, content_repo);
-
- record.addAttribute("keyA", "");
-
- REQUIRE(true == record.Serialize());
-
- repository->stop();
-}
-
-TEST_CASE("Test Repo Empty Key Attribute ", "[TestFFR2]") {
- TestController testController;
- char format[] = "/tmp/testRepo.XXXXXX";
- char *dir = testController.createTempDirectory(format);
- std::shared_ptr<core::repository::FlowFileRepository> repository = std::make_shared<core::repository::FlowFileRepository>("ff", dir, 0, 0, 1);
-
- repository->initialize(std::make_shared<minifi::Configure>());
- std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>();
- minifi::FlowFileRecord record(repository, content_repo);
-
- record.addAttribute("keyA", "hasdgasdgjsdgasgdsgsadaskgasd");
-
- record.addAttribute("", "hasdgasdgjsdgasgdsgsadaskgasd");
-
- REQUIRE(true == record.Serialize());
-
- repository->stop();
-}
-
-TEST_CASE("Test Repo Key Attribute Verify ", "[TestFFR3]") {
- TestController testController;
- char format[] = "/tmp/testRepo.XXXXXX";
- char *dir = testController.createTempDirectory(format);
- std::shared_ptr<core::repository::FlowFileRepository> repository = std::make_shared<core::repository::FlowFileRepository>("ff", dir, 0, 0, 1);
-
- repository->initialize(std::make_shared<org::apache::nifi::minifi::Configure>());
-
- std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>();
- minifi::FlowFileRecord record(repository, content_repo);
-
- minifi::FlowFileRecord record2(repository, content_repo);
-
- std::string uuid = record.getUUIDStr();
-
- record.addAttribute("keyA", "hasdgasdgjsdgasgdsgsadaskgasd");
-
- record.addAttribute("keyB", "");
-
- record.addAttribute("", "");
-
- record.updateAttribute("", "hasdgasdgjsdgasgdsgsadaskgasd2");
-
- record.addAttribute("", "sdgsdg");
-
- REQUIRE(true == record.Serialize());
-
- repository->stop();
-
- record2.DeSerialize(uuid);
-
- std::string value;
- REQUIRE(true == record2.getAttribute("", value));
-
- REQUIRE("hasdgasdgjsdgasgdsgsadaskgasd2" == value);
-
- REQUIRE(false == record2.getAttribute("key", value));
- REQUIRE(true == record2.getAttribute("keyA", value));
- REQUIRE("hasdgasdgjsdgasgdsgsadaskgasd" == value);
-
- REQUIRE(true == record2.getAttribute("keyB", value));
- REQUIRE("" == value);
-}
-
-TEST_CASE("Test Delete Content ", "[TestFFR4]") {
- TestController testController;
- char format[] = "/tmp/testRepo.XXXXXX";
- LogTestController::getInstance().setDebug<core::ContentRepository>();
- LogTestController::getInstance().setDebug<core::repository::FileSystemRepository>();
- LogTestController::getInstance().setDebug<core::repository::FlowFileRepository>();
-
- char *dir = testController.createTempDirectory(format);
-
- std::shared_ptr<core::repository::FlowFileRepository> repository = std::make_shared<core::repository::FlowFileRepository>("ff", dir, 0, 0, 1);
-
- std::map<std::string, std::string> attributes;
-
- std::fstream file;
- std::stringstream ss;
- ss << dir << "/" << "tstFile.ext";
- file.open(ss.str(), std::ios::out);
- file << "tempFile";
- file.close();
-
- std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::FileSystemRepository>();
-
- repository->initialize(std::make_shared<minifi::Configure>());
-
- repository->loadComponent(content_repo);
-
- std::shared_ptr<minifi::ResourceClaim> claim = std::make_shared<minifi::ResourceClaim>(ss.str(), content_repo);
-
- minifi::FlowFileRecord record(repository, content_repo, attributes, claim);
-
- record.addAttribute("keyA", "hasdgasdgjsdgasgdsgsadaskgasd");
-
- record.addAttribute("", "hasdgasdgjsdgasgdsgsadaskgasd");
-
- REQUIRE(true == record.Serialize());
-
- claim->decreaseFlowFileRecordOwnedCount();
-
- claim->decreaseFlowFileRecordOwnedCount();
-
- repository->Delete(record.getUUIDStr());
-
- repository->flush();
-
- repository->stop();
-
- std::ifstream fileopen(ss.str());
- REQUIRE(false == fileopen.good());
-
- LogTestController::getInstance().reset();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/main/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt
index 249dd8e..ef2bbbc 100644
--- a/main/CMakeLists.txt
+++ b/main/CMakeLists.txt
@@ -23,7 +23,18 @@ IF(POLICY CMP0048)
CMAKE_POLICY(SET CMP0048 OLD)
ENDIF(POLICY CMP0048)
-include_directories(../libminifi/include ../libminifi/include/c2 ../libminifi/include/c2/protocols/ ../libminifi/include/core/state ./libminifi/include/core/statemanagement/metrics ../libminifi/include/core/yaml ../libminifi/include/core ../thirdparty/spdlog-20170710/include ../thirdparty/concurrentqueue ../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../thirdparty/civetweb-1.9.1/include ../thirdparty/jsoncpp/include ../thirdparty/leveldb-1.18/include ../thirdparty/)
+include_directories(../libminifi/include ../libminifi/include/c2 ../libminifi/include/c2/protocols/ ../libminifi/include/core/state ./libminifi/include/core/statemanagement/metrics ../libminifi/include/core/yaml ../libminifi/include/core ../thirdparty/spdlog-20170710/include ../thirdparty/concurrentqueue ../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../thirdparty/civetweb-1.9.1/include ../thirdparty/jsoncpp/include ../thirdparty/)
+
+include(CheckCXXCompilerFlag)
+CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11)
+CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X)
+if(COMPILER_SUPPORTS_CXX11)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Os")
+elseif(COMPILER_SUPPORTS_CXX0X)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x -Os")
+else()
+ message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.")
+endif()
find_package(Boost REQUIRED)
include_directories(${Boost_INCLUDE_DIRS})
@@ -43,23 +54,28 @@ find_package(UUID REQUIRED)
find_package(OpenSSL REQUIRED)
include_directories(${OPENSSL_INCLUDE_DIR})
-# Link against minifi, yaml-cpp, civetweb-cpp, uuid, openssl, jsoncpp and leveldb
-target_link_libraries(minifiexe minifi yaml-cpp c-library civetweb-cpp ${JSON_CPP_LIB} ${UUID_LIBRARIES} ${LEVELDB_LIBRARIES} ${OPENSSL_LIBRARIES})
-message("**** HTTP Curl is from libminifi ${HTTP-CURL}")
+# Link against minifi, yaml-cpp, civetweb-cpp, uuid, openssl, jsoncpp and rocksdb
+target_link_libraries(minifiexe minifi yaml-cpp c-library civetweb-cpp ${JSON_CPP_LIB} ${UUID_LIBRARIES} ${OPENSSL_LIBRARIES})
if (APPLE)
if (HTTP-CURL)
target_link_libraries (minifiexe -Wl,-all_load ${HTTP-CURL})
endif()
+ if (ROCKSDB-REPOS)
+ target_link_libraries (minifiexe -Wl,-all_load ${ROCKSDB-REPOS})
+ endif()
+
else ()
if (HTTP-CURL)
target_link_libraries (minifiexe -Wl,--whole-archive ${HTTP-CURL} -Wl,--no-whole-archive)
- endif()
+ endif()
+ if (ROCKSDB-REPOS)
+ target_link_libraries (minifiexe -Wl,--whole-archive ${ROCKSDB-REPOS} -Wl,--no-whole-archive)
+ endif()
endif ()
set_property(TARGET minifiexe PROPERTY INTERPROCEDURAL_OPTIMIZATION True)
-#SET (MYLIB -Wl,-all_load minifi-accumulo)
-#TARGET_LINK_LIBRARIES(minifiexe ${MYLIB} )
+
set_target_properties(minifiexe
PROPERTIES OUTPUT_NAME minifi)
@@ -68,4 +84,8 @@ set_target_properties(minifiexe
install(TARGETS minifiexe
RUNTIME
DESTINATION bin
- COMPONENT bin)
\ No newline at end of file
+ COMPONENT bin)
+
+
+add_custom_command(TARGET minifiexe POST_BUILD
+ COMMAND cat ${CMAKE_BINARY_DIR}/all.log)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/main/MiNiFiMain.cpp
----------------------------------------------------------------------
diff --git a/main/MiNiFiMain.cpp b/main/MiNiFiMain.cpp
index 3afa06e..4f29636 100644
--- a/main/MiNiFiMain.cpp
+++ b/main/MiNiFiMain.cpp
@@ -86,6 +86,9 @@ int main(int argc, char **argv) {
uint16_t stop_wait_time = STOP_WAIT_TIME_MS;
+ // initialize static functions that were defined apriori
+ core::FlowConfiguration::initialize_static_functions();
+
std::string graceful_shutdown_seconds = "";
std::string prov_repo_class = "provenancerepository";
std::string flow_repo_class = "flowfilerepository";
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/.gitignore
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/.gitignore b/thirdparty/leveldb-1.18/.gitignore
deleted file mode 100755
index 8d3a4c4..0000000
--- a/thirdparty/leveldb-1.18/.gitignore
+++ /dev/null
@@ -1,33 +0,0 @@
-build_config.mk
-
-db_bench
-leveldbutil
-*_test
-
-*.slo
-*.lo
-*.o
-
-# Compiled Dynamic libraries
-*.so
-*.so.*
-*.dylib*
-
-# Compiled Static libraries
-*.lai
-*.la
-*.a
-
-
-/project/
-
-# CMake
-/CMakeFiles/
-*.dir/
-cmake_install.cmake
-/CMakeCache.txt
-
-# Visual Studio
-*.sln
-*.vcxproj
-*.vcxproj.filters
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/AUTHORS
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/AUTHORS b/thirdparty/leveldb-1.18/AUTHORS
deleted file mode 100755
index 2439d7a..0000000
--- a/thirdparty/leveldb-1.18/AUTHORS
+++ /dev/null
@@ -1,12 +0,0 @@
-# Names should be added to this file like so:
-# Name or Organization <email address>
-
-Google Inc.
-
-# Initial version authors:
-Jeffrey Dean <je...@google.com>
-Sanjay Ghemawat <sa...@google.com>
-
-# Partial list of contributors:
-Kevin Regan <ke...@gmail.com>
-Johan Bilien <jo...@litl.com>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/CMakeLists.txt b/thirdparty/leveldb-1.18/CMakeLists.txt
deleted file mode 100755
index f053739..0000000
--- a/thirdparty/leveldb-1.18/CMakeLists.txt
+++ /dev/null
@@ -1,186 +0,0 @@
-cmake_minimum_required(VERSION 3.0.2 FATAL_ERROR)
-
-project(leveldb CXX)
-set(CMAKE_DEBUG_POSTFIX "d")
-
-set(Boost_USE_STATIC_LIBS ON)
-set(Boost_USE_MULTITHREAD ON)
-set(Boost_USE_STATIC_RUNTIME OFF)
-
-find_package(Boost COMPONENTS
- date_time
- filesystem
- system
- REQUIRED)
-
-set(SNAPPY_LIBRARY "")
-
-if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
- add_definitions(-DOS_LINUX)
-elseif(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
- add_definitions(-DOS_FREEBSD)
-elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
- add_definitions(-DOS_MACOSX)
-endif()
-
-string(REGEX MATCH "clang" CLANG ${CMAKE_CXX_COMPILER})
-
-if(CMAKE_COMPILER_IS_GNUCXX OR CLANG)
- find_library(Pthread_LIBRARY pthread)
- find_library(Realtime_LIBRARY rt)
- # find library can be problematic with stdc++ which is why we hardwire the link
- set(Stdcpp_LIBRARY stdc++)
-else(CMAKE_COMPILER_IS_GNUCXX OR CLANG)
- set(Pthread_LIBRARY "")
- set(Realtime_LIBRARY "")
- set(Stdcpp_LIBRARY "")
-endif(CMAKE_COMPILER_IS_GNUCXX OR CLANG)
-
-include_directories(${Boost_INCLUDE_DIRS}
- ${CMAKE_CURRENT_SOURCE_DIR}
- include)
-
-if(MSVC)
- add_compile_options(
- /D_CRT_SECURE_NO_WARNINGS
- /wd4389 # signed/unsigned mismatch
- /wd4800 # constructor never returns, potential memory leak because of a singleton pattern
- /wd4722 # unreachable code because of singleton pattern
- /wd4702 # bool cast performance warning
- )
-else()
- add_compile_options(
- -Wno-sign-compare
- -std=c++11
- )
-endif()
-
-add_definitions(
- -DLEVELDB_ATOMIC_PRESENT
-)
-
-set(LEVEL_DB_FILES
- include/leveldb/c.h
- include/leveldb/cache.h
- include/leveldb/comparator.h
- include/leveldb/db.h
- include/leveldb/dumpfile.h
- include/leveldb/env.h
- include/leveldb/iterator.h
- include/leveldb/filter_policy.h
- include/leveldb/iterator.h
- include/leveldb/options.h
- include/leveldb/slice.h
- include/leveldb/status.h
- include/leveldb/table.h
- include/leveldb/table_builder.h
- include/leveldb/write_batch.h
- db/builder.cc
- db/builder.h
- db/db_impl.cc
- db/db_impl.h
- db/db_iter.cc
- db/db_iter.h
- db/dbformat.cc
- db/dbformat.h
- db/dumpfile.cc
- db/filename.cc
- db/filename.h
- db/log_format.h
- db/log_reader.cc
- db/log_reader.h
- db/log_writer.cc
- db/log_writer.h
- db/skiplist.h
- db/snapshot.h
- db/memtable.cc
- db/memtable.h
- db/repair.cc
- db/table_cache.cc
- db/table_cache.h
- db/version_edit.cc
- db/version_edit.h
- db/version_set.cc
- db/version_set.h
- db/write_batch.cc
- table/block.cc
- table/block.h
- table/block_builder.cc
- table/block_builder.h
- table/filter_block.cc
- table/filter_block.h
- table/format.cc
- table/format.h
- table/iterator.cc
- table/iterator_wrapper.h
- table/merger.cc
- table/merger.h
- table/table.cc
- table/table_builder.cc
- table/two_level_iterator.cc
- table/two_level_iterator.h
- util/arena.cc
- util/arena.h
- util/bloom.cc
- util/cache.cc
- util/coding.cc
- util/coding.h
- util/comparator.cc
- util/crc32c.cc
- util/crc32c.h
- util/env.cc
- util/filter_policy.cc
- util/hash.cc
- util/hash.h
- util/histogram.cc
- util/histogram.h
- util/logging.cc
- util/logging.h
- util/mutexlock.h
- util/options.cc
- util/random.h
- util/status.cc
- port/port.h)
-
-if(WIN32)
- list(APPEND LEVEL_DB_FILES
- port/port_win.h
- port/port_win.cc
- util/win_logger.h
- util/win_logger.cc
- util/env_boost.cc)
-else()
- list(APPEND LEVEL_DB_FILES
- port/port_posix.h
- port/port_posix.cc
- util/posix_logger.h
- util/env_posix.cc)
-endif()
-
-add_library(leveldb ${LEVEL_DB_FILES})
-
-target_include_directories(leveldb
- PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
- PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
-)
-
-target_link_libraries(leveldb
- PRIVATE
- ${Boost_LIBRARIES}
- ${Pthread_LIBRARY}
-)
-
-add_executable(leveldbutil
- db/leveldb_main.cc)
-
-target_link_libraries(leveldbutil
- leveldb)
-
-set_target_properties(leveldbutil PROPERTIES
- DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX})
-
-# we distribute the leveldbutil as it might be useful
-install(TARGETS leveldbutil
- RUNTIME DESTINATION bin
- LIBRARY DESTINATION lib
- ARCHIVE DESTINATION lib)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/LICENSE
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/LICENSE b/thirdparty/leveldb-1.18/LICENSE
deleted file mode 100755
index 8e80208..0000000
--- a/thirdparty/leveldb-1.18/LICENSE
+++ /dev/null
@@ -1,27 +0,0 @@
-Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/NEWS
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/NEWS b/thirdparty/leveldb-1.18/NEWS
deleted file mode 100755
index 3fd9924..0000000
--- a/thirdparty/leveldb-1.18/NEWS
+++ /dev/null
@@ -1,17 +0,0 @@
-Release 1.2 2011-05-16
-----------------------
-
-Fixes for larger databases (tested up to one billion 100-byte entries,
-i.e., ~100GB).
-
-(1) Place hard limit on number of level-0 files. This fixes errors
-of the form "too many open files".
-
-(2) Fixed memtable management. Before the fix, a heavy write burst
-could cause unbounded memory usage.
-
-A fix for a logging bug where the reader would incorrectly complain
-about corruption.
-
-Allow public access to WriteBatch contents so that users can easily
-wrap a DB.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/README
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/README b/thirdparty/leveldb-1.18/README
deleted file mode 100755
index 3618ade..0000000
--- a/thirdparty/leveldb-1.18/README
+++ /dev/null
@@ -1,51 +0,0 @@
-leveldb: A key-value store
-Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
-
-The code under this directory implements a system for maintaining a
-persistent key/value store.
-
-See doc/index.html for more explanation.
-See doc/impl.html for a brief overview of the implementation.
-
-The public interface is in include/*.h. Callers should not include or
-rely on the details of any other header files in this package. Those
-internal APIs may be changed without warning.
-
-Guide to header files:
-
-include/db.h
- Main interface to the DB: Start here
-
-include/options.h
- Control over the behavior of an entire database, and also
- control over the behavior of individual reads and writes.
-
-include/comparator.h
- Abstraction for user-specified comparison function. If you want
- just bytewise comparison of keys, you can use the default comparator,
- but clients can write their own comparator implementations if they
- want custom ordering (e.g. to handle different character
- encodings, etc.)
-
-include/iterator.h
- Interface for iterating over data. You can get an iterator
- from a DB object.
-
-include/write_batch.h
- Interface for atomically applying multiple updates to a database.
-
-include/slice.h
- A simple module for maintaining a pointer and a length into some
- other byte array.
-
-include/status.h
- Status is returned from many of the public interfaces and is used
- to report success and various kinds of errors.
-
-include/env.h
- Abstraction of the OS environment. A posix implementation of
- this interface is in util/env_posix.cc
-
-include/table.h
-include/table_builder.h
- Lower-level modules that most clients probably won't use directly
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/README.md
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/README.md b/thirdparty/leveldb-1.18/README.md
deleted file mode 100755
index 760cb10..0000000
--- a/thirdparty/leveldb-1.18/README.md
+++ /dev/null
@@ -1,20 +0,0 @@
-LevelDB qdb branch
-==================
-
-Current version: 1.18
-
-quasardb [LevelDB](http://code.google.com/p/leveldb/) branch with full Windows support. This is not an official LevelDB branch, but the branch we use in our product, [quasardb](https://www.quasardb.net/).
-
-* Full Windows support: everything builds, all tests pass;
-* [CMake](http://www.cmake.org/) based build
-* Explicit (thread unsafe) de-allocation routines for "clean exits". Helps a lot when running your application into a leak detector;
-* The Windows build requires [Boost](http://www.boost.org/);
-* Our code is C++11ish and may require a recent compiler;
-* Lots of warnings fixed;
-* Is not 100% compliant with Google coding style.
-
-Tested on [FreeBSD](http://www.freebsd.org/), Linux and Windows (32-bit & 64-bit).
-
-Might contains trace of nuts.
-
-Comments? Questions? Suggestions? Pull!
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/TODO
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/TODO b/thirdparty/leveldb-1.18/TODO
deleted file mode 100755
index e603c07..0000000
--- a/thirdparty/leveldb-1.18/TODO
+++ /dev/null
@@ -1,14 +0,0 @@
-ss
-- Stats
-
-db
-- Maybe implement DB::BulkDeleteForRange(start_key, end_key)
- that would blow away files whose ranges are entirely contained
- within [start_key..end_key]? For Chrome, deletion of obsolete
- object stores, etc. can be done in the background anyway, so
- probably not that important.
-- There have been requests for MultiGet.
-
-After a range is completely deleted, what gets rid of the
-corresponding files if we do no future changes to that range. Make
-the conditions for triggering compactions fire in more situations?
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/WINDOWS
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/WINDOWS b/thirdparty/leveldb-1.18/WINDOWS
deleted file mode 100755
index 9435809..0000000
--- a/thirdparty/leveldb-1.18/WINDOWS
+++ /dev/null
@@ -1,42 +0,0 @@
-INSTRUCTIONS FOR BUILDING LEVELDB ON WINDOWS / MSVC++
-(Tested with Microsoft Visual C++ 2010 Express)
-
-1. Install Boost
- You'll need the Boost libraries to compile LevelDB on Windows:
- http://www.boost.org/users/download/
- You'll need at least the following Boost packages:
- - date_time
- - filesystem
- - thread
- - interprocess
-
-2. To create your LevelDB project, choose:
- New -> "Project From Existing Code"
- and point Visual Studio to the leveldb root directory.
-
-3. To just build the benchmarking tools, choose:
- Project Type: "Console application project"
-
-4. In the configuration settings, make sure you include
- Preprocessor definitions: LEVELDB_PLATFORM_WINDOWS;OS_WIN
- You can also add these later in:
- Project -> Properties ->
- Configuration Properties -> C/C++ -> Preprocessor Definitions
-
- Include the root directory of your LevelDB sources in header search paths.
- You can also add this later in:
- Project -> Properties ->
- Configuration Properties -> C/C++ -> Additional Include Directories
-
-5. Add boost/lib directory to Linker paths:
- Project -> Properties -> Linker -> General -> Additional Library Dependencies
-
-6. Manually exclude the following files from the build
- (Solution Explorer -> right-click on file -> Exclude from Project)
- - port/port_android.cc
- - port/port_posix.cc
- - util/env_chromium.cc
- - util/env_posix.cc
-
-7. Manually exclude all the *_test.cc and *_bench.cc files you don't want
- to build. There should only be one .cc file with a main() in your project.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/build_detect_platform
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/build_detect_platform b/thirdparty/leveldb-1.18/build_detect_platform
deleted file mode 100755
index bb76c4f..0000000
--- a/thirdparty/leveldb-1.18/build_detect_platform
+++ /dev/null
@@ -1,228 +0,0 @@
-#!/bin/sh
-#
-# Detects OS we're compiling on and outputs a file specified by the first
-# argument, which in turn gets read while processing Makefile.
-#
-# The output will set the following variables:
-# CC C Compiler path
-# CXX C++ Compiler path
-# PLATFORM_LDFLAGS Linker flags
-# PLATFORM_LIBS Libraries flags
-# PLATFORM_SHARED_EXT Extension for shared libraries
-# PLATFORM_SHARED_LDFLAGS Flags for building shared library
-# This flag is embedded just before the name
-# of the shared library without intervening spaces
-# PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library
-# PLATFORM_CCFLAGS C compiler flags
-# PLATFORM_CXXFLAGS C++ compiler flags. Will contain:
-# PLATFORM_SHARED_VERSIONED Set to 'true' if platform supports versioned
-# shared libraries, empty otherwise.
-#
-# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
-#
-# -DLEVELDB_ATOMIC_PRESENT if <atomic> is present
-# -DLEVELDB_PLATFORM_POSIX for Posix-based platforms
-# -DSNAPPY if the Snappy library is present
-#
-
-OUTPUT=$1
-PREFIX=$2
-if test -z "$OUTPUT" || test -z "$PREFIX"; then
- echo "usage: $0 <output-filename> <directory_prefix>" >&2
- exit 1
-fi
-
-# Delete existing output, if it exists
-rm -f $OUTPUT
-touch $OUTPUT
-
-if test -z "$CC"; then
- CC=cc
-fi
-
-if test -z "$CXX"; then
- CXX=g++
-fi
-
-if test -z "$TMPDIR"; then
- TMPDIR=/tmp
-fi
-
-# Detect OS
-if test -z "$TARGET_OS"; then
- TARGET_OS=`uname -s`
-fi
-
-COMMON_FLAGS=
-CROSS_COMPILE=
-PLATFORM_CCFLAGS=
-PLATFORM_CXXFLAGS=
-PLATFORM_LDFLAGS=
-PLATFORM_LIBS=
-PLATFORM_SHARED_EXT="so"
-PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl,"
-PLATFORM_SHARED_CFLAGS="-fPIC"
-PLATFORM_SHARED_VERSIONED=true
-
-MEMCMP_FLAG=
-if [ "$CXX" = "g++" ]; then
- # Use libc's memcmp instead of GCC's memcmp. This results in ~40%
- # performance improvement on readrandom under gcc 4.4.3 on Linux/x86.
- MEMCMP_FLAG="-fno-builtin-memcmp"
-fi
-
-case "$TARGET_OS" in
- CYGWIN_*)
- PLATFORM=OS_LINUX
- COMMON_FLAGS="$MEMCMP_FLAG -lpthread -DOS_LINUX -DCYGWIN"
- PLATFORM_LDFLAGS="-lpthread"
- PORT_FILE=port/port_posix.cc
- ;;
- Darwin)
- PLATFORM=OS_MACOSX
- COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
- PLATFORM_SHARED_EXT=dylib
- [ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd`
- PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name $INSTALL_PATH/"
- PORT_FILE=port/port_posix.cc
- ;;
- Linux)
- PLATFORM=OS_LINUX
- COMMON_FLAGS="$MEMCMP_FLAG -pthread -DOS_LINUX"
- PLATFORM_LDFLAGS="-pthread"
- PORT_FILE=port/port_posix.cc
- ;;
- SunOS)
- PLATFORM=OS_SOLARIS
- COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_SOLARIS"
- PLATFORM_LIBS="-lpthread -lrt"
- PORT_FILE=port/port_posix.cc
- ;;
- FreeBSD)
- PLATFORM=OS_FREEBSD
- COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_FREEBSD"
- PLATFORM_LIBS="-lpthread"
- PORT_FILE=port/port_posix.cc
- ;;
- NetBSD)
- PLATFORM=OS_NETBSD
- COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD"
- PLATFORM_LIBS="-lpthread -lgcc_s"
- PORT_FILE=port/port_posix.cc
- ;;
- OpenBSD)
- PLATFORM=OS_OPENBSD
- COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_OPENBSD"
- PLATFORM_LDFLAGS="-pthread"
- PORT_FILE=port/port_posix.cc
- ;;
- DragonFly)
- PLATFORM=OS_DRAGONFLYBSD
- COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_DRAGONFLYBSD"
- PLATFORM_LIBS="-lpthread"
- PORT_FILE=port/port_posix.cc
- ;;
- OS_ANDROID_CROSSCOMPILE)
- PLATFORM=OS_ANDROID
- COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
- PLATFORM_LDFLAGS="" # All pthread features are in the Android C library
- PORT_FILE=port/port_posix.cc
- CROSS_COMPILE=true
- ;;
- HP-UX)
- PLATFORM=OS_HPUX
- COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_HPUX"
- PLATFORM_LDFLAGS="-pthread"
- PORT_FILE=port/port_posix.cc
- # man ld: +h internal_name
- PLATFORM_SHARED_LDFLAGS="-shared -Wl,+h -Wl,"
- ;;
- IOS)
- PLATFORM=IOS
- COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
- [ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd`
- PORT_FILE=port/port_posix.cc
- PLATFORM_SHARED_EXT=
- PLATFORM_SHARED_LDFLAGS=
- PLATFORM_SHARED_CFLAGS=
- PLATFORM_SHARED_VERSIONED=
- ;;
- *)
- echo "Unknown platform!" >&2
- exit 1
-esac
-
-# We want to make a list of all cc files within util, db, table, and helpers
-# except for the test and benchmark files. By default, find will output a list
-# of all files matching either rule, so we need to append -print to make the
-# prune take effect.
-DIRS="$PREFIX/db $PREFIX/util $PREFIX/table"
-
-set -f # temporarily disable globbing so that our patterns aren't expanded
-PRUNE_TEST="-name *test*.cc -prune"
-PRUNE_BENCH="-name *_bench.cc -prune"
-PRUNE_TOOL="-name leveldb_main.cc -prune"
-PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`
-
-set +f # re-enable globbing
-
-# The sources consist of the portable files, plus the platform-specific port
-# file.
-echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT
-echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT
-
-if [ "$CROSS_COMPILE" = "true" ]; then
- # Cross-compiling; do not try any compilation tests.
- true
-else
- CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$"
-
- # If -std=c++0x works, use <atomic> as fallback for when memory barriers
- # are not available.
- $CXX $CXXFLAGS -std=c++0x -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
- #include <atomic>
- int main() {}
-EOF
- if [ "$?" = 0 ]; then
- COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_ATOMIC_PRESENT"
- PLATFORM_CXXFLAGS="-std=c++0x"
- else
- COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
- fi
-
- # Test whether Snappy library is installed
- # http://code.google.com/p/snappy/
- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
- #include <snappy.h>
- int main() {}
-EOF
- if [ "$?" = 0 ]; then
- COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
- PLATFORM_LIBS="$PLATFORM_LIBS -lsnappy"
- fi
-
- # Test whether tcmalloc is available
- $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -ltcmalloc 2>/dev/null <<EOF
- int main() {}
-EOF
- if [ "$?" = 0 ]; then
- PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc"
- fi
-
- rm -f $CXXOUTPUT 2>/dev/null
-fi
-
-PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
-PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
-
-echo "CC=$CC" >> $OUTPUT
-echo "CXX=$CXX" >> $OUTPUT
-echo "PLATFORM=$PLATFORM" >> $OUTPUT
-echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT
-echo "PLATFORM_LIBS=$PLATFORM_LIBS" >> $OUTPUT
-echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT
-echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT
-echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
-echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT
-echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT
-echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/build_detect_platform.cmake
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/build_detect_platform.cmake b/thirdparty/leveldb-1.18/build_detect_platform.cmake
deleted file mode 100755
index 6a69be5..0000000
--- a/thirdparty/leveldb-1.18/build_detect_platform.cmake
+++ /dev/null
@@ -1,7 +0,0 @@
-if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
- add_definitions(-DOS_LINUX)
-elseif(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
- add_definitions(-DOS_FREEBSD)
-elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
- add_definitions(-DOS_MACOSX)
-endif()
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/autocompact_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/autocompact_test.cc b/thirdparty/leveldb-1.18/db/autocompact_test.cc
deleted file mode 100755
index 026873f..0000000
--- a/thirdparty/leveldb-1.18/db/autocompact_test.cc
+++ /dev/null
@@ -1,118 +0,0 @@
-// Copyright (c) 2013 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/db.h"
-#include "db/db_impl.h"
-#include "leveldb/cache.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-class AutoCompactTest {
- public:
- std::string dbname_;
- Cache* tiny_cache_;
- Options options_;
- DB* db_;
-
- AutoCompactTest() {
- dbname_ = test::TmpDir() + "/autocompact_test";
- tiny_cache_ = NewLRUCache(100);
- options_.block_cache = tiny_cache_;
- DestroyDB(dbname_, options_);
- options_.create_if_missing = true;
- options_.compression = kNoCompression;
- ASSERT_OK(DB::Open(options_, dbname_, &db_));
- }
-
- ~AutoCompactTest() {
- delete db_;
- DestroyDB(dbname_, Options());
- delete tiny_cache_;
- }
-
- std::string Key(int i) {
- char buf[100];
- snprintf(buf, sizeof(buf), "key%06d", i);
- return std::string(buf);
- }
-
- uint64_t Size(const Slice& start, const Slice& limit) {
- Range r(start, limit);
- uint64_t size;
- db_->GetApproximateSizes(&r, 1, &size);
- return size;
- }
-
- void DoReads(int n);
-};
-
-static const int kValueSize = 200 * 1024;
-static const int kTotalSize = 100 * 1024 * 1024;
-static const int kCount = kTotalSize / kValueSize;
-
-// Read through the first n keys repeatedly and check that they get
-// compacted (verified by checking the size of the key space).
-void AutoCompactTest::DoReads(int n) {
- std::string value(kValueSize, 'x');
- DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-
- // Fill database
- for (int i = 0; i < kCount; i++) {
- ASSERT_OK(db_->Put(WriteOptions(), Key(i), value));
- }
- ASSERT_OK(dbi->TEST_CompactMemTable());
-
- // Delete everything
- for (int i = 0; i < kCount; i++) {
- ASSERT_OK(db_->Delete(WriteOptions(), Key(i)));
- }
- ASSERT_OK(dbi->TEST_CompactMemTable());
-
- // Get initial measurement of the space we will be reading.
- const int64_t initial_size = Size(Key(0), Key(n));
- const int64_t initial_other_size = Size(Key(n), Key(kCount));
-
- // Read until size drops significantly.
- std::string limit_key = Key(n);
- for (int read = 0; true; read++) {
- ASSERT_LT(read, 100) << "Taking too long to compact";
- Iterator* iter = db_->NewIterator(ReadOptions());
- for (iter->SeekToFirst();
- iter->Valid() && iter->key().ToString() < limit_key;
- iter->Next()) {
- // Drop data
- }
- delete iter;
- // Wait a little bit to allow any triggered compactions to complete.
- Env::Default()->SleepForMicroseconds(1000000);
- uint64_t size = Size(Key(0), Key(n));
- fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n",
- read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0);
- if (size <= static_cast<uint64_t>(initial_size/4)) { // 10 is too much, put it to 4
- break;
- }
- }
-
- // Verify that the size of the key space not touched by the reads
- // is pretty much unchanged.
- const int64_t final_other_size = Size(Key(n), Key(kCount));
- ASSERT_LE(final_other_size, initial_other_size + 1048576);
- ASSERT_GE(final_other_size, initial_other_size/5 - 1048576);
-}
-
-TEST(AutoCompactTest, ReadAll) {
- DoReads(kCount);
-}
-
-TEST(AutoCompactTest, ReadHalf) {
- DoReads(kCount/2);
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/builder.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/builder.cc b/thirdparty/leveldb-1.18/db/builder.cc
deleted file mode 100755
index f419882..0000000
--- a/thirdparty/leveldb-1.18/db/builder.cc
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/builder.h"
-
-#include "db/filename.h"
-#include "db/dbformat.h"
-#include "db/table_cache.h"
-#include "db/version_edit.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
-
-namespace leveldb {
-
-Status BuildTable(const std::string& dbname,
- Env* env,
- const Options& options,
- TableCache* table_cache,
- Iterator* iter,
- FileMetaData* meta) {
- Status s;
- meta->file_size = 0;
- iter->SeekToFirst();
-
- std::string fname = TableFileName(dbname, meta->number);
- if (iter->Valid()) {
- WritableFile* file;
- s = env->NewWritableFile(fname, &file);
- if (!s.ok()) {
- return s;
- }
-
- TableBuilder* builder = new TableBuilder(options, file);
- meta->smallest.DecodeFrom(iter->key());
- for (; iter->Valid(); iter->Next()) {
- Slice key = iter->key();
- meta->largest.DecodeFrom(key);
- builder->Add(key, iter->value());
- }
-
- // Finish and check for builder errors
- if (s.ok()) {
- s = builder->Finish();
- if (s.ok()) {
- meta->file_size = builder->FileSize();
- assert(meta->file_size > 0);
- }
- } else {
- builder->Abandon();
- }
- delete builder;
-
- // Finish and check for file errors
- if (s.ok()) {
- s = file->Sync();
- }
- if (s.ok()) {
- s = file->Close();
- }
- delete file;
- file = NULL;
-
- if (s.ok()) {
- // Verify that the table is usable
- Iterator* it = table_cache->NewIterator(ReadOptions(),
- meta->number,
- meta->file_size);
- s = it->status();
- delete it;
- }
- }
-
- // Check for input iterator errors
- if (!iter->status().ok()) {
- s = iter->status();
- }
-
- if (s.ok() && meta->file_size > 0) {
- // Keep it
- } else {
- env->DeleteFile(fname);
- }
- return s;
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/builder.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/builder.h b/thirdparty/leveldb-1.18/db/builder.h
deleted file mode 100755
index 62431fc..0000000
--- a/thirdparty/leveldb-1.18/db/builder.h
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_BUILDER_H_
-#define STORAGE_LEVELDB_DB_BUILDER_H_
-
-#include "leveldb/status.h"
-
-namespace leveldb {
-
-struct Options;
-struct FileMetaData;
-
-class Env;
-class Iterator;
-class TableCache;
-class VersionEdit;
-
-// Build a Table file from the contents of *iter. The generated file
-// will be named according to meta->number. On success, the rest of
-// *meta will be filled with metadata about the generated table.
-// If no data is present in *iter, meta->file_size will be set to
-// zero, and no Table file will be produced.
-extern Status BuildTable(const std::string& dbname,
- Env* env,
- const Options& options,
- TableCache* table_cache,
- Iterator* iter,
- FileMetaData* meta);
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_BUILDER_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/c.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/c.cc b/thirdparty/leveldb-1.18/db/c.cc
deleted file mode 100755
index 08ff0ad..0000000
--- a/thirdparty/leveldb-1.18/db/c.cc
+++ /dev/null
@@ -1,595 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/c.h"
-
-#include <stdlib.h>
-#include <unistd.h>
-#include "leveldb/cache.h"
-#include "leveldb/comparator.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/filter_policy.h"
-#include "leveldb/iterator.h"
-#include "leveldb/options.h"
-#include "leveldb/status.h"
-#include "leveldb/write_batch.h"
-
-using leveldb::Cache;
-using leveldb::Comparator;
-using leveldb::CompressionType;
-using leveldb::DB;
-using leveldb::Env;
-using leveldb::FileLock;
-using leveldb::FilterPolicy;
-using leveldb::Iterator;
-using leveldb::kMajorVersion;
-using leveldb::kMinorVersion;
-using leveldb::Logger;
-using leveldb::NewBloomFilterPolicy;
-using leveldb::NewLRUCache;
-using leveldb::Options;
-using leveldb::RandomAccessFile;
-using leveldb::Range;
-using leveldb::ReadOptions;
-using leveldb::SequentialFile;
-using leveldb::Slice;
-using leveldb::Snapshot;
-using leveldb::Status;
-using leveldb::WritableFile;
-using leveldb::WriteBatch;
-using leveldb::WriteOptions;
-
-extern "C" {
-
-struct leveldb_t { DB* rep; };
-struct leveldb_iterator_t { Iterator* rep; };
-struct leveldb_writebatch_t { WriteBatch rep; };
-struct leveldb_snapshot_t { const Snapshot* rep; };
-struct leveldb_readoptions_t { ReadOptions rep; };
-struct leveldb_writeoptions_t { WriteOptions rep; };
-struct leveldb_options_t { Options rep; };
-struct leveldb_cache_t { Cache* rep; };
-struct leveldb_seqfile_t { SequentialFile* rep; };
-struct leveldb_randomfile_t { RandomAccessFile* rep; };
-struct leveldb_writablefile_t { WritableFile* rep; };
-struct leveldb_logger_t { Logger* rep; };
-struct leveldb_filelock_t { FileLock* rep; };
-
-struct leveldb_comparator_t : public Comparator {
- void* state_;
- void (*destructor_)(void*);
- int (*compare_)(
- void*,
- const char* a, size_t alen,
- const char* b, size_t blen);
- const char* (*name_)(void*);
-
- virtual ~leveldb_comparator_t() {
- (*destructor_)(state_);
- }
-
- virtual int Compare(const Slice& a, const Slice& b) const {
- return (*compare_)(state_, a.data(), a.size(), b.data(), b.size());
- }
-
- virtual const char* Name() const {
- return (*name_)(state_);
- }
-
- // No-ops since the C binding does not support key shortening methods.
- virtual void FindShortestSeparator(std::string*, const Slice&) const { }
- virtual void FindShortSuccessor(std::string* key) const { }
-};
-
-struct leveldb_filterpolicy_t : public FilterPolicy {
- void* state_;
- void (*destructor_)(void*);
- const char* (*name_)(void*);
- char* (*create_)(
- void*,
- const char* const* key_array, const size_t* key_length_array,
- int num_keys,
- size_t* filter_length);
- unsigned char (*key_match_)(
- void*,
- const char* key, size_t length,
- const char* filter, size_t filter_length);
-
- virtual ~leveldb_filterpolicy_t() {
- (*destructor_)(state_);
- }
-
- virtual const char* Name() const {
- return (*name_)(state_);
- }
-
- virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {
- std::vector<const char*> key_pointers(n);
- std::vector<size_t> key_sizes(n);
- for (int i = 0; i < n; i++) {
- key_pointers[i] = keys[i].data();
- key_sizes[i] = keys[i].size();
- }
- size_t len;
- char* filter = (*create_)(state_, &key_pointers[0], &key_sizes[0], n, &len);
- dst->append(filter, len);
- free(filter);
- }
-
- virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const {
- return (*key_match_)(state_, key.data(), key.size(),
- filter.data(), filter.size());
- }
-};
-
-struct leveldb_env_t {
- Env* rep;
- bool is_default;
-};
-
-static bool SaveError(char** errptr, const Status& s) {
- assert(errptr != NULL);
- if (s.ok()) {
- return false;
- } else if (*errptr == NULL) {
- *errptr = strdup(s.ToString().c_str());
- } else {
- // TODO(sanjay): Merge with existing error?
- free(*errptr);
- *errptr = strdup(s.ToString().c_str());
- }
- return true;
-}
-
-static char* CopyString(const std::string& str) {
- char* result = reinterpret_cast<char*>(malloc(sizeof(char) * str.size()));
- memcpy(result, str.data(), sizeof(char) * str.size());
- return result;
-}
-
-leveldb_t* leveldb_open(
- const leveldb_options_t* options,
- const char* name,
- char** errptr) {
- DB* db;
- if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) {
- return NULL;
- }
- leveldb_t* result = new leveldb_t;
- result->rep = db;
- return result;
-}
-
-void leveldb_close(leveldb_t* db) {
- delete db->rep;
- delete db;
-}
-
-void leveldb_put(
- leveldb_t* db,
- const leveldb_writeoptions_t* options,
- const char* key, size_t keylen,
- const char* val, size_t vallen,
- char** errptr) {
- SaveError(errptr,
- db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen)));
-}
-
-void leveldb_delete(
- leveldb_t* db,
- const leveldb_writeoptions_t* options,
- const char* key, size_t keylen,
- char** errptr) {
- SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen)));
-}
-
-
-void leveldb_write(
- leveldb_t* db,
- const leveldb_writeoptions_t* options,
- leveldb_writebatch_t* batch,
- char** errptr) {
- SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
-}
-
-char* leveldb_get(
- leveldb_t* db,
- const leveldb_readoptions_t* options,
- const char* key, size_t keylen,
- size_t* vallen,
- char** errptr) {
- char* result = NULL;
- std::string tmp;
- Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp);
- if (s.ok()) {
- *vallen = tmp.size();
- result = CopyString(tmp);
- } else {
- *vallen = 0;
- if (!s.IsNotFound()) {
- SaveError(errptr, s);
- }
- }
- return result;
-}
-
-leveldb_iterator_t* leveldb_create_iterator(
- leveldb_t* db,
- const leveldb_readoptions_t* options) {
- leveldb_iterator_t* result = new leveldb_iterator_t;
- result->rep = db->rep->NewIterator(options->rep);
- return result;
-}
-
-const leveldb_snapshot_t* leveldb_create_snapshot(
- leveldb_t* db) {
- leveldb_snapshot_t* result = new leveldb_snapshot_t;
- result->rep = db->rep->GetSnapshot();
- return result;
-}
-
-void leveldb_release_snapshot(
- leveldb_t* db,
- const leveldb_snapshot_t* snapshot) {
- db->rep->ReleaseSnapshot(snapshot->rep);
- delete snapshot;
-}
-
-char* leveldb_property_value(
- leveldb_t* db,
- const char* propname) {
- std::string tmp;
- if (db->rep->GetProperty(Slice(propname), &tmp)) {
- // We use strdup() since we expect human readable output.
- return strdup(tmp.c_str());
- } else {
- return NULL;
- }
-}
-
-void leveldb_approximate_sizes(
- leveldb_t* db,
- int num_ranges,
- const char* const* range_start_key, const size_t* range_start_key_len,
- const char* const* range_limit_key, const size_t* range_limit_key_len,
- uint64_t* sizes) {
- Range* ranges = new Range[num_ranges];
- for (int i = 0; i < num_ranges; i++) {
- ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]);
- ranges[i].limit = Slice(range_limit_key[i], range_limit_key_len[i]);
- }
- db->rep->GetApproximateSizes(ranges, num_ranges, sizes);
- delete[] ranges;
-}
-
-void leveldb_compact_range(
- leveldb_t* db,
- const char* start_key, size_t start_key_len,
- const char* limit_key, size_t limit_key_len) {
- Slice a, b;
- db->rep->CompactRange(
- // Pass NULL Slice if corresponding "const char*" is NULL
- (start_key ? (a = Slice(start_key, start_key_len), &a) : NULL),
- (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : NULL));
-}
-
-void leveldb_destroy_db(
- const leveldb_options_t* options,
- const char* name,
- char** errptr) {
- SaveError(errptr, DestroyDB(name, options->rep));
-}
-
-void leveldb_repair_db(
- const leveldb_options_t* options,
- const char* name,
- char** errptr) {
- SaveError(errptr, RepairDB(name, options->rep));
-}
-
-void leveldb_iter_destroy(leveldb_iterator_t* iter) {
- delete iter->rep;
- delete iter;
-}
-
-unsigned char leveldb_iter_valid(const leveldb_iterator_t* iter) {
- return iter->rep->Valid();
-}
-
-void leveldb_iter_seek_to_first(leveldb_iterator_t* iter) {
- iter->rep->SeekToFirst();
-}
-
-void leveldb_iter_seek_to_last(leveldb_iterator_t* iter) {
- iter->rep->SeekToLast();
-}
-
-void leveldb_iter_seek(leveldb_iterator_t* iter, const char* k, size_t klen) {
- iter->rep->Seek(Slice(k, klen));
-}
-
-void leveldb_iter_next(leveldb_iterator_t* iter) {
- iter->rep->Next();
-}
-
-void leveldb_iter_prev(leveldb_iterator_t* iter) {
- iter->rep->Prev();
-}
-
-const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) {
- Slice s = iter->rep->key();
- *klen = s.size();
- return s.data();
-}
-
-const char* leveldb_iter_value(const leveldb_iterator_t* iter, size_t* vlen) {
- Slice s = iter->rep->value();
- *vlen = s.size();
- return s.data();
-}
-
-void leveldb_iter_get_error(const leveldb_iterator_t* iter, char** errptr) {
- SaveError(errptr, iter->rep->status());
-}
-
-leveldb_writebatch_t* leveldb_writebatch_create() {
- return new leveldb_writebatch_t;
-}
-
-void leveldb_writebatch_destroy(leveldb_writebatch_t* b) {
- delete b;
-}
-
-void leveldb_writebatch_clear(leveldb_writebatch_t* b) {
- b->rep.Clear();
-}
-
-void leveldb_writebatch_put(
- leveldb_writebatch_t* b,
- const char* key, size_t klen,
- const char* val, size_t vlen) {
- b->rep.Put(Slice(key, klen), Slice(val, vlen));
-}
-
-void leveldb_writebatch_delete(
- leveldb_writebatch_t* b,
- const char* key, size_t klen) {
- b->rep.Delete(Slice(key, klen));
-}
-
-void leveldb_writebatch_iterate(
- leveldb_writebatch_t* b,
- void* state,
- void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
- void (*deleted)(void*, const char* k, size_t klen)) {
- class H : public WriteBatch::Handler {
- public:
- void* state_;
- void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
- void (*deleted_)(void*, const char* k, size_t klen);
- virtual void Put(const Slice& key, const Slice& value) {
- (*put_)(state_, key.data(), key.size(), value.data(), value.size());
- }
- virtual void Delete(const Slice& key) {
- (*deleted_)(state_, key.data(), key.size());
- }
- };
- H handler;
- handler.state_ = state;
- handler.put_ = put;
- handler.deleted_ = deleted;
- b->rep.Iterate(&handler);
-}
-
-leveldb_options_t* leveldb_options_create() {
- return new leveldb_options_t;
-}
-
-void leveldb_options_destroy(leveldb_options_t* options) {
- delete options;
-}
-
-void leveldb_options_set_comparator(
- leveldb_options_t* opt,
- leveldb_comparator_t* cmp) {
- opt->rep.comparator = cmp;
-}
-
-void leveldb_options_set_filter_policy(
- leveldb_options_t* opt,
- leveldb_filterpolicy_t* policy) {
- opt->rep.filter_policy = policy;
-}
-
-void leveldb_options_set_create_if_missing(
- leveldb_options_t* opt, unsigned char v) {
- opt->rep.create_if_missing = v;
-}
-
-void leveldb_options_set_error_if_exists(
- leveldb_options_t* opt, unsigned char v) {
- opt->rep.error_if_exists = v;
-}
-
-void leveldb_options_set_paranoid_checks(
- leveldb_options_t* opt, unsigned char v) {
- opt->rep.paranoid_checks = v;
-}
-
-void leveldb_options_set_env(leveldb_options_t* opt, leveldb_env_t* env) {
- opt->rep.env = (env ? env->rep : NULL);
-}
-
-void leveldb_options_set_info_log(leveldb_options_t* opt, leveldb_logger_t* l) {
- opt->rep.info_log = (l ? l->rep : NULL);
-}
-
-void leveldb_options_set_write_buffer_size(leveldb_options_t* opt, size_t s) {
- opt->rep.write_buffer_size = s;
-}
-
-void leveldb_options_set_max_open_files(leveldb_options_t* opt, int n) {
- opt->rep.max_open_files = n;
-}
-
-void leveldb_options_set_cache(leveldb_options_t* opt, leveldb_cache_t* c) {
- opt->rep.block_cache = c->rep;
-}
-
-void leveldb_options_set_block_size(leveldb_options_t* opt, size_t s) {
- opt->rep.block_size = s;
-}
-
-void leveldb_options_set_block_restart_interval(leveldb_options_t* opt, int n) {
- opt->rep.block_restart_interval = n;
-}
-
-void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
- opt->rep.compression = static_cast<CompressionType>(t);
-}
-
-leveldb_comparator_t* leveldb_comparator_create(
- void* state,
- void (*destructor)(void*),
- int (*compare)(
- void*,
- const char* a, size_t alen,
- const char* b, size_t blen),
- const char* (*name)(void*)) {
- leveldb_comparator_t* result = new leveldb_comparator_t;
- result->state_ = state;
- result->destructor_ = destructor;
- result->compare_ = compare;
- result->name_ = name;
- return result;
-}
-
-void leveldb_comparator_destroy(leveldb_comparator_t* cmp) {
- delete cmp;
-}
-
-leveldb_filterpolicy_t* leveldb_filterpolicy_create(
- void* state,
- void (*destructor)(void*),
- char* (*create_filter)(
- void*,
- const char* const* key_array, const size_t* key_length_array,
- int num_keys,
- size_t* filter_length),
- unsigned char (*key_may_match)(
- void*,
- const char* key, size_t length,
- const char* filter, size_t filter_length),
- const char* (*name)(void*)) {
- leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t;
- result->state_ = state;
- result->destructor_ = destructor;
- result->create_ = create_filter;
- result->key_match_ = key_may_match;
- result->name_ = name;
- return result;
-}
-
-void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t* filter) {
- delete filter;
-}
-
-leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) {
- // Make a leveldb_filterpolicy_t, but override all of its methods so
- // they delegate to a NewBloomFilterPolicy() instead of user
- // supplied C functions.
- struct Wrapper : public leveldb_filterpolicy_t {
- const FilterPolicy* rep_;
- ~Wrapper() { delete rep_; }
- const char* Name() const { return rep_->Name(); }
- void CreateFilter(const Slice* keys, int n, std::string* dst) const {
- return rep_->CreateFilter(keys, n, dst);
- }
- bool KeyMayMatch(const Slice& key, const Slice& filter) const {
- return rep_->KeyMayMatch(key, filter);
- }
- static void DoNothing(void*) { }
- };
- Wrapper* wrapper = new Wrapper;
- wrapper->rep_ = NewBloomFilterPolicy(bits_per_key);
- wrapper->state_ = NULL;
- wrapper->destructor_ = &Wrapper::DoNothing;
- return wrapper;
-}
-
-leveldb_readoptions_t* leveldb_readoptions_create() {
- return new leveldb_readoptions_t;
-}
-
-void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) {
- delete opt;
-}
-
-void leveldb_readoptions_set_verify_checksums(
- leveldb_readoptions_t* opt,
- unsigned char v) {
- opt->rep.verify_checksums = v;
-}
-
-void leveldb_readoptions_set_fill_cache(
- leveldb_readoptions_t* opt, unsigned char v) {
- opt->rep.fill_cache = v;
-}
-
-void leveldb_readoptions_set_snapshot(
- leveldb_readoptions_t* opt,
- const leveldb_snapshot_t* snap) {
- opt->rep.snapshot = (snap ? snap->rep : NULL);
-}
-
-leveldb_writeoptions_t* leveldb_writeoptions_create() {
- return new leveldb_writeoptions_t;
-}
-
-void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) {
- delete opt;
-}
-
-void leveldb_writeoptions_set_sync(
- leveldb_writeoptions_t* opt, unsigned char v) {
- opt->rep.sync = v;
-}
-
-leveldb_cache_t* leveldb_cache_create_lru(size_t capacity) {
- leveldb_cache_t* c = new leveldb_cache_t;
- c->rep = NewLRUCache(capacity);
- return c;
-}
-
-void leveldb_cache_destroy(leveldb_cache_t* cache) {
- delete cache->rep;
- delete cache;
-}
-
-leveldb_env_t* leveldb_create_default_env() {
- leveldb_env_t* result = new leveldb_env_t;
- result->rep = Env::Default();
- result->is_default = true;
- return result;
-}
-
-void leveldb_env_destroy(leveldb_env_t* env) {
- if (!env->is_default) delete env->rep;
- delete env;
-}
-
-void leveldb_free(void* ptr) {
- free(ptr);
-}
-
-int leveldb_major_version() {
- return kMajorVersion;
-}
-
-int leveldb_minor_version() {
- return kMinorVersion;
-}
-
-} // end extern "C"
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/c_test.c
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/c_test.c b/thirdparty/leveldb-1.18/db/c_test.c
deleted file mode 100755
index 7cd5ee0..0000000
--- a/thirdparty/leveldb-1.18/db/c_test.c
+++ /dev/null
@@ -1,390 +0,0 @@
-/* Copyright (c) 2011 The LevelDB Authors. All rights reserved.
- Use of this source code is governed by a BSD-style license that can be
- found in the LICENSE file. See the AUTHORS file for names of contributors. */
-
-#include "leveldb/c.h"
-
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-const char* phase = "";
-static char dbname[200];
-
-static void StartPhase(const char* name) {
- fprintf(stderr, "=== Test %s\n", name);
- phase = name;
-}
-
-static const char* GetTempDir(void) {
- const char* ret = getenv("TEST_TMPDIR");
- if (ret == NULL || ret[0] == '\0')
- ret = "/tmp";
- return ret;
-}
-
-#define CheckNoError(err) \
- if ((err) != NULL) { \
- fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \
- abort(); \
- }
-
-#define CheckCondition(cond) \
- if (!(cond)) { \
- fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \
- abort(); \
- }
-
-static void CheckEqual(const char* expected, const char* v, size_t n) {
- if (expected == NULL && v == NULL) {
- // ok
- } else if (expected != NULL && v != NULL && n == strlen(expected) &&
- memcmp(expected, v, n) == 0) {
- // ok
- return;
- } else {
- fprintf(stderr, "%s: expected '%s', got '%s'\n",
- phase,
- (expected ? expected : "(null)"),
- (v ? v : "(null"));
- abort();
- }
-}
-
-static void Free(char** ptr) {
- if (*ptr) {
- free(*ptr);
- *ptr = NULL;
- }
-}
-
-static void CheckGet(
- leveldb_t* db,
- const leveldb_readoptions_t* options,
- const char* key,
- const char* expected) {
- char* err = NULL;
- size_t val_len;
- char* val;
- val = leveldb_get(db, options, key, strlen(key), &val_len, &err);
- CheckNoError(err);
- CheckEqual(expected, val, val_len);
- Free(&val);
-}
-
-static void CheckIter(leveldb_iterator_t* iter,
- const char* key, const char* val) {
- size_t len;
- const char* str;
- str = leveldb_iter_key(iter, &len);
- CheckEqual(key, str, len);
- str = leveldb_iter_value(iter, &len);
- CheckEqual(val, str, len);
-}
-
-// Callback from leveldb_writebatch_iterate()
-static void CheckPut(void* ptr,
- const char* k, size_t klen,
- const char* v, size_t vlen) {
- int* state = (int*) ptr;
- CheckCondition(*state < 2);
- switch (*state) {
- case 0:
- CheckEqual("bar", k, klen);
- CheckEqual("b", v, vlen);
- break;
- case 1:
- CheckEqual("box", k, klen);
- CheckEqual("c", v, vlen);
- break;
- }
- (*state)++;
-}
-
-// Callback from leveldb_writebatch_iterate()
-static void CheckDel(void* ptr, const char* k, size_t klen) {
- int* state = (int*) ptr;
- CheckCondition(*state == 2);
- CheckEqual("bar", k, klen);
- (*state)++;
-}
-
-static void CmpDestroy(void* arg) { }
-
-static int CmpCompare(void* arg, const char* a, size_t alen,
- const char* b, size_t blen) {
- int n = (alen < blen) ? alen : blen;
- int r = memcmp(a, b, n);
- if (r == 0) {
- if (alen < blen) r = -1;
- else if (alen > blen) r = +1;
- }
- return r;
-}
-
-static const char* CmpName(void* arg) {
- return "foo";
-}
-
-// Custom filter policy
-static unsigned char fake_filter_result = 1;
-static void FilterDestroy(void* arg) { }
-static const char* FilterName(void* arg) {
- return "TestFilter";
-}
-static char* FilterCreate(
- void* arg,
- const char* const* key_array, const size_t* key_length_array,
- int num_keys,
- size_t* filter_length) {
- *filter_length = 4;
- char* result = malloc(4);
- memcpy(result, "fake", 4);
- return result;
-}
-unsigned char FilterKeyMatch(
- void* arg,
- const char* key, size_t length,
- const char* filter, size_t filter_length) {
- CheckCondition(filter_length == 4);
- CheckCondition(memcmp(filter, "fake", 4) == 0);
- return fake_filter_result;
-}
-
-int main(int argc, char** argv) {
- leveldb_t* db;
- leveldb_comparator_t* cmp;
- leveldb_cache_t* cache;
- leveldb_env_t* env;
- leveldb_options_t* options;
- leveldb_readoptions_t* roptions;
- leveldb_writeoptions_t* woptions;
- char* err = NULL;
- int run = -1;
-
- CheckCondition(leveldb_major_version() >= 1);
- CheckCondition(leveldb_minor_version() >= 1);
-
- snprintf(dbname, sizeof(dbname),
- "%s/leveldb_c_test-%d",
- GetTempDir(),
- ((int) geteuid()));
-
- StartPhase("create_objects");
- cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
- env = leveldb_create_default_env();
- cache = leveldb_cache_create_lru(100000);
-
- options = leveldb_options_create();
- leveldb_options_set_comparator(options, cmp);
- leveldb_options_set_error_if_exists(options, 1);
- leveldb_options_set_cache(options, cache);
- leveldb_options_set_env(options, env);
- leveldb_options_set_info_log(options, NULL);
- leveldb_options_set_write_buffer_size(options, 100000);
- leveldb_options_set_paranoid_checks(options, 1);
- leveldb_options_set_max_open_files(options, 10);
- leveldb_options_set_block_size(options, 1024);
- leveldb_options_set_block_restart_interval(options, 8);
- leveldb_options_set_compression(options, leveldb_no_compression);
-
- roptions = leveldb_readoptions_create();
- leveldb_readoptions_set_verify_checksums(roptions, 1);
- leveldb_readoptions_set_fill_cache(roptions, 0);
-
- woptions = leveldb_writeoptions_create();
- leveldb_writeoptions_set_sync(woptions, 1);
-
- StartPhase("destroy");
- leveldb_destroy_db(options, dbname, &err);
- Free(&err);
-
- StartPhase("open_error");
- db = leveldb_open(options, dbname, &err);
- CheckCondition(err != NULL);
- Free(&err);
-
- StartPhase("leveldb_free");
- db = leveldb_open(options, dbname, &err);
- CheckCondition(err != NULL);
- leveldb_free(err);
- err = NULL;
-
- StartPhase("open");
- leveldb_options_set_create_if_missing(options, 1);
- db = leveldb_open(options, dbname, &err);
- CheckNoError(err);
- CheckGet(db, roptions, "foo", NULL);
-
- StartPhase("put");
- leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
- CheckNoError(err);
- CheckGet(db, roptions, "foo", "hello");
-
- StartPhase("compactall");
- leveldb_compact_range(db, NULL, 0, NULL, 0);
- CheckGet(db, roptions, "foo", "hello");
-
- StartPhase("compactrange");
- leveldb_compact_range(db, "a", 1, "z", 1);
- CheckGet(db, roptions, "foo", "hello");
-
- StartPhase("writebatch");
- {
- leveldb_writebatch_t* wb = leveldb_writebatch_create();
- leveldb_writebatch_put(wb, "foo", 3, "a", 1);
- leveldb_writebatch_clear(wb);
- leveldb_writebatch_put(wb, "bar", 3, "b", 1);
- leveldb_writebatch_put(wb, "box", 3, "c", 1);
- leveldb_writebatch_delete(wb, "bar", 3);
- leveldb_write(db, woptions, wb, &err);
- CheckNoError(err);
- CheckGet(db, roptions, "foo", "hello");
- CheckGet(db, roptions, "bar", NULL);
- CheckGet(db, roptions, "box", "c");
- int pos = 0;
- leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
- CheckCondition(pos == 3);
- leveldb_writebatch_destroy(wb);
- }
-
- StartPhase("iter");
- {
- leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
- CheckCondition(!leveldb_iter_valid(iter));
- leveldb_iter_seek_to_first(iter);
- CheckCondition(leveldb_iter_valid(iter));
- CheckIter(iter, "box", "c");
- leveldb_iter_next(iter);
- CheckIter(iter, "foo", "hello");
- leveldb_iter_prev(iter);
- CheckIter(iter, "box", "c");
- leveldb_iter_prev(iter);
- CheckCondition(!leveldb_iter_valid(iter));
- leveldb_iter_seek_to_last(iter);
- CheckIter(iter, "foo", "hello");
- leveldb_iter_seek(iter, "b", 1);
- CheckIter(iter, "box", "c");
- leveldb_iter_get_error(iter, &err);
- CheckNoError(err);
- leveldb_iter_destroy(iter);
- }
-
- StartPhase("approximate_sizes");
- {
- int i;
- int n = 20000;
- char keybuf[100];
- char valbuf[100];
- uint64_t sizes[2];
- const char* start[2] = { "a", "k00000000000000010000" };
- size_t start_len[2] = { 1, 21 };
- const char* limit[2] = { "k00000000000000010000", "z" };
- size_t limit_len[2] = { 21, 1 };
- leveldb_writeoptions_set_sync(woptions, 0);
- for (i = 0; i < n; i++) {
- snprintf(keybuf, sizeof(keybuf), "k%020d", i);
- snprintf(valbuf, sizeof(valbuf), "v%020d", i);
- leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
- &err);
- CheckNoError(err);
- }
- leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
- CheckCondition(sizes[0] > 0);
- CheckCondition(sizes[1] > 0);
- }
-
- StartPhase("property");
- {
- char* prop = leveldb_property_value(db, "nosuchprop");
- CheckCondition(prop == NULL);
- prop = leveldb_property_value(db, "leveldb.stats");
- CheckCondition(prop != NULL);
- Free(&prop);
- }
-
- StartPhase("snapshot");
- {
- const leveldb_snapshot_t* snap;
- snap = leveldb_create_snapshot(db);
- leveldb_delete(db, woptions, "foo", 3, &err);
- CheckNoError(err);
- leveldb_readoptions_set_snapshot(roptions, snap);
- CheckGet(db, roptions, "foo", "hello");
- leveldb_readoptions_set_snapshot(roptions, NULL);
- CheckGet(db, roptions, "foo", NULL);
- leveldb_release_snapshot(db, snap);
- }
-
- StartPhase("repair");
- {
- leveldb_close(db);
- leveldb_options_set_create_if_missing(options, 0);
- leveldb_options_set_error_if_exists(options, 0);
- leveldb_repair_db(options, dbname, &err);
- CheckNoError(err);
- db = leveldb_open(options, dbname, &err);
- CheckNoError(err);
- CheckGet(db, roptions, "foo", NULL);
- CheckGet(db, roptions, "bar", NULL);
- CheckGet(db, roptions, "box", "c");
- leveldb_options_set_create_if_missing(options, 1);
- leveldb_options_set_error_if_exists(options, 1);
- }
-
- StartPhase("filter");
- for (run = 0; run < 2; run++) {
- // First run uses custom filter, second run uses bloom filter
- CheckNoError(err);
- leveldb_filterpolicy_t* policy;
- if (run == 0) {
- policy = leveldb_filterpolicy_create(
- NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName);
- } else {
- policy = leveldb_filterpolicy_create_bloom(10);
- }
-
- // Create new database
- leveldb_close(db);
- leveldb_destroy_db(options, dbname, &err);
- leveldb_options_set_filter_policy(options, policy);
- db = leveldb_open(options, dbname, &err);
- CheckNoError(err);
- leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err);
- CheckNoError(err);
- leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err);
- CheckNoError(err);
- leveldb_compact_range(db, NULL, 0, NULL, 0);
-
- fake_filter_result = 1;
- CheckGet(db, roptions, "foo", "foovalue");
- CheckGet(db, roptions, "bar", "barvalue");
- if (phase == 0) {
- // Must not find value when custom filter returns false
- fake_filter_result = 0;
- CheckGet(db, roptions, "foo", NULL);
- CheckGet(db, roptions, "bar", NULL);
- fake_filter_result = 1;
-
- CheckGet(db, roptions, "foo", "foovalue");
- CheckGet(db, roptions, "bar", "barvalue");
- }
- leveldb_options_set_filter_policy(options, NULL);
- leveldb_filterpolicy_destroy(policy);
- }
-
- StartPhase("cleanup");
- leveldb_close(db);
- leveldb_options_destroy(options);
- leveldb_readoptions_destroy(roptions);
- leveldb_writeoptions_destroy(woptions);
- leveldb_cache_destroy(cache);
- leveldb_comparator_destroy(cmp);
- leveldb_env_destroy(env);
-
- fprintf(stderr, "PASS\n");
- return 0;
-}
[27/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compaction.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compaction.cc b/thirdparty/rocksdb/db/compaction.cc
new file mode 100644
index 0000000..9ea332d
--- /dev/null
+++ b/thirdparty/rocksdb/db/compaction.cc
@@ -0,0 +1,480 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/compaction.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <vector>
+
+#include "db/column_family.h"
+#include "rocksdb/compaction_filter.h"
+#include "util/string_util.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+uint64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
+ uint64_t sum = 0;
+ for (size_t i = 0; i < files.size() && files[i]; i++) {
+ sum += files[i]->fd.GetFileSize();
+ }
+ return sum;
+}
+
+void Compaction::SetInputVersion(Version* _input_version) {
+ input_version_ = _input_version;
+ cfd_ = input_version_->cfd();
+
+ cfd_->Ref();
+ input_version_->Ref();
+ edit_.SetColumnFamily(cfd_->GetID());
+}
+
+void Compaction::GetBoundaryKeys(
+ VersionStorageInfo* vstorage,
+ const std::vector<CompactionInputFiles>& inputs, Slice* smallest_user_key,
+ Slice* largest_user_key) {
+ bool initialized = false;
+ const Comparator* ucmp = vstorage->InternalComparator()->user_comparator();
+ for (size_t i = 0; i < inputs.size(); ++i) {
+ if (inputs[i].files.empty()) {
+ continue;
+ }
+ if (inputs[i].level == 0) {
+ // we need to consider all files on level 0
+ for (const auto* f : inputs[i].files) {
+ const Slice& start_user_key = f->smallest.user_key();
+ if (!initialized ||
+ ucmp->Compare(start_user_key, *smallest_user_key) < 0) {
+ *smallest_user_key = start_user_key;
+ }
+ const Slice& end_user_key = f->largest.user_key();
+ if (!initialized ||
+ ucmp->Compare(end_user_key, *largest_user_key) > 0) {
+ *largest_user_key = end_user_key;
+ }
+ initialized = true;
+ }
+ } else {
+ // we only need to consider the first and last file
+ const Slice& start_user_key = inputs[i].files[0]->smallest.user_key();
+ if (!initialized ||
+ ucmp->Compare(start_user_key, *smallest_user_key) < 0) {
+ *smallest_user_key = start_user_key;
+ }
+ const Slice& end_user_key = inputs[i].files.back()->largest.user_key();
+ if (!initialized || ucmp->Compare(end_user_key, *largest_user_key) > 0) {
+ *largest_user_key = end_user_key;
+ }
+ initialized = true;
+ }
+ }
+}
+
+// helper function to determine if compaction is creating files at the
+// bottommost level
+bool Compaction::IsBottommostLevel(
+ int output_level, VersionStorageInfo* vstorage,
+ const std::vector<CompactionInputFiles>& inputs) {
+ if (inputs[0].level == 0 &&
+ inputs[0].files.back() != vstorage->LevelFiles(0).back()) {
+ return false;
+ }
+
+ Slice smallest_key, largest_key;
+ GetBoundaryKeys(vstorage, inputs, &smallest_key, &largest_key);
+
+ // Checks whether there are files living beyond the output_level.
+ // If lower levels have files, it checks for overlap between files
+ // if the compaction process and those files.
+ // Bottomlevel optimizations can be made if there are no files in
+ // lower levels or if there is no overlap with the files in
+ // the lower levels.
+ for (int i = output_level + 1; i < vstorage->num_levels(); i++) {
+ // It is not the bottommost level if there are files in higher
+ // levels when the output level is 0 or if there are files in
+ // higher levels which overlap with files to be compacted.
+ // output_level == 0 means that we want it to be considered
+ // s the bottommost level only if the last file on the level
+ // is a part of the files to be compacted - this is verified by
+ // the first if condition in this function
+ if (vstorage->NumLevelFiles(i) > 0 &&
+ (output_level == 0 ||
+ vstorage->OverlapInLevel(i, &smallest_key, &largest_key))) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// test function to validate the functionality of IsBottommostLevel()
+// function -- determines if compaction with inputs and storage is bottommost
+bool Compaction::TEST_IsBottommostLevel(
+ int output_level, VersionStorageInfo* vstorage,
+ const std::vector<CompactionInputFiles>& inputs) {
+ return IsBottommostLevel(output_level, vstorage, inputs);
+}
+
+bool Compaction::IsFullCompaction(
+ VersionStorageInfo* vstorage,
+ const std::vector<CompactionInputFiles>& inputs) {
+ size_t num_files_in_compaction = 0;
+ size_t total_num_files = 0;
+ for (int l = 0; l < vstorage->num_levels(); l++) {
+ total_num_files += vstorage->NumLevelFiles(l);
+ }
+ for (size_t i = 0; i < inputs.size(); i++) {
+ num_files_in_compaction += inputs[i].size();
+ }
+ return num_files_in_compaction == total_num_files;
+}
+
+Compaction::Compaction(VersionStorageInfo* vstorage,
+ const ImmutableCFOptions& _immutable_cf_options,
+ const MutableCFOptions& _mutable_cf_options,
+ std::vector<CompactionInputFiles> _inputs,
+ int _output_level, uint64_t _target_file_size,
+ uint64_t _max_compaction_bytes, uint32_t _output_path_id,
+ CompressionType _compression,
+ std::vector<FileMetaData*> _grandparents,
+ bool _manual_compaction, double _score,
+ bool _deletion_compaction,
+ CompactionReason _compaction_reason)
+ : input_vstorage_(vstorage),
+ start_level_(_inputs[0].level),
+ output_level_(_output_level),
+ max_output_file_size_(_target_file_size),
+ max_compaction_bytes_(_max_compaction_bytes),
+ immutable_cf_options_(_immutable_cf_options),
+ mutable_cf_options_(_mutable_cf_options),
+ input_version_(nullptr),
+ number_levels_(vstorage->num_levels()),
+ cfd_(nullptr),
+ output_path_id_(_output_path_id),
+ output_compression_(_compression),
+ deletion_compaction_(_deletion_compaction),
+ inputs_(std::move(_inputs)),
+ grandparents_(std::move(_grandparents)),
+ score_(_score),
+ bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
+ is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
+ is_manual_compaction_(_manual_compaction),
+ is_trivial_move_(false),
+ compaction_reason_(_compaction_reason) {
+ MarkFilesBeingCompacted(true);
+ if (is_manual_compaction_) {
+ compaction_reason_ = CompactionReason::kManualCompaction;
+ }
+
+#ifndef NDEBUG
+ for (size_t i = 1; i < inputs_.size(); ++i) {
+ assert(inputs_[i].level > inputs_[i - 1].level);
+ }
+#endif
+
+ // setup input_levels_
+ {
+ input_levels_.resize(num_input_levels());
+ for (size_t which = 0; which < num_input_levels(); which++) {
+ DoGenerateLevelFilesBrief(&input_levels_[which], inputs_[which].files,
+ &arena_);
+ }
+ }
+
+ GetBoundaryKeys(vstorage, inputs_, &smallest_user_key_, &largest_user_key_);
+}
+
+Compaction::~Compaction() {
+ if (input_version_ != nullptr) {
+ input_version_->Unref();
+ }
+ if (cfd_ != nullptr) {
+ if (cfd_->Unref()) {
+ delete cfd_;
+ }
+ }
+}
+
+bool Compaction::InputCompressionMatchesOutput() const {
+ int base_level = input_vstorage_->base_level();
+ bool matches = (GetCompressionType(immutable_cf_options_, input_vstorage_,
+ mutable_cf_options_, start_level_,
+ base_level) == output_compression_);
+ if (matches) {
+ TEST_SYNC_POINT("Compaction::InputCompressionMatchesOutput:Matches");
+ return true;
+ }
+ TEST_SYNC_POINT("Compaction::InputCompressionMatchesOutput:DidntMatch");
+ return matches;
+}
+
+bool Compaction::IsTrivialMove() const {
+ // Avoid a move if there is lots of overlapping grandparent data.
+ // Otherwise, the move could create a parent file that will require
+ // a very expensive merge later on.
+ // If start_level_== output_level_, the purpose is to force compaction
+ // filter to be applied to that level, and thus cannot be a trivial move.
+
+ // Check if start level have files with overlapping ranges
+ if (start_level_ == 0 && input_vstorage_->level0_non_overlapping() == false) {
+ // We cannot move files from L0 to L1 if the files are overlapping
+ return false;
+ }
+
+ if (is_manual_compaction_ &&
+ (immutable_cf_options_.compaction_filter != nullptr ||
+ immutable_cf_options_.compaction_filter_factory != nullptr)) {
+ // This is a manual compaction and we have a compaction filter that should
+ // be executed, we cannot do a trivial move
+ return false;
+ }
+
+ // Used in universal compaction, where trivial move can be done if the
+ // input files are non overlapping
+ if ((immutable_cf_options_.compaction_options_universal.allow_trivial_move) &&
+ (output_level_ != 0)) {
+ return is_trivial_move_;
+ }
+
+ if (!(start_level_ != output_level_ && num_input_levels() == 1 &&
+ input(0, 0)->fd.GetPathId() == output_path_id() &&
+ InputCompressionMatchesOutput())) {
+ return false;
+ }
+
+ // assert inputs_.size() == 1
+
+ for (const auto& file : inputs_.front().files) {
+ std::vector<FileMetaData*> file_grand_parents;
+ if (output_level_ + 1 >= number_levels_) {
+ continue;
+ }
+ input_vstorage_->GetOverlappingInputs(output_level_ + 1, &file->smallest,
+ &file->largest, &file_grand_parents);
+ const auto compaction_size =
+ file->fd.GetFileSize() + TotalFileSize(file_grand_parents);
+ if (compaction_size > max_compaction_bytes_) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void Compaction::AddInputDeletions(VersionEdit* out_edit) {
+ for (size_t which = 0; which < num_input_levels(); which++) {
+ for (size_t i = 0; i < inputs_[which].size(); i++) {
+ out_edit->DeleteFile(level(which), inputs_[which][i]->fd.GetNumber());
+ }
+ }
+}
+
+bool Compaction::KeyNotExistsBeyondOutputLevel(
+ const Slice& user_key, std::vector<size_t>* level_ptrs) const {
+ assert(input_version_ != nullptr);
+ assert(level_ptrs != nullptr);
+ assert(level_ptrs->size() == static_cast<size_t>(number_levels_));
+ assert(cfd_->ioptions()->compaction_style != kCompactionStyleFIFO);
+ if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
+ return bottommost_level_;
+ }
+ if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel &&
+ output_level_ == 0) {
+ return false;
+ }
+ // Maybe use binary search to find right entry instead of linear search?
+ const Comparator* user_cmp = cfd_->user_comparator();
+ for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
+ const std::vector<FileMetaData*>& files = input_vstorage_->LevelFiles(lvl);
+ for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
+ auto* f = files[level_ptrs->at(lvl)];
+ if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
+ // We've advanced far enough
+ if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
+ // Key falls in this file's range, so definitely
+ // exists beyond output level
+ return false;
+ }
+ break;
+ }
+ }
+ }
+ return true;
+}
+
+// Mark (or clear) each file that is being compacted
+void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
+ for (size_t i = 0; i < num_input_levels(); i++) {
+ for (size_t j = 0; j < inputs_[i].size(); j++) {
+ assert(mark_as_compacted ? !inputs_[i][j]->being_compacted
+ : inputs_[i][j]->being_compacted);
+ inputs_[i][j]->being_compacted = mark_as_compacted;
+ }
+ }
+}
+
+// Sample output:
+// If compacting 3 L0 files, 2 L3 files and 1 L4 file, and outputting to L5,
+// print: "3@0 + 2@3 + 1@4 files to L5"
+const char* Compaction::InputLevelSummary(
+ InputLevelSummaryBuffer* scratch) const {
+ int len = 0;
+ bool is_first = true;
+ for (auto& input_level : inputs_) {
+ if (input_level.empty()) {
+ continue;
+ }
+ if (!is_first) {
+ len +=
+ snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, " + ");
+ } else {
+ is_first = false;
+ }
+ len += snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len,
+ "%" ROCKSDB_PRIszt "@%d", input_level.size(),
+ input_level.level);
+ }
+ snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len,
+ " files to L%d", output_level());
+
+ return scratch->buffer;
+}
+
+uint64_t Compaction::CalculateTotalInputSize() const {
+ uint64_t size = 0;
+ for (auto& input_level : inputs_) {
+ for (auto f : input_level.files) {
+ size += f->fd.GetFileSize();
+ }
+ }
+ return size;
+}
+
+void Compaction::ReleaseCompactionFiles(Status status) {
+ MarkFilesBeingCompacted(false);
+ cfd_->compaction_picker()->ReleaseCompactionFiles(this, status);
+}
+
+void Compaction::ResetNextCompactionIndex() {
+ assert(input_version_ != nullptr);
+ input_vstorage_->ResetNextCompactionIndex(start_level_);
+}
+
+namespace {
+int InputSummary(const std::vector<FileMetaData*>& files, char* output,
+ int len) {
+ *output = '\0';
+ int write = 0;
+ for (size_t i = 0; i < files.size(); i++) {
+ int sz = len - write;
+ int ret;
+ char sztxt[16];
+ AppendHumanBytes(files.at(i)->fd.GetFileSize(), sztxt, 16);
+ ret = snprintf(output + write, sz, "%" PRIu64 "(%s) ",
+ files.at(i)->fd.GetNumber(), sztxt);
+ if (ret < 0 || ret >= sz) break;
+ write += ret;
+ }
+ // if files.size() is non-zero, overwrite the last space
+ return write - !!files.size();
+}
+} // namespace
+
+void Compaction::Summary(char* output, int len) {
+ int write =
+ snprintf(output, len, "Base version %" PRIu64 " Base level %d, inputs: [",
+ input_version_->GetVersionNumber(), start_level_);
+ if (write < 0 || write >= len) {
+ return;
+ }
+
+ for (size_t level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
+ if (level_iter > 0) {
+ write += snprintf(output + write, len - write, "], [");
+ if (write < 0 || write >= len) {
+ return;
+ }
+ }
+ write +=
+ InputSummary(inputs_[level_iter].files, output + write, len - write);
+ if (write < 0 || write >= len) {
+ return;
+ }
+ }
+
+ snprintf(output + write, len - write, "]");
+}
+
+uint64_t Compaction::OutputFilePreallocationSize() const {
+ uint64_t preallocation_size = 0;
+
+ if (max_output_file_size_ != port::kMaxUint64 &&
+ (cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||
+ output_level() > 0)) {
+ preallocation_size = max_output_file_size_;
+ } else {
+ for (const auto& level_files : inputs_) {
+ for (const auto& file : level_files.files) {
+ preallocation_size += file->fd.GetFileSize();
+ }
+ }
+ }
+ // Over-estimate slightly so we don't end up just barely crossing
+ // the threshold
+ return preallocation_size + (preallocation_size / 10);
+}
+
+std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
+ if (!cfd_->ioptions()->compaction_filter_factory) {
+ return nullptr;
+ }
+
+ CompactionFilter::Context context;
+ context.is_full_compaction = is_full_compaction_;
+ context.is_manual_compaction = is_manual_compaction_;
+ context.column_family_id = cfd_->GetID();
+ return cfd_->ioptions()->compaction_filter_factory->CreateCompactionFilter(
+ context);
+}
+
+bool Compaction::IsOutputLevelEmpty() const {
+ return inputs_.back().level != output_level_ || inputs_.back().empty();
+}
+
+bool Compaction::ShouldFormSubcompactions() const {
+ if (immutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) {
+ return false;
+ }
+ if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
+ return start_level_ == 0 && output_level_ > 0 && !IsOutputLevelEmpty();
+ } else if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
+ return number_levels_ > 1 && output_level_ > 0;
+ } else {
+ return false;
+ }
+}
+
+uint64_t Compaction::MaxInputFileCreationTime() const {
+ uint64_t max_creation_time = 0;
+ for (const auto& file : inputs_[0].files) {
+ if (file->fd.table_reader != nullptr &&
+ file->fd.table_reader->GetTableProperties() != nullptr) {
+ uint64_t creation_time =
+ file->fd.table_reader->GetTableProperties()->creation_time;
+ max_creation_time = std::max(max_creation_time, creation_time);
+ }
+ }
+ return max_creation_time;
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compaction.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compaction.h b/thirdparty/rocksdb/db/compaction.h
new file mode 100644
index 0000000..7be6df2
--- /dev/null
+++ b/thirdparty/rocksdb/db/compaction.h
@@ -0,0 +1,325 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include "db/version_set.h"
+#include "options/cf_options.h"
+#include "util/arena.h"
+#include "util/autovector.h"
+
+namespace rocksdb {
+
+// The structure that manages compaction input files associated
+// with the same physical level.
+struct CompactionInputFiles {
+ int level;
+ std::vector<FileMetaData*> files;
+ inline bool empty() const { return files.empty(); }
+ inline size_t size() const { return files.size(); }
+ inline void clear() { files.clear(); }
+ inline FileMetaData* operator[](size_t i) const { return files[i]; }
+};
+
+class Version;
+class ColumnFamilyData;
+class VersionStorageInfo;
+class CompactionFilter;
+
+// A Compaction encapsulates information about a compaction.
+class Compaction {
+ public:
+ Compaction(VersionStorageInfo* input_version,
+ const ImmutableCFOptions& immutable_cf_options,
+ const MutableCFOptions& mutable_cf_options,
+ std::vector<CompactionInputFiles> inputs, int output_level,
+ uint64_t target_file_size, uint64_t max_compaction_bytes,
+ uint32_t output_path_id, CompressionType compression,
+ std::vector<FileMetaData*> grandparents,
+ bool manual_compaction = false, double score = -1,
+ bool deletion_compaction = false,
+ CompactionReason compaction_reason = CompactionReason::kUnknown);
+
+ // No copying allowed
+ Compaction(const Compaction&) = delete;
+ void operator=(const Compaction&) = delete;
+
+ ~Compaction();
+
+ // Returns the level associated to the specified compaction input level.
+ // If compaction_input_level is not specified, then input_level is set to 0.
+ int level(size_t compaction_input_level = 0) const {
+ return inputs_[compaction_input_level].level;
+ }
+
+ int start_level() const { return start_level_; }
+
+ // Outputs will go to this level
+ int output_level() const { return output_level_; }
+
+ // Returns the number of input levels in this compaction.
+ size_t num_input_levels() const { return inputs_.size(); }
+
+ // Return the object that holds the edits to the descriptor done
+ // by this compaction.
+ VersionEdit* edit() { return &edit_; }
+
+ // Returns the number of input files associated to the specified
+ // compaction input level.
+ // The function will return 0 if when "compaction_input_level" < 0
+ // or "compaction_input_level" >= "num_input_levels()".
+ size_t num_input_files(size_t compaction_input_level) const {
+ if (compaction_input_level < inputs_.size()) {
+ return inputs_[compaction_input_level].size();
+ }
+ return 0;
+ }
+
+ // Returns input version of the compaction
+ Version* input_version() const { return input_version_; }
+
+ // Returns the ColumnFamilyData associated with the compaction.
+ ColumnFamilyData* column_family_data() const { return cfd_; }
+
+ // Returns the file meta data of the 'i'th input file at the
+ // specified compaction input level.
+ // REQUIREMENT: "compaction_input_level" must be >= 0 and
+ // < "input_levels()"
+ FileMetaData* input(size_t compaction_input_level, size_t i) const {
+ assert(compaction_input_level < inputs_.size());
+ return inputs_[compaction_input_level][i];
+ }
+
+ // Returns the list of file meta data of the specified compaction
+ // input level.
+ // REQUIREMENT: "compaction_input_level" must be >= 0 and
+ // < "input_levels()"
+ const std::vector<FileMetaData*>* inputs(
+ size_t compaction_input_level) const {
+ assert(compaction_input_level < inputs_.size());
+ return &inputs_[compaction_input_level].files;
+ }
+
+ const std::vector<CompactionInputFiles>* inputs() { return &inputs_; }
+
+ // Returns the LevelFilesBrief of the specified compaction input level.
+ const LevelFilesBrief* input_levels(size_t compaction_input_level) const {
+ return &input_levels_[compaction_input_level];
+ }
+
+ // Maximum size of files to build during this compaction.
+ uint64_t max_output_file_size() const { return max_output_file_size_; }
+
+ // What compression for output
+ CompressionType output_compression() const { return output_compression_; }
+
+ // Whether need to write output file to second DB path.
+ uint32_t output_path_id() const { return output_path_id_; }
+
+ // Is this a trivial compaction that can be implemented by just
+ // moving a single input file to the next level (no merging or splitting)
+ bool IsTrivialMove() const;
+
+ // If true, then the compaction can be done by simply deleting input files.
+ bool deletion_compaction() const { return deletion_compaction_; }
+
+ // Add all inputs to this compaction as delete operations to *edit.
+ void AddInputDeletions(VersionEdit* edit);
+
+ // Returns true if the available information we have guarantees that
+ // the input "user_key" does not exist in any level beyond "output_level()".
+ bool KeyNotExistsBeyondOutputLevel(const Slice& user_key,
+ std::vector<size_t>* level_ptrs) const;
+
+ // Clear all files to indicate that they are not being compacted
+ // Delete this compaction from the list of running compactions.
+ //
+ // Requirement: DB mutex held
+ void ReleaseCompactionFiles(Status status);
+
+ // Returns the summary of the compaction in "output" with maximum "len"
+ // in bytes. The caller is responsible for the memory management of
+ // "output".
+ void Summary(char* output, int len);
+
+ // Return the score that was used to pick this compaction run.
+ double score() const { return score_; }
+
+ // Is this compaction creating a file in the bottom most level?
+ bool bottommost_level() const { return bottommost_level_; }
+
+ // Does this compaction include all sst files?
+ bool is_full_compaction() const { return is_full_compaction_; }
+
+ // Was this compaction triggered manually by the client?
+ bool is_manual_compaction() const { return is_manual_compaction_; }
+
+ // Used when allow_trivial_move option is set in
+ // Universal compaction. If all the input files are
+ // non overlapping, then is_trivial_move_ variable
+ // will be set true, else false
+ void set_is_trivial_move(bool trivial_move) {
+ is_trivial_move_ = trivial_move;
+ }
+
+ // Used when allow_trivial_move option is set in
+ // Universal compaction. Returns true, if the input files
+ // are non-overlapping and can be trivially moved.
+ bool is_trivial_move() const { return is_trivial_move_; }
+
+ // How many total levels are there?
+ int number_levels() const { return number_levels_; }
+
+ // Return the ImmutableCFOptions that should be used throughout the compaction
+ // procedure
+ const ImmutableCFOptions* immutable_cf_options() const {
+ return &immutable_cf_options_;
+ }
+
+ // Return the MutableCFOptions that should be used throughout the compaction
+ // procedure
+ const MutableCFOptions* mutable_cf_options() const {
+ return &mutable_cf_options_;
+ }
+
+ // Returns the size in bytes that the output file should be preallocated to.
+ // In level compaction, that is max_file_size_. In universal compaction, that
+ // is the sum of all input file sizes.
+ uint64_t OutputFilePreallocationSize() const;
+
+ void SetInputVersion(Version* input_version);
+
+ struct InputLevelSummaryBuffer {
+ char buffer[128];
+ };
+
+ const char* InputLevelSummary(InputLevelSummaryBuffer* scratch) const;
+
+ uint64_t CalculateTotalInputSize() const;
+
+ // In case of compaction error, reset the nextIndex that is used
+ // to pick up the next file to be compacted from files_by_size_
+ void ResetNextCompactionIndex();
+
+ // Create a CompactionFilter from compaction_filter_factory
+ std::unique_ptr<CompactionFilter> CreateCompactionFilter() const;
+
+ // Is the input level corresponding to output_level_ empty?
+ bool IsOutputLevelEmpty() const;
+
+ // Should this compaction be broken up into smaller ones run in parallel?
+ bool ShouldFormSubcompactions() const;
+
+ // test function to validate the functionality of IsBottommostLevel()
+ // function -- determines if compaction with inputs and storage is bottommost
+ static bool TEST_IsBottommostLevel(
+ int output_level, VersionStorageInfo* vstorage,
+ const std::vector<CompactionInputFiles>& inputs);
+
+ TablePropertiesCollection GetOutputTableProperties() const {
+ return output_table_properties_;
+ }
+
+ void SetOutputTableProperties(TablePropertiesCollection tp) {
+ output_table_properties_ = std::move(tp);
+ }
+
+ Slice GetSmallestUserKey() const { return smallest_user_key_; }
+
+ Slice GetLargestUserKey() const { return largest_user_key_; }
+
+ CompactionReason compaction_reason() { return compaction_reason_; }
+
+ const std::vector<FileMetaData*>& grandparents() const {
+ return grandparents_;
+ }
+
+ uint64_t max_compaction_bytes() const { return max_compaction_bytes_; }
+
+ uint64_t MaxInputFileCreationTime() const;
+
+ private:
+ // mark (or clear) all files that are being compacted
+ void MarkFilesBeingCompacted(bool mark_as_compacted);
+
+ // get the smallest and largest key present in files to be compacted
+ static void GetBoundaryKeys(VersionStorageInfo* vstorage,
+ const std::vector<CompactionInputFiles>& inputs,
+ Slice* smallest_key, Slice* largest_key);
+
+ // helper function to determine if compaction with inputs and storage is
+ // bottommost
+ static bool IsBottommostLevel(
+ int output_level, VersionStorageInfo* vstorage,
+ const std::vector<CompactionInputFiles>& inputs);
+
+ static bool IsFullCompaction(VersionStorageInfo* vstorage,
+ const std::vector<CompactionInputFiles>& inputs);
+
+ VersionStorageInfo* input_vstorage_;
+
+ const int start_level_; // the lowest level to be compacted
+ const int output_level_; // levels to which output files are stored
+ uint64_t max_output_file_size_;
+ uint64_t max_compaction_bytes_;
+ const ImmutableCFOptions immutable_cf_options_;
+ const MutableCFOptions mutable_cf_options_;
+ Version* input_version_;
+ VersionEdit edit_;
+ const int number_levels_;
+ ColumnFamilyData* cfd_;
+ Arena arena_; // Arena used to allocate space for file_levels_
+
+ const uint32_t output_path_id_;
+ CompressionType output_compression_;
+ // If true, then the comaction can be done by simply deleting input files.
+ const bool deletion_compaction_;
+
+ // Compaction input files organized by level. Constant after construction
+ const std::vector<CompactionInputFiles> inputs_;
+
+ // A copy of inputs_, organized more closely in memory
+ autovector<LevelFilesBrief, 2> input_levels_;
+
+ // State used to check for number of overlapping grandparent files
+ // (grandparent == "output_level_ + 1")
+ std::vector<FileMetaData*> grandparents_;
+ const double score_; // score that was used to pick this compaction.
+
+ // Is this compaction creating a file in the bottom most level?
+ const bool bottommost_level_;
+ // Does this compaction include all sst files?
+ const bool is_full_compaction_;
+
+ // Is this compaction requested by the client?
+ const bool is_manual_compaction_;
+
+ // True if we can do trivial move in Universal multi level
+ // compaction
+ bool is_trivial_move_;
+
+ // Does input compression match the output compression?
+ bool InputCompressionMatchesOutput() const;
+
+ // table properties of output files
+ TablePropertiesCollection output_table_properties_;
+
+ // smallest user keys in compaction
+ Slice smallest_user_key_;
+
+ // largest user keys in compaction
+ Slice largest_user_key_;
+
+ // Reason for compaction
+ CompactionReason compaction_reason_;
+};
+
+// Utility function
+extern uint64_t TotalFileSize(const std::vector<FileMetaData*>& files);
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compaction_iteration_stats.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compaction_iteration_stats.h b/thirdparty/rocksdb/db/compaction_iteration_stats.h
new file mode 100644
index 0000000..52a666e
--- /dev/null
+++ b/thirdparty/rocksdb/db/compaction_iteration_stats.h
@@ -0,0 +1,33 @@
+// Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+struct CompactionIterationStats {
+ // Compaction statistics
+
+ // Doesn't include records skipped because of
+ // CompactionFilter::Decision::kRemoveAndSkipUntil.
+ int64_t num_record_drop_user = 0;
+
+ int64_t num_record_drop_hidden = 0;
+ int64_t num_record_drop_obsolete = 0;
+ int64_t num_record_drop_range_del = 0;
+ int64_t num_range_del_drop_obsolete = 0;
+ uint64_t total_filter_time = 0;
+
+ // Input statistics
+ // TODO(noetzli): The stats are incomplete. They are lacking everything
+ // consumed by MergeHelper.
+ uint64_t num_input_records = 0;
+ uint64_t num_input_deletion_records = 0;
+ uint64_t num_input_corrupt_records = 0;
+ uint64_t total_input_raw_key_bytes = 0;
+ uint64_t total_input_raw_value_bytes = 0;
+
+ // Single-Delete diagnostics for exceptional situations
+ uint64_t num_single_del_fallthru = 0;
+ uint64_t num_single_del_mismatch = 0;
+};
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compaction_iterator.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compaction_iterator.cc b/thirdparty/rocksdb/db/compaction_iterator.cc
new file mode 100644
index 0000000..08ae197
--- /dev/null
+++ b/thirdparty/rocksdb/db/compaction_iterator.cc
@@ -0,0 +1,576 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "db/compaction_iterator.h"
+#include "rocksdb/listener.h"
+#include "table/internal_iterator.h"
+
+namespace rocksdb {
+
+#ifndef ROCKSDB_LITE
+CompactionEventListener::CompactionListenerValueType fromInternalValueType(
+ ValueType vt) {
+ switch (vt) {
+ case kTypeDeletion:
+ return CompactionEventListener::CompactionListenerValueType::kDelete;
+ case kTypeValue:
+ return CompactionEventListener::CompactionListenerValueType::kValue;
+ case kTypeMerge:
+ return CompactionEventListener::CompactionListenerValueType::
+ kMergeOperand;
+ case kTypeSingleDeletion:
+ return CompactionEventListener::CompactionListenerValueType::
+ kSingleDelete;
+ case kTypeRangeDeletion:
+ return CompactionEventListener::CompactionListenerValueType::kRangeDelete;
+ default:
+ assert(false);
+ return CompactionEventListener::CompactionListenerValueType::kInvalid;
+ }
+}
+#endif // ROCKSDB_LITE
+
+CompactionIterator::CompactionIterator(
+ InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
+ SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+ SequenceNumber earliest_write_conflict_snapshot, Env* env,
+ bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
+ const Compaction* compaction, const CompactionFilter* compaction_filter,
+ CompactionEventListener* compaction_listener,
+ const std::atomic<bool>* shutting_down)
+ : CompactionIterator(
+ input, cmp, merge_helper, last_sequence, snapshots,
+ earliest_write_conflict_snapshot, env, expect_valid_internal_key,
+ range_del_agg,
+ std::unique_ptr<CompactionProxy>(
+ compaction ? new CompactionProxy(compaction) : nullptr),
+ compaction_filter, compaction_listener, shutting_down) {}
+
+CompactionIterator::CompactionIterator(
+ InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
+ SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+ SequenceNumber earliest_write_conflict_snapshot, Env* env,
+ bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
+ std::unique_ptr<CompactionProxy> compaction,
+ const CompactionFilter* compaction_filter,
+ CompactionEventListener* compaction_listener,
+ const std::atomic<bool>* shutting_down)
+ : input_(input),
+ cmp_(cmp),
+ merge_helper_(merge_helper),
+ snapshots_(snapshots),
+ earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
+ env_(env),
+ expect_valid_internal_key_(expect_valid_internal_key),
+ range_del_agg_(range_del_agg),
+ compaction_(std::move(compaction)),
+ compaction_filter_(compaction_filter),
+#ifndef ROCKSDB_LITE
+ compaction_listener_(compaction_listener),
+#endif // ROCKSDB_LITE
+ shutting_down_(shutting_down),
+ ignore_snapshots_(false),
+ merge_out_iter_(merge_helper_) {
+ assert(compaction_filter_ == nullptr || compaction_ != nullptr);
+ bottommost_level_ =
+ compaction_ == nullptr ? false : compaction_->bottommost_level();
+ if (compaction_ != nullptr) {
+ level_ptrs_ = std::vector<size_t>(compaction_->number_levels(), 0);
+ }
+
+ if (snapshots_->size() == 0) {
+ // optimize for fast path if there are no snapshots
+ visible_at_tip_ = true;
+ earliest_snapshot_ = kMaxSequenceNumber;
+ latest_snapshot_ = 0;
+ } else {
+ visible_at_tip_ = false;
+ earliest_snapshot_ = snapshots_->at(0);
+ latest_snapshot_ = snapshots_->back();
+ }
+ if (compaction_filter_ != nullptr) {
+ if (compaction_filter_->IgnoreSnapshots()) {
+ ignore_snapshots_ = true;
+ }
+ } else {
+ ignore_snapshots_ = false;
+ }
+ input_->SetPinnedItersMgr(&pinned_iters_mgr_);
+}
+
+CompactionIterator::~CompactionIterator() {
+ // input_ Iteartor lifetime is longer than pinned_iters_mgr_ lifetime
+ input_->SetPinnedItersMgr(nullptr);
+}
+
+void CompactionIterator::ResetRecordCounts() {
+ iter_stats_.num_record_drop_user = 0;
+ iter_stats_.num_record_drop_hidden = 0;
+ iter_stats_.num_record_drop_obsolete = 0;
+ iter_stats_.num_record_drop_range_del = 0;
+ iter_stats_.num_range_del_drop_obsolete = 0;
+}
+
+void CompactionIterator::SeekToFirst() {
+ NextFromInput();
+ PrepareOutput();
+}
+
+void CompactionIterator::Next() {
+ // If there is a merge output, return it before continuing to process the
+ // input.
+ if (merge_out_iter_.Valid()) {
+ merge_out_iter_.Next();
+
+ // Check if we returned all records of the merge output.
+ if (merge_out_iter_.Valid()) {
+ key_ = merge_out_iter_.key();
+ value_ = merge_out_iter_.value();
+ bool valid_key __attribute__((__unused__)) =
+ ParseInternalKey(key_, &ikey_);
+ // MergeUntil stops when it encounters a corrupt key and does not
+ // include them in the result, so we expect the keys here to be valid.
+ assert(valid_key);
+ // Keep current_key_ in sync.
+ current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
+ key_ = current_key_.GetInternalKey();
+ ikey_.user_key = current_key_.GetUserKey();
+ valid_ = true;
+ } else {
+ // We consumed all pinned merge operands, release pinned iterators
+ pinned_iters_mgr_.ReleasePinnedData();
+ // MergeHelper moves the iterator to the first record after the merged
+ // records, so even though we reached the end of the merge output, we do
+ // not want to advance the iterator.
+ NextFromInput();
+ }
+ } else {
+ // Only advance the input iterator if there is no merge output and the
+ // iterator is not already at the next record.
+ if (!at_next_) {
+ input_->Next();
+ }
+ NextFromInput();
+ }
+
+ if (valid_) {
+ // Record that we've outputted a record for the current key.
+ has_outputted_key_ = true;
+ }
+
+ PrepareOutput();
+}
+
+void CompactionIterator::NextFromInput() {
+ at_next_ = false;
+ valid_ = false;
+
+ while (!valid_ && input_->Valid() && !IsShuttingDown()) {
+ key_ = input_->key();
+ value_ = input_->value();
+ iter_stats_.num_input_records++;
+
+ if (!ParseInternalKey(key_, &ikey_)) {
+ // If `expect_valid_internal_key_` is false, return the corrupted key
+ // and let the caller decide what to do with it.
+ // TODO(noetzli): We should have a more elegant solution for this.
+ if (expect_valid_internal_key_) {
+ assert(!"Corrupted internal key not expected.");
+ status_ = Status::Corruption("Corrupted internal key not expected.");
+ break;
+ }
+ key_ = current_key_.SetInternalKey(key_);
+ has_current_user_key_ = false;
+ current_user_key_sequence_ = kMaxSequenceNumber;
+ current_user_key_snapshot_ = 0;
+ iter_stats_.num_input_corrupt_records++;
+ valid_ = true;
+ break;
+ }
+
+ // Update input statistics
+ if (ikey_.type == kTypeDeletion || ikey_.type == kTypeSingleDeletion) {
+ iter_stats_.num_input_deletion_records++;
+ }
+ iter_stats_.total_input_raw_key_bytes += key_.size();
+ iter_stats_.total_input_raw_value_bytes += value_.size();
+
+ // If need_skip is true, we should seek the input iterator
+ // to internal key skip_until and continue from there.
+ bool need_skip = false;
+ // Points either into compaction_filter_skip_until_ or into
+ // merge_helper_->compaction_filter_skip_until_.
+ Slice skip_until;
+
+ // Check whether the user key changed. After this if statement current_key_
+ // is a copy of the current input key (maybe converted to a delete by the
+ // compaction filter). ikey_.user_key is pointing to the copy.
+ if (!has_current_user_key_ ||
+ !cmp_->Equal(ikey_.user_key, current_user_key_)) {
+ // First occurrence of this user key
+ // Copy key for output
+ key_ = current_key_.SetInternalKey(key_, &ikey_);
+ current_user_key_ = ikey_.user_key;
+ has_current_user_key_ = true;
+ has_outputted_key_ = false;
+ current_user_key_sequence_ = kMaxSequenceNumber;
+ current_user_key_snapshot_ = 0;
+
+#ifndef ROCKSDB_LITE
+ if (compaction_listener_) {
+ compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
+ fromInternalValueType(ikey_.type),
+ value_, ikey_.sequence, true);
+ }
+#endif // ROCKSDB_LITE
+
+ // apply the compaction filter to the first occurrence of the user key
+ if (compaction_filter_ != nullptr && ikey_.type == kTypeValue &&
+ (visible_at_tip_ || ikey_.sequence > latest_snapshot_ ||
+ ignore_snapshots_)) {
+ // If the user has specified a compaction filter and the sequence
+ // number is greater than any external snapshot, then invoke the
+ // filter. If the return value of the compaction filter is true,
+ // replace the entry with a deletion marker.
+ CompactionFilter::Decision filter;
+ compaction_filter_value_.clear();
+ compaction_filter_skip_until_.Clear();
+ {
+ StopWatchNano timer(env_, true);
+ filter = compaction_filter_->FilterV2(
+ compaction_->level(), ikey_.user_key,
+ CompactionFilter::ValueType::kValue, value_,
+ &compaction_filter_value_, compaction_filter_skip_until_.rep());
+ iter_stats_.total_filter_time +=
+ env_ != nullptr ? timer.ElapsedNanos() : 0;
+ }
+
+ if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil &&
+ cmp_->Compare(*compaction_filter_skip_until_.rep(),
+ ikey_.user_key) <= 0) {
+ // Can't skip to a key smaller than the current one.
+ // Keep the key as per FilterV2 documentation.
+ filter = CompactionFilter::Decision::kKeep;
+ }
+
+ if (filter == CompactionFilter::Decision::kRemove) {
+ // convert the current key to a delete; key_ is pointing into
+ // current_key_ at this point, so updating current_key_ updates key()
+ ikey_.type = kTypeDeletion;
+ current_key_.UpdateInternalKey(ikey_.sequence, kTypeDeletion);
+ // no value associated with delete
+ value_.clear();
+ iter_stats_.num_record_drop_user++;
+ } else if (filter == CompactionFilter::Decision::kChangeValue) {
+ value_ = compaction_filter_value_;
+ } else if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil) {
+ need_skip = true;
+ compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber,
+ kValueTypeForSeek);
+ skip_until = compaction_filter_skip_until_.Encode();
+ }
+ }
+ } else {
+#ifndef ROCKSDB_LITE
+ if (compaction_listener_) {
+ compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
+ fromInternalValueType(ikey_.type),
+ value_, ikey_.sequence, false);
+ }
+#endif // ROCKSDB_LITE
+
+ // Update the current key to reflect the new sequence number/type without
+ // copying the user key.
+ // TODO(rven): Compaction filter does not process keys in this path
+ // Need to have the compaction filter process multiple versions
+ // if we have versions on both sides of a snapshot
+ current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
+ key_ = current_key_.GetInternalKey();
+ ikey_.user_key = current_key_.GetUserKey();
+ }
+
+ // If there are no snapshots, then this kv affect visibility at tip.
+ // Otherwise, search though all existing snapshots to find the earliest
+ // snapshot that is affected by this kv.
+ SequenceNumber last_sequence __attribute__((__unused__)) =
+ current_user_key_sequence_;
+ current_user_key_sequence_ = ikey_.sequence;
+ SequenceNumber last_snapshot = current_user_key_snapshot_;
+ SequenceNumber prev_snapshot = 0; // 0 means no previous snapshot
+ current_user_key_snapshot_ =
+ visible_at_tip_
+ ? earliest_snapshot_
+ : findEarliestVisibleSnapshot(ikey_.sequence, &prev_snapshot);
+
+ if (need_skip) {
+ // This case is handled below.
+ } else if (clear_and_output_next_key_) {
+ // In the previous iteration we encountered a single delete that we could
+ // not compact out. We will keep this Put, but can drop it's data.
+ // (See Optimization 3, below.)
+ assert(ikey_.type == kTypeValue);
+ assert(current_user_key_snapshot_ == last_snapshot);
+
+ value_.clear();
+ valid_ = true;
+ clear_and_output_next_key_ = false;
+ } else if (ikey_.type == kTypeSingleDeletion) {
+ // We can compact out a SingleDelete if:
+ // 1) We encounter the corresponding PUT -OR- we know that this key
+ // doesn't appear past this output level
+ // =AND=
+ // 2) We've already returned a record in this snapshot -OR-
+ // there are no earlier earliest_write_conflict_snapshot.
+ //
+ // Rule 1 is needed for SingleDelete correctness. Rule 2 is needed to
+ // allow Transactions to do write-conflict checking (if we compacted away
+ // all keys, then we wouldn't know that a write happened in this
+ // snapshot). If there is no earlier snapshot, then we know that there
+ // are no active transactions that need to know about any writes.
+ //
+ // Optimization 3:
+ // If we encounter a SingleDelete followed by a PUT and Rule 2 is NOT
+ // true, then we must output a SingleDelete. In this case, we will decide
+ // to also output the PUT. While we are compacting less by outputting the
+ // PUT now, hopefully this will lead to better compaction in the future
+ // when Rule 2 is later true (Ie, We are hoping we can later compact out
+ // both the SingleDelete and the Put, while we couldn't if we only
+ // outputted the SingleDelete now).
+ // In this case, we can save space by removing the PUT's value as it will
+ // never be read.
+ //
+ // Deletes and Merges are not supported on the same key that has a
+ // SingleDelete as it is not possible to correctly do any partial
+ // compaction of such a combination of operations. The result of mixing
+ // those operations for a given key is documented as being undefined. So
+ // we can choose how to handle such a combinations of operations. We will
+ // try to compact out as much as we can in these cases.
+ // We will report counts on these anomalous cases.
+
+ // The easiest way to process a SingleDelete during iteration is to peek
+ // ahead at the next key.
+ ParsedInternalKey next_ikey;
+ input_->Next();
+
+ // Check whether the next key exists, is not corrupt, and is the same key
+ // as the single delete.
+ if (input_->Valid() && ParseInternalKey(input_->key(), &next_ikey) &&
+ cmp_->Equal(ikey_.user_key, next_ikey.user_key)) {
+ // Check whether the next key belongs to the same snapshot as the
+ // SingleDelete.
+ if (prev_snapshot == 0 || next_ikey.sequence > prev_snapshot) {
+ if (next_ikey.type == kTypeSingleDeletion) {
+ // We encountered two SingleDeletes in a row. This could be due to
+ // unexpected user input.
+ // Skip the first SingleDelete and let the next iteration decide how
+ // to handle the second SingleDelete
+
+ // First SingleDelete has been skipped since we already called
+ // input_->Next().
+ ++iter_stats_.num_record_drop_obsolete;
+ ++iter_stats_.num_single_del_mismatch;
+ } else if ((ikey_.sequence <= earliest_write_conflict_snapshot_) ||
+ has_outputted_key_) {
+ // Found a matching value, we can drop the single delete and the
+ // value. It is safe to drop both records since we've already
+ // outputted a key in this snapshot, or there is no earlier
+ // snapshot (Rule 2 above).
+
+ // Note: it doesn't matter whether the second key is a Put or if it
+ // is an unexpected Merge or Delete. We will compact it out
+ // either way. We will maintain counts of how many mismatches
+ // happened
+ if (next_ikey.type != kTypeValue) {
+ ++iter_stats_.num_single_del_mismatch;
+ }
+
+ ++iter_stats_.num_record_drop_hidden;
+ ++iter_stats_.num_record_drop_obsolete;
+ // Already called input_->Next() once. Call it a second time to
+ // skip past the second key.
+ input_->Next();
+ } else {
+ // Found a matching value, but we cannot drop both keys since
+ // there is an earlier snapshot and we need to leave behind a record
+ // to know that a write happened in this snapshot (Rule 2 above).
+ // Clear the value and output the SingleDelete. (The value will be
+ // outputted on the next iteration.)
+
+ // Setting valid_ to true will output the current SingleDelete
+ valid_ = true;
+
+ // Set up the Put to be outputted in the next iteration.
+ // (Optimization 3).
+ clear_and_output_next_key_ = true;
+ }
+ } else {
+ // We hit the next snapshot without hitting a put, so the iterator
+ // returns the single delete.
+ valid_ = true;
+ }
+ } else {
+ // We are at the end of the input, could not parse the next key, or hit
+ // a different key. The iterator returns the single delete if the key
+ // possibly exists beyond the current output level. We set
+ // has_current_user_key to false so that if the iterator is at the next
+ // key, we do not compare it again against the previous key at the next
+ // iteration. If the next key is corrupt, we return before the
+ // comparison, so the value of has_current_user_key does not matter.
+ has_current_user_key_ = false;
+ if (compaction_ != nullptr && ikey_.sequence <= earliest_snapshot_ &&
+ compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
+ &level_ptrs_)) {
+ // Key doesn't exist outside of this range.
+ // Can compact out this SingleDelete.
+ ++iter_stats_.num_record_drop_obsolete;
+ ++iter_stats_.num_single_del_fallthru;
+ } else {
+ // Output SingleDelete
+ valid_ = true;
+ }
+ }
+
+ if (valid_) {
+ at_next_ = true;
+ }
+ } else if (last_snapshot == current_user_key_snapshot_) {
+ // If the earliest snapshot is which this key is visible in
+ // is the same as the visibility of a previous instance of the
+ // same key, then this kv is not visible in any snapshot.
+ // Hidden by an newer entry for same user key
+ // TODO(noetzli): why not > ?
+ //
+ // Note: Dropping this key will not affect TransactionDB write-conflict
+ // checking since there has already been a record returned for this key
+ // in this snapshot.
+ assert(last_sequence >= current_user_key_sequence_);
+ ++iter_stats_.num_record_drop_hidden; // (A)
+ input_->Next();
+ } else if (compaction_ != nullptr && ikey_.type == kTypeDeletion &&
+ ikey_.sequence <= earliest_snapshot_ &&
+ compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
+ &level_ptrs_)) {
+ // TODO(noetzli): This is the only place where we use compaction_
+ // (besides the constructor). We should probably get rid of this
+ // dependency and find a way to do similar filtering during flushes.
+ //
+ // For this user key:
+ // (1) there is no data in higher levels
+ // (2) data in lower levels will have larger sequence numbers
+ // (3) data in layers that are being compacted here and have
+ // smaller sequence numbers will be dropped in the next
+ // few iterations of this loop (by rule (A) above).
+ // Therefore this deletion marker is obsolete and can be dropped.
+ //
+ // Note: Dropping this Delete will not affect TransactionDB
+ // write-conflict checking since it is earlier than any snapshot.
+ ++iter_stats_.num_record_drop_obsolete;
+ input_->Next();
+ } else if (ikey_.type == kTypeMerge) {
+ if (!merge_helper_->HasOperator()) {
+ status_ = Status::InvalidArgument(
+ "merge_operator is not properly initialized.");
+ return;
+ }
+
+ pinned_iters_mgr_.StartPinning();
+ // We know the merge type entry is not hidden, otherwise we would
+ // have hit (A)
+ // We encapsulate the merge related state machine in a different
+ // object to minimize change to the existing flow.
+ Status s = merge_helper_->MergeUntil(input_, range_del_agg_,
+ prev_snapshot, bottommost_level_);
+ merge_out_iter_.SeekToFirst();
+
+ if (!s.ok() && !s.IsMergeInProgress()) {
+ status_ = s;
+ return;
+ } else if (merge_out_iter_.Valid()) {
+ // NOTE: key, value, and ikey_ refer to old entries.
+ // These will be correctly set below.
+ key_ = merge_out_iter_.key();
+ value_ = merge_out_iter_.value();
+ bool valid_key __attribute__((__unused__)) =
+ ParseInternalKey(key_, &ikey_);
+ // MergeUntil stops when it encounters a corrupt key and does not
+ // include them in the result, so we expect the keys here to valid.
+ assert(valid_key);
+ // Keep current_key_ in sync.
+ current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
+ key_ = current_key_.GetInternalKey();
+ ikey_.user_key = current_key_.GetUserKey();
+ valid_ = true;
+ } else {
+ // all merge operands were filtered out. reset the user key, since the
+ // batch consumed by the merge operator should not shadow any keys
+ // coming after the merges
+ has_current_user_key_ = false;
+ pinned_iters_mgr_.ReleasePinnedData();
+
+ if (merge_helper_->FilteredUntil(&skip_until)) {
+ need_skip = true;
+ }
+ }
+ } else {
+ // 1. new user key -OR-
+ // 2. different snapshot stripe
+ bool should_delete = range_del_agg_->ShouldDelete(
+ key_, RangeDelAggregator::RangePositioningMode::kForwardTraversal);
+ if (should_delete) {
+ ++iter_stats_.num_record_drop_hidden;
+ ++iter_stats_.num_record_drop_range_del;
+ input_->Next();
+ } else {
+ valid_ = true;
+ }
+ }
+
+ if (need_skip) {
+ input_->Seek(skip_until);
+ }
+ }
+
+ if (!valid_ && IsShuttingDown()) {
+ status_ = Status::ShutdownInProgress();
+ }
+}
+
+void CompactionIterator::PrepareOutput() {
+ // Zeroing out the sequence number leads to better compression.
+ // If this is the bottommost level (no files in lower levels)
+ // and the earliest snapshot is larger than this seqno
+ // and the userkey differs from the last userkey in compaction
+ // then we can squash the seqno to zero.
+
+ // This is safe for TransactionDB write-conflict checking since transactions
+ // only care about sequence number larger than any active snapshots.
+ if ((compaction_ != nullptr && !compaction_->allow_ingest_behind()) &&
+ bottommost_level_ && valid_ && ikey_.sequence <= earliest_snapshot_ &&
+ ikey_.type != kTypeMerge &&
+ !cmp_->Equal(compaction_->GetLargestUserKey(), ikey_.user_key)) {
+ assert(ikey_.type != kTypeDeletion && ikey_.type != kTypeSingleDeletion);
+ ikey_.sequence = 0;
+ current_key_.UpdateInternalKey(0, ikey_.type);
+ }
+}
+
+inline SequenceNumber CompactionIterator::findEarliestVisibleSnapshot(
+ SequenceNumber in, SequenceNumber* prev_snapshot) {
+ assert(snapshots_->size());
+ SequenceNumber prev __attribute__((__unused__)) = kMaxSequenceNumber;
+ for (const auto cur : *snapshots_) {
+ assert(prev == kMaxSequenceNumber || prev <= cur);
+ if (cur >= in) {
+ *prev_snapshot = prev == kMaxSequenceNumber ? 0 : prev;
+ return cur;
+ }
+ prev = cur;
+ assert(prev < kMaxSequenceNumber);
+ }
+ *prev_snapshot = prev;
+ return kMaxSequenceNumber;
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compaction_iterator.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compaction_iterator.h b/thirdparty/rocksdb/db/compaction_iterator.h
new file mode 100644
index 0000000..cad2386
--- /dev/null
+++ b/thirdparty/rocksdb/db/compaction_iterator.h
@@ -0,0 +1,197 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+#pragma once
+
+#include <algorithm>
+#include <deque>
+#include <string>
+#include <vector>
+
+#include "db/compaction.h"
+#include "db/compaction_iteration_stats.h"
+#include "db/merge_helper.h"
+#include "db/pinned_iterators_manager.h"
+#include "db/range_del_aggregator.h"
+#include "options/cf_options.h"
+#include "rocksdb/compaction_filter.h"
+
+namespace rocksdb {
+
+class CompactionEventListener;
+
+class CompactionIterator {
+ public:
+ // A wrapper around Compaction. Has a much smaller interface, only what
+ // CompactionIterator uses. Tests can override it.
+ class CompactionProxy {
+ public:
+ explicit CompactionProxy(const Compaction* compaction)
+ : compaction_(compaction) {}
+
+ virtual ~CompactionProxy() = default;
+ virtual int level(size_t compaction_input_level = 0) const {
+ return compaction_->level();
+ }
+ virtual bool KeyNotExistsBeyondOutputLevel(
+ const Slice& user_key, std::vector<size_t>* level_ptrs) const {
+ return compaction_->KeyNotExistsBeyondOutputLevel(user_key, level_ptrs);
+ }
+ virtual bool bottommost_level() const {
+ return compaction_->bottommost_level();
+ }
+ virtual int number_levels() const { return compaction_->number_levels(); }
+ virtual Slice GetLargestUserKey() const {
+ return compaction_->GetLargestUserKey();
+ }
+ virtual bool allow_ingest_behind() const {
+ return compaction_->immutable_cf_options()->allow_ingest_behind;
+ }
+
+ protected:
+ CompactionProxy() = default;
+
+ private:
+ const Compaction* compaction_;
+ };
+
+ CompactionIterator(InternalIterator* input, const Comparator* cmp,
+ MergeHelper* merge_helper, SequenceNumber last_sequence,
+ std::vector<SequenceNumber>* snapshots,
+ SequenceNumber earliest_write_conflict_snapshot, Env* env,
+ bool expect_valid_internal_key,
+ RangeDelAggregator* range_del_agg,
+ const Compaction* compaction = nullptr,
+ const CompactionFilter* compaction_filter = nullptr,
+ CompactionEventListener* compaction_listener = nullptr,
+ const std::atomic<bool>* shutting_down = nullptr);
+
+ // Constructor with custom CompactionProxy, used for tests.
+ CompactionIterator(InternalIterator* input, const Comparator* cmp,
+ MergeHelper* merge_helper, SequenceNumber last_sequence,
+ std::vector<SequenceNumber>* snapshots,
+ SequenceNumber earliest_write_conflict_snapshot, Env* env,
+ bool expect_valid_internal_key,
+ RangeDelAggregator* range_del_agg,
+ std::unique_ptr<CompactionProxy> compaction,
+ const CompactionFilter* compaction_filter = nullptr,
+ CompactionEventListener* compaction_listener = nullptr,
+ const std::atomic<bool>* shutting_down = nullptr);
+
+ ~CompactionIterator();
+
+ void ResetRecordCounts();
+
+ // Seek to the beginning of the compaction iterator output.
+ //
+ // REQUIRED: Call only once.
+ void SeekToFirst();
+
+ // Produces the next record in the compaction.
+ //
+ // REQUIRED: SeekToFirst() has been called.
+ void Next();
+
+ // Getters
+ const Slice& key() const { return key_; }
+ const Slice& value() const { return value_; }
+ const Status& status() const { return status_; }
+ const ParsedInternalKey& ikey() const { return ikey_; }
+ bool Valid() const { return valid_; }
+ const Slice& user_key() const { return current_user_key_; }
+ const CompactionIterationStats& iter_stats() const { return iter_stats_; }
+
+ private:
+ // Processes the input stream to find the next output
+ void NextFromInput();
+
+ // Do last preparations before presenting the output to the callee. At this
+ // point this only zeroes out the sequence number if possible for better
+ // compression.
+ void PrepareOutput();
+
+ // Given a sequence number, return the sequence number of the
+ // earliest snapshot that this sequence number is visible in.
+ // The snapshots themselves are arranged in ascending order of
+ // sequence numbers.
+ // Employ a sequential search because the total number of
+ // snapshots are typically small.
+ inline SequenceNumber findEarliestVisibleSnapshot(
+ SequenceNumber in, SequenceNumber* prev_snapshot);
+
+ InternalIterator* input_;
+ const Comparator* cmp_;
+ MergeHelper* merge_helper_;
+ const std::vector<SequenceNumber>* snapshots_;
+ const SequenceNumber earliest_write_conflict_snapshot_;
+ Env* env_;
+ bool expect_valid_internal_key_;
+ RangeDelAggregator* range_del_agg_;
+ std::unique_ptr<CompactionProxy> compaction_;
+ const CompactionFilter* compaction_filter_;
+#ifndef ROCKSDB_LITE
+ CompactionEventListener* compaction_listener_;
+#endif // ROCKSDB_LITE
+ const std::atomic<bool>* shutting_down_;
+ bool bottommost_level_;
+ bool valid_ = false;
+ bool visible_at_tip_;
+ SequenceNumber earliest_snapshot_;
+ SequenceNumber latest_snapshot_;
+ bool ignore_snapshots_;
+
+ // State
+ //
+ // Points to a copy of the current compaction iterator output (current_key_)
+ // if valid_.
+ Slice key_;
+ // Points to the value in the underlying iterator that corresponds to the
+ // current output.
+ Slice value_;
+ // The status is OK unless compaction iterator encounters a merge operand
+ // while not having a merge operator defined.
+ Status status_;
+ // Stores the user key, sequence number and type of the current compaction
+ // iterator output (or current key in the underlying iterator during
+ // NextFromInput()).
+ ParsedInternalKey ikey_;
+ // Stores whether ikey_.user_key is valid. If set to false, the user key is
+ // not compared against the current key in the underlying iterator.
+ bool has_current_user_key_ = false;
+ bool at_next_ = false; // If false, the iterator
+ // Holds a copy of the current compaction iterator output (or current key in
+ // the underlying iterator during NextFromInput()).
+ IterKey current_key_;
+ Slice current_user_key_;
+ SequenceNumber current_user_key_sequence_;
+ SequenceNumber current_user_key_snapshot_;
+
+ // True if the iterator has already returned a record for the current key.
+ bool has_outputted_key_ = false;
+
+ // truncated the value of the next key and output it without applying any
+ // compaction rules. This is used for outputting a put after a single delete.
+ bool clear_and_output_next_key_ = false;
+
+ MergeOutputIterator merge_out_iter_;
+ // PinnedIteratorsManager used to pin input_ Iterator blocks while reading
+ // merge operands and then releasing them after consuming them.
+ PinnedIteratorsManager pinned_iters_mgr_;
+ std::string compaction_filter_value_;
+ InternalKey compaction_filter_skip_until_;
+ // "level_ptrs" holds indices that remember which file of an associated
+ // level we were last checking during the last call to compaction->
+ // KeyNotExistsBeyondOutputLevel(). This allows future calls to the function
+ // to pick off where it left off since each subcompaction's key range is
+ // increasing so a later call to the function must be looking for a key that
+ // is in or beyond the last file checked during the previous call
+ std::vector<size_t> level_ptrs_;
+ CompactionIterationStats iter_stats_;
+
+ bool IsShuttingDown() {
+ // This is a best-effort facility, so memory_order_relaxed is sufficient.
+ return shutting_down_ && shutting_down_->load(std::memory_order_relaxed);
+ }
+};
+} // namespace rocksdb
[05/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_docs/faq.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_docs/faq.md b/thirdparty/rocksdb/docs/_docs/faq.md
new file mode 100644
index 0000000..0887a09
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_docs/faq.md
@@ -0,0 +1,48 @@
+---
+docid: support-faq
+title: FAQ
+layout: docs
+permalink: /docs/support/faq.html
+---
+
+Here is an ever-growing list of frequently asked questions around RocksDB
+
+## What is RocksDB?
+
+RocksDB is an embeddable persistent key-value store for fast storage. RocksDB can also be the foundation for a client-server database but our current focus is on embedded workloads.
+
+RocksDB builds on [LevelDB](https://code.google.com/p/leveldb/) to be scalable to run on servers with many CPU cores, to efficiently use fast storage, to support IO-bound, in-memory and write-once workloads, and to be flexible to allow for innovation.
+
+For the latest details, watch [Mark Callaghan’s and Igor Canadi’s talk at CMU on 10/2015](https://scs.hosted.panopto.com/Panopto/Pages/Viewer.aspx?id=f4e0eb37-ae18-468f-9248-cb73edad3e56). [Dhruba Borthakur’s introductory talk](https://github.com/facebook/rocksdb/blob/gh-pages-old/intro.pdf?raw=true) from the Data @ Scale 2013 conference provides some perspective about how RocksDB has evolved.
+
+## How does performance compare?
+
+We benchmarked LevelDB and found that it was unsuitable for our server workloads. The [benchmark results](http://leveldb.googlecode.com/svn/trunk/doc/benchmark.html) look awesome at first sight, but we quickly realized that those results were for a database whose size was smaller than the size of RAM on the test machine – where the entire database could fit in the OS page cache. When we performed the same benchmarks on a database that was at least 5 times larger than main memory, the performance results were dismal.
+
+By contrast, we’ve published the [RocksDB benchmark results](https://github.com/facebook/rocksdb/wiki/Performance-Benchmarks) for server side workloads on Flash. We also measured the performance of LevelDB on these server-workload benchmarks and found that RocksDB solidly outperforms LevelDB for these IO bound workloads. We found that LevelDB’s single-threaded compaction process was insufficient to drive server workloads. We saw frequent write-stalls with LevelDB that caused 99-percentile latency to be tremendously large. We found that mmap-ing a file into the OS cache introduced performance bottlenecks for reads. We could not make LevelDB consume all the IOs offered by the underlying Flash storage.
+
+## What is RocksDB suitable for?
+
+RocksDB can be used by applications that need low latency database accesses. Possibilities include:
+
+* A user-facing application that stores the viewing history and state of users of a website.
+* A spam detection application that needs fast access to big data sets.
+* A graph-search query that needs to scan a data set in realtime.
+* A cache data from Hadoop, thereby allowing applications to query Hadoop data in realtime.
+* A message-queue that supports a high number of inserts and deletes.
+
+## How big is RocksDB adoption?
+
+RocksDB is an embedded storage engine that is used in a number of backend systems at Facebook. In the Facebook newsfeed’s backend, it replaced another internal storage engine called Centrifuge and is one of the many components used. ZippyDB, a distributed key value store service used by Facebook products relies RocksDB. Details on ZippyDB are in [Muthu Annamalai’s talk at Data@Scale in Seattle](https://youtu.be/DfiN7pG0D0k). Dragon, a distributed graph query engine part of the social graph infrastructure, is using RocksDB to store data. Parse has been running [MongoDB on RocksDB in production](http://blog.parse.com/announcements/mongodb-rocksdb-parse/) since early 2015.
+
+RocksDB is proving to be a useful component for a lot of other groups in the industry. For a list of projects currently using RocksDB, take a look at our USERS.md list on github.
+
+## How good is RocksDB as a database storage engine?
+
+Our engineering team at Facebook firmly believes that RocksDB has great potential as storage engine for databases. It has been proven in production with MongoDB: [MongoRocks](https://github.com/mongodb-partners/mongo-rocks) is the RocksDB based storage engine for MongoDB.
+
+[MyRocks](https://code.facebook.com/posts/190251048047090/myrocks-a-space-and-write-optimized-mysql-database/) is the RocksDB based storage engine for MySQL. Using RocksDB we have managed to achieve 2x better compression and 10x less write amplification for our benchmarks compared to our existing MySQL setup. Given our current results, work is currently underway to develop MyRocks into a production ready solution for web-scale MySQL workloads. Follow along on [GitHub](https://github.com/facebook/mysql-5.6)!
+
+## Why is RocksDB open sourced?
+
+We are open sourcing this project on [GitHub](http://github.com/facebook/rocksdb) because we think it will be useful beyond Facebook. We are hoping that software programmers and database developers will use, enhance, and customize RocksDB for their use-cases. We would also like to engage with the academic community on topics related to efficiency for modern database algorithms.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_docs/getting-started.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_docs/getting-started.md b/thirdparty/rocksdb/docs/_docs/getting-started.md
new file mode 100644
index 0000000..8b01dfe
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_docs/getting-started.md
@@ -0,0 +1,78 @@
+---
+docid: getting-started
+title: Getting started
+layout: docs
+permalink: /docs/getting-started.html
+---
+
+## Overview
+
+The RocksDB library provides a persistent key value store. Keys and values are arbitrary byte arrays. The keys are ordered within the key value store according to a user-specified comparator function.
+
+The library is maintained by the Facebook Database Engineering Team, and is based on [LevelDB](https://github.com/google/leveldb), by Sanjay Ghemawat and Jeff Dean at Google.
+
+This overview gives some simple examples of how RocksDB is used. For the story of why RocksDB was created in the first place, see [Dhruba Borthakur’s introductory talk](https://github.com/facebook/rocksdb/blob/gh-pages-old/intro.pdf?raw=true) from the Data @ Scale 2013 conference.
+
+## Opening A Database
+
+A rocksdb database has a name which corresponds to a file system directory. All of the contents of database are stored in this directory. The following example shows how to open a database, creating it if necessary:
+
+```c++
+#include <assert>
+#include "rocksdb/db.h"
+
+rocksdb::DB* db;
+rocksdb::Options options;
+options.create_if_missing = true;
+rocksdb::Status status =
+ rocksdb::DB::Open(options, "/tmp/testdb", &db);
+assert(status.ok());
+...
+```
+
+If you want to raise an error if the database already exists, add the following line before the rocksdb::DB::Open call:
+
+```c++
+options.error_if_exists = true;
+```
+
+## Status
+
+You may have noticed the `rocksdb::Status` type above. Values of this type are returned by most functions in RocksDB that may encounter
+an error. You can check if such a result is ok, and also print an associated error message:
+
+```c++
+rocksdb::Status s = ...;
+if (!s.ok()) cerr << s.ToString() << endl;
+```
+
+## Closing A Database
+
+When you are done with a database, just delete the database object. For example:
+
+```c++
+/* open the db as described above */
+/* do something with db */
+delete db;
+```
+
+## Reads And Writes
+
+The database provides Put, Delete, and Get methods to modify/query the database. For example, the following code moves the value stored under `key1` to `key2`.
+
+```c++
+std::string value;
+rocksdb::Status s = db->Get(rocksdb::ReadOptions(), key1, &value);
+if (s.ok()) s = db->Put(rocksdb::WriteOptions(), key2, value);
+if (s.ok()) s = db->Delete(rocksdb::WriteOptions(), key1);
+```
+
+## Further documentation
+
+These are just simple examples of how RocksDB is used. The full documentation is currently on the [GitHub wiki](https://github.com/facebook/rocksdb/wiki).
+
+Here are some specific details about the RocksDB implementation:
+
+- [Architecture Guide](https://github.com/facebook/rocksdb/wiki/Rocksdb-Architecture-Guide)
+- [Format of an immutable Table file](https://github.com/facebook/rocksdb/wiki/Rocksdb-Table-Format)
+- [Format of a log file](https://github.com/facebook/rocksdb/wiki/Write-Ahead-Log-File-Format)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/blog_pagination.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/blog_pagination.html b/thirdparty/rocksdb/docs/_includes/blog_pagination.html
new file mode 100644
index 0000000..6a1f334
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/blog_pagination.html
@@ -0,0 +1,28 @@
+<!-- Pagination links - copied from http://jekyllrb.com/docs/pagination/ -->
+{% if paginator.total_pages > 1 %}
+<br />
+<div class="pagination">
+ {% if paginator.previous_page %}
+ <a href="{{ paginator.previous_page_path | replace: '//', '/' }}">« Prev</a>
+ {% else %}
+ <span>« Prev</span>
+ {% endif %}
+
+ {% for page in (1..paginator.total_pages) %}
+ {% if page == paginator.page %}
+ <em>{{ page }}</em>
+ {% elsif page == 1 %}
+ <a href="{{ '/blog' }}">{{ page }}</a>
+ {% else %}
+ <a href="{{ site.paginate_path | replace: '//', '/' | replace: ':num', page }}">{{ page }}</a>
+ {% endif %}
+ {% endfor %}
+
+ {% if paginator.next_page %}
+ <a href="{{ paginator.next_page_path | replace: '//', '/' }}">Next »</a>
+ {% else %}
+ <span>Next »</span>
+ {% endif %}
+</div>
+<br />
+{% endif %}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/content/gridblocks.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/content/gridblocks.html b/thirdparty/rocksdb/docs/_includes/content/gridblocks.html
new file mode 100644
index 0000000..49c5e59
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/content/gridblocks.html
@@ -0,0 +1,5 @@
+<div class="gridBlock">
+{% for item in {{include.data_source}} %}
+ {% include content/items/gridblock.html item=item layout=include.layout imagealign=include.imagealign align=include.align %}
+{% endfor %}
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/content/items/gridblock.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/content/items/gridblock.html b/thirdparty/rocksdb/docs/_includes/content/items/gridblock.html
new file mode 100644
index 0000000..58c9e7f
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/content/items/gridblock.html
@@ -0,0 +1,37 @@
+{% if include.layout == "fourColumn" %}
+ {% assign layout = "fourByGridBlock" %}
+{% else %}
+ {% assign layout = "twoByGridBlock" %}
+{% endif %}
+
+{% if include.imagealign == "side" %}
+ {% assign imagealign = "imageAlignSide" %}
+{% else %}
+ {% if item.image %}
+ {% assign imagealign = "imageAlignTop" %}
+ {% else %}
+ {% assign imagealign = "" %}
+ {% endif %}
+{% endif %}
+
+{% if include.align == "right" %}
+ {% assign align = "alignRight" %}
+{% elsif include.align == "center" %}
+ {% assign align = "alignCenter" %}
+{% else %}
+ {% assign align = "alignLeft" %}
+{% endif %}
+
+<div class="blockElement {{ layout }} {{ imagealign }} {{ align }}">
+ {% if item.image %}
+ <div class="blockImage">
+ <img src="/static/{{ item.image }}" alt="{{ item.title }}" title="{{ item.title }}" />
+ </div>
+ {% endif %}
+ <div class="blockContent">
+ <h3>{{ item.title }}</h3>
+ {% if item.text %}
+ {{ item.text | markdownify }}
+ {% endif %}
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/doc.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/doc.html b/thirdparty/rocksdb/docs/_includes/doc.html
new file mode 100644
index 0000000..a795000
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/doc.html
@@ -0,0 +1,25 @@
+<div class="post">
+ <header class="post-header">
+ <h1 class="post-title">{% if include.truncate %}<a href="{{ page.url | absolute_url }}">{{ page.title }}</a>{% else %}{{ page.title }}{% endif %}</h1>
+ </header>
+
+ <article class="post-content">
+ {% if include.truncate %}
+ {% if page.content contains '<!--truncate-->' %}
+ {{ page.content | split:'<!--truncate-->' | first }}
+ <div class="read-more">
+ <a href="{{ page.url | absolute_url }}" >
+ ...Read More
+ </a>
+ </div>
+ {% else %}
+ {{ page.content }}
+ {% endif %}
+ {% else %}
+ {{ content }}
+
+ <p><a class="edit-page-link" href="https://github.com/{{ site.ghrepo }}/blob/master/docs/{{ page.path }}" target="_blank">Edit on GitHub</a></p>
+ {% endif %}
+ </article>
+ {% include doc_paging.html %}
+</div>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/doc_paging.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/doc_paging.html b/thirdparty/rocksdb/docs/_includes/doc_paging.html
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/footer.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/footer.html b/thirdparty/rocksdb/docs/_includes/footer.html
new file mode 100644
index 0000000..dd9494a
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/footer.html
@@ -0,0 +1,33 @@
+<div class="footerContainer">
+ <div id="footer_wrap" class="wrapper footerWrapper">
+ <div class="footerBlocks">
+ <div id="fb_oss" class="footerSection fbOpenSourceFooter">
+ <svg class="facebookOSSLogoSvg" viewBox="0 0 1133.9 1133.9" x="0px" y="0px">
+ <g>
+ <path class="logoRing outerRing" d="M 498.3 3.7 c 153.6 88.9 307.3 177.7 461.1 266.2 c 7.6 4.4 10.3 9.1 10.3 17.8 c -0.3 179.1 -0.2 358.3 0 537.4 c 0 8.1 -2.4 12.8 -9.7 17.1 c -154.5 88.9 -308.8 178.1 -462.9 267.5 c -9 5.2 -15.5 5.3 -24.6 0.1 c -153.9 -89.2 -307.9 -178 -462.1 -266.8 C 3 838.8 0 833.9 0 825.1 c 0.3 -179.1 0.2 -358.3 0 -537.4 c 0 -8.6 2.6 -13.6 10.2 -18 C 164.4 180.9 318.4 92 472.4 3 C 477 -1.5 494.3 -0.7 498.3 3.7 Z M 48.8 555.3 c 0 79.9 0.2 159.9 -0.2 239.8 c -0.1 10 3 15.6 11.7 20.6 c 137.2 78.8 274.2 157.8 411 237.3 c 9.9 5.7 17 5.7 26.8 0.1 c 137.5 -79.8 275.2 -159.2 412.9 -238.5 c 7.4 -4.3 10.5 -8.9 10.5 -17.8 c -0.3 -160.2 -0.3 -320.5 0 -480.7 c 0 -8.8 -2.8 -13.6 -10.3 -18 C 772.1 218 633.1 137.8 494.2 57.4 c -6.5 -3.8 -11.5 -4.5 -18.5 -0.5 C 336.8 137.4 197.9 217.7 58.8 297.7 c -7.7 4.4 -10.2 9.2 -10.2 17.9 C 48.9 395.5 48.8 475.4 48.8 555.3 Z" />
+ <path class="logoRing middleRing" d="M 184.4 555.9 c 0 -33.3 -1 -66.7 0.3 -100 c 1.9 -48 24.1 -86 64.7 -110.9 c 54.8 -33.6 110.7 -65.5 167 -96.6 c 45.7 -25.2 92.9 -24.7 138.6 1 c 54.4 30.6 108.7 61.5 162.2 93.7 c 44 26.5 67.3 66.8 68 118.4 c 0.9 63.2 0.9 126.5 0 189.7 c -0.7 50.6 -23.4 90.7 -66.6 116.9 c -55 33.4 -110.8 65.4 -167.1 96.5 c -43.4 24 -89 24.2 -132.3 0.5 c -57.5 -31.3 -114.2 -64 -170 -98.3 c -41 -25.1 -62.9 -63.7 -64.5 -112.2 C 183.5 621.9 184.3 588.9 184.4 555.9 Z M 232.9 556.3 c 0 29.5 0.5 59.1 -0.1 88.6 c -0.8 39.2 16.9 67.1 50.2 86.2 c 51.2 29.4 102.2 59.2 153.4 88.4 c 31.4 17.9 63.6 18.3 95 0.6 c 53.7 -30.3 107.1 -61.2 160.3 -92.5 c 29.7 -17.5 45 -44.5 45.3 -78.8 c 0.6 -61.7 0.5 -123.5 0 -185.2 c -0.3 -34.4 -15.3 -61.5 -44.9 -79 C 637.7 352.6 583 320.8 527.9 290 c -27.5 -15.4 -57.2 -16.1 -84.7 -0.7 c -56.9 31.6 -113.4 64 -169.1 97.6 c -26.4 15.9 -40.7 41.3 -41.1 72.9 C 232.6 491.9 232.9 524.1 232.9 556.3 Z" />
+ <path class="logoRing innerRing" d="M 484.9 424.4 c 69.8 -2.8 133.2 57.8 132.6 132 C 617 630 558.5 688.7 484.9 689.1 c -75.1 0.4 -132.6 -63.6 -132.7 -132.7 C 352.1 485 413.4 421.5 484.9 424.4 Z M 401.3 556.7 c -3.4 37.2 30.5 83.6 83 84.1 c 46.6 0.4 84.8 -37.6 84.9 -84 c 0.1 -46.6 -37.2 -84.4 -84.2 -84.6 C 432.2 472.1 397.9 518.3 401.3 556.7 Z" />
+ </g>
+ </svg>
+ <h2>Facebook Open Source</h2>
+ </div>
+ <div class="footerSection">
+ <a class="footerLink" href="https://code.facebook.com/projects/" target="_blank">Open Source Projects</a>
+ <a class="footerLink" href="https://github.com/facebook/" target="_blank">GitHub</a>
+ <a class="footerLink" href="https://twitter.com/fbOpenSource" target="_blank">Twitter</a>
+ </div>
+ <div class="footerSection rightAlign">
+ <a class="footerLink" href="https://github.com/{{ site.ghrepo }}" target="_blank">Contribute to this project on GitHub</a>
+ </div>
+ </div>
+ </div>
+</div>
+<script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', '{{ site.gacode }}', 'auto');
+ ga('send', 'pageview');
+</script>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/head.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/head.html b/thirdparty/rocksdb/docs/_includes/head.html
new file mode 100644
index 0000000..10845ec
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/head.html
@@ -0,0 +1,23 @@
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+
+ <meta property="og:url" content="{{ page.url | replace:'index.html','' | absolute_url }}" />
+ <meta property="og:site_name" content="{{ site.title }}"/>
+ <meta property="og:title" content="{% if page.title %}{{ page.title }}{% else %}{{ site.title }}{% endif %}" />
+ <meta property="og:image" content="{{ '/static/og_image.png' | absolute_url }}" />
+ <meta property="og:description" content="{% if page.excerpt %}{{ page.excerpt | strip_html | strip_newlines | truncate: 160 }}{% else %}{{ site.description }}{% endif %}" />
+
+ <link rel="stylesheet" href="{{ '/css/main.css' }}" media="screen">
+ <link rel="icon" href="{{ '/static/favicon.png' }}" type="image/x-icon">
+ {% if site.searchconfig %}
+ <link rel="stylesheet" href="https://cdn.jsdelivr.net/docsearch.js/1/docsearch.min.css" />
+ {% endif %}
+
+ <title>{% if page.title %}{{ page.title }} | {{ site.title }}{% else %}{{ site.title }}{% endif %}</title>
+ <meta name="description" content="{% if page.excerpt %}{{ page.excerpt | strip_html | strip_newlines | truncate: 160 }}{% else %}{{ site.description }}{% endif %}">
+
+ <link rel="canonical" href="{{ page.url | replace:'index.html','' | absolute_url }}">
+ <link rel="alternate" type="application/rss+xml" title="{{ site.title }}" href="{{ '/feed.xml' | absolute_url }}" />
+</head>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/header.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/header.html b/thirdparty/rocksdb/docs/_includes/header.html
new file mode 100644
index 0000000..8108d22
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/header.html
@@ -0,0 +1,19 @@
+<div class="headerContainer">
+ <div id="header_wrap" class="wrapper headerWrapper">
+ <div class="inner">
+ <img class="projectLogo" height="200px" src="{{ '/static/logo.svg' }}" alt="{{ site.title }}" title="{{ site.title }}" />
+ <h1 id="project_title">{{ site.title }}</h1>
+ <h2 id="project_tagline" class="fbossFontLight">{{ site.tagline }}</h2>
+
+ <section id="intro">
+ <p>{% if page.excerpt %}{{ page.excerpt | strip_html }}{% else %}{{ site.description }}{% endif %}</p>
+ </section>
+ <div id="promo" class="section promoSection">
+ {% for promo in site.data.promo %}
+ {% include plugins/{{promo.type}}.html button_href=promo.href button_text=promo.text %}
+ <div class="gridClear"></div>
+ {% endfor %}
+ </div>
+ </div>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/hero.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/hero.html b/thirdparty/rocksdb/docs/_includes/hero.html
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/home_header.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/home_header.html b/thirdparty/rocksdb/docs/_includes/home_header.html
new file mode 100644
index 0000000..90880d1
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/home_header.html
@@ -0,0 +1,22 @@
+<div class="homeContainer">
+ <div class="homeSplashFade">
+ <div id="home_wrap" class="wrapper homeWrapper">
+ <div id="inner">
+ <h2 id="project_tagline">{{ site.tagline }}</h2>
+ <section id="intro">
+ <p>{% if page.excerpt %}{{ page.excerpt | strip_html }}{% else %}{{ site.description }}{% endif %}</p>
+ </section>
+ <div id="promo" class="section promoSection">
+ {% for promo in site.data.promo %}
+ <div class="promoRow">
+ {% include plugins/{{promo.type}}.html href=promo.href text=promo.text children=promo.children %}
+ </div>
+ {% endfor %}
+ </div>
+ </div>
+ <div class="projectLogo">
+ <img src="{{ '/static/logo.svg' }}" alt="{{ site.title }}">
+ </div>
+ </div>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/katex_import.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/katex_import.html b/thirdparty/rocksdb/docs/_includes/katex_import.html
new file mode 100644
index 0000000..6d6b7cf
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/katex_import.html
@@ -0,0 +1,3 @@
+<script src="//code.jquery.com/jquery-1.11.1.min.js"></script>
+<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/KaTeX/0.2.0/katex.min.css">
+<script src="//cdnjs.cloudflare.com/ajax/libs/KaTeX/0.2.0/katex.min.js"></script>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/katex_render.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/katex_render.html b/thirdparty/rocksdb/docs/_includes/katex_render.html
new file mode 100644
index 0000000..56e2e89
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/katex_render.html
@@ -0,0 +1,210 @@
+<script type="text/javascript">
+/* global katex */
+
+var findEndOfMath = function(delimiter, text, startIndex) {
+ // Adapted from
+ // https://github.com/Khan/perseus/blob/master/src/perseus-markdown.jsx
+ var index = startIndex;
+ var braceLevel = 0;
+
+ var delimLength = delimiter.length;
+
+ while (index < text.length) {
+ var character = text[index];
+
+ if (braceLevel <= 0 &&
+ text.slice(index, index + delimLength) === delimiter) {
+ return index;
+ } else if (character === "\\") {
+ index++;
+ } else if (character === "{") {
+ braceLevel++;
+ } else if (character === "}") {
+ braceLevel--;
+ }
+
+ index++;
+ }
+
+ return -1;
+};
+
+var splitAtDelimiters = function(startData, leftDelim, rightDelim, display) {
+ var finalData = [];
+
+ for (var i = 0; i < startData.length; i++) {
+ if (startData[i].type === "text") {
+ var text = startData[i].data;
+
+ var lookingForLeft = true;
+ var currIndex = 0;
+ var nextIndex;
+
+ nextIndex = text.indexOf(leftDelim);
+ if (nextIndex !== -1) {
+ currIndex = nextIndex;
+ finalData.push({
+ type: "text",
+ data: text.slice(0, currIndex)
+ });
+ lookingForLeft = false;
+ }
+
+ while (true) {
+ if (lookingForLeft) {
+ nextIndex = text.indexOf(leftDelim, currIndex);
+ if (nextIndex === -1) {
+ break;
+ }
+
+ finalData.push({
+ type: "text",
+ data: text.slice(currIndex, nextIndex)
+ });
+
+ currIndex = nextIndex;
+ } else {
+ nextIndex = findEndOfMath(
+ rightDelim,
+ text,
+ currIndex + leftDelim.length);
+ if (nextIndex === -1) {
+ break;
+ }
+
+ finalData.push({
+ type: "math",
+ data: text.slice(
+ currIndex + leftDelim.length,
+ nextIndex),
+ rawData: text.slice(
+ currIndex,
+ nextIndex + rightDelim.length),
+ display: display
+ });
+
+ currIndex = nextIndex + rightDelim.length;
+ }
+
+ lookingForLeft = !lookingForLeft;
+ }
+
+ finalData.push({
+ type: "text",
+ data: text.slice(currIndex)
+ });
+ } else {
+ finalData.push(startData[i]);
+ }
+ }
+
+ return finalData;
+};
+
+var splitWithDelimiters = function(text, delimiters) {
+ var data = [{type: "text", data: text}];
+ for (var i = 0; i < delimiters.length; i++) {
+ var delimiter = delimiters[i];
+ data = splitAtDelimiters(
+ data, delimiter.left, delimiter.right,
+ delimiter.display || false);
+ }
+ return data;
+};
+
+var renderMathInText = function(text, delimiters) {
+ var data = splitWithDelimiters(text, delimiters);
+
+ var fragment = document.createDocumentFragment();
+
+ for (var i = 0; i < data.length; i++) {
+ if (data[i].type === "text") {
+ fragment.appendChild(document.createTextNode(data[i].data));
+ } else {
+ var span = document.createElement("span");
+ var math = data[i].data;
+ try {
+ katex.render(math, span, {
+ displayMode: data[i].display
+ });
+ } catch (e) {
+ if (!(e instanceof katex.ParseError)) {
+ throw e;
+ }
+ console.error(
+ "KaTeX auto-render: Failed to parse `" + data[i].data +
+ "` with ",
+ e
+ );
+ fragment.appendChild(document.createTextNode(data[i].rawData));
+ continue;
+ }
+ fragment.appendChild(span);
+ }
+ }
+
+ return fragment;
+};
+
+var renderElem = function(elem, delimiters, ignoredTags) {
+ for (var i = 0; i < elem.childNodes.length; i++) {
+ var childNode = elem.childNodes[i];
+ if (childNode.nodeType === 3) {
+ // Text node
+ var frag = renderMathInText(childNode.textContent, delimiters);
+ i += frag.childNodes.length - 1;
+ elem.replaceChild(frag, childNode);
+ } else if (childNode.nodeType === 1) {
+ // Element node
+ var shouldRender = ignoredTags.indexOf(
+ childNode.nodeName.toLowerCase()) === -1;
+
+ if (shouldRender) {
+ renderElem(childNode, delimiters, ignoredTags);
+ }
+ }
+ // Otherwise, it's something else, and ignore it.
+ }
+};
+
+var defaultOptions = {
+ delimiters: [
+ {left: "$$", right: "$$", display: true},
+ {left: "\\[", right: "\\]", display: true},
+ {left: "\\(", right: "\\)", display: false}
+ // LaTeX uses this, but it ruins the display of normal `$` in text:
+ // {left: "$", right: "$", display: false}
+ ],
+
+ ignoredTags: [
+ "script", "noscript", "style", "textarea", "pre", "code"
+ ]
+};
+
+var extend = function(obj) {
+ // Adapted from underscore.js' `_.extend`. See LICENSE.txt for license.
+ var source, prop;
+ for (var i = 1, length = arguments.length; i < length; i++) {
+ source = arguments[i];
+ for (prop in source) {
+ if (Object.prototype.hasOwnProperty.call(source, prop)) {
+ obj[prop] = source[prop];
+ }
+ }
+ }
+ return obj;
+};
+
+var renderMathInElement = function(elem, options) {
+ if (!elem) {
+ throw new Error("No element provided to render");
+ }
+
+ options = extend({}, defaultOptions, options);
+
+ renderElem(elem, options.delimiters, options.ignoredTags);
+};
+
+renderMathInElement(document.body);
+
+</script>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/nav.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/nav.html b/thirdparty/rocksdb/docs/_includes/nav.html
new file mode 100644
index 0000000..9c6fed0
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/nav.html
@@ -0,0 +1,37 @@
+<div id="fixed_header" class="fixedHeaderContainer{% if include.alwayson %} visible{% endif %}">
+ <div class="headerWrapper wrapper">
+ <header>
+ <a href="{{ '/' | absolute_url }}">
+ <img src="{{ '/static/logo.svg' }}">
+ <h2>{{ site.title }}</h2>
+ </a>
+
+ <div class="navigationWrapper navigationFull" id="flat_nav">
+ <nav class="navigation">
+ <ul>
+ {% for item in site.data.nav %}
+ <li class="navItem{% if page.collection == item.category or page.category == item.category %} navItemActive{% endif %}">
+ {% if item.category == "external" %}
+ <a href="{{ item.href }}">{{ item.title }}</a>
+ {% else %}
+ {% comment %}
+ I removed `relative_url` from here for now until the problem we are having with
+ GitHub pages is resolved. Yes, I know this is exactly the same as the if above.
+ See: https://github.com/facebook/rocksdb/commit/800e51553ee029f29581f7f338cbc988c7f6da62
+ {% endcomment %}
+ <a href="{{ item.href }}">{{ item.title }}</a>
+ {% endif %}
+ </li>
+ {% endfor %}
+ {% if site.searchconfig %}
+ {% include nav_search.html inputselector="search_input" %}
+ {% endif %}
+ </ul>
+ </nav>
+ </div>
+ <div class="navigationWrapper navigationSlider" id="navigation_wrap">
+ {% include nav/header_nav.html %}
+ </div>
+ </header>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/nav/collection_nav.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/nav/collection_nav.html b/thirdparty/rocksdb/docs/_includes/nav/collection_nav.html
new file mode 100644
index 0000000..a3c7a2d
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/nav/collection_nav.html
@@ -0,0 +1,64 @@
+<div class="docsNavContainer">
+ <nav class="toc" id="doc_nav">
+ <div class="toggleNav" id="collection_nav">
+ <section class="navWrapper wrapper">
+ <div class="navBreadcrumb wrapper">
+ <div class="navToggle" id="collection_nav_toggler">
+ <i></i>
+ </div>
+ <h2>
+ <a href="{{ include.sectionpath }}">{{ include.sectiontitle }}</a>
+ {% if include.currentgroup %}
+ <i>›</i>
+ <span>{{ include.currentgroup }}</span>
+ {% endif %}
+ </h2>
+ </div>
+ <div class="navGroups">
+ {% if include.type == "blog" %}
+ {% assign grouptitle = "All Posts" %}
+ {% assign groupitems = include.navdata %}
+ {% include nav/collection_nav_group.html %}
+ {% else %}
+ {% for group in include.navdata %}
+ {% assign grouptitle = group.title %}
+ {% for item in group.items %}
+ {% if item.id == page.docid %}
+ {% assign currentgroup = group %}
+ {% endif %}
+ {% endfor %}
+ {% include nav/collection_nav_group.html %}
+ {% endfor %}
+ {% endif %}
+ </div>
+ </section>
+ </div>
+ </nav>
+</div>
+<script>
+ var docsevent = document.createEvent('Event');
+ docsevent.initEvent('docs_slide', true, true);
+ document.addEventListener('docs_slide', function (e) {
+ document.body.classList.toggle('docsSliderActive');
+ }, false);
+
+ var collectionNav = document.getElementById('collection_nav');
+ var collectionNavToggler =
+ document.getElementById('collection_nav_toggler');
+ collectionNavToggler.addEventListener('click', function(e) {
+ collectionNav.classList.toggle('toggleNavActive');
+ document.dispatchEvent(docsevent);
+ });
+
+ var groups = document.getElementsByClassName('navGroup');
+ for(var i = 0; i < groups.length; i++) {
+ var thisGroup = groups[i];
+ thisGroup.onclick = function() {
+ for(var j = 0; j < groups.length; j++) {
+ var group = groups[j];
+ group.classList.remove('navGroupActive');
+ }
+ this.classList.add('navGroupActive');
+ }
+ }
+</script>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/nav/collection_nav_group.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/nav/collection_nav_group.html b/thirdparty/rocksdb/docs/_includes/nav/collection_nav_group.html
new file mode 100644
index 0000000..b236ac5
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/nav/collection_nav_group.html
@@ -0,0 +1,19 @@
+<div class="navGroup{% if currentgroup == group %} navGroupActive navGroupCurrent{% endif %}">
+ <h3><i>+</i><span>{{ grouptitle }}</span></h3>
+ <ul>
+ {% if include.data_collection %}
+ {% for item in group.items %}
+ {% for collectionitem in include.data_collection %}
+ {% if collectionitem.docid == item.id %}
+ {% assign groupitem = collectionitem %}
+ {% include nav/collection_nav_group_item.html %}
+ {% endif %}
+ {% endfor %}
+ {% endfor %}
+ {% else %}
+ {% for groupitem in groupitems %}
+ {% include nav/collection_nav_group_item.html %}
+ {% endfor %}
+ {% endif %}
+ </ul>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/nav/collection_nav_group_item.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/nav/collection_nav_group_item.html b/thirdparty/rocksdb/docs/_includes/nav/collection_nav_group_item.html
new file mode 100644
index 0000000..fbb063d
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/nav/collection_nav_group_item.html
@@ -0,0 +1 @@
+<li class="navListItem"><a class="navItem" href="{{ groupitem.url | absolute_url }}">{{ groupitem.title }}</a></li>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/nav/header_nav.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/nav/header_nav.html b/thirdparty/rocksdb/docs/_includes/nav/header_nav.html
new file mode 100644
index 0000000..0fe945c
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/nav/header_nav.html
@@ -0,0 +1,30 @@
+<div id="header_nav">
+ <div class="navSlideout">
+ <i class="menuExpand" id="header_nav_expander"><span></span><span></span><span></span></i>
+ </div>
+ <nav class="slidingNav">
+ <ul>
+ {% for item in site.data.nav %}
+ <li class="navItem">
+ <a href="{{ item.href }}"{% if item.category == "external" %} target="_blank"{% endif %}>{{ item.title }}</a>
+ </li>
+ {% endfor %}
+ {% if site.searchconfig %}
+ {% include nav_search.html inputselector="search_input_react" %}
+ {% endif %}
+ </ul>
+ </nav>
+</div>
+<script>
+ var event = document.createEvent('Event');
+ event.initEvent('slide', true, true);
+ document.addEventListener('slide', function (e) {
+ document.body.classList.toggle('sliderActive');
+ }, false);
+ var headerNav = document.getElementById('header_nav');
+ var headerNavExpander = document.getElementById('header_nav_expander');
+ headerNavExpander.addEventListener('click', function(e) {
+ headerNav.classList.toggle('navSlideoutActive');
+ document.dispatchEvent(event);
+ }, false);
+</script>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/nav_search.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/nav_search.html b/thirdparty/rocksdb/docs/_includes/nav_search.html
new file mode 100644
index 0000000..84956b9
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/nav_search.html
@@ -0,0 +1,15 @@
+<li class="navSearchWrapper">
+ <input id="{{ include.inputselector }}" type="search" />
+</li>
+<script type="text/javascript" src="https://cdn.jsdelivr.net/docsearch.js/1/docsearch.min.js"></script>
+<script>
+// For Algolia search
+(function() {
+ // Algolia
+ docsearch({
+ apiKey: '{{ site.searchconfig.apikey }}',
+ indexName: '{{ site.searchconfig.indexname }}',
+ inputSelector: '#{{ include.inputselector }}',
+ });
+}());
+</script>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/all_share.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/all_share.html b/thirdparty/rocksdb/docs/_includes/plugins/all_share.html
new file mode 100644
index 0000000..59b00d6
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/all_share.html
@@ -0,0 +1,3 @@
+<div class="pluginBlock allShareBlock">
+ {% include plugins/like_button.html %}{% include plugins/twitter_share.html %}{% include plugins/google_share.html %}
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/ascii_cinema.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/ascii_cinema.html b/thirdparty/rocksdb/docs/_includes/plugins/ascii_cinema.html
new file mode 100644
index 0000000..7d3f971
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/ascii_cinema.html
@@ -0,0 +1,2 @@
+<div class="ascii-cinema pluginBlock"></div>
+<script type="text/javascript" src="https://asciinema.org/a/{{ include.href }}.js" id="asciicast-{{ include.href }}" async data-autoplay="true" data-loop="true" data-speed="2" data-t="23"></script>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/button.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/button.html b/thirdparty/rocksdb/docs/_includes/plugins/button.html
new file mode 100644
index 0000000..9e499fe
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/button.html
@@ -0,0 +1,6 @@
+<div class="pluginWrapper buttonWrapper">
+ <a
+ class="button"
+ href="{{ include.href }}"
+ >{{ include.text }}</a>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/github_star.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/github_star.html b/thirdparty/rocksdb/docs/_includes/plugins/github_star.html
new file mode 100644
index 0000000..6aea70f
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/github_star.html
@@ -0,0 +1,4 @@
+<div class="pluginWrapper ghStarWrapper">
+ <a aria-label="Star {{ site.ghrepo }} on GitHub" data-count-aria-label="# stargazers on GitHub" data-count-api="/repos/{{ site.ghrepo }}#stargazers_count" data-count-href="/{{ site.ghrepo }}/stargazers" data-style="mega" data-icon="octicon-star" href="https://github.com/{{ site.ghrepo }}" class="github-button">Star</a>
+</div>
+<script async defer id="github-bjs" src="https://buttons.github.io/buttons.js"></script>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/github_watch.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/github_watch.html b/thirdparty/rocksdb/docs/_includes/plugins/github_watch.html
new file mode 100644
index 0000000..64233b5
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/github_watch.html
@@ -0,0 +1,4 @@
+<div class="pluginWrapper ghWatchWrapper">
+ <a aria-label="Watch {{ site.ghrepo }} on GitHub" data-count-aria-label="# watchers on GitHub" data-count-api="/repos/{{ site.ghrepo }}#subscribers_count" data-count-href="/{{ site.ghrepo }}/watchers" data-style="mega" data-icon="octicon-eye" href="https://github.com/{{ site.ghrepo }}" class="github-button">Watch</a>
+</div>
+<script async defer id="github-bjs" src="https://buttons.github.io/buttons.js"></script>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/google_share.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/google_share.html b/thirdparty/rocksdb/docs/_includes/plugins/google_share.html
new file mode 100644
index 0000000..1b557db
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/google_share.html
@@ -0,0 +1,5 @@
+<div class="pluginBlock">
+ <div class="g-plusone" data-size="medium"></div>
+</div>
+
+<script src="https://apis.google.com/js/platform.js" async defer></script>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/iframe.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/iframe.html b/thirdparty/rocksdb/docs/_includes/plugins/iframe.html
new file mode 100644
index 0000000..525b59f
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/iframe.html
@@ -0,0 +1,6 @@
+<div class="iframeContent">
+ <iframe class="pluginIframe" src="{{ include.href }}" seamless></iframe>
+</div>
+<div class="iframePreview">
+ {% include plugins/button.html href=include.href text=include.text %}
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/like_button.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/like_button.html b/thirdparty/rocksdb/docs/_includes/plugins/like_button.html
new file mode 100644
index 0000000..bcb8a7b
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/like_button.html
@@ -0,0 +1,18 @@
+<div class="fb-like pluginWrapper likeButtonWrapper" data-layout="button_count" data-action="like" data-show-faces="true" data-share="true"></div>
+<script>
+ window.fbAsyncInit = function() {
+ FB.init({
+ appId : '{{ site.fbappid }}',
+ xfbml : true,
+ version : 'v2.3'
+ });
+ };
+
+ (function(d, s, id){
+ var js, fjs = d.getElementsByTagName(s)[0];
+ if (d.getElementById(id)) {return;}
+ js = d.createElement(s); js.id = id;
+ js.src = "//connect.facebook.net/en_US/sdk.js";
+ fjs.parentNode.insertBefore(js, fjs);
+ }(document, 'script', 'facebook-jssdk'));
+</script>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/plugin_row.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/plugin_row.html b/thirdparty/rocksdb/docs/_includes/plugins/plugin_row.html
new file mode 100644
index 0000000..800f50b
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/plugin_row.html
@@ -0,0 +1,5 @@
+<div class="pluginRowBlock">
+{% for child in include.children %}
+ {% include plugins/{{child.type}}.html href=child.href text=child.text %}
+{% endfor %}
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/post_social_plugins.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/post_social_plugins.html b/thirdparty/rocksdb/docs/_includes/plugins/post_social_plugins.html
new file mode 100644
index 0000000..cb4d6e2
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/post_social_plugins.html
@@ -0,0 +1,34 @@
+<div class="postSocialPlugins">
+ <a
+ href="https://twitter.com/share"
+ class="twitter-share-button"
+ data-url="{{ page.url | replace:'index.html','' | absolute_url }}"
+ data-text="{% if page.title %}{{ page.title }}{% else %}{{ site.title }}{% endif %}"
+ data-hashtags="flowtype">Tweet</a>
+ <div
+ class="fb-like"
+ data-href="{{ page.url | replace:'index.html','' | absolute_url }}"
+ data-layout="button_count"
+ data-action="like"
+ data-show-faces="false"
+ data-share="true"></div>
+</div>
+<script>
+ window.fbAsyncInit = function() {
+ FB.init({
+ appId : '{{ site.fbappid }}',
+ xfbml : true,
+ version : 'v2.2'
+ });
+ };
+
+ (function(d, s, id){
+ var js, fjs = d.getElementsByTagName(s)[0];
+ if (d.getElementById(id)) {return;}
+ js = d.createElement(s); js.id = id;
+ js.src = "//connect.facebook.net/en_US/sdk.js";
+ fjs.parentNode.insertBefore(js, fjs);
+ }(document, 'script', 'facebook-jssdk'));
+</script>
+
+<script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs');</script>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/slideshow.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/slideshow.html b/thirdparty/rocksdb/docs/_includes/plugins/slideshow.html
new file mode 100644
index 0000000..69fa2b3
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/slideshow.html
@@ -0,0 +1,88 @@
+<div class="slideshowBlock pluginWrapper" id="slideshow"></div>
+<script>
+ var slideshowData = [
+ {% for image in site.data.slideshow %}
+ {
+ id : "{{ image.id }}",
+ imagesrc : "{{ image.src }}",
+ tooltip : "{{ image.tooltip }}",
+ href : "{{ image.link }}",
+ },
+ {% endfor %}
+ ];
+</script>
+<script src="http://fb.me/react-with-addons-0.13.1.min.js"></script>
+<script type="text/javascript">
+ var Slideshow = React.createClass({displayName: "Slideshow",
+ getInitialState: function() {
+ return {
+ currentSlide: 0,
+ };
+ },
+ getDefaultProps: function() {
+ return {
+ data: slideshowData,
+ };
+ },
+ handleSelect: function(id) {
+ var index = this.props.data.map(function (el, elIndex) {
+ return (
+ elIndex
+ );
+ });
+ var currentIndex = index.indexOf(id);
+ this.setState({
+ currentSlide: currentIndex,
+ });
+ },
+ render: function() {
+ return (
+ React.createElement("div", {className: "slideshow"},
+ React.createElement("div", {className: "slides"},
+ this.props.data.map(this.renderSlide)
+ ),
+ React.createElement("div", {className: "pagination"},
+ this.props.data.map(this.renderPager)
+ )
+ )
+ );
+ },
+ renderSlide: function(child, index) {
+ var classes = React.addons.classSet({
+ 'slide': true,
+ 'slideActive': this.state.currentSlide === index,
+ });
+ if (child.href) {
+ return (
+ React.createElement("div", {key: index, className: classes},
+ React.createElement("a", {href: child.href, alt: child.tooltip, title: child.tooltip},
+ React.createElement("img", {src: child.imagesrc, alt: child.tooltip, title: child.tooltip})
+ )
+ )
+ );
+ }
+ return (
+ React.createElement("div", {key: index, className: classes},
+ React.createElement("img", {src: child.imagesrc, alt: child.tooltip})
+ )
+ );
+ },
+ renderPager: function(child, index) {
+ var classes = React.addons.classSet({
+ 'pager': true,
+ 'pagerActive': this.state.currentSlide === index,
+ });
+ return (
+ React.createElement("span", {key: index, className: classes, onClick: this.handleSelect.bind(this, index)})
+ );
+ },
+ });
+
+ function render(slideshowData) {
+ React.render(
+ React.createElement(Slideshow, {data: slideshowData}),
+ document.getElementById('slideshow')
+ );
+ }
+ render(slideshowData);
+</script>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/twitter_follow.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/twitter_follow.html b/thirdparty/rocksdb/docs/_includes/plugins/twitter_follow.html
new file mode 100644
index 0000000..9443db3
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/twitter_follow.html
@@ -0,0 +1,5 @@
+<div class="pluginBlock">
+ <a href="https://twitter.com/{{ include.href }}" class="twitter-follow-button pluginBlock" data-show-count="false">Follow @{{ include.href }}</a>
+</div>
+
+<script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs');</script>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/plugins/twitter_share.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/plugins/twitter_share.html b/thirdparty/rocksdb/docs/_includes/plugins/twitter_share.html
new file mode 100644
index 0000000..67f1d62
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/plugins/twitter_share.html
@@ -0,0 +1,4 @@
+<div class="pluginWrapper twitterSharePlugin">
+ <a href="https://twitter.com/share" class="twitter-share-button" data-hashtags="{{ site.title| replace: ' ', '' }}">Tweet</a>
+</div>
+<script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs');</script>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/post.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/post.html b/thirdparty/rocksdb/docs/_includes/post.html
new file mode 100644
index 0000000..4c4b5b1
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/post.html
@@ -0,0 +1,35 @@
+<div class="post">
+ {% assign author = site.data.authors[page.author] %}
+ <header class="post-header">
+ {% if author.fbid %}
+ <div class="authorPhoto">
+ <img src="http://graph.facebook.com/{{ author.fbid }}/picture/" alt="{{ author.fullname }}" title="{{ author.fullname }}" />
+ </div>
+ {% endif %}
+ {% if author.full_name %}
+ <p class="post-authorName">{{ author.full_name }}</p>
+ {% endif %}
+ <h1 class="post-title">{% if include.truncate %}<a href="{{ page.url | absolute_url }}">{{ page.title }}</a>{% else %}{{ page.title }}{% endif %}</h1>
+ <p class="post-meta">Posted {{ page.date | date: '%B %d, %Y' }}{% if page.meta %} • {{ page.meta }}{% endif %}</p>
+ </header>
+
+ <article class="post-content">
+ {% if include.truncate %}
+ {% if page.content contains '<!--truncate-->' %}
+ {{ page.content | split:'<!--truncate-->' | first | markdownify }}
+ <div class="read-more">
+ <a href="{{ page.url | absolute_url }}" >
+ Read More
+ </a>
+ </div>
+ {% else %}
+ {{ page.content | markdownify }}
+ {% endif %}
+ {% else %}
+ {{ content }}
+ {% endif %}
+ {% unless include.truncate %}
+ {% include plugins/like_button.html %}
+ {% endunless %}
+ </article>
+</div>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/powered_by.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/powered_by.html b/thirdparty/rocksdb/docs/_includes/powered_by.html
new file mode 100644
index 0000000..c629429
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/powered_by.html
@@ -0,0 +1,28 @@
+{% if site.data.powered_by.first.items or site.data.powered_by_highlight.first.items %}
+<div class="poweredByContainer">
+ <div class="wrapper mainWrapper poweredByWrapper">
+ {% if site.data.powered_by_highlight.first.title %}
+ <h2>{{ site.data.powered_by_highlight.first.title }}</h2>
+ {% else %}
+ <h2>{{ site.data.powered_by.first.title }}</h2>
+ {% endif %}
+ {% if site.data.powered_by_highlight.first.items %}
+ <div class="poweredByItems">
+ {% for item in site.data.powered_by_highlight.first.items %}
+ <div class="poweredByItem itemLarge">
+ <a href="{{ item.url }}" target="_blank"><img src="{{ item.img }}" alt="{{ item.name }}" /></a>
+ </div>
+ {% endfor %}
+ </div>
+ {% endif %}
+ <div class="poweredByItems">
+ {% for item in site.data.powered_by.first.items %}
+ <div class="poweredByItem itemSmall">
+ <a href="{{ item.url }}" target="_blank">{{ item.name }}</a>
+ </div>
+ {% endfor %}
+ </div>
+ <div class="poweredByMessage">Does your app use {{ site.title }}? Add it to this list with <a href="https://github.com/{{ site.ghrepo }}/edit/gh-pages/_data/powered_by.yml" target="_blank">a pull request!</a></div>
+ </div>
+</div>
+{% endif %}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/social_plugins.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/social_plugins.html b/thirdparty/rocksdb/docs/_includes/social_plugins.html
new file mode 100644
index 0000000..db4f1ae
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/social_plugins.html
@@ -0,0 +1,24 @@
+<a
+ href="https://twitter.com/share"
+ class="twitter-share-button"
+ data-url="http://facebook.github.io/fresco{{ page.url }}"
+ data-text="Fresco | {{ page.title }}"
+ data-hashtags="fresco">Tweet</a>
+<div
+ class="fb-like"
+ data-href="http://facebook.github.io/fresco{{ page.url }}"
+ data-layout="standard"
+ data-action="like"
+ data-show-faces="true"
+ data-share="true"></div>
+
+<div id="fb-root"></div>
+<script>(function(d, s, id) {
+ var js, fjs = d.getElementsByTagName(s)[0];
+ if (d.getElementById(id)) return;
+ js = d.createElement(s); js.id = id;
+ js.src = "//connect.facebook.net/en_US/sdk.js#xfbml=1&version=v2.0";
+ fjs.parentNode.insertBefore(js, fjs);
+}(document, 'script', 'facebook-jssdk'));</script>
+
+<script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs');</script>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_includes/ui/button.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_includes/ui/button.html b/thirdparty/rocksdb/docs/_includes/ui/button.html
new file mode 100644
index 0000000..729ccc3
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_includes/ui/button.html
@@ -0,0 +1 @@
+<span class="buttonWrap {{ include.align }}"><a class="button blockButton fbossFontLight pluginBlock margin{{ include.margin }}" target="{{ include.button_target }}" href="{{ include.button_href }}">{{ include.button_text }}</a></span>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/basic.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/basic.html b/thirdparty/rocksdb/docs/_layouts/basic.html
new file mode 100644
index 0000000..65bd210
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/basic.html
@@ -0,0 +1,12 @@
+---
+layout: doc_default
+---
+
+<div class="mainContainer blogContainer postContainer">
+ <div id="main_wrap" class="wrapper mainWrapper">
+ <div class="post basicPost">
+ {{ content }}
+ </div>
+ </div>
+</div>
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/blog.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/blog.html b/thirdparty/rocksdb/docs/_layouts/blog.html
new file mode 100644
index 0000000..1b0da41
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/blog.html
@@ -0,0 +1,11 @@
+---
+category: blog
+layout: blog_default
+---
+
+<div class="mainContainer blogContainer postContainer">
+ <div id="main_wrap" class="wrapper mainWrapper">
+ {{ content }}
+ </div>
+</div>
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/blog_default.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/blog_default.html b/thirdparty/rocksdb/docs/_layouts/blog_default.html
new file mode 100644
index 0000000..a29d58d
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/blog_default.html
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+ {% include head.html %}
+ <body class="docsNavVisible">
+ {% include nav.html alwayson=true %}
+ <div class="navPusher">
+ <div class="docMainWrapper wrapper">
+ {% include nav/collection_nav.html navdata=site.posts type="blog" sectionpath="/blog/" sectiontitle="Blog" %}
+ {{ content }}
+ </div>
+ {% include footer.html %}
+ </div>
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/default.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/default.html b/thirdparty/rocksdb/docs/_layouts/default.html
new file mode 100644
index 0000000..0167d9f
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/default.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+ {% include head.html %}
+ <body>
+ {% include nav.html alwayson=true %}
+ <div class="navPusher">
+ {{ content }}
+ {% include footer.html %}
+ </div>
+ </body>
+
+</html>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/doc_default.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/doc_default.html b/thirdparty/rocksdb/docs/_layouts/doc_default.html
new file mode 100644
index 0000000..4a41392
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/doc_default.html
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+ {% include head.html %}
+ <body class="docsNavVisible">
+ {% include nav.html alwayson=true %}
+ <div class="navPusher">
+ <div class="docMainWrapper wrapper">
+ {% include nav/collection_nav.html navdata=site.data.nav_docs type="docs" sectionpath="/docs/" sectiontitle="Docs" data_collection=site.docs %}
+ {{ content }}
+ </div>
+ {% include footer.html %}
+ </div>
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/doc_page.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/doc_page.html b/thirdparty/rocksdb/docs/_layouts/doc_page.html
new file mode 100644
index 0000000..dba761e
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/doc_page.html
@@ -0,0 +1,10 @@
+---
+layout: doc_default
+---
+
+<div class="mainContainer documentContainer postContainer">
+ <div id="main_wrap" class="wrapper mainWrapper">
+ {{ content }}
+ </div>
+</div>
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/docs.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/docs.html b/thirdparty/rocksdb/docs/_layouts/docs.html
new file mode 100644
index 0000000..749dafa
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/docs.html
@@ -0,0 +1,5 @@
+---
+layout: doc_page
+---
+
+{% include doc.html %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/home.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/home.html b/thirdparty/rocksdb/docs/_layouts/home.html
new file mode 100644
index 0000000..e3c320f
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/home.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+ {% include head.html %}
+ <body>
+ {% include nav.html alwayson=true %}
+ <div class="navPusher">
+ {% include home_header.html %}
+ <div class="mainContainer">
+ <div id="main_wrap" class="wrapper mainWrapper">
+ {{ content }}
+ </div>
+ {% include powered_by.html %}
+ </div>
+ {% include footer.html %}
+ </div>
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/page.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/page.html b/thirdparty/rocksdb/docs/_layouts/page.html
new file mode 100644
index 0000000..bec3680
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/page.html
@@ -0,0 +1,3 @@
+---
+layout: blog
+---
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/plain.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/plain.html b/thirdparty/rocksdb/docs/_layouts/plain.html
new file mode 100644
index 0000000..fccc02c
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/plain.html
@@ -0,0 +1,10 @@
+---
+layout: default
+---
+
+<div class="mainContainer blogContainer postContainer">
+ <div id="main_wrap" class="wrapper mainWrapper">
+ {{ content }}
+ </div>
+</div>
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/post.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/post.html b/thirdparty/rocksdb/docs/_layouts/post.html
new file mode 100644
index 0000000..4c92cf2
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/post.html
@@ -0,0 +1,8 @@
+---
+collection: blog
+layout: blog
+---
+
+<div class="lonePost">
+{% include post.html %}
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/redirect.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/redirect.html b/thirdparty/rocksdb/docs/_layouts/redirect.html
new file mode 100644
index 0000000..c24f817
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/redirect.html
@@ -0,0 +1,6 @@
+<html>
+<head>
+ <meta http-equiv="refresh" content="0; {{ page.destination }}">
+</head>
+<body></body>
+</html>
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_layouts/top-level.html
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_layouts/top-level.html b/thirdparty/rocksdb/docs/_layouts/top-level.html
new file mode 100644
index 0000000..fccc02c
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_layouts/top-level.html
@@ -0,0 +1,10 @@
+---
+layout: default
+---
+
+<div class="mainContainer blogContainer postContainer">
+ <div id="main_wrap" class="wrapper mainWrapper">
+ {{ content }}
+ </div>
+</div>
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-03-27-how-to-backup-rocksdb.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-03-27-how-to-backup-rocksdb.markdown b/thirdparty/rocksdb/docs/_posts/2014-03-27-how-to-backup-rocksdb.markdown
new file mode 100644
index 0000000..f9e4a54
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-03-27-how-to-backup-rocksdb.markdown
@@ -0,0 +1,135 @@
+---
+title: How to backup RocksDB?
+layout: post
+author: icanadi
+category: blog
+redirect_from:
+ - /blog/191/how-to-backup-rocksdb/
+---
+
+In RocksDB, we have implemented an easy way to backup your DB. Here is a simple example:
+
+
+
+ #include "rocksdb/db.h"
+ #include "utilities/backupable_db.h"
+ using namespace rocksdb;
+
+ DB* db;
+ DB::Open(Options(), "/tmp/rocksdb", &db);
+ BackupableDB* backupable_db = new BackupableDB(db, BackupableDBOptions("/tmp/rocksdb_backup"));
+ backupable_db->Put(...); // do your thing
+ backupable_db->CreateNewBackup();
+ delete backupable_db; // no need to also delete db
+
+<!--truncate-->
+
+
+This simple example will create a backup of your DB in "/tmp/rocksdb_backup". Creating new BackupableDB consumes DB* and you should be calling all the DB methods on object `backupable_db` going forward.
+
+Restoring is also easy:
+
+
+
+ RestoreBackupableDB* restore = new RestoreBackupableDB(Env::Default(), BackupableDBOptions("/tmp/rocksdb_backup"));
+ restore->RestoreDBFromLatestBackup("/tmp/rocksdb", "/tmp/rocksdb");
+ delete restore;
+
+
+
+
+This code will restore the backup back to "/tmp/rocksdb". The second parameter is the location of log files (In some DBs they are different from DB directory, but usually they are the same. See Options::wal_dir for more info).
+
+An alternative API for backups is to use BackupEngine directly:
+
+
+
+ #include "rocksdb/db.h"
+ #include "utilities/backupable_db.h"
+ using namespace rocksdb;
+
+ DB* db;
+ DB::Open(Options(), "/tmp/rocksdb", &db);
+ db->Put(...); // do your thing
+ BackupEngine* backup_engine = BackupEngine::NewBackupEngine(Env::Default(), BackupableDBOptions("/tmp/rocksdb_backup"));
+ backup_engine->CreateNewBackup(db);
+ delete db;
+ delete backup_engine;
+
+
+
+
+Restoring with BackupEngine is similar to RestoreBackupableDB:
+
+
+
+ BackupEngine* backup_engine = BackupEngine::NewBackupEngine(Env::Default(), BackupableDBOptions("/tmp/rocksdb_backup"));
+ backup_engine->RestoreDBFromLatestBackup("/tmp/rocksdb", "/tmp/rocksdb");
+ delete backup_engine;
+
+
+
+
+Backups are incremental. You can create a new backup with `CreateNewBackup()` and only the new data will be copied to backup directory (for more details on what gets copied, see "Under the hood"). Checksum is always calculated for any backuped file (including sst, log, and etc). It is used to make sure files are kept sound in the file system. Checksum is also verified for files from the previous backups even though they do not need to be copied. A checksum mismatch aborts the current backup (see "Under the hood" for more details). Once you have more backups saved, you can issue `GetBackupInfo()` call to get a list of all backups together with information on timestamp of the backup and the size (please note that sum of all backups' sizes is bigger than the actual size of the backup directory because some data is shared by multiple backups). Backups are identified by their always-increasing IDs. `GetBackupInfo()` is available both in `BackupableDB` and `RestoreBackupableDB`.
+
+You probably want to keep around only small number of backups. To delete old backups, just call `PurgeOldBackups(N)`, where N is how many backups you'd like to keep. All backups except the N newest ones will be deleted. You can also choose to delete arbitrary backup with call `DeleteBackup(id)`.
+
+`RestoreDBFromLatestBackup()` will restore the DB from the latest consistent backup. An alternative is `RestoreDBFromBackup()` which takes a backup ID and restores that particular backup. Checksum is calculated for any restored file and compared against the one stored during the backup time. If a checksum mismatch is detected, the restore process is aborted and `Status::Corruption` is returned. Very important thing to note here: Let's say you have backups 1, 2, 3, 4. If you restore from backup 2 and start writing more data to your database, newly created backup will delete old backups 3 and 4 and create new backup 3 on top of 2.
+
+
+
+## Advanced usage
+
+
+Let's say you want to backup your DB to HDFS. There is an option in `BackupableDBOptions` to set `backup_env`, which will be used for all file I/O related to backup dir (writes when backuping, reads when restoring). If you set it to HDFS Env, all the backups will be stored in HDFS.
+
+`BackupableDBOptions::info_log` is a Logger object that is used to print out LOG messages if not-nullptr.
+
+If `BackupableDBOptions::sync` is true, we will sync data to disk after every file write, guaranteeing that backups will be consistent after a reboot or if machine crashes. Setting it to false will speed things up a bit, but some (newer) backups might be inconsistent. In most cases, everything should be fine, though.
+
+If you set `BackupableDBOptions::destroy_old_data` to true, creating new `BackupableDB` will delete all the old backups in the backup directory.
+
+`BackupableDB::CreateNewBackup()` method takes a parameter `flush_before_backup`, which is false by default. When `flush_before_backup` is true, `BackupableDB` will first issue a memtable flush and only then copy the DB files to the backup directory. Doing so will prevent log files from being copied to the backup directory (since flush will delete them). If `flush_before_backup` is false, backup will not issue flush before starting the backup. In that case, the backup will also include log files corresponding to live memtables. Backup will be consistent with current state of the database regardless of `flush_before_backup` parameter.
+
+
+
+## Under the hood
+
+
+`BackupableDB` implements `DB` interface and adds four methods to it: `CreateNewBackup()`, `GetBackupInfo()`, `PurgeOldBackups()`, `DeleteBackup()`. Any `DB` interface calls will get forwarded to underlying `DB` object.
+
+When you call `BackupableDB::CreateNewBackup()`, it does the following:
+
+
+
+
+
+ 1. Disable file deletions
+
+
+
+ 2. Get live files (this includes table files, current and manifest file).
+
+
+
+ 3. Copy live files to the backup directory. Since table files are immutable and filenames unique, we don't copy a table file that is already present in the backup directory. For example, if there is a file `00050.sst` already backed up and `GetLiveFiles()` returns `00050.sst`, we will not copy that file to the backup directory. However, checksum is calculated for all files regardless if a file needs to be copied or not. If a file is already present, the calculated checksum is compared against previously calculated checksum to make sure nothing crazy happened between backups. If a mismatch is detected, backup is aborted and the system is restored back to the state before `BackupableDB::CreateNewBackup()` is called. One thing to note is that a backup abortion could mean a corruption from a file in backup directory or the corresponding live file in current DB. Both manifest and current files are copied, since they are not immutable.
+
+
+
+ 4. If `flush_before_backup` was set to false, we also need to copy log files to the backup directory. We call `GetSortedWalFiles()` and copy all live files to the backup directory.
+
+
+
+ 5. Enable file deletions
+
+
+
+
+Backup IDs are always increasing and we have a file `LATEST_BACKUP` that contains the ID of the latest backup. If we crash in middle of backing up, on a restart we will detect that there are newer backup files than `LATEST_BACKUP` claims there are. In that case, we will delete any backup newer than `LATEST_BACKUP` and clean up all the files since some of the table files might be corrupted. Having corrupted table files in the backup directory is dangerous because of our deduplication strategy.
+
+
+
+## Further reading
+
+
+For the API details, see `include/utilities/backupable_db.h`. For the implementation, see `utilities/backupable/backupable_db.cc`.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-03-27-how-to-persist-in-memory-rocksdb-database.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-03-27-how-to-persist-in-memory-rocksdb-database.markdown b/thirdparty/rocksdb/docs/_posts/2014-03-27-how-to-persist-in-memory-rocksdb-database.markdown
new file mode 100644
index 0000000..89ffb2d
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-03-27-how-to-persist-in-memory-rocksdb-database.markdown
@@ -0,0 +1,54 @@
+---
+title: How to persist in-memory RocksDB database?
+layout: post
+author: icanadi
+category: blog
+redirect_from:
+ - /blog/245/how-to-persist-in-memory-rocksdb-database/
+---
+
+In recent months, we have focused on optimizing RocksDB for in-memory workloads. With growing RAM sizes and strict low-latency requirements, lots of applications decide to keep their entire data in memory. Running in-memory database with RocksDB is easy -- just mount your RocksDB directory on tmpfs or ramfs [1]. Even if the process crashes, RocksDB can recover all of your data from in-memory filesystem. However, what happens if the machine reboots?
+
+<!--truncate-->
+
+In this article we will explain how you can recover your in-memory RocksDB database even after a machine reboot.
+
+Every update to RocksDB is written to two places - one is an in-memory data structure called memtable and second is write-ahead log. Write-ahead log can be used to completely recover the data in memtable. By default, when we flush the memtable to table file, we also delete the current log, since we don't need it anymore for recovery (the data from the log is "persisted" in the table file -- we say that the log file is obsolete). However, if your table file is stored in in-memory file system, you may need the obsolete write-ahead log to recover the data after the machine reboots. Here's how you can do that.
+
+Options::wal_dir is the directory where RocksDB stores write-ahead log files. If you configure this directory to be on flash or disk, you will not lose current log file on machine reboot.
+Options::WAL_ttl_seconds is the timeout when we delete the archived log files. If the timeout is non-zero, obsolete log files will be moved to `archive/` directory under Options::wal_dir. Those archived log files will only be deleted after the specified timeout.
+
+Let's assume Options::wal_dir is a directory on persistent storage and Options::WAL_ttl_seconds is set to one day. To fully recover the DB, we also need to backup the current snapshot of the database (containing table and metadata files) with a frequency of less than one day. RocksDB provides an utility that enables you to easily backup the snapshot of your database. You can learn more about it here: [How to backup RocksDB?](https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F)
+
+You should configure the backup process to avoid backing up log files, since they are already stored in persistent storage. To do that, set BackupableDBOptions::backup_log_files to false.
+
+Restore process by default cleans up entire DB and WAL directory. Since we didn't include log files in the backup, we need to make sure that restoring the database doesn't delete log files in WAL directory. When restoring, configure RestoreOptions::keep_log_file to true. That option will also move any archived log files back to WAL directory, enabling RocksDB to replay all archived log files and rebuild the in-memory database state.
+
+To reiterate, here's what you have to do:
+
+
+
+
+ * Set DB directory to tmpfs or ramfs mounted drive
+
+
+
+ * Set Options::wal_log to a directory on persistent storage
+
+
+
+ * Set Options::WAL_ttl_seconds to T seconds
+
+
+
+ * Backup RocksDB every T/2 seconds, with BackupableDBOptions::backup_log_files = false
+
+
+
+ * When you lose data, restore from backup with RestoreOptions::keep_log_file = true
+
+
+
+
+
+[1] You might also want to consider using [PlainTable format](https://github.com/facebook/rocksdb/wiki/PlainTable-Format) for table files
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-04-02-the-1st-rocksdb-local-meetup-held-on-march-27-2014.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-04-02-the-1st-rocksdb-local-meetup-held-on-march-27-2014.markdown b/thirdparty/rocksdb/docs/_posts/2014-04-02-the-1st-rocksdb-local-meetup-held-on-march-27-2014.markdown
new file mode 100644
index 0000000..7ccbdba
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-04-02-the-1st-rocksdb-local-meetup-held-on-march-27-2014.markdown
@@ -0,0 +1,53 @@
+---
+title: The 1st RocksDB Local Meetup Held on March 27, 2014
+layout: post
+author: xjin
+category: blog
+redirect_from:
+ - /blog/323/the-1st-rocksdb-local-meetup-held-on-march-27-2014/
+---
+
+On Mar 27, 2014, RocksDB team @ Facebook held the 1st RocksDB local meetup in FB HQ (Menlo Park, California). We invited around 80 guests from 20+ local companies, including LinkedIn, Twitter, Dropbox, Square, Pinterest, MapR, Microsoft and IBM. Finally around 50 guests showed up, totaling around 60% show-up rate.
+
+<!--truncate-->
+
+[](/static/images/Resize-of-20140327_200754-300x225.jpg)
+
+RocksDB team @ Facebook gave four talks about the latest progress and experience on RocksDB:
+
+
+
+
+ * [Supporting a 1PB In-Memory Workload](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Haobo-RocksDB-In-Memory.pdf)
+
+
+
+
+ * [Column Families in RocksDB](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Igor-Column-Families.pdf)
+
+
+
+
+ * ["Lockless" Get() in RocksDB?](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Lei-Lockless-Get.pdf)
+
+
+
+
+ * [Prefix Hashing in RocksDB](https://github.com/facebook/rocksdb/raw/gh-pages/talks/2014-03-27-RocksDB-Meetup-Siying-Prefix-Hash.pdf)
+
+
+A very interesting question asked by a massive number of guests is: does RocksDB plan to provide replication functionality? Obviously, many applications need a resilient and distributed storage solution, not just single-node storage. We are considering how to approach this issue.
+
+When will be the next meetup? We haven't decided yet. We will see whether the community is interested in it and how it can help RocksDB grow.
+
+If you have any questions or feedback for the meetup or RocksDB, please let us know in [our Facebook group](https://www.facebook.com/groups/rocksdb.dev/).
+
+### Comments
+
+**[Rajiv](geetasen@gmail.com)**
+
+Have any of these talks been recorded and if so will they be published?
+
+**[Igor Canadi](icanadi@fb.com)**
+
+Yes, I think we plan to publish them soon.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-04-07-rocksdb-2-8-release.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-04-07-rocksdb-2-8-release.markdown b/thirdparty/rocksdb/docs/_posts/2014-04-07-rocksdb-2-8-release.markdown
new file mode 100644
index 0000000..7be7842
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-04-07-rocksdb-2-8-release.markdown
@@ -0,0 +1,40 @@
+---
+title: RocksDB 2.8 release
+layout: post
+author: icanadi
+category: blog
+redirect_from:
+ - /blog/371/rocksdb-2-8-release/
+---
+
+Check out the new RocksDB 2.8 release on [Github](https://github.com/facebook/rocksdb/releases/tag/2.8.fb).
+
+RocksDB 2.8. is mostly focused on improving performance for in-memory workloads. We are seeing read QPS as high as 5M (we will write a separate blog post on this).
+
+<!--truncate-->
+
+Here is the summary of new features:
+
+ * Added a new table format called PlainTable, which is optimized for RAM storage (ramfs or tmpfs). You can read more details about it on [our wiki](https://github.com/facebook/rocksdb/wiki/PlainTable-Format).
+
+
+ * New prefixed memtable format HashLinkedList, which is optimized for cases where there are only a few keys for each prefix.
+
+
+ * Merge operator supports a new function PartialMergeMulti() that allows users to do partial merges against multiple operands. This function enables big speedups for workloads that use merge operators.
+
+
+ * Added a V2 compaction filter interface. It buffers the kv-pairs sharing the same key prefix, process them in batches, and return the batched results back to DB.
+
+
+ * Geo-spatial support for locations and radial-search.
+
+
+ * Improved read performance using thread local cache for frequently accessed data.
+
+
+ * Stability improvements -- we're now ignoring partially written tailing record to MANIFEST or WAL files.
+
+
+
+We have also introduced small incompatible API changes (mostly for advanced users). You can see full release notes in our [HISTORY.my](https://github.com/facebook/rocksdb/blob/2.8.fb/HISTORY.md) file.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_posts/2014-04-21-indexing-sst-files-for-better-lookup-performance.markdown
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_posts/2014-04-21-indexing-sst-files-for-better-lookup-performance.markdown b/thirdparty/rocksdb/docs/_posts/2014-04-21-indexing-sst-files-for-better-lookup-performance.markdown
new file mode 100644
index 0000000..368055d
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_posts/2014-04-21-indexing-sst-files-for-better-lookup-performance.markdown
@@ -0,0 +1,28 @@
+---
+title: Indexing SST Files for Better Lookup Performance
+layout: post
+author: leijin
+category: blog
+redirect_from:
+ - /blog/431/indexing-sst-files-for-better-lookup-performance/
+---
+
+For a `Get()` request, RocksDB goes through mutable memtable, list of immutable memtables, and SST files to look up the target key. SST files are organized in levels.
+
+On level 0, files are sorted based on the time they are flushed. Their key range (as defined by FileMetaData.smallest and FileMetaData.largest) are mostly overlapped with each other. So it needs to look up every L0 file.
+
+<!--truncate-->
+
+Compaction is scheduled periodically to pick up files from an upper level and merges them with files from lower level. As a result, key/values are moved from L0 down the LSM tree gradually. Compaction sorts key/values and split them into files. From level 1 and below, SST files are sorted based on key. Their key range are mutually exclusive. Instead of scanning through each SST file and checking if a key falls into its range, RocksDB performs a binary search based on FileMetaData.largest to locate a candidate file that can potentially contain the target key. This reduces complexity from O(N) to O(log(N)). However, log(N) can still be large for bottom levels. For a fan-out ratio of 10, level 3 can have 1000 files. That requires 10 comparisons to locate a candidate file. This is a significant cost for an in-memory database when you can do [several million gets per second](https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks).
+
+One observation to this problem is that: after the LSM tree is built, an SST file's position in its level is fixed. Furthermore, its order relative to files from the next level is also fixed. Based on this idea, we can perform [fractional cascading](http://en.wikipedia.org/wiki/Fractional_cascading) kind of optimization to narrow down the binary search range. Here is an example:
+
+[](/static/images/tree_example1.png)
+
+Level 1 has 2 files and level 2 has 8 files. Now, we want to look up key 80. A binary search based FileMetaData.largest tells you file 1 is the candidate. Then key 80 is compared with its FileMetaData.smallest and FileMetaData.largest to decide if it falls into the range. The comparison shows 80 is less than FileMetaData.smallest (100), so file 1 does not possibly contain key 80. We to proceed to check level 2. Usually, we need to do binary search among all 8 files on level 2. But since we already know target key 80 is less than 100 and only file 1 to file 3 can contain key less than 100, we can safely exclude other files from the search. As a result we cut down the search space from 8 files to 3 files.
+
+Let's look at another example. We want to get key 230. A binary search on level 1 locates to file 2 (this also implies key 230 is larger than file 1's FileMetaData.largest 200). A comparison with file 2's range shows the target key is smaller than file 2's FileMetaData.smallest 300. Even though, we couldn't find key on level 1, we have derived hints that target key is in range between 200 and 300. Any files on level 2 that cannot overlap with [200, 300] can be safely excluded. As a result, we only need to look at file 5 and file 6 on level 2.
+
+Inspired by this concept, we pre-build pointers at compaction time on level 1 files that point to a range of files on level 2. For example, file 1 on level 1 points to file 3 (on level 2) on the left and file 4 on the right. File 2 will point to level 2 files 6 and 7. At query time, these pointers are used to determine the actual binary search range based on comparison result.
+
+Our benchmark shows that this optimization improves lookup QPS by ~5% for similar setup mentioned [here](https://github.com/facebook/rocksdb/wiki/RocksDB-In-Memory-Workload-Performance-Benchmarks).
[25/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compaction_picker.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compaction_picker.cc b/thirdparty/rocksdb/db/compaction_picker.cc
new file mode 100644
index 0000000..c6a5674
--- /dev/null
+++ b/thirdparty/rocksdb/db/compaction_picker.cc
@@ -0,0 +1,1591 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/compaction_picker.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <limits>
+#include <queue>
+#include <string>
+#include <utility>
+#include "db/column_family.h"
+#include "monitoring/statistics.h"
+#include "util/filename.h"
+#include "util/log_buffer.h"
+#include "util/random.h"
+#include "util/string_util.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+namespace {
+uint64_t TotalCompensatedFileSize(const std::vector<FileMetaData*>& files) {
+ uint64_t sum = 0;
+ for (size_t i = 0; i < files.size() && files[i]; i++) {
+ sum += files[i]->compensated_file_size;
+ }
+ return sum;
+}
+
+bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
+ size_t min_files_to_compact,
+ uint64_t max_compact_bytes_per_del_file,
+ CompactionInputFiles* comp_inputs) {
+ size_t compact_bytes = level_files[0]->fd.file_size;
+ size_t compact_bytes_per_del_file = port::kMaxSizet;
+ // compaction range will be [0, span_len).
+ size_t span_len;
+ // pull in files until the amount of compaction work per deleted file begins
+ // increasing.
+ size_t new_compact_bytes_per_del_file = 0;
+ for (span_len = 1; span_len < level_files.size(); ++span_len) {
+ compact_bytes += level_files[span_len]->fd.file_size;
+ new_compact_bytes_per_del_file = compact_bytes / span_len;
+ if (level_files[span_len]->being_compacted ||
+ new_compact_bytes_per_del_file > compact_bytes_per_del_file) {
+ break;
+ }
+ compact_bytes_per_del_file = new_compact_bytes_per_del_file;
+ }
+
+ if (span_len >= min_files_to_compact &&
+ new_compact_bytes_per_del_file < max_compact_bytes_per_del_file) {
+ assert(comp_inputs != nullptr);
+ comp_inputs->level = 0;
+ for (size_t i = 0; i < span_len; ++i) {
+ comp_inputs->files.push_back(level_files[i]);
+ }
+ return true;
+ }
+ return false;
+}
+} // anonymous namespace
+
+// Determine compression type, based on user options, level of the output
+// file and whether compression is disabled.
+// If enable_compression is false, then compression is always disabled no
+// matter what the values of the other two parameters are.
+// Otherwise, the compression type is determined based on options and level.
+CompressionType GetCompressionType(const ImmutableCFOptions& ioptions,
+ const VersionStorageInfo* vstorage,
+ const MutableCFOptions& mutable_cf_options,
+ int level, int base_level,
+ const bool enable_compression) {
+ if (!enable_compression) {
+ // disable compression
+ return kNoCompression;
+ }
+
+ // If bottommost_compression is set and we are compacting to the
+ // bottommost level then we should use it.
+ if (ioptions.bottommost_compression != kDisableCompressionOption &&
+ level > base_level && level >= (vstorage->num_non_empty_levels() - 1)) {
+ return ioptions.bottommost_compression;
+ }
+ // If the user has specified a different compression level for each level,
+ // then pick the compression for that level.
+ if (!ioptions.compression_per_level.empty()) {
+ assert(level == 0 || level >= base_level);
+ int idx = (level == 0) ? 0 : level - base_level + 1;
+
+ const int n = static_cast<int>(ioptions.compression_per_level.size()) - 1;
+ // It is possible for level_ to be -1; in that case, we use level
+ // 0's compression. This occurs mostly in backwards compatibility
+ // situations when the builder doesn't know what level the file
+ // belongs to. Likewise, if level is beyond the end of the
+ // specified compression levels, use the last value.
+ return ioptions.compression_per_level[std::max(0, std::min(idx, n))];
+ } else {
+ return mutable_cf_options.compression;
+ }
+}
+
+CompactionPicker::CompactionPicker(const ImmutableCFOptions& ioptions,
+ const InternalKeyComparator* icmp)
+ : ioptions_(ioptions), icmp_(icmp) {}
+
+CompactionPicker::~CompactionPicker() {}
+
+// Delete this compaction from the list of running compactions.
+void CompactionPicker::ReleaseCompactionFiles(Compaction* c, Status status) {
+ UnregisterCompaction(c);
+ if (!status.ok()) {
+ c->ResetNextCompactionIndex();
+ }
+}
+
+void CompactionPicker::GetRange(const CompactionInputFiles& inputs,
+ InternalKey* smallest,
+ InternalKey* largest) const {
+ const int level = inputs.level;
+ assert(!inputs.empty());
+ smallest->Clear();
+ largest->Clear();
+
+ if (level == 0) {
+ for (size_t i = 0; i < inputs.size(); i++) {
+ FileMetaData* f = inputs[i];
+ if (i == 0) {
+ *smallest = f->smallest;
+ *largest = f->largest;
+ } else {
+ if (icmp_->Compare(f->smallest, *smallest) < 0) {
+ *smallest = f->smallest;
+ }
+ if (icmp_->Compare(f->largest, *largest) > 0) {
+ *largest = f->largest;
+ }
+ }
+ }
+ } else {
+ *smallest = inputs[0]->smallest;
+ *largest = inputs[inputs.size() - 1]->largest;
+ }
+}
+
+void CompactionPicker::GetRange(const CompactionInputFiles& inputs1,
+ const CompactionInputFiles& inputs2,
+ InternalKey* smallest,
+ InternalKey* largest) const {
+ assert(!inputs1.empty() || !inputs2.empty());
+ if (inputs1.empty()) {
+ GetRange(inputs2, smallest, largest);
+ } else if (inputs2.empty()) {
+ GetRange(inputs1, smallest, largest);
+ } else {
+ InternalKey smallest1, smallest2, largest1, largest2;
+ GetRange(inputs1, &smallest1, &largest1);
+ GetRange(inputs2, &smallest2, &largest2);
+ *smallest =
+ icmp_->Compare(smallest1, smallest2) < 0 ? smallest1 : smallest2;
+ *largest = icmp_->Compare(largest1, largest2) < 0 ? largest2 : largest1;
+ }
+}
+
+void CompactionPicker::GetRange(const std::vector<CompactionInputFiles>& inputs,
+ InternalKey* smallest,
+ InternalKey* largest) const {
+ InternalKey current_smallest;
+ InternalKey current_largest;
+ bool initialized = false;
+ for (const auto& in : inputs) {
+ if (in.empty()) {
+ continue;
+ }
+ GetRange(in, ¤t_smallest, ¤t_largest);
+ if (!initialized) {
+ *smallest = current_smallest;
+ *largest = current_largest;
+ initialized = true;
+ } else {
+ if (icmp_->Compare(current_smallest, *smallest) < 0) {
+ *smallest = current_smallest;
+ }
+ if (icmp_->Compare(current_largest, *largest) > 0) {
+ *largest = current_largest;
+ }
+ }
+ }
+ assert(initialized);
+}
+
+bool CompactionPicker::ExpandInputsToCleanCut(const std::string& cf_name,
+ VersionStorageInfo* vstorage,
+ CompactionInputFiles* inputs) {
+ // This isn't good compaction
+ assert(!inputs->empty());
+
+ const int level = inputs->level;
+ // GetOverlappingInputs will always do the right thing for level-0.
+ // So we don't need to do any expansion if level == 0.
+ if (level == 0) {
+ return true;
+ }
+
+ InternalKey smallest, largest;
+
+ // Keep expanding inputs until we are sure that there is a "clean cut"
+ // boundary between the files in input and the surrounding files.
+ // This will ensure that no parts of a key are lost during compaction.
+ int hint_index = -1;
+ size_t old_size;
+ do {
+ old_size = inputs->size();
+ GetRange(*inputs, &smallest, &largest);
+ inputs->clear();
+ vstorage->GetOverlappingInputs(level, &smallest, &largest, &inputs->files,
+ hint_index, &hint_index);
+ } while (inputs->size() > old_size);
+
+ // we started off with inputs non-empty and the previous loop only grew
+ // inputs. thus, inputs should be non-empty here
+ assert(!inputs->empty());
+
+ // If, after the expansion, there are files that are already under
+ // compaction, then we must drop/cancel this compaction.
+ if (AreFilesInCompaction(inputs->files)) {
+ return false;
+ }
+ return true;
+}
+
+bool CompactionPicker::RangeOverlapWithCompaction(
+ const Slice& smallest_user_key, const Slice& largest_user_key,
+ int level) const {
+ const Comparator* ucmp = icmp_->user_comparator();
+ for (Compaction* c : compactions_in_progress_) {
+ if (c->output_level() == level &&
+ ucmp->Compare(smallest_user_key, c->GetLargestUserKey()) <= 0 &&
+ ucmp->Compare(largest_user_key, c->GetSmallestUserKey()) >= 0) {
+ // Overlap
+ return true;
+ }
+ }
+ // Did not overlap with any running compaction in level `level`
+ return false;
+}
+
+bool CompactionPicker::FilesRangeOverlapWithCompaction(
+ const std::vector<CompactionInputFiles>& inputs, int level) const {
+ bool is_empty = true;
+ for (auto& in : inputs) {
+ if (!in.empty()) {
+ is_empty = false;
+ break;
+ }
+ }
+ if (is_empty) {
+ // No files in inputs
+ return false;
+ }
+
+ InternalKey smallest, largest;
+ GetRange(inputs, &smallest, &largest);
+ return RangeOverlapWithCompaction(smallest.user_key(), largest.user_key(),
+ level);
+}
+
+// Returns true if any one of specified files are being compacted
+bool CompactionPicker::AreFilesInCompaction(
+ const std::vector<FileMetaData*>& files) {
+ for (size_t i = 0; i < files.size(); i++) {
+ if (files[i]->being_compacted) {
+ return true;
+ }
+ }
+ return false;
+}
+
+Compaction* CompactionPicker::CompactFiles(
+ const CompactionOptions& compact_options,
+ const std::vector<CompactionInputFiles>& input_files, int output_level,
+ VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options,
+ uint32_t output_path_id) {
+ assert(input_files.size());
+
+ // TODO(rven ): we might be able to run concurrent level 0 compaction
+ // if the key ranges of the two compactions do not overlap, but for now
+ // we do not allow it.
+ if ((input_files[0].level == 0) && !level0_compactions_in_progress_.empty()) {
+ return nullptr;
+ }
+ // This compaction output could overlap with a running compaction
+ if (FilesRangeOverlapWithCompaction(input_files, output_level)) {
+ return nullptr;
+ }
+ auto c =
+ new Compaction(vstorage, ioptions_, mutable_cf_options, input_files,
+ output_level, compact_options.output_file_size_limit,
+ mutable_cf_options.max_compaction_bytes, output_path_id,
+ compact_options.compression, /* grandparents */ {}, true);
+
+ // If it's level 0 compaction, make sure we don't execute any other level 0
+ // compactions in parallel
+ RegisterCompaction(c);
+ return c;
+}
+
+Status CompactionPicker::GetCompactionInputsFromFileNumbers(
+ std::vector<CompactionInputFiles>* input_files,
+ std::unordered_set<uint64_t>* input_set, const VersionStorageInfo* vstorage,
+ const CompactionOptions& compact_options) const {
+ if (input_set->size() == 0U) {
+ return Status::InvalidArgument(
+ "Compaction must include at least one file.");
+ }
+ assert(input_files);
+
+ std::vector<CompactionInputFiles> matched_input_files;
+ matched_input_files.resize(vstorage->num_levels());
+ int first_non_empty_level = -1;
+ int last_non_empty_level = -1;
+ // TODO(yhchiang): use a lazy-initialized mapping from
+ // file_number to FileMetaData in Version.
+ for (int level = 0; level < vstorage->num_levels(); ++level) {
+ for (auto file : vstorage->LevelFiles(level)) {
+ auto iter = input_set->find(file->fd.GetNumber());
+ if (iter != input_set->end()) {
+ matched_input_files[level].files.push_back(file);
+ input_set->erase(iter);
+ last_non_empty_level = level;
+ if (first_non_empty_level == -1) {
+ first_non_empty_level = level;
+ }
+ }
+ }
+ }
+
+ if (!input_set->empty()) {
+ std::string message(
+ "Cannot find matched SST files for the following file numbers:");
+ for (auto fn : *input_set) {
+ message += " ";
+ message += ToString(fn);
+ }
+ return Status::InvalidArgument(message);
+ }
+
+ for (int level = first_non_empty_level; level <= last_non_empty_level;
+ ++level) {
+ matched_input_files[level].level = level;
+ input_files->emplace_back(std::move(matched_input_files[level]));
+ }
+
+ return Status::OK();
+}
+
+// Returns true if any one of the parent files are being compacted
+bool CompactionPicker::IsRangeInCompaction(VersionStorageInfo* vstorage,
+ const InternalKey* smallest,
+ const InternalKey* largest,
+ int level, int* level_index) {
+ std::vector<FileMetaData*> inputs;
+ assert(level < NumberLevels());
+
+ vstorage->GetOverlappingInputs(level, smallest, largest, &inputs,
+ *level_index, level_index);
+ return AreFilesInCompaction(inputs);
+}
+
+// Populates the set of inputs of all other levels that overlap with the
+// start level.
+// Now we assume all levels except start level and output level are empty.
+// Will also attempt to expand "start level" if that doesn't expand
+// "output level" or cause "level" to include a file for compaction that has an
+// overlapping user-key with another file.
+// REQUIRES: input_level and output_level are different
+// REQUIRES: inputs->empty() == false
+// Returns false if files on parent level are currently in compaction, which
+// means that we can't compact them
+bool CompactionPicker::SetupOtherInputs(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, CompactionInputFiles* inputs,
+ CompactionInputFiles* output_level_inputs, int* parent_index,
+ int base_index) {
+ assert(!inputs->empty());
+ assert(output_level_inputs->empty());
+ const int input_level = inputs->level;
+ const int output_level = output_level_inputs->level;
+ assert(input_level != output_level);
+
+ // For now, we only support merging two levels, start level and output level.
+ // We need to assert other levels are empty.
+ for (int l = input_level + 1; l < output_level; l++) {
+ assert(vstorage->NumLevelFiles(l) == 0);
+ }
+
+ InternalKey smallest, largest;
+
+ // Get the range one last time.
+ GetRange(*inputs, &smallest, &largest);
+
+ // Populate the set of next-level files (inputs_GetOutputLevelInputs()) to
+ // include in compaction
+ vstorage->GetOverlappingInputs(output_level, &smallest, &largest,
+ &output_level_inputs->files, *parent_index,
+ parent_index);
+ if (AreFilesInCompaction(output_level_inputs->files)) {
+ return false;
+ }
+ if (!output_level_inputs->empty()) {
+ if (!ExpandInputsToCleanCut(cf_name, vstorage, output_level_inputs)) {
+ return false;
+ }
+ }
+
+ // See if we can further grow the number of inputs in "level" without
+ // changing the number of "level+1" files we pick up. We also choose NOT
+ // to expand if this would cause "level" to include some entries for some
+ // user key, while excluding other entries for the same user key. This
+ // can happen when one user key spans multiple files.
+ if (!output_level_inputs->empty()) {
+ const uint64_t limit = mutable_cf_options.max_compaction_bytes;
+ const uint64_t output_level_inputs_size =
+ TotalCompensatedFileSize(output_level_inputs->files);
+ const uint64_t inputs_size = TotalCompensatedFileSize(inputs->files);
+ bool expand_inputs = false;
+
+ CompactionInputFiles expanded_inputs;
+ expanded_inputs.level = input_level;
+ // Get closed interval of output level
+ InternalKey all_start, all_limit;
+ GetRange(*inputs, *output_level_inputs, &all_start, &all_limit);
+ bool try_overlapping_inputs = true;
+ vstorage->GetOverlappingInputs(input_level, &all_start, &all_limit,
+ &expanded_inputs.files, base_index, nullptr);
+ uint64_t expanded_inputs_size =
+ TotalCompensatedFileSize(expanded_inputs.files);
+ if (!ExpandInputsToCleanCut(cf_name, vstorage, &expanded_inputs)) {
+ try_overlapping_inputs = false;
+ }
+ if (try_overlapping_inputs && expanded_inputs.size() > inputs->size() &&
+ output_level_inputs_size + expanded_inputs_size < limit &&
+ !AreFilesInCompaction(expanded_inputs.files)) {
+ InternalKey new_start, new_limit;
+ GetRange(expanded_inputs, &new_start, &new_limit);
+ CompactionInputFiles expanded_output_level_inputs;
+ expanded_output_level_inputs.level = output_level;
+ vstorage->GetOverlappingInputs(output_level, &new_start, &new_limit,
+ &expanded_output_level_inputs.files,
+ *parent_index, parent_index);
+ assert(!expanded_output_level_inputs.empty());
+ if (!AreFilesInCompaction(expanded_output_level_inputs.files) &&
+ ExpandInputsToCleanCut(cf_name, vstorage,
+ &expanded_output_level_inputs) &&
+ expanded_output_level_inputs.size() == output_level_inputs->size()) {
+ expand_inputs = true;
+ }
+ }
+ if (!expand_inputs) {
+ vstorage->GetCleanInputsWithinInterval(input_level, &all_start,
+ &all_limit, &expanded_inputs.files,
+ base_index, nullptr);
+ expanded_inputs_size = TotalCompensatedFileSize(expanded_inputs.files);
+ if (expanded_inputs.size() > inputs->size() &&
+ output_level_inputs_size + expanded_inputs_size < limit &&
+ !AreFilesInCompaction(expanded_inputs.files)) {
+ expand_inputs = true;
+ }
+ }
+ if (expand_inputs) {
+ ROCKS_LOG_INFO(ioptions_.info_log,
+ "[%s] Expanding@%d %" ROCKSDB_PRIszt "+%" ROCKSDB_PRIszt
+ "(%" PRIu64 "+%" PRIu64 " bytes) to %" ROCKSDB_PRIszt
+ "+%" ROCKSDB_PRIszt " (%" PRIu64 "+%" PRIu64 "bytes)\n",
+ cf_name.c_str(), input_level, inputs->size(),
+ output_level_inputs->size(), inputs_size,
+ output_level_inputs_size, expanded_inputs.size(),
+ output_level_inputs->size(), expanded_inputs_size,
+ output_level_inputs_size);
+ inputs->files = expanded_inputs.files;
+ }
+ }
+ return true;
+}
+
+void CompactionPicker::GetGrandparents(
+ VersionStorageInfo* vstorage, const CompactionInputFiles& inputs,
+ const CompactionInputFiles& output_level_inputs,
+ std::vector<FileMetaData*>* grandparents) {
+ InternalKey start, limit;
+ GetRange(inputs, output_level_inputs, &start, &limit);
+ // Compute the set of grandparent files that overlap this compaction
+ // (parent == level+1; grandparent == level+2)
+ if (output_level_inputs.level + 1 < NumberLevels()) {
+ vstorage->GetOverlappingInputs(output_level_inputs.level + 1, &start,
+ &limit, grandparents);
+ }
+}
+
+Compaction* CompactionPicker::CompactRange(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, int input_level, int output_level,
+ uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
+ InternalKey** compaction_end, bool* manual_conflict) {
+ // CompactionPickerFIFO has its own implementation of compact range
+ assert(ioptions_.compaction_style != kCompactionStyleFIFO);
+
+ if (input_level == ColumnFamilyData::kCompactAllLevels) {
+ assert(ioptions_.compaction_style == kCompactionStyleUniversal);
+
+ // Universal compaction with more than one level always compacts all the
+ // files together to the last level.
+ assert(vstorage->num_levels() > 1);
+ // DBImpl::CompactRange() set output level to be the last level
+ if (ioptions_.allow_ingest_behind) {
+ assert(output_level == vstorage->num_levels() - 2);
+ } else {
+ assert(output_level == vstorage->num_levels() - 1);
+ }
+ // DBImpl::RunManualCompaction will make full range for universal compaction
+ assert(begin == nullptr);
+ assert(end == nullptr);
+ *compaction_end = nullptr;
+
+ int start_level = 0;
+ for (; start_level < vstorage->num_levels() &&
+ vstorage->NumLevelFiles(start_level) == 0;
+ start_level++) {
+ }
+ if (start_level == vstorage->num_levels()) {
+ return nullptr;
+ }
+
+ if ((start_level == 0) && (!level0_compactions_in_progress_.empty())) {
+ *manual_conflict = true;
+ // Only one level 0 compaction allowed
+ return nullptr;
+ }
+
+ std::vector<CompactionInputFiles> inputs(vstorage->num_levels() -
+ start_level);
+ for (int level = start_level; level < vstorage->num_levels(); level++) {
+ inputs[level - start_level].level = level;
+ auto& files = inputs[level - start_level].files;
+ for (FileMetaData* f : vstorage->LevelFiles(level)) {
+ files.push_back(f);
+ }
+ if (AreFilesInCompaction(files)) {
+ *manual_conflict = true;
+ return nullptr;
+ }
+ }
+
+ // 2 non-exclusive manual compactions could run at the same time producing
+ // overlaping outputs in the same level.
+ if (FilesRangeOverlapWithCompaction(inputs, output_level)) {
+ // This compaction output could potentially conflict with the output
+ // of a currently running compaction, we cannot run it.
+ *manual_conflict = true;
+ return nullptr;
+ }
+
+ Compaction* c = new Compaction(
+ vstorage, ioptions_, mutable_cf_options, std::move(inputs),
+ output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
+ /* max_compaction_bytes */ LLONG_MAX, output_path_id,
+ GetCompressionType(ioptions_, vstorage, mutable_cf_options,
+ output_level, 1),
+ /* grandparents */ {}, /* is manual */ true);
+ RegisterCompaction(c);
+ return c;
+ }
+
+ CompactionInputFiles inputs;
+ inputs.level = input_level;
+ bool covering_the_whole_range = true;
+
+ // All files are 'overlapping' in universal style compaction.
+ // We have to compact the entire range in one shot.
+ if (ioptions_.compaction_style == kCompactionStyleUniversal) {
+ begin = nullptr;
+ end = nullptr;
+ }
+
+ vstorage->GetOverlappingInputs(input_level, begin, end, &inputs.files);
+ if (inputs.empty()) {
+ return nullptr;
+ }
+
+ if ((input_level == 0) && (!level0_compactions_in_progress_.empty())) {
+ // Only one level 0 compaction allowed
+ TEST_SYNC_POINT("CompactionPicker::CompactRange:Conflict");
+ *manual_conflict = true;
+ return nullptr;
+ }
+
+ // Avoid compacting too much in one shot in case the range is large.
+ // But we cannot do this for level-0 since level-0 files can overlap
+ // and we must not pick one file and drop another older file if the
+ // two files overlap.
+ if (input_level > 0) {
+ const uint64_t limit = mutable_cf_options.max_compaction_bytes;
+ uint64_t total = 0;
+ for (size_t i = 0; i + 1 < inputs.size(); ++i) {
+ uint64_t s = inputs[i]->compensated_file_size;
+ total += s;
+ if (total >= limit) {
+ **compaction_end = inputs[i + 1]->smallest;
+ covering_the_whole_range = false;
+ inputs.files.resize(i + 1);
+ break;
+ }
+ }
+ }
+ assert(output_path_id < static_cast<uint32_t>(ioptions_.db_paths.size()));
+
+ if (ExpandInputsToCleanCut(cf_name, vstorage, &inputs) == false) {
+ // manual compaction is now multi-threaded, so it can
+ // happen that ExpandWhileOverlapping fails
+ // we handle it higher in RunManualCompaction
+ *manual_conflict = true;
+ return nullptr;
+ }
+
+ if (covering_the_whole_range) {
+ *compaction_end = nullptr;
+ }
+
+ CompactionInputFiles output_level_inputs;
+ if (output_level == ColumnFamilyData::kCompactToBaseLevel) {
+ assert(input_level == 0);
+ output_level = vstorage->base_level();
+ assert(output_level > 0);
+ }
+ output_level_inputs.level = output_level;
+ if (input_level != output_level) {
+ int parent_index = -1;
+ if (!SetupOtherInputs(cf_name, mutable_cf_options, vstorage, &inputs,
+ &output_level_inputs, &parent_index, -1)) {
+ // manual compaction is now multi-threaded, so it can
+ // happen that SetupOtherInputs fails
+ // we handle it higher in RunManualCompaction
+ *manual_conflict = true;
+ return nullptr;
+ }
+ }
+
+ std::vector<CompactionInputFiles> compaction_inputs({inputs});
+ if (!output_level_inputs.empty()) {
+ compaction_inputs.push_back(output_level_inputs);
+ }
+ for (size_t i = 0; i < compaction_inputs.size(); i++) {
+ if (AreFilesInCompaction(compaction_inputs[i].files)) {
+ *manual_conflict = true;
+ return nullptr;
+ }
+ }
+
+ // 2 non-exclusive manual compactions could run at the same time producing
+ // overlaping outputs in the same level.
+ if (FilesRangeOverlapWithCompaction(compaction_inputs, output_level)) {
+ // This compaction output could potentially conflict with the output
+ // of a currently running compaction, we cannot run it.
+ *manual_conflict = true;
+ return nullptr;
+ }
+
+ std::vector<FileMetaData*> grandparents;
+ GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents);
+ Compaction* compaction = new Compaction(
+ vstorage, ioptions_, mutable_cf_options, std::move(compaction_inputs),
+ output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
+ mutable_cf_options.max_compaction_bytes, output_path_id,
+ GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level,
+ vstorage->base_level()),
+ std::move(grandparents), /* is manual compaction */ true);
+
+ TEST_SYNC_POINT_CALLBACK("CompactionPicker::CompactRange:Return", compaction);
+ RegisterCompaction(compaction);
+
+ // Creating a compaction influences the compaction score because the score
+ // takes running compactions into account (by skipping files that are already
+ // being compacted). Since we just changed compaction score, we recalculate it
+ // here
+ vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options);
+
+ return compaction;
+}
+
+#ifndef ROCKSDB_LITE
+namespace {
+// Test whether two files have overlapping key-ranges.
+bool HaveOverlappingKeyRanges(const Comparator* c, const SstFileMetaData& a,
+ const SstFileMetaData& b) {
+ if (c->Compare(a.smallestkey, b.smallestkey) >= 0) {
+ if (c->Compare(a.smallestkey, b.largestkey) <= 0) {
+ // b.smallestkey <= a.smallestkey <= b.largestkey
+ return true;
+ }
+ } else if (c->Compare(a.largestkey, b.smallestkey) >= 0) {
+ // a.smallestkey < b.smallestkey <= a.largestkey
+ return true;
+ }
+ if (c->Compare(a.largestkey, b.largestkey) <= 0) {
+ if (c->Compare(a.largestkey, b.smallestkey) >= 0) {
+ // b.smallestkey <= a.largestkey <= b.largestkey
+ return true;
+ }
+ } else if (c->Compare(a.smallestkey, b.largestkey) <= 0) {
+ // a.smallestkey <= b.largestkey < a.largestkey
+ return true;
+ }
+ return false;
+}
+} // namespace
+
+Status CompactionPicker::SanitizeCompactionInputFilesForAllLevels(
+ std::unordered_set<uint64_t>* input_files,
+ const ColumnFamilyMetaData& cf_meta, const int output_level) const {
+ auto& levels = cf_meta.levels;
+ auto comparator = icmp_->user_comparator();
+
+ // TODO(yhchiang): If there is any input files of L1 or up and there
+ // is at least one L0 files. All L0 files older than the L0 file needs
+ // to be included. Otherwise, it is a false conditoin
+
+ // TODO(yhchiang): add is_adjustable to CompactionOptions
+
+ // the smallest and largest key of the current compaction input
+ std::string smallestkey;
+ std::string largestkey;
+ // a flag for initializing smallest and largest key
+ bool is_first = false;
+ const int kNotFound = -1;
+
+ // For each level, it does the following things:
+ // 1. Find the first and the last compaction input files
+ // in the current level.
+ // 2. Include all files between the first and the last
+ // compaction input files.
+ // 3. Update the compaction key-range.
+ // 4. For all remaining levels, include files that have
+ // overlapping key-range with the compaction key-range.
+ for (int l = 0; l <= output_level; ++l) {
+ auto& current_files = levels[l].files;
+ int first_included = static_cast<int>(current_files.size());
+ int last_included = kNotFound;
+
+ // identify the first and the last compaction input files
+ // in the current level.
+ for (size_t f = 0; f < current_files.size(); ++f) {
+ if (input_files->find(TableFileNameToNumber(current_files[f].name)) !=
+ input_files->end()) {
+ first_included = std::min(first_included, static_cast<int>(f));
+ last_included = std::max(last_included, static_cast<int>(f));
+ if (is_first == false) {
+ smallestkey = current_files[f].smallestkey;
+ largestkey = current_files[f].largestkey;
+ is_first = true;
+ }
+ }
+ }
+ if (last_included == kNotFound) {
+ continue;
+ }
+
+ if (l != 0) {
+ // expend the compaction input of the current level if it
+ // has overlapping key-range with other non-compaction input
+ // files in the same level.
+ while (first_included > 0) {
+ if (comparator->Compare(current_files[first_included - 1].largestkey,
+ current_files[first_included].smallestkey) <
+ 0) {
+ break;
+ }
+ first_included--;
+ }
+
+ while (last_included < static_cast<int>(current_files.size()) - 1) {
+ if (comparator->Compare(current_files[last_included + 1].smallestkey,
+ current_files[last_included].largestkey) > 0) {
+ break;
+ }
+ last_included++;
+ }
+ }
+
+ // include all files between the first and the last compaction input files.
+ for (int f = first_included; f <= last_included; ++f) {
+ if (current_files[f].being_compacted) {
+ return Status::Aborted("Necessary compaction input file " +
+ current_files[f].name +
+ " is currently being compacted.");
+ }
+ input_files->insert(TableFileNameToNumber(current_files[f].name));
+ }
+
+ // update smallest and largest key
+ if (l == 0) {
+ for (int f = first_included; f <= last_included; ++f) {
+ if (comparator->Compare(smallestkey, current_files[f].smallestkey) >
+ 0) {
+ smallestkey = current_files[f].smallestkey;
+ }
+ if (comparator->Compare(largestkey, current_files[f].largestkey) < 0) {
+ largestkey = current_files[f].largestkey;
+ }
+ }
+ } else {
+ if (comparator->Compare(smallestkey,
+ current_files[first_included].smallestkey) > 0) {
+ smallestkey = current_files[first_included].smallestkey;
+ }
+ if (comparator->Compare(largestkey,
+ current_files[last_included].largestkey) < 0) {
+ largestkey = current_files[last_included].largestkey;
+ }
+ }
+
+ SstFileMetaData aggregated_file_meta;
+ aggregated_file_meta.smallestkey = smallestkey;
+ aggregated_file_meta.largestkey = largestkey;
+
+ // For all lower levels, include all overlapping files.
+ // We need to add overlapping files from the current level too because even
+ // if there no input_files in level l, we would still need to add files
+ // which overlap with the range containing the input_files in levels 0 to l
+ // Level 0 doesn't need to be handled this way because files are sorted by
+ // time and not by key
+ for (int m = std::max(l, 1); m <= output_level; ++m) {
+ for (auto& next_lv_file : levels[m].files) {
+ if (HaveOverlappingKeyRanges(comparator, aggregated_file_meta,
+ next_lv_file)) {
+ if (next_lv_file.being_compacted) {
+ return Status::Aborted(
+ "File " + next_lv_file.name +
+ " that has overlapping key range with one of the compaction "
+ " input file is currently being compacted.");
+ }
+ input_files->insert(TableFileNameToNumber(next_lv_file.name));
+ }
+ }
+ }
+ }
+ return Status::OK();
+}
+
+Status CompactionPicker::SanitizeCompactionInputFiles(
+ std::unordered_set<uint64_t>* input_files,
+ const ColumnFamilyMetaData& cf_meta, const int output_level) const {
+ assert(static_cast<int>(cf_meta.levels.size()) - 1 ==
+ cf_meta.levels[cf_meta.levels.size() - 1].level);
+ if (output_level >= static_cast<int>(cf_meta.levels.size())) {
+ return Status::InvalidArgument(
+ "Output level for column family " + cf_meta.name +
+ " must between [0, " +
+ ToString(cf_meta.levels[cf_meta.levels.size() - 1].level) + "].");
+ }
+
+ if (output_level > MaxOutputLevel()) {
+ return Status::InvalidArgument(
+ "Exceed the maximum output level defined by "
+ "the current compaction algorithm --- " +
+ ToString(MaxOutputLevel()));
+ }
+
+ if (output_level < 0) {
+ return Status::InvalidArgument("Output level cannot be negative.");
+ }
+
+ if (input_files->size() == 0) {
+ return Status::InvalidArgument(
+ "A compaction must contain at least one file.");
+ }
+
+ Status s = SanitizeCompactionInputFilesForAllLevels(input_files, cf_meta,
+ output_level);
+
+ if (!s.ok()) {
+ return s;
+ }
+
+ // for all input files, check whether the file number matches
+ // any currently-existing files.
+ for (auto file_num : *input_files) {
+ bool found = false;
+ for (auto level_meta : cf_meta.levels) {
+ for (auto file_meta : level_meta.files) {
+ if (file_num == TableFileNameToNumber(file_meta.name)) {
+ if (file_meta.being_compacted) {
+ return Status::Aborted("Specified compaction input file " +
+ MakeTableFileName("", file_num) +
+ " is already being compacted.");
+ }
+ found = true;
+ break;
+ }
+ }
+ if (found) {
+ break;
+ }
+ }
+ if (!found) {
+ return Status::InvalidArgument(
+ "Specified compaction input file " + MakeTableFileName("", file_num) +
+ " does not exist in column family " + cf_meta.name + ".");
+ }
+ }
+
+ return Status::OK();
+}
+#endif // !ROCKSDB_LITE
+
+void CompactionPicker::RegisterCompaction(Compaction* c) {
+ if (c == nullptr) {
+ return;
+ }
+ assert(ioptions_.compaction_style != kCompactionStyleLevel ||
+ c->output_level() == 0 ||
+ !FilesRangeOverlapWithCompaction(*c->inputs(), c->output_level()));
+ if (c->start_level() == 0 ||
+ ioptions_.compaction_style == kCompactionStyleUniversal) {
+ level0_compactions_in_progress_.insert(c);
+ }
+ compactions_in_progress_.insert(c);
+}
+
+void CompactionPicker::UnregisterCompaction(Compaction* c) {
+ if (c == nullptr) {
+ return;
+ }
+ if (c->start_level() == 0 ||
+ ioptions_.compaction_style == kCompactionStyleUniversal) {
+ level0_compactions_in_progress_.erase(c);
+ }
+ compactions_in_progress_.erase(c);
+}
+
+bool LevelCompactionPicker::NeedsCompaction(
+ const VersionStorageInfo* vstorage) const {
+ if (!vstorage->FilesMarkedForCompaction().empty()) {
+ return true;
+ }
+ for (int i = 0; i <= vstorage->MaxInputLevel(); i++) {
+ if (vstorage->CompactionScore(i) >= 1) {
+ return true;
+ }
+ }
+ return false;
+}
+
+namespace {
+// A class to build a leveled compaction step-by-step.
+class LevelCompactionBuilder {
+ public:
+ LevelCompactionBuilder(const std::string& cf_name,
+ VersionStorageInfo* vstorage,
+ CompactionPicker* compaction_picker,
+ LogBuffer* log_buffer,
+ const MutableCFOptions& mutable_cf_options,
+ const ImmutableCFOptions& ioptions)
+ : cf_name_(cf_name),
+ vstorage_(vstorage),
+ compaction_picker_(compaction_picker),
+ log_buffer_(log_buffer),
+ mutable_cf_options_(mutable_cf_options),
+ ioptions_(ioptions) {}
+
+ // Pick and return a compaction.
+ Compaction* PickCompaction();
+
+ // Pick the initial files to compact to the next level. (or together
+ // in Intra-L0 compactions)
+ void SetupInitialFiles();
+
+ // If the initial files are from L0 level, pick other L0
+ // files if needed.
+ bool SetupOtherL0FilesIfNeeded();
+
+ // Based on initial files, setup other files need to be compacted
+ // in this compaction, accordingly.
+ bool SetupOtherInputsIfNeeded();
+
+ Compaction* GetCompaction();
+
+ // For the specfied level, pick a file that we want to compact.
+ // Returns false if there is no file to compact.
+ // If it returns true, inputs->files.size() will be exactly one.
+ // If level is 0 and there is already a compaction on that level, this
+ // function will return false.
+ bool PickFileToCompact();
+
+ // For L0->L0, picks the longest span of files that aren't currently
+ // undergoing compaction for which work-per-deleted-file decreases. The span
+ // always starts from the newest L0 file.
+ //
+ // Intra-L0 compaction is independent of all other files, so it can be
+ // performed even when L0->base_level compactions are blocked.
+ //
+ // Returns true if `inputs` is populated with a span of files to be compacted;
+ // otherwise, returns false.
+ bool PickIntraL0Compaction();
+
+ // If there is any file marked for compaction, put put it into inputs.
+ void PickFilesMarkedForCompaction();
+
+ const std::string& cf_name_;
+ VersionStorageInfo* vstorage_;
+ CompactionPicker* compaction_picker_;
+ LogBuffer* log_buffer_;
+ int start_level_ = -1;
+ int output_level_ = -1;
+ int parent_index_ = -1;
+ int base_index_ = -1;
+ double start_level_score_ = 0;
+ bool is_manual_ = false;
+ CompactionInputFiles start_level_inputs_;
+ std::vector<CompactionInputFiles> compaction_inputs_;
+ CompactionInputFiles output_level_inputs_;
+ std::vector<FileMetaData*> grandparents_;
+ CompactionReason compaction_reason_ = CompactionReason::kUnknown;
+
+ const MutableCFOptions& mutable_cf_options_;
+ const ImmutableCFOptions& ioptions_;
+ // Pick a path ID to place a newly generated file, with its level
+ static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& mutable_cf_options,
+ int level);
+
+ static const int kMinFilesForIntraL0Compaction = 4;
+};
+
+void LevelCompactionBuilder::PickFilesMarkedForCompaction() {
+ if (vstorage_->FilesMarkedForCompaction().empty()) {
+ return;
+ }
+
+ auto continuation = [&](std::pair<int, FileMetaData*> level_file) {
+ // If it's being compacted it has nothing to do here.
+ // If this assert() fails that means that some function marked some
+ // files as being_compacted, but didn't call ComputeCompactionScore()
+ assert(!level_file.second->being_compacted);
+ start_level_ = level_file.first;
+ output_level_ =
+ (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1;
+
+ if (start_level_ == 0 &&
+ !compaction_picker_->level0_compactions_in_progress()->empty()) {
+ return false;
+ }
+
+ start_level_inputs_.files = {level_file.second};
+ start_level_inputs_.level = start_level_;
+ return compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+ &start_level_inputs_);
+ };
+
+ // take a chance on a random file first
+ Random64 rnd(/* seed */ reinterpret_cast<uint64_t>(vstorage_));
+ size_t random_file_index = static_cast<size_t>(rnd.Uniform(
+ static_cast<uint64_t>(vstorage_->FilesMarkedForCompaction().size())));
+
+ if (continuation(vstorage_->FilesMarkedForCompaction()[random_file_index])) {
+ // found the compaction!
+ return;
+ }
+
+ for (auto& level_file : vstorage_->FilesMarkedForCompaction()) {
+ if (continuation(level_file)) {
+ // found the compaction!
+ return;
+ }
+ }
+ start_level_inputs_.files.clear();
+}
+
+void LevelCompactionBuilder::SetupInitialFiles() {
+ // Find the compactions by size on all levels.
+ bool skipped_l0_to_base = false;
+ for (int i = 0; i < compaction_picker_->NumberLevels() - 1; i++) {
+ start_level_score_ = vstorage_->CompactionScore(i);
+ start_level_ = vstorage_->CompactionScoreLevel(i);
+ assert(i == 0 || start_level_score_ <= vstorage_->CompactionScore(i - 1));
+ if (start_level_score_ >= 1) {
+ if (skipped_l0_to_base && start_level_ == vstorage_->base_level()) {
+ // If L0->base_level compaction is pending, don't schedule further
+ // compaction from base level. Otherwise L0->base_level compaction
+ // may starve.
+ continue;
+ }
+ output_level_ =
+ (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1;
+ if (PickFileToCompact()) {
+ // found the compaction!
+ if (start_level_ == 0) {
+ // L0 score = `num L0 files` / `level0_file_num_compaction_trigger`
+ compaction_reason_ = CompactionReason::kLevelL0FilesNum;
+ } else {
+ // L1+ score = `Level files size` / `MaxBytesForLevel`
+ compaction_reason_ = CompactionReason::kLevelMaxLevelSize;
+ }
+ break;
+ } else {
+ // didn't find the compaction, clear the inputs
+ start_level_inputs_.clear();
+ if (start_level_ == 0) {
+ skipped_l0_to_base = true;
+ // L0->base_level may be blocked due to ongoing L0->base_level
+ // compactions. It may also be blocked by an ongoing compaction from
+ // base_level downwards.
+ //
+ // In these cases, to reduce L0 file count and thus reduce likelihood
+ // of write stalls, we can attempt compacting a span of files within
+ // L0.
+ if (PickIntraL0Compaction()) {
+ output_level_ = 0;
+ compaction_reason_ = CompactionReason::kLevelL0FilesNum;
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ // if we didn't find a compaction, check if there are any files marked for
+ // compaction
+ if (start_level_inputs_.empty()) {
+ is_manual_ = true;
+ parent_index_ = base_index_ = -1;
+ PickFilesMarkedForCompaction();
+ if (!start_level_inputs_.empty()) {
+ compaction_reason_ = CompactionReason::kFilesMarkedForCompaction;
+ }
+ }
+}
+
+bool LevelCompactionBuilder::SetupOtherL0FilesIfNeeded() {
+ if (start_level_ == 0 && output_level_ != 0) {
+ // Two level 0 compaction won't run at the same time, so don't need to worry
+ // about files on level 0 being compacted.
+ assert(compaction_picker_->level0_compactions_in_progress()->empty());
+ InternalKey smallest, largest;
+ compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest);
+ // Note that the next call will discard the file we placed in
+ // c->inputs_[0] earlier and replace it with an overlapping set
+ // which will include the picked file.
+ start_level_inputs_.files.clear();
+ vstorage_->GetOverlappingInputs(0, &smallest, &largest,
+ &start_level_inputs_.files);
+
+ // If we include more L0 files in the same compaction run it can
+ // cause the 'smallest' and 'largest' key to get extended to a
+ // larger range. So, re-invoke GetRange to get the new key range
+ compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest);
+ if (compaction_picker_->IsRangeInCompaction(
+ vstorage_, &smallest, &largest, output_level_, &parent_index_)) {
+ return false;
+ }
+ }
+ assert(!start_level_inputs_.files.empty());
+
+ return true;
+}
+
+bool LevelCompactionBuilder::SetupOtherInputsIfNeeded() {
+ // Setup input files from output level. For output to L0, we only compact
+ // spans of files that do not interact with any pending compactions, so don't
+ // need to consider other levels.
+ if (output_level_ != 0) {
+ output_level_inputs_.level = output_level_;
+ if (!compaction_picker_->SetupOtherInputs(
+ cf_name_, mutable_cf_options_, vstorage_, &start_level_inputs_,
+ &output_level_inputs_, &parent_index_, base_index_)) {
+ return false;
+ }
+
+ compaction_inputs_.push_back(start_level_inputs_);
+ if (!output_level_inputs_.empty()) {
+ compaction_inputs_.push_back(output_level_inputs_);
+ }
+
+ // In some edge cases we could pick a compaction that will be compacting
+ // a key range that overlap with another running compaction, and both
+ // of them have the same output level. This could happen if
+ // (1) we are running a non-exclusive manual compaction
+ // (2) AddFile ingest a new file into the LSM tree
+ // We need to disallow this from happening.
+ if (compaction_picker_->FilesRangeOverlapWithCompaction(compaction_inputs_,
+ output_level_)) {
+ // This compaction output could potentially conflict with the output
+ // of a currently running compaction, we cannot run it.
+ return false;
+ }
+ compaction_picker_->GetGrandparents(vstorage_, start_level_inputs_,
+ output_level_inputs_, &grandparents_);
+ } else {
+ compaction_inputs_.push_back(start_level_inputs_);
+ }
+ return true;
+}
+
+Compaction* LevelCompactionBuilder::PickCompaction() {
+ // Pick up the first file to start compaction. It may have been extended
+ // to a clean cut.
+ SetupInitialFiles();
+ if (start_level_inputs_.empty()) {
+ return nullptr;
+ }
+ assert(start_level_ >= 0 && output_level_ >= 0);
+
+ // If it is a L0 -> base level compaction, we need to set up other L0
+ // files if needed.
+ if (!SetupOtherL0FilesIfNeeded()) {
+ return nullptr;
+ }
+
+ // Pick files in the output level and expand more files in the start level
+ // if needed.
+ if (!SetupOtherInputsIfNeeded()) {
+ return nullptr;
+ }
+
+ // Form a compaction object containing the files we picked.
+ Compaction* c = GetCompaction();
+
+ TEST_SYNC_POINT_CALLBACK("LevelCompactionPicker::PickCompaction:Return", c);
+
+ return c;
+}
+
+Compaction* LevelCompactionBuilder::GetCompaction() {
+ auto c = new Compaction(
+ vstorage_, ioptions_, mutable_cf_options_, std::move(compaction_inputs_),
+ output_level_, mutable_cf_options_.MaxFileSizeForLevel(output_level_),
+ mutable_cf_options_.max_compaction_bytes,
+ GetPathId(ioptions_, mutable_cf_options_, output_level_),
+ GetCompressionType(ioptions_, vstorage_, mutable_cf_options_,
+ output_level_, vstorage_->base_level()),
+ std::move(grandparents_), is_manual_, start_level_score_,
+ false /* deletion_compaction */, compaction_reason_);
+
+ // If it's level 0 compaction, make sure we don't execute any other level 0
+ // compactions in parallel
+ compaction_picker_->RegisterCompaction(c);
+
+ // Creating a compaction influences the compaction score because the score
+ // takes running compactions into account (by skipping files that are already
+ // being compacted). Since we just changed compaction score, we recalculate it
+ // here
+ vstorage_->ComputeCompactionScore(ioptions_, mutable_cf_options_);
+ return c;
+}
+
+/*
+ * Find the optimal path to place a file
+ * Given a level, finds the path where levels up to it will fit in levels
+ * up to and including this path
+ */
+uint32_t LevelCompactionBuilder::GetPathId(
+ const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& mutable_cf_options, int level) {
+ uint32_t p = 0;
+ assert(!ioptions.db_paths.empty());
+
+ // size remaining in the most recent path
+ uint64_t current_path_size = ioptions.db_paths[0].target_size;
+
+ uint64_t level_size;
+ int cur_level = 0;
+
+ level_size = mutable_cf_options.max_bytes_for_level_base;
+
+ // Last path is the fallback
+ while (p < ioptions.db_paths.size() - 1) {
+ if (level_size <= current_path_size) {
+ if (cur_level == level) {
+ // Does desired level fit in this path?
+ return p;
+ } else {
+ current_path_size -= level_size;
+ level_size = static_cast<uint64_t>(
+ level_size * mutable_cf_options.max_bytes_for_level_multiplier);
+ cur_level++;
+ continue;
+ }
+ }
+ p++;
+ current_path_size = ioptions.db_paths[p].target_size;
+ }
+ return p;
+}
+
+bool LevelCompactionBuilder::PickFileToCompact() {
+ // level 0 files are overlapping. So we cannot pick more
+ // than one concurrent compactions at this level. This
+ // could be made better by looking at key-ranges that are
+ // being compacted at level 0.
+ if (start_level_ == 0 &&
+ !compaction_picker_->level0_compactions_in_progress()->empty()) {
+ TEST_SYNC_POINT("LevelCompactionPicker::PickCompactionBySize:0");
+ return false;
+ }
+
+ start_level_inputs_.clear();
+
+ assert(start_level_ >= 0);
+
+ // Pick the largest file in this level that is not already
+ // being compacted
+ const std::vector<int>& file_size =
+ vstorage_->FilesByCompactionPri(start_level_);
+ const std::vector<FileMetaData*>& level_files =
+ vstorage_->LevelFiles(start_level_);
+
+ unsigned int cmp_idx;
+ for (cmp_idx = vstorage_->NextCompactionIndex(start_level_);
+ cmp_idx < file_size.size(); cmp_idx++) {
+ int index = file_size[cmp_idx];
+ auto* f = level_files[index];
+
+ // do not pick a file to compact if it is being compacted
+ // from n-1 level.
+ if (f->being_compacted) {
+ continue;
+ }
+
+ start_level_inputs_.files.push_back(f);
+ start_level_inputs_.level = start_level_;
+ if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+ &start_level_inputs_) ||
+ compaction_picker_->FilesRangeOverlapWithCompaction(
+ {start_level_inputs_}, output_level_)) {
+ // A locked (pending compaction) input-level file was pulled in due to
+ // user-key overlap.
+ start_level_inputs_.clear();
+ continue;
+ }
+
+ // Now that input level is fully expanded, we check whether any output files
+ // are locked due to pending compaction.
+ //
+ // Note we rely on ExpandInputsToCleanCut() to tell us whether any output-
+ // level files are locked, not just the extra ones pulled in for user-key
+ // overlap.
+ InternalKey smallest, largest;
+ compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest);
+ CompactionInputFiles output_level_inputs;
+ output_level_inputs.level = output_level_;
+ vstorage_->GetOverlappingInputs(output_level_, &smallest, &largest,
+ &output_level_inputs.files);
+ if (!output_level_inputs.empty() &&
+ !compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+ &output_level_inputs)) {
+ start_level_inputs_.clear();
+ continue;
+ }
+ base_index_ = index;
+ break;
+ }
+
+ // store where to start the iteration in the next call to PickCompaction
+ vstorage_->SetNextCompactionIndex(start_level_, cmp_idx);
+
+ return start_level_inputs_.size() > 0;
+}
+
+bool LevelCompactionBuilder::PickIntraL0Compaction() {
+ start_level_inputs_.clear();
+ const std::vector<FileMetaData*>& level_files =
+ vstorage_->LevelFiles(0 /* level */);
+ if (level_files.size() <
+ static_cast<size_t>(
+ mutable_cf_options_.level0_file_num_compaction_trigger + 2) ||
+ level_files[0]->being_compacted) {
+ // If L0 isn't accumulating much files beyond the regular trigger, don't
+ // resort to L0->L0 compaction yet.
+ return false;
+ }
+ return FindIntraL0Compaction(level_files, kMinFilesForIntraL0Compaction,
+ port::kMaxUint64, &start_level_inputs_);
+}
+} // namespace
+
+Compaction* LevelCompactionPicker::PickCompaction(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
+ LevelCompactionBuilder builder(cf_name, vstorage, this, log_buffer,
+ mutable_cf_options, ioptions_);
+ return builder.PickCompaction();
+}
+
+#ifndef ROCKSDB_LITE
+bool FIFOCompactionPicker::NeedsCompaction(
+ const VersionStorageInfo* vstorage) const {
+ const int kLevel0 = 0;
+ return vstorage->CompactionScore(kLevel0) >= 1;
+}
+
+namespace {
+uint64_t GetTotalFilesSize(
+ const std::vector<FileMetaData*>& files) {
+ uint64_t total_size = 0;
+ for (const auto& f : files) {
+ total_size += f->fd.file_size;
+ }
+ return total_size;
+}
+} // anonymous namespace
+
+Compaction* FIFOCompactionPicker::PickTTLCompaction(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
+ assert(ioptions_.compaction_options_fifo.ttl > 0);
+
+ const int kLevel0 = 0;
+ const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
+ uint64_t total_size = GetTotalFilesSize(level_files);
+
+ int64_t _current_time;
+ auto status = ioptions_.env->GetCurrentTime(&_current_time);
+ if (!status.ok()) {
+ ROCKS_LOG_BUFFER(log_buffer,
+ "[%s] FIFO compaction: Couldn't get current time: %s. "
+ "Not doing compactions based on TTL. ",
+ cf_name.c_str(), status.ToString().c_str());
+ return nullptr;
+ }
+ const uint64_t current_time = static_cast<uint64_t>(_current_time);
+
+ std::vector<CompactionInputFiles> inputs;
+ inputs.emplace_back();
+ inputs[0].level = 0;
+
+ for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
+ auto f = *ritr;
+ if (f->fd.table_reader != nullptr &&
+ f->fd.table_reader->GetTableProperties() != nullptr) {
+ auto creation_time =
+ f->fd.table_reader->GetTableProperties()->creation_time;
+ if (creation_time == 0 ||
+ creation_time >=
+ (current_time - ioptions_.compaction_options_fifo.ttl)) {
+ break;
+ }
+ total_size -= f->compensated_file_size;
+ inputs[0].files.push_back(f);
+ }
+ }
+
+ // Return a nullptr and proceed to size-based FIFO compaction if:
+ // 1. there are no files older than ttl OR
+ // 2. there are a few files older than ttl, but deleting them will not bring
+ // the total size to be less than max_table_files_size threshold.
+ if (inputs[0].files.empty() ||
+ total_size > ioptions_.compaction_options_fifo.max_table_files_size) {
+ return nullptr;
+ }
+
+ for (const auto& f : inputs[0].files) {
+ ROCKS_LOG_BUFFER(log_buffer,
+ "[%s] FIFO compaction: picking file %" PRIu64
+ " with creation time %" PRIu64 " for deletion",
+ cf_name.c_str(), f->fd.GetNumber(),
+ f->fd.table_reader->GetTableProperties()->creation_time);
+ }
+
+ Compaction* c = new Compaction(
+ vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
+ kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
+ /* is deletion compaction */ true, CompactionReason::kFIFOTtl);
+ return c;
+}
+
+Compaction* FIFOCompactionPicker::PickSizeCompaction(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
+ const int kLevel0 = 0;
+ const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
+ uint64_t total_size = GetTotalFilesSize(level_files);
+
+ if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size ||
+ level_files.size() == 0) {
+ // total size not exceeded
+ if (ioptions_.compaction_options_fifo.allow_compaction &&
+ level_files.size() > 0) {
+ CompactionInputFiles comp_inputs;
+ if (FindIntraL0Compaction(
+ level_files,
+ mutable_cf_options
+ .level0_file_num_compaction_trigger /* min_files_to_compact */,
+ mutable_cf_options.write_buffer_size, &comp_inputs)) {
+ Compaction* c = new Compaction(
+ vstorage, ioptions_, mutable_cf_options, {comp_inputs}, 0,
+ 16 * 1024 * 1024 /* output file size limit */,
+ 0 /* max compaction bytes, not applicable */,
+ 0 /* output path ID */, mutable_cf_options.compression, {},
+ /* is manual */ false, vstorage->CompactionScore(0),
+ /* is deletion compaction */ false,
+ CompactionReason::kFIFOReduceNumFiles);
+ return c;
+ }
+ }
+
+ ROCKS_LOG_BUFFER(log_buffer,
+ "[%s] FIFO compaction: nothing to do. Total size %" PRIu64
+ ", max size %" PRIu64 "\n",
+ cf_name.c_str(), total_size,
+ ioptions_.compaction_options_fifo.max_table_files_size);
+ return nullptr;
+ }
+
+ if (!level0_compactions_in_progress_.empty()) {
+ ROCKS_LOG_BUFFER(
+ log_buffer,
+ "[%s] FIFO compaction: Already executing compaction. No need "
+ "to run parallel compactions since compactions are very fast",
+ cf_name.c_str());
+ return nullptr;
+ }
+
+ std::vector<CompactionInputFiles> inputs;
+ inputs.emplace_back();
+ inputs[0].level = 0;
+
+ for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
+ auto f = *ritr;
+ total_size -= f->compensated_file_size;
+ inputs[0].files.push_back(f);
+ char tmp_fsize[16];
+ AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize));
+ ROCKS_LOG_BUFFER(log_buffer,
+ "[%s] FIFO compaction: picking file %" PRIu64
+ " with size %s for deletion",
+ cf_name.c_str(), f->fd.GetNumber(), tmp_fsize);
+ if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size) {
+ break;
+ }
+ }
+
+ Compaction* c = new Compaction(
+ vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
+ kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
+ /* is deletion compaction */ true, CompactionReason::kFIFOMaxSize);
+ return c;
+}
+
+Compaction* FIFOCompactionPicker::PickCompaction(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
+ assert(vstorage->num_levels() == 1);
+
+ Compaction* c = nullptr;
+ if (ioptions_.compaction_options_fifo.ttl > 0) {
+ c = PickTTLCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
+ }
+ if (c == nullptr) {
+ c = PickSizeCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
+ }
+ RegisterCompaction(c);
+ return c;
+}
+
+Compaction* FIFOCompactionPicker::CompactRange(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, int input_level, int output_level,
+ uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
+ InternalKey** compaction_end, bool* manual_conflict) {
+ assert(input_level == 0);
+ assert(output_level == 0);
+ *compaction_end = nullptr;
+ LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, ioptions_.info_log);
+ Compaction* c =
+ PickCompaction(cf_name, mutable_cf_options, vstorage, &log_buffer);
+ log_buffer.FlushBufferToLog();
+ return c;
+}
+
+#endif // !ROCKSDB_LITE
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/compaction_picker.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/compaction_picker.h b/thirdparty/rocksdb/db/compaction_picker.h
new file mode 100644
index 0000000..f44139c
--- /dev/null
+++ b/thirdparty/rocksdb/db/compaction_picker.h
@@ -0,0 +1,298 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "db/compaction.h"
+#include "db/version_set.h"
+#include "options/cf_options.h"
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+#include "rocksdb/status.h"
+
+namespace rocksdb {
+
+class LogBuffer;
+class Compaction;
+class VersionStorageInfo;
+struct CompactionInputFiles;
+
+class CompactionPicker {
+ public:
+ CompactionPicker(const ImmutableCFOptions& ioptions,
+ const InternalKeyComparator* icmp);
+ virtual ~CompactionPicker();
+
+ // Pick level and inputs for a new compaction.
+ // Returns nullptr if there is no compaction to be done.
+ // Otherwise returns a pointer to a heap-allocated object that
+ // describes the compaction. Caller should delete the result.
+ virtual Compaction* PickCompaction(const std::string& cf_name,
+ const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage,
+ LogBuffer* log_buffer) = 0;
+
+ // Return a compaction object for compacting the range [begin,end] in
+ // the specified level. Returns nullptr if there is nothing in that
+ // level that overlaps the specified range. Caller should delete
+ // the result.
+ //
+ // The returned Compaction might not include the whole requested range.
+ // In that case, compaction_end will be set to the next key that needs
+ // compacting. In case the compaction will compact the whole range,
+ // compaction_end will be set to nullptr.
+ // Client is responsible for compaction_end storage -- when called,
+ // *compaction_end should point to valid InternalKey!
+ virtual Compaction* CompactRange(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, int input_level, int output_level,
+ uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
+ InternalKey** compaction_end, bool* manual_conflict);
+
+ // The maximum allowed output level. Default value is NumberLevels() - 1.
+ virtual int MaxOutputLevel() const { return NumberLevels() - 1; }
+
+ virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const = 0;
+
+// Sanitize the input set of compaction input files.
+// When the input parameters do not describe a valid compaction, the
+// function will try to fix the input_files by adding necessary
+// files. If it's not possible to conver an invalid input_files
+// into a valid one by adding more files, the function will return a
+// non-ok status with specific reason.
+#ifndef ROCKSDB_LITE
+ Status SanitizeCompactionInputFiles(std::unordered_set<uint64_t>* input_files,
+ const ColumnFamilyMetaData& cf_meta,
+ const int output_level) const;
+#endif // ROCKSDB_LITE
+
+ // Free up the files that participated in a compaction
+ //
+ // Requirement: DB mutex held
+ void ReleaseCompactionFiles(Compaction* c, Status status);
+
+ // Returns true if any one of the specified files are being compacted
+ bool AreFilesInCompaction(const std::vector<FileMetaData*>& files);
+
+ // Takes a list of CompactionInputFiles and returns a (manual) Compaction
+ // object.
+ Compaction* CompactFiles(const CompactionOptions& compact_options,
+ const std::vector<CompactionInputFiles>& input_files,
+ int output_level, VersionStorageInfo* vstorage,
+ const MutableCFOptions& mutable_cf_options,
+ uint32_t output_path_id);
+
+ // Converts a set of compaction input file numbers into
+ // a list of CompactionInputFiles.
+ Status GetCompactionInputsFromFileNumbers(
+ std::vector<CompactionInputFiles>* input_files,
+ std::unordered_set<uint64_t>* input_set,
+ const VersionStorageInfo* vstorage,
+ const CompactionOptions& compact_options) const;
+
+ // Is there currently a compaction involving level 0 taking place
+ bool IsLevel0CompactionInProgress() const {
+ return !level0_compactions_in_progress_.empty();
+ }
+
+ // Return true if the passed key range overlap with a compaction output
+ // that is currently running.
+ bool RangeOverlapWithCompaction(const Slice& smallest_user_key,
+ const Slice& largest_user_key,
+ int level) const;
+
+ // Stores the minimal range that covers all entries in inputs in
+ // *smallest, *largest.
+ // REQUIRES: inputs is not empty
+ void GetRange(const CompactionInputFiles& inputs, InternalKey* smallest,
+ InternalKey* largest) const;
+
+ // Stores the minimal range that covers all entries in inputs1 and inputs2
+ // in *smallest, *largest.
+ // REQUIRES: inputs is not empty
+ void GetRange(const CompactionInputFiles& inputs1,
+ const CompactionInputFiles& inputs2, InternalKey* smallest,
+ InternalKey* largest) const;
+
+ // Stores the minimal range that covers all entries in inputs
+ // in *smallest, *largest.
+ // REQUIRES: inputs is not empty (at least on entry have one file)
+ void GetRange(const std::vector<CompactionInputFiles>& inputs,
+ InternalKey* smallest, InternalKey* largest) const;
+
+ int NumberLevels() const { return ioptions_.num_levels; }
+
+ // Add more files to the inputs on "level" to make sure that
+ // no newer version of a key is compacted to "level+1" while leaving an older
+ // version in a "level". Otherwise, any Get() will search "level" first,
+ // and will likely return an old/stale value for the key, since it always
+ // searches in increasing order of level to find the value. This could
+ // also scramble the order of merge operands. This function should be
+ // called any time a new Compaction is created, and its inputs_[0] are
+ // populated.
+ //
+ // Will return false if it is impossible to apply this compaction.
+ bool ExpandInputsToCleanCut(const std::string& cf_name,
+ VersionStorageInfo* vstorage,
+ CompactionInputFiles* inputs);
+
+ // Returns true if any one of the parent files are being compacted
+ bool IsRangeInCompaction(VersionStorageInfo* vstorage,
+ const InternalKey* smallest,
+ const InternalKey* largest, int level, int* index);
+
+ // Returns true if the key range that `inputs` files cover overlap with the
+ // key range of a currently running compaction.
+ bool FilesRangeOverlapWithCompaction(
+ const std::vector<CompactionInputFiles>& inputs, int level) const;
+
+ bool SetupOtherInputs(const std::string& cf_name,
+ const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage,
+ CompactionInputFiles* inputs,
+ CompactionInputFiles* output_level_inputs,
+ int* parent_index, int base_index);
+
+ void GetGrandparents(VersionStorageInfo* vstorage,
+ const CompactionInputFiles& inputs,
+ const CompactionInputFiles& output_level_inputs,
+ std::vector<FileMetaData*>* grandparents);
+
+ // Register this compaction in the set of running compactions
+ void RegisterCompaction(Compaction* c);
+
+ // Remove this compaction from the set of running compactions
+ void UnregisterCompaction(Compaction* c);
+
+ std::set<Compaction*>* level0_compactions_in_progress() {
+ return &level0_compactions_in_progress_;
+ }
+ std::unordered_set<Compaction*>* compactions_in_progress() {
+ return &compactions_in_progress_;
+ }
+
+ protected:
+ const ImmutableCFOptions& ioptions_;
+
+// A helper function to SanitizeCompactionInputFiles() that
+// sanitizes "input_files" by adding necessary files.
+#ifndef ROCKSDB_LITE
+ virtual Status SanitizeCompactionInputFilesForAllLevels(
+ std::unordered_set<uint64_t>* input_files,
+ const ColumnFamilyMetaData& cf_meta, const int output_level) const;
+#endif // ROCKSDB_LITE
+
+ // Keeps track of all compactions that are running on Level0.
+ // Protected by DB mutex
+ std::set<Compaction*> level0_compactions_in_progress_;
+
+ // Keeps track of all compactions that are running.
+ // Protected by DB mutex
+ std::unordered_set<Compaction*> compactions_in_progress_;
+
+ const InternalKeyComparator* const icmp_;
+};
+
+class LevelCompactionPicker : public CompactionPicker {
+ public:
+ LevelCompactionPicker(const ImmutableCFOptions& ioptions,
+ const InternalKeyComparator* icmp)
+ : CompactionPicker(ioptions, icmp) {}
+ virtual Compaction* PickCompaction(const std::string& cf_name,
+ const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage,
+ LogBuffer* log_buffer) override;
+
+ virtual bool NeedsCompaction(
+ const VersionStorageInfo* vstorage) const override;
+};
+
+#ifndef ROCKSDB_LITE
+class FIFOCompactionPicker : public CompactionPicker {
+ public:
+ FIFOCompactionPicker(const ImmutableCFOptions& ioptions,
+ const InternalKeyComparator* icmp)
+ : CompactionPicker(ioptions, icmp) {}
+
+ virtual Compaction* PickCompaction(const std::string& cf_name,
+ const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* version,
+ LogBuffer* log_buffer) override;
+
+ virtual Compaction* CompactRange(
+ const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, int input_level, int output_level,
+ uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
+ InternalKey** compaction_end, bool* manual_conflict) override;
+
+ // The maximum allowed output level. Always returns 0.
+ virtual int MaxOutputLevel() const override { return 0; }
+
+ virtual bool NeedsCompaction(
+ const VersionStorageInfo* vstorage) const override;
+
+ private:
+ Compaction* PickTTLCompaction(const std::string& cf_name,
+ const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* version,
+ LogBuffer* log_buffer);
+
+ Compaction* PickSizeCompaction(const std::string& cf_name,
+ const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* version,
+ LogBuffer* log_buffer);
+};
+
+class NullCompactionPicker : public CompactionPicker {
+ public:
+ NullCompactionPicker(const ImmutableCFOptions& ioptions,
+ const InternalKeyComparator* icmp)
+ : CompactionPicker(ioptions, icmp) {}
+ virtual ~NullCompactionPicker() {}
+
+ // Always return "nullptr"
+ Compaction* PickCompaction(const std::string& cf_name,
+ const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage,
+ LogBuffer* log_buffer) override {
+ return nullptr;
+ }
+
+ // Always return "nullptr"
+ Compaction* CompactRange(const std::string& cf_name,
+ const MutableCFOptions& mutable_cf_options,
+ VersionStorageInfo* vstorage, int input_level,
+ int output_level, uint32_t output_path_id,
+ const InternalKey* begin, const InternalKey* end,
+ InternalKey** compaction_end,
+ bool* manual_conflict) override {
+ return nullptr;
+ }
+
+ // Always returns false.
+ virtual bool NeedsCompaction(
+ const VersionStorageInfo* vstorage) const override {
+ return false;
+ }
+};
+#endif // !ROCKSDB_LITE
+
+CompressionType GetCompressionType(const ImmutableCFOptions& ioptions,
+ const VersionStorageInfo* vstorage,
+ const MutableCFOptions& mutable_cf_options,
+ int level, int base_level,
+ const bool enable_compression = true);
+
+} // namespace rocksdb
[38/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/table_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/table_test.cc b/thirdparty/leveldb-1.18/table/table_test.cc
deleted file mode 100755
index d46c079..0000000
--- a/thirdparty/leveldb-1.18/table/table_test.cc
+++ /dev/null
@@ -1,868 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/table.h"
-
-#include <map>
-#include <string>
-#include "db/dbformat.h"
-#include "db/memtable.h"
-#include "db/write_batch_internal.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
-#include "leveldb/table_builder.h"
-#include "table/block.h"
-#include "table/block_builder.h"
-#include "table/format.h"
-#include "util/random.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-// Return reverse of "key".
-// Used to test non-lexicographic comparators.
-static std::string Reverse(const Slice& key) {
- std::string str(key.ToString());
- std::string rev("");
- for (std::string::reverse_iterator rit = str.rbegin();
- rit != str.rend(); ++rit) {
- rev.push_back(*rit);
- }
- return rev;
-}
-
-namespace {
-class ReverseKeyComparator : public Comparator {
- public:
- virtual const char* Name() const {
- return "leveldb.ReverseBytewiseComparator";
- }
-
- virtual int Compare(const Slice& a, const Slice& b) const {
- return BytewiseComparator()->Compare(Reverse(a), Reverse(b));
- }
-
- virtual void FindShortestSeparator(
- std::string* start,
- const Slice& limit) const {
- std::string s = Reverse(*start);
- std::string l = Reverse(limit);
- BytewiseComparator()->FindShortestSeparator(&s, l);
- *start = Reverse(s);
- }
-
- virtual void FindShortSuccessor(std::string* key) const {
- std::string s = Reverse(*key);
- BytewiseComparator()->FindShortSuccessor(&s);
- *key = Reverse(s);
- }
-};
-} // namespace
-static ReverseKeyComparator reverse_key_comparator;
-
-static void Increment(const Comparator* cmp, std::string* key) {
- if (cmp == BytewiseComparator()) {
- key->push_back('\0');
- } else {
- assert(cmp == &reverse_key_comparator);
- std::string rev = Reverse(*key);
- rev.push_back('\0');
- *key = Reverse(rev);
- }
-}
-
-// An STL comparator that uses a Comparator
-namespace {
-struct STLLessThan {
- const Comparator* cmp;
-
- STLLessThan() : cmp(BytewiseComparator()) { }
- STLLessThan(const Comparator* c) : cmp(c) { }
- bool operator()(const std::string& a, const std::string& b) const {
- return cmp->Compare(Slice(a), Slice(b)) < 0;
- }
-};
-} // namespace
-
-class StringSink: public WritableFile {
- public:
- ~StringSink() { }
-
- const std::string& contents() const { return contents_; }
-
- virtual Status Close() { return Status::OK(); }
- virtual Status Flush() { return Status::OK(); }
- virtual Status Sync() { return Status::OK(); }
-
- virtual Status Append(const Slice& data) {
- contents_.append(data.data(), data.size());
- return Status::OK();
- }
-
- private:
- std::string contents_;
-};
-
-
-class StringSource: public RandomAccessFile {
- public:
- StringSource(const Slice& contents)
- : contents_(contents.data(), contents.size()) {
- }
-
- virtual ~StringSource() { }
-
- uint64_t Size() const { return contents_.size(); }
-
- virtual Status Read(uint64_t offset, size_t n, Slice* result,
- char* scratch) const {
- if (offset > contents_.size()) {
- return Status::InvalidArgument("invalid Read offset");
- }
- if (offset + n > contents_.size()) {
- n = contents_.size() - offset;
- }
- memcpy(scratch, &contents_[offset], n);
- *result = Slice(scratch, n);
- return Status::OK();
- }
-
- private:
- std::string contents_;
-};
-
-typedef std::map<std::string, std::string, STLLessThan> KVMap;
-
-// Helper class for tests to unify the interface between
-// BlockBuilder/TableBuilder and Block/Table.
-class Constructor {
- public:
- explicit Constructor(const Comparator* cmp) : data_(STLLessThan(cmp)) { }
- virtual ~Constructor() { }
-
- void Add(const std::string& key, const Slice& value) {
- data_[key] = value.ToString();
- }
-
- // Finish constructing the data structure with all the keys that have
- // been added so far. Returns the keys in sorted order in "*keys"
- // and stores the key/value pairs in "*kvmap"
- void Finish(const Options& options,
- std::vector<std::string>* keys,
- KVMap* kvmap) {
- *kvmap = data_;
- keys->clear();
- for (KVMap::const_iterator it = data_.begin();
- it != data_.end();
- ++it) {
- keys->push_back(it->first);
- }
- data_.clear();
- Status s = FinishImpl(options, *kvmap);
- ASSERT_TRUE(s.ok()) << s.ToString();
- }
-
- // Construct the data structure from the data in "data"
- virtual Status FinishImpl(const Options& options, const KVMap& data) = 0;
-
- virtual Iterator* NewIterator() const = 0;
-
- virtual const KVMap& data() { return data_; }
-
- virtual DB* db() const { return NULL; } // Overridden in DBConstructor
-
- private:
- KVMap data_;
-};
-
-class BlockConstructor: public Constructor {
- public:
- explicit BlockConstructor(const Comparator* cmp)
- : Constructor(cmp),
- comparator_(cmp),
- block_(NULL) { }
- ~BlockConstructor() {
- delete block_;
- }
- virtual Status FinishImpl(const Options& options, const KVMap& data) {
- delete block_;
- block_ = NULL;
- BlockBuilder builder(&options);
-
- for (KVMap::const_iterator it = data.begin();
- it != data.end();
- ++it) {
- builder.Add(it->first, it->second);
- }
- // Open the block
- data_ = builder.Finish().ToString();
- BlockContents contents;
- contents.data = data_;
- contents.cachable = false;
- contents.heap_allocated = false;
- block_ = new Block(contents);
- return Status::OK();
- }
- virtual Iterator* NewIterator() const {
- return block_->NewIterator(comparator_);
- }
-
- private:
- const Comparator* comparator_;
- std::string data_;
- Block* block_;
-
- BlockConstructor();
-};
-
-class TableConstructor: public Constructor {
- public:
- TableConstructor(const Comparator* cmp)
- : Constructor(cmp),
- source_(NULL), table_(NULL) {
- }
- ~TableConstructor() {
- Reset();
- }
- virtual Status FinishImpl(const Options& options, const KVMap& data) {
- Reset();
- StringSink sink;
- TableBuilder builder(options, &sink);
-
- for (KVMap::const_iterator it = data.begin();
- it != data.end();
- ++it) {
- builder.Add(it->first, it->second);
- ASSERT_TRUE(builder.status().ok());
- }
- Status s = builder.Finish();
- ASSERT_TRUE(s.ok()) << s.ToString();
-
- ASSERT_EQ(sink.contents().size(), builder.FileSize());
-
- // Open the table
- source_ = new StringSource(sink.contents());
- Options table_options;
- table_options.comparator = options.comparator;
- return Table::Open(table_options, source_, sink.contents().size(), &table_);
- }
-
- virtual Iterator* NewIterator() const {
- return table_->NewIterator(ReadOptions());
- }
-
- uint64_t ApproximateOffsetOf(const Slice& key) const {
- return table_->ApproximateOffsetOf(key);
- }
-
- private:
- void Reset() {
- delete table_;
- delete source_;
- table_ = NULL;
- source_ = NULL;
- }
-
- StringSource* source_;
- Table* table_;
-
- TableConstructor();
-};
-
-// A helper class that converts internal format keys into user keys
-class KeyConvertingIterator: public Iterator {
- public:
- explicit KeyConvertingIterator(Iterator* iter) : iter_(iter) { }
- virtual ~KeyConvertingIterator() { delete iter_; }
- virtual bool Valid() const { return iter_->Valid(); }
- virtual void Seek(const Slice& target) {
- ParsedInternalKey ikey(target, kMaxSequenceNumber, kTypeValue);
- std::string encoded;
- AppendInternalKey(&encoded, ikey);
- iter_->Seek(encoded);
- }
- virtual void SeekToFirst() { iter_->SeekToFirst(); }
- virtual void SeekToLast() { iter_->SeekToLast(); }
- virtual void Next() { iter_->Next(); }
- virtual void Prev() { iter_->Prev(); }
-
- virtual Slice key() const {
- assert(Valid());
- ParsedInternalKey key;
- if (!ParseInternalKey(iter_->key(), &key)) {
- status_ = Status::Corruption("malformed internal key");
- return Slice("corrupted key");
- }
- return key.user_key;
- }
-
- virtual Slice value() const { return iter_->value(); }
- virtual Status status() const {
- return status_.ok() ? iter_->status() : status_;
- }
-
- private:
- mutable Status status_;
- Iterator* iter_;
-
- // No copying allowed
- KeyConvertingIterator(const KeyConvertingIterator&);
- void operator=(const KeyConvertingIterator&);
-};
-
-class MemTableConstructor: public Constructor {
- public:
- explicit MemTableConstructor(const Comparator* cmp)
- : Constructor(cmp),
- internal_comparator_(cmp) {
- memtable_ = new MemTable(internal_comparator_);
- memtable_->Ref();
- }
- ~MemTableConstructor() {
- memtable_->Unref();
- }
- virtual Status FinishImpl(const Options& options, const KVMap& data) {
- memtable_->Unref();
- memtable_ = new MemTable(internal_comparator_);
- memtable_->Ref();
- int seq = 1;
- for (KVMap::const_iterator it = data.begin();
- it != data.end();
- ++it) {
- memtable_->Add(seq, kTypeValue, it->first, it->second);
- seq++;
- }
- return Status::OK();
- }
- virtual Iterator* NewIterator() const {
- return new KeyConvertingIterator(memtable_->NewIterator());
- }
-
- private:
- InternalKeyComparator internal_comparator_;
- MemTable* memtable_;
-};
-
-class DBConstructor: public Constructor {
- public:
- explicit DBConstructor(const Comparator* cmp)
- : Constructor(cmp),
- comparator_(cmp) {
- db_ = NULL;
- NewDB();
- }
- ~DBConstructor() {
- delete db_;
- }
- virtual Status FinishImpl(const Options& options, const KVMap& data) {
- delete db_;
- db_ = NULL;
- NewDB();
- for (KVMap::const_iterator it = data.begin();
- it != data.end();
- ++it) {
- WriteBatch batch;
- batch.Put(it->first, it->second);
- ASSERT_TRUE(db_->Write(WriteOptions(), &batch).ok());
- }
- return Status::OK();
- }
- virtual Iterator* NewIterator() const {
- return db_->NewIterator(ReadOptions());
- }
-
- virtual DB* db() const { return db_; }
-
- private:
- void NewDB() {
- std::string name = test::TmpDir() + "/table_testdb";
-
- Options options;
- options.comparator = comparator_;
- Status status = DestroyDB(name, options);
- ASSERT_TRUE(status.ok()) << status.ToString();
-
- options.create_if_missing = true;
- options.error_if_exists = true;
- options.write_buffer_size = 10000; // Something small to force merging
- status = DB::Open(options, name, &db_);
- ASSERT_TRUE(status.ok()) << status.ToString();
- }
-
- const Comparator* comparator_;
- DB* db_;
-};
-
-enum TestType {
- TABLE_TEST,
- BLOCK_TEST,
- MEMTABLE_TEST,
- DB_TEST
-};
-
-struct TestArgs {
- TestType type;
- bool reverse_compare;
- int restart_interval;
-};
-
-static const TestArgs kTestArgList[] = {
- { TABLE_TEST, false, 16 },
- { TABLE_TEST, false, 1 },
- { TABLE_TEST, false, 1024 },
- { TABLE_TEST, true, 16 },
- { TABLE_TEST, true, 1 },
- { TABLE_TEST, true, 1024 },
-
- { BLOCK_TEST, false, 16 },
- { BLOCK_TEST, false, 1 },
- { BLOCK_TEST, false, 1024 },
- { BLOCK_TEST, true, 16 },
- { BLOCK_TEST, true, 1 },
- { BLOCK_TEST, true, 1024 },
-
- // Restart interval does not matter for memtables
- { MEMTABLE_TEST, false, 16 },
- { MEMTABLE_TEST, true, 16 },
-
- // Do not bother with restart interval variations for DB
- { DB_TEST, false, 16 },
- { DB_TEST, true, 16 },
-};
-static const int kNumTestArgs = sizeof(kTestArgList) / sizeof(kTestArgList[0]);
-
-class Harness {
- public:
- Harness() : constructor_(NULL) { }
-
- void Init(const TestArgs& args) {
- delete constructor_;
- constructor_ = NULL;
- options_ = Options();
-
- options_.block_restart_interval = args.restart_interval;
- // Use shorter block size for tests to exercise block boundary
- // conditions more.
- options_.block_size = 256;
- if (args.reverse_compare) {
- options_.comparator = &reverse_key_comparator;
- }
- switch (args.type) {
- case TABLE_TEST:
- constructor_ = new TableConstructor(options_.comparator);
- break;
- case BLOCK_TEST:
- constructor_ = new BlockConstructor(options_.comparator);
- break;
- case MEMTABLE_TEST:
- constructor_ = new MemTableConstructor(options_.comparator);
- break;
- case DB_TEST:
- constructor_ = new DBConstructor(options_.comparator);
- break;
- }
- }
-
- ~Harness() {
- delete constructor_;
- }
-
- void Add(const std::string& key, const std::string& value) {
- constructor_->Add(key, value);
- }
-
- void Test(Random* rnd) {
- std::vector<std::string> keys;
- KVMap data;
- constructor_->Finish(options_, &keys, &data);
-
- TestForwardScan(keys, data);
- TestBackwardScan(keys, data);
- TestRandomAccess(rnd, keys, data);
- }
-
- void TestForwardScan(const std::vector<std::string>& keys,
- const KVMap& data) {
- Iterator* iter = constructor_->NewIterator();
- ASSERT_TRUE(!iter->Valid());
- iter->SeekToFirst();
- for (KVMap::const_iterator model_iter = data.begin();
- model_iter != data.end();
- ++model_iter) {
- ASSERT_EQ(ToString(data, model_iter), ToString(iter));
- iter->Next();
- }
- ASSERT_TRUE(!iter->Valid());
- delete iter;
- }
-
- void TestBackwardScan(const std::vector<std::string>& keys,
- const KVMap& data) {
- Iterator* iter = constructor_->NewIterator();
- ASSERT_TRUE(!iter->Valid());
- iter->SeekToLast();
- for (KVMap::const_reverse_iterator model_iter = data.rbegin();
- model_iter != data.rend();
- ++model_iter) {
- ASSERT_EQ(ToString(data, model_iter), ToString(iter));
- iter->Prev();
- }
- ASSERT_TRUE(!iter->Valid());
- delete iter;
- }
-
- void TestRandomAccess(Random* rnd,
- const std::vector<std::string>& keys,
- const KVMap& data) {
- static const bool kVerbose = false;
- Iterator* iter = constructor_->NewIterator();
- ASSERT_TRUE(!iter->Valid());
- KVMap::const_iterator model_iter = data.begin();
- if (kVerbose) fprintf(stderr, "---\n");
- for (int i = 0; i < 200; i++) {
- const int toss = rnd->Uniform(5);
- switch (toss) {
- case 0: {
- if (iter->Valid()) {
- if (kVerbose) fprintf(stderr, "Next\n");
- iter->Next();
- ++model_iter;
- ASSERT_EQ(ToString(data, model_iter), ToString(iter));
- }
- break;
- }
-
- case 1: {
- if (kVerbose) fprintf(stderr, "SeekToFirst\n");
- iter->SeekToFirst();
- model_iter = data.begin();
- ASSERT_EQ(ToString(data, model_iter), ToString(iter));
- break;
- }
-
- case 2: {
- std::string key = PickRandomKey(rnd, keys);
- model_iter = data.lower_bound(key);
- if (kVerbose) fprintf(stderr, "Seek '%s'\n",
- EscapeString(key).c_str());
- iter->Seek(Slice(key));
- ASSERT_EQ(ToString(data, model_iter), ToString(iter));
- break;
- }
-
- case 3: {
- if (iter->Valid()) {
- if (kVerbose) fprintf(stderr, "Prev\n");
- iter->Prev();
- if (model_iter == data.begin()) {
- model_iter = data.end(); // Wrap around to invalid value
- } else {
- --model_iter;
- }
- ASSERT_EQ(ToString(data, model_iter), ToString(iter));
- }
- break;
- }
-
- case 4: {
- if (kVerbose) fprintf(stderr, "SeekToLast\n");
- iter->SeekToLast();
- if (keys.empty()) {
- model_iter = data.end();
- } else {
- std::string last = data.rbegin()->first;
- model_iter = data.lower_bound(last);
- }
- ASSERT_EQ(ToString(data, model_iter), ToString(iter));
- break;
- }
- }
- }
- delete iter;
- }
-
- std::string ToString(const KVMap& data, const KVMap::const_iterator& it) {
- if (it == data.end()) {
- return "END";
- } else {
- return "'" + it->first + "->" + it->second + "'";
- }
- }
-
- std::string ToString(const KVMap& data,
- const KVMap::const_reverse_iterator& it) {
- if (it == data.rend()) {
- return "END";
- } else {
- return "'" + it->first + "->" + it->second + "'";
- }
- }
-
- std::string ToString(const Iterator* it) {
- if (!it->Valid()) {
- return "END";
- } else {
- return "'" + it->key().ToString() + "->" + it->value().ToString() + "'";
- }
- }
-
- std::string PickRandomKey(Random* rnd, const std::vector<std::string>& keys) {
- if (keys.empty()) {
- return "foo";
- } else {
- const int index = rnd->Uniform(static_cast<int>(keys.size()));
- std::string result = keys[index];
- switch (rnd->Uniform(3)) {
- case 0:
- // Return an existing key
- break;
- case 1: {
- // Attempt to return something smaller than an existing key
- if (result.size() > 0 && result[result.size()-1] > '\0') {
- result[result.size()-1]--;
- }
- break;
- }
- case 2: {
- // Return something larger than an existing key
- Increment(options_.comparator, &result);
- break;
- }
- }
- return result;
- }
- }
-
- // Returns NULL if not running against a DB
- DB* db() const { return constructor_->db(); }
-
- private:
- Options options_;
- Constructor* constructor_;
-};
-
-// Test empty table/block.
-TEST(Harness, Empty) {
- for (int i = 0; i < kNumTestArgs; i++) {
- Init(kTestArgList[i]);
- Random rnd(test::RandomSeed() + 1);
- Test(&rnd);
- }
-}
-
-// Special test for a block with no restart entries. The C++ leveldb
-// code never generates such blocks, but the Java version of leveldb
-// seems to.
-TEST(Harness, ZeroRestartPointsInBlock) {
- char data[sizeof(uint32_t)];
- memset(data, 0, sizeof(data));
- BlockContents contents;
- contents.data = Slice(data, sizeof(data));
- contents.cachable = false;
- contents.heap_allocated = false;
- Block block(contents);
- Iterator* iter = block.NewIterator(BytewiseComparator());
- iter->SeekToFirst();
- ASSERT_TRUE(!iter->Valid());
- iter->SeekToLast();
- ASSERT_TRUE(!iter->Valid());
- iter->Seek("foo");
- ASSERT_TRUE(!iter->Valid());
- delete iter;
-}
-
-// Test the empty key
-TEST(Harness, SimpleEmptyKey) {
- for (int i = 0; i < kNumTestArgs; i++) {
- Init(kTestArgList[i]);
- Random rnd(test::RandomSeed() + 1);
- Add("", "v");
- Test(&rnd);
- }
-}
-
-TEST(Harness, SimpleSingle) {
- for (int i = 0; i < kNumTestArgs; i++) {
- Init(kTestArgList[i]);
- Random rnd(test::RandomSeed() + 2);
- Add("abc", "v");
- Test(&rnd);
- }
-}
-
-TEST(Harness, SimpleMulti) {
- for (int i = 0; i < kNumTestArgs; i++) {
- Init(kTestArgList[i]);
- Random rnd(test::RandomSeed() + 3);
- Add("abc", "v");
- Add("abcd", "v");
- Add("ac", "v2");
- Test(&rnd);
- }
-}
-
-TEST(Harness, SimpleSpecialKey) {
- for (int i = 0; i < kNumTestArgs; i++) {
- Init(kTestArgList[i]);
- Random rnd(test::RandomSeed() + 4);
- Add("\xff\xff", "v3");
- Test(&rnd);
- }
-}
-
-TEST(Harness, Randomized) {
- for (int i = 0; i < kNumTestArgs; i++) {
- Init(kTestArgList[i]);
- Random rnd(test::RandomSeed() + 5);
- for (int num_entries = 0; num_entries < 2000;
- num_entries += (num_entries < 50 ? 1 : 200)) {
- if ((num_entries % 10) == 0) {
- fprintf(stderr, "case %d of %d: num_entries = %d\n",
- (i + 1), int(kNumTestArgs), num_entries);
- }
- for (int e = 0; e < num_entries; e++) {
- std::string v;
- Add(test::RandomKey(&rnd, rnd.Skewed(4)),
- test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
- }
- Test(&rnd);
- }
- }
-}
-
-TEST(Harness, RandomizedLongDB) {
- Random rnd(test::RandomSeed());
- TestArgs args = { DB_TEST, false, 16 };
- Init(args);
- int num_entries = 100000;
- for (int e = 0; e < num_entries; e++) {
- std::string v;
- Add(test::RandomKey(&rnd, rnd.Skewed(4)),
- test::RandomString(&rnd, rnd.Skewed(5), &v).ToString());
- }
- Test(&rnd);
-
- // We must have created enough data to force merging
- int files = 0;
- for (int level = 0; level < config::kNumLevels; level++) {
- std::string value;
- char name[100];
- snprintf(name, sizeof(name), "leveldb.num-files-at-level%d", level);
- ASSERT_TRUE(db()->GetProperty(name, &value));
- files += atoi(value.c_str());
- }
- ASSERT_GT(files, 0);
-}
-
-class MemTableTest { };
-
-TEST(MemTableTest, Simple) {
- InternalKeyComparator cmp(BytewiseComparator());
- MemTable* memtable = new MemTable(cmp);
- memtable->Ref();
- WriteBatch batch;
- WriteBatchInternal::SetSequence(&batch, 100);
- batch.Put(std::string("k1"), std::string("v1"));
- batch.Put(std::string("k2"), std::string("v2"));
- batch.Put(std::string("k3"), std::string("v3"));
- batch.Put(std::string("largekey"), std::string("vlarge"));
- ASSERT_TRUE(WriteBatchInternal::InsertInto(&batch, memtable).ok());
-
- Iterator* iter = memtable->NewIterator();
- iter->SeekToFirst();
- while (iter->Valid()) {
- fprintf(stderr, "key: '%s' -> '%s'\n",
- iter->key().ToString().c_str(),
- iter->value().ToString().c_str());
- iter->Next();
- }
-
- delete iter;
- memtable->Unref();
-}
-
-static bool Between(uint64_t val, uint64_t low, uint64_t high) {
- bool result = (val >= low) && (val <= high);
- if (!result) {
- fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
- (unsigned long long)(val),
- (unsigned long long)(low),
- (unsigned long long)(high));
- }
- return result;
-}
-
-class TableTest { };
-
-TEST(TableTest, ApproximateOffsetOfPlain) {
- TableConstructor c(BytewiseComparator());
- c.Add("k01", "hello");
- c.Add("k02", "hello2");
- c.Add("k03", std::string(10000, 'x'));
- c.Add("k04", std::string(200000, 'x'));
- c.Add("k05", std::string(300000, 'x'));
- c.Add("k06", "hello3");
- c.Add("k07", std::string(100000, 'x'));
- std::vector<std::string> keys;
- KVMap kvmap;
- Options options;
- options.block_size = 1024;
- options.compression = kNoCompression;
- c.Finish(options, &keys, &kvmap);
-
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01a"), 0, 0));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 0, 0));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 10000, 11000));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04a"), 210000, 211000));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k05"), 210000, 211000));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k06"), 510000, 511000));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k07"), 510000, 511000));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 610000, 612000));
-
-}
-
-static bool SnappyCompressionSupported() {
- std::string out;
- Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
- return port::Snappy_Compress(in.data(), in.size(), &out);
-}
-
-TEST(TableTest, ApproximateOffsetOfCompressed) {
- if (!SnappyCompressionSupported()) {
- fprintf(stderr, "skipping compression tests\n");
- return;
- }
-
- Random rnd(301);
- TableConstructor c(BytewiseComparator());
- std::string tmp;
- c.Add("k01", "hello");
- c.Add("k02", test::CompressibleString(&rnd, 0.25, 10000, &tmp));
- c.Add("k03", "hello3");
- c.Add("k04", test::CompressibleString(&rnd, 0.25, 10000, &tmp));
- std::vector<std::string> keys;
- KVMap kvmap;
- Options options;
- options.block_size = 1024;
- options.compression = kSnappyCompression;
- c.Finish(options, &keys, &kvmap);
-
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, 0));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, 0));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, 0));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), 2000, 3000));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), 2000, 3000));
- ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 4000, 6000));
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/two_level_iterator.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/two_level_iterator.cc b/thirdparty/leveldb-1.18/table/two_level_iterator.cc
deleted file mode 100755
index 7822eba..0000000
--- a/thirdparty/leveldb-1.18/table/two_level_iterator.cc
+++ /dev/null
@@ -1,182 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "table/two_level_iterator.h"
-
-#include "leveldb/table.h"
-#include "table/block.h"
-#include "table/format.h"
-#include "table/iterator_wrapper.h"
-
-namespace leveldb {
-
-namespace {
-
-typedef Iterator* (*BlockFunction)(void*, const ReadOptions&, const Slice&);
-
-class TwoLevelIterator: public Iterator {
- public:
- TwoLevelIterator(
- Iterator* index_iter,
- BlockFunction block_function,
- void* arg,
- const ReadOptions& options);
-
- virtual ~TwoLevelIterator();
-
- virtual void Seek(const Slice& target);
- virtual void SeekToFirst();
- virtual void SeekToLast();
- virtual void Next();
- virtual void Prev();
-
- virtual bool Valid() const {
- return data_iter_.Valid();
- }
- virtual Slice key() const {
- assert(Valid());
- return data_iter_.key();
- }
- virtual Slice value() const {
- assert(Valid());
- return data_iter_.value();
- }
- virtual Status status() const {
- // It'd be nice if status() returned a const Status& instead of a Status
- if (!index_iter_.status().ok()) {
- return index_iter_.status();
- } else if (data_iter_.iter() != NULL && !data_iter_.status().ok()) {
- return data_iter_.status();
- } else {
- return status_;
- }
- }
-
- private:
- void SaveError(const Status& s) {
- if (status_.ok() && !s.ok()) status_ = s;
- }
- void SkipEmptyDataBlocksForward();
- void SkipEmptyDataBlocksBackward();
- void SetDataIterator(Iterator* data_iter);
- void InitDataBlock();
-
- BlockFunction block_function_;
- void* arg_;
- const ReadOptions options_;
- Status status_;
- IteratorWrapper index_iter_;
- IteratorWrapper data_iter_; // May be NULL
- // If data_iter_ is non-NULL, then "data_block_handle_" holds the
- // "index_value" passed to block_function_ to create the data_iter_.
- std::string data_block_handle_;
-};
-
-TwoLevelIterator::TwoLevelIterator(
- Iterator* index_iter,
- BlockFunction block_function,
- void* arg,
- const ReadOptions& options)
- : block_function_(block_function),
- arg_(arg),
- options_(options),
- index_iter_(index_iter),
- data_iter_(NULL) {
-}
-
-TwoLevelIterator::~TwoLevelIterator() {
-}
-
-void TwoLevelIterator::Seek(const Slice& target) {
- index_iter_.Seek(target);
- InitDataBlock();
- if (data_iter_.iter() != NULL) data_iter_.Seek(target);
- SkipEmptyDataBlocksForward();
-}
-
-void TwoLevelIterator::SeekToFirst() {
- index_iter_.SeekToFirst();
- InitDataBlock();
- if (data_iter_.iter() != NULL) data_iter_.SeekToFirst();
- SkipEmptyDataBlocksForward();
-}
-
-void TwoLevelIterator::SeekToLast() {
- index_iter_.SeekToLast();
- InitDataBlock();
- if (data_iter_.iter() != NULL) data_iter_.SeekToLast();
- SkipEmptyDataBlocksBackward();
-}
-
-void TwoLevelIterator::Next() {
- assert(Valid());
- data_iter_.Next();
- SkipEmptyDataBlocksForward();
-}
-
-void TwoLevelIterator::Prev() {
- assert(Valid());
- data_iter_.Prev();
- SkipEmptyDataBlocksBackward();
-}
-
-
-void TwoLevelIterator::SkipEmptyDataBlocksForward() {
- while (data_iter_.iter() == NULL || !data_iter_.Valid()) {
- // Move to next block
- if (!index_iter_.Valid()) {
- SetDataIterator(NULL);
- return;
- }
- index_iter_.Next();
- InitDataBlock();
- if (data_iter_.iter() != NULL) data_iter_.SeekToFirst();
- }
-}
-
-void TwoLevelIterator::SkipEmptyDataBlocksBackward() {
- while (data_iter_.iter() == NULL || !data_iter_.Valid()) {
- // Move to next block
- if (!index_iter_.Valid()) {
- SetDataIterator(NULL);
- return;
- }
- index_iter_.Prev();
- InitDataBlock();
- if (data_iter_.iter() != NULL) data_iter_.SeekToLast();
- }
-}
-
-void TwoLevelIterator::SetDataIterator(Iterator* data_iter) {
- if (data_iter_.iter() != NULL) SaveError(data_iter_.status());
- data_iter_.Set(data_iter);
-}
-
-void TwoLevelIterator::InitDataBlock() {
- if (!index_iter_.Valid()) {
- SetDataIterator(NULL);
- } else {
- Slice handle = index_iter_.value();
- if (data_iter_.iter() != NULL && handle.compare(data_block_handle_) == 0) {
- // data_iter_ is already constructed with this iterator, so
- // no need to change anything
- } else {
- Iterator* iter = (*block_function_)(arg_, options_, handle);
- data_block_handle_.assign(handle.data(), handle.size());
- SetDataIterator(iter);
- }
- }
-}
-
-} // namespace
-
-Iterator* NewTwoLevelIterator(
- Iterator* index_iter,
- BlockFunction block_function,
- void* arg,
- const ReadOptions& options) {
- return new TwoLevelIterator(index_iter, block_function, arg, options);
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/table/two_level_iterator.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/table/two_level_iterator.h b/thirdparty/leveldb-1.18/table/two_level_iterator.h
deleted file mode 100755
index 629ca34..0000000
--- a/thirdparty/leveldb-1.18/table/two_level_iterator.h
+++ /dev/null
@@ -1,34 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_
-#define STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_
-
-#include "leveldb/iterator.h"
-
-namespace leveldb {
-
-struct ReadOptions;
-
-// Return a new two level iterator. A two-level iterator contains an
-// index iterator whose values point to a sequence of blocks where
-// each block is itself a sequence of key,value pairs. The returned
-// two-level iterator yields the concatenation of all key/value pairs
-// in the sequence of blocks. Takes ownership of "index_iter" and
-// will delete it when no longer needed.
-//
-// Uses a supplied function to convert an index_iter value into
-// an iterator over the contents of the corresponding block.
-extern Iterator* NewTwoLevelIterator(
- Iterator* index_iter,
- Iterator* (*block_function)(
- void* arg,
- const ReadOptions& options,
- const Slice& index_value),
- void* arg,
- const ReadOptions& options);
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_TABLE_TWO_LEVEL_ITERATOR_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/arena.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/arena.cc b/thirdparty/leveldb-1.18/util/arena.cc
deleted file mode 100755
index 9367f71..0000000
--- a/thirdparty/leveldb-1.18/util/arena.cc
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "util/arena.h"
-#include <assert.h>
-
-namespace leveldb {
-
-static const int kBlockSize = 4096;
-
-Arena::Arena() {
- blocks_memory_ = 0;
- alloc_ptr_ = NULL; // First allocation will allocate a block
- alloc_bytes_remaining_ = 0;
-}
-
-Arena::~Arena() {
- for (size_t i = 0; i < blocks_.size(); i++) {
- delete[] blocks_[i];
- }
-}
-
-char* Arena::AllocateFallback(size_t bytes) {
- if (bytes > kBlockSize / 4) {
- // Object is more than a quarter of our block size. Allocate it separately
- // to avoid wasting too much space in leftover bytes.
- char* result = AllocateNewBlock(bytes);
- return result;
- }
-
- // We waste the remaining space in the current block.
- alloc_ptr_ = AllocateNewBlock(kBlockSize);
- alloc_bytes_remaining_ = kBlockSize;
-
- char* result = alloc_ptr_;
- alloc_ptr_ += bytes;
- alloc_bytes_remaining_ -= bytes;
- return result;
-}
-
-char* Arena::AllocateAligned(size_t bytes) {
- const int align = (sizeof(void*) > 8) ? sizeof(void*) : 8;
- assert((align & (align-1)) == 0); // Pointer size should be a power of 2
- size_t current_mod = reinterpret_cast<uintptr_t>(alloc_ptr_) & (align-1);
- size_t slop = (current_mod == 0 ? 0 : align - current_mod);
- size_t needed = bytes + slop;
- char* result;
- if (needed <= alloc_bytes_remaining_) {
- result = alloc_ptr_ + slop;
- alloc_ptr_ += needed;
- alloc_bytes_remaining_ -= needed;
- } else {
- // AllocateFallback always returned aligned memory
- result = AllocateFallback(bytes);
- }
- assert((reinterpret_cast<uintptr_t>(result) & (align-1)) == 0);
- return result;
-}
-
-char* Arena::AllocateNewBlock(size_t block_bytes) {
- char* result = new char[block_bytes];
- blocks_memory_ += block_bytes;
- blocks_.push_back(result);
- return result;
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/arena.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/arena.h b/thirdparty/leveldb-1.18/util/arena.h
deleted file mode 100755
index 73bbf1c..0000000
--- a/thirdparty/leveldb-1.18/util/arena.h
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_UTIL_ARENA_H_
-#define STORAGE_LEVELDB_UTIL_ARENA_H_
-
-#include <vector>
-#include <assert.h>
-#include <stddef.h>
-#include <stdint.h>
-
-namespace leveldb {
-
-class Arena {
- public:
- Arena();
- ~Arena();
-
- // Return a pointer to a newly allocated memory block of "bytes" bytes.
- char* Allocate(size_t bytes);
-
- // Allocate memory with the normal alignment guarantees provided by malloc
- char* AllocateAligned(size_t bytes);
-
- // Returns an estimate of the total memory usage of data allocated
- // by the arena (including space allocated but not yet used for user
- // allocations).
- size_t MemoryUsage() const {
- return blocks_memory_ + blocks_.capacity() * sizeof(char*);
- }
-
- private:
- char* AllocateFallback(size_t bytes);
- char* AllocateNewBlock(size_t block_bytes);
-
- // Allocation state
- char* alloc_ptr_;
- size_t alloc_bytes_remaining_;
-
- // Array of new[] allocated memory blocks
- std::vector<char*> blocks_;
-
- // Bytes of memory in blocks allocated so far
- size_t blocks_memory_;
-
- // No copying allowed
- Arena(const Arena&);
- void operator=(const Arena&);
-};
-
-inline char* Arena::Allocate(size_t bytes) {
- // The semantics of what to return are a bit messy if we allow
- // 0-byte allocations, so we disallow them here (we don't need
- // them for our internal use).
- assert(bytes > 0);
- if (bytes <= alloc_bytes_remaining_) {
- char* result = alloc_ptr_;
- alloc_ptr_ += bytes;
- alloc_bytes_remaining_ -= bytes;
- return result;
- }
- return AllocateFallback(bytes);
-}
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_UTIL_ARENA_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/arena_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/arena_test.cc b/thirdparty/leveldb-1.18/util/arena_test.cc
deleted file mode 100755
index 58e870e..0000000
--- a/thirdparty/leveldb-1.18/util/arena_test.cc
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "util/arena.h"
-
-#include "util/random.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-class ArenaTest { };
-
-TEST(ArenaTest, Empty) {
- Arena arena;
-}
-
-TEST(ArenaTest, Simple) {
- std::vector<std::pair<size_t, char*> > allocated;
- Arena arena;
- const int N = 100000;
- size_t bytes = 0;
- Random rnd(301);
- for (int i = 0; i < N; i++) {
- size_t s;
- if (i % (N / 10) == 0) {
- s = i;
- } else {
- s = rnd.OneIn(4000) ? rnd.Uniform(6000) :
- (rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20));
- }
- if (s == 0) {
- // Our arena disallows size 0 allocations.
- s = 1;
- }
- char* r;
- if (rnd.OneIn(10)) {
- r = arena.AllocateAligned(s);
- } else {
- r = arena.Allocate(s);
- }
-
- for (size_t b = 0; b < s; b++) {
- // Fill the "i"th allocation with a known bit pattern
- r[b] = i % 256;
- }
- bytes += s;
- allocated.push_back(std::make_pair(s, r));
- ASSERT_GE(arena.MemoryUsage(), bytes);
- if (i > N/10) {
- ASSERT_LE(arena.MemoryUsage(), bytes * 1.10);
- }
- }
- for (size_t i = 0; i < allocated.size(); i++) {
- size_t num_bytes = allocated[i].first;
- const char* p = allocated[i].second;
- for (size_t b = 0; b < num_bytes; b++) {
- // Check the "i"th allocation for the known bit pattern
- ASSERT_EQ(int(p[b]) & 0xff, i % 256);
- }
- }
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/bloom.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/bloom.cc b/thirdparty/leveldb-1.18/util/bloom.cc
deleted file mode 100755
index 27a2b08..0000000
--- a/thirdparty/leveldb-1.18/util/bloom.cc
+++ /dev/null
@@ -1,95 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/filter_policy.h"
-
-#include "leveldb/slice.h"
-#include "util/hash.h"
-
-namespace leveldb {
-
-namespace {
-static uint32_t BloomHash(const Slice& key) {
- return Hash(reinterpret_cast<const unsigned char *>(key.data()), key.size(), 0xbc9f1d34);
-}
-
-class BloomFilterPolicy : public FilterPolicy {
- private:
- size_t bits_per_key_;
- size_t k_;
-
- public:
- explicit BloomFilterPolicy(int bits_per_key)
- : bits_per_key_(bits_per_key) {
- // We intentionally round down to reduce probing cost a little bit
- k_ = static_cast<size_t>(bits_per_key * 0.69); // 0.69 =~ ln(2)
- if (k_ < 1) k_ = 1;
- if (k_ > 30) k_ = 30;
- }
-
- virtual const char* Name() const {
- return "leveldb.BuiltinBloomFilter2";
- }
-
- virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {
- // Compute bloom filter size (in both bits and bytes)
- size_t bits = n * bits_per_key_;
-
- // For small n, we can see a very high false positive rate. Fix it
- // by enforcing a minimum bloom filter length.
- if (bits < 64) bits = 64;
-
- size_t bytes = (bits + 7) / 8;
- bits = bytes * 8;
-
- const size_t init_size = dst->size();
- dst->resize(init_size + bytes, 0);
- dst->push_back(static_cast<char>(k_)); // Remember # of probes in filter
- char* array = &(*dst)[init_size];
- for (size_t i = 0; i < n; i++) {
- // Use double-hashing to generate a sequence of hash values.
- // See analysis in [Kirsch,Mitzenmacher 2006].
- uint32_t h = BloomHash(keys[i]);
- const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
- for (size_t j = 0; j < k_; j++) {
- const uint32_t bitpos = h % bits;
- array[bitpos/8] |= (1 << (bitpos % 8));
- h += delta;
- }
- }
- }
-
- virtual bool KeyMayMatch(const Slice& key, const Slice& bloom_filter) const {
- const size_t len = bloom_filter.size();
- if (len < 2) return false;
-
- const char* array = bloom_filter.data();
- const size_t bits = (len - 1) * 8;
-
- // Use the encoded k so that we can read filters generated by
- // bloom filters created using different parameters.
- const size_t k = array[len-1];
- if (k > 30) {
- // Reserved for potentially new encodings for short bloom filters.
- // Consider it a match.
- return true;
- }
-
- uint32_t h = BloomHash(key);
- const uint32_t delta = (h >> 17) | (h << 15); // Rotate right 17 bits
- for (size_t j = 0; j < k; j++) {
- const uint32_t bitpos = h % bits;
- if ((array[bitpos/8] & (1 << (bitpos % 8))) == 0) return false;
- h += delta;
- }
- return true;
- }
-};
-}
-
-const FilterPolicy* NewBloomFilterPolicy(int bits_per_key) {
- return new BloomFilterPolicy(bits_per_key);
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/bloom_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/bloom_test.cc b/thirdparty/leveldb-1.18/util/bloom_test.cc
deleted file mode 100755
index bfb9306..0000000
--- a/thirdparty/leveldb-1.18/util/bloom_test.cc
+++ /dev/null
@@ -1,161 +0,0 @@
-// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/filter_policy.h"
-
-#include "util/coding.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-static const int kVerbose = 1;
-
-static Slice Key(int i, char* buffer) {
- EncodeFixed32(buffer, i);
- return Slice(buffer, sizeof(uint32_t));
-}
-
-class BloomTest {
- private:
- const FilterPolicy* policy_;
- std::string filter_;
- std::vector<std::string> keys_;
-
- public:
- BloomTest() : policy_(NewBloomFilterPolicy(10)) { }
-
- ~BloomTest() {
- delete policy_;
- }
-
- void Reset() {
- keys_.clear();
- filter_.clear();
- }
-
- void Add(const Slice& s) {
- keys_.push_back(s.ToString());
- }
-
- void Build() {
- std::vector<Slice> key_slices;
- for (size_t i = 0; i < keys_.size(); i++) {
- key_slices.push_back(Slice(keys_[i]));
- }
- filter_.clear();
- policy_->CreateFilter(&key_slices[0], static_cast<int>(key_slices.size()), &filter_);
- keys_.clear();
- if (kVerbose >= 2) DumpFilter();
- }
-
- size_t FilterSize() const {
- return filter_.size();
- }
-
- void DumpFilter() {
- fprintf(stderr, "F(");
- for (size_t i = 0; i+1 < filter_.size(); i++) {
- const unsigned int c = static_cast<unsigned int>(filter_[i]);
- for (int j = 0; j < 8; j++) {
- fprintf(stderr, "%c", (c & (1 <<j)) ? '1' : '.');
- }
- }
- fprintf(stderr, ")\n");
- }
-
- bool Matches(const Slice& s) {
- if (!keys_.empty()) {
- Build();
- }
- return policy_->KeyMayMatch(s, filter_);
- }
-
- double FalsePositiveRate() {
- char buffer[sizeof(int)];
- int result = 0;
- for (int i = 0; i < 10000; i++) {
- if (Matches(Key(i + 1000000000, buffer))) {
- result++;
- }
- }
- return result / 10000.0;
- }
-};
-
-TEST(BloomTest, EmptyFilter) {
- ASSERT_TRUE(! Matches("hello"));
- ASSERT_TRUE(! Matches("world"));
-}
-
-TEST(BloomTest, Small) {
- Add("hello");
- Add("world");
- ASSERT_TRUE(Matches("hello"));
- ASSERT_TRUE(Matches("world"));
- ASSERT_TRUE(! Matches("x"));
- ASSERT_TRUE(! Matches("foo"));
-}
-
-static int NextLength(int length) {
- if (length < 10) {
- length += 1;
- } else if (length < 100) {
- length += 10;
- } else if (length < 1000) {
- length += 100;
- } else {
- length += 1000;
- }
- return length;
-}
-
-TEST(BloomTest, VaryingLengths) {
- char buffer[sizeof(int)];
-
- // Count number of filters that significantly exceed the false positive rate
- int mediocre_filters = 0;
- int good_filters = 0;
-
- for (int length = 1; length <= 10000; length = NextLength(length)) {
- Reset();
- for (int i = 0; i < length; i++) {
- Add(Key(i, buffer));
- }
- Build();
-
- ASSERT_LE(FilterSize(), static_cast<size_t>((length * 10 / 8) + 40))
- << length;
-
- // All added keys must match
- for (int i = 0; i < length; i++) {
- ASSERT_TRUE(Matches(Key(i, buffer)))
- << "Length " << length << "; key " << i;
- }
-
- // Check false positive rate
- double rate = FalsePositiveRate();
- if (kVerbose >= 1) {
- fprintf(stderr, "False positives: %5.2f%% @ length = %6d ; bytes = %6d\n",
- rate*100.0, length, static_cast<int>(FilterSize()));
- }
- ASSERT_LE(rate, 0.02); // Must not be over 2%
- if (rate > 0.0125) mediocre_filters++; // Allowed, but not too often
- else good_filters++;
- }
- if (kVerbose >= 1) {
- fprintf(stderr, "Filters: %d good, %d mediocre\n",
- good_filters, mediocre_filters);
- }
- ASSERT_LE(mediocre_filters, good_filters/5);
-}
-
-// Different bits-per-byte
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/cache.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/cache.cc b/thirdparty/leveldb-1.18/util/cache.cc
deleted file mode 100755
index ff8d2fc..0000000
--- a/thirdparty/leveldb-1.18/util/cache.cc
+++ /dev/null
@@ -1,327 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <assert.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "leveldb/cache.h"
-#include "port/port.h"
-#include "util/hash.h"
-#include "util/mutexlock.h"
-
-namespace leveldb {
-
-Cache::~Cache() {
-}
-
-namespace {
-
-// LRU cache implementation
-
-// An entry is a variable length heap-allocated structure. Entries
-// are kept in a circular doubly linked list ordered by access time.
-struct LRUHandle {
- void* value;
- void (*deleter)(const Slice&, void* value);
- LRUHandle* next_hash;
- LRUHandle* next;
- LRUHandle* prev;
- size_t charge; // TODO(opt): Only allow uint32_t?
- size_t key_length;
- uint32_t refs;
- uint32_t hash; // Hash of key(); used for fast sharding and comparisons
- char key_data[1]; // Beginning of key
-
- Slice key() const {
- // For cheaper lookups, we allow a temporary Handle object
- // to store a pointer to a key in "value".
- if (next == this) {
- return *(reinterpret_cast<Slice*>(value));
- } else {
- return Slice(key_data, key_length);
- }
- }
-};
-
-// We provide our own simple hash table since it removes a whole bunch
-// of porting hacks and is also faster than some of the built-in hash
-// table implementations in some of the compiler/runtime combinations
-// we have tested. E.g., readrandom speeds up by ~5% over the g++
-// 4.4.3's builtin hashtable.
-class HandleTable {
- public:
- HandleTable() : length_(0), elems_(0), list_(NULL) { Resize(); }
- ~HandleTable() { delete[] list_; }
-
- LRUHandle* Lookup(const Slice& key, uint32_t hash) {
- return *FindPointer(key, hash);
- }
-
- LRUHandle* Insert(LRUHandle* h) {
- LRUHandle** ptr = FindPointer(h->key(), h->hash);
- LRUHandle* old = *ptr;
- h->next_hash = (old == NULL ? NULL : old->next_hash);
- *ptr = h;
- if (old == NULL) {
- ++elems_;
- if (elems_ > length_) {
- // Since each cache entry is fairly large, we aim for a small
- // average linked list length (<= 1).
- Resize();
- }
- }
- return old;
- }
-
- LRUHandle* Remove(const Slice& key, uint32_t hash) {
- LRUHandle** ptr = FindPointer(key, hash);
- LRUHandle* result = *ptr;
- if (result != NULL) {
- *ptr = result->next_hash;
- --elems_;
- }
- return result;
- }
-
- private:
- // The table consists of an array of buckets where each bucket is
- // a linked list of cache entries that hash into the bucket.
- uint32_t length_;
- uint32_t elems_;
- LRUHandle** list_;
-
- // Return a pointer to slot that points to a cache entry that
- // matches key/hash. If there is no such cache entry, return a
- // pointer to the trailing slot in the corresponding linked list.
- LRUHandle** FindPointer(const Slice& key, uint32_t hash) {
- LRUHandle** ptr = &list_[hash & (length_ - 1)];
- while (*ptr != NULL &&
- ((*ptr)->hash != hash || key != (*ptr)->key())) {
- ptr = &(*ptr)->next_hash;
- }
- return ptr;
- }
-
- void Resize() {
- uint32_t new_length = 4;
- while (new_length < elems_) {
- new_length *= 2;
- }
- LRUHandle** new_list = new LRUHandle*[new_length];
- memset(new_list, 0, sizeof(new_list[0]) * new_length);
- uint32_t count = 0;
- for (uint32_t i = 0; i < length_; i++) {
- LRUHandle* h = list_[i];
- while (h != NULL) {
- LRUHandle* next = h->next_hash;
- uint32_t hash = h->hash;
- LRUHandle** ptr = &new_list[hash & (new_length - 1)];
- h->next_hash = *ptr;
- *ptr = h;
- h = next;
- count++;
- }
- }
- assert(elems_ == count);
- delete[] list_;
- list_ = new_list;
- length_ = new_length;
- }
-};
-
-// A single shard of sharded cache.
-class LRUCache {
- public:
- LRUCache();
- ~LRUCache();
-
- // Separate from constructor so caller can easily make an array of LRUCache
- void SetCapacity(size_t capacity) { capacity_ = capacity; }
-
- // Like Cache methods, but with an extra "hash" parameter.
- Cache::Handle* Insert(const Slice& key, uint32_t hash,
- void* value, size_t charge,
- void (*deleter)(const Slice& key, void* value));
- Cache::Handle* Lookup(const Slice& key, uint32_t hash);
- void Release(Cache::Handle* handle);
- void Erase(const Slice& key, uint32_t hash);
-
- private:
- void LRU_Remove(LRUHandle* e);
- void LRU_Append(LRUHandle* e);
- void Unref(LRUHandle* e);
-
- // Initialized before use.
- size_t capacity_;
-
- // mutex_ protects the following state.
- port::Mutex mutex_;
- size_t usage_;
-
- // Dummy head of LRU list.
- // lru.prev is newest entry, lru.next is oldest entry.
- LRUHandle lru_;
-
- HandleTable table_;
-};
-
-LRUCache::LRUCache()
- : usage_(0) {
- // Make empty circular linked list
- lru_.next = &lru_;
- lru_.prev = &lru_;
-}
-
-LRUCache::~LRUCache() {
- for (LRUHandle* e = lru_.next; e != &lru_; ) {
- LRUHandle* next = e->next;
- assert(e->refs == 1); // Error if caller has an unreleased handle
- Unref(e);
- e = next;
- }
-}
-
-void LRUCache::Unref(LRUHandle* e) {
- assert(e->refs > 0);
- e->refs--;
- if (e->refs <= 0) {
- usage_ -= e->charge;
- (*e->deleter)(e->key(), e->value);
- free(e);
- }
-}
-
-void LRUCache::LRU_Remove(LRUHandle* e) {
- e->next->prev = e->prev;
- e->prev->next = e->next;
-}
-
-void LRUCache::LRU_Append(LRUHandle* e) {
- // Make "e" newest entry by inserting just before lru_
- e->next = &lru_;
- e->prev = lru_.prev;
- e->prev->next = e;
- e->next->prev = e;
-}
-
-Cache::Handle* LRUCache::Lookup(const Slice& key, uint32_t hash) {
- MutexLock l(&mutex_);
- LRUHandle* e = table_.Lookup(key, hash);
- if (e != NULL) {
- e->refs++;
- LRU_Remove(e);
- LRU_Append(e);
- }
- return reinterpret_cast<Cache::Handle*>(e);
-}
-
-void LRUCache::Release(Cache::Handle* handle) {
- MutexLock l(&mutex_);
- Unref(reinterpret_cast<LRUHandle*>(handle));
-}
-
-Cache::Handle* LRUCache::Insert(
- const Slice& key, uint32_t hash, void* value, size_t charge,
- void (*deleter)(const Slice& key, void* value)) {
- MutexLock l(&mutex_);
-
- LRUHandle* e = reinterpret_cast<LRUHandle*>(
- malloc(sizeof(LRUHandle)-1 + key.size()));
- e->value = value;
- e->deleter = deleter;
- e->charge = charge;
- e->key_length = key.size();
- e->hash = hash;
- e->refs = 2; // One from LRUCache, one for the returned handle
- memcpy(e->key_data, key.data(), key.size());
- LRU_Append(e);
- usage_ += charge;
-
- {
- LRUHandle* old = table_.Insert(e);
- if (old != NULL) {
- LRU_Remove(old);
- Unref(old);
- }
- }
-
- while (usage_ > capacity_ && lru_.next != &lru_) {
- LRUHandle* old = lru_.next;
- LRU_Remove(old);
- table_.Remove(old->key(), old->hash);
- Unref(old);
- }
-
- return reinterpret_cast<Cache::Handle*>(e);
-}
-
-void LRUCache::Erase(const Slice& key, uint32_t hash) {
- MutexLock l(&mutex_);
- LRUHandle* e = table_.Remove(key, hash);
- if (e != NULL) {
- LRU_Remove(e);
- Unref(e);
- }
-}
-
-static const int kNumShardBits = 4;
-static const int kNumShards = 1 << kNumShardBits;
-
-class ShardedLRUCache : public Cache {
- private:
- LRUCache shard_[kNumShards];
- port::Mutex id_mutex_;
- uint64_t last_id_;
-
- static inline uint32_t HashSlice(const Slice& s) {
- return Hash(reinterpret_cast<const unsigned char *>(s.data()), s.size(), 0);
- }
-
- static uint32_t Shard(uint32_t hash) {
- return hash >> (32 - kNumShardBits);
- }
-
- public:
- explicit ShardedLRUCache(size_t capacity)
- : last_id_(0) {
- const size_t per_shard = (capacity + (kNumShards - 1)) / kNumShards;
- for (int s = 0; s < kNumShards; s++) {
- shard_[s].SetCapacity(per_shard);
- }
- }
- virtual ~ShardedLRUCache() { }
- virtual Handle* Insert(const Slice& key, void* value, size_t charge,
- void (*deleter)(const Slice& key, void* value)) {
- const uint32_t hash = HashSlice(key);
- return shard_[Shard(hash)].Insert(key, hash, value, charge, deleter);
- }
- virtual Handle* Lookup(const Slice& key) {
- const uint32_t hash = HashSlice(key);
- return shard_[Shard(hash)].Lookup(key, hash);
- }
- virtual void Release(Handle* handle) {
- LRUHandle* h = reinterpret_cast<LRUHandle*>(handle);
- shard_[Shard(h->hash)].Release(handle);
- }
- virtual void Erase(const Slice& key) {
- const uint32_t hash = HashSlice(key);
- shard_[Shard(hash)].Erase(key, hash);
- }
- virtual void* Value(Handle* handle) {
- return reinterpret_cast<LRUHandle*>(handle)->value;
- }
- virtual uint64_t NewId() {
- MutexLock l(&id_mutex_);
- return ++(last_id_);
- }
-};
-
-} // end anonymous namespace
-
-Cache* NewLRUCache(size_t capacity) {
- return new ShardedLRUCache(capacity);
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/cache_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/cache_test.cc b/thirdparty/leveldb-1.18/util/cache_test.cc
deleted file mode 100755
index d9b9057..0000000
--- a/thirdparty/leveldb-1.18/util/cache_test.cc
+++ /dev/null
@@ -1,186 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/cache.h"
-
-#include <vector>
-#include "util/coding.h"
-#include "util/testharness.h"
-
-namespace leveldb {
-
-// Conversions between numeric keys/values and the types expected by Cache.
-static std::string EncodeKey(intptr_t k) {
- std::string result;
- PutFixed32(&result, static_cast<uint32_t>(k));
- return result;
-}
-static int DecodeKey(const Slice& k) {
- assert(k.size() == 4);
- return DecodeFixed32(k.data());
-}
-static void* EncodeValue(intptr_t v) { return reinterpret_cast<void*>(v); }
-static intptr_t DecodeValue(void* v) { return reinterpret_cast<intptr_t>(v); }
-
-class CacheTest {
- public:
- static CacheTest* current_;
-
- static void Deleter(const Slice& key, void* v) {
- current_->deleted_keys_.push_back(DecodeKey(key));
- current_->deleted_values_.push_back(DecodeValue(v));
- }
-
- static const int kCacheSize = 1000;
- std::vector<intptr_t> deleted_keys_;
- std::vector<intptr_t> deleted_values_;
- Cache* cache_;
-
- CacheTest() : cache_(NewLRUCache(kCacheSize)) {
- current_ = this;
- }
-
- ~CacheTest() {
- delete cache_;
- }
-
- intptr_t Lookup(int key) {
- Cache::Handle* handle = cache_->Lookup(EncodeKey(key));
- const intptr_t r = (handle == NULL) ? -1 : DecodeValue(cache_->Value(handle));
- if (handle != NULL) {
- cache_->Release(handle);
- }
- return r;
- }
-
- void Insert(int key, int value, int charge = 1) {
- cache_->Release(cache_->Insert(EncodeKey(key), EncodeValue(value), charge,
- &CacheTest::Deleter));
- }
-
- void Erase(int key) {
- cache_->Erase(EncodeKey(key));
- }
-};
-CacheTest* CacheTest::current_;
-
-TEST(CacheTest, HitAndMiss) {
- ASSERT_EQ(-1, Lookup(100));
-
- Insert(100, 101);
- ASSERT_EQ(101, Lookup(100));
- ASSERT_EQ(-1, Lookup(200));
- ASSERT_EQ(-1, Lookup(300));
-
- Insert(200, 201);
- ASSERT_EQ(101, Lookup(100));
- ASSERT_EQ(201, Lookup(200));
- ASSERT_EQ(-1, Lookup(300));
-
- Insert(100, 102);
- ASSERT_EQ(102, Lookup(100));
- ASSERT_EQ(201, Lookup(200));
- ASSERT_EQ(-1, Lookup(300));
-
- ASSERT_EQ(1, deleted_keys_.size());
- ASSERT_EQ(100, deleted_keys_[0]);
- ASSERT_EQ(101, deleted_values_[0]);
-}
-
-TEST(CacheTest, Erase) {
- Erase(200);
- ASSERT_EQ(0, deleted_keys_.size());
-
- Insert(100, 101);
- Insert(200, 201);
- Erase(100);
- ASSERT_EQ(-1, Lookup(100));
- ASSERT_EQ(201, Lookup(200));
- ASSERT_EQ(1, deleted_keys_.size());
- ASSERT_EQ(100, deleted_keys_[0]);
- ASSERT_EQ(101, deleted_values_[0]);
-
- Erase(100);
- ASSERT_EQ(-1, Lookup(100));
- ASSERT_EQ(201, Lookup(200));
- ASSERT_EQ(1, deleted_keys_.size());
-}
-
-TEST(CacheTest, EntriesArePinned) {
- Insert(100, 101);
- Cache::Handle* h1 = cache_->Lookup(EncodeKey(100));
- ASSERT_EQ(101, DecodeValue(cache_->Value(h1)));
-
- Insert(100, 102);
- Cache::Handle* h2 = cache_->Lookup(EncodeKey(100));
- ASSERT_EQ(102, DecodeValue(cache_->Value(h2)));
- ASSERT_EQ(0, deleted_keys_.size());
-
- cache_->Release(h1);
- ASSERT_EQ(1, deleted_keys_.size());
- ASSERT_EQ(100, deleted_keys_[0]);
- ASSERT_EQ(101, deleted_values_[0]);
-
- Erase(100);
- ASSERT_EQ(-1, Lookup(100));
- ASSERT_EQ(1, deleted_keys_.size());
-
- cache_->Release(h2);
- ASSERT_EQ(2, deleted_keys_.size());
- ASSERT_EQ(100, deleted_keys_[1]);
- ASSERT_EQ(102, deleted_values_[1]);
-}
-
-TEST(CacheTest, EvictionPolicy) {
- Insert(100, 101);
- Insert(200, 201);
-
- // Frequently used entry must be kept around
- for (int i = 0; i < kCacheSize + 100; i++) {
- Insert(1000+i, 2000+i);
- ASSERT_EQ(2000+i, Lookup(1000+i));
- ASSERT_EQ(101, Lookup(100));
- }
- ASSERT_EQ(101, Lookup(100));
- ASSERT_EQ(-1, Lookup(200));
-}
-
-TEST(CacheTest, HeavyEntries) {
- // Add a bunch of light and heavy entries and then count the combined
- // size of items still in the cache, which must be approximately the
- // same as the total capacity.
- const int kLight = 1;
- const int kHeavy = 10;
- int added = 0;
- int index = 0;
- while (added < 2*kCacheSize) {
- const int weight = (index & 1) ? kLight : kHeavy;
- Insert(index, 1000+index, weight);
- added += weight;
- index++;
- }
-
- intptr_t cached_weight = 0;
- for (int i = 0; i < index; i++) {
- const intptr_t weight = (i & 1 ? kLight : kHeavy);
- intptr_t r = Lookup(i);
- if (r >= 0) {
- cached_weight += weight;
- ASSERT_EQ(1000+i, r);
- }
- }
- ASSERT_LE(cached_weight, kCacheSize + kCacheSize/10);
-}
-
-TEST(CacheTest, NewId) {
- uint64_t a = cache_->NewId();
- uint64_t b = cache_->NewId();
- ASSERT_NE(a, b);
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/coding.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/coding.cc b/thirdparty/leveldb-1.18/util/coding.cc
deleted file mode 100755
index dfa826a..0000000
--- a/thirdparty/leveldb-1.18/util/coding.cc
+++ /dev/null
@@ -1,194 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "util/coding.h"
-
-namespace leveldb {
-
-void EncodeFixed32(char* buf, uint32_t value) {
- if (port::kLittleEndian) {
- memcpy(buf, &value, sizeof(value));
- } else {
- buf[0] = value & 0xff;
- buf[1] = (value >> 8) & 0xff;
- buf[2] = (value >> 16) & 0xff;
- buf[3] = (value >> 24) & 0xff;
- }
-}
-
-void EncodeFixed64(char* buf, uint64_t value) {
- if (port::kLittleEndian) {
- memcpy(buf, &value, sizeof(value));
- } else {
- buf[0] = value & 0xff;
- buf[1] = (value >> 8) & 0xff;
- buf[2] = (value >> 16) & 0xff;
- buf[3] = (value >> 24) & 0xff;
- buf[4] = (value >> 32) & 0xff;
- buf[5] = (value >> 40) & 0xff;
- buf[6] = (value >> 48) & 0xff;
- buf[7] = (value >> 56) & 0xff;
- }
-}
-
-void PutFixed32(std::string* dst, uint32_t value) {
- char buf[sizeof(value)];
- EncodeFixed32(buf, value);
- dst->append(buf, sizeof(buf));
-}
-
-void PutFixed64(std::string* dst, uint64_t value) {
- char buf[sizeof(value)];
- EncodeFixed64(buf, value);
- dst->append(buf, sizeof(buf));
-}
-
-char* EncodeVarint32(char* dst, uint32_t v) {
- // Operate on characters as unsigneds
- unsigned char* ptr = reinterpret_cast<unsigned char*>(dst);
- static const int B = 128;
- if (v < (1<<7)) {
- *(ptr++) = v;
- } else if (v < (1<<14)) {
- *(ptr++) = v | B;
- *(ptr++) = v>>7;
- } else if (v < (1<<21)) {
- *(ptr++) = v | B;
- *(ptr++) = (v>>7) | B;
- *(ptr++) = v>>14;
- } else if (v < (1<<28)) {
- *(ptr++) = v | B;
- *(ptr++) = (v>>7) | B;
- *(ptr++) = (v>>14) | B;
- *(ptr++) = v>>21;
- } else {
- *(ptr++) = v | B;
- *(ptr++) = (v>>7) | B;
- *(ptr++) = (v>>14) | B;
- *(ptr++) = (v>>21) | B;
- *(ptr++) = v>>28;
- }
- return reinterpret_cast<char*>(ptr);
-}
-
-void PutVarint32(std::string* dst, uint32_t v) {
- char buf[5];
- char* ptr = EncodeVarint32(buf, v);
- dst->append(buf, ptr - buf);
-}
-
-char* EncodeVarint64(char* dst, uint64_t v) {
- static const int B = 128;
- unsigned char* ptr = reinterpret_cast<unsigned char*>(dst);
- while (v >= B) {
- *(ptr++) = (v & (B-1)) | B;
- v >>= 7;
- }
- *(ptr++) = static_cast<unsigned char>(v);
- return reinterpret_cast<char*>(ptr);
-}
-
-void PutVarint64(std::string* dst, uint64_t v) {
- char buf[10];
- char* ptr = EncodeVarint64(buf, v);
- dst->append(buf, ptr - buf);
-}
-
-void PutLengthPrefixedSlice(std::string* dst, const Slice& value) {
- PutVarint32(dst, static_cast<uint32_t>(value.size()));
- dst->append(value.data(), value.size());
-}
-
-int VarintLength(uint64_t v) {
- int len = 1;
- while (v >= 128) {
- v >>= 7;
- len++;
- }
- return len;
-}
-
-const char* GetVarint32PtrFallback(const char* p,
- const char* limit,
- uint32_t* value) {
- uint32_t result = 0;
- for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) {
- uint32_t byte = *(reinterpret_cast<const unsigned char*>(p));
- p++;
- if (byte & 128) {
- // More bytes are present
- result |= ((byte & 127) << shift);
- } else {
- result |= (byte << shift);
- *value = result;
- return reinterpret_cast<const char*>(p);
- }
- }
- return NULL;
-}
-
-bool GetVarint32(Slice* input, uint32_t* value) {
- const char* p = input->data();
- const char* limit = p + input->size();
- const char* q = GetVarint32Ptr(p, limit, value);
- if (q == NULL) {
- return false;
- } else {
- *input = Slice(q, limit - q);
- return true;
- }
-}
-
-const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* value) {
- uint64_t result = 0;
- for (uint32_t shift = 0; shift <= 63 && p < limit; shift += 7) {
- uint64_t byte = *(reinterpret_cast<const unsigned char*>(p));
- p++;
- if (byte & 128) {
- // More bytes are present
- result |= ((byte & 127) << shift);
- } else {
- result |= (byte << shift);
- *value = result;
- return reinterpret_cast<const char*>(p);
- }
- }
- return NULL;
-}
-
-bool GetVarint64(Slice* input, uint64_t* value) {
- const char* p = input->data();
- const char* limit = p + input->size();
- const char* q = GetVarint64Ptr(p, limit, value);
- if (q == NULL) {
- return false;
- } else {
- *input = Slice(q, limit - q);
- return true;
- }
-}
-
-const char* GetLengthPrefixedSlice(const char* p, const char* limit,
- Slice* result) {
- uint32_t len;
- p = GetVarint32Ptr(p, limit, &len);
- if (p == NULL) return NULL;
- if (p + len > limit) return NULL;
- *result = Slice(p, len);
- return p + len;
-}
-
-bool GetLengthPrefixedSlice(Slice* input, Slice* result) {
- uint32_t len;
- if (GetVarint32(input, &len) &&
- input->size() >= len) {
- *result = Slice(input->data(), len);
- input->remove_prefix(len);
- return true;
- } else {
- return false;
- }
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/coding.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/coding.h b/thirdparty/leveldb-1.18/util/coding.h
deleted file mode 100755
index 3da94df..0000000
--- a/thirdparty/leveldb-1.18/util/coding.h
+++ /dev/null
@@ -1,107 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// Endian-neutral encoding:
-// * Fixed-length numbers are encoded with least-significant byte first
-// * In addition we support variable length "varint" encoding
-// * Strings are encoded prefixed by their length in varint format
-
-#ifndef STORAGE_LEVELDB_UTIL_CODING_H_
-#define STORAGE_LEVELDB_UTIL_CODING_H_
-
-#include <stdint.h>
-#include <string.h>
-#include <string>
-#include "leveldb/slice.h"
-#include "port/port.h"
-
-namespace leveldb {
-
-// Standard Put... routines append to a string
-extern void PutFixed32(std::string* dst, uint32_t value);
-extern void PutFixed64(std::string* dst, uint64_t value);
-extern void PutVarint32(std::string* dst, uint32_t value);
-extern void PutVarint64(std::string* dst, uint64_t value);
-extern void PutLengthPrefixedSlice(std::string* dst, const Slice& value);
-
-// Standard Get... routines parse a value from the beginning of a Slice
-// and advance the slice past the parsed value.
-extern bool GetVarint32(Slice* input, uint32_t* value);
-extern bool GetVarint64(Slice* input, uint64_t* value);
-extern bool GetLengthPrefixedSlice(Slice* input, Slice* result);
-
-// Pointer-based variants of GetVarint... These either store a value
-// in *v and return a pointer just past the parsed value, or return
-// NULL on error. These routines only look at bytes in the range
-// [p..limit-1]
-extern const char* GetVarint32Ptr(const char* p,const char* limit, uint32_t* v);
-extern const char* GetVarint64Ptr(const char* p,const char* limit, uint64_t* v);
-
-// Returns the length of the varint32 or varint64 encoding of "v"
-extern int VarintLength(uint64_t v);
-
-// Lower-level versions of Put... that write directly into a character buffer
-// REQUIRES: dst has enough space for the value being written
-extern void EncodeFixed32(char* dst, uint32_t value);
-extern void EncodeFixed64(char* dst, uint64_t value);
-
-// Lower-level versions of Put... that write directly into a character buffer
-// and return a pointer just past the last byte written.
-// REQUIRES: dst has enough space for the value being written
-extern char* EncodeVarint32(char* dst, uint32_t value);
-extern char* EncodeVarint64(char* dst, uint64_t value);
-
-// Lower-level versions of Get... that read directly from a character buffer
-// without any bounds checking.
-
-inline uint32_t DecodeFixed32(const void* ptr) {
- if (port::kLittleEndian) {
- // Load the raw bytes
- uint32_t result;
- memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load
- return result;
- } else {
- const unsigned char * const p = static_cast<const unsigned char *>(ptr);
-
- return ((static_cast<uint32_t>(p[0]))
- | (static_cast<uint32_t>(p[1]) << 8)
- | (static_cast<uint32_t>(p[2]) << 16)
- | (static_cast<uint32_t>(p[3]) << 24));
- }
-}
-
-inline uint64_t DecodeFixed64(const void* ptr) {
- if (port::kLittleEndian) {
- // Load the raw bytes
- uint64_t result;
- memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load
- return result;
- } else {
- const unsigned char * const p = static_cast<const unsigned char *>(ptr);
- uint64_t lo = DecodeFixed32(p);
- uint64_t hi = DecodeFixed32(p + 4);
- return (hi << 32) | lo;
- }
-}
-
-// Internal routine for use by fallback path of GetVarint32Ptr
-extern const char* GetVarint32PtrFallback(const char* p,
- const char* limit,
- uint32_t* value);
-inline const char* GetVarint32Ptr(const char* p,
- const char* limit,
- uint32_t* value) {
- if (p < limit) {
- uint32_t result = *(reinterpret_cast<const unsigned char*>(p));
- if ((result & 128) == 0) {
- *value = result;
- return p + 1;
- }
- }
- return GetVarint32PtrFallback(p, limit, value);
-}
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_UTIL_CODING_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/coding_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/coding_test.cc b/thirdparty/leveldb-1.18/util/coding_test.cc
deleted file mode 100755
index 521541e..0000000
--- a/thirdparty/leveldb-1.18/util/coding_test.cc
+++ /dev/null
@@ -1,196 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "util/coding.h"
-
-#include "util/testharness.h"
-
-namespace leveldb {
-
-class Coding { };
-
-TEST(Coding, Fixed32) {
- std::string s;
- for (uint32_t v = 0; v < 100000; v++) {
- PutFixed32(&s, v);
- }
-
- const char* p = s.data();
- for (uint32_t v = 0; v < 100000; v++) {
- uint32_t actual = DecodeFixed32(p);
- ASSERT_EQ(v, actual);
- p += sizeof(uint32_t);
- }
-}
-
-TEST(Coding, Fixed64) {
- std::string s;
- for (int power = 0; power <= 63; power++) {
- uint64_t v = static_cast<uint64_t>(1) << power;
- PutFixed64(&s, v - 1);
- PutFixed64(&s, v + 0);
- PutFixed64(&s, v + 1);
- }
-
- const char* p = s.data();
- for (int power = 0; power <= 63; power++) {
- uint64_t v = static_cast<uint64_t>(1) << power;
- uint64_t actual;
- actual = DecodeFixed64(p);
- ASSERT_EQ(v-1, actual);
- p += sizeof(uint64_t);
-
- actual = DecodeFixed64(p);
- ASSERT_EQ(v+0, actual);
- p += sizeof(uint64_t);
-
- actual = DecodeFixed64(p);
- ASSERT_EQ(v+1, actual);
- p += sizeof(uint64_t);
- }
-}
-
-// Test that encoding routines generate little-endian encodings
-TEST(Coding, EncodingOutput) {
- std::string dst;
- PutFixed32(&dst, 0x04030201);
- ASSERT_EQ(4, dst.size());
- ASSERT_EQ(0x01, static_cast<int>(dst[0]));
- ASSERT_EQ(0x02, static_cast<int>(dst[1]));
- ASSERT_EQ(0x03, static_cast<int>(dst[2]));
- ASSERT_EQ(0x04, static_cast<int>(dst[3]));
-
- dst.clear();
- PutFixed64(&dst, 0x0807060504030201ull);
- ASSERT_EQ(8, dst.size());
- ASSERT_EQ(0x01, static_cast<int>(dst[0]));
- ASSERT_EQ(0x02, static_cast<int>(dst[1]));
- ASSERT_EQ(0x03, static_cast<int>(dst[2]));
- ASSERT_EQ(0x04, static_cast<int>(dst[3]));
- ASSERT_EQ(0x05, static_cast<int>(dst[4]));
- ASSERT_EQ(0x06, static_cast<int>(dst[5]));
- ASSERT_EQ(0x07, static_cast<int>(dst[6]));
- ASSERT_EQ(0x08, static_cast<int>(dst[7]));
-}
-
-TEST(Coding, Varint32) {
- std::string s;
- for (uint32_t i = 0; i < (32 * 32); i++) {
- uint32_t v = (i / 32) << (i % 32);
- PutVarint32(&s, v);
- }
-
- const char* p = s.data();
- const char* limit = p + s.size();
- for (uint32_t i = 0; i < (32 * 32); i++) {
- uint32_t expected = (i / 32) << (i % 32);
- uint32_t actual;
- const char* start = p;
- p = GetVarint32Ptr(p, limit, &actual);
- ASSERT_TRUE(p != NULL);
- ASSERT_EQ(expected, actual);
- ASSERT_EQ(VarintLength(actual), p - start);
- }
- ASSERT_EQ(p, s.data() + s.size());
-}
-
-TEST(Coding, Varint64) {
- // Construct the list of values to check
- std::vector<uint64_t> values;
- // Some special values
- values.push_back(0);
- values.push_back(100);
- values.push_back(~static_cast<uint64_t>(0));
- values.push_back(~static_cast<uint64_t>(0) - 1);
- for (uint32_t k = 0; k < 64; k++) {
- // Test values near powers of two
- const uint64_t power = 1ull << k;
- values.push_back(power);
- values.push_back(power-1);
- values.push_back(power+1);
- }
-
- std::string s;
- for (size_t i = 0; i < values.size(); i++) {
- PutVarint64(&s, values[i]);
- }
-
- const char* p = s.data();
- const char* limit = p + s.size();
- for (size_t i = 0; i < values.size(); i++) {
- ASSERT_TRUE(p < limit);
- uint64_t actual;
- const char* start = p;
- p = GetVarint64Ptr(p, limit, &actual);
- ASSERT_TRUE(p != NULL);
- ASSERT_EQ(values[i], actual);
- ASSERT_EQ(VarintLength(actual), p - start);
- }
- ASSERT_EQ(p, limit);
-
-}
-
-TEST(Coding, Varint32Overflow) {
- uint32_t result;
- std::string input("\x81\x82\x83\x84\x85\x11");
- ASSERT_TRUE(GetVarint32Ptr(input.data(), input.data() + input.size(), &result)
- == NULL);
-}
-
-TEST(Coding, Varint32Truncation) {
- uint32_t large_value = (1u << 31) + 100;
- std::string s;
- PutVarint32(&s, large_value);
- uint32_t result;
- for (size_t len = 0; len < s.size() - 1; len++) {
- ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + len, &result) == NULL);
- }
- ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + s.size(), &result) != NULL);
- ASSERT_EQ(large_value, result);
-}
-
-TEST(Coding, Varint64Overflow) {
- uint64_t result;
- std::string input("\x81\x82\x83\x84\x85\x81\x82\x83\x84\x85\x11");
- ASSERT_TRUE(GetVarint64Ptr(input.data(), input.data() + input.size(), &result)
- == NULL);
-}
-
-TEST(Coding, Varint64Truncation) {
- uint64_t large_value = (1ull << 63) + 100ull;
- std::string s;
- PutVarint64(&s, large_value);
- uint64_t result;
- for (size_t len = 0; len < s.size() - 1; len++) {
- ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + len, &result) == NULL);
- }
- ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + s.size(), &result) != NULL);
- ASSERT_EQ(large_value, result);
-}
-
-TEST(Coding, Strings) {
- std::string s;
- PutLengthPrefixedSlice(&s, Slice(""));
- PutLengthPrefixedSlice(&s, Slice("foo"));
- PutLengthPrefixedSlice(&s, Slice("bar"));
- PutLengthPrefixedSlice(&s, Slice(std::string(200, 'x')));
-
- Slice input(s);
- Slice v;
- ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v));
- ASSERT_EQ("", v.ToString());
- ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v));
- ASSERT_EQ("foo", v.ToString());
- ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v));
- ASSERT_EQ("bar", v.ToString());
- ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v));
- ASSERT_EQ(std::string(200, 'x'), v.ToString());
- ASSERT_EQ("", input.ToString());
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/util/comparator.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/util/comparator.cc b/thirdparty/leveldb-1.18/util/comparator.cc
deleted file mode 100755
index 5397929..0000000
--- a/thirdparty/leveldb-1.18/util/comparator.cc
+++ /dev/null
@@ -1,88 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <algorithm>
-#include <stdint.h>
-#include "leveldb/comparator.h"
-#include "leveldb/slice.h"
-#include "port/port.h"
-#include "util/logging.h"
-
-namespace leveldb {
-
-Comparator::~Comparator() { }
-
-namespace {
-class BytewiseComparatorImpl : public Comparator {
- public:
- BytewiseComparatorImpl() { }
-
- virtual const char* Name() const {
- return "leveldb.BytewiseComparator";
- }
-
- virtual int Compare(const Slice& a, const Slice& b) const {
- return a.compare(b);
- }
-
- virtual void FindShortestSeparator(
- std::string* start,
- const Slice& limit) const {
- // Find length of common prefix
- size_t min_length = std::min(start->size(), limit.size());
- size_t diff_index = 0;
- while ((diff_index < min_length) &&
- ((*start)[diff_index] == limit[diff_index])) {
- diff_index++;
- }
-
- if (diff_index >= min_length) {
- // Do not shorten if one string is a prefix of the other
- } else {
- uint8_t diff_byte = static_cast<uint8_t>((*start)[diff_index]);
- if (diff_byte < static_cast<uint8_t>(0xff) &&
- diff_byte + 1 < static_cast<uint8_t>(limit[diff_index])) {
- (*start)[diff_index]++;
- start->resize(diff_index + 1);
- assert(Compare(*start, limit) < 0);
- }
- }
- }
-
- virtual void FindShortSuccessor(std::string* key) const {
- // Find first character that can be incremented
- size_t n = key->size();
- for (size_t i = 0; i < n; i++) {
- const uint8_t byte = (*key)[i];
- if (byte != static_cast<uint8_t>(0xff)) {
- (*key)[i] = byte + 1;
- key->resize(i+1);
- return;
- }
- }
- // *key is a run of 0xffs. Leave it alone.
- }
-};
-} // namespace
-
-static port::OnceType once = LEVELDB_ONCE_INIT;
-static const Comparator* bytewise;
-
-static void InitModule() {
- bytewise = new BytewiseComparatorImpl;
-}
-
-const Comparator* BytewiseComparator() {
- port::InitOnce(&once, InitModule);
- return bytewise;
-}
-
-void UnsafeDeallocateBytewiseComparator()
-{
- once = LEVELDB_ONCE_INIT;
- delete bytewise;
- bytewise = nullptr;
-}
-
-} // namespace leveldb
[34/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/TARGETS
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/TARGETS b/thirdparty/rocksdb/TARGETS
new file mode 100644
index 0000000..3fac4a7
--- /dev/null
+++ b/thirdparty/rocksdb/TARGETS
@@ -0,0 +1,533 @@
+
+import os
+
+TARGETS_PATH = os.path.dirname(__file__)
+REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
+BUCK_BINS = "buck-out/gen/" + REPO_PATH
+TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
+rocksdb_compiler_flags = [
+ "-fno-builtin-memcmp",
+ "-DROCKSDB_PLATFORM_POSIX",
+ "-DROCKSDB_LIB_IO_POSIX",
+ "-DROCKSDB_FALLOCATE_PRESENT",
+ "-DROCKSDB_MALLOC_USABLE_SIZE",
+ "-DROCKSDB_RANGESYNC_PRESENT",
+ "-DROCKSDB_SCHED_GETCPU_PRESENT",
+ "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+ "-DOS_LINUX",
+ "-DROCKSDB_UBSAN_RUN",
+ # Flags to enable libs we include
+ "-DSNAPPY",
+ "-DZLIB",
+ "-DBZIP2",
+ "-DLZ4",
+ "-DZSTD",
+ "-DGFLAGS=gflags",
+ "-DNUMA",
+ "-DTBB",
+ # Needed to compile in fbcode
+ "-Wno-expansion-to-defined",
+]
+
+rocksdb_external_deps = [
+ ('bzip2', None, 'bz2'),
+ ('snappy', None, "snappy"),
+ ('zlib', None, 'z'),
+ ('gflags', None, 'gflags'),
+ ('lz4', None, 'lz4'),
+ ('zstd', None),
+ ('tbb', None),
+ ("numa", None, "numa"),
+ ("googletest", None, "gtest"),
+]
+
+rocksdb_preprocessor_flags = [
+ # Directories with files for #include
+ "-I" + REPO_PATH + "include/",
+ "-I" + REPO_PATH,
+]
+
+rocksdb_arch_preprocessor_flags = {
+ "x86_64": ["-DHAVE_SSE42"],
+}
+
+cpp_library(
+ name = "rocksdb_lib",
+ headers = AutoHeaders.RECURSIVE_GLOB,
+ srcs = [
+ "cache/clock_cache.cc",
+ "cache/lru_cache.cc",
+ "cache/sharded_cache.cc",
+ "db/builder.cc",
+ "db/c.cc",
+ "db/column_family.cc",
+ "db/compacted_db_impl.cc",
+ "db/compaction.cc",
+ "db/compaction_iterator.cc",
+ "db/compaction_job.cc",
+ "db/compaction_picker.cc",
+ "db/compaction_picker_universal.cc",
+ "db/convenience.cc",
+ "db/db_filesnapshot.cc",
+ "db/db_impl.cc",
+ "db/db_impl_write.cc",
+ "db/db_impl_compaction_flush.cc",
+ "db/db_impl_files.cc",
+ "db/db_impl_open.cc",
+ "db/db_impl_debug.cc",
+ "db/db_impl_experimental.cc",
+ "db/db_impl_readonly.cc",
+ "db/db_info_dumper.cc",
+ "db/db_iter.cc",
+ "db/dbformat.cc",
+ "db/event_helpers.cc",
+ "db/experimental.cc",
+ "db/external_sst_file_ingestion_job.cc",
+ "db/file_indexer.cc",
+ "db/flush_job.cc",
+ "db/flush_scheduler.cc",
+ "db/forward_iterator.cc",
+ "db/internal_stats.cc",
+ "db/log_reader.cc",
+ "db/log_writer.cc",
+ "db/malloc_stats.cc",
+ "db/managed_iterator.cc",
+ "db/memtable.cc",
+ "db/memtable_list.cc",
+ "db/merge_helper.cc",
+ "db/merge_operator.cc",
+ "db/range_del_aggregator.cc",
+ "db/repair.cc",
+ "db/snapshot_impl.cc",
+ "db/table_cache.cc",
+ "db/table_properties_collector.cc",
+ "db/transaction_log_impl.cc",
+ "db/version_builder.cc",
+ "db/version_edit.cc",
+ "db/version_set.cc",
+ "db/wal_manager.cc",
+ "db/write_batch.cc",
+ "db/write_batch_base.cc",
+ "db/write_controller.cc",
+ "db/write_thread.cc",
+ "env/env.cc",
+ "env/env_chroot.cc",
+ "env/env_encryption.cc",
+ "env/env_hdfs.cc",
+ "env/env_posix.cc",
+ "env/io_posix.cc",
+ "env/mock_env.cc",
+ "memtable/alloc_tracker.cc",
+ "memtable/hash_cuckoo_rep.cc",
+ "memtable/hash_linklist_rep.cc",
+ "memtable/hash_skiplist_rep.cc",
+ "memtable/skiplistrep.cc",
+ "memtable/vectorrep.cc",
+ "memtable/write_buffer_manager.cc",
+ "monitoring/histogram.cc",
+ "monitoring/histogram_windowing.cc",
+ "monitoring/instrumented_mutex.cc",
+ "monitoring/iostats_context.cc",
+ "monitoring/perf_context.cc",
+ "monitoring/perf_level.cc",
+ "monitoring/statistics.cc",
+ "monitoring/thread_status_impl.cc",
+ "monitoring/thread_status_updater.cc",
+ "monitoring/thread_status_updater_debug.cc",
+ "monitoring/thread_status_util.cc",
+ "monitoring/thread_status_util_debug.cc",
+ "options/cf_options.cc",
+ "options/db_options.cc",
+ "options/options.cc",
+ "options/options_helper.cc",
+ "options/options_parser.cc",
+ "options/options_sanity_check.cc",
+ "port/port_posix.cc",
+ "port/stack_trace.cc",
+ "table/adaptive_table_factory.cc",
+ "table/block.cc",
+ "table/block_based_filter_block.cc",
+ "table/block_based_table_builder.cc",
+ "table/block_based_table_factory.cc",
+ "table/block_based_table_reader.cc",
+ "table/block_builder.cc",
+ "table/block_prefix_index.cc",
+ "table/bloom_block.cc",
+ "table/cuckoo_table_builder.cc",
+ "table/cuckoo_table_factory.cc",
+ "table/cuckoo_table_reader.cc",
+ "table/flush_block_policy.cc",
+ "table/format.cc",
+ "table/full_filter_block.cc",
+ "table/get_context.cc",
+ "table/index_builder.cc",
+ "table/iterator.cc",
+ "table/merging_iterator.cc",
+ "table/meta_blocks.cc",
+ "table/partitioned_filter_block.cc",
+ "table/persistent_cache_helper.cc",
+ "table/plain_table_builder.cc",
+ "table/plain_table_factory.cc",
+ "table/plain_table_index.cc",
+ "table/plain_table_key_coding.cc",
+ "table/plain_table_reader.cc",
+ "table/sst_file_writer.cc",
+ "table/table_properties.cc",
+ "table/two_level_iterator.cc",
+ "tools/dump/db_dump_tool.cc",
+ "util/arena.cc",
+ "util/auto_roll_logger.cc",
+ "util/bloom.cc",
+ "util/build_version.cc",
+ "util/coding.cc",
+ "util/compaction_job_stats_impl.cc",
+ "util/comparator.cc",
+ "util/concurrent_arena.cc",
+ "util/crc32c.cc",
+ "util/delete_scheduler.cc",
+ "util/dynamic_bloom.cc",
+ "util/event_logger.cc",
+ "util/file_reader_writer.cc",
+ "util/file_util.cc",
+ "util/filename.cc",
+ "util/filter_policy.cc",
+ "util/hash.cc",
+ "util/log_buffer.cc",
+ "util/murmurhash.cc",
+ "util/random.cc",
+ "util/rate_limiter.cc",
+ "util/slice.cc",
+ "util/sst_file_manager_impl.cc",
+ "util/status.cc",
+ "util/status_message.cc",
+ "util/string_util.cc",
+ "util/sync_point.cc",
+ "util/thread_local.cc",
+ "util/threadpool_imp.cc",
+ "util/transaction_test_util.cc",
+ "util/xxhash.cc",
+ "utilities/backupable/backupable_db.cc",
+ "utilities/blob_db/blob_db.cc",
+ "utilities/blob_db/blob_db_impl.cc",
+ "utilities/blob_db/blob_file.cc",
+ "utilities/blob_db/blob_log_reader.cc",
+ "utilities/blob_db/blob_log_writer.cc",
+ "utilities/blob_db/blob_log_format.cc",
+ "utilities/blob_db/ttl_extractor.cc",
+ "utilities/cassandra/cassandra_compaction_filter.cc",
+ "utilities/cassandra/format.cc",
+ "utilities/cassandra/merge_operator.cc",
+ "utilities/checkpoint/checkpoint_impl.cc",
+ "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
+ "utilities/convenience/info_log_finder.cc",
+ "utilities/date_tiered/date_tiered_db_impl.cc",
+ "utilities/debug.cc",
+ "utilities/document/document_db.cc",
+ "utilities/document/json_document.cc",
+ "utilities/document/json_document_builder.cc",
+ "utilities/env_mirror.cc",
+ "utilities/env_timed.cc",
+ "utilities/geodb/geodb_impl.cc",
+ "utilities/leveldb_options/leveldb_options.cc",
+ "utilities/lua/rocks_lua_compaction_filter.cc",
+ "utilities/memory/memory_util.cc",
+ "utilities/merge_operators/max.cc",
+ "utilities/merge_operators/put.cc",
+ "utilities/merge_operators/string_append/stringappend.cc",
+ "utilities/merge_operators/string_append/stringappend2.cc",
+ "utilities/merge_operators/uint64add.cc",
+ "utilities/option_change_migration/option_change_migration.cc",
+ "utilities/options/options_util.cc",
+ "utilities/persistent_cache/block_cache_tier.cc",
+ "utilities/persistent_cache/block_cache_tier_file.cc",
+ "utilities/persistent_cache/block_cache_tier_metadata.cc",
+ "utilities/persistent_cache/persistent_cache_tier.cc",
+ "utilities/persistent_cache/volatile_tier_impl.cc",
+ "utilities/redis/redis_lists.cc",
+ "utilities/simulator_cache/sim_cache.cc",
+ "utilities/spatialdb/spatial_db.cc",
+ "utilities/table_properties_collectors/compact_on_deletion_collector.cc",
+ "utilities/transactions/optimistic_transaction_db_impl.cc",
+ "utilities/transactions/optimistic_transaction.cc",
+ "utilities/transactions/transaction_base.cc",
+ "utilities/transactions/pessimistic_transaction_db.cc",
+ "utilities/transactions/transaction_db_mutex_impl.cc",
+ "utilities/transactions/pessimistic_transaction.cc",
+ "utilities/transactions/transaction_lock_mgr.cc",
+ "utilities/transactions/transaction_util.cc",
+ "utilities/transactions/write_prepared_txn.cc",
+ "utilities/ttl/db_ttl_impl.cc",
+ "utilities/write_batch_with_index/write_batch_with_index.cc",
+ "utilities/write_batch_with_index/write_batch_with_index_internal.cc",
+ "tools/ldb_cmd.cc",
+ "tools/ldb_tool.cc",
+ "tools/sst_dump_tool.cc",
+ "utilities/blob_db/blob_dump_tool.cc",
+ ],
+ deps = [],
+ preprocessor_flags = rocksdb_preprocessor_flags,
+ arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+ compiler_flags = rocksdb_compiler_flags,
+ external_deps = rocksdb_external_deps,
+)
+
+cpp_library(
+ name = "rocksdb_test_lib",
+ headers = AutoHeaders.RECURSIVE_GLOB,
+ srcs = [
+ "table/mock_table.cc",
+ "util/fault_injection_test_env.cc",
+ "util/testharness.cc",
+ "util/testutil.cc",
+ "db/db_test_util.cc",
+ "utilities/cassandra/test_utils.cc",
+ "utilities/col_buf_encoder.cc",
+ "utilities/col_buf_decoder.cc",
+ "utilities/column_aware_encoding_util.cc",
+ ],
+ deps = [":rocksdb_lib"],
+ preprocessor_flags = rocksdb_preprocessor_flags,
+ arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+ compiler_flags = rocksdb_compiler_flags,
+ external_deps = rocksdb_external_deps,
+)
+
+cpp_library(
+ name = "rocksdb_tools_lib",
+ headers = AutoHeaders.RECURSIVE_GLOB,
+ srcs = [
+ "tools/db_bench_tool.cc",
+ "util/testutil.cc",
+ ],
+ deps = [":rocksdb_lib"],
+ preprocessor_flags = rocksdb_preprocessor_flags,
+ arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+ compiler_flags = rocksdb_compiler_flags,
+ external_deps = rocksdb_external_deps,
+)
+
+cpp_library(
+ name = "env_basic_test_lib",
+ headers = AutoHeaders.RECURSIVE_GLOB,
+ srcs = ["env/env_basic_test.cc"],
+ deps = [":rocksdb_test_lib"],
+ preprocessor_flags = rocksdb_preprocessor_flags,
+ arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+ compiler_flags = rocksdb_compiler_flags,
+ external_deps = rocksdb_external_deps,
+)
+
+# [test_name, test_src, test_type]
+ROCKS_TESTS = [['arena_test', 'util/arena_test.cc', 'serial'],
+ ['auto_roll_logger_test', 'util/auto_roll_logger_test.cc', 'serial'],
+ ['autovector_test', 'util/autovector_test.cc', 'serial'],
+ ['backupable_db_test',
+ 'utilities/backupable/backupable_db_test.cc',
+ 'parallel'],
+ ['blob_db_test', 'utilities/blob_db/blob_db_test.cc', 'serial'],
+ ['block_based_filter_block_test',
+ 'table/block_based_filter_block_test.cc',
+ 'serial'],
+ ['block_test', 'table/block_test.cc', 'serial'],
+ ['bloom_test', 'util/bloom_test.cc', 'serial'],
+ ['c_test', 'db/c_test.c', 'serial'],
+ ['cache_test', 'cache/cache_test.cc', 'serial'],
+ ['cassandra_format_test',
+ 'utilities/cassandra/cassandra_format_test.cc',
+ 'serial'],
+ ['cassandra_functional_test',
+ 'utilities/cassandra/cassandra_functional_test.cc',
+ 'serial'],
+ ['cassandra_row_merge_test',
+ 'utilities/cassandra/cassandra_row_merge_test.cc',
+ 'serial'],
+ ['cassandra_serialize_test',
+ 'utilities/cassandra/cassandra_serialize_test.cc',
+ 'serial'],
+ ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
+ ['cleanable_test', 'table/cleanable_test.cc', 'serial'],
+ ['coding_test', 'util/coding_test.cc', 'serial'],
+ ['column_aware_encoding_test',
+ 'utilities/column_aware_encoding_test.cc',
+ 'serial'],
+ ['column_family_test', 'db/column_family_test.cc', 'serial'],
+ ['compact_files_test', 'db/compact_files_test.cc', 'serial'],
+ ['compact_on_deletion_collector_test',
+ 'utilities/table_properties_collectors/compact_on_deletion_collector_test.cc',
+ 'serial'],
+ ['compaction_iterator_test', 'db/compaction_iterator_test.cc', 'serial'],
+ ['compaction_job_stats_test', 'db/compaction_job_stats_test.cc', 'serial'],
+ ['compaction_job_test', 'db/compaction_job_test.cc', 'serial'],
+ ['compaction_picker_test', 'db/compaction_picker_test.cc', 'serial'],
+ ['comparator_db_test', 'db/comparator_db_test.cc', 'serial'],
+ ['corruption_test', 'db/corruption_test.cc', 'serial'],
+ ['crc32c_test', 'util/crc32c_test.cc', 'serial'],
+ ['cuckoo_table_builder_test', 'table/cuckoo_table_builder_test.cc', 'serial'],
+ ['cuckoo_table_db_test', 'db/cuckoo_table_db_test.cc', 'serial'],
+ ['cuckoo_table_reader_test', 'table/cuckoo_table_reader_test.cc', 'serial'],
+ ['date_tiered_test', 'utilities/date_tiered/date_tiered_test.cc', 'serial'],
+ ['db_basic_test', 'db/db_basic_test.cc', 'serial'],
+ ['db_block_cache_test', 'db/db_block_cache_test.cc', 'serial'],
+ ['db_bloom_filter_test', 'db/db_bloom_filter_test.cc', 'serial'],
+ ['db_compaction_filter_test', 'db/db_compaction_filter_test.cc', 'parallel'],
+ ['db_compaction_test', 'db/db_compaction_test.cc', 'parallel'],
+ ['db_dynamic_level_test', 'db/db_dynamic_level_test.cc', 'serial'],
+ ['db_encryption_test', 'db/db_encryption_test.cc', 'serial'],
+ ['db_flush_test', 'db/db_flush_test.cc', 'serial'],
+ ['db_inplace_update_test', 'db/db_inplace_update_test.cc', 'serial'],
+ ['db_io_failure_test', 'db/db_io_failure_test.cc', 'serial'],
+ ['db_iter_test', 'db/db_iter_test.cc', 'serial'],
+ ['db_iterator_test', 'db/db_iterator_test.cc', 'serial'],
+ ['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'],
+ ['db_memtable_test', 'db/db_memtable_test.cc', 'serial'],
+ ['db_merge_operator_test', 'db/db_merge_operator_test.cc', 'serial'],
+ ['db_options_test', 'db/db_options_test.cc', 'serial'],
+ ['db_properties_test', 'db/db_properties_test.cc', 'serial'],
+ ['db_range_del_test', 'db/db_range_del_test.cc', 'serial'],
+ ['db_sst_test', 'db/db_sst_test.cc', 'parallel'],
+ ['db_statistics_test', 'db/db_statistics_test.cc', 'serial'],
+ ['db_table_properties_test', 'db/db_table_properties_test.cc', 'serial'],
+ ['db_tailing_iter_test', 'db/db_tailing_iter_test.cc', 'serial'],
+ ['db_test', 'db/db_test.cc', 'parallel'],
+ ['db_test2', 'db/db_test2.cc', 'serial'],
+ ['db_universal_compaction_test',
+ 'db/db_universal_compaction_test.cc',
+ 'parallel'],
+ ['db_wal_test', 'db/db_wal_test.cc', 'parallel'],
+ ['db_write_test', 'db/db_write_test.cc', 'serial'],
+ ['dbformat_test', 'db/dbformat_test.cc', 'serial'],
+ ['delete_scheduler_test', 'util/delete_scheduler_test.cc', 'serial'],
+ ['deletefile_test', 'db/deletefile_test.cc', 'serial'],
+ ['document_db_test', 'utilities/document/document_db_test.cc', 'serial'],
+ ['dynamic_bloom_test', 'util/dynamic_bloom_test.cc', 'serial'],
+ ['env_basic_test', 'env/env_basic_test.cc', 'serial'],
+ ['env_test', 'env/env_test.cc', 'serial'],
+ ['env_timed_test', 'utilities/env_timed_test.cc', 'serial'],
+ ['event_logger_test', 'util/event_logger_test.cc', 'serial'],
+ ['external_sst_file_basic_test',
+ 'db/external_sst_file_basic_test.cc',
+ 'serial'],
+ ['external_sst_file_test', 'db/external_sst_file_test.cc', 'parallel'],
+ ['fault_injection_test', 'db/fault_injection_test.cc', 'parallel'],
+ ['file_indexer_test', 'db/file_indexer_test.cc', 'serial'],
+ ['file_reader_writer_test', 'util/file_reader_writer_test.cc', 'serial'],
+ ['filelock_test', 'util/filelock_test.cc', 'serial'],
+ ['filename_test', 'db/filename_test.cc', 'serial'],
+ ['flush_job_test', 'db/flush_job_test.cc', 'serial'],
+ ['full_filter_block_test', 'table/full_filter_block_test.cc', 'serial'],
+ ['geodb_test', 'utilities/geodb/geodb_test.cc', 'serial'],
+ ['hash_table_test',
+ 'utilities/persistent_cache/hash_table_test.cc',
+ 'serial'],
+ ['hash_test', 'util/hash_test.cc', 'serial'],
+ ['heap_test', 'util/heap_test.cc', 'serial'],
+ ['histogram_test', 'monitoring/histogram_test.cc', 'serial'],
+ ['inlineskiplist_test', 'memtable/inlineskiplist_test.cc', 'parallel'],
+ ['iostats_context_test', 'monitoring/iostats_context_test.cc', 'serial'],
+ ['json_document_test', 'utilities/document/json_document_test.cc', 'serial'],
+ ['ldb_cmd_test', 'tools/ldb_cmd_test.cc', 'serial'],
+ ['listener_test', 'db/listener_test.cc', 'serial'],
+ ['log_test', 'db/log_test.cc', 'serial'],
+ ['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
+ ['manual_compaction_test', 'db/manual_compaction_test.cc', 'parallel'],
+ ['memory_test', 'utilities/memory/memory_test.cc', 'serial'],
+ ['memtable_list_test', 'db/memtable_list_test.cc', 'serial'],
+ ['merge_helper_test', 'db/merge_helper_test.cc', 'serial'],
+ ['merge_test', 'db/merge_test.cc', 'serial'],
+ ['merger_test', 'table/merger_test.cc', 'serial'],
+ ['mock_env_test', 'env/mock_env_test.cc', 'serial'],
+ ['object_registry_test', 'utilities/object_registry_test.cc', 'serial'],
+ ['optimistic_transaction_test',
+ 'utilities/transactions/optimistic_transaction_test.cc',
+ 'serial'],
+ ['option_change_migration_test',
+ 'utilities/option_change_migration/option_change_migration_test.cc',
+ 'serial'],
+ ['options_file_test', 'db/options_file_test.cc', 'serial'],
+ ['options_settable_test', 'options/options_settable_test.cc', 'serial'],
+ ['options_test', 'options/options_test.cc', 'serial'],
+ ['options_util_test', 'utilities/options/options_util_test.cc', 'serial'],
+ ['partitioned_filter_block_test',
+ 'table/partitioned_filter_block_test.cc',
+ 'serial'],
+ ['perf_context_test', 'db/perf_context_test.cc', 'serial'],
+ ['persistent_cache_test',
+ 'utilities/persistent_cache/persistent_cache_test.cc',
+ 'parallel'],
+ ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial'],
+ ['prefix_test', 'db/prefix_test.cc', 'serial'],
+ ['range_del_aggregator_test', 'db/range_del_aggregator_test.cc', 'serial'],
+ ['rate_limiter_test', 'util/rate_limiter_test.cc', 'serial'],
+ ['reduce_levels_test', 'tools/reduce_levels_test.cc', 'serial'],
+ ['repair_test', 'db/repair_test.cc', 'serial'],
+ ['sim_cache_test', 'utilities/simulator_cache/sim_cache_test.cc', 'serial'],
+ ['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
+ ['slice_transform_test', 'util/slice_transform_test.cc', 'serial'],
+ ['spatial_db_test', 'utilities/spatialdb/spatial_db_test.cc', 'serial'],
+ ['sst_dump_test', 'tools/sst_dump_test.cc', 'serial'],
+ ['statistics_test', 'monitoring/statistics_test.cc', 'serial'],
+ ['stringappend_test',
+ 'utilities/merge_operators/string_append/stringappend_test.cc',
+ 'serial'],
+ ['table_properties_collector_test',
+ 'db/table_properties_collector_test.cc',
+ 'serial'],
+ ['table_test', 'table/table_test.cc', 'parallel'],
+ ['thread_list_test', 'util/thread_list_test.cc', 'serial'],
+ ['thread_local_test', 'util/thread_local_test.cc', 'serial'],
+ ['timer_queue_test', 'util/timer_queue_test.cc', 'serial'],
+ ['transaction_test', 'utilities/transactions/transaction_test.cc', 'serial'],
+ ['ttl_test', 'utilities/ttl/ttl_test.cc', 'serial'],
+ ['util_merge_operators_test',
+ 'utilities/util_merge_operators_test.cc',
+ 'serial'],
+ ['version_builder_test', 'db/version_builder_test.cc', 'serial'],
+ ['version_edit_test', 'db/version_edit_test.cc', 'serial'],
+ ['version_set_test', 'db/version_set_test.cc', 'serial'],
+ ['wal_manager_test', 'db/wal_manager_test.cc', 'serial'],
+ ['write_batch_test', 'db/write_batch_test.cc', 'serial'],
+ ['write_batch_with_index_test',
+ 'utilities/write_batch_with_index/write_batch_with_index_test.cc',
+ 'serial'],
+ ['write_buffer_manager_test',
+ 'memtable/write_buffer_manager_test.cc',
+ 'serial'],
+ ['write_callback_test', 'db/write_callback_test.cc', 'serial'],
+ ['write_controller_test', 'db/write_controller_test.cc', 'serial']]
+
+
+# Generate a test rule for each entry in ROCKS_TESTS
+for test_cfg in ROCKS_TESTS:
+ test_name = test_cfg[0]
+ test_cc = test_cfg[1]
+ ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
+ test_bin = test_name + "_bin"
+
+ cpp_binary (
+ name = test_bin,
+ srcs = [test_cc],
+ deps = [":rocksdb_test_lib"],
+ preprocessor_flags = rocksdb_preprocessor_flags,
+ arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+ compiler_flags = rocksdb_compiler_flags,
+ external_deps = rocksdb_external_deps,
+ )
+
+ custom_unittest(
+ name = test_name,
+ type = ttype,
+ deps = [":" + test_bin],
+ command = [TEST_RUNNER, BUCK_BINS + test_bin]
+ )
+
+custom_unittest(
+ name = "make_rocksdbjavastatic",
+ type = "simple",
+ command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
+)
+
+custom_unittest(
+ name = "make_rocksdb_lite_release",
+ type = "simple",
+ command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
+)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/USERS.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/USERS.md b/thirdparty/rocksdb/USERS.md
new file mode 100644
index 0000000..7be093f
--- /dev/null
+++ b/thirdparty/rocksdb/USERS.md
@@ -0,0 +1,85 @@
+This document lists users of RocksDB and their use cases. If you are using RocksDB, please open a pull request and add yourself to the list.
+
+## Facebook
+At Facebook, we use RocksDB as storage engines in multiple data management services and a backend for many different stateful services, including:
+
+1. MyRocks -- https://github.com/MySQLOnRocksDB/mysql-5.6
+2. MongoRocks -- https://github.com/mongodb-partners/mongo-rocks
+3. ZippyDB -- Facebook's distributed key-value store with Paxos-style replication, built on top of RocksDB.[*] https://www.youtube.com/watch?v=DfiN7pG0D0khtt
+4. Laser -- Laser is a high query throughput, low (millisecond) latency, key-value storage service built on top of RocksDB.[*]
+4. Dragon -- a distributed graph query engine. https://code.facebook.com/posts/1737605303120405/dragon-a-distributed-graph-query-engine/
+5. Stylus -- a low-level stream processing framework writtenin C++.[*]
+
+[*] https://research.facebook.com/publications/realtime-data-processing-at-facebook/
+
+## LinkedIn
+Two different use cases at Linkedin are using RocksDB as a storage engine:
+
+1. LinkedIn's follow feed for storing user's activities. Check out the blog post: https://engineering.linkedin.com/blog/2016/03/followfeed--linkedin-s-feed-made-faster-and-smarter
+2. Apache Samza, open source framework for stream processing
+
+Learn more about those use cases in a Tech Talk by Ankit Gupta and Naveen Somasundaram: http://www.youtube.com/watch?v=plqVp_OnSzg
+
+## Yahoo
+Yahoo is using RocksDB as a storage engine for their biggest distributed data store Sherpa. Learn more about it here: http://yahooeng.tumblr.com/post/120730204806/sherpa-scales-new-heights
+
+## CockroachDB
+CockroachDB is an open-source geo-replicated transactional database (still in development). They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach
+
+## DNANexus
+DNANexus is using RocksDB to speed up processing of genomics data.
+You can learn more from this great blog post by Mike Lin: http://devblog.dnanexus.com/faster-bam-sorting-with-samtools-and-rocksdb/
+
+## Iron.io
+Iron.io is using RocksDB as a storage engine for their distributed queueing system.
+Learn more from Tech Talk by Reed Allman: http://www.youtube.com/watch?v=HTjt6oj-RL4
+
+## Tango Me
+Tango is using RocksDB as a graph storage to store all users' connection data and other social activity data.
+
+## Turn
+Turn is using RocksDB as a storage layer for their key/value store, serving at peak 2.4MM QPS out of different datacenters.
+Check out our RocksDB Protobuf merge operator at: https://github.com/vladb38/rocksdb_protobuf
+
+## Santanader UK/Cloudera Profession Services
+Check out their blog post: http://blog.cloudera.com/blog/2015/08/inside-santanders-near-real-time-data-ingest-architecture/
+
+## Airbnb
+Airbnb is using RocksDB as a storage engine for their personalized search service. You can learn more about it here: https://www.youtube.com/watch?v=ASQ6XMtogMs
+
+## Pinterest
+Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtube.com/watch?v=MtFEVEs_2Vo
+
+## Smyte
+[Smyte](https://www.smyte.com/) uses RocksDB as the storage layer for their core key-value storage, high-performance counters and time-windowed HyperLogLog services.
+
+## Rakuten Marketing
+[Rakuten Marketing](https://marketing.rakuten.com/) uses RocksDB as the disk cache layer for the real-time bidding service in their Performance DSP.
+
+## VWO, Wingify
+[VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed.
+
+## quasardb
+[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark.
+quasardb uses a heavily tuned RocksDB as its persistence layer.
+
+## Netflix
+[Netflix](http://techblog.netflix.com/2016/05/application-data-caching-using-ssds.html) Netflix uses RocksDB on AWS EC2 instances with local SSD drives to cache application data.
+
+## TiKV
+[TiKV](https://github.com/pingcap/tikv) is a GEO-replicated, high-performance, distributed, transactional key-value database. TiKV is powered by Rust and Raft. TiKV uses RocksDB as its persistence layer.
+
+## Apache Flink
+[Apache Flink](https://flink.apache.org/news/2016/03/08/release-1.0.0.html) uses RocksDB to store state locally on a machine.
+
+## Dgraph
+[Dgraph](https://github.com/dgraph-io/dgraph) is an open-source, scalable, distributed, low latency, high throughput Graph database .They use RocksDB to store state locally on a machine.
+
+## Uber
+[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and scalable task queue.
+
+## 360 Pika
+[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company
+
+## LzLabs
+LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/Vagrantfile
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/Vagrantfile b/thirdparty/rocksdb/Vagrantfile
new file mode 100644
index 0000000..d7c2991
--- /dev/null
+++ b/thirdparty/rocksdb/Vagrantfile
@@ -0,0 +1,34 @@
+# Vagrant file
+Vagrant.configure("2") do |config|
+
+ config.vm.provider "virtualbox" do |v|
+ v.memory = 4096
+ v.cpus = 2
+ end
+
+ config.vm.define "ubuntu14" do |box|
+ box.vm.box = "ubuntu/trusty64"
+ end
+
+ config.vm.define "centos65" do |box|
+ box.vm.box = "chef/centos-6.5"
+ end
+
+ config.vm.define "FreeBSD10" do |box|
+ box.vm.guest = :freebsd
+ box.vm.box = "robin/freebsd-10"
+ # FreeBSD does not support 'mount_virtualbox_shared_folder', use NFS
+ box.vm.synced_folder ".", "/vagrant", :nfs => true, id: "vagrant-root"
+ box.vm.network "private_network", ip: "10.0.1.10"
+
+ # build everything after creating VM, skip using --no-provision
+ box.vm.provision "shell", inline: <<-SCRIPT
+ pkg install -y gmake clang35
+ export CXX=/usr/local/bin/clang++35
+ cd /vagrant
+ gmake clean
+ gmake all OPT=-g
+ SCRIPT
+ end
+
+end
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/WINDOWS_PORT.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/WINDOWS_PORT.md b/thirdparty/rocksdb/WINDOWS_PORT.md
new file mode 100644
index 0000000..a0fe1fe
--- /dev/null
+++ b/thirdparty/rocksdb/WINDOWS_PORT.md
@@ -0,0 +1,228 @@
+# Microsoft Contribution Notes
+
+## Contributors
+* Alexander Zinoviev https://github.com/zinoale
+* Dmitri Smirnov https://github.com/yuslepukhin
+* Praveen Rao https://github.com/PraveenSinghRao
+* Sherlock Huang https://github.com/SherlockNoMad
+
+## Introduction
+RocksDB is a well proven open source key-value persistent store, optimized for fast storage. It provides scalability with number of CPUs and storage IOPS, to support IO-bound, in-memory and write-once workloads, most importantly, to be flexible to allow for innovation.
+
+As Microsoft Bing team we have been continuously pushing hard to improve the scalability, efficiency of platform and eventually benefit Bing end-user satisfaction. We would like to explore the opportunity to embrace open source, RocksDB here, to use, enhance and customize for our usage, and also contribute back to the RocksDB community. Herein, we are pleased to offer this RocksDB port for Windows platform.
+
+These notes describe some decisions and changes we had to make with regards to porting RocksDB on Windows. We hope this will help both reviewers and users of the Windows port.
+We are open for comments and improvements.
+
+## OS specifics
+All of the porting, testing and benchmarking was done on Windows Server 2012 R2 Datacenter 64-bit but to the best of our knowledge there is not a specific API we used during porting that is unsupported on other Windows OS after Vista.
+
+## Porting goals
+We strive to achieve the following goals:
+* make use of the existing porting interface of RocksDB
+* make minimum [WY2]modifications within platform independent code.
+* make all unit test pass both in debug and release builds.
+ * Note: latest introduction of SyncPoint seems to disable running db_test in Release.
+* make performance on par with published benchmarks accounting for HW differences
+* we would like to keep the port code inline with the master branch with no forking
+
+## Build system
+We have chosen CMake as a widely accepted build system to build the Windows port. It is very fast and convenient.
+
+At the same time it generates Visual Studio projects that are both usable from a command line and IDE.
+
+The top-level CMakeLists.txt file contains description of all targets and build rules. It also provides brief instructions on how to build the software for Windows. One more build related file is thirdparty.inc that also resides on the top level. This file must be edited to point to actual third party libraries location.
+We think that it would be beneficial to merge the existing make-based build system and the new cmake-based build system into a single one to use on all platforms.
+
+All building and testing was done for 64-bit. We have not conducted any testing for 32-bit and early reports indicate that it will not run on 32-bit.
+
+## C++ and STL notes
+We had to make some minimum changes within the portable files that either account for OS differences or the shortcomings of C++11 support in the current version of the MS compiler. Most or all of them are expected to be fixed in the upcoming compiler releases.
+
+We plan to use this port for our business purposes here at Bing and this provided business justification for this port. This also means, we do not have at present to choose the compiler version at will.
+
+* Certain headers that are not present and not necessary on Windows were simply `#ifndef OS_WIN` in a few places (`unistd.h`)
+* All posix specific headers were replaced to port/port.h which worked well
+* Replaced `dirent.h` for `port/dirent.h` (very few places) with the implementation of the relevant interfaces within `rocksdb::port` namespace
+* Replaced `sys/time.h` to `port/sys_time.h` (few places) implemented equivalents within `rocksdb::port`
+* `printf %z` specification is not supported on Windows. To imitate existing standards we came up with a string macro `ROCKSDB_PRIszt` which expands to `%z` on posix systems and to Iu on windows.
+* in class member initialization were moved to a __ctors in some cases
+* `constexpr` is not supported. We had to replace `std::numeric_limits<>::max/min()` to its C macros for constants. Sometimes we had to make class members `static const` and place a definition within a .cc file.
+* `constexpr` for functions was replaced to a template specialization (1 place)
+* Union members that have non-trivial constructors were replaced to `char[]` in one place along with bug fixes (spatial experimental feature)
+* Zero-sized arrays are deemed a non-standard extension which we converted to 1 size array and that should work well for the purposes of these classes.
+* `std::chrono` lacks nanoseconds support (fixed in the upcoming release of the STL) and we had to use `QueryPerfCounter()` within env_win.cc
+* Function local statics initialization is still not safe. Used `std::once` to mitigate within WinEnv.
+
+## Windows Environments notes
+We endeavored to make it functionally on par with posix_env. This means we replicated the functionality of the thread pool and other things as precise as possible, including:
+* Replicate posix logic using std:thread primitives.
+* Implement all posix_env disk access functionality.
+* Set `use_os_buffer=false` to disable OS disk buffering for WinWritableFile and WinRandomAccessFile.
+* Replace `pread/pwrite` with `WriteFile/ReadFile` with `OVERLAPPED` structure.
+* Use `SetFileInformationByHandle` to compensate absence of `fallocate`.
+
+### In detail
+Even though Windows provides its own efficient thread-pool implementation we chose to replicate posix logic using `std::thread` primitives. This allows anyone to quickly detect any changes within the posix source code and replicate them within windows env. This has proven to work very well. At the same time for anyone who wishes to replace the built-in thread-pool can do so using RocksDB stackable environments.
+
+For disk access we implemented all of the functionality present within the posix_env which includes memory mapped files, random access, rate-limiter support etc.
+The `use_os_buffer` flag on Posix platforms currently denotes disabling read-ahead log via `fadvise` mechanism. Windows does not have `fadvise` system call. What is more, it implements disk cache in a way that differs from Linux greatly. It�s not an uncommon practice on Windows to perform un-buffered disk access to gain control of the memory consumption. We think that in our use case this may also be a good configuration option at the expense of disk throughput. To compensate one may increase the configured in-memory cache size instead. Thus we have chosen `use_os_buffer=false` to disable OS disk buffering for `WinWritableFile` and `WinRandomAccessFile`. The OS imposes restrictions on the alignment of the disk offsets, buffers used and the amount of data that is read/written when accessing files in un-buffered mode. When the option is true, the classes behave in a standard way. This allows to perform writes and reads in cases when un-buffered access does not make sense such as WA
L and MANIFEST.
+
+We have replaced `pread/pwrite` with `WriteFile/ReadFile` with `OVERLAPPED` structure so we can atomically seek to the position of the disk operation but still perform the operation synchronously. Thus we able to emulate that functionality of `pread/pwrite` reasonably well. The only difference is that the file pointer is not returned to its original position but that hardly matters given the random nature of access.
+
+We used `SetFileInformationByHandle` both to truncate files after writing a full final page to disk and to pre-allocate disk space for faster I/O thus compensating for the absence of `fallocate` although some differences remain. For example, the pre-allocated space is not filled with zeros like on Linux, however, on a positive note, the end of file position is also not modified after pre-allocation.
+
+RocksDB renames, copies and deletes files at will even though they may be opened with another handle at the same time. We had to relax and allow nearly all the concurrent access permissions possible.
+
+## Thread-Local Storage
+Thread-Local storage plays a significant role for RocksDB performance. Rather than creating a separate implementation we chose to create inline wrappers that forward `pthread_specific` calls to Windows `Tls` interfaces within `rocksdb::port` namespace. This leaves the existing meat of the logic in tact and unchanged and just as maintainable.
+
+To mitigate the lack of thread local storage cleanup on thread-exit we added a limited amount of windows specific code within the same thread_local.cc file that injects a cleanup callback into a `"__tls"` structure within `".CRT$XLB"` data segment. This approach guarantees that the callback is invoked regardless of whether RocksDB used within an executable, standalone DLL or within another DLL.
+
+## Jemalloc usage
+
+When RocksDB is used with Jemalloc the latter needs to be initialized before any of the C++ globals or statics. To accomplish that we injected an initialization routine into `".CRT$XCT"` that is automatically invoked by the runtime before initializing static objects. je-uninit is queued to `atexit()`.
+
+The jemalloc redirecting `new/delete` global operators are used by the linker providing certain conditions are met. See build section in these notes.
+
+## Stack Trace and Unhandled Exception Handler
+
+We decided not to implement these two features because the hosting program as a rule has these two things in it.
+We experienced no inconveniences debugging issues in the debugger or analyzing process dumps if need be and thus we did not
+see this as a priority.
+
+## Performance results
+### Setup
+All of the benchmarks are run on the same set of machines. Here are the details of the test setup:
+* 2 Intel(R) Xeon(R) E5 2450 0 @ 2.10 GHz (total 16 cores)
+* 2 XK0480GDQPH SSD Device, total 894GB free disk
+* Machine has 128 GB of RAM
+* Operating System: Windows Server 2012 R2 Datacenter
+* 100 Million keys; each key is of size 10 bytes, each value is of size 800 bytes
+* total database size is ~76GB
+* The performance result is based on RocksDB 3.11.
+* The parameters used, unless specified, were exactly the same as published in the GitHub Wiki page.
+
+### RocksDB on flash storage
+
+#### Test 1. Bulk Load of keys in Random Order
+
+Version 3.11
+
+* Total Run Time: 17.6 min
+* Fillrandom: 5.480 micros/op 182465 ops/sec; 142.0 MB/s
+* Compact: 486056544.000 micros/op 0 ops/sec
+
+Version 3.10
+
+* Total Run Time: 16.2 min
+* Fillrandom: 5.018 micros/op 199269 ops/sec; 155.1 MB/s
+* Compact: 441313173.000 micros/op 0 ops/sec;
+
+
+#### Test 2. Bulk Load of keys in Sequential Order
+
+Version 3.11
+
+* Fillseq: 4.944 micros/op 202k ops/sec; 157.4 MB/s
+
+Version 3.10
+
+* Fillseq: 4.105 micros/op 243.6k ops/sec; 189.6 MB/s
+
+
+#### Test 3. Random Write
+
+Version 3.11
+
+* Unbuffered I/O enabled
+* Overwrite: 52.661 micros/op 18.9k ops/sec; 14.8 MB/s
+
+Version 3.10
+
+* Unbuffered I/O enabled
+* Overwrite: 52.661 micros/op 18.9k ops/sec;
+
+
+#### Test 4. Random Read
+
+Version 3.11
+
+* Unbuffered I/O enabled
+* Readrandom: 15.716 micros/op 63.6k ops/sec; 49.5 MB/s
+
+Version 3.10
+
+* Unbuffered I/O enabled
+* Readrandom: 15.548 micros/op 64.3k ops/sec;
+
+
+#### Test 5. Multi-threaded read and single-threaded write
+
+Version 3.11
+
+* Unbuffered I/O enabled
+* Readwhilewriting: 25.128 micros/op 39.7k ops/sec;
+
+Version 3.10
+
+* Unbuffered I/O enabled
+* Readwhilewriting: 24.854 micros/op 40.2k ops/sec;
+
+
+### RocksDB In Memory
+
+#### Test 1. Point Lookup
+
+Version 3.11
+
+80K writes/sec
+* Write Rate Achieved: 40.5k write/sec;
+* Readwhilewriting: 0.314 micros/op 3187455 ops/sec; 364.8 MB/s (715454999 of 715454999 found)
+
+Version 3.10
+
+* Write Rate Achieved: 50.6k write/sec
+* Readwhilewriting: 0.316 micros/op 3162028 ops/sec; (719576999 of 719576999 found)
+
+
+*10K writes/sec*
+
+Version 3.11
+
+* Write Rate Achieved: 5.8k/s write/sec
+* Readwhilewriting: 0.246 micros/op 4062669 ops/sec; 464.9 MB/s (915481999 of 915481999 found)
+
+Version 3.10
+
+* Write Rate Achieved: 5.8k/s write/sec
+* Readwhilewriting: 0.244 micros/op 4106253 ops/sec; (927986999 of 927986999 found)
+
+
+#### Test 2. Prefix Range Query
+
+Version 3.11
+
+80K writes/sec
+* Write Rate Achieved: 46.3k/s write/sec
+* Readwhilewriting: 0.362 micros/op 2765052 ops/sec; 316.4 MB/s (611549999 of 611549999 found)
+
+Version 3.10
+
+* Write Rate Achieved: 45.8k/s write/sec
+* Readwhilewriting: 0.317 micros/op 3154941 ops/sec; (708158999 of 708158999 found)
+
+Version 3.11
+
+10K writes/sec
+* Write Rate Achieved: 5.78k write/sec
+* Readwhilewriting: 0.269 micros/op 3716692 ops/sec; 425.3 MB/s (837401999 of 837401999 found)
+
+Version 3.10
+
+* Write Rate Achieved: 5.7k write/sec
+* Readwhilewriting: 0.261 micros/op 3830152 ops/sec; (863482999 of 863482999 found)
+
+
+We think that there is still big room to improve the performance, which will be an ongoing effort for us.
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/appveyor.yml
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/appveyor.yml b/thirdparty/rocksdb/appveyor.yml
new file mode 100644
index 0000000..f582bb1
--- /dev/null
+++ b/thirdparty/rocksdb/appveyor.yml
@@ -0,0 +1,15 @@
+version: 1.0.{build}
+image: Visual Studio 2015
+before_build:
+- md %APPVEYOR_BUILD_FOLDER%\build
+- cd %APPVEYOR_BUILD_FOLDER%\build
+- cmake -G "Visual Studio 14 2015 Win64" -DOPTDBG=1 -DXPRESS=1 ..
+- cd ..
+build:
+ project: build\rocksdb.sln
+ parallel: true
+ verbosity: minimal
+test:
+test_script:
+- ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test -Concurrency 8
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/arcanist_util/INTERNAL_ONLY_DIR
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/arcanist_util/INTERNAL_ONLY_DIR b/thirdparty/rocksdb/arcanist_util/INTERNAL_ONLY_DIR
new file mode 100644
index 0000000..e55aa3b
--- /dev/null
+++ b/thirdparty/rocksdb/arcanist_util/INTERNAL_ONLY_DIR
@@ -0,0 +1,2 @@
+arcanist_util are only used internaly, If you want to change it please check
+<internal_rocksdb_repo>/arcanist_util
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/buckifier/buckify_rocksdb.py
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/buckifier/buckify_rocksdb.py b/thirdparty/rocksdb/buckifier/buckify_rocksdb.py
new file mode 100644
index 0000000..a3c8be3
--- /dev/null
+++ b/thirdparty/rocksdb/buckifier/buckify_rocksdb.py
@@ -0,0 +1,172 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from targets_builder import TARGETSBuilder
+from optparse import OptionParser
+import os
+import fnmatch
+import sys
+import tempfile
+
+from util import ColorString
+import util
+
+# tests to export as libraries for inclusion in other projects
+_EXPORTED_TEST_LIBS = ["env_basic_test"]
+
+# Parse src.mk files as a Dictionary of
+# VAR_NAME => list of files
+def parse_src_mk(repo_path):
+ src_mk = repo_path + "/src.mk"
+ src_files = {}
+ for line in open(src_mk):
+ line = line.strip()
+ if len(line) == 0 or line[0] == '#':
+ continue
+ if '=' in line:
+ current_src = line.split('=')[0].strip()
+ src_files[current_src] = []
+ elif '.cc' in line:
+ src_path = line.split('.cc')[0].strip() + '.cc'
+ src_files[current_src].append(src_path)
+ return src_files
+
+
+# get all .cc / .c files
+def get_cc_files(repo_path):
+ cc_files = []
+ for root, dirnames, filenames in os.walk(repo_path):
+ root = root[(len(repo_path) + 1):]
+ if "java" in root:
+ # Skip java
+ continue
+ for filename in fnmatch.filter(filenames, '*.cc'):
+ cc_files.append(os.path.join(root, filename))
+ for filename in fnmatch.filter(filenames, '*.c'):
+ cc_files.append(os.path.join(root, filename))
+ return cc_files
+
+
+# Get tests from Makefile
+def get_tests(repo_path):
+ Makefile = repo_path + "/Makefile"
+
+ # Dictionary TEST_NAME => IS_PARALLEL
+ tests = {}
+
+ found_tests = False
+ for line in open(Makefile):
+ line = line.strip()
+ if line.startswith("TESTS ="):
+ found_tests = True
+ elif found_tests:
+ if line.endswith("\\"):
+ # remove the trailing \
+ line = line[:-1]
+ line = line.strip()
+ tests[line] = False
+ else:
+ # we consumed all the tests
+ break
+
+ found_parallel_tests = False
+ for line in open(Makefile):
+ line = line.strip()
+ if line.startswith("PARALLEL_TEST ="):
+ found_parallel_tests = True
+ elif found_parallel_tests:
+ if line.endswith("\\"):
+ # remove the trailing \
+ line = line[:-1]
+ line = line.strip()
+ tests[line] = True
+ else:
+ # we consumed all the parallel tests
+ break
+
+ return tests
+
+
+# Prepare TARGETS file for buck
+def generate_targets(repo_path):
+ print(ColorString.info("Generating TARGETS"))
+ # parsed src.mk file
+ src_mk = parse_src_mk(repo_path)
+ # get all .cc files
+ cc_files = get_cc_files(repo_path)
+ # get tests from Makefile
+ tests = get_tests(repo_path)
+
+ if src_mk is None or cc_files is None or tests is None:
+ return False
+
+ TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
+ # rocksdb_lib
+ TARGETS.add_library(
+ "rocksdb_lib",
+ src_mk["LIB_SOURCES"] +
+ src_mk["TOOL_LIB_SOURCES"])
+ # rocksdb_test_lib
+ TARGETS.add_library(
+ "rocksdb_test_lib",
+ src_mk.get("MOCK_LIB_SOURCES", []) +
+ src_mk.get("TEST_LIB_SOURCES", []) +
+ src_mk.get("EXP_LIB_SOURCES", []),
+ [":rocksdb_lib"])
+ # rocksdb_tools_lib
+ TARGETS.add_library(
+ "rocksdb_tools_lib",
+ src_mk.get("BENCH_LIB_SOURCES", []) +
+ ["util/testutil.cc"],
+ [":rocksdb_lib"])
+
+ # test for every test we found in the Makefile
+ for test in sorted(tests):
+ match_src = [src for src in cc_files if ("/%s.c" % test) in src]
+ if len(match_src) == 0:
+ print(ColorString.warning("Cannot find .cc file for %s" % test))
+ continue
+ elif len(match_src) > 1:
+ print(ColorString.warning("Found more than one .cc for %s" % test))
+ print(match_src)
+ continue
+
+ assert(len(match_src) == 1)
+ is_parallel = tests[test]
+ TARGETS.register_test(test, match_src[0], is_parallel)
+
+ if test in _EXPORTED_TEST_LIBS:
+ test_library = "%s_lib" % test
+ TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
+ TARGETS.flush_tests()
+
+ print(ColorString.info("Generated TARGETS Summary:"))
+ print(ColorString.info("- %d libs" % TARGETS.total_lib))
+ print(ColorString.info("- %d binarys" % TARGETS.total_bin))
+ print(ColorString.info("- %d tests" % TARGETS.total_test))
+ return True
+
+
+def get_rocksdb_path():
+ # rocksdb = {script_dir}/..
+ script_dir = os.path.dirname(sys.argv[0])
+ script_dir = os.path.abspath(script_dir)
+ rocksdb_path = os.path.abspath(
+ os.path.join(script_dir, "../"))
+
+ return rocksdb_path
+
+def exit_with_error(msg):
+ print(ColorString.error(msg))
+ sys.exit(1)
+
+
+def main():
+ # Generate TARGETS file for buck
+ ok = generate_targets(get_rocksdb_path())
+ if not ok:
+ exit_with_error("Failed to generate TARGETS files")
+
+if __name__ == "__main__":
+ main()
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/buckifier/targets_builder.py
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/buckifier/targets_builder.py b/thirdparty/rocksdb/buckifier/targets_builder.py
new file mode 100644
index 0000000..7d47d2d
--- /dev/null
+++ b/thirdparty/rocksdb/buckifier/targets_builder.py
@@ -0,0 +1,65 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+import targets_cfg
+import pprint
+
+# TODO(tec): replace this with PrettyPrinter
+def pretty_list(lst, indent=6):
+ if lst is None or len(lst) == 0:
+ return ""
+
+ if len(lst) == 1:
+ return "\"%s\"" % lst[0]
+
+ separator = "\",\n%s\"" % (" " * indent)
+ res = separator.join(lst)
+ res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 2))
+ return res
+
+
+class TARGETSBuilder:
+ def __init__(self, path):
+ self.path = path
+ self.targets_file = open(path, 'w')
+ self.targets_file.write(targets_cfg.rocksdb_target_header)
+ self.total_lib = 0
+ self.total_bin = 0
+ self.total_test = 0
+ self.tests_cfg = []
+
+ def __del__(self):
+ self.targets_file.close()
+
+ def add_library(self, name, srcs, deps=None, headers=None):
+ if headers is None:
+ headers = "AutoHeaders.RECURSIVE_GLOB"
+ self.targets_file.write(targets_cfg.library_template % (
+ name,
+ headers,
+ pretty_list(srcs),
+ pretty_list(deps)))
+ self.total_lib = self.total_lib + 1
+
+ def add_binary(self, name, srcs, deps=None):
+ self.targets_file.write(targets_cfg.binary_template % (
+ name,
+ pretty_list(srcs),
+ pretty_list(deps)))
+ self.total_bin = self.total_bin + 1
+
+ def register_test(self, test_name, src, is_parallel):
+ exec_mode = "serial"
+ if is_parallel:
+ exec_mode = "parallel"
+ self.tests_cfg.append([test_name, str(src), str(exec_mode)])
+
+ self.total_test = self.total_test + 1
+
+ def flush_tests(self):
+ self.targets_file.write(targets_cfg.unittests_template % (
+ pprint.PrettyPrinter().pformat(self.tests_cfg)
+ ))
+
+ self.tests_cfg = []
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/buckifier/targets_cfg.py
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/buckifier/targets_cfg.py b/thirdparty/rocksdb/buckifier/targets_cfg.py
new file mode 100644
index 0000000..33023a5
--- /dev/null
+++ b/thirdparty/rocksdb/buckifier/targets_cfg.py
@@ -0,0 +1,124 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+rocksdb_target_header = """
+import os
+
+TARGETS_PATH = os.path.dirname(__file__)
+REPO_PATH = "rocksdb/src/"
+BUCK_BINS = "buck-out/gen/" + REPO_PATH
+TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
+rocksdb_compiler_flags = [
+ "-fno-builtin-memcmp",
+ "-DROCKSDB_PLATFORM_POSIX",
+ "-DROCKSDB_LIB_IO_POSIX",
+ "-DROCKSDB_FALLOCATE_PRESENT",
+ "-DROCKSDB_MALLOC_USABLE_SIZE",
+ "-DROCKSDB_RANGESYNC_PRESENT",
+ "-DROCKSDB_SCHED_GETCPU_PRESENT",
+ "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+ "-DOS_LINUX",
+ # Flags to enable libs we include
+ "-DSNAPPY",
+ "-DZLIB",
+ "-DBZIP2",
+ "-DLZ4",
+ "-DZSTD",
+ "-DGFLAGS=gflags",
+ "-DNUMA",
+ "-DTBB",
+ # Needed to compile in fbcode
+ "-Wno-expansion-to-defined",
+]
+
+rocksdb_external_deps = [
+ ('bzip2', None, 'bz2'),
+ ('snappy', None, "snappy"),
+ ('zlib', None, 'z'),
+ ('gflags', None, 'gflags'),
+ ('lz4', None, 'lz4'),
+ ('zstd', None),
+ ('tbb', None),
+ ("numa", None, "numa"),
+ ("googletest", None, "gtest"),
+]
+
+rocksdb_preprocessor_flags = [
+ # Directories with files for #include
+ "-I" + REPO_PATH + "include/",
+ "-I" + REPO_PATH,
+]
+
+rocksdb_arch_preprocessor_flags = {
+ "x86_64": ["-DHAVE_SSE42"],
+}
+"""
+
+
+library_template = """
+cpp_library(
+ name = "%s",
+ headers = %s,
+ srcs = [%s],
+ deps = [%s],
+ preprocessor_flags = rocksdb_preprocessor_flags,
+ arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+ compiler_flags = rocksdb_compiler_flags,
+ external_deps = rocksdb_external_deps,
+)
+"""
+
+binary_template = """
+cpp_binary(
+ name = "%s",
+ srcs = [%s],
+ deps = [%s],
+ preprocessor_flags = rocksdb_preprocessor_flags,
+ arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+ compiler_flags = rocksdb_compiler_flags,
+ external_deps = rocksdb_external_deps,
+)
+"""
+
+unittests_template = """
+# [test_name, test_src, test_type]
+ROCKS_TESTS = %s
+
+
+# Generate a test rule for each entry in ROCKS_TESTS
+for test_cfg in ROCKS_TESTS:
+ test_name = test_cfg[0]
+ test_cc = test_cfg[1]
+ ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
+ test_bin = test_name + "_bin"
+
+ cpp_binary (
+ name = test_bin,
+ srcs = [test_cc],
+ deps = [":rocksdb_test_lib"],
+ preprocessor_flags = rocksdb_preprocessor_flags,
+ arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+ compiler_flags = rocksdb_compiler_flags,
+ external_deps = rocksdb_external_deps,
+ )
+
+ custom_unittest(
+ name = test_name,
+ type = ttype,
+ deps = [":" + test_bin],
+ command = [TEST_RUNNER, BUCK_BINS + test_bin]
+ )
+
+custom_unittest(
+ name = "make_rocksdbjavastatic",
+ type = "simple",
+ command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
+)
+
+custom_unittest(
+ name = "make_rocksdb_lite_release",
+ type = "simple",
+ command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
+)
+"""
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/buckifier/util.py
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/buckifier/util.py b/thirdparty/rocksdb/buckifier/util.py
new file mode 100644
index 0000000..350b733
--- /dev/null
+++ b/thirdparty/rocksdb/buckifier/util.py
@@ -0,0 +1,107 @@
+"""
+This module keeps commonly used components.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+import subprocess
+import os
+import time
+
+class ColorString:
+ """ Generate colorful strings on terminal """
+ HEADER = '\033[95m'
+ BLUE = '\033[94m'
+ GREEN = '\033[92m'
+ WARNING = '\033[93m'
+ FAIL = '\033[91m'
+ ENDC = '\033[0m'
+
+ @staticmethod
+ def _make_color_str(text, color):
+ return "".join([color, text.encode('utf-8'), ColorString.ENDC])
+
+ @staticmethod
+ def ok(text):
+ if ColorString.is_disabled:
+ return text
+ return ColorString._make_color_str(text, ColorString.GREEN)
+
+ @staticmethod
+ def info(text):
+ if ColorString.is_disabled:
+ return text
+ return ColorString._make_color_str(text, ColorString.BLUE)
+
+ @staticmethod
+ def header(text):
+ if ColorString.is_disabled:
+ return text
+ return ColorString._make_color_str(text, ColorString.HEADER)
+
+ @staticmethod
+ def error(text):
+ if ColorString.is_disabled:
+ return text
+ return ColorString._make_color_str(text, ColorString.FAIL)
+
+ @staticmethod
+ def warning(text):
+ if ColorString.is_disabled:
+ return text
+ return ColorString._make_color_str(text, ColorString.WARNING)
+
+ is_disabled = False
+
+
+def run_shell_command(shell_cmd, cmd_dir=None):
+ """ Run a single shell command.
+ @returns a tuple of shell command return code, stdout, stderr """
+
+ if cmd_dir is not None and not os.path.exists(cmd_dir):
+ run_shell_command("mkdir -p %s" % cmd_dir)
+
+ start = time.time()
+ print("\t>>> Running: " + shell_cmd)
+ p = subprocess.Popen(shell_cmd,
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=cmd_dir)
+ stdout, stderr = p.communicate()
+ end = time.time()
+
+ # Report time if we spent more than 5 minutes executing a command
+ execution_time = end - start
+ if execution_time > (60 * 5):
+ mins = (execution_time / 60)
+ secs = (execution_time % 60)
+ print("\t>time spent: %d minutes %d seconds" % (mins, secs))
+
+
+ return p.returncode, stdout, stderr
+
+
+def run_shell_commands(shell_cmds, cmd_dir=None, verbose=False):
+ """ Execute a sequence of shell commands, which is equivalent to
+ running `cmd1 && cmd2 && cmd3`
+ @returns boolean indication if all commands succeeds.
+ """
+
+ if cmd_dir:
+ print("\t=== Set current working directory => %s" % cmd_dir)
+
+ for shell_cmd in shell_cmds:
+ ret_code, stdout, stderr = run_shell_command(shell_cmd, cmd_dir)
+ if stdout:
+ if verbose or ret_code != 0:
+ print(ColorString.info("stdout: \n"), stdout)
+ if stderr:
+ # contents in stderr is not necessarily to be error messages.
+ if verbose or ret_code != 0:
+ print(ColorString.error("stderr: \n"), stderr)
+ if ret_code != 0:
+ return False
+
+ return True
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/RocksDBCommonHelper.php
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/RocksDBCommonHelper.php b/thirdparty/rocksdb/build_tools/RocksDBCommonHelper.php
new file mode 100644
index 0000000..41d1e21
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/RocksDBCommonHelper.php
@@ -0,0 +1,365 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+// Name of the environment variables which need to be set by the entity which
+// triggers continuous runs so that code at the end of the file gets executed
+// and Sandcastle run starts.
+define("ENV_POST_RECEIVE_HOOK", "POST_RECEIVE_HOOK");
+define("ENV_HTTPS_APP_VALUE", "HTTPS_APP_VALUE");
+define("ENV_HTTPS_TOKEN_VALUE", "HTTPS_TOKEN_VALUE");
+
+define("PRIMARY_TOKEN_FILE", '/home/krad/.sandcastle');
+define("CONT_RUN_ALIAS", "leveldb");
+
+//////////////////////////////////////////////////////////////////////
+/* Run tests in sandcastle */
+function postURL($diffID, $url) {
+ assert(strlen($diffID) > 0);
+ assert(is_numeric($diffID));
+ assert(strlen($url) > 0);
+
+ $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
+ . '"name":"click here for sandcastle tests for D' . $diffID . '", '
+ . '"link":"' . $url . '"}\' | '
+ . 'arc call-conduit '
+ . 'differential.updateunitresults';
+ shell_exec($cmd);
+}
+
+function buildUpdateTestStatusCmd($diffID, $test, $status) {
+ assert(strlen($diffID) > 0);
+ assert(is_numeric($diffID));
+ assert(strlen($test) > 0);
+ assert(strlen($status) > 0);
+
+ $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
+ . '"name":"' . $test . '", '
+ . '"result":"' . $status . '"}\' | '
+ . 'arc call-conduit '
+ . 'differential.updateunitresults';
+ return $cmd;
+}
+
+function updateTestStatus($diffID, $test) {
+ assert(strlen($diffID) > 0);
+ assert(is_numeric($diffID));
+ assert(strlen($test) > 0);
+
+ shell_exec(buildUpdateTestStatusCmd($diffID, $test, "waiting"));
+}
+
+function getSteps($applyDiff, $diffID, $username, $test) {
+ assert(strlen($username) > 0);
+ assert(strlen($test) > 0);
+
+ if ($applyDiff) {
+ assert(strlen($diffID) > 0);
+ assert(is_numeric($diffID));
+
+ $arcrc_content = (PHP_OS == "Darwin" ?
+ exec("cat ~/.arcrc | gzip -f | base64") :
+ exec("cat ~/.arcrc | gzip -f | base64 -w0"));
+ assert(strlen($arcrc_content) > 0);
+
+ // Sandcastle machines don't have arc setup. We copy the user certificate
+ // and authenticate using that in Sandcastle.
+ $setup = array(
+ "name" => "Setup arcrc",
+ "shell" => "echo " . $arcrc_content . " | base64 --decode"
+ . " | gzip -d > ~/.arcrc",
+ "user" => "root"
+ );
+
+ // arc demands certain permission on its config.
+ // also fix the sticky bit issue in sandcastle
+ $fix_permission = array(
+ "name" => "Fix environment",
+ "shell" => "chmod 600 ~/.arcrc && chmod +t /dev/shm",
+ "user" => "root"
+ );
+
+ // Construct the steps in the order of execution.
+ $steps[] = $setup;
+ $steps[] = $fix_permission;
+ }
+
+ // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
+ // Git thinks it is an uncommited change.
+ $fix_git_ignore = array(
+ "name" => "Fix git ignore",
+ "shell" => "echo fbcode >> .git/info/exclude",
+ "user" => "root"
+ );
+
+ // This fixes "FATAL: ThreadSanitizer can not mmap the shadow memory"
+ // Source:
+ // https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual#FAQ
+ $fix_kernel_issue = array(
+ "name" => "Fix kernel issue with tsan",
+ "shell" => "echo 2 >/proc/sys/kernel/randomize_va_space",
+ "user" => "root"
+ );
+
+ $steps[] = $fix_git_ignore;
+ $steps[] = $fix_kernel_issue;
+
+ // This will be the command used to execute particular type of tests.
+ $cmd = "";
+
+ if ($applyDiff) {
+ // Patch the code (keep your fingures crossed).
+ $patch = array(
+ "name" => "Patch " . $diffID,
+ "shell" => "arc --arcrc-file ~/.arcrc "
+ . "patch --nocommit --diff " . $diffID,
+ "user" => "root"
+ );
+
+ $steps[] = $patch;
+
+ updateTestStatus($diffID, $test);
+ $cmd = buildUpdateTestStatusCmd($diffID, $test, "running") . "; ";
+ }
+
+ // Run the actual command.
+ $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " . $test
+ . "; exit_code=$?; ";
+
+ if ($applyDiff) {
+ $cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&"
+ . buildUpdateTestStatusCmd($diffID, $test, "pass") . ")"
+ . "||" . buildUpdateTestStatusCmd($diffID, $test, "fail")
+ . "; ";
+ }
+
+ // shell command to sort the tests based on exit code and print
+ // the output of the log files.
+ $cat_sorted_logs = "
+ while read code log_file;
+ do echo \"################ cat \$log_file [exit_code : \$code] ################\";
+ cat \$log_file;
+ done < <(tail -n +2 LOG | sort -k7,7n -k4,4gr | awk '{print \$7,\$NF}')";
+
+ // Shell command to cat all log files
+ $cat_all_logs = "for f in `ls t/!(run-*)`; do echo \$f;cat \$f; done";
+
+ // If LOG file exist use it to cat log files sorted by exit code, otherwise
+ // cat everything
+ $logs_cmd = "if [ -f LOG ]; then {$cat_sorted_logs}; else {$cat_all_logs}; fi";
+
+ $cmd = $cmd . " cat /tmp/precommit-check.log"
+ . "; shopt -s extglob; {$logs_cmd}"
+ . "; shopt -u extglob; [[ \$exit_code -eq 0 ]]";
+ assert(strlen($cmd) > 0);
+
+ $run_test = array(
+ "name" => "Run " . $test,
+ "shell" => $cmd,
+ "user" => "root",
+ "parser" => "python build_tools/error_filter.py " . $test,
+ );
+
+ $steps[] = $run_test;
+
+ if ($applyDiff) {
+ // Clean up the user arc config we are using.
+ $cleanup = array(
+ "name" => "Arc cleanup",
+ "shell" => "rm -f ~/.arcrc",
+ "user" => "root"
+ );
+
+ $steps[] = $cleanup;
+ }
+
+ assert(count($steps) > 0);
+ return $steps;
+}
+
+function getSandcastleConfig() {
+ $sandcastle_config = array();
+
+ $cwd = getcwd();
+ $cwd_token_file = "{$cwd}/.sandcastle";
+ // This is a case when we're executed from a continuous run. Fetch the values
+ // from the environment.
+ if (getenv(ENV_POST_RECEIVE_HOOK)) {
+ $sandcastle_config[0] = getenv(ENV_HTTPS_APP_VALUE);
+ $sandcastle_config[1] = getenv(ENV_HTTPS_TOKEN_VALUE);
+ } else {
+ // This is a typical `[p]arc diff` case. Fetch the values from the specific
+ // configuration files.
+ for ($i = 0; $i < 50; $i++) {
+ if (file_exists(PRIMARY_TOKEN_FILE) ||
+ file_exists($cwd_token_file)) {
+ break;
+ }
+ // If we failed to fetch the tokens, sleep for 0.2 second and try again
+ usleep(200000);
+ }
+ assert(file_exists(PRIMARY_TOKEN_FILE) ||
+ file_exists($cwd_token_file));
+
+ // Try the primary location first, followed by a secondary.
+ if (file_exists(PRIMARY_TOKEN_FILE)) {
+ $cmd = 'cat ' . PRIMARY_TOKEN_FILE;
+ } else {
+ $cmd = 'cat ' . $cwd_token_file;
+ }
+
+ assert(strlen($cmd) > 0);
+ $sandcastle_config = explode(':', rtrim(shell_exec($cmd)));
+ }
+
+ // In this case be very explicit about the implications.
+ if (count($sandcastle_config) != 2) {
+ echo "Sandcastle configuration files don't contain valid information " .
+ "or the necessary environment variables aren't defined. Unable " .
+ "to validate the code changes.";
+ exit(1);
+ }
+
+ assert(strlen($sandcastle_config[0]) > 0);
+ assert(strlen($sandcastle_config[1]) > 0);
+ assert(count($sandcastle_config) > 0);
+
+ return $sandcastle_config;
+}
+
+// This function can be called either from `[p]arc diff` command or during
+// the Git post-receive hook.
+ function startTestsInSandcastle($applyDiff, $workflow, $diffID) {
+ // Default options don't terminate on failure, but that's what we want. In
+ // the current case we use assertions intentionally as "terminate on failure
+ // invariants".
+ assert_options(ASSERT_BAIL, true);
+
+ // In case of a diff we'll send notificatios to the author. Else it'll go to
+ // the entire team because failures indicate that build quality has regressed.
+ $username = $applyDiff ? exec("whoami") : CONT_RUN_ALIAS;
+ assert(strlen($username) > 0);
+
+ if ($applyDiff) {
+ assert($workflow);
+ assert(strlen($diffID) > 0);
+ assert(is_numeric($diffID));
+ }
+
+ // List of tests we want to run in Sandcastle.
+ $tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan",
+ "asan", "lite_test", "valgrind", "release", "release_481",
+ "clang_release", "clang_analyze", "code_cov",
+ "java_build", "no_compression", "unity", "ubsan");
+
+ $send_email_template = array(
+ 'type' => 'email',
+ 'triggers' => array('fail'),
+ 'emails' => array($username . '@fb.com'),
+ );
+
+ // Construct a job definition for each test and add it to the master plan.
+ foreach ($tests as $test) {
+ $stepName = "RocksDB diff " . $diffID . " test " . $test;
+
+ if (!$applyDiff) {
+ $stepName = "RocksDB continuous integration test " . $test;
+ }
+
+ $arg[] = array(
+ "name" => $stepName,
+ "report" => array($send_email_template),
+ "steps" => getSteps($applyDiff, $diffID, $username, $test)
+ );
+ }
+
+ // We cannot submit the parallel execution master plan to Sandcastle and
+ // need supply the job plan as a determinator. So we construct a small job
+ // that will spit out the master job plan which Sandcastle will parse and
+ // execute. Why compress the job definitions? Otherwise we run over the max
+ // string size.
+ $cmd = "echo " . base64_encode(json_encode($arg))
+ . (PHP_OS == "Darwin" ?
+ " | gzip -f | base64" :
+ " | gzip -f | base64 -w0");
+ assert(strlen($cmd) > 0);
+
+ $arg_encoded = shell_exec($cmd);
+ assert(strlen($arg_encoded) > 0);
+
+ $runName = "Run diff " . $diffID . "for user " . $username;
+
+ if (!$applyDiff) {
+ $runName = "RocksDB continuous integration build and test run";
+ }
+
+ $command = array(
+ "name" => $runName,
+ "steps" => array()
+ );
+
+ $command["steps"][] = array(
+ "name" => "Generate determinator",
+ "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d"
+ . " | base64 --decode",
+ "determinator" => true,
+ "user" => "root"
+ );
+
+ // Submit to Sandcastle.
+ $url = 'https://interngraph.intern.facebook.com/sandcastle/create';
+
+ $job = array(
+ 'command' => 'SandcastleUniversalCommand',
+ 'args' => $command,
+ 'capabilities' => array(
+ 'vcs' => 'rocksdb-int-git',
+ 'type' => 'lego',
+ ),
+ 'hash' => 'origin/master',
+ 'user' => $username,
+ 'alias' => 'rocksdb-precommit',
+ 'tags' => array('rocksdb'),
+ 'description' => 'Rocksdb precommit job',
+ );
+
+ // Fetch the configuration necessary to submit a successful HTTPS request.
+ $sandcastle_config = getSandcastleConfig();
+
+ $app = $sandcastle_config[0];
+ $token = $sandcastle_config[1];
+
+ $cmd = 'curl -s -k -F app=' . $app . ' '
+ . '-F token=' . $token . ' -F job=\'' . json_encode($job)
+ .'\' "' . $url . '"';
+
+ $output = shell_exec($cmd);
+ assert(strlen($output) > 0);
+
+ // Extract Sandcastle URL from the response.
+ preg_match('/url": "(.+)"/', $output, $sandcastle_url);
+
+ assert(count($sandcastle_url) > 0, "Unable to submit Sandcastle request.");
+ assert(strlen($sandcastle_url[1]) > 0, "Unable to extract Sandcastle URL.");
+
+ if ($applyDiff) {
+ echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n";
+ // Ask Phabricator to display it on the diff UI.
+ postURL($diffID, $sandcastle_url[1]);
+ } else {
+ echo "Continuous integration started Sandcastle tests. You can look at ";
+ echo "the progress at:\n" . $sandcastle_url[1] . "\n";
+ }
+}
+
+// Continuous run cript will set the environment variable and based on that
+// we'll trigger the execution of tests in Sandcastle. In that case we don't
+// need to apply any diffs and there's no associated workflow either.
+if (getenv(ENV_POST_RECEIVE_HOOK)) {
+ startTestsInSandcastle(
+ false /* $applyDiff */,
+ NULL /* $workflow */,
+ NULL /* $diffID */);
+}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/amalgamate.py
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/amalgamate.py b/thirdparty/rocksdb/build_tools/amalgamate.py
new file mode 100755
index 0000000..548b1e8
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/amalgamate.py
@@ -0,0 +1,110 @@
+#!/usr/bin/python
+
+# amalgamate.py creates an amalgamation from a unity build.
+# It can be run with either Python 2 or 3.
+# An amalgamation consists of a header that includes the contents of all public
+# headers and a source file that includes the contents of all source files and
+# private headers.
+#
+# This script works by starting with the unity build file and recursively expanding
+# #include directives. If the #include is found in a public include directory,
+# that header is expanded into the amalgamation header.
+#
+# A particular header is only expanded once, so this script will
+# break if there are multiple inclusions of the same header that are expected to
+# expand differently. Similarly, this type of code causes issues:
+#
+# #ifdef FOO
+# #include "bar.h"
+# // code here
+# #else
+# #include "bar.h" // oops, doesn't get expanded
+# // different code here
+# #endif
+#
+# The solution is to move the include out of the #ifdef.
+
+from __future__ import print_function
+
+import argparse
+from os import path
+import re
+import sys
+
+include_re = re.compile('^[ \t]*#include[ \t]+"(.*)"[ \t]*$')
+included = set()
+excluded = set()
+
+def find_header(name, abs_path, include_paths):
+ samedir = path.join(path.dirname(abs_path), name)
+ if path.exists(samedir):
+ return samedir
+ for include_path in include_paths:
+ include_path = path.join(include_path, name)
+ if path.exists(include_path):
+ return include_path
+ return None
+
+def expand_include(include_path, f, abs_path, source_out, header_out, include_paths, public_include_paths):
+ if include_path in included:
+ return False
+
+ included.add(include_path)
+ with open(include_path) as f:
+ print('#line 1 "{}"'.format(include_path), file=source_out)
+ process_file(f, include_path, source_out, header_out, include_paths, public_include_paths)
+ return True
+
+def process_file(f, abs_path, source_out, header_out, include_paths, public_include_paths):
+ for (line, text) in enumerate(f):
+ m = include_re.match(text)
+ if m:
+ filename = m.groups()[0]
+ # first check private headers
+ include_path = find_header(filename, abs_path, include_paths)
+ if include_path:
+ if include_path in excluded:
+ source_out.write(text)
+ expanded = False
+ else:
+ expanded = expand_include(include_path, f, abs_path, source_out, header_out, include_paths, public_include_paths)
+ else:
+ # now try public headers
+ include_path = find_header(filename, abs_path, public_include_paths)
+ if include_path:
+ # found public header
+ expanded = False
+ if include_path in excluded:
+ source_out.write(text)
+ else:
+ expand_include(include_path, f, abs_path, header_out, None, public_include_paths, [])
+ else:
+ sys.exit("unable to find {}, included in {} on line {}".format(filename, abs_path, line))
+
+ if expanded:
+ print('#line {} "{}"'.format(line+1, abs_path), file=source_out)
+ elif text != "#pragma once\n":
+ source_out.write(text)
+
+def main():
+ parser = argparse.ArgumentParser(description="Transform a unity build into an amalgamation")
+ parser.add_argument("source", help="source file")
+ parser.add_argument("-I", action="append", dest="include_paths", help="include paths for private headers")
+ parser.add_argument("-i", action="append", dest="public_include_paths", help="include paths for public headers")
+ parser.add_argument("-x", action="append", dest="excluded", help="excluded header files")
+ parser.add_argument("-o", dest="source_out", help="output C++ file", required=True)
+ parser.add_argument("-H", dest="header_out", help="output C++ header file", required=True)
+ args = parser.parse_args()
+
+ include_paths = list(map(path.abspath, args.include_paths or []))
+ public_include_paths = list(map(path.abspath, args.public_include_paths or []))
+ excluded.update(map(path.abspath, args.excluded or []))
+ filename = args.source
+ abs_path = path.abspath(filename)
+ with open(filename) as f, open(args.source_out, 'w') as source_out, open(args.header_out, 'w') as header_out:
+ print('#line 1 "{}"'.format(filename), file=source_out)
+ print('#include "{}"'.format(header_out.name), file=source_out)
+ process_file(f, abs_path, source_out, header_out, include_paths, public_include_paths)
+
+if __name__ == "__main__":
+ main()
[35/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/HISTORY.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/HISTORY.md b/thirdparty/rocksdb/HISTORY.md
new file mode 100644
index 0000000..29f0f3f
--- /dev/null
+++ b/thirdparty/rocksdb/HISTORY.md
@@ -0,0 +1,568 @@
+# Rocksdb Change Log
+## Unreleased
+### New Features
+* Add Iterator::Refresh(), which allows users to update the iterator state so that they can avoid some initialization costs of recreating iterators.
+* Replace dynamic_cast<> (except unit test) so people can choose to build with RTTI off. With make, release mode is by default built with -fno-rtti and debug mode is built without it. Users can override it by setting USE_RTTI=0 or 1.
+* Universal compactions including the bottom level can be executed in a dedicated thread pool. This alleviates head-of-line blocking in the compaction queue, which cause write stalling, particularly in multi-instance use cases. Users can enable this feature via `Env::SetBackgroundThreads(N, Env::Priority::BOTTOM)`, where `N > 0`.
+
+### Bug Fixes
+* Fix wrong latencies in `rocksdb.db.get.micros`, `rocksdb.db.write.micros`, and `rocksdb.sst.read.micros`.
+* Fix incorrect dropping of deletions during intra-L0 compaction.
+
+## 5.7.0 (07/13/2017)
+### Public API Change
+* DB property "rocksdb.sstables" now prints keys in hex form.
+
+### New Features
+* Measure estimated number of reads per file. The information can be accessed through DB::GetColumnFamilyMetaData or "rocksdb.sstables" DB property.
+* RateLimiter support for throttling background reads, or throttling the sum of background reads and writes. This can give more predictable I/O usage when compaction reads more data than it writes, e.g., due to lots of deletions.
+* [Experimental] FIFO compaction with TTL support. It can be enabled by setting CompactionOptionsFIFO.ttl > 0.
+* Introduce `EventListener::OnBackgroundError()` callback. Users can implement it to be notified of errors causing the DB to enter read-only mode, and optionally override them.
+* Partitioned Index/Filters exiting the experimental mode. To enable partitioned indexes set index_type to kTwoLevelIndexSearch and to further enable partitioned filters set partition_filters to true. To configure the partition size set metadata_block_size.
+
+
+### Bug Fixes
+* Fix discarding empty compaction output files when `DeleteRange()` is used together with subcompactions.
+
+## 5.6.0 (06/06/2017)
+### Public API Change
+* Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads.
+* Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction.
+* options.delayed_write_rate by default take the value of options.rate_limiter rate.
+* Replace global variable `IOStatsContext iostats_context` with `IOStatsContext* get_iostats_context()`; replace global variable `PerfContext perf_context` with `PerfContext* get_perf_context()`.
+
+### New Features
+* Change ticker/histogram statistics implementations to use core-local storage. This improves aggregation speed compared to our previous thread-local approach, particularly for applications with many threads.
+* Users can pass a cache object to write buffer manager, so that they can cap memory usage for memtable and block cache using one single limit.
+* Flush will be triggered when 7/8 of the limit introduced by write_buffer_manager or db_write_buffer_size is triggered, so that the hard threshold is hard to hit.
+* Introduce WriteOptions.low_pri. If it is true, low priority writes will be throttled if the compaction is behind.
+* `DB::IngestExternalFile()` now supports ingesting files into a database containing range deletions.
+
+### Bug Fixes
+* Shouldn't ignore return value of fsync() in flush.
+
+## 5.5.0 (05/17/2017)
+### New Features
+* FIFO compaction to support Intra L0 compaction too with CompactionOptionsFIFO.allow_compaction=true.
+* DB::ResetStats() to reset internal stats.
+* Statistics::Reset() to reset user stats.
+* ldb add option --try_load_options, which will open DB with its own option file.
+* Introduce WriteBatch::PopSavePoint to pop the most recent save point explicitly.
+* Support dynamically change `max_open_files` option via SetDBOptions()
+* Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families.
+* Add debugging function `GetAllKeyVersions` to see internal versions of a range of keys.
+* Support file ingestion with universal compaction style
+* Support file ingestion behind with option `allow_ingest_behind`
+* New option enable_pipelined_write which may improve write throughput in case writing from multiple threads and WAL enabled.
+
+### Bug Fixes
+* Fix the bug that Direct I/O uses direct reads for non-SST file
+
+## 5.4.0 (04/11/2017)
+### Public API Change
+* random_access_max_buffer_size no longer has any effect
+* Removed Env::EnableReadAhead(), Env::ShouldForwardRawRequest()
+* Support dynamically change `stats_dump_period_sec` option via SetDBOptions().
+* Added ReadOptions::max_skippable_internal_keys to set a threshold to fail a request as incomplete when too many keys are being skipped when using iterators.
+* DB::Get in place of std::string accepts PinnableSlice, which avoids the extra memcpy of value to std::string in most of cases.
+ * PinnableSlice releases the pinned resources that contain the value when it is destructed or when ::Reset() is called on it.
+ * The old API that accepts std::string, although discouraged, is still supported.
+* Replace Options::use_direct_writes with Options::use_direct_io_for_flush_and_compaction. Read Direct IO wiki for details.
+* Added CompactionEventListener and EventListener::OnFlushBegin interfaces.
+
+### New Features
+* Memtable flush can be avoided during checkpoint creation if total log file size is smaller than a threshold specified by the user.
+* Introduce level-based L0->L0 compactions to reduce file count, so write delays are incurred less often.
+* (Experimental) Partitioning filters which creates an index on the partitions. The feature can be enabled by setting partition_filters when using kFullFilter. Currently the feature also requires two-level indexing to be enabled. Number of partitions is the same as the number of partitions for indexes, which is controlled by metadata_block_size.
+
+## 5.3.0 (03/08/2017)
+### Public API Change
+* Remove disableDataSync option.
+* Remove timeout_hint_us option from WriteOptions. The option has been deprecated and has no effect since 3.13.0.
+* Remove option min_partial_merge_operands. Partial merge operands will always be merged in flush or compaction if there are more than one.
+* Remove option verify_checksums_in_compaction. Compaction will always verify checksum.
+
+### Bug Fixes
+* Fix the bug that iterator may skip keys
+
+## 5.2.0 (02/08/2017)
+### Public API Change
+* NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explict provide one.
+* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
+* Options::use_direct_writes and Options::use_direct_reads are now ready to use.
+* (Experimental) Two-level indexing that partition the index and creates a 2nd level index on the partitions. The feature can be enabled by setting kTwoLevelIndexSearch as IndexType and configuring index_per_partition.
+
+### New Features
+* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations.
+* Added new function GetApproximateMemTableStats that approximates both number of records and size of memtables.
+* Add Direct I/O mode for SST file I/O
+
+### Bug Fixes
+* RangeSync() should work if ROCKSDB_FALLOCATE_PRESENT is not set
+* Fix wrong results in a data race case in Get()
+* Some fixes related to 2PC.
+* Fix bugs of data corruption in direct I/O
+
+## 5.1.0 (01/13/2017)
+* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
+* Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully.
+* BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env.
+
+### Bug Fixes
+* Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions.
+* When file copying is needed when creating checkpoints or bulk loading files, fsync the file after the file copying.
+
+## 5.0.0 (11/17/2016)
+### Public API Change
+* Options::max_bytes_for_level_multiplier is now a double along with all getters and setters.
+* Support dynamically change `delayed_write_rate` and `max_total_wal_size` options via SetDBOptions().
+* Introduce DB::DeleteRange for optimized deletion of large ranges of contiguous keys.
+* Support dynamically change `delayed_write_rate` option via SetDBOptions().
+* Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default.
+* Remove Tickers::SEQUENCE_NUMBER to avoid confusion if statistics object is shared among RocksDB instance. Alternatively DB::GetLatestSequenceNumber() can be used to get the same value.
+* Options.level0_stop_writes_trigger default value changes from 24 to 32.
+* New compaction filter API: CompactionFilter::FilterV2(). Allows to drop ranges of keys.
+* Removed flashcache support.
+* DB::AddFile() is deprecated and is replaced with DB::IngestExternalFile(). DB::IngestExternalFile() remove all the restrictions that existed for DB::AddFile.
+
+### New Features
+* Add avoid_flush_during_shutdown option, which speeds up DB shutdown by not flushing unpersisted data (i.e. with disableWAL = true). Unpersisted data will be lost. The options is dynamically changeable via SetDBOptions().
+* Add memtable_insert_with_hint_prefix_extractor option. The option is mean to reduce CPU usage for inserting keys into memtable, if keys can be group by prefix and insert for each prefix are sequential or almost sequential. See include/rocksdb/options.h for more details.
+* Add LuaCompactionFilter in utilities. This allows developers to write compaction filters in Lua. To use this feature, LUA_PATH needs to be set to the root directory of Lua.
+* No longer populate "LATEST_BACKUP" file in backup directory, which formerly contained the number of the latest backup. The latest backup can be determined by finding the highest numbered file in the "meta/" subdirectory.
+
+## 4.13.0 (10/18/2016)
+### Public API Change
+* DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference.
+* Added Statistics::getAndResetTickerCount().
+
+### New Features
+* Add DB::SetDBOptions() to dynamic change base_background_compactions and max_background_compactions.
+* Added Iterator::SeekForPrev(). This new API will seek to the last key that less than or equal to the target key.
+
+## 4.12.0 (9/12/2016)
+### Public API Change
+* CancelAllBackgroundWork() flushes all memtables for databases containing writes that have bypassed the WAL (writes issued with WriteOptions::disableWAL=true) before shutting down background threads.
+* Merge options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor into max_compaction_bytes.
+* Remove ImmutableCFOptions.
+* Add a compression type ZSTD, which can work with ZSTD 0.8.0 or up. Still keep ZSTDNotFinal for compatibility reasons.
+
+### New Features
+* Introduce NewClockCache, which is based on CLOCK algorithm with better concurrent performance in some cases. It can be used to replace the default LRU-based block cache and table cache. To use it, RocksDB need to be linked with TBB lib.
+* Change ticker/histogram statistics implementations to accumulate data in thread-local storage, which improves CPU performance by reducing cache coherency costs. Callers of CreateDBStatistics do not need to change anything to use this feature.
+* Block cache mid-point insertion, where index and filter block are inserted into LRU block cache with higher priority. The feature can be enabled by setting BlockBasedTableOptions::cache_index_and_filter_blocks_with_high_priority to true and high_pri_pool_ratio > 0 when creating NewLRUCache.
+
+## 4.11.0 (8/1/2016)
+### Public API Change
+* options.memtable_prefix_bloom_huge_page_tlb_size => memtable_huge_page_size. When it is set, RocksDB will try to allocate memory from huge page for memtable too, rather than just memtable bloom filter.
+
+### New Features
+* A tool to migrate DB after options change. See include/rocksdb/utilities/option_change_migration.h.
+* Add ReadOptions.background_purge_on_iterator_cleanup. If true, we avoid file deletion when destorying iterators.
+
+## 4.10.0 (7/5/2016)
+### Public API Change
+* options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes
+* enum type CompressionType and PerfLevel changes from char to unsigned char. Value of all PerfLevel shift by one.
+* Deprecate options.filter_deletes.
+
+### New Features
+* Add avoid_flush_during_recovery option.
+* Add a read option background_purge_on_iterator_cleanup to avoid deleting files in foreground when destroying iterators. Instead, a job is scheduled in high priority queue and would be executed in a separate background thread.
+* RepairDB support for column families. RepairDB now associates data with non-default column families using information embedded in the SST/WAL files (4.7 or later). For data written by 4.6 or earlier, RepairDB associates it with the default column family.
+* Add options.write_buffer_manager which allows users to control total memtable sizes across multiple DB instances.
+
+## 4.9.0 (6/9/2016)
+### Public API changes
+* Add bottommost_compression option, This option can be used to set a specific compression algorithm for the bottommost level (Last level containing files in the DB).
+* Introduce CompactionJobInfo::compression, This field state the compression algorithm used to generate the output files of the compaction.
+* Deprecate BlockBaseTableOptions.hash_index_allow_collision=false
+* Deprecate options builder (GetOptions()).
+
+### New Features
+* Introduce NewSimCache() in rocksdb/utilities/sim_cache.h. This function creates a block cache that is able to give simulation results (mainly hit rate) of simulating block behavior with a configurable cache size.
+
+## 4.8.0 (5/2/2016)
+### Public API Change
+* Allow preset compression dictionary for improved compression of block-based tables. This is supported for zlib, zstd, and lz4. The compression dictionary's size is configurable via CompressionOptions::max_dict_bytes.
+* Delete deprecated classes for creating backups (BackupableDB) and restoring from backups (RestoreBackupableDB). Now, BackupEngine should be used for creating backups, and BackupEngineReadOnly should be used for restorations. For more details, see https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F
+* Expose estimate of per-level compression ratio via DB property: "rocksdb.compression-ratio-at-levelN".
+* Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
+
+### New Features
+* Add ReadOptions::readahead_size. If non-zero, NewIterator will create a new table reader which performs reads of the given size.
+
+## 4.7.0 (4/8/2016)
+### Public API Change
+* rename options compaction_measure_io_stats to report_bg_io_stats and include flush too.
+* Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.
+
+## 4.6.0 (3/10/2016)
+### Public API Changes
+* Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier.
+* Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signature of Cache::Insert() is updated accordingly.
+* Tickers [NUMBER_DB_NEXT, NUMBER_DB_PREV, NUMBER_DB_NEXT_FOUND, NUMBER_DB_PREV_FOUND, ITER_BYTES_READ] are not updated immediately. The are updated when the Iterator is deleted.
+* Add monotonically increasing counter (DB property "rocksdb.current-super-version-number") that increments upon any change to the LSM tree.
+
+### New Features
+* Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification.
+* Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned"
+
+## 4.5.0 (2/5/2016)
+### Public API Changes
+* Add a new perf context level between kEnableCount and kEnableTime. Level 2 now does not include timers for mutexes.
+* Statistics of mutex operation durations will not be measured by default. If you want to have them enabled, you need to set Statistics::stats_level_ to kAll.
+* DBOptions::delete_scheduler and NewDeleteScheduler() are removed, please use DBOptions::sst_file_manager and NewSstFileManager() instead
+
+### New Features
+* ldb tool now supports operations to non-default column families.
+* Add kPersistedTier to ReadTier. This option allows Get and MultiGet to read only the persited data and skip mem-tables if writes were done with disableWAL = true.
+* Add DBOptions::sst_file_manager. Use NewSstFileManager() in include/rocksdb/sst_file_manager.h to create a SstFileManager that can be used to track the total size of SST files and control the SST files deletion rate.
+
+## 4.4.0 (1/14/2016)
+### Public API Changes
+* Change names in CompactionPri and add a new one.
+* Deprecate options.soft_rate_limit and add options.soft_pending_compaction_bytes_limit.
+* If options.max_write_buffer_number > 3, writes will be slowed down when writing to the last write buffer to delay a full stop.
+* Introduce CompactionJobInfo::compaction_reason, this field include the reason to trigger the compaction.
+* After slow down is triggered, if estimated pending compaction bytes keep increasing, slowdown more.
+* Increase default options.delayed_write_rate to 2MB/s.
+* Added a new parameter --path to ldb tool. --path accepts the name of either MANIFEST, SST or a WAL file. Either --db or --path can be used when calling ldb.
+
+## 4.3.0 (12/8/2015)
+### New Features
+* CompactionFilter has new member function called IgnoreSnapshots which allows CompactionFilter to be called even if there are snapshots later than the key.
+* RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions.
+* Introduce LoadLatestOptions() in rocksdb/utilities/options_util.h. This function can construct the latest DBOptions / ColumnFamilyOptions used by the specified RocksDB intance.
+* Introduce CheckOptionsCompatibility() in rocksdb/utilities/options_util.h. This function checks whether the input set of options is able to open the specified DB successfully.
+
+### Public API Changes
+* When options.db_write_buffer_size triggers, only the column family with the largest column family size will be flushed, not all the column families.
+
+## 4.2.0 (11/9/2015)
+### New Features
+* Introduce CreateLoggerFromOptions(), this function create a Logger for provided DBOptions.
+* Add GetAggregatedIntProperty(), which returns the sum of the GetIntProperty of all the column families.
+* Add MemoryUtil in rocksdb/utilities/memory.h. It currently offers a way to get the memory usage by type from a list rocksdb instances.
+
+### Public API Changes
+* CompactionFilter::Context includes information of Column Family ID
+* The need-compaction hint given by TablePropertiesCollector::NeedCompact() will be persistent and recoverable after DB recovery. This introduces a breaking format change. If you use this experimental feature, including NewCompactOnDeletionCollectorFactory() in the new version, you may not be able to directly downgrade the DB back to version 4.0 or lower.
+* TablePropertiesCollectorFactory::CreateTablePropertiesCollector() now takes an option Context, containing the information of column family ID for the file being written.
+* Remove DefaultCompactionFilterFactory.
+
+
+## 4.1.0 (10/8/2015)
+### New Features
+* Added single delete operation as a more efficient way to delete keys that have not been overwritten.
+* Added experimental AddFile() to DB interface that allow users to add files created by SstFileWriter into an empty Database, see include/rocksdb/sst_file_writer.h and DB::AddFile() for more info.
+* Added support for opening SST files with .ldb suffix which enables opening LevelDB databases.
+* CompactionFilter now supports filtering of merge operands and merge results.
+
+### Public API Changes
+* Added SingleDelete() to the DB interface.
+* Added AddFile() to DB interface.
+* Added SstFileWriter class.
+* CompactionFilter has a new method FilterMergeOperand() that RocksDB applies to every merge operand during compaction to decide whether to filter the operand.
+* We removed CompactionFilterV2 interfaces from include/rocksdb/compaction_filter.h. The functionality was deprecated already in version 3.13.
+
+## 4.0.0 (9/9/2015)
+### New Features
+* Added support for transactions. See include/rocksdb/utilities/transaction.h for more info.
+* DB::GetProperty() now accepts "rocksdb.aggregated-table-properties" and "rocksdb.aggregated-table-properties-at-levelN", in which case it returns aggregated table properties of the target column family, or the aggregated table properties of the specified level N if the "at-level" version is used.
+* Add compression option kZSTDNotFinalCompression for people to experiment ZSTD although its format is not finalized.
+* We removed the need for LATEST_BACKUP file in BackupEngine. We still keep writing it when we create new backups (because of backward compatibility), but we don't read it anymore.
+
+### Public API Changes
+* Removed class Env::RandomRWFile and Env::NewRandomRWFile().
+* Renamed DBOptions.num_subcompactions to DBOptions.max_subcompactions to make the name better match the actual functionality of the option.
+* Added Equal() method to the Comparator interface that can optionally be overwritten in cases where equality comparisons can be done more efficiently than three-way comparisons.
+* Previous 'experimental' OptimisticTransaction class has been replaced by Transaction class.
+
+## 3.13.0 (8/6/2015)
+### New Features
+* RollbackToSavePoint() in WriteBatch/WriteBatchWithIndex
+* Add NewCompactOnDeletionCollectorFactory() in utilities/table_properties_collectors, which allows rocksdb to mark a SST file as need-compaction when it observes at least D deletion entries in any N consecutive entries in that SST file. Note that this feature depends on an experimental NeedCompact() API --- the result of this API will not persist after DB restart.
+* Add DBOptions::delete_scheduler. Use NewDeleteScheduler() in include/rocksdb/delete_scheduler.h to create a DeleteScheduler that can be shared among multiple RocksDB instances to control the file deletion rate of SST files that exist in the first db_path.
+
+### Public API Changes
+* Deprecated WriteOptions::timeout_hint_us. We no longer support write timeout. If you really need this option, talk to us and we might consider returning it.
+* Deprecated purge_redundant_kvs_while_flush option.
+* Removed BackupEngine::NewBackupEngine() and NewReadOnlyBackupEngine() that were deprecated in RocksDB 3.8. Please use BackupEngine::Open() instead.
+* Deprecated Compaction Filter V2. We are not aware of any existing use-cases. If you use this filter, your compile will break with RocksDB 3.13. Please let us know if you use it and we'll put it back in RocksDB 3.14.
+* Env::FileExists now returns a Status instead of a boolean
+* Add statistics::getHistogramString() to print detailed distribution of a histogram metric.
+* Add DBOptions::skip_stats_update_on_db_open. When it is on, DB::Open() will run faster as it skips the random reads required for loading necessary stats from SST files to optimize compaction.
+
+## 3.12.0 (7/2/2015)
+### New Features
+* Added experimental support for optimistic transactions. See include/rocksdb/utilities/optimistic_transaction.h for more info.
+* Added a new way to report QPS from db_bench (check out --report_file and --report_interval_seconds)
+* Added a cache for individual rows. See DBOptions::row_cache for more info.
+* Several new features on EventListener (see include/rocksdb/listener.h):
+ - OnCompationCompleted() now returns per-compaction job statistics, defined in include/rocksdb/compaction_job_stats.h.
+ - Added OnTableFileCreated() and OnTableFileDeleted().
+* Add compaction_options_universal.enable_trivial_move to true, to allow trivial move while performing universal compaction. Trivial move will happen only when all the input files are non overlapping.
+
+### Public API changes
+* EventListener::OnFlushCompleted() now passes FlushJobInfo instead of a list of parameters.
+* DB::GetDbIdentity() is now a const function. If this function is overridden in your application, be sure to also make GetDbIdentity() const to avoid compile error.
+* Move listeners from ColumnFamilyOptions to DBOptions.
+* Add max_write_buffer_number_to_maintain option
+* DB::CompactRange()'s parameter reduce_level is changed to change_level, to allow users to move levels to lower levels if allowed. It can be used to migrate a DB from options.level_compaction_dynamic_level_bytes=false to options.level_compaction_dynamic_level_bytes.true.
+* Change default value for options.compaction_filter_factory and options.compaction_filter_factory_v2 to nullptr instead of DefaultCompactionFilterFactory and DefaultCompactionFilterFactoryV2.
+* If CancelAllBackgroundWork is called without doing a flush after doing loads with WAL disabled, the changes which haven't been flushed before the call to CancelAllBackgroundWork will be lost.
+* WBWIIterator::Entry() now returns WriteEntry instead of `const WriteEntry&`
+* options.hard_rate_limit is deprecated.
+* When options.soft_rate_limit or options.level0_slowdown_writes_trigger is triggered, the way to slow down writes is changed to: write rate to DB is limited to to options.delayed_write_rate.
+* DB::GetApproximateSizes() adds a parameter to allow the estimation to include data in mem table, with default to be not to include. It is now only supported in skip list mem table.
+* DB::CompactRange() now accept CompactRangeOptions instead of multiple parameters. CompactRangeOptions is defined in include/rocksdb/options.h.
+* CompactRange() will now skip bottommost level compaction for level based compaction if there is no compaction filter, bottommost_level_compaction is introduced in CompactRangeOptions to control when it's possible to skip bottommost level compaction. This mean that if you want the compaction to produce a single file you need to set bottommost_level_compaction to BottommostLevelCompaction::kForce.
+* Add Cache.GetPinnedUsage() to get the size of memory occupied by entries that are in use by the system.
+* DB:Open() will fail if the compression specified in Options is not linked with the binary. If you see this failure, recompile RocksDB with compression libraries present on your system. Also, previously our default compression was snappy. This behavior is now changed. Now, the default compression is snappy only if it's available on the system. If it isn't we change the default to kNoCompression.
+* We changed how we account for memory used in block cache. Previously, we only counted the sum of block sizes currently present in block cache. Now, we count the actual memory usage of the blocks. For example, a block of size 4.5KB will use 8KB memory with jemalloc. This might decrease your memory usage and possibly decrease performance. Increase block cache size if you see this happening after an upgrade.
+* Add BackupEngineImpl.options_.max_background_operations to specify the maximum number of operations that may be performed in parallel. Add support for parallelized backup and restore.
+* Add DB::SyncWAL() that does a WAL sync without blocking writers.
+
+## 3.11.0 (5/19/2015)
+### New Features
+* Added a new API Cache::SetCapacity(size_t capacity) to dynamically change the maximum configured capacity of the cache. If the new capacity is less than the existing cache usage, the implementation will try to lower the usage by evicting the necessary number of elements following a strict LRU policy.
+* Added an experimental API for handling flashcache devices (blacklists background threads from caching their reads) -- NewFlashcacheAwareEnv
+* If universal compaction is used and options.num_levels > 1, compact files are tried to be stored in none-L0 with smaller files based on options.target_file_size_base. The limitation of DB size when using universal compaction is greatly mitigated by using more levels. You can set num_levels = 1 to make universal compaction behave as before. If you set num_levels > 1 and want to roll back to a previous version, you need to compact all files to a big file in level 0 (by setting target_file_size_base to be large and CompactRange(<cf_handle>, nullptr, nullptr, true, 0) and reopen the DB with the same version to rewrite the manifest, and then you can open it using previous releases.
+* More information about rocksdb background threads are available in Env::GetThreadList(), including the number of bytes read / written by a compaction job, mem-table size and current number of bytes written by a flush job and many more. Check include/rocksdb/thread_status.h for more detail.
+
+### Public API changes
+* TablePropertiesCollector::AddUserKey() is added to replace TablePropertiesCollector::Add(). AddUserKey() exposes key type, sequence number and file size up to now to users.
+* DBOptions::bytes_per_sync used to apply to both WAL and table files. As of 3.11 it applies only to table files. If you want to use this option to sync WAL in the background, please use wal_bytes_per_sync
+
+## 3.10.0 (3/24/2015)
+### New Features
+* GetThreadStatus() is now able to report detailed thread status, including:
+ - Thread Operation including flush and compaction.
+ - The stage of the current thread operation.
+ - The elapsed time in micros since the current thread operation started.
+ More information can be found in include/rocksdb/thread_status.h. In addition, when running db_bench with --thread_status_per_interval, db_bench will also report thread status periodically.
+* Changed the LRU caching algorithm so that referenced blocks (by iterators) are never evicted. This change made parameter removeScanCountLimit obsolete. Because of that NewLRUCache doesn't take three arguments anymore. table_cache_remove_scan_limit option is also removed
+* By default we now optimize the compilation for the compilation platform (using -march=native). If you want to build portable binary, use 'PORTABLE=1' before the make command.
+* We now allow level-compaction to place files in different paths by
+ specifying them in db_paths along with the target_size.
+ Lower numbered levels will be placed earlier in the db_paths and higher
+ numbered levels will be placed later in the db_paths vector.
+* Potentially big performance improvements if you're using RocksDB with lots of column families (100-1000)
+* Added BlockBasedTableOptions.format_version option, which allows user to specify which version of block based table he wants. As a general guideline, newer versions have more features, but might not be readable by older versions of RocksDB.
+* Added new block based table format (version 2), which you can enable by setting BlockBasedTableOptions.format_version = 2. This format changes how we encode size information in compressed blocks and should help with memory allocations if you're using Zlib or BZip2 compressions.
+* MemEnv (env that stores data in memory) is now available in default library build. You can create it by calling NewMemEnv().
+* Add SliceTransform.SameResultWhenAppended() to help users determine it is safe to apply prefix bloom/hash.
+* Block based table now makes use of prefix bloom filter if it is a full fulter.
+* Block based table remembers whether a whole key or prefix based bloom filter is supported in SST files. Do a sanity check when reading the file with users' configuration.
+* Fixed a bug in ReadOnlyBackupEngine that deleted corrupted backups in some cases, even though the engine was ReadOnly
+* options.level_compaction_dynamic_level_bytes, a feature to allow RocksDB to pick dynamic base of bytes for levels. With this feature turned on, we will automatically adjust max bytes for each level. The goal of this feature is to have lower bound on size amplification. For more details, see comments in options.h.
+* Added an abstract base class WriteBatchBase for write batches
+* Fixed a bug where we start deleting files of a dropped column families even if there are still live references to it
+
+### Public API changes
+* Deprecated skip_log_error_on_recovery and table_cache_remove_scan_count_limit options.
+* Logger method logv with log level parameter is now virtual
+
+### RocksJava
+* Added compression per level API.
+* MemEnv is now available in RocksJava via RocksMemEnv class.
+* lz4 compression is now included in rocksjava static library when running `make rocksdbjavastatic`.
+* Overflowing a size_t when setting rocksdb options now throws an IllegalArgumentException, which removes the necessity for a developer to catch these Exceptions explicitly.
+
+## 3.9.0 (12/8/2014)
+
+### New Features
+* Add rocksdb::GetThreadList(), which in the future will return the current status of all
+ rocksdb-related threads. We will have more code instruments in the following RocksDB
+ releases.
+* Change convert function in rocksdb/utilities/convenience.h to return Status instead of boolean.
+ Also add support for nested options in convert function
+
+### Public API changes
+* New API to create a checkpoint added. Given a directory name, creates a new
+ database which is an image of the existing database.
+* New API LinkFile added to Env. If you implement your own Env class, an
+ implementation of the API LinkFile will have to be provided.
+* MemTableRep takes MemTableAllocator instead of Arena
+
+### Improvements
+* RocksDBLite library now becomes smaller and will be compiled with -fno-exceptions flag.
+
+## 3.8.0 (11/14/2014)
+
+### Public API changes
+* BackupEngine::NewBackupEngine() was deprecated; please use BackupEngine::Open() from now on.
+* BackupableDB/RestoreBackupableDB have new GarbageCollect() methods, which will clean up files from corrupt and obsolete backups.
+* BackupableDB/RestoreBackupableDB have new GetCorruptedBackups() methods which list corrupt backups.
+
+### Cleanup
+* Bunch of code cleanup, some extra warnings turned on (-Wshadow, -Wshorten-64-to-32, -Wnon-virtual-dtor)
+
+### New features
+* CompactFiles and EventListener, although they are still in experimental state
+* Full ColumnFamily support in RocksJava.
+
+## 3.7.0 (11/6/2014)
+### Public API changes
+* Introduce SetOptions() API to allow adjusting a subset of options dynamically online
+* Introduce 4 new convenient functions for converting Options from string: GetColumnFamilyOptionsFromMap(), GetColumnFamilyOptionsFromString(), GetDBOptionsFromMap(), GetDBOptionsFromString()
+* Remove WriteBatchWithIndex.Delete() overloads using SliceParts
+* When opening a DB, if options.max_background_compactions is larger than the existing low pri pool of options.env, it will enlarge it. Similarly, options.max_background_flushes is larger than the existing high pri pool of options.env, it will enlarge it.
+
+## 3.6.0 (10/7/2014)
+### Disk format changes
+* If you're using RocksDB on ARM platforms and you're using default bloom filter, there is a disk format change you need to be aware of. There are three steps you need to do when you convert to new release: 1. turn off filter policy, 2. compact the whole database, 3. turn on filter policy
+
+### Behavior changes
+* We have refactored our system of stalling writes. Any stall-related statistics' meanings are changed. Instead of per-write stall counts, we now count stalls per-epoch, where epochs are periods between flushes and compactions. You'll find more information in our Tuning Perf Guide once we release RocksDB 3.6.
+* When disableDataSync=true, we no longer sync the MANIFEST file.
+* Add identity_as_first_hash property to CuckooTable. SST file needs to be rebuilt to be opened by reader properly.
+
+### Public API changes
+* Change target_file_size_base type to uint64_t from int.
+* Remove allow_thread_local. This feature was proved to be stable, so we are turning it always-on.
+
+## 3.5.0 (9/3/2014)
+### New Features
+* Add include/utilities/write_batch_with_index.h, providing a utility class to query data out of WriteBatch when building it.
+* Move BlockBasedTable related options to BlockBasedTableOptions from Options. Change corresponding JNI interface. Options affected include:
+ no_block_cache, block_cache, block_cache_compressed, block_size, block_size_deviation, block_restart_interval, filter_policy, whole_key_filtering. filter_policy is changed to shared_ptr from a raw pointer.
+* Remove deprecated options: disable_seek_compaction and db_stats_log_interval
+* OptimizeForPointLookup() takes one parameter for block cache size. It now builds hash index, bloom filter, and block cache.
+
+### Public API changes
+* The Prefix Extractor used with V2 compaction filters is now passed user key to SliceTransform::Transform instead of unparsed RocksDB key.
+
+## 3.4.0 (8/18/2014)
+### New Features
+* Support Multiple DB paths in universal style compactions
+* Add feature of storing plain table index and bloom filter in SST file.
+* CompactRange() will never output compacted files to level 0. This used to be the case when all the compaction input files were at level 0.
+* Added iterate_upper_bound to define the extent upto which the forward iterator will return entries. This will prevent iterating over delete markers and overwritten entries for edge cases where you want to break out the iterator anyways. This may improve performance in case there are a large number of delete markers or overwritten entries.
+
+### Public API changes
+* DBOptions.db_paths now is a vector of a DBPath structure which indicates both of path and target size
+* NewPlainTableFactory instead of bunch of parameters now accepts PlainTableOptions, which is defined in include/rocksdb/table.h
+* Moved include/utilities/*.h to include/rocksdb/utilities/*.h
+* Statistics APIs now take uint32_t as type instead of Tickers. Also make two access functions getTickerCount and histogramData const
+* Add DB property rocksdb.estimate-num-keys, estimated number of live keys in DB.
+* Add DB::GetIntProperty(), which returns DB properties that are integer as uint64_t.
+* The Prefix Extractor used with V2 compaction filters is now passed user key to SliceTransform::Transform instead of unparsed RocksDB key.
+
+## 3.3.0 (7/10/2014)
+### New Features
+* Added JSON API prototype.
+* HashLinklist reduces performance outlier caused by skewed bucket by switching data in the bucket from linked list to skip list. Add parameter threshold_use_skiplist in NewHashLinkListRepFactory().
+* RocksDB is now able to reclaim storage space more effectively during the compaction process. This is done by compensating the size of each deletion entry by the 2X average value size, which makes compaction to be triggered by deletion entries more easily.
+* Add TimeOut API to write. Now WriteOptions have a variable called timeout_hint_us. With timeout_hint_us set to non-zero, any write associated with this timeout_hint_us may be aborted when it runs longer than the specified timeout_hint_us, and it is guaranteed that any write completes earlier than the specified time-out will not be aborted due to the time-out condition.
+* Add a rate_limiter option, which controls total throughput of flush and compaction. The throughput is specified in bytes/sec. Flush always has precedence over compaction when available bandwidth is constrained.
+
+### Public API changes
+* Removed NewTotalOrderPlainTableFactory because it is not used and implemented semantically incorrect.
+
+## 3.2.0 (06/20/2014)
+
+### Public API changes
+* We removed seek compaction as a concept from RocksDB because:
+1) It makes more sense for spinning disk workloads, while RocksDB is primarily designed for flash and memory,
+2) It added some complexity to the important code-paths,
+3) None of our internal customers were really using it.
+Because of that, Options::disable_seek_compaction is now obsolete. It is still a parameter in Options, so it does not break the build, but it does not have any effect. We plan to completely remove it at some point, so we ask users to please remove this option from your code base.
+* Add two parameters to NewHashLinkListRepFactory() for logging on too many entries in a hash bucket when flushing.
+* Added new option BlockBasedTableOptions::hash_index_allow_collision. When enabled, prefix hash index for block-based table will not store prefix and allow hash collision, reducing memory consumption.
+
+### New Features
+* PlainTable now supports a new key encoding: for keys of the same prefix, the prefix is only written once. It can be enabled through encoding_type parameter of NewPlainTableFactory()
+* Add AdaptiveTableFactory, which is used to convert from a DB of PlainTable to BlockBasedTabe, or vise versa. It can be created using NewAdaptiveTableFactory()
+
+### Performance Improvements
+* Tailing Iterator re-implemeted with ForwardIterator + Cascading Search Hint , see ~20% throughput improvement.
+
+## 3.1.0 (05/21/2014)
+
+### Public API changes
+* Replaced ColumnFamilyOptions::table_properties_collectors with ColumnFamilyOptions::table_properties_collector_factories
+
+### New Features
+* Hash index for block-based table will be materialized and reconstructed more efficiently. Previously hash index is constructed by scanning the whole table during every table open.
+* FIFO compaction style
+
+## 3.0.0 (05/05/2014)
+
+### Public API changes
+* Added _LEVEL to all InfoLogLevel enums
+* Deprecated ReadOptions.prefix and ReadOptions.prefix_seek. Seek() defaults to prefix-based seek when Options.prefix_extractor is supplied. More detail is documented in https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes
+* MemTableRepFactory::CreateMemTableRep() takes info logger as an extra parameter.
+
+### New Features
+* Column family support
+* Added an option to use different checksum functions in BlockBasedTableOptions
+* Added ApplyToAllCacheEntries() function to Cache
+
+## 2.8.0 (04/04/2014)
+
+* Removed arena.h from public header files.
+* By default, checksums are verified on every read from database
+* Change default value of several options, including: paranoid_checks=true, max_open_files=5000, level0_slowdown_writes_trigger=20, level0_stop_writes_trigger=24, disable_seek_compaction=true, max_background_flushes=1 and allow_mmap_writes=false
+* Added is_manual_compaction to CompactionFilter::Context
+* Added "virtual void WaitForJoin()" in class Env. Default operation is no-op.
+* Removed BackupEngine::DeleteBackupsNewerThan() function
+* Added new option -- verify_checksums_in_compaction
+* Changed Options.prefix_extractor from raw pointer to shared_ptr (take ownership)
+ Changed HashSkipListRepFactory and HashLinkListRepFactory constructor to not take SliceTransform object (use Options.prefix_extractor implicitly)
+* Added Env::GetThreadPoolQueueLen(), which returns the waiting queue length of thread pools
+* Added a command "checkconsistency" in ldb tool, which checks
+ if file system state matches DB state (file existence and file sizes)
+* Separate options related to block based table to a new struct BlockBasedTableOptions.
+* WriteBatch has a new function Count() to return total size in the batch, and Data() now returns a reference instead of a copy
+* Add more counters to perf context.
+* Supports several more DB properties: compaction-pending, background-errors and cur-size-active-mem-table.
+
+### New Features
+* If we find one truncated record at the end of the MANIFEST or WAL files,
+ we will ignore it. We assume that writers of these records were interrupted
+ and that we can safely ignore it.
+* A new SST format "PlainTable" is added, which is optimized for memory-only workloads. It can be created through NewPlainTableFactory() or NewTotalOrderPlainTableFactory().
+* A new mem table implementation hash linked list optimizing for the case that there are only few keys for each prefix, which can be created through NewHashLinkListRepFactory().
+* Merge operator supports a new function PartialMergeMulti() to allow users to do partial merges against multiple operands.
+* Now compaction filter has a V2 interface. It buffers the kv-pairs sharing the same key prefix, process them in batches, and return the batched results back to DB. The new interface uses a new structure CompactionFilterContext for the same purpose as CompactionFilter::Context in V1.
+* Geo-spatial support for locations and radial-search.
+
+## 2.7.0 (01/28/2014)
+
+### Public API changes
+
+* Renamed `StackableDB::GetRawDB()` to `StackableDB::GetBaseDB()`.
+* Renamed `WriteBatch::Data()` `const std::string& Data() const`.
+* Renamed class `TableStats` to `TableProperties`.
+* Deleted class `PrefixHashRepFactory`. Please use `NewHashSkipListRepFactory()` instead.
+* Supported multi-threaded `EnableFileDeletions()` and `DisableFileDeletions()`.
+* Added `DB::GetOptions()`.
+* Added `DB::GetDbIdentity()`.
+
+### New Features
+
+* Added [BackupableDB](https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F)
+* Implemented [TailingIterator](https://github.com/facebook/rocksdb/wiki/Tailing-Iterator), a special type of iterator that
+ doesn't create a snapshot (can be used to read newly inserted data)
+ and is optimized for doing sequential reads.
+* Added property block for table, which allows (1) a table to store
+ its metadata and (2) end user to collect and store properties they
+ are interested in.
+* Enabled caching index and filter block in block cache (turned off by default).
+* Supported error report when doing manual compaction.
+* Supported additional Linux platform flavors and Mac OS.
+* Put with `SliceParts` - Variant of `Put()` that gathers output like `writev(2)`
+* Bug fixes and code refactor for compatibility with upcoming Column
+ Family feature.
+
+### Performance Improvements
+
+* Huge benchmark performance improvements by multiple efforts. For example, increase in readonly QPS from about 530k in 2.6 release to 1.1 million in 2.7 [1]
+* Speeding up a way RocksDB deleted obsolete files - no longer listing the whole directory under a lock -- decrease in p99
+* Use raw pointer instead of shared pointer for statistics: [5b825d](https://github.com/facebook/rocksdb/commit/5b825d6964e26ec3b4bb6faa708ebb1787f1d7bd) -- huge increase in performance -- shared pointers are slow
+* Optimized locking for `Get()` -- [1fdb3f](https://github.com/facebook/rocksdb/commit/1fdb3f7dc60e96394e3e5b69a46ede5d67fb976c) -- 1.5x QPS increase for some workloads
+* Cache speedup - [e8d40c3](https://github.com/facebook/rocksdb/commit/e8d40c31b3cca0c3e1ae9abe9b9003b1288026a9)
+* Implemented autovector, which allocates first N elements on stack. Most of vectors in RocksDB are small. Also, we never want to allocate heap objects while holding a mutex. -- [c01676e4](https://github.com/facebook/rocksdb/commit/c01676e46d3be08c3c140361ef1f5884f47d3b3c)
+* Lots of efforts to move malloc, memcpy and IO outside of locks
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/INSTALL.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/INSTALL.md b/thirdparty/rocksdb/INSTALL.md
new file mode 100644
index 0000000..04f0eb2
--- /dev/null
+++ b/thirdparty/rocksdb/INSTALL.md
@@ -0,0 +1,144 @@
+## Compilation
+
+**Important**: If you plan to run RocksDB in production, don't compile using default
+`make` or `make all`. That will compile RocksDB in debug mode, which is much slower
+than release mode.
+
+RocksDB's library should be able to compile without any dependency installed,
+although we recommend installing some compression libraries (see below).
+We do depend on newer gcc/clang with C++11 support.
+
+There are few options when compiling RocksDB:
+
+* [recommended] `make static_lib` will compile librocksdb.a, RocksDB static library. Compiles static library in release mode.
+
+* `make shared_lib` will compile librocksdb.so, RocksDB shared library. Compiles shared library in release mode.
+
+* `make check` will compile and run all the unit tests. `make check` will compile RocksDB in debug mode.
+
+* `make all` will compile our static library, and all our tools and unit tests. Our tools
+depend on gflags. You will need to have gflags installed to run `make all`. This will compile RocksDB in debug mode. Don't
+use binaries compiled by `make all` in production.
+
+* By default the binary we produce is optimized for the platform you're compiling on
+(`-march=native` or the equivalent). SSE4.2 will thus be enabled automatically if your
+CPU supports it. To print a warning if your CPU does not support SSE4.2, build with
+`USE_SSE=1 make static_lib` or, if using CMake, `cmake -DFORCE_SSE42=ON`. If you want
+to build a portable binary, add `PORTABLE=1` before your make commands, like this:
+`PORTABLE=1 make static_lib`.
+
+## Dependencies
+
+* You can link RocksDB with following compression libraries:
+ - [zlib](http://www.zlib.net/) - a library for data compression.
+ - [bzip2](http://www.bzip.org/) - a library for data compression.
+ - [lz4](https://github.com/lz4/lz4) - a library for extremely fast data compression.
+ - [snappy](http://google.github.io/snappy/) - a library for fast
+ data compression.
+ - [zstandard](http://www.zstd.net) - Fast real-time compression
+ algorithm.
+
+* All our tools depend on:
+ - [gflags](https://gflags.github.io/gflags/) - a library that handles
+ command line flags processing. You can compile rocksdb library even
+ if you don't have gflags installed.
+
+## Supported platforms
+
+* **Linux - Ubuntu**
+ * Upgrade your gcc to version at least 4.8 to get C++11 support.
+ * Install gflags. First, try: `sudo apt-get install libgflags-dev`
+ If this doesn't work and you're using Ubuntu, here's a nice tutorial:
+ (http://askubuntu.com/questions/312173/installing-gflags-12-04)
+ * Install snappy. This is usually as easy as:
+ `sudo apt-get install libsnappy-dev`.
+ * Install zlib. Try: `sudo apt-get install zlib1g-dev`.
+ * Install bzip2: `sudo apt-get install libbz2-dev`.
+ * Install lz4: `sudo apt-get install liblz4-dev`.
+ * Install zstandard: `sudo apt-get install libzstd-dev`.
+
+* **Linux - CentOS / RHEL**
+ * Upgrade your gcc to version at least 4.8 to get C++11 support:
+ `yum install gcc48-c++`
+ * Install gflags:
+
+ git clone https://github.com/gflags/gflags.git
+ cd gflags
+ git checkout v2.0
+ ./configure && make && sudo make install
+
+ **Notice**: Once installed, please add the include path for gflags to your `CPATH` environment variable and the
+ lib path to `LIBRARY_PATH`. If installed with default settings, the include path will be `/usr/local/include`
+ and the lib path will be `/usr/local/lib`.
+
+ * Install snappy:
+
+ sudo yum install snappy snappy-devel
+
+ * Install zlib:
+
+ sudo yum install zlib zlib-devel
+
+ * Install bzip2:
+
+ sudo yum install bzip2 bzip2-devel
+
+ * Install lz4:
+
+ sudo yum install lz4-devel
+
+ * Install ASAN (optional for debugging):
+
+ sudo yum install libasan
+
+ * Install zstandard:
+
+ wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
+ mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
+ tar zxvf zstd-1.1.3.tar.gz
+ cd zstd-1.1.3
+ make && sudo make install
+
+* **OS X**:
+ * Install latest C++ compiler that supports C++ 11:
+ * Update XCode: run `xcode-select --install` (or install it from XCode App's settting).
+ * Install via [homebrew](http://brew.sh/).
+ * If you're first time developer in MacOS, you still need to run: `xcode-select --install` in your command line.
+ * run `brew tap homebrew/versions; brew install gcc48 --use-llvm` to install gcc 4.8 (or higher).
+ * run `brew install rocksdb`
+
+* **iOS**:
+ * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.
+
+* **Windows**:
+ * For building with MS Visual Studio 13 you will need Update 4 installed.
+ * Read and follow the instructions at CMakeLists.txt
+ * Or install via [vcpkg](https://github.com/microsoft/vcpkg)
+ * run `vcpkg install rocksdb`
+
+* **AIX 6.1**
+ * Install AIX Toolbox rpms with gcc
+ * Use these environment variables:
+
+ export PORTABLE=1
+ export CC=gcc
+ export AR="ar -X64"
+ export EXTRA_ARFLAGS=-X64
+ export EXTRA_CFLAGS=-maix64
+ export EXTRA_CXXFLAGS=-maix64
+ export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
+ export LIBPATH=/opt/freeware/lib
+ export JAVA_HOME=/usr/java8_64
+ export PATH=/opt/freeware/bin:$PATH
+
+* **Solaris Sparc**
+ * Install GCC 4.8.2 and higher.
+ * Use these environment variables:
+
+ export CC=gcc
+ export EXTRA_CFLAGS=-m64
+ export EXTRA_CXXFLAGS=-m64
+ export EXTRA_LDFLAGS=-m64
+ export PORTABLE=1
+ export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/LANGUAGE-BINDINGS.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/LANGUAGE-BINDINGS.md b/thirdparty/rocksdb/LANGUAGE-BINDINGS.md
new file mode 100644
index 0000000..d28035b
--- /dev/null
+++ b/thirdparty/rocksdb/LANGUAGE-BINDINGS.md
@@ -0,0 +1,13 @@
+This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it.
+
+* Java - https://github.com/facebook/rocksdb/tree/master/java
+* Python - http://pyrocksdb.readthedocs.org/en/latest/
+* Perl - https://metacpan.org/pod/RocksDB
+* Node.js - https://npmjs.org/package/rocksdb
+* Go - https://github.com/tecbot/gorocksdb
+* Ruby - http://rubygems.org/gems/rocksdb-ruby
+* Haskell - https://hackage.haskell.org/package/rocksdb-haskell
+* PHP - https://github.com/Photonios/rocksdb-php
+* C# - https://github.com/warrenfalk/rocksdb-sharp
+* Rust - https://github.com/spacejam/rust-rocksdb
+* D programming language - https://github.com/b1naryth1ef/rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/LICENSE.Apache
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/LICENSE.Apache b/thirdparty/rocksdb/LICENSE.Apache
new file mode 100644
index 0000000..d645695
--- /dev/null
+++ b/thirdparty/rocksdb/LICENSE.Apache
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/LICENSE.leveldb
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/LICENSE.leveldb b/thirdparty/rocksdb/LICENSE.leveldb
new file mode 100644
index 0000000..7108b0b
--- /dev/null
+++ b/thirdparty/rocksdb/LICENSE.leveldb
@@ -0,0 +1,29 @@
+This contains code that is from LevelDB, and that code is under the following license:
+
+Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/README.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/README.md b/thirdparty/rocksdb/README.md
new file mode 100644
index 0000000..550c352
--- /dev/null
+++ b/thirdparty/rocksdb/README.md
@@ -0,0 +1,27 @@
+## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage
+
+[](https://travis-ci.org/facebook/rocksdb)
+[](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
+
+
+RocksDB is developed and maintained by Facebook Database Engineering Team.
+It is built on earlier work on LevelDB by Sanjay Ghemawat (sanjay@google.com)
+and Jeff Dean (jeff@google.com)
+
+This code is a library that forms the core building block for a fast
+key value server, especially suited for storing data on flash drives.
+It has a Log-Structured-Merge-Database (LSM) design with flexible tradeoffs
+between Write-Amplification-Factor (WAF), Read-Amplification-Factor (RAF)
+and Space-Amplification-Factor (SAF). It has multi-threaded compactions,
+making it specially suitable for storing multiple terabytes of data in a
+single database.
+
+Start with example usage here: https://github.com/facebook/rocksdb/tree/master/examples
+
+See the [github wiki](https://github.com/facebook/rocksdb/wiki) for more explanation.
+
+The public interface is in `include/`. Callers should not include or
+rely on the details of any other header files in this package. Those
+internal APIs may be changed without warning.
+
+Design discussions are conducted in https://www.facebook.com/groups/rocksdb.dev/
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/ROCKSDB_LITE.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/ROCKSDB_LITE.md b/thirdparty/rocksdb/ROCKSDB_LITE.md
new file mode 100644
index 0000000..41cfbec
--- /dev/null
+++ b/thirdparty/rocksdb/ROCKSDB_LITE.md
@@ -0,0 +1,21 @@
+# RocksDBLite
+
+RocksDBLite is a project focused on mobile use cases, which don't need a lot of fancy things we've built for server workloads and they are very sensitive to binary size. For that reason, we added a compile flag ROCKSDB_LITE that comments out a lot of the nonessential code and keeps the binary lean.
+
+Some examples of the features disabled by ROCKSDB_LITE:
+* compiled-in support for LDB tool
+* No backupable DB
+* No support for replication (which we provide in form of TrasactionalIterator)
+* No advanced monitoring tools
+* No special-purpose memtables that are highly optimized for specific use cases
+* No Transactions
+
+When adding a new big feature to RocksDB, please add ROCKSDB_LITE compile guard if:
+* Nobody from mobile really needs your feature,
+* Your feature is adding a lot of weight to the binary.
+
+Don't add ROCKSDB_LITE compile guard if:
+* It would introduce a lot of code complexity. Compile guards make code harder to read. It's a trade-off.
+* Your feature is not adding a lot of weight.
+
+If unsure, ask. :)
[46/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/db_impl.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/db_impl.h b/thirdparty/leveldb-1.18/db/db_impl.h
deleted file mode 100755
index cfc9981..0000000
--- a/thirdparty/leveldb-1.18/db/db_impl.h
+++ /dev/null
@@ -1,211 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_
-#define STORAGE_LEVELDB_DB_DB_IMPL_H_
-
-#include <deque>
-#include <set>
-#include "db/dbformat.h"
-#include "db/log_writer.h"
-#include "db/snapshot.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "port/port.h"
-#include "port/thread_annotations.h"
-
-namespace leveldb {
-
-class MemTable;
-class TableCache;
-class Version;
-class VersionEdit;
-class VersionSet;
-
-class DBImpl : public DB {
- public:
- DBImpl(const Options& options, const std::string& dbname);
- virtual ~DBImpl();
-
- // Implementations of the DB interface
- virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value);
- virtual Status Delete(const WriteOptions&, const Slice& key);
- virtual Status Write(const WriteOptions& options, WriteBatch* updates);
- virtual Status Get(const ReadOptions& options,
- const Slice& key,
- std::string* value);
- virtual Iterator* NewIterator(const ReadOptions&);
- virtual const Snapshot* GetSnapshot();
- virtual void ReleaseSnapshot(const Snapshot* snapshot);
- virtual bool GetProperty(const Slice& property, std::string* value);
- virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
- virtual void CompactRange(const Slice* begin, const Slice* end);
-
- // Extra methods (for testing) that are not in the public DB interface
-
- // Compact any files in the named level that overlap [*begin,*end]
- void TEST_CompactRange(int level, const Slice* begin, const Slice* end);
-
- // Force current memtable contents to be compacted.
- Status TEST_CompactMemTable();
-
- // Return an internal iterator over the current state of the database.
- // The keys of this iterator are internal keys (see format.h).
- // The returned iterator should be deleted when no longer needed.
- Iterator* TEST_NewInternalIterator();
-
- // Return the maximum overlapping data (in bytes) at next level for any
- // file at a level >= 1.
- int64_t TEST_MaxNextLevelOverlappingBytes();
-
- // Record a sample of bytes read at the specified internal key.
- // Samples are taken approximately once every config::kReadBytesPeriod
- // bytes.
- void RecordReadSample(Slice key);
-
- private:
- friend class DB;
- struct CompactionState;
- struct Writer;
-
- Iterator* NewInternalIterator(const ReadOptions&,
- SequenceNumber* latest_snapshot,
- uint32_t* seed);
-
- Status NewDB();
-
- // Recover the descriptor from persistent storage. May do a significant
- // amount of work to recover recently logged updates. Any changes to
- // be made to the descriptor are added to *edit.
- Status Recover(VersionEdit* edit) EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
- void MaybeIgnoreError(Status* s) const;
-
- // Delete any unneeded files and stale in-memory entries.
- void DeleteObsoleteFiles();
-
- // Compact the in-memory write buffer to disk. Switches to a new
- // log-file/memtable and writes a new descriptor iff successful.
- // Errors are recorded in bg_error_.
- void CompactMemTable() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
- Status RecoverLogFile(uint64_t log_number,
- VersionEdit* edit,
- SequenceNumber* max_sequence)
- EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
- Status WriteLevel0Table(MemTable* mem, VersionEdit* edit, Version* base)
- EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
- Status MakeRoomForWrite(bool force /* compact even if there is room? */)
- EXCLUSIVE_LOCKS_REQUIRED(mutex_);
- WriteBatch* BuildBatchGroup(Writer** last_writer);
-
- void RecordBackgroundError(const Status& s);
-
- void MaybeScheduleCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
- static void BGWork(void* db);
- void BackgroundCall();
- void BackgroundCompaction() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
- void CleanupCompaction(CompactionState* compact)
- EXCLUSIVE_LOCKS_REQUIRED(mutex_);
- Status DoCompactionWork(CompactionState* compact)
- EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
- Status OpenCompactionOutputFile(CompactionState* compact);
- Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input);
- Status InstallCompactionResults(CompactionState* compact)
- EXCLUSIVE_LOCKS_REQUIRED(mutex_);
-
- // Constant after construction
- Env* const env_;
- const InternalKeyComparator internal_comparator_;
- const InternalFilterPolicy internal_filter_policy_;
- const Options options_; // options_.comparator == &internal_comparator_
- bool owns_info_log_;
- bool owns_cache_;
- const std::string dbname_;
-
- // table_cache_ provides its own synchronization
- TableCache* table_cache_;
-
- // Lock over the persistent DB state. Non-NULL iff successfully acquired.
- FileLock* db_lock_;
-
- // State below is protected by mutex_
- port::Mutex mutex_;
- port::AtomicPointer shutting_down_;
- port::CondVar bg_cv_; // Signalled when background work finishes
- MemTable* mem_;
- MemTable* imm_; // Memtable being compacted
- port::AtomicPointer has_imm_; // So bg thread can detect non-NULL imm_
- WritableFile* logfile_;
- uint64_t logfile_number_;
- log::Writer* log_;
- uint32_t seed_; // For sampling.
-
- // Queue of writers.
- std::deque<Writer*> writers_;
- WriteBatch* tmp_batch_;
-
- SnapshotList snapshots_;
-
- // Set of table files to protect from deletion because they are
- // part of ongoing compactions.
- std::set<uint64_t> pending_outputs_;
-
- // Has a background compaction been scheduled or is running?
- bool bg_compaction_scheduled_;
-
- // Information for a manual compaction
- struct ManualCompaction {
- int level;
- bool done;
- const InternalKey* begin; // NULL means beginning of key range
- const InternalKey* end; // NULL means end of key range
- InternalKey tmp_storage; // Used to keep track of compaction progress
- };
- ManualCompaction* manual_compaction_;
-
- VersionSet* versions_;
-
- // Have we encountered a background error in paranoid mode?
- Status bg_error_;
-
- // Per level compaction stats. stats_[level] stores the stats for
- // compactions that produced data for the specified "level".
- struct CompactionStats {
- int64_t micros;
- int64_t bytes_read;
- int64_t bytes_written;
-
- CompactionStats() : micros(0), bytes_read(0), bytes_written(0) { }
-
- void Add(const CompactionStats& c) {
- this->micros += c.micros;
- this->bytes_read += c.bytes_read;
- this->bytes_written += c.bytes_written;
- }
- };
- CompactionStats stats_[config::kNumLevels];
-
- // No copying allowed
- DBImpl(const DBImpl&);
- void operator=(const DBImpl&);
-
- const Comparator* user_comparator() const {
- return internal_comparator_.user_comparator();
- }
-};
-
-// Sanitize db options. The caller should delete result.info_log if
-// it is not equal to src.info_log.
-extern Options SanitizeOptions(const std::string& db,
- const InternalKeyComparator* icmp,
- const InternalFilterPolicy* ipolicy,
- const Options& src);
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_DB_IMPL_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/db_iter.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/db_iter.cc b/thirdparty/leveldb-1.18/db/db_iter.cc
deleted file mode 100755
index bd29f30..0000000
--- a/thirdparty/leveldb-1.18/db/db_iter.cc
+++ /dev/null
@@ -1,317 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/db_iter.h"
-
-#include "db/filename.h"
-#include "db/db_impl.h"
-#include "db/dbformat.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
-#include "port/port.h"
-#include "util/logging.h"
-#include "util/mutexlock.h"
-#include "util/random.h"
-
-namespace leveldb {
-
-#if 0
-static void DumpInternalIter(Iterator* iter) {
- for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
- ParsedInternalKey k;
- if (!ParseInternalKey(iter->key(), &k)) {
- fprintf(stderr, "Corrupt '%s'\n", EscapeString(iter->key()).c_str());
- } else {
- fprintf(stderr, "@ '%s'\n", k.DebugString().c_str());
- }
- }
-}
-#endif
-
-namespace {
-
-// Memtables and sstables that make the DB representation contain
-// (userkey,seq,type) => uservalue entries. DBIter
-// combines multiple entries for the same userkey found in the DB
-// representation into a single entry while accounting for sequence
-// numbers, deletion markers, overwrites, etc.
-class DBIter: public Iterator {
- public:
- // Which direction is the iterator currently moving?
- // (1) When moving forward, the internal iterator is positioned at
- // the exact entry that yields this->key(), this->value()
- // (2) When moving backwards, the internal iterator is positioned
- // just before all entries whose user key == this->key().
- enum Direction {
- kForward,
- kReverse
- };
-
- DBIter(DBImpl* db, const Comparator* cmp, Iterator* iter, SequenceNumber s,
- uint32_t seed)
- : db_(db),
- user_comparator_(cmp),
- iter_(iter),
- sequence_(s),
- direction_(kForward),
- valid_(false),
- rnd_(seed),
- bytes_counter_(RandomPeriod()) {
- }
- virtual ~DBIter() {
- delete iter_;
- }
- virtual bool Valid() const { return valid_; }
- virtual Slice key() const {
- assert(valid_);
- return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_;
- }
- virtual Slice value() const {
- assert(valid_);
- return (direction_ == kForward) ? iter_->value() : saved_value_;
- }
- virtual Status status() const {
- if (status_.ok()) {
- return iter_->status();
- } else {
- return status_;
- }
- }
-
- virtual void Next();
- virtual void Prev();
- virtual void Seek(const Slice& target);
- virtual void SeekToFirst();
- virtual void SeekToLast();
-
- private:
- void FindNextUserEntry(bool skipping, std::string* skip);
- void FindPrevUserEntry();
- bool ParseKey(ParsedInternalKey* key);
-
- inline void SaveKey(const Slice& k, std::string* dst) {
- dst->assign(k.data(), k.size());
- }
-
- inline void ClearSavedValue() {
- if (saved_value_.capacity() > 1048576) {
- std::string empty;
- swap(empty, saved_value_);
- } else {
- saved_value_.clear();
- }
- }
-
- // Pick next gap with average value of config::kReadBytesPeriod.
- int64_t RandomPeriod() {
- return rnd_.Uniform(2*config::kReadBytesPeriod);
- }
-
- DBImpl* db_;
- const Comparator* const user_comparator_;
- Iterator* const iter_;
- SequenceNumber const sequence_;
-
- Status status_;
- std::string saved_key_; // == current key when direction_==kReverse
- std::string saved_value_; // == current raw value when direction_==kReverse
- Direction direction_;
- bool valid_;
-
- Random rnd_;
- int64_t bytes_counter_;
-
- // No copying allowed
- DBIter(const DBIter&);
- void operator=(const DBIter&);
-};
-
-inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
- Slice k = iter_->key();
- int64_t n = k.size() + iter_->value().size();
- bytes_counter_ -= n;
- while (bytes_counter_ < 0) {
- bytes_counter_ += RandomPeriod();
- db_->RecordReadSample(k);
- }
- if (!ParseInternalKey(k, ikey)) {
- status_ = Status::Corruption("corrupted internal key in DBIter");
- return false;
- } else {
- return true;
- }
-}
-
-void DBIter::Next() {
- assert(valid_);
-
- if (direction_ == kReverse) { // Switch directions?
- direction_ = kForward;
- // iter_ is pointing just before the entries for this->key(),
- // so advance into the range of entries for this->key() and then
- // use the normal skipping code below.
- if (!iter_->Valid()) {
- iter_->SeekToFirst();
- } else {
- iter_->Next();
- }
- if (!iter_->Valid()) {
- valid_ = false;
- saved_key_.clear();
- return;
- }
- // saved_key_ already contains the key to skip past.
- } else {
- // Store in saved_key_ the current key so we skip it below.
- SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
- }
-
- FindNextUserEntry(true, &saved_key_);
-}
-
-void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
- // Loop until we hit an acceptable entry to yield
- assert(iter_->Valid());
- assert(direction_ == kForward);
- do {
- ParsedInternalKey ikey;
- if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
- switch (ikey.type) {
- case kTypeDeletion:
- // Arrange to skip all upcoming entries for this key since
- // they are hidden by this deletion.
- SaveKey(ikey.user_key, skip);
- skipping = true;
- break;
- case kTypeValue:
- if (skipping &&
- user_comparator_->Compare(ikey.user_key, *skip) <= 0) {
- // Entry hidden
- } else {
- valid_ = true;
- saved_key_.clear();
- return;
- }
- break;
- }
- }
- iter_->Next();
- } while (iter_->Valid());
- saved_key_.clear();
- valid_ = false;
-}
-
-void DBIter::Prev() {
- assert(valid_);
-
- if (direction_ == kForward) { // Switch directions?
- // iter_ is pointing at the current entry. Scan backwards until
- // the key changes so we can use the normal reverse scanning code.
- assert(iter_->Valid()); // Otherwise valid_ would have been false
- SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
- while (true) {
- iter_->Prev();
- if (!iter_->Valid()) {
- valid_ = false;
- saved_key_.clear();
- ClearSavedValue();
- return;
- }
- if (user_comparator_->Compare(ExtractUserKey(iter_->key()),
- saved_key_) < 0) {
- break;
- }
- }
- direction_ = kReverse;
- }
-
- FindPrevUserEntry();
-}
-
-void DBIter::FindPrevUserEntry() {
- assert(direction_ == kReverse);
-
- ValueType value_type = kTypeDeletion;
- if (iter_->Valid()) {
- do {
- ParsedInternalKey ikey;
- if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
- if ((value_type != kTypeDeletion) &&
- user_comparator_->Compare(ikey.user_key, saved_key_) < 0) {
- // We encountered a non-deleted value in entries for previous keys,
- break;
- }
- value_type = ikey.type;
- if (value_type == kTypeDeletion) {
- saved_key_.clear();
- ClearSavedValue();
- } else {
- Slice raw_value = iter_->value();
- if (saved_value_.capacity() > raw_value.size() + 1048576) {
- std::string empty;
- swap(empty, saved_value_);
- }
- SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
- saved_value_.assign(raw_value.data(), raw_value.size());
- }
- }
- iter_->Prev();
- } while (iter_->Valid());
- }
-
- if (value_type == kTypeDeletion) {
- // End
- valid_ = false;
- saved_key_.clear();
- ClearSavedValue();
- direction_ = kForward;
- } else {
- valid_ = true;
- }
-}
-
-void DBIter::Seek(const Slice& target) {
- direction_ = kForward;
- ClearSavedValue();
- saved_key_.clear();
- AppendInternalKey(
- &saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek));
- iter_->Seek(saved_key_);
- if (iter_->Valid()) {
- FindNextUserEntry(false, &saved_key_ /* temporary storage */);
- } else {
- valid_ = false;
- }
-}
-
-void DBIter::SeekToFirst() {
- direction_ = kForward;
- ClearSavedValue();
- iter_->SeekToFirst();
- if (iter_->Valid()) {
- FindNextUserEntry(false, &saved_key_ /* temporary storage */);
- } else {
- valid_ = false;
- }
-}
-
-void DBIter::SeekToLast() {
- direction_ = kReverse;
- ClearSavedValue();
- iter_->SeekToLast();
- FindPrevUserEntry();
-}
-
-} // anonymous namespace
-
-Iterator* NewDBIterator(
- DBImpl* db,
- const Comparator* user_key_comparator,
- Iterator* internal_iter,
- SequenceNumber sequence,
- uint32_t seed) {
- return new DBIter(db, user_key_comparator, internal_iter, sequence, seed);
-}
-
-} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/db_iter.h
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/db_iter.h b/thirdparty/leveldb-1.18/db/db_iter.h
deleted file mode 100755
index 04927e9..0000000
--- a/thirdparty/leveldb-1.18/db/db_iter.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_DB_ITER_H_
-#define STORAGE_LEVELDB_DB_DB_ITER_H_
-
-#include <stdint.h>
-#include "leveldb/db.h"
-#include "db/dbformat.h"
-
-namespace leveldb {
-
-class DBImpl;
-
-// Return a new iterator that converts internal keys (yielded by
-// "*internal_iter") that were live at the specified "sequence" number
-// into appropriate user keys.
-extern Iterator* NewDBIterator(
- DBImpl* db,
- const Comparator* user_key_comparator,
- Iterator* internal_iter,
- SequenceNumber sequence,
- uint32_t seed);
-
-} // namespace leveldb
-
-#endif // STORAGE_LEVELDB_DB_DB_ITER_H_
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/db_test.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/db_test.cc b/thirdparty/leveldb-1.18/db/db_test.cc
deleted file mode 100755
index 02e7d90..0000000
--- a/thirdparty/leveldb-1.18/db/db_test.cc
+++ /dev/null
@@ -1,2129 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "leveldb/db.h"
-#include "leveldb/filter_policy.h"
-#include "db/db_impl.h"
-#include "db/filename.h"
-#include "db/version_set.h"
-#include "db/write_batch_internal.h"
-#include "leveldb/cache.h"
-#include "leveldb/env.h"
-#include "leveldb/table.h"
-#include "util/hash.h"
-#include "util/logging.h"
-#include "util/mutexlock.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace leveldb {
-
-static std::string RandomString(Random* rnd, int len) {
- std::string r;
- test::RandomString(rnd, len, &r);
- return r;
-}
-
-namespace {
-class AtomicCounter {
- private:
- port::Mutex mu_;
- int count_;
- public:
- AtomicCounter() : count_(0) { }
- void Increment() {
- IncrementBy(1);
- }
- void IncrementBy(int count) {
- MutexLock l(&mu_);
- count_ += count;
- }
- int Read() {
- MutexLock l(&mu_);
- return count_;
- }
- void Reset() {
- MutexLock l(&mu_);
- count_ = 0;
- }
-};
-
-void DelayMilliseconds(int millis) {
- Env::Default()->SleepForMicroseconds(millis * 1000);
-}
-}
-
-// Special Env used to delay background operations
-class SpecialEnv : public EnvWrapper {
- public:
- // sstable/log Sync() calls are blocked while this pointer is non-NULL.
- port::AtomicPointer delay_data_sync_;
-
- // sstable/log Sync() calls return an error.
- port::AtomicPointer data_sync_error_;
-
- // Simulate no-space errors while this pointer is non-NULL.
- port::AtomicPointer no_space_;
-
- // Simulate non-writable file system while this pointer is non-NULL
- port::AtomicPointer non_writable_;
-
- // Force sync of manifest files to fail while this pointer is non-NULL
- port::AtomicPointer manifest_sync_error_;
-
- // Force write to manifest files to fail while this pointer is non-NULL
- port::AtomicPointer manifest_write_error_;
-
- bool count_random_reads_;
- AtomicCounter random_read_counter_;
-
- explicit SpecialEnv(Env* base) : EnvWrapper(base) {
- delay_data_sync_.Release_Store(NULL);
- data_sync_error_.Release_Store(NULL);
- no_space_.Release_Store(NULL);
- non_writable_.Release_Store(NULL);
- count_random_reads_ = false;
- manifest_sync_error_.Release_Store(NULL);
- manifest_write_error_.Release_Store(NULL);
- }
-
- Status NewWritableFile(const std::string& f, WritableFile** r) {
- class DataFile : public WritableFile {
- private:
- SpecialEnv* env_;
- WritableFile* base_;
-
- public:
- DataFile(SpecialEnv* env, WritableFile* base)
- : env_(env),
- base_(base) {
- }
- ~DataFile() { delete base_; }
- Status Append(const Slice& data) {
- if (env_->no_space_.Acquire_Load() != NULL) {
- // Drop writes on the floor
- return Status::OK();
- } else {
- return base_->Append(data);
- }
- }
- Status Close() { return base_->Close(); }
- Status Flush() { return base_->Flush(); }
- Status Sync() {
- if (env_->data_sync_error_.Acquire_Load() != NULL) {
- return Status::IOError("simulated data sync error");
- }
- while (env_->delay_data_sync_.Acquire_Load() != NULL) {
- DelayMilliseconds(100);
- }
- return base_->Sync();
- }
- };
- class ManifestFile : public WritableFile {
- private:
- SpecialEnv* env_;
- WritableFile* base_;
- public:
- ManifestFile(SpecialEnv* env, WritableFile* b) : env_(env), base_(b) { }
- ~ManifestFile() { delete base_; }
- Status Append(const Slice& data) {
- if (env_->manifest_write_error_.Acquire_Load() != NULL) {
- return Status::IOError("simulated writer error");
- } else {
- return base_->Append(data);
- }
- }
- Status Close() { return base_->Close(); }
- Status Flush() { return base_->Flush(); }
- Status Sync() {
- if (env_->manifest_sync_error_.Acquire_Load() != NULL) {
- return Status::IOError("simulated sync error");
- } else {
- return base_->Sync();
- }
- }
- };
-
- if (non_writable_.Acquire_Load() != NULL) {
- return Status::IOError("simulated write error");
- }
-
- Status s = target()->NewWritableFile(f, r);
- if (s.ok()) {
- if (strstr(f.c_str(), ".ldb") != NULL ||
- strstr(f.c_str(), ".log") != NULL) {
- *r = new DataFile(this, *r);
- } else if (strstr(f.c_str(), "MANIFEST") != NULL) {
- *r = new ManifestFile(this, *r);
- }
- }
- return s;
- }
-
- Status NewRandomAccessFile(const std::string& f, RandomAccessFile** r) {
- class CountingFile : public RandomAccessFile {
- private:
- RandomAccessFile* target_;
- AtomicCounter* counter_;
- public:
- CountingFile(RandomAccessFile* target, AtomicCounter* counter)
- : target_(target), counter_(counter) {
- }
- virtual ~CountingFile() { delete target_; }
- virtual Status Read(uint64_t offset, size_t n, Slice* result,
- char* scratch) const {
- counter_->Increment();
- return target_->Read(offset, n, result, scratch);
- }
- };
-
- Status s = target()->NewRandomAccessFile(f, r);
- if (s.ok() && count_random_reads_) {
- *r = new CountingFile(*r, &random_read_counter_);
- }
- return s;
- }
-};
-
-class DBTest {
- private:
- const FilterPolicy* filter_policy_;
-
- // Sequence of option configurations to try
- enum OptionConfig {
- kDefault,
- kFilter,
- kUncompressed,
- kEnd
- };
- int option_config_;
-
- public:
- std::string dbname_;
- SpecialEnv* env_;
- DB* db_;
-
- Options last_options_;
-
- DBTest() : option_config_(kDefault),
- env_(new SpecialEnv(Env::Default())) {
- filter_policy_ = NewBloomFilterPolicy(10);
- dbname_ = test::TmpDir() + "/db_test";
- DestroyDB(dbname_, Options());
- db_ = NULL;
- Reopen();
- }
-
- ~DBTest() {
- delete db_;
- DestroyDB(dbname_, Options());
- delete env_;
- delete filter_policy_;
- }
-
- // Switch to a fresh database with the next option configuration to
- // test. Return false if there are no more configurations to test.
- bool ChangeOptions() {
- option_config_++;
- if (option_config_ >= kEnd) {
- return false;
- } else {
- DestroyAndReopen();
- return true;
- }
- }
-
- // Return the current option configuration.
- Options CurrentOptions() {
- Options options;
- switch (option_config_) {
- case kFilter:
- options.filter_policy = filter_policy_;
- break;
- case kUncompressed:
- options.compression = kNoCompression;
- break;
- default:
- break;
- }
- return options;
- }
-
- DBImpl* dbfull() {
- return reinterpret_cast<DBImpl*>(db_);
- }
-
- void Reopen(Options* options = NULL) {
- ASSERT_OK(TryReopen(options));
- }
-
- void Close() {
- delete db_;
- db_ = NULL;
- }
-
- void DestroyAndReopen(Options* options = NULL) {
- delete db_;
- db_ = NULL;
- DestroyDB(dbname_, Options());
- ASSERT_OK(TryReopen(options));
- }
-
- Status TryReopen(Options* options) {
- delete db_;
- db_ = NULL;
- Options opts;
- if (options != NULL) {
- opts = *options;
- } else {
- opts = CurrentOptions();
- opts.create_if_missing = true;
- }
- last_options_ = opts;
-
- return DB::Open(opts, dbname_, &db_);
- }
-
- Status Put(const std::string& k, const std::string& v) {
- return db_->Put(WriteOptions(), k, v);
- }
-
- Status Delete(const std::string& k) {
- return db_->Delete(WriteOptions(), k);
- }
-
- std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
- ReadOptions options;
- options.snapshot = snapshot;
- std::string result;
- Status s = db_->Get(options, k, &result);
- if (s.IsNotFound()) {
- result = "NOT_FOUND";
- } else if (!s.ok()) {
- result = s.ToString();
- }
- return result;
- }
-
- // Return a string that contains all key,value pairs in order,
- // formatted like "(k1->v1)(k2->v2)".
- std::string Contents() {
- std::vector<std::string> forward;
- std::string result;
- Iterator* iter = db_->NewIterator(ReadOptions());
- for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
- std::string s = IterStatus(iter);
- result.push_back('(');
- result.append(s);
- result.push_back(')');
- forward.push_back(s);
- }
-
- // Check reverse iteration results are the reverse of forward results
- size_t matched = 0;
- for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
- ASSERT_LT(matched, forward.size());
- ASSERT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]);
- matched++;
- }
- ASSERT_EQ(matched, forward.size());
-
- delete iter;
- return result;
- }
-
- std::string AllEntriesFor(const Slice& user_key) {
- Iterator* iter = dbfull()->TEST_NewInternalIterator();
- InternalKey target(user_key, kMaxSequenceNumber, kTypeValue);
- iter->Seek(target.Encode());
- std::string result;
- if (!iter->status().ok()) {
- result = iter->status().ToString();
- } else {
- result = "[ ";
- bool first = true;
- while (iter->Valid()) {
- ParsedInternalKey ikey;
- if (!ParseInternalKey(iter->key(), &ikey)) {
- result += "CORRUPTED";
- } else {
- if (last_options_.comparator->Compare(ikey.user_key, user_key) != 0) {
- break;
- }
- if (!first) {
- result += ", ";
- }
- first = false;
- switch (ikey.type) {
- case kTypeValue:
- result += iter->value().ToString();
- break;
- case kTypeDeletion:
- result += "DEL";
- break;
- }
- }
- iter->Next();
- }
- if (!first) {
- result += " ";
- }
- result += "]";
- }
- delete iter;
- return result;
- }
-
- int NumTableFilesAtLevel(int level) {
- std::string property;
- ASSERT_TRUE(
- db_->GetProperty("leveldb.num-files-at-level" + NumberToString(level),
- &property));
- return atoi(property.c_str());
- }
-
- int TotalTableFiles() {
- int result = 0;
- for (int level = 0; level < config::kNumLevels; level++) {
- result += NumTableFilesAtLevel(level);
- }
- return result;
- }
-
- // Return spread of files per level
- std::string FilesPerLevel() {
- std::string result;
- int last_non_zero_offset = 0;
- for (int level = 0; level < config::kNumLevels; level++) {
- int f = NumTableFilesAtLevel(level);
- char buf[100];
- snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
- result += buf;
- if (f > 0) {
- last_non_zero_offset = static_cast<int>(result.size());
- }
- }
- result.resize(last_non_zero_offset);
- return result;
- }
-
- int CountFiles() {
- std::vector<std::string> files;
- env_->GetChildren(dbname_, &files);
- return static_cast<int>(files.size());
- }
-
- uint64_t Size(const Slice& start, const Slice& limit) {
- Range r(start, limit);
- uint64_t size;
- db_->GetApproximateSizes(&r, 1, &size);
- return size;
- }
-
- void Compact(const Slice& start, const Slice& limit) {
- db_->CompactRange(&start, &limit);
- }
-
- // Do n memtable compactions, each of which produces an sstable
- // covering the range [small,large].
- void MakeTables(int n, const std::string& small, const std::string& large) {
- for (int i = 0; i < n; i++) {
- Put(small, "begin");
- Put(large, "end");
- dbfull()->TEST_CompactMemTable();
- }
- }
-
- // Prevent pushing of new sstables into deeper levels by adding
- // tables that cover a specified range to all levels.
- void FillLevels(const std::string& smallest, const std::string& largest) {
- MakeTables(config::kNumLevels, smallest, largest);
- }
-
- void DumpFileCounts(const char* label) {
- fprintf(stderr, "---\n%s:\n", label);
- fprintf(stderr, "maxoverlap: %lld\n",
- static_cast<long long>(
- dbfull()->TEST_MaxNextLevelOverlappingBytes()));
- for (int level = 0; level < config::kNumLevels; level++) {
- int num = NumTableFilesAtLevel(level);
- if (num > 0) {
- fprintf(stderr, " level %3d : %d files\n", level, num);
- }
- }
- }
-
- std::string DumpSSTableList() {
- std::string property;
- db_->GetProperty("leveldb.sstables", &property);
- return property;
- }
-
- std::string IterStatus(Iterator* iter) {
- std::string result;
- if (iter->Valid()) {
- result = iter->key().ToString() + "->" + iter->value().ToString();
- } else {
- result = "(invalid)";
- }
- return result;
- }
-
- bool DeleteAnSSTFile() {
- std::vector<std::string> filenames;
- ASSERT_OK(env_->GetChildren(dbname_, &filenames));
- uint64_t number;
- FileType type;
- for (size_t i = 0; i < filenames.size(); i++) {
- if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) {
- ASSERT_OK(env_->DeleteFile(TableFileName(dbname_, number)));
- return true;
- }
- }
- return false;
- }
-
- // Returns number of files renamed.
- int RenameLDBToSST() {
- std::vector<std::string> filenames;
- ASSERT_OK(env_->GetChildren(dbname_, &filenames));
- uint64_t number;
- FileType type;
- int files_renamed = 0;
- for (size_t i = 0; i < filenames.size(); i++) {
- if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) {
- const std::string from = TableFileName(dbname_, number);
- const std::string to = SSTTableFileName(dbname_, number);
- ASSERT_OK(env_->RenameFile(from, to));
- files_renamed++;
- }
- }
- return files_renamed;
- }
-};
-
-TEST(DBTest, Empty) {
- do {
- ASSERT_TRUE(db_ != NULL);
- ASSERT_EQ("NOT_FOUND", Get("foo"));
- } while (ChangeOptions());
-}
-
-TEST(DBTest, ReadWrite) {
- do {
- ASSERT_OK(Put("foo", "v1"));
- ASSERT_EQ("v1", Get("foo"));
- ASSERT_OK(Put("bar", "v2"));
- ASSERT_OK(Put("foo", "v3"));
- ASSERT_EQ("v3", Get("foo"));
- ASSERT_EQ("v2", Get("bar"));
- } while (ChangeOptions());
-}
-
-TEST(DBTest, PutDeleteGet) {
- do {
- ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
- ASSERT_EQ("v1", Get("foo"));
- ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
- ASSERT_EQ("v2", Get("foo"));
- ASSERT_OK(db_->Delete(WriteOptions(), "foo"));
- ASSERT_EQ("NOT_FOUND", Get("foo"));
- } while (ChangeOptions());
-}
-
-TEST(DBTest, GetFromImmutableLayer) {
- do {
- Options options = CurrentOptions();
- options.env = env_;
- options.write_buffer_size = 100000; // Small write buffer
- Reopen(&options);
-
- ASSERT_OK(Put("foo", "v1"));
- ASSERT_EQ("v1", Get("foo"));
-
- env_->delay_data_sync_.Release_Store(env_); // Block sync calls
- Put("k1", std::string(100000, 'x')); // Fill memtable
- Put("k2", std::string(100000, 'y')); // Trigger compaction
- ASSERT_EQ("v1", Get("foo"));
- env_->delay_data_sync_.Release_Store(NULL); // Release sync calls
- } while (ChangeOptions());
-}
-
-TEST(DBTest, GetFromVersions) {
- do {
- ASSERT_OK(Put("foo", "v1"));
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("v1", Get("foo"));
- } while (ChangeOptions());
-}
-
-TEST(DBTest, GetSnapshot) {
- do {
- // Try with both a short key and a long key
- for (int i = 0; i < 2; i++) {
- std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
- ASSERT_OK(Put(key, "v1"));
- const Snapshot* s1 = db_->GetSnapshot();
- ASSERT_OK(Put(key, "v2"));
- ASSERT_EQ("v2", Get(key));
- ASSERT_EQ("v1", Get(key, s1));
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("v2", Get(key));
- ASSERT_EQ("v1", Get(key, s1));
- db_->ReleaseSnapshot(s1);
- }
- } while (ChangeOptions());
-}
-
-TEST(DBTest, GetLevel0Ordering) {
- do {
- // Check that we process level-0 files in correct order. The code
- // below generates two level-0 files where the earlier one comes
- // before the later one in the level-0 file list since the earlier
- // one has a smaller "smallest" key.
- ASSERT_OK(Put("bar", "b"));
- ASSERT_OK(Put("foo", "v1"));
- dbfull()->TEST_CompactMemTable();
- ASSERT_OK(Put("foo", "v2"));
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("v2", Get("foo"));
- } while (ChangeOptions());
-}
-
-TEST(DBTest, GetOrderedByLevels) {
- do {
- ASSERT_OK(Put("foo", "v1"));
- Compact("a", "z");
- ASSERT_EQ("v1", Get("foo"));
- ASSERT_OK(Put("foo", "v2"));
- ASSERT_EQ("v2", Get("foo"));
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("v2", Get("foo"));
- } while (ChangeOptions());
-}
-
-TEST(DBTest, GetPicksCorrectFile) {
- do {
- // Arrange to have multiple files in a non-level-0 level.
- ASSERT_OK(Put("a", "va"));
- Compact("a", "b");
- ASSERT_OK(Put("x", "vx"));
- Compact("x", "y");
- ASSERT_OK(Put("f", "vf"));
- Compact("f", "g");
- ASSERT_EQ("va", Get("a"));
- ASSERT_EQ("vf", Get("f"));
- ASSERT_EQ("vx", Get("x"));
- } while (ChangeOptions());
-}
-
-TEST(DBTest, GetEncountersEmptyLevel) {
- do {
- // Arrange for the following to happen:
- // * sstable A in level 0
- // * nothing in level 1
- // * sstable B in level 2
- // Then do enough Get() calls to arrange for an automatic compaction
- // of sstable A. A bug would cause the compaction to be marked as
- // occurring at level 1 (instead of the correct level 0).
-
- // Step 1: First place sstables in levels 0 and 2
- int compaction_count = 0;
- while (NumTableFilesAtLevel(0) == 0 ||
- NumTableFilesAtLevel(2) == 0) {
- ASSERT_LE(compaction_count, 100) << "could not fill levels 0 and 2";
- compaction_count++;
- Put("a", "begin");
- Put("z", "end");
- dbfull()->TEST_CompactMemTable();
- }
-
- // Step 2: clear level 1 if necessary.
- dbfull()->TEST_CompactRange(1, NULL, NULL);
- ASSERT_EQ(NumTableFilesAtLevel(0), 1);
- ASSERT_EQ(NumTableFilesAtLevel(1), 0);
- ASSERT_EQ(NumTableFilesAtLevel(2), 1);
-
- // Step 3: read a bunch of times
- for (int i = 0; i < 1000; i++) {
- ASSERT_EQ("NOT_FOUND", Get("missing"));
- }
-
- // Step 4: Wait for compaction to finish
- DelayMilliseconds(1000);
-
- ASSERT_EQ(NumTableFilesAtLevel(0), 0);
- } while (ChangeOptions());
-}
-
-TEST(DBTest, IterEmpty) {
- Iterator* iter = db_->NewIterator(ReadOptions());
-
- iter->SeekToFirst();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- iter->SeekToLast();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- iter->Seek("foo");
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- delete iter;
-}
-
-TEST(DBTest, IterSingle) {
- ASSERT_OK(Put("a", "va"));
- Iterator* iter = db_->NewIterator(ReadOptions());
-
- iter->SeekToFirst();
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
- iter->SeekToFirst();
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- iter->SeekToLast();
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
- iter->SeekToLast();
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- iter->Seek("");
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- iter->Seek("a");
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- iter->Seek("b");
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- delete iter;
-}
-
-TEST(DBTest, IterMulti) {
- ASSERT_OK(Put("a", "va"));
- ASSERT_OK(Put("b", "vb"));
- ASSERT_OK(Put("c", "vc"));
- Iterator* iter = db_->NewIterator(ReadOptions());
-
- iter->SeekToFirst();
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "b->vb");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "c->vc");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
- iter->SeekToFirst();
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- iter->SeekToLast();
- ASSERT_EQ(IterStatus(iter), "c->vc");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "b->vb");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
- iter->SeekToLast();
- ASSERT_EQ(IterStatus(iter), "c->vc");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- iter->Seek("");
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Seek("a");
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Seek("ax");
- ASSERT_EQ(IterStatus(iter), "b->vb");
- iter->Seek("b");
- ASSERT_EQ(IterStatus(iter), "b->vb");
- iter->Seek("z");
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- // Switch from reverse to forward
- iter->SeekToLast();
- iter->Prev();
- iter->Prev();
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "b->vb");
-
- // Switch from forward to reverse
- iter->SeekToFirst();
- iter->Next();
- iter->Next();
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "b->vb");
-
- // Make sure iter stays at snapshot
- ASSERT_OK(Put("a", "va2"));
- ASSERT_OK(Put("a2", "va3"));
- ASSERT_OK(Put("b", "vb2"));
- ASSERT_OK(Put("c", "vc2"));
- ASSERT_OK(Delete("b"));
- iter->SeekToFirst();
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "b->vb");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "c->vc");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
- iter->SeekToLast();
- ASSERT_EQ(IterStatus(iter), "c->vc");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "b->vb");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- delete iter;
-}
-
-TEST(DBTest, IterSmallAndLargeMix) {
- ASSERT_OK(Put("a", "va"));
- ASSERT_OK(Put("b", std::string(100000, 'b')));
- ASSERT_OK(Put("c", "vc"));
- ASSERT_OK(Put("d", std::string(100000, 'd')));
- ASSERT_OK(Put("e", std::string(100000, 'e')));
-
- Iterator* iter = db_->NewIterator(ReadOptions());
-
- iter->SeekToFirst();
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "c->vc");
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
- iter->Next();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- iter->SeekToLast();
- ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "c->vc");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "a->va");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "(invalid)");
-
- delete iter;
-}
-
-TEST(DBTest, IterMultiWithDelete) {
- do {
- ASSERT_OK(Put("a", "va"));
- ASSERT_OK(Put("b", "vb"));
- ASSERT_OK(Put("c", "vc"));
- ASSERT_OK(Delete("b"));
- ASSERT_EQ("NOT_FOUND", Get("b"));
-
- Iterator* iter = db_->NewIterator(ReadOptions());
- iter->Seek("c");
- ASSERT_EQ(IterStatus(iter), "c->vc");
- iter->Prev();
- ASSERT_EQ(IterStatus(iter), "a->va");
- delete iter;
- } while (ChangeOptions());
-}
-
-TEST(DBTest, Recover) {
- do {
- ASSERT_OK(Put("foo", "v1"));
- ASSERT_OK(Put("baz", "v5"));
-
- Reopen();
- ASSERT_EQ("v1", Get("foo"));
-
- ASSERT_EQ("v1", Get("foo"));
- ASSERT_EQ("v5", Get("baz"));
- ASSERT_OK(Put("bar", "v2"));
- ASSERT_OK(Put("foo", "v3"));
-
- Reopen();
- ASSERT_EQ("v3", Get("foo"));
- ASSERT_OK(Put("foo", "v4"));
- ASSERT_EQ("v4", Get("foo"));
- ASSERT_EQ("v2", Get("bar"));
- ASSERT_EQ("v5", Get("baz"));
- } while (ChangeOptions());
-}
-
-TEST(DBTest, RecoveryWithEmptyLog) {
- do {
- ASSERT_OK(Put("foo", "v1"));
- ASSERT_OK(Put("foo", "v2"));
- Reopen();
- Reopen();
- ASSERT_OK(Put("foo", "v3"));
- Reopen();
- ASSERT_EQ("v3", Get("foo"));
- } while (ChangeOptions());
-}
-
-// Check that writes done during a memtable compaction are recovered
-// if the database is shutdown during the memtable compaction.
-TEST(DBTest, RecoverDuringMemtableCompaction) {
- do {
- Options options = CurrentOptions();
- options.env = env_;
- options.write_buffer_size = 1000000;
- Reopen(&options);
-
- // Trigger a long memtable compaction and reopen the database during it
- ASSERT_OK(Put("foo", "v1")); // Goes to 1st log file
- ASSERT_OK(Put("big1", std::string(10000000, 'x'))); // Fills memtable
- ASSERT_OK(Put("big2", std::string(1000, 'y'))); // Triggers compaction
- ASSERT_OK(Put("bar", "v2")); // Goes to new log file
-
- Reopen(&options);
- ASSERT_EQ("v1", Get("foo"));
- ASSERT_EQ("v2", Get("bar"));
- ASSERT_EQ(std::string(10000000, 'x'), Get("big1"));
- ASSERT_EQ(std::string(1000, 'y'), Get("big2"));
- } while (ChangeOptions());
-}
-
-static std::string Key(int i) {
- char buf[100];
- snprintf(buf, sizeof(buf), "key%06d", i);
- return std::string(buf);
-}
-
-TEST(DBTest, MinorCompactionsHappen) {
- Options options = CurrentOptions();
- options.write_buffer_size = 10000;
- Reopen(&options);
-
- const int N = 500;
-
- int starting_num_tables = TotalTableFiles();
- for (int i = 0; i < N; i++) {
- ASSERT_OK(Put(Key(i), Key(i) + std::string(1000, 'v')));
- }
- int ending_num_tables = TotalTableFiles();
- ASSERT_GT(ending_num_tables, starting_num_tables);
-
- for (int i = 0; i < N; i++) {
- ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
- }
-
- Reopen();
-
- for (int i = 0; i < N; i++) {
- ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
- }
-}
-
-TEST(DBTest, RecoverWithLargeLog) {
- {
- Options options = CurrentOptions();
- Reopen(&options);
- ASSERT_OK(Put("big1", std::string(200000, '1')));
- ASSERT_OK(Put("big2", std::string(200000, '2')));
- ASSERT_OK(Put("small3", std::string(10, '3')));
- ASSERT_OK(Put("small4", std::string(10, '4')));
- ASSERT_EQ(NumTableFilesAtLevel(0), 0);
- }
-
- // Make sure that if we re-open with a small write buffer size that
- // we flush table files in the middle of a large log file.
- Options options = CurrentOptions();
- options.write_buffer_size = 100000;
- Reopen(&options);
- ASSERT_EQ(NumTableFilesAtLevel(0), 3);
- ASSERT_EQ(std::string(200000, '1'), Get("big1"));
- ASSERT_EQ(std::string(200000, '2'), Get("big2"));
- ASSERT_EQ(std::string(10, '3'), Get("small3"));
- ASSERT_EQ(std::string(10, '4'), Get("small4"));
- ASSERT_GT(NumTableFilesAtLevel(0), 1);
-}
-
-TEST(DBTest, CompactionsGenerateMultipleFiles) {
- Options options = CurrentOptions();
- options.write_buffer_size = 100000000; // Large write buffer
- Reopen(&options);
-
- Random rnd(301);
-
- // Write 8MB (80 values, each 100K)
- ASSERT_EQ(NumTableFilesAtLevel(0), 0);
- std::vector<std::string> values;
- for (int i = 0; i < 80; i++) {
- values.push_back(RandomString(&rnd, 100000));
- ASSERT_OK(Put(Key(i), values[i]));
- }
-
- // Reopening moves updates to level-0
- Reopen(&options);
- dbfull()->TEST_CompactRange(0, NULL, NULL);
-
- ASSERT_EQ(NumTableFilesAtLevel(0), 0);
- ASSERT_GT(NumTableFilesAtLevel(1), 1);
- for (int i = 0; i < 80; i++) {
- ASSERT_EQ(Get(Key(i)), values[i]);
- }
-}
-
-TEST(DBTest, RepeatedWritesToSameKey) {
- Options options = CurrentOptions();
- options.env = env_;
- options.write_buffer_size = 100000; // Small write buffer
- Reopen(&options);
-
- // We must have at most one file per level except for level-0,
- // which may have up to kL0_StopWritesTrigger files.
- const int kMaxFiles = config::kNumLevels + config::kL0_StopWritesTrigger;
-
- Random rnd(301);
- std::string value = RandomString(&rnd, static_cast<int>(2 * options.write_buffer_size));
- for (int i = 0; i < 5 * kMaxFiles; i++) {
- Put("key", value);
- ASSERT_LE(TotalTableFiles(), kMaxFiles);
- fprintf(stderr, "after %d: %d files\n", int(i+1), TotalTableFiles());
- }
-}
-
-TEST(DBTest, SparseMerge) {
- Options options = CurrentOptions();
- options.compression = kNoCompression;
- Reopen(&options);
-
- FillLevels("A", "Z");
-
- // Suppose there is:
- // small amount of data with prefix A
- // large amount of data with prefix B
- // small amount of data with prefix C
- // and that recent updates have made small changes to all three prefixes.
- // Check that we do not do a compaction that merges all of B in one shot.
- const std::string value(1000, 'x');
- Put("A", "va");
- // Write approximately 100MB of "B" values
- for (int i = 0; i < 100000; i++) {
- char key[100];
- snprintf(key, sizeof(key), "B%010d", i);
- Put(key, value);
- }
- Put("C", "vc");
- dbfull()->TEST_CompactMemTable();
- dbfull()->TEST_CompactRange(0, NULL, NULL);
-
- // Make sparse update
- Put("A", "va2");
- Put("B100", "bvalue2");
- Put("C", "vc2");
- dbfull()->TEST_CompactMemTable();
-
- // Compactions should not cause us to create a situation where
- // a file overlaps too much data at the next level.
- ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
- dbfull()->TEST_CompactRange(0, NULL, NULL);
- ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
- dbfull()->TEST_CompactRange(1, NULL, NULL);
- ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
-}
-
-static bool Between(uint64_t val, uint64_t low, uint64_t high) {
- bool result = (val >= low) && (val <= high);
- if (!result) {
- fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
- (unsigned long long)(val),
- (unsigned long long)(low),
- (unsigned long long)(high));
- }
- return result;
-}
-
-TEST(DBTest, ApproximateSizes) {
- do {
- Options options = CurrentOptions();
- options.write_buffer_size = 100000000; // Large write buffer
- options.compression = kNoCompression;
- DestroyAndReopen();
-
- ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
- Reopen(&options);
- ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
-
- // Write 8MB (80 values, each 100K)
- ASSERT_EQ(NumTableFilesAtLevel(0), 0);
- const int N = 80;
- static const int S1 = 100000;
- static const int S2 = 105000; // Allow some expansion from metadata
- Random rnd(301);
- for (int i = 0; i < N; i++) {
- ASSERT_OK(Put(Key(i), RandomString(&rnd, S1)));
- }
-
- // 0 because GetApproximateSizes() does not account for memtable space
- ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
-
- // Check sizes across recovery by reopening a few times
- for (int run = 0; run < 3; run++) {
- Reopen(&options);
-
- for (int compact_start = 0; compact_start < N; compact_start += 10) {
- for (int i = 0; i < N; i += 10) {
- ASSERT_TRUE(Between(Size("", Key(i)), S1*i, S2*i));
- ASSERT_TRUE(Between(Size("", Key(i)+".suffix"), S1*(i+1), S2*(i+1)));
- ASSERT_TRUE(Between(Size(Key(i), Key(i+10)), S1*10, S2*10));
- }
- ASSERT_TRUE(Between(Size("", Key(50)), S1*50, S2*50));
- ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), S1*50, S2*50));
-
- std::string cstart_str = Key(compact_start);
- std::string cend_str = Key(compact_start + 9);
- Slice cstart = cstart_str;
- Slice cend = cend_str;
- dbfull()->TEST_CompactRange(0, &cstart, &cend);
- }
-
- ASSERT_EQ(NumTableFilesAtLevel(0), 0);
- ASSERT_GT(NumTableFilesAtLevel(1), 0);
- }
- } while (ChangeOptions());
-}
-
-TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
- do {
- Options options = CurrentOptions();
- options.compression = kNoCompression;
- Reopen();
-
- Random rnd(301);
- std::string big1 = RandomString(&rnd, 100000);
- ASSERT_OK(Put(Key(0), RandomString(&rnd, 10000)));
- ASSERT_OK(Put(Key(1), RandomString(&rnd, 10000)));
- ASSERT_OK(Put(Key(2), big1));
- ASSERT_OK(Put(Key(3), RandomString(&rnd, 10000)));
- ASSERT_OK(Put(Key(4), big1));
- ASSERT_OK(Put(Key(5), RandomString(&rnd, 10000)));
- ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
- ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));
-
- // Check sizes across recovery by reopening a few times
- for (int run = 0; run < 3; run++) {
- Reopen(&options);
-
- ASSERT_TRUE(Between(Size("", Key(0)), 0, 0));
- ASSERT_TRUE(Between(Size("", Key(1)), 10000, 11000));
- ASSERT_TRUE(Between(Size("", Key(2)), 20000, 21000));
- ASSERT_TRUE(Between(Size("", Key(3)), 120000, 121000));
- ASSERT_TRUE(Between(Size("", Key(4)), 130000, 131000));
- ASSERT_TRUE(Between(Size("", Key(5)), 230000, 231000));
- ASSERT_TRUE(Between(Size("", Key(6)), 240000, 241000));
- ASSERT_TRUE(Between(Size("", Key(7)), 540000, 541000));
- ASSERT_TRUE(Between(Size("", Key(8)), 550000, 560000));
-
- ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000));
-
- dbfull()->TEST_CompactRange(0, NULL, NULL);
- }
- } while (ChangeOptions());
-}
-
-TEST(DBTest, IteratorPinsRef) {
- Put("foo", "hello");
-
- // Get iterator that will yield the current contents of the DB.
- Iterator* iter = db_->NewIterator(ReadOptions());
-
- // Write to force compactions
- Put("foo", "newvalue1");
- for (int i = 0; i < 100; i++) {
- ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values
- }
- Put("foo", "newvalue2");
-
- iter->SeekToFirst();
- ASSERT_TRUE(iter->Valid());
- ASSERT_EQ("foo", iter->key().ToString());
- ASSERT_EQ("hello", iter->value().ToString());
- iter->Next();
- ASSERT_TRUE(!iter->Valid());
- delete iter;
-}
-
-TEST(DBTest, Snapshot) {
- do {
- Put("foo", "v1");
- const Snapshot* s1 = db_->GetSnapshot();
- Put("foo", "v2");
- const Snapshot* s2 = db_->GetSnapshot();
- Put("foo", "v3");
- const Snapshot* s3 = db_->GetSnapshot();
-
- Put("foo", "v4");
- ASSERT_EQ("v1", Get("foo", s1));
- ASSERT_EQ("v2", Get("foo", s2));
- ASSERT_EQ("v3", Get("foo", s3));
- ASSERT_EQ("v4", Get("foo"));
-
- db_->ReleaseSnapshot(s3);
- ASSERT_EQ("v1", Get("foo", s1));
- ASSERT_EQ("v2", Get("foo", s2));
- ASSERT_EQ("v4", Get("foo"));
-
- db_->ReleaseSnapshot(s1);
- ASSERT_EQ("v2", Get("foo", s2));
- ASSERT_EQ("v4", Get("foo"));
-
- db_->ReleaseSnapshot(s2);
- ASSERT_EQ("v4", Get("foo"));
- } while (ChangeOptions());
-}
-
-TEST(DBTest, HiddenValuesAreRemoved) {
- do {
- Random rnd(301);
- FillLevels("a", "z");
-
- std::string big = RandomString(&rnd, 50000);
- Put("foo", big);
- Put("pastfoo", "v");
- const Snapshot* snapshot = db_->GetSnapshot();
- Put("foo", "tiny");
- Put("pastfoo2", "v2"); // Advance sequence number one more
-
- ASSERT_OK(dbfull()->TEST_CompactMemTable());
- ASSERT_GT(NumTableFilesAtLevel(0), 0);
-
- ASSERT_EQ(big, Get("foo", snapshot));
- ASSERT_TRUE(Between(Size("", "pastfoo"), 50000, 60000));
- db_->ReleaseSnapshot(snapshot);
- ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]");
- Slice x("x");
- dbfull()->TEST_CompactRange(0, NULL, &x);
- ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
- ASSERT_EQ(NumTableFilesAtLevel(0), 0);
- ASSERT_GE(NumTableFilesAtLevel(1), 1);
- dbfull()->TEST_CompactRange(1, NULL, &x);
- ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
-
- ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000));
- } while (ChangeOptions());
-}
-
-TEST(DBTest, DeletionMarkers1) {
- Put("foo", "v1");
- ASSERT_OK(dbfull()->TEST_CompactMemTable());
- const int last = config::kMaxMemCompactLevel;
- ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
-
- // Place a table at level last-1 to prevent merging with preceding mutation
- Put("a", "begin");
- Put("z", "end");
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ(NumTableFilesAtLevel(last), 1);
- ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
-
- Delete("foo");
- Put("foo", "v2");
- ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
- ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
- ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
- Slice z("z");
- dbfull()->TEST_CompactRange(last-2, NULL, &z);
- // DEL eliminated, but v1 remains because we aren't compacting that level
- // (DEL can be eliminated because v2 hides v1).
- ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]");
- dbfull()->TEST_CompactRange(last-1, NULL, NULL);
- // Merging last-1 w/ last, so we are the base level for "foo", so
- // DEL is removed. (as is v1).
- ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]");
-}
-
-TEST(DBTest, DeletionMarkers2) {
- Put("foo", "v1");
- ASSERT_OK(dbfull()->TEST_CompactMemTable());
- const int last = config::kMaxMemCompactLevel;
- ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
-
- // Place a table at level last-1 to prevent merging with preceding mutation
- Put("a", "begin");
- Put("z", "end");
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ(NumTableFilesAtLevel(last), 1);
- ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
-
- Delete("foo");
- ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
- ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
- ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
- dbfull()->TEST_CompactRange(last-2, NULL, NULL);
- // DEL kept: "last" file overlaps
- ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
- dbfull()->TEST_CompactRange(last-1, NULL, NULL);
- // Merging last-1 w/ last, so we are the base level for "foo", so
- // DEL is removed. (as is v1).
- ASSERT_EQ(AllEntriesFor("foo"), "[ ]");
-}
-
-TEST(DBTest, OverlapInLevel0) {
- do {
- ASSERT_EQ(config::kMaxMemCompactLevel, 2) << "Fix test to match config";
-
- // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
- ASSERT_OK(Put("100", "v100"));
- ASSERT_OK(Put("999", "v999"));
- dbfull()->TEST_CompactMemTable();
- ASSERT_OK(Delete("100"));
- ASSERT_OK(Delete("999"));
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("0,1,1", FilesPerLevel());
-
- // Make files spanning the following ranges in level-0:
- // files[0] 200 .. 900
- // files[1] 300 .. 500
- // Note that files are sorted by smallest key.
- ASSERT_OK(Put("300", "v300"));
- ASSERT_OK(Put("500", "v500"));
- dbfull()->TEST_CompactMemTable();
- ASSERT_OK(Put("200", "v200"));
- ASSERT_OK(Put("600", "v600"));
- ASSERT_OK(Put("900", "v900"));
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("2,1,1", FilesPerLevel());
-
- // Compact away the placeholder files we created initially
- dbfull()->TEST_CompactRange(1, NULL, NULL);
- dbfull()->TEST_CompactRange(2, NULL, NULL);
- ASSERT_EQ("2", FilesPerLevel());
-
- // Do a memtable compaction. Before bug-fix, the compaction would
- // not detect the overlap with level-0 files and would incorrectly place
- // the deletion in a deeper level.
- ASSERT_OK(Delete("600"));
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("3", FilesPerLevel());
- ASSERT_EQ("NOT_FOUND", Get("600"));
- } while (ChangeOptions());
-}
-
-TEST(DBTest, L0_CompactionBug_Issue44_a) {
- Reopen();
- ASSERT_OK(Put("b", "v"));
- Reopen();
- ASSERT_OK(Delete("b"));
- ASSERT_OK(Delete("a"));
- Reopen();
- ASSERT_OK(Delete("a"));
- Reopen();
- ASSERT_OK(Put("a", "v"));
- Reopen();
- Reopen();
- ASSERT_EQ("(a->v)", Contents());
- DelayMilliseconds(1000); // Wait for compaction to finish
- ASSERT_EQ("(a->v)", Contents());
-}
-
-TEST(DBTest, L0_CompactionBug_Issue44_b) {
- Reopen();
- Put("","");
- Reopen();
- Delete("e");
- Put("","");
- Reopen();
- Put("c", "cv");
- Reopen();
- Put("","");
- Reopen();
- Put("","");
- DelayMilliseconds(1000); // Wait for compaction to finish
- Reopen();
- Put("d","dv");
- Reopen();
- Put("","");
- Reopen();
- Delete("d");
- Delete("b");
- Reopen();
- ASSERT_EQ("(->)(c->cv)", Contents());
- DelayMilliseconds(1000); // Wait for compaction to finish
- ASSERT_EQ("(->)(c->cv)", Contents());
-}
-
-TEST(DBTest, ComparatorCheck) {
- class NewComparator : public Comparator {
- public:
- virtual const char* Name() const { return "leveldb.NewComparator"; }
- virtual int Compare(const Slice& a, const Slice& b) const {
- return BytewiseComparator()->Compare(a, b);
- }
- virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
- BytewiseComparator()->FindShortestSeparator(s, l);
- }
- virtual void FindShortSuccessor(std::string* key) const {
- BytewiseComparator()->FindShortSuccessor(key);
- }
- };
- NewComparator cmp;
- Options new_options = CurrentOptions();
- new_options.comparator = &cmp;
- Status s = TryReopen(&new_options);
- ASSERT_TRUE(!s.ok());
- ASSERT_TRUE(s.ToString().find("comparator") != std::string::npos)
- << s.ToString();
-}
-
-TEST(DBTest, CustomComparator) {
- class NumberComparator : public Comparator {
- public:
- virtual const char* Name() const { return "test.NumberComparator"; }
- virtual int Compare(const Slice& a, const Slice& b) const {
- return ToNumber(a) - ToNumber(b);
- }
- virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
- ToNumber(*s); // Check format
- ToNumber(l); // Check format
- }
- virtual void FindShortSuccessor(std::string* key) const {
- ToNumber(*key); // Check format
- }
- private:
- static int ToNumber(const Slice& x) {
- // Check that there are no extra characters.
- ASSERT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size()-1] == ']')
- << EscapeString(x);
- int val;
- char ignored;
- ASSERT_TRUE(sscanf(x.ToString().c_str(), "[%i]%c", &val, &ignored) == 1)
- << EscapeString(x);
- return val;
- }
- };
- NumberComparator cmp;
- Options new_options = CurrentOptions();
- new_options.create_if_missing = true;
- new_options.comparator = &cmp;
- new_options.filter_policy = NULL; // Cannot use bloom filters
- new_options.write_buffer_size = 1000; // Compact more often
- DestroyAndReopen(&new_options);
- ASSERT_OK(Put("[10]", "ten"));
- ASSERT_OK(Put("[0x14]", "twenty"));
- for (int i = 0; i < 2; i++) {
- ASSERT_EQ("ten", Get("[10]"));
- ASSERT_EQ("ten", Get("[0xa]"));
- ASSERT_EQ("twenty", Get("[20]"));
- ASSERT_EQ("twenty", Get("[0x14]"));
- ASSERT_EQ("NOT_FOUND", Get("[15]"));
- ASSERT_EQ("NOT_FOUND", Get("[0xf]"));
- Compact("[0]", "[9999]");
- }
-
- for (int run = 0; run < 2; run++) {
- for (int i = 0; i < 1000; i++) {
- char buf[100];
- snprintf(buf, sizeof(buf), "[%d]", i*10);
- ASSERT_OK(Put(buf, buf));
- }
- Compact("[0]", "[1000000]");
- }
-}
-
-TEST(DBTest, ManualCompaction) {
- ASSERT_EQ(config::kMaxMemCompactLevel, 2)
- << "Need to update this test to match kMaxMemCompactLevel";
-
- MakeTables(3, "p", "q");
- ASSERT_EQ("1,1,1", FilesPerLevel());
-
- // Compaction range falls before files
- Compact("", "c");
- ASSERT_EQ("1,1,1", FilesPerLevel());
-
- // Compaction range falls after files
- Compact("r", "z");
- ASSERT_EQ("1,1,1", FilesPerLevel());
-
- // Compaction range overlaps files
- Compact("p1", "p9");
- ASSERT_EQ("0,0,1", FilesPerLevel());
-
- // Populate a different range
- MakeTables(3, "c", "e");
- ASSERT_EQ("1,1,2", FilesPerLevel());
-
- // Compact just the new range
- Compact("b", "f");
- ASSERT_EQ("0,0,2", FilesPerLevel());
-
- // Compact all
- MakeTables(1, "a", "z");
- ASSERT_EQ("0,1,2", FilesPerLevel());
- db_->CompactRange(NULL, NULL);
- ASSERT_EQ("0,0,1", FilesPerLevel());
-}
-
-TEST(DBTest, DBOpen_Options) {
- std::string dbname = test::TmpDir() + "/db_options_test";
- DestroyDB(dbname, Options());
-
- // Does not exist, and create_if_missing == false: error
- DB* db = NULL;
- Options opts;
- opts.create_if_missing = false;
- Status s = DB::Open(opts, dbname, &db);
- ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != NULL);
- ASSERT_TRUE(db == NULL);
-
- // Does not exist, and create_if_missing == true: OK
- opts.create_if_missing = true;
- s = DB::Open(opts, dbname, &db);
- ASSERT_OK(s);
- ASSERT_TRUE(db != NULL);
-
- delete db;
- db = NULL;
-
- // Does exist, and error_if_exists == true: error
- opts.create_if_missing = false;
- opts.error_if_exists = true;
- s = DB::Open(opts, dbname, &db);
- ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != NULL);
- ASSERT_TRUE(db == NULL);
-
- // Does exist, and error_if_exists == false: OK
- opts.create_if_missing = true;
- opts.error_if_exists = false;
- s = DB::Open(opts, dbname, &db);
- ASSERT_OK(s);
- ASSERT_TRUE(db != NULL);
-
- delete db;
- db = NULL;
-}
-
-TEST(DBTest, Locking) {
- DB* db2 = NULL;
- Status s = DB::Open(CurrentOptions(), dbname_, &db2);
- ASSERT_TRUE(!s.ok()) << "Locking did not prevent re-opening db";
-}
-
-// Check that number of files does not grow when we are out of space
-TEST(DBTest, NoSpace) {
- Options options = CurrentOptions();
- options.env = env_;
- Reopen(&options);
-
- ASSERT_OK(Put("foo", "v1"));
- ASSERT_EQ("v1", Get("foo"));
- Compact("a", "z");
- const int num_files = CountFiles();
- env_->no_space_.Release_Store(env_); // Force out-of-space errors
- for (int i = 0; i < 10; i++) {
- for (int level = 0; level < config::kNumLevels-1; level++) {
- dbfull()->TEST_CompactRange(level, NULL, NULL);
- }
- }
- env_->no_space_.Release_Store(NULL);
- ASSERT_LT(CountFiles(), num_files + 3);
-}
-
-TEST(DBTest, NonWritableFileSystem) {
- Options options = CurrentOptions();
- options.write_buffer_size = 1000;
- options.env = env_;
- Reopen(&options);
- ASSERT_OK(Put("foo", "v1"));
- env_->non_writable_.Release_Store(env_); // Force errors for new files
- std::string big(100000, 'x');
- int errors = 0;
- for (int i = 0; i < 20; i++) {
- fprintf(stderr, "iter %d; errors %d\n", i, errors);
- if (!Put("foo", big).ok()) {
- errors++;
- DelayMilliseconds(100);
- }
- }
- ASSERT_GT(errors, 0);
- env_->non_writable_.Release_Store(NULL);
-}
-
-TEST(DBTest, WriteSyncError) {
- // Check that log sync errors cause the DB to disallow future writes.
-
- // (a) Cause log sync calls to fail
- Options options = CurrentOptions();
- options.env = env_;
- Reopen(&options);
- env_->data_sync_error_.Release_Store(env_);
-
- // (b) Normal write should succeed
- WriteOptions w;
- ASSERT_OK(db_->Put(w, "k1", "v1"));
- ASSERT_EQ("v1", Get("k1"));
-
- // (c) Do a sync write; should fail
- w.sync = true;
- ASSERT_TRUE(!db_->Put(w, "k2", "v2").ok());
- ASSERT_EQ("v1", Get("k1"));
- ASSERT_EQ("NOT_FOUND", Get("k2"));
-
- // (d) make sync behave normally
- env_->data_sync_error_.Release_Store(NULL);
-
- // (e) Do a non-sync write; should fail
- w.sync = false;
- ASSERT_TRUE(!db_->Put(w, "k3", "v3").ok());
- ASSERT_EQ("v1", Get("k1"));
- ASSERT_EQ("NOT_FOUND", Get("k2"));
- ASSERT_EQ("NOT_FOUND", Get("k3"));
-}
-
-TEST(DBTest, ManifestWriteError) {
- // Test for the following problem:
- // (a) Compaction produces file F
- // (b) Log record containing F is written to MANIFEST file, but Sync() fails
- // (c) GC deletes F
- // (d) After reopening DB, reads fail since deleted F is named in log record
-
- // We iterate twice. In the second iteration, everything is the
- // same except the log record never makes it to the MANIFEST file.
- for (int iter = 0; iter < 2; iter++) {
- port::AtomicPointer* error_type = (iter == 0)
- ? &env_->manifest_sync_error_
- : &env_->manifest_write_error_;
-
- // Insert foo=>bar mapping
- Options options = CurrentOptions();
- options.env = env_;
- options.create_if_missing = true;
- options.error_if_exists = false;
- DestroyAndReopen(&options);
- ASSERT_OK(Put("foo", "bar"));
- ASSERT_EQ("bar", Get("foo"));
-
- // Memtable compaction (will succeed)
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("bar", Get("foo"));
- const int last = config::kMaxMemCompactLevel;
- ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level
-
- // Merging compaction (will fail)
- error_type->Release_Store(env_);
- dbfull()->TEST_CompactRange(last, NULL, NULL); // Should fail
- ASSERT_EQ("bar", Get("foo"));
-
- // Recovery: should not lose data
- error_type->Release_Store(NULL);
- Reopen(&options);
- ASSERT_EQ("bar", Get("foo"));
- }
-}
-
-TEST(DBTest, MissingSSTFile) {
- ASSERT_OK(Put("foo", "bar"));
- ASSERT_EQ("bar", Get("foo"));
-
- // Dump the memtable to disk.
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("bar", Get("foo"));
-
- Close();
- ASSERT_TRUE(DeleteAnSSTFile());
- Options options = CurrentOptions();
- options.paranoid_checks = true;
- Status s = TryReopen(&options);
- ASSERT_TRUE(!s.ok());
- ASSERT_TRUE(s.ToString().find("issing") != std::string::npos)
- << s.ToString();
-}
-
-TEST(DBTest, StillReadSST) {
- ASSERT_OK(Put("foo", "bar"));
- ASSERT_EQ("bar", Get("foo"));
-
- // Dump the memtable to disk.
- dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("bar", Get("foo"));
- Close();
- ASSERT_GT(RenameLDBToSST(), 0);
- Options options = CurrentOptions();
- options.paranoid_checks = true;
- Status s = TryReopen(&options);
- ASSERT_TRUE(s.ok());
- ASSERT_EQ("bar", Get("foo"));
-}
-
-TEST(DBTest, FilesDeletedAfterCompaction) {
- ASSERT_OK(Put("foo", "v2"));
- Compact("a", "z");
- const int num_files = CountFiles();
- for (int i = 0; i < 10; i++) {
- ASSERT_OK(Put("foo", "v2"));
- Compact("a", "z");
- }
- ASSERT_EQ(CountFiles(), num_files);
-}
-
-TEST(DBTest, BloomFilter) {
- env_->count_random_reads_ = true;
- Options options = CurrentOptions();
- options.env = env_;
- options.block_cache = NewLRUCache(0); // Prevent cache hits
- options.filter_policy = NewBloomFilterPolicy(10);
- Reopen(&options);
-
- // Populate multiple layers
- const int N = 10000;
- for (int i = 0; i < N; i++) {
- ASSERT_OK(Put(Key(i), Key(i)));
- }
- Compact("a", "z");
- for (int i = 0; i < N; i += 100) {
- ASSERT_OK(Put(Key(i), Key(i)));
- }
- dbfull()->TEST_CompactMemTable();
-
- // Prevent auto compactions triggered by seeks
- env_->delay_data_sync_.Release_Store(env_);
-
- // Lookup present keys. Should rarely read from small sstable.
- env_->random_read_counter_.Reset();
- for (int i = 0; i < N; i++) {
- ASSERT_EQ(Key(i), Get(Key(i)));
- }
- int reads = env_->random_read_counter_.Read();
- fprintf(stderr, "%d present => %d reads\n", N, reads);
- ASSERT_GE(reads, N);
- ASSERT_LE(reads, N + 2*N/100);
-
- // Lookup present keys. Should rarely read from either sstable.
- env_->random_read_counter_.Reset();
- for (int i = 0; i < N; i++) {
- ASSERT_EQ("NOT_FOUND", Get(Key(i) + ".missing"));
- }
- reads = env_->random_read_counter_.Read();
- fprintf(stderr, "%d missing => %d reads\n", N, reads);
- ASSERT_LE(reads, 3*N/100);
-
- env_->delay_data_sync_.Release_Store(NULL);
- Close();
- delete options.block_cache;
- delete options.filter_policy;
-}
-
-// Multi-threaded test:
-namespace {
-
-static const int kNumThreads = 4;
-static const int kTestSeconds = 10;
-static const int kNumKeys = 1000;
-
-struct MTState {
- DBTest* test;
- port::AtomicPointer stop;
- port::AtomicPointer counter[kNumThreads];
- port::AtomicPointer thread_done[kNumThreads];
-};
-
-struct MTThread {
- MTState* state;
- int id;
-};
-
-static void MTThreadBody(void* arg) {
- MTThread* t = reinterpret_cast<MTThread*>(arg);
- int id = t->id;
- DB* db = t->state->test->db_;
- uintptr_t counter = 0;
- fprintf(stderr, "... starting thread %d\n", id);
- Random rnd(1000 + id);
- std::string value;
- char valbuf[1500];
- while (t->state->stop.Acquire_Load() == NULL) {
- t->state->counter[id].Release_Store(reinterpret_cast<void*>(counter));
-
- int key = rnd.Uniform(kNumKeys);
- char keybuf[20];
- snprintf(keybuf, sizeof(keybuf), "%016d", key);
-
- if (rnd.OneIn(2)) {
- // Write values of the form <key, my id, counter>.
- // We add some padding for force compactions.
- snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d",
- key, id, static_cast<int>(counter));
- ASSERT_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf)));
- } else {
- // Read a value and verify that it matches the pattern written above.
- Status s = db->Get(ReadOptions(), Slice(keybuf), &value);
- if (s.IsNotFound()) {
- // Key has not yet been written
- } else {
- // Check that the writer thread counter is >= the counter in the value
- ASSERT_OK(s);
- int k, w, c;
- ASSERT_EQ(3, sscanf(value.c_str(), "%d.%d.%d", &k, &w, &c)) << value;
- ASSERT_EQ(k, key);
- ASSERT_GE(w, 0);
- ASSERT_LT(w, kNumThreads);
- ASSERT_LE(static_cast<uintptr_t>(c), reinterpret_cast<uintptr_t>(
- t->state->counter[w].Acquire_Load()));
- }
- }
- counter++;
- }
- t->state->thread_done[id].Release_Store(t);
- fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
-}
-
-} // namespace
-
-TEST(DBTest, MultiThreaded) {
- do {
- // Initialize state
- MTState mt;
- mt.test = this;
- mt.stop.Release_Store(0);
- for (int id = 0; id < kNumThreads; id++) {
- mt.counter[id].Release_Store(0);
- mt.thread_done[id].Release_Store(0);
- }
-
- // Start threads
- MTThread thread[kNumThreads];
- for (int id = 0; id < kNumThreads; id++) {
- thread[id].state = &mt;
- thread[id].id = id;
- std::thread t(std::bind(&MTThreadBody, &thread[id]));
- t.detach();
- }
-
- // Let them run for a while
- DelayMilliseconds(kTestSeconds * 1000);
-
- // Stop the threads and wait for them to finish
- mt.stop.Release_Store(&mt);
- for (int id = 0; id < kNumThreads; id++) {
- while (mt.thread_done[id].Acquire_Load() == NULL) {
- DelayMilliseconds(100);
- }
- }
- } while (ChangeOptions());
-}
-
-namespace {
-typedef std::map<std::string, std::string> KVMap;
-}
-
-class ModelDB: public DB {
- public:
- class ModelSnapshot : public Snapshot {
- public:
- KVMap map_;
- };
-
- explicit ModelDB(const Options& options): options_(options) { }
- ~ModelDB() { }
- virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) {
- return DB::Put(o, k, v);
- }
- virtual Status Delete(const WriteOptions& o, const Slice& key) {
- return DB::Delete(o, key);
- }
- virtual Status Get(const ReadOptions& options,
- const Slice& key, std::string* value) {
- assert(false); // Not implemented
- return Status::NotFound(key);
- }
- virtual Iterator* NewIterator(const ReadOptions& options) {
- if (options.snapshot == NULL) {
- KVMap* saved = new KVMap;
- *saved = map_;
- return new ModelIter(saved, true);
- } else {
- const KVMap* snapshot_state =
- &(reinterpret_cast<const ModelSnapshot*>(options.snapshot)->map_);
- return new ModelIter(snapshot_state, false);
- }
- }
- virtual const Snapshot* GetSnapshot() {
- ModelSnapshot* snapshot = new ModelSnapshot;
- snapshot->map_ = map_;
- return snapshot;
- }
-
- virtual void ReleaseSnapshot(const Snapshot* snapshot) {
- delete reinterpret_cast<const ModelSnapshot*>(snapshot);
- }
- virtual Status Write(const WriteOptions& options, WriteBatch* batch) {
- class Handler : public WriteBatch::Handler {
- public:
- KVMap* map_;
- virtual void Put(const Slice& key, const Slice& value) {
- (*map_)[key.ToString()] = value.ToString();
- }
- virtual void Delete(const Slice& key) {
- map_->erase(key.ToString());
- }
- };
- Handler handler;
- handler.map_ = &map_;
- return batch->Iterate(&handler);
- }
-
- virtual bool GetProperty(const Slice& property, std::string* value) {
- return false;
- }
- virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) {
- for (int i = 0; i < n; i++) {
- sizes[i] = 0;
- }
- }
- virtual void CompactRange(const Slice* start, const Slice* end) {
- }
-
- private:
- class ModelIter: public Iterator {
- public:
- ModelIter(const KVMap* map, bool owned)
- : map_(map), owned_(owned), iter_(map_->end()) {
- }
- ~ModelIter() {
- if (owned_) delete map_;
- }
- virtual bool Valid() const { return iter_ != map_->end(); }
- virtual void SeekToFirst() { iter_ = map_->begin(); }
- virtual void SeekToLast() {
- if (map_->empty()) {
- iter_ = map_->end();
- } else {
- iter_ = map_->find(map_->rbegin()->first);
- }
- }
- virtual void Seek(const Slice& k) {
- iter_ = map_->lower_bound(k.ToString());
- }
- virtual void Next() { ++iter_; }
- virtual void Prev() { --iter_; }
- virtual Slice key() const { return iter_->first; }
- virtual Slice value() const { return iter_->second; }
- virtual Status status() const { return Status::OK(); }
- private:
- const KVMap* const map_;
- const bool owned_; // Do we own map_
- KVMap::const_iterator iter_;
- };
- const Options options_;
- KVMap map_;
-};
-
-static std::string RandomKey(Random* rnd) {
- int len = (rnd->OneIn(3)
- ? 1 // Short sometimes to encourage collisions
- : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
- return test::RandomKey(rnd, len);
-}
-
-static bool CompareIterators(int step,
- DB* model,
- DB* db,
- const Snapshot* model_snap,
- const Snapshot* db_snap) {
- ReadOptions options;
- options.snapshot = model_snap;
- Iterator* miter = model->NewIterator(options);
- options.snapshot = db_snap;
- Iterator* dbiter = db->NewIterator(options);
- bool ok = true;
- int count = 0;
- for (miter->SeekToFirst(), dbiter->SeekToFirst();
- ok && miter->Valid() && dbiter->Valid();
- miter->Next(), dbiter->Next()) {
- count++;
- if (miter->key().compare(dbiter->key()) != 0) {
- fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n",
- step,
- EscapeString(miter->key()).c_str(),
- EscapeString(dbiter->key()).c_str());
- ok = false;
- break;
- }
-
- if (miter->value().compare(dbiter->value()) != 0) {
- fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n",
- step,
- EscapeString(miter->key()).c_str(),
- EscapeString(miter->value()).c_str(),
- EscapeString(miter->value()).c_str());
- ok = false;
- }
- }
-
- if (ok) {
- if (miter->Valid() != dbiter->Valid()) {
- fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n",
- step, miter->Valid(), dbiter->Valid());
- ok = false;
- }
- }
- fprintf(stderr, "%d entries compared: ok=%d\n", count, ok);
- delete miter;
- delete dbiter;
- return ok;
-}
-
-TEST(DBTest, Randomized) {
- Random rnd(test::RandomSeed());
- do {
- ModelDB model(CurrentOptions());
- const int N = 10000;
- const Snapshot* model_snap = NULL;
- const Snapshot* db_snap = NULL;
- std::string k, v;
- for (int step = 0; step < N; step++) {
- if (step % 100 == 0) {
- fprintf(stderr, "Step %d of %d\n", step, N);
- }
- // TODO(sanjay): Test Get() works
- int p = rnd.Uniform(100);
- if (p < 45) { // Put
- k = RandomKey(&rnd);
- v = RandomString(&rnd,
- rnd.OneIn(20)
- ? 100 + rnd.Uniform(100)
- : rnd.Uniform(8));
- ASSERT_OK(model.Put(WriteOptions(), k, v));
- ASSERT_OK(db_->Put(WriteOptions(), k, v));
-
- } else if (p < 90) { // Delete
- k = RandomKey(&rnd);
- ASSERT_OK(model.Delete(WriteOptions(), k));
- ASSERT_OK(db_->Delete(WriteOptions(), k));
-
-
- } else { // Multi-element batch
- WriteBatch b;
- const int num = rnd.Uniform(8);
- for (int i = 0; i < num; i++) {
- if (i == 0 || !rnd.OneIn(10)) {
- k = RandomKey(&rnd);
- } else {
- // Periodically re-use the same key from the previous iter, so
- // we have multiple entries in the write batch for the same key
- }
- if (rnd.OneIn(2)) {
- v = RandomString(&rnd, rnd.Uniform(10));
- b.Put(k, v);
- } else {
- b.Delete(k);
- }
- }
- ASSERT_OK(model.Write(WriteOptions(), &b));
- ASSERT_OK(db_->Write(WriteOptions(), &b));
- }
-
- if ((step % 100) == 0) {
- ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
- ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap));
- // Save a snapshot from each DB this time that we'll use next
- // time we compare things, to make sure the current state is
- // preserved with the snapshot
- if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
- if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
-
- Reopen();
- ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
-
- model_snap = model.GetSnapshot();
- db_snap = db_->GetSnapshot();
- }
- }
- if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
- if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
- } while (ChangeOptions());
-}
-
-std::string MakeKey(unsigned int num) {
- char buf[30];
- snprintf(buf, sizeof(buf), "%016u", num);
- return std::string(buf);
-}
-
-void BM_LogAndApply(int iters, int num_base_files) {
- std::string dbname = test::TmpDir() + "/leveldb_test_benchmark";
- DestroyDB(dbname, Options());
-
- DB* db = NULL;
- Options opts;
- opts.create_if_missing = true;
- Status s = DB::Open(opts, dbname, &db);
- ASSERT_OK(s);
- ASSERT_TRUE(db != NULL);
-
- delete db;
- db = NULL;
-
- Env* env = Env::Default();
-
- port::Mutex mu;
- MutexLock l(&mu);
-
- InternalKeyComparator cmp(BytewiseComparator());
- Options options;
- VersionSet vset(dbname, &options, NULL, &cmp);
- ASSERT_OK(vset.Recover());
- VersionEdit vbase;
- uint64_t fnum = 1;
- for (int i = 0; i < num_base_files; i++) {
- InternalKey start(MakeKey(static_cast<unsigned int>(2*fnum)), 1, kTypeValue);
- InternalKey limit(MakeKey(static_cast<unsigned int>(2*fnum+1)), 1, kTypeDeletion);
- vbase.AddFile(2, fnum++, 1 /* file size */, start, limit);
- }
- ASSERT_OK(vset.LogAndApply(&vbase, &mu));
-
- uint64_t start_micros = env->NowMicros();
-
- for (int i = 0; i < iters; i++) {
- VersionEdit vedit;
- vedit.DeleteFile(2, fnum);
- InternalKey start(MakeKey(static_cast<unsigned int>(2*fnum)), 1, kTypeValue);
- InternalKey limit(MakeKey(static_cast<unsigned int>(2*fnum+1)), 1, kTypeDeletion);
- vedit.AddFile(2, fnum++, 1 /* file size */, start, limit);
- vset.LogAndApply(&vedit, &mu);
- }
- uint64_t stop_micros = env->NowMicros();
- unsigned int us = static_cast<unsigned int>(stop_micros - start_micros);
- char buf[16];
- snprintf(buf, sizeof(buf), "%d", num_base_files);
- fprintf(stderr,
- "BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n",
- buf, iters, us, ((float)us) / iters);
-}
-
-} // namespace leveldb
-
-int main(int argc, char** argv) {
- if (argc > 1 && std::string(argv[1]) == "--benchmark") {
- leveldb::BM_LogAndApply(1000, 1);
- leveldb::BM_LogAndApply(1000, 100);
- leveldb::BM_LogAndApply(1000, 10000);
- leveldb::BM_LogAndApply(100, 100000);
- return 0;
- }
-
- return leveldb::test::RunAllTests();
-}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/leveldb-1.18/db/dbformat.cc
----------------------------------------------------------------------
diff --git a/thirdparty/leveldb-1.18/db/dbformat.cc b/thirdparty/leveldb-1.18/db/dbformat.cc
deleted file mode 100755
index f5975b3..0000000
--- a/thirdparty/leveldb-1.18/db/dbformat.cc
+++ /dev/null
@@ -1,140 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <stdio.h>
-#include "db/dbformat.h"
-#include "port/port.h"
-#include "util/coding.h"
-
-namespace leveldb {
-
-static uint64_t PackSequenceAndType(uint64_t seq, ValueType t) {
- assert(seq <= kMaxSequenceNumber);
- assert(t <= kValueTypeForSeek);
- return (seq << 8) | t;
-}
-
-void AppendInternalKey(std::string* result, const ParsedInternalKey& key) {
- result->append(key.user_key.data(), key.user_key.size());
- PutFixed64(result, PackSequenceAndType(key.sequence, key.type));
-}
-
-std::string ParsedInternalKey::DebugString() const {
- char buf[50];
- snprintf(buf, sizeof(buf), "' @ %llu : %d",
- (unsigned long long) sequence,
- int(type));
- std::string result = "'";
- result += EscapeString(user_key.ToString());
- result += buf;
- return result;
-}
-
-std::string InternalKey::DebugString() const {
- std::string result;
- ParsedInternalKey parsed;
- if (ParseInternalKey(rep_, &parsed)) {
- result = parsed.DebugString();
- } else {
- result = "(bad)";
- result.append(EscapeString(rep_));
- }
- return result;
-}
-
-const char* InternalKeyComparator::Name() const {
- return "leveldb.InternalKeyComparator";
-}
-
-int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
- // Order by:
- // increasing user key (according to user-supplied comparator)
- // decreasing sequence number
- // decreasing type (though sequence# should be enough to disambiguate)
- int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
- if (r == 0) {
- const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
- const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
- if (anum > bnum) {
- r = -1;
- } else if (anum < bnum) {
- r = +1;
- }
- }
- return r;
-}
-
-void InternalKeyComparator::FindShortestSeparator(
- std::string* start,
- const Slice& limit) const {
- // Attempt to shorten the user portion of the key
- Slice user_start = ExtractUserKey(*start);
- Slice user_limit = ExtractUserKey(limit);
- std::string tmp(user_start.data(), user_start.size());
- user_comparator_->FindShortestSeparator(&tmp, user_limit);
- if (tmp.size() < user_start.size() &&
- user_comparator_->Compare(user_start, tmp) < 0) {
- // User key has become shorter physically, but larger logically.
- // Tack on the earliest possible number to the shortened user key.
- PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
- assert(this->Compare(*start, tmp) < 0);
- assert(this->Compare(tmp, limit) < 0);
- start->swap(tmp);
- }
-}
-
-void InternalKeyComparator::FindShortSuccessor(std::string* key) const {
- Slice user_key = ExtractUserKey(*key);
- std::string tmp(user_key.data(), user_key.size());
- user_comparator_->FindShortSuccessor(&tmp);
- if (tmp.size() < user_key.size() &&
- user_comparator_->Compare(user_key, tmp) < 0) {
- // User key has become shorter physically, but larger logically.
- // Tack on the earliest possible number to the shortened user key.
- PutFixed64(&tmp, PackSequenceAndType(kMaxSequenceNumber,kValueTypeForSeek));
- assert(this->Compare(*key, tmp) < 0);
- key->swap(tmp);
- }
-}
-
-const char* InternalFilterPolicy::Name() const {
- return user_policy_->Name();
-}
-
-void InternalFilterPolicy::CreateFilter(const Slice* keys, int n,
- std::string* dst) const {
- // We rely on the fact that the code in table.cc does not mind us
- // adjusting keys[].
- Slice* mkey = const_cast<Slice*>(keys);
- for (int i = 0; i < n; i++) {
- mkey[i] = ExtractUserKey(keys[i]);
- // TODO(sanjay): Suppress dups?
- }
- user_policy_->CreateFilter(keys, n, dst);
-}
-
-bool InternalFilterPolicy::KeyMayMatch(const Slice& key, const Slice& f) const {
- return user_policy_->KeyMayMatch(ExtractUserKey(key), f);
-}
-
-LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
- uint32_t usize = static_cast<uint32_t>(user_key.size());
- uint32_t needed = usize + 13; // A conservative estimate
- char* dst;
- if (needed <= sizeof(space_)) {
- dst = space_;
- } else {
- dst = new char[needed];
- }
- start_ = dst;
- dst = EncodeVarint32(dst, usize + 8);
- kstart_ = dst;
- memcpy(dst, user_key.data(), usize);
- dst += usize;
- EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek));
- dst += 8;
- end_ = dst;
-}
-
-} // namespace leveldb
[33/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/build_detect_platform
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/build_detect_platform b/thirdparty/rocksdb/build_tools/build_detect_platform
new file mode 100755
index 0000000..c7ddb7c
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/build_detect_platform
@@ -0,0 +1,532 @@
+#!/bin/sh
+#
+# Detects OS we're compiling on and outputs a file specified by the first
+# argument, which in turn gets read while processing Makefile.
+#
+# The output will set the following variables:
+# CC C Compiler path
+# CXX C++ Compiler path
+# PLATFORM_LDFLAGS Linker flags
+# JAVA_LDFLAGS Linker flags for RocksDBJava
+# JAVA_STATIC_LDFLAGS Linker flags for RocksDBJava static build
+# PLATFORM_SHARED_EXT Extension for shared libraries
+# PLATFORM_SHARED_LDFLAGS Flags for building shared library
+# PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library
+# PLATFORM_CCFLAGS C compiler flags
+# PLATFORM_CXXFLAGS C++ compiler flags. Will contain:
+# PLATFORM_SHARED_VERSIONED Set to 'true' if platform supports versioned
+# shared libraries, empty otherwise.
+#
+# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
+#
+# -DROCKSDB_PLATFORM_POSIX if posix-platform based
+# -DSNAPPY if the Snappy library is present
+# -DLZ4 if the LZ4 library is present
+# -DZSTD if the ZSTD library is present
+# -DNUMA if the NUMA library is present
+# -DTBB if the TBB library is present
+#
+# Using gflags in rocksdb:
+# Our project depends on gflags, which requires users to take some extra steps
+# before they can compile the whole repository:
+# 1. Install gflags. You may download it from here:
+# https://gflags.github.io/gflags/ (Mac users can `brew install gflags`)
+# 2. Once installed, add the include path for gflags to your CPATH env var and
+# the lib path to LIBRARY_PATH. If installed with default settings, the lib
+# will be /usr/local/lib and the include path will be /usr/local/include
+
+OUTPUT=$1
+if test -z "$OUTPUT"; then
+ echo "usage: $0 <output-filename>" >&2
+ exit 1
+fi
+
+# we depend on C++11
+PLATFORM_CXXFLAGS="-std=c++11"
+# we currently depend on POSIX platform
+COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX"
+
+# Default to fbcode gcc on internal fb machines
+if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
+ FBCODE_BUILD="true"
+ # If we're compiling with TSAN we need pic build
+ PIC_BUILD=$COMPILE_WITH_TSAN
+ if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
+ source "$PWD/build_tools/fbcode_config.sh"
+ else
+ # we need this to build with MySQL. Don't use for other purposes.
+ source "$PWD/build_tools/fbcode_config4.8.1.sh"
+ fi
+fi
+
+# Delete existing output, if it exists
+rm -f "$OUTPUT"
+touch "$OUTPUT"
+
+if test -z "$CC"; then
+ CC=cc
+fi
+
+if test -z "$CXX"; then
+ CXX=g++
+fi
+
+# Detect OS
+if test -z "$TARGET_OS"; then
+ TARGET_OS=`uname -s`
+fi
+
+if test -z "$TARGET_ARCHITECTURE"; then
+ TARGET_ARCHITECTURE=`uname -m`
+fi
+
+if test -z "$CLANG_SCAN_BUILD"; then
+ CLANG_SCAN_BUILD=scan-build
+fi
+
+if test -z "$CLANG_ANALYZER"; then
+ CLANG_ANALYZER=$(which clang++ 2> /dev/null)
+fi
+
+COMMON_FLAGS="$COMMON_FLAGS ${CFLAGS}"
+CROSS_COMPILE=
+PLATFORM_CCFLAGS=
+PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS"
+PLATFORM_SHARED_EXT="so"
+PLATFORM_SHARED_LDFLAGS="-Wl,--no-as-needed -shared -Wl,-soname -Wl,"
+PLATFORM_SHARED_CFLAGS="-fPIC"
+PLATFORM_SHARED_VERSIONED=true
+
+# generic port files (working on all platform by #ifdef) go directly in /port
+GENERIC_PORT_FILES=`cd "$ROCKSDB_ROOT"; find port -name '*.cc' | tr "\n" " "`
+
+# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
+case "$TARGET_OS" in
+ Darwin)
+ PLATFORM=OS_MACOSX
+ COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX"
+ PLATFORM_SHARED_EXT=dylib
+ PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
+ # PORT_FILES=port/darwin/darwin_specific.cc
+ ;;
+ IOS)
+ PLATFORM=IOS
+ COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX -DIOS_CROSS_COMPILE -DROCKSDB_LITE"
+ PLATFORM_SHARED_EXT=dylib
+ PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
+ CROSS_COMPILE=true
+ PLATFORM_SHARED_VERSIONED=
+ ;;
+ Linux)
+ PLATFORM=OS_LINUX
+ COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX"
+ if [ -z "$USE_CLANG" ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
+ fi
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
+ # PORT_FILES=port/linux/linux_specific.cc
+ ;;
+ SunOS)
+ PLATFORM=OS_SOLARIS
+ COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -static-libstdc++ -static-libgcc -m64"
+ # PORT_FILES=port/sunos/sunos_specific.cc
+ ;;
+ AIX)
+ PLATFORM=OS_AIX
+ CC=gcc
+ COMMON_FLAGS="$COMMON_FLAGS -maix64 -pthread -fno-builtin-memcmp -D_REENTRANT -DOS_AIX -D__STDC_FORMAT_MACROS"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread -lpthread -lrt -maix64 -static-libstdc++ -static-libgcc"
+ # PORT_FILES=port/aix/aix_specific.cc
+ ;;
+ FreeBSD)
+ PLATFORM=OS_FREEBSD
+ COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
+ # PORT_FILES=port/freebsd/freebsd_specific.cc
+ ;;
+ NetBSD)
+ PLATFORM=OS_NETBSD
+ COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lgcc_s"
+ # PORT_FILES=port/netbsd/netbsd_specific.cc
+ ;;
+ OpenBSD)
+ PLATFORM=OS_OPENBSD
+ COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread"
+ # PORT_FILES=port/openbsd/openbsd_specific.cc
+ ;;
+ DragonFly)
+ PLATFORM=OS_DRAGONFLYBSD
+ COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
+ # PORT_FILES=port/dragonfly/dragonfly_specific.cc
+ ;;
+ Cygwin)
+ PLATFORM=CYGWIN
+ PLATFORM_SHARED_CFLAGS=""
+ PLATFORM_CXXFLAGS="-std=gnu++11"
+ COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN"
+ if [ -z "$USE_CLANG" ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
+ fi
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
+ # PORT_FILES=port/linux/linux_specific.cc
+ ;;
+ OS_ANDROID_CROSSCOMPILE)
+ PLATFORM=OS_ANDROID
+ COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DROCKSDB_PLATFORM_POSIX"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS " # All pthread features are in the Android C library
+ # PORT_FILES=port/android/android.cc
+ CROSS_COMPILE=true
+ ;;
+ *)
+ echo "Unknown platform!" >&2
+ exit 1
+esac
+
+PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}"
+JAVA_LDFLAGS="$PLATFORM_LDFLAGS"
+JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS"
+
+if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
+ # Cross-compiling; do not try any compilation tests.
+ # Also don't need any compilation tests if compiling on fbcode
+ true
+else
+ if ! test $ROCKSDB_DISABLE_FALLOCATE; then
+ # Test whether fallocate is available
+ $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <fcntl.h>
+ #include <linux/falloc.h>
+ int main() {
+ int fd = open("/dev/null", 0);
+ fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
+ }
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
+ fi
+ fi
+
+ # Test whether Snappy library is installed
+ # http://code.google.com/p/snappy/
+ $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <snappy.h>
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
+ JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy"
+ fi
+
+ # Test whether gflags library is installed
+ # http://gflags.github.io/gflags/
+ # check if the namespace is gflags
+ $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+ #include <gflags/gflags.h>
+ using namespace gflags;
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=gflags"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+ else
+ # check if namespace is google
+ $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+ #include <gflags/gflags.h>
+ using namespace google;
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+ fi
+ fi
+
+ # Test whether zlib library is installed
+ $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <zlib.h>
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DZLIB"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lz"
+ JAVA_LDFLAGS="$JAVA_LDFLAGS -lz"
+ fi
+
+ # Test whether bzip library is installed
+ $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <bzlib.h>
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DBZIP2"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2"
+ JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2"
+ fi
+
+ # Test whether lz4 library is installed
+ $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <lz4.h>
+ #include <lz4hc.h>
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DLZ4"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"
+ JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
+ fi
+
+ # Test whether zstd library is installed
+ $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <zstd.h>
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
+ JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
+ fi
+
+ # Test whether numa is available
+ $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null <<EOF
+ #include <numa.h>
+ #include <numaif.h>
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DNUMA"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma"
+ JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"
+ fi
+
+ # Test whether tbb is available
+ $CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null <<EOF
+ #include <tbb/tbb.h>
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DTBB"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"
+ JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"
+ fi
+
+ # Test whether jemalloc is available
+ if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
+ 2>/dev/null; then
+ # This will enable some preprocessor identifiers in the Makefile
+ JEMALLOC=1
+ # JEMALLOC can be enabled either using the flag (like here) or by
+ # providing direct link to the jemalloc library
+ WITH_JEMALLOC_FLAG=1
+ else
+ # jemalloc is not available. Let's try tcmalloc
+ if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
+ -ltcmalloc 2>/dev/null; then
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
+ JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc"
+ fi
+ fi
+
+ # Test whether malloc_usable_size is available
+ $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <malloc.h>
+ int main() {
+ size_t res = malloc_usable_size(0);
+ return 0;
+ }
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE"
+ fi
+
+ # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
+ $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <pthread.h>
+ int main() {
+ int x = PTHREAD_MUTEX_ADAPTIVE_NP;
+ return 0;
+ }
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX"
+ fi
+
+ # Test whether backtrace is available
+ $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <execinfo.h>>
+ int main() {
+ void* frames[1];
+ backtrace_symbols(frames, backtrace(frames, 1));
+ return 0;
+ }
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
+ else
+ # Test whether execinfo library is installed
+ $CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <execinfo.h>
+ int main() {
+ void* frames[1];
+ backtrace_symbols(frames, backtrace(frames, 1));
+ }
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo"
+ JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo"
+ fi
+ fi
+
+ # Test if -pg is supported
+ $CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null <<EOF
+ int main() {
+ return 0;
+ }
+EOF
+ if [ "$?" = 0 ]; then
+ PROFILING_FLAGS=-pg
+ fi
+
+ # Test whether sync_file_range is supported for compatibility with an old glibc
+ $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <fcntl.h>
+ int main() {
+ int fd = open("/dev/null", 0);
+ sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
+ }
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
+ fi
+
+ # Test whether sched_getcpu is supported
+ $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <sched.h>
+ int main() {
+ int cpuid = sched_getcpu();
+ }
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
+ fi
+fi
+
+# TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
+# -Wshorten-64-to-32 breaks compilation on FreeBSD i386
+if ! [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then
+ # Test whether -Wshorten-64-to-32 is available
+ $CXX $CFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null <<EOF
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -Wshorten-64-to-32"
+ fi
+fi
+
+# shall we use HDFS?
+
+if test "$USE_HDFS"; then
+ if test -z "$JAVA_HOME"; then
+ echo "JAVA_HOME has to be set for HDFS usage."
+ exit 1
+ fi
+ HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS"
+ HDFS_LDFLAGS="$HDFS_LDFLAGS -lhdfs -L$JAVA_HOME/jre/lib/amd64"
+ HDFS_LDFLAGS="$HDFS_LDFLAGS -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib"
+ HDFS_LDFLAGS="$HDFS_LDFLAGS -ldl -lverify -ljava -ljvm"
+ COMMON_FLAGS="$COMMON_FLAGS $HDFS_CCFLAGS"
+ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $HDFS_LDFLAGS"
+ JAVA_LDFLAGS="$JAVA_LDFLAGS $HDFS_LDFLAGS"
+fi
+
+if test "$USE_SSE"; then
+ COMMON_FLAGS="$COMMON_FLAGS -msse4.2"
+elif test -z "$PORTABLE"; then
+ if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
+ # Tune for this POWER processor, treating '+' models as base models
+ POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
+ COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
+ elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
+ COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
+ elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -march=native "
+ fi
+fi
+
+$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #include <cstdint>
+ #include <nmmintrin.h>
+ int main() {
+ volatile uint32_t x = _mm_crc32_u32(0, 0);
+ }
+EOF
+if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DHAVE_SSE42"
+elif test "$USE_SSE"; then
+ echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling"
+fi
+
+# iOS doesn't support thread-local storage, but this check would erroneously
+# succeed because the cross-compiler flags are added by the Makefile, not this
+# script.
+if [ "$PLATFORM" != IOS ]; then
+ $CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+ #if defined(_MSC_VER) && !defined(__thread)
+ #define __thread __declspec(thread)
+ #endif
+ int main() {
+ static __thread int tls;
+ }
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SUPPORT_THREAD_LOCAL"
+ fi
+fi
+
+PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
+PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
+
+VALGRIND_VER="$VALGRIND_VER"
+
+ROCKSDB_MAJOR=`build_tools/version.sh major`
+ROCKSDB_MINOR=`build_tools/version.sh minor`
+ROCKSDB_PATCH=`build_tools/version.sh patch`
+
+echo "CC=$CC" >> "$OUTPUT"
+echo "CXX=$CXX" >> "$OUTPUT"
+echo "PLATFORM=$PLATFORM" >> "$OUTPUT"
+echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT"
+echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT"
+echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT"
+echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT"
+echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT"
+echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT"
+echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> "$OUTPUT"
+echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> "$OUTPUT"
+echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> "$OUTPUT"
+echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> "$OUTPUT"
+echo "EXEC_LDFLAGS=$EXEC_LDFLAGS" >> "$OUTPUT"
+echo "JEMALLOC_INCLUDE=$JEMALLOC_INCLUDE" >> "$OUTPUT"
+echo "JEMALLOC_LIB=$JEMALLOC_LIB" >> "$OUTPUT"
+echo "ROCKSDB_MAJOR=$ROCKSDB_MAJOR" >> "$OUTPUT"
+echo "ROCKSDB_MINOR=$ROCKSDB_MINOR" >> "$OUTPUT"
+echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
+echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
+echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
+echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
+# This will enable some related identifiers for the preprocessor
+if test -n "$JEMALLOC"; then
+ echo "JEMALLOC=1" >> "$OUTPUT"
+fi
+# Indicates that jemalloc should be enabled using -ljemalloc flag
+# The alternative is to porvide a direct link to the library via JEMALLOC_LIB
+# and JEMALLOC_INCLUDE
+if test -n "$WITH_JEMALLOC_FLAG"; then
+ echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT"
+fi
+echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/cont_integration.sh
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/cont_integration.sh b/thirdparty/rocksdb/build_tools/cont_integration.sh
new file mode 100755
index 0000000..06f25c5
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/cont_integration.sh
@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2016, Facebook. All rights reserved.
+#
+# Overall wrapper script for RocksDB continuous builds. The implementation is a
+# trivial pulling scheme. We loop infinitely, check if any new changes have been
+# committed, if yes then trigger a Sandcastle run, and finally go to sleep again
+# for a certain interval.
+#
+
+SRC_GIT_REPO=/data/git/rocksdb-public
+error=0
+
+function log {
+ DATE=`date +%Y-%m-%d:%H:%M:%S`
+ echo $DATE $@
+}
+
+function log_err {
+ log "ERROR: $@ Error code: $error."
+}
+
+function update_repo_status {
+ # Update the parent first.
+ pushd $SRC_GIT_REPO
+
+ # This is a fatal error. Something in the environment isn't right and we will
+ # terminate the execution.
+ error=$?
+ if [ ! $error -eq 0 ]; then
+ log_err "Where is $SRC_GIT_REPO?"
+ exit $error
+ fi
+
+ HTTPS_PROXY=fwdproxy:8080 git fetch -f
+
+ error=$?
+ if [ ! $error -eq 0 ]; then
+ log_err "git fetch -f failed."
+ popd
+ return $error
+ fi
+
+ git update-ref refs/heads/master refs/remotes/origin/master
+
+ error=$?
+ if [ ! $error -eq 0 ]; then
+ log_err "git update-ref failed."
+ popd
+ return $error
+ fi
+
+ popd
+
+ # We're back in an instance-specific directory. Get the latest changes.
+ git pull --rebase
+
+ error=$?
+ if [ ! $error -eq 0 ]; then
+ log_err "git pull --rebase failed."
+ return $error
+ fi
+}
+
+#
+# Execution starts here.
+#
+
+# Path to the determinator from the root of the RocksDB repo.
+CONTRUN_DETERMINATOR=./build_tools/RocksDBCommonHelper.php
+
+# Value of the previous commit.
+PREV_COMMIT=
+
+log "Starting to monitor for new RocksDB changes ..."
+log "Running under `pwd` as `whoami`."
+
+# Paranoia. Make sure that we're using the right branch.
+git checkout master
+
+error=$?
+if [ ! $error -eq 0 ]; then
+ log_err "This is not good. Can't checkout master. Bye-bye!"
+ exit 1
+fi
+
+# We'll run forever and let the execution environment terminate us if we'll
+# exceed whatever timeout is set for the job.
+while true;
+do
+ # Get the latest changes committed.
+ update_repo_status
+
+ error=$?
+ if [ $error -eq 0 ]; then
+ LAST_COMMIT=`git log -1 | head -1 | grep commit | awk '{ print $2; }'`
+
+ log "Last commit is '$LAST_COMMIT', previous commit is '$PREV_COMMIT'."
+
+ if [ "$PREV_COMMIT" == "$LAST_COMMIT" ]; then
+ log "There were no changes since the last time I checked. Going to sleep."
+ else
+ if [ ! -z "$LAST_COMMIT" ]; then
+ log "New code has been committed or previous commit not known. " \
+ "Will trigger the tests."
+
+ PREV_COMMIT=$LAST_COMMIT
+ log "Updated previous commit to '$PREV_COMMIT'."
+
+ #
+ # This is where we'll trigger the Sandcastle run. The values for
+ # HTTPS_APP_VALUE and HTTPS_APP_VALUE will be set in the container we're
+ # running in.
+ #
+ POST_RECEIVE_HOOK=1 php $CONTRUN_DETERMINATOR
+
+ error=$?
+ if [ $error -eq 0 ]; then
+ log "Sandcastle run successfully triggered."
+ else
+ log_err "Failed to trigger Sandcastle run."
+ fi
+ else
+ log_err "Previous commit not updated. Don't know what the last one is."
+ fi
+ fi
+ else
+ log_err "Getting latest changes failed. Will skip running tests for now."
+ fi
+
+ # Always sleep, even if errors happens while trying to determine the latest
+ # commit. This will prevent us terminating in case of transient errors.
+ log "Will go to sleep for 5 minutes."
+ sleep 5m
+done
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/dependencies.sh
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/dependencies.sh b/thirdparty/rocksdb/build_tools/dependencies.sh
new file mode 100644
index 0000000..868753b
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/dependencies.sh
@@ -0,0 +1,18 @@
+GCC_BASE=/mnt/gvfs/third-party2/gcc/2928bb3ed95bf64f5b388ee88c30dc74710c3b35/5.x/centos6-native/f4950a1
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/a5fea028cb7ba43498976e1f8054b0b2e790c295/stable/centos6-native/6aaf4de
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/7a9099f6587ee4378c0b1fa32bb8934019d30ca4/5.x/gcc-5-glibc-2.23/339d858
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/3b7c6469854dfc7832a1c3cc5b86919a84e5f865/2.23/gcc-5-glibc-2.23/ca1d1c0
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-5-glibc-2.23/9bc6787
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/d7861abe6f0e27ab98c9303b95a662f0e4cdedb5/1.2.8/gcc-5-glibc-2.23/9bc6787
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-5-glibc-2.23/9bc6787
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0815d59804160c96caac5f27ca004f51af893dc6/r131/gcc-5-glibc-2.23/9bc6787
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/c15a4f5f619a2930478d01e2e34dc1e0652b0873/1.1.4/gcc-5-glibc-2.23/03859b5
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f905a5e1032fb30c05db3d3752319857388c0c49/2.2.0/gcc-5-glibc-2.23/9bc6787
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/8d60633d822a2a55849c73db24e74a25e52b71db/master/gcc-5-glibc-2.23/1c32b4b
+NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-5-glibc-2.23/9bc6787
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/8db74270cd6d0212ac92d69e7fc7beefe617d772/trunk/gcc-5-glibc-2.23/b1847cb
+TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-5-glibc-2.23/9bc6787
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/90c9734afc5579c9d1db529fa788d09f97763b85/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/9e829389ef61b92c62de8748c80169aaf25ce1f0/2.26.1/centos6-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.11.0/gcc-5-glibc-2.23/9bc6787
+LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/gcc-5-glibc-2.23/65372bd
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/dependencies_4.8.1.sh
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/dependencies_4.8.1.sh b/thirdparty/rocksdb/build_tools/dependencies_4.8.1.sh
new file mode 100644
index 0000000..ef0cda2
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/dependencies_4.8.1.sh
@@ -0,0 +1,18 @@
+GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/d282e6e8f3d20f4e40a516834847bdc038e07973/2.17/gcc-4.8.1-glibc-2.17/99df8fc
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-4.8.1-glibc-2.17/c3f970a
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-4.8.1-glibc-2.17/c3f970a
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-4.8.1-glibc-2.17/8d31e51
+NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/ad576de2a1ea560c4d3434304f0fc4e079bede42/trunk/gcc-4.8.1-glibc-2.17/675d945
+TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-4.8.1-glibc-2.17/c3f970a
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-4.8.1-glibc-2.17/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
+LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/centos6-native/730f94e
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/dockerbuild.sh
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/dockerbuild.sh b/thirdparty/rocksdb/build_tools/dockerbuild.sh
new file mode 100755
index 0000000..02f6094
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/dockerbuild.sh
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+docker run -v $PWD:/rocks -w /rocks buildpack-deps make
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/error_filter.py
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/error_filter.py b/thirdparty/rocksdb/build_tools/error_filter.py
new file mode 100644
index 0000000..9f619cf
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/error_filter.py
@@ -0,0 +1,167 @@
+# Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+# This source code is licensed under both the GPLv2 (found in the
+# COPYING file in the root directory) and Apache 2.0 License
+# (found in the LICENSE.Apache file in the root directory).
+
+'''Filter for error messages in test output:
+ - Receives merged stdout/stderr from test on stdin
+ - Finds patterns of known error messages for test name (first argument)
+ - Prints those error messages to stdout
+'''
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import re
+import sys
+
+
+class ErrorParserBase(object):
+ def parse_error(self, line):
+ '''Parses a line of test output. If it contains an error, returns a
+ formatted message describing the error; otherwise, returns None.
+ Subclasses must override this method.
+ '''
+ raise NotImplementedError
+
+
+class GTestErrorParser(ErrorParserBase):
+ '''A parser that remembers the last test that began running so it can print
+ that test's name upon detecting failure.
+ '''
+ _GTEST_NAME_PATTERN = re.compile(r'\[ RUN \] (\S+)$')
+ # format: '<filename or "unknown file">:<line #>: Failure'
+ _GTEST_FAIL_PATTERN = re.compile(r'(unknown file|\S+:\d+): Failure$')
+
+ def __init__(self):
+ self._last_gtest_name = 'Unknown test'
+
+ def parse_error(self, line):
+ gtest_name_match = self._GTEST_NAME_PATTERN.match(line)
+ if gtest_name_match:
+ self._last_gtest_name = gtest_name_match.group(1)
+ return None
+ gtest_fail_match = self._GTEST_FAIL_PATTERN.match(line)
+ if gtest_fail_match:
+ return '%s failed: %s' % (
+ self._last_gtest_name, gtest_fail_match.group(1))
+ return None
+
+
+class MatchErrorParser(ErrorParserBase):
+ '''A simple parser that returns the whole line if it matches the pattern.
+ '''
+ def __init__(self, pattern):
+ self._pattern = re.compile(pattern)
+
+ def parse_error(self, line):
+ if self._pattern.match(line):
+ return line
+ return None
+
+
+class CompilerErrorParser(MatchErrorParser):
+ def __init__(self):
+ # format: '<filename>:<line #>:<column #>: error: <error msg>'
+ super(CompilerErrorParser, self).__init__(r'\S+:\d+:\d+: error:')
+
+
+class ScanBuildErrorParser(MatchErrorParser):
+ def __init__(self):
+ super(ScanBuildErrorParser, self).__init__(
+ r'scan-build: \d+ bugs found.$')
+
+
+class DbCrashErrorParser(MatchErrorParser):
+ def __init__(self):
+ super(DbCrashErrorParser, self).__init__(r'\*\*\*.*\^$|TEST FAILED.')
+
+
+class WriteStressErrorParser(MatchErrorParser):
+ def __init__(self):
+ super(WriteStressErrorParser, self).__init__(
+ r'ERROR: write_stress died with exitcode=\d+')
+
+
+class AsanErrorParser(MatchErrorParser):
+ def __init__(self):
+ super(AsanErrorParser, self).__init__(
+ r'==\d+==ERROR: AddressSanitizer:')
+
+
+class UbsanErrorParser(MatchErrorParser):
+ def __init__(self):
+ # format: '<filename>:<line #>:<column #>: runtime error: <error msg>'
+ super(UbsanErrorParser, self).__init__(r'\S+:\d+:\d+: runtime error:')
+
+
+class ValgrindErrorParser(MatchErrorParser):
+ def __init__(self):
+ # just grab the summary, valgrind doesn't clearly distinguish errors
+ # from other log messages.
+ super(ValgrindErrorParser, self).__init__(r'==\d+== ERROR SUMMARY:')
+
+
+class CompatErrorParser(MatchErrorParser):
+ def __init__(self):
+ super(CompatErrorParser, self).__init__(r'==== .*[Ee]rror.* ====$')
+
+
+class TsanErrorParser(MatchErrorParser):
+ def __init__(self):
+ super(TsanErrorParser, self).__init__(r'WARNING: ThreadSanitizer:')
+
+
+_TEST_NAME_TO_PARSERS = {
+ 'punit': [CompilerErrorParser, GTestErrorParser],
+ 'unit': [CompilerErrorParser, GTestErrorParser],
+ 'release': [CompilerErrorParser, GTestErrorParser],
+ 'unit_481': [CompilerErrorParser, GTestErrorParser],
+ 'release_481': [CompilerErrorParser, GTestErrorParser],
+ 'clang_unit': [CompilerErrorParser, GTestErrorParser],
+ 'clang_release': [CompilerErrorParser, GTestErrorParser],
+ 'clang_analyze': [CompilerErrorParser, ScanBuildErrorParser],
+ 'code_cov': [CompilerErrorParser, GTestErrorParser],
+ 'unity': [CompilerErrorParser, GTestErrorParser],
+ 'lite': [CompilerErrorParser],
+ 'lite_test': [CompilerErrorParser, GTestErrorParser],
+ 'stress_crash': [CompilerErrorParser, DbCrashErrorParser],
+ 'write_stress': [CompilerErrorParser, WriteStressErrorParser],
+ 'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser],
+ 'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
+ 'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser],
+ 'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
+ 'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser],
+ 'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser],
+ 'format_compatible': [CompilerErrorParser, CompatErrorParser],
+ 'run_format_compatible': [CompilerErrorParser, CompatErrorParser],
+ 'no_compression': [CompilerErrorParser, GTestErrorParser],
+ 'run_no_compression': [CompilerErrorParser, GTestErrorParser],
+ 'regression': [CompilerErrorParser],
+ 'run_regression': [CompilerErrorParser],
+}
+
+
+def main():
+ if len(sys.argv) != 2:
+ return 'Usage: %s <test name>' % sys.argv[0]
+ test_name = sys.argv[1]
+ if test_name not in _TEST_NAME_TO_PARSERS:
+ return 'Unknown test name: %s' % test_name
+
+ error_parsers = []
+ for parser_cls in _TEST_NAME_TO_PARSERS[test_name]:
+ error_parsers.append(parser_cls())
+
+ for line in sys.stdin:
+ line = line.strip()
+ for error_parser in error_parsers:
+ error_msg = error_parser.parse_error(line)
+ if error_msg is not None:
+ print(error_msg)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/fb_compile_mongo.sh
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/fb_compile_mongo.sh b/thirdparty/rocksdb/build_tools/fb_compile_mongo.sh
new file mode 100755
index 0000000..c087f81
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/fb_compile_mongo.sh
@@ -0,0 +1,55 @@
+#!/bin/sh
+
+# fail early
+set -e
+
+if test -z $ROCKSDB_PATH; then
+ ROCKSDB_PATH=~/rocksdb
+fi
+source $ROCKSDB_PATH/build_tools/fbcode_config4.8.1.sh
+
+EXTRA_LDFLAGS=""
+
+if test -z $ALLOC; then
+ # default
+ ALLOC=tcmalloc
+elif [[ $ALLOC == "jemalloc" ]]; then
+ ALLOC=system
+ EXTRA_LDFLAGS+=" -Wl,--whole-archive $JEMALLOC_LIB -Wl,--no-whole-archive"
+fi
+
+# we need to force mongo to use static library, not shared
+STATIC_LIB_DEP_DIR='build/static_library_dependencies'
+test -d $STATIC_LIB_DEP_DIR || mkdir $STATIC_LIB_DEP_DIR
+test -h $STATIC_LIB_DEP_DIR/`basename $SNAPPY_LIBS` || ln -s $SNAPPY_LIBS $STATIC_LIB_DEP_DIR
+test -h $STATIC_LIB_DEP_DIR/`basename $LZ4_LIBS` || ln -s $LZ4_LIBS $STATIC_LIB_DEP_DIR
+
+EXTRA_LDFLAGS+=" -L $STATIC_LIB_DEP_DIR"
+
+set -x
+
+EXTRA_CMD=""
+if ! test -e version.json; then
+ # this is Mongo 3.0
+ EXTRA_CMD="--rocksdb \
+ --variant-dir=linux2/norm
+ --cxx=${CXX} \
+ --cc=${CC} \
+ --use-system-zlib" # add this line back to normal code path
+ # when https://jira.mongodb.org/browse/SERVER-19123 is resolved
+fi
+
+scons \
+ LINKFLAGS="$EXTRA_LDFLAGS $EXEC_LDFLAGS $PLATFORM_LDFLAGS" \
+ CCFLAGS="$CXXFLAGS -L $STATIC_LIB_DEP_DIR" \
+ LIBS="lz4 gcc stdc++" \
+ LIBPATH="$ROCKSDB_PATH" \
+ CPPPATH="$ROCKSDB_PATH/include" \
+ -j32 \
+ --allocator=$ALLOC \
+ --nostrip \
+ --opt=on \
+ --disable-minimum-compiler-version-enforcement \
+ --use-system-snappy \
+ --disable-warnings-as-errors \
+ $EXTRA_CMD $*
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/fbcode_config.sh
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/fbcode_config.sh b/thirdparty/rocksdb/build_tools/fbcode_config.sh
new file mode 100644
index 0000000..b8609a1
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/fbcode_config.sh
@@ -0,0 +1,156 @@
+#!/bin/sh
+#
+# Set environment variables so that we can compile rocksdb using
+# fbcode settings. It uses the latest g++ and clang compilers and also
+# uses jemalloc
+# Environment variables that change the behavior of this script:
+# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
+
+
+BASEDIR=`dirname $BASH_SOURCE`
+source "$BASEDIR/dependencies.sh"
+
+CFLAGS=""
+
+# libgcc
+LIBGCC_INCLUDE="$LIBGCC_BASE/include"
+LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
+
+# glibc
+GLIBC_INCLUDE="$GLIBC_BASE/include"
+GLIBC_LIBS=" -L $GLIBC_BASE/lib"
+
+# snappy
+SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
+if test -z $PIC_BUILD; then
+ SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
+else
+ SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
+fi
+CFLAGS+=" -DSNAPPY"
+
+if test -z $PIC_BUILD; then
+ # location of zlib headers and libraries
+ ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
+ ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
+ CFLAGS+=" -DZLIB"
+
+ # location of bzip headers and libraries
+ BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
+ BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
+ CFLAGS+=" -DBZIP2"
+
+ LZ4_INCLUDE=" -I $LZ4_BASE/include/"
+ LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
+ CFLAGS+=" -DLZ4"
+
+ ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
+ ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+ CFLAGS+=" -DZSTD"
+fi
+
+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
+if test -z $PIC_BUILD; then
+ GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
+else
+ GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
+fi
+CFLAGS+=" -DGFLAGS=gflags"
+
+# location of jemalloc
+JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
+JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
+
+if test -z $PIC_BUILD; then
+ # location of numa
+ NUMA_INCLUDE=" -I $NUMA_BASE/include/"
+ NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
+ CFLAGS+=" -DNUMA"
+
+ # location of libunwind
+ LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
+fi
+
+# location of TBB
+TBB_INCLUDE=" -isystem $TBB_BASE/include/"
+if test -z $PIC_BUILD; then
+ TBB_LIBS="$TBB_BASE/lib/libtbb.a"
+else
+ TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
+fi
+CFLAGS+=" -DTBB"
+
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
+
+BINUTILS="$BINUTILS_BASE/bin"
+AR="$BINUTILS/ar"
+
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
+
+STDLIBS="-L $GCC_BASE/lib64"
+
+CLANG_BIN="$CLANG_BASE/bin"
+CLANG_LIB="$CLANG_BASE/lib"
+CLANG_SRC="$CLANG_BASE/../../src"
+
+CLANG_ANALYZER="$CLANG_BIN/clang++"
+CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
+
+if [ -z "$USE_CLANG" ]; then
+ # gcc
+ CC="$GCC_BASE/bin/gcc"
+ CXX="$GCC_BASE/bin/g++"
+
+ CFLAGS+=" -B$BINUTILS/gold"
+ CFLAGS+=" -isystem $GLIBC_INCLUDE"
+ CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+ JEMALLOC=1
+else
+ # clang
+ CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
+ CC="$CLANG_BIN/clang"
+ CXX="$CLANG_BIN/clang++"
+
+ KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
+
+ CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
+ CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x "
+ CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x/x86_64-facebook-linux "
+ CFLAGS+=" -isystem $GLIBC_INCLUDE"
+ CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+ CFLAGS+=" -isystem $CLANG_INCLUDE"
+ CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
+ CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
+ CFLAGS+=" -Wno-expansion-to-defined "
+ CXXFLAGS="-nostdinc++"
+fi
+
+CFLAGS+=" $DEPS_INCLUDE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
+CXXFLAGS+=" $CFLAGS"
+
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
+EXEC_LDFLAGS+=" -B$BINUTILS/gold"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-5-glibc-2.23/lib/ld.so"
+EXEC_LDFLAGS+=" $LIBUNWIND"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-5-glibc-2.23/lib"
+# required by libtbb
+EXEC_LDFLAGS+=" -ldl"
+
+PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
+
+VALGRIND_VER="$VALGRIND_BASE/bin/"
+
+LUA_PATH="$LUA_BASE"
+
+if test -z $PIC_BUILD; then
+ LUA_LIB=" $LUA_PATH/lib/liblua.a"
+else
+ LUA_LIB=" $LUA_PATH/lib/liblua_pic.a"
+fi
+
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/fbcode_config4.8.1.sh
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/fbcode_config4.8.1.sh b/thirdparty/rocksdb/build_tools/fbcode_config4.8.1.sh
new file mode 100644
index 0000000..f5b8334
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/fbcode_config4.8.1.sh
@@ -0,0 +1,115 @@
+#!/bin/sh
+#
+# Set environment variables so that we can compile rocksdb using
+# fbcode settings. It uses the latest g++ compiler and also
+# uses jemalloc
+
+BASEDIR=`dirname $BASH_SOURCE`
+source "$BASEDIR/dependencies_4.8.1.sh"
+
+# location of libgcc
+LIBGCC_INCLUDE="$LIBGCC_BASE/include"
+LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
+
+# location of glibc
+GLIBC_INCLUDE="$GLIBC_BASE/include"
+GLIBC_LIBS=" -L $GLIBC_BASE/lib"
+
+# location of snappy headers and libraries
+SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include"
+SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
+
+# location of zlib headers and libraries
+ZLIB_INCLUDE=" -I $ZLIB_BASE/include"
+ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
+
+# location of bzip headers and libraries
+BZIP2_INCLUDE=" -I $BZIP2_BASE/include/"
+BZIP2_LIBS=" $BZIP2_BASE/lib/libbz2.a"
+
+LZ4_INCLUDE=" -I $LZ4_BASE/include"
+LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
+
+ZSTD_INCLUDE=" -I $ZSTD_BASE/include"
+ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+
+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
+GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
+
+# location of jemalloc
+JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include"
+JEMALLOC_LIB="$JEMALLOC_BASE/lib/libjemalloc.a"
+
+# location of numa
+NUMA_INCLUDE=" -I $NUMA_BASE/include/"
+NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
+
+# location of libunwind
+LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
+
+# location of tbb
+TBB_INCLUDE=" -isystem $TBB_BASE/include/"
+TBB_LIBS="$TBB_BASE/lib/libtbb.a"
+
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
+
+BINUTILS="$BINUTILS_BASE/bin"
+AR="$BINUTILS/ar"
+
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP2_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
+
+STDLIBS="-L $GCC_BASE/lib64"
+
+if [ -z "$USE_CLANG" ]; then
+ # gcc
+ CC="$GCC_BASE/bin/gcc"
+ CXX="$GCC_BASE/bin/g++"
+
+ CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic"
+ CFLAGS+=" -isystem $GLIBC_INCLUDE"
+ CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+ JEMALLOC=1
+else
+ # clang
+ CLANG_BIN="$CLANG_BASE/bin"
+ CLANG_LIB="$CLANG_BASE/lib"
+ CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
+ CC="$CLANG_BIN/clang"
+ CXX="$CLANG_BIN/clang++"
+
+ KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/"
+
+ CFLAGS="-B$BINUTILS/gold -nostdinc -nostdlib"
+ CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 "
+ CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1/x86_64-facebook-linux "
+ CFLAGS+=" -isystem $GLIBC_INCLUDE"
+ CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+ CFLAGS+=" -isystem $CLANG_INCLUDE"
+ CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
+ CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
+ CXXFLAGS="-nostdinc++"
+fi
+
+CFLAGS+=" $DEPS_INCLUDE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
+CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB"
+CXXFLAGS+=" $CFLAGS"
+
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
+EXEC_LDFLAGS+=" $LIBUNWIND"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib"
+# required by libtbb
+EXEC_LDFLAGS+=" -ldl"
+
+PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
+
+VALGRIND_VER="$VALGRIND_BASE/bin/"
+
+LUA_PATH="$LUA_BASE"
+
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE LUA_PATH
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/build_tools/format-diff.sh
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/build_tools/format-diff.sh b/thirdparty/rocksdb/build_tools/format-diff.sh
new file mode 100755
index 0000000..81221ed
--- /dev/null
+++ b/thirdparty/rocksdb/build_tools/format-diff.sh
@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# If clang_format_diff.py command is not specfied, we assume we are able to
+# access directly without any path.
+if [ -z $CLANG_FORMAT_DIFF ]
+then
+CLANG_FORMAT_DIFF="clang-format-diff.py"
+fi
+
+# Check clang-format-diff.py
+if ! which $CLANG_FORMAT_DIFF &> /dev/null
+then
+ echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
+ echo "You can download clang-format-diff.py by running: "
+ echo " curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}"
+ echo "You can download clang-format by running: "
+ echo " brew install clang-format"
+ echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}"
+ exit 128
+fi
+
+# Check argparse, a library that clang-format-diff.py requires.
+python 2>/dev/null << EOF
+import argparse
+EOF
+
+if [ "$?" != 0 ]
+then
+ echo "To run clang-format-diff.py, we'll need the library "argparse" to be"
+ echo "installed. You can try either of the follow ways to install it:"
+ echo " 1. Manually download argparse: https://pypi.python.org/pypi/argparse"
+ echo " 2. easy_install argparse (if you have easy_install)"
+ echo " 3. pip install argparse (if you have pip)"
+ exit 129
+fi
+
+# TODO(kailiu) following work is not complete since we still need to figure
+# out how to add the modified files done pre-commit hook to git's commit index.
+#
+# Check if this script has already been added to pre-commit hook.
+# Will suggest user to add this script to pre-commit hook if their pre-commit
+# is empty.
+# PRE_COMMIT_SCRIPT_PATH="`git rev-parse --show-toplevel`/.git/hooks/pre-commit"
+# if ! ls $PRE_COMMIT_SCRIPT_PATH &> /dev/null
+# then
+# echo "Would you like to add this script to pre-commit hook, which will do "
+# echo -n "the format check for all the affected lines before you check in (y/n):"
+# read add_to_hook
+# if [ "$add_to_hook" == "y" ]
+# then
+# ln -s `git rev-parse --show-toplevel`/build_tools/format-diff.sh $PRE_COMMIT_SCRIPT_PATH
+# fi
+# fi
+set -e
+
+uncommitted_code=`git diff HEAD`
+LAST_MASTER=`git merge-base master HEAD`
+
+# If there's no uncommitted changes, we assume user are doing post-commit
+# format check, in which case we'll check the modified lines since last commit
+# from master. Otherwise, we'll check format of the uncommitted code only.
+if [ -z "$uncommitted_code" ]
+then
+ # Check the format of last commit
+ diffs=$(git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -p 1)
+else
+ # Check the format of uncommitted lines,
+ diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1)
+fi
+
+if [ -z "$diffs" ]
+then
+ echo "Nothing needs to be reformatted!"
+ exit 0
+fi
+
+# Highlight the insertion/deletion from the clang-format-diff.py's output
+COLOR_END="\033[0m"
+COLOR_RED="\033[0;31m"
+COLOR_GREEN="\033[0;32m"
+
+echo -e "Detect lines that doesn't follow the format rules:\r"
+# Add the color to the diff. lines added will be green; lines removed will be red.
+echo "$diffs" |
+ sed -e "s/\(^-.*$\)/`echo -e \"$COLOR_RED\1$COLOR_END\"`/" |
+ sed -e "s/\(^+.*$\)/`echo -e \"$COLOR_GREEN\1$COLOR_END\"`/"
+
+if [[ "$OPT" == *"-DTRAVIS"* ]]
+then
+ exit 1
+fi
+
+echo -e "Would you like to fix the format automatically (y/n): \c"
+
+# Make sure under any mode, we can read user input.
+exec < /dev/tty
+read to_fix
+
+if [ "$to_fix" != "y" ]
+then
+ exit 1
+fi
+
+# Do in-place format adjustment.
+if [ -z "$uncommitted_code" ]
+then
+ git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -i -p 1
+else
+ git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1
+fi
+echo "Files reformatted!"
+
+# Amend to last commit if user do the post-commit format check
+if [ -z "$uncommitted_code" ]; then
+ echo -e "Would you like to amend the changes to last commit (`git log HEAD --oneline | head -1`)? (y/n): \c"
+ read to_amend
+
+ if [ "$to_amend" == "y" ]
+ then
+ git commit -a --amend --reuse-message HEAD
+ echo "Amended to last commit"
+ fi
+fi
[50/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/cmake/BuildTests.cmake
----------------------------------------------------------------------
diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake
index 4f4bed0..e76802d 100644
--- a/cmake/BuildTests.cmake
+++ b/cmake/BuildTests.cmake
@@ -31,13 +31,11 @@ ENDMACRO()
function(createTests testName)
message ("-- Adding test: ${testName}")
- target_include_directories(${testName} PRIVATE BEFORE ${UUID_INCLUDE_DIRS})
target_include_directories(${testName} PRIVATE BEFORE "thirdparty/catch")
target_include_directories(${testName} PRIVATE BEFORE "thirdparty/spdlog-20170710/include")
target_include_directories(${testName} PRIVATE BEFORE "thirdparty/yaml-cpp-yaml-cpp-0.5.3/include")
target_include_directories(${testName} PRIVATE BEFORE "thirdparty/jsoncpp/include")
target_include_directories(${testName} PRIVATE BEFORE "thirdparty/civetweb-1.9.1/include")
- target_include_directories(${testName} PRIVATE BEFORE ${LEVELDB_INCLUDE_DIRS})
target_include_directories(${testName} PRIVATE BEFORE "include")
target_include_directories(${testName} PRIVATE BEFORE "libminifi/include/")
target_include_directories(${testName} PRIVATE BEFORE "libminifi/include/c2/protocols")
@@ -52,8 +50,8 @@ function(createTests testName)
target_include_directories(${testName} PRIVATE BEFORE "libminifi/include/utils")
target_include_directories(${testName} PRIVATE BEFORE "libminifi/include/processors")
target_include_directories(${testName} PRIVATE BEFORE "libminifi/include/provenance")
- target_link_libraries(${testName} ${CMAKE_THREAD_LIBS_INIT} ${UUID_LIBRARIES} ${LEVELDB_LIBRARIES} ${OPENSSL_LIBRARIES} minifi yaml-cpp c-library civetweb-cpp ${JSON_CPP_LIB})
- if (HTTP-CURL)
+ target_link_libraries(${testName} ${CMAKE_THREAD_LIBS_INIT} ${OPENSSL_LIBRARIES} minifi yaml-cpp c-library civetweb-cpp ${JSON_CPP_LIB})
+ if (CURL_FOUND)
target_include_directories(${testName} PRIVATE BEFORE ${CURL_INCLUDE_DIRS})
target_include_directories(${testName} PRIVATE BEFORE "extensions/http-curl/")
target_include_directories(${testName} PRIVATE BEFORE "extensions/http-curl/client/")
@@ -61,7 +59,7 @@ function(createTests testName)
target_include_directories(${testName} PRIVATE BEFORE "extensions/http-curl/protocols/")
target_link_libraries(${testName} ${CURL_LIBRARIES} )
if (APPLE)
- target_link_libraries (${testName} -Wl,-all_load ${HTTP-CURL})
+ target_link_libraries (${testName} -Wl,-all_load ${HTTP-CURL})
else ()
target_link_libraries (${testName} -Wl,--whole-archive ${HTTP-CURL} -Wl,--no-whole-archive)
endif ()
@@ -77,6 +75,7 @@ SET(TEST_RESOURCES ${TEST_DIR}/resources)
GETSOURCEFILES(UNIT_TESTS "${TEST_DIR}/unit/")
GETSOURCEFILES(INTEGRATION_TESTS "${TEST_DIR}/integration/")
GETSOURCEFILES(CURL_INTEGRATION_TESTS "${TEST_DIR}/curl-tests/")
+GETSOURCEFILES(ROCKSDB_INTEGRATION_TESTS "${TEST_DIR}/rocksdb-tests/")
SET(UNIT_TEST_COUNT 0)
FOREACH(testfile ${UNIT_TESTS})
@@ -93,11 +92,28 @@ FOREACH(testfile ${INTEGRATION_TESTS})
get_filename_component(testfilename "${testfile}" NAME_WE)
add_executable("${testfilename}" "${TEST_DIR}/integration/${testfile}" ${SPD_SOURCES} "${TEST_DIR}/TestBase.cpp")
createTests("${testfilename}")
- #message("Adding ${testfilename} from ${testfile}")
MATH(EXPR INT_TEST_COUNT "${INT_TEST_COUNT}+1")
ENDFOREACH()
message("-- Finished building ${INT_TEST_COUNT} integration test file(s)...")
+if (ROCKSDB-REPOS)
+SET(ROCKSDB_TEST_COUNT 0)
+FOREACH(testfile ${ROCKSDB_INTEGRATION_TESTS})
+ get_filename_component(testfilename "${testfile}" NAME_WE)
+ add_executable("${testfilename}" "${TEST_DIR}/rocksdb-tests/${testfile}" ${SPD_SOURCES} "${TEST_DIR}/TestBase.cpp")
+ target_include_directories(${testfilename} PRIVATE BEFORE "extensions/rocksdb-repos/")
+ target_include_directories(${testfilename} PRIVATE BEFORE "thirdparty/rocksdb/include")
+ if (APPLE)
+ target_link_libraries (${testfilename} -Wl,-all_load ${ROCKSDB-REPOS})
+ else ()
+ target_link_libraries (${testfilename} -Wl,--whole-archive ${ROCKSDB-REPOS} -Wl,--no-whole-archive)
+ endif ()
+ createTests("${testfilename}")
+ MATH(EXPR ROCKSDB_TEST_COUNT "${ROCKSDB_TEST_COUNT}+1")
+ENDFOREACH()
+message("-- Finished building ${ROCKSDB_TEST_COUNT} RocksDB related test file(s)...")
+endif(ROCKSDB-REPOS)
+
if (HTTP-CURL)
SET(CURL_INT_TEST_COUNT 0)
@@ -124,8 +140,6 @@ add_test(NAME HttpPostIntegrationTestChunked COMMAND HttpPostIntegrationTest "${
add_test(NAME C2VerifyServeResults COMMAND C2VerifyServeResults "${TEST_RESOURCES}/TestHTTPGet.yml" "${TEST_RESOURCES}/")
-add_test(NAME C2NullConfiguration COMMAND C2NullConfiguration "${TEST_RESOURCES}/TestHTTPGet.yml" "${TEST_RESOURCES}/")
-
add_test(NAME C2VerifyHeartbeatAndStop COMMAND C2VerifyHeartbeatAndStop "${TEST_RESOURCES}/TestHTTPGet.yml" "${TEST_RESOURCES}/")
add_test(NAME SiteToSiteRestTest COMMAND SiteToSiteRestTest "${TEST_RESOURCES}/TestSite2SiteRest.yml" "${TEST_RESOURCES}/" "http://localhost:8082/nifi-api/controller")
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/cmake/FindLeveldb.cmake
----------------------------------------------------------------------
diff --git a/cmake/FindLeveldb.cmake b/cmake/FindLeveldb.cmake
deleted file mode 100644
index a6d94c0..0000000
--- a/cmake/FindLeveldb.cmake
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-# Find module for Leveldb library and includes
-# LEVELDB_FOUND - if system found LEVELDB library
-# LEVELDB_INCLUDE_DIRS - The LEVELDB include directories
-# LEVELDB_LIBRARIES - The libraries needed to use LEVELDB
-# LEVELDB_DEFINITIONS - Compiler switches required for using LEVELDB
-
-# For OS X do not attempt to use the OS X application frameworks or bundles.
-set (CMAKE_FIND_FRAMEWORK NEVER)
-set (CMAKE_FIND_APPBUNDLE NEVER)
-
-find_path(LEVELDB_INCLUDE_DIR
- NAMES leveldb/db.h
- PATHS /usr/local/include /usr/include
- DOC "LevelDB include header"
-)
-
-find_library(LEVELDB_LIBRARY
- NAMES libleveldb.dylib libleveldb.so
- PATHS /usr/local/lib /usr/lib/x86_64-linux-gnu
- DOC "LevelDB library"
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(LEVELDB DEFAULT_MSG LEVELDB_INCLUDE_DIR LEVELDB_LIBRARY)
-
-if (LEVELDB_FOUND)
- set(LEVELDB_LIBRARIES ${LEVELDB_LIBRARY} )
- set(LEVELDB_INCLUDE_DIRS ${LEVELDB_INCLUDE_DIR} )
- set(LEVELDB_DEFINITIONS )
-endif()
-
-mark_as_advanced(LEVELDB_ROOT_DIR LEVELDB_INCLUDE_DIR LEVELDB_LIBRARY)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/cmake/FindRocksDB.cmake
----------------------------------------------------------------------
diff --git a/cmake/FindRocksDB.cmake b/cmake/FindRocksDB.cmake
new file mode 100644
index 0000000..db9c2d1
--- /dev/null
+++ b/cmake/FindRocksDB.cmake
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ROCKSDB_FOUND System has RocksDB library/headers.
+# ROCKSDB_LIBRARIES The RocksDB library.
+# ROCKSDB_INCLUDE_DIR The location of RocksDB headers.
+
+find_path(ROCKSDB_ROOT_DIR
+ NAMES include/rocksdb/db.h
+)
+
+find_library(ROCKSDB_LIBRARIES
+ NAMES rocksdb
+ HINTS ${ROCKSDB_ROOT_DIR}/lib
+)
+
+find_path(ROCKSDB_INCLUDE_DIR
+ NAMES rocksdb/db.h
+ HINTS ${ROCKSDB_ROOT_DIR}/include
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(RocksDB DEFAULT_MSG
+ ROCKSDB_LIBRARIES
+ ROCKSDB_INCLUDE_DIR
+)
+
+mark_as_advanced(
+ ROCKSDB_ROOT_DIR
+ ROCKSDB_LIBRARIES
+ ROCKSDB_INCLUDE_DIR
+)
+
+if(ROCKSDB_INCLUDE_DIR AND ROCKSDB_LIBRARIES)
+ set(ROCKSDB_FOUND "YES")
+ message(STATUS "Found RocksDB...${ROCKSDB_LIBRARIES}")
+endif()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 7688439..641f0e2 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -17,7 +17,7 @@
#
# First stage: the build environment
-# Edge required for leveldb
+# Edge required for rocksdb
FROM alpine:edge AS builder
MAINTAINER Apache NiFi <de...@nifi.apache.org>
@@ -27,7 +27,7 @@ ARG MINIFI_VERSION
ARG MINIFI_SOURCE_CODE
# Install the system dependencies needed for a build
-# Add testing repo for leveldb
+# Add testing repo for rocksdb
RUN echo 'http://dl-cdn.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
RUN apk --update --no-cache upgrade && apk --update --no-cache add gcc \
g++ \
@@ -38,7 +38,7 @@ RUN apk --update --no-cache upgrade && apk --update --no-cache add gcc \
boost-dev \
vim \
util-linux-dev \
- leveldb-dev \
+ rocksdb-dev \
curl-dev \
cmake \
git \
@@ -70,7 +70,7 @@ RUN cd $MINIFI_BASE_DIR \
&& tar -xzvf $MINIFI_BASE_DIR/build/nifi-minifi-cpp-$MINIFI_VERSION-bin.tar.gz -C $MINIFI_BASE_DIR
# Second stage: the runtime image
-# Edge required for leveldb
+# Edge required for rocksdb
FROM alpine:edge
ARG UID
@@ -78,12 +78,12 @@ ARG GID
ARG MINIFI_VERSION
ARG MINIFI_SOURCE_CODE
-# Add testing repo for leveldb
+# Add testing repo for rocksdb
RUN echo 'http://dl-cdn.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
RUN apk --update --no-cache upgrade && apk add --update --no-cache \
util-linux \
- leveldb \
+ rocksdb \
curl \
unzip \
gpsd \
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/http-curl/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/extensions/http-curl/CMakeLists.txt b/extensions/http-curl/CMakeLists.txt
index cb91683..b8b0629 100644
--- a/extensions/http-curl/CMakeLists.txt
+++ b/extensions/http-curl/CMakeLists.txt
@@ -24,7 +24,7 @@ find_package(CURL REQUIRED)
set(CMAKE_EXE_LINKER_FLAGS "-Wl,--export-all-symbols")
set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--export-symbols")
-include_directories(../../libminifi/include ../../libminifi/include/c2 ../../libminifi/include/c2/protocols/ ../../libminifi/include/core/state ./libminifi/include/core/statemanagement/metrics ../../libminifi/include/core/yaml ../../libminifi/include/core ../../thirdparty/spdlog-20170710/include ../../thirdparty/concurrentqueue ../../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../../thirdparty/civetweb-1.9.1/include ../../thirdparty/jsoncpp/include ../../thirdparty/leveldb-1.18/include ../../thirdparty/)
+include_directories(../../libminifi/include ../../libminifi/include/c2 ../../libminifi/include/c2/protocols/ ../../libminifi/include/core/state ./libminifi/include/core/statemanagement/metrics ../../libminifi/include/core/yaml ../../libminifi/include/core ../../thirdparty/spdlog-20170710/include ../../thirdparty/concurrentqueue ../../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../../thirdparty/civetweb-1.9.1/include ../../thirdparty/jsoncpp/include ../../thirdparty/)
find_package(Boost REQUIRED)
include_directories(${Boost_INCLUDE_DIRS})
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/rocksdb-repos/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/CMakeLists.txt b/extensions/rocksdb-repos/CMakeLists.txt
new file mode 100644
index 0000000..82df6db
--- /dev/null
+++ b/extensions/rocksdb-repos/CMakeLists.txt
@@ -0,0 +1,84 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+cmake_minimum_required(VERSION 2.6)
+
+find_package(CURL REQUIRED)
+
+set(CMAKE_EXE_LINKER_FLAGS "-Wl,--export-all-symbols")
+set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--export-symbols")
+
+include_directories(../../libminifi/include ../../libminifi/include/c2 ../../libminifi/include/c2/protocols/ ../../libminifi/include/core/state ./libminifi/include/core/statemanagement/metrics ../../libminifi/include/core/yaml ../../libminifi/include/core ../../thirdparty/spdlog-20170710/include ../../thirdparty/concurrentqueue ../../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../../thirdparty/civetweb-1.9.1/include ../../thirdparty/jsoncpp/include ../../thirdparty/rocksdb/include ../../thirdparty/)
+
+find_package(Boost REQUIRED)
+find_package(RocksDB)
+
+SET_PACKAGE_INFO("HTTP-CURL" "libcURL implementation code that supports " )
+
+include_directories(${Boost_INCLUDE_DIRS})
+
+file(GLOB SOURCES "*.cpp")
+
+add_library(minifi-rocksdb-repos STATIC ${SOURCES})
+set_property(TARGET minifi-rocksdb-repos PROPERTY POSITION_INDEPENDENT_CODE ON)
+if(THREADS_HAVE_PTHREAD_ARG)
+ target_compile_options(PUBLIC minifi-rocksdb-repos "-pthread")
+endif()
+if(CMAKE_THREAD_LIBS_INIT)
+ target_link_libraries(minifi-rocksdb-repos "${CMAKE_THREAD_LIBS_INIT}")
+endif()
+
+if (CURL_FOUND)
+ include_directories(${CURL_INCLUDE_DIRS})
+ target_link_libraries (minifi-rocksdb-repos ${CURL_LIBRARIES})
+endif(CURL_FOUND)
+
+# Include UUID
+find_package(UUID REQUIRED)
+target_link_libraries(minifi-rocksdb-repos ${LIBMINIFI} ${UUID_LIBRARIES} ${JSONCPP_LIB})
+add_dependencies(minifi-rocksdb-repos jsoncpp_project)
+find_package(OpenSSL REQUIRED)
+include_directories(${OPENSSL_INCLUDE_DIR})
+target_link_libraries(minifi-rocksdb-repos ${CMAKE_DL_LIBS} )
+if (ROCKSDB_FOUND)
+ target_link_libraries(minifi-rocksdb-repos ${ROCKSDB_LIBRARIES} )
+else()
+ target_link_libraries(minifi-rocksdb-repos rocksdb )
+endif()
+find_package(ZLIB REQUIRED)
+include_directories(${ZLIB_INCLUDE_DIRS})
+target_link_libraries (minifi-rocksdb-repos ${ZLIB_LIBRARIES})
+if (WIN32)
+ set_target_properties(minifi-rocksdb-repos PROPERTIES
+ LINK_FLAGS "/WHOLEARCHIVE"
+ )
+elseif (APPLE)
+ set_target_properties(minifi-rocksdb-repos PROPERTIES
+ LINK_FLAGS "-Wl,-all_load"
+ )
+else ()
+ set_target_properties(minifi-rocksdb-repos PROPERTIES
+ LINK_FLAGS "-Wl,--whole-archive"
+ )
+endif ()
+
+
+SET (ROCKSDB-REPOS minifi-rocksdb-repos PARENT_SCOPE)
+
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/rocksdb-repos/DatabaseContentRepository.cpp
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/DatabaseContentRepository.cpp b/extensions/rocksdb-repos/DatabaseContentRepository.cpp
new file mode 100644
index 0000000..50f007f
--- /dev/null
+++ b/extensions/rocksdb-repos/DatabaseContentRepository.cpp
@@ -0,0 +1,112 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "DatabaseContentRepository.h"
+#include <memory>
+#include <string>
+#include "RocksDbStream.h"
+#include "rocksdb/merge_operator.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace core {
+namespace repository {
+
+bool DatabaseContentRepository::initialize(const std::shared_ptr<minifi::Configure> &configuration) {
+ std::string value;
+ if (configuration->get(Configure::nifi_dbcontent_repository_directory_default, value)) {
+ directory_ = value;
+ } else {
+ directory_ = "dbcontentrepository";
+ }
+ rocksdb::Options options;
+ options.create_if_missing = true;
+ options.use_direct_io_for_flush_and_compaction = true;
+ options.use_direct_reads = true;
+ options.merge_operator = std::make_shared<StringAppender>();
+ options.error_if_exists = false;
+ options.max_successive_merges = 0;
+ rocksdb::Status status = rocksdb::DB::Open(options, directory_.c_str(), &db_);
+ if (status.ok()) {
+ logger_->log_debug("NiFi Content DB Repository database open %s success", directory_.c_str());
+ is_valid_ = true;
+ } else {
+ logger_->log_error("NiFi Content DB Repository database open %s fail", directory_.c_str());
+ is_valid_ = false;
+ }
+ return is_valid_;
+}
+void DatabaseContentRepository::stop() {
+ if (db_) {
+ db_->FlushWAL(true);
+ delete db_;
+ db_ = nullptr;
+ }
+}
+
+std::shared_ptr<io::BaseStream> DatabaseContentRepository::write(const std::shared_ptr<minifi::ResourceClaim> &claim) {
+ // the traditional approach with these has been to return -1 from the stream; however, since we have the ability here
+ // we can simply return a nullptr, which is also valid from the API when this stream is not valid.
+ if (nullptr == claim || !is_valid_ || !db_)
+ return nullptr;
+ return std::make_shared<io::RocksDbStream>(claim->getContentFullPath(), db_, true);
+}
+
+std::shared_ptr<io::BaseStream> DatabaseContentRepository::read(const std::shared_ptr<minifi::ResourceClaim> &claim) {
+ // the traditional approach with these has been to return -1 from the stream; however, since we have the ability here
+ // we can simply return a nullptr, which is also valid from the API when this stream is not valid.
+ if (nullptr == claim || !is_valid_ || !db_)
+ return nullptr;
+ return std::make_shared<io::RocksDbStream>(claim->getContentFullPath(), db_, false);
+}
+
+bool DatabaseContentRepository::exists(const std::shared_ptr<minifi::ResourceClaim> &streamId) {
+ std::string value;
+ rocksdb::Status status;
+ status = db_->Get(rocksdb::ReadOptions(), streamId->getContentFullPath(), &value);
+ if (status.ok()) {
+ logger_->log_debug("%s exists", streamId->getContentFullPath());
+ return true;
+ } else {
+ logger_->log_debug("%s does not exist", streamId->getContentFullPath());
+ return false;
+ }
+}
+
+bool DatabaseContentRepository::remove(const std::shared_ptr<minifi::ResourceClaim> &claim) {
+ if (nullptr == claim || !is_valid_ || !db_)
+ return false;
+ rocksdb::Status status;
+ status = db_->Delete(rocksdb::WriteOptions(), claim->getContentFullPath());
+ if (status.ok()) {
+ logger_->log_debug("Deleted %s", claim->getContentFullPath());
+ return true;
+ } else {
+ logger_->log_debug("Attempted, but could not delete %s", claim->getContentFullPath());
+ return false;
+ }
+}
+
+} /* namespace repository */
+} /* namespace core */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/rocksdb-repos/DatabaseContentRepository.h
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/DatabaseContentRepository.h b/extensions/rocksdb-repos/DatabaseContentRepository.h
new file mode 100644
index 0000000..e43ff35
--- /dev/null
+++ b/extensions/rocksdb-repos/DatabaseContentRepository.h
@@ -0,0 +1,132 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef LIBMINIFI_INCLUDE_CORE_REPOSITORY_DatabaseContentRepository_H_
+#define LIBMINIFI_INCLUDE_CORE_REPOSITORY_DatabaseContentRepository_H_
+
+#include "rocksdb/db.h"
+#include "rocksdb/merge_operator.h"
+#include "core/Core.h"
+#include "core/Connectable.h"
+#include "../ContentRepository.h"
+#include "properties/Configure.h"
+#include "core/logging/LoggerConfiguration.h"
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace core {
+namespace repository {
+
+class StringAppender : public rocksdb::AssociativeMergeOperator {
+ public:
+ // Constructor: specify delimiter
+ explicit StringAppender() {
+
+ }
+
+ virtual bool Merge(const rocksdb::Slice& key, const rocksdb::Slice* existing_value, const rocksdb::Slice& value, std::string* new_value, rocksdb::Logger* logger) const {
+ // Clear the *new_value for writing.
+ assert(new_value);
+ new_value->clear();
+
+ if (!existing_value) {
+ // No existing_value. Set *new_value = value
+ new_value->assign(value.data(), value.size());
+ } else {
+ new_value->reserve(existing_value->size() + value.size());
+ new_value->assign(existing_value->data(), existing_value->size());
+ new_value->append(value.data(), value.size());
+ }
+
+ return true;
+ }
+
+ virtual const char* Name() const {
+ return "StringAppender";
+ }
+
+ private:
+
+};
+
+/**
+ * DatabaseContentRepository is a content repository that stores data onto the local file system.
+ */
+class DatabaseContentRepository : public core::ContentRepository, public core::Connectable {
+ public:
+
+ DatabaseContentRepository(std::string name = getClassName<DatabaseContentRepository>(), uuid_t uuid = 0)
+ : core::Connectable(name, uuid),
+ logger_(logging::LoggerFactory<DatabaseContentRepository>::getLogger()),
+ is_valid_(false),
+ db_(nullptr) {
+ }
+ virtual ~DatabaseContentRepository() {
+ stop();
+ }
+
+ virtual bool initialize(const std::shared_ptr<minifi::Configure> &configuration);
+
+ virtual void stop();
+
+ virtual std::shared_ptr<io::BaseStream> write(const std::shared_ptr<minifi::ResourceClaim> &claim);
+
+ virtual std::shared_ptr<io::BaseStream> read(const std::shared_ptr<minifi::ResourceClaim> &claim);
+
+ virtual bool close(const std::shared_ptr<minifi::ResourceClaim> &claim) {
+ return remove(claim);
+ }
+
+ virtual bool remove(const std::shared_ptr<minifi::ResourceClaim> &claim);
+
+ virtual bool exists(const std::shared_ptr<minifi::ResourceClaim> &streamId);
+
+ virtual void yield() {
+
+ }
+
+ /**
+ * Determines if we are connected and operating
+ */
+ virtual bool isRunning() {
+ return true;
+ }
+
+ /**
+ * Determines if work is available by this connectable
+ * @return boolean if work is available.
+ */
+ virtual bool isWorkAvailable() {
+ return true;
+ }
+
+ private:
+ bool is_valid_;
+ std::string directory_;
+ rocksdb::DB* db_;
+ std::shared_ptr<logging::Logger> logger_;
+};
+
+} /* namespace repository */
+} /* namespace core */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif /* LIBMINIFI_INCLUDE_CORE_REPOSITORY_DatabaseContentRepository_H_ */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/rocksdb-repos/FlowFileRepository.cpp
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/FlowFileRepository.cpp b/extensions/rocksdb-repos/FlowFileRepository.cpp
new file mode 100644
index 0000000..df2be6e
--- /dev/null
+++ b/extensions/rocksdb-repos/FlowFileRepository.cpp
@@ -0,0 +1,140 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "FlowFileRepository.h"
+#include "rocksdb/write_batch.h"
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+#include "FlowFileRecord.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace core {
+namespace repository {
+
+void FlowFileRepository::flush() {
+ rocksdb::WriteBatch batch;
+ std::string key;
+ std::string value;
+ rocksdb::ReadOptions options;
+
+ std::vector<std::shared_ptr<FlowFileRecord>> purgeList;
+
+ uint64_t decrement_total = 0;
+ while (keys_to_delete.size_approx() > 0) {
+ if (keys_to_delete.try_dequeue(key)) {
+ db_->Get(options, key, &value);
+ decrement_total += value.size();
+ std::shared_ptr<FlowFileRecord> eventRead = std::make_shared<FlowFileRecord>(shared_from_this(), content_repo_);
+ if (eventRead->DeSerialize(reinterpret_cast<const uint8_t *>(value.data()), value.size())) {
+ purgeList.push_back(eventRead);
+ }
+ logger_->log_info("Issuing batch delete, including %s, Content path %s", eventRead->getUUIDStr(), eventRead->getContentFullPath());
+ batch.Delete(key);
+ }
+ }
+ if (db_->Write(rocksdb::WriteOptions(), &batch).ok()) {
+ logger_->log_info("Decrementing %u from a repo size of %u", decrement_total, repo_size_.load());
+ if (decrement_total > repo_size_.load()) {
+ repo_size_ = 0;
+ } else {
+ repo_size_ -= decrement_total;
+ }
+ }
+
+ if (nullptr != content_repo_) {
+ for (const auto &ffr : purgeList) {
+ auto claim = ffr->getResourceClaim();
+ if (claim != nullptr) {
+ content_repo_->removeIfOrphaned(claim);
+ }
+ }
+ }
+}
+
+void FlowFileRepository::run() {
+ // threshold for purge
+ uint64_t purgeThreshold = max_partition_bytes_ * 3 / 4;
+
+ while (running_) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(purge_period_));
+ uint64_t curTime = getTimeMillis();
+
+ flush();
+
+ uint64_t size = getRepoSize();
+
+ if (size > max_partition_bytes_)
+ repo_full_ = true;
+ else
+ repo_full_ = false;
+ }
+}
+
+void FlowFileRepository::loadComponent(const std::shared_ptr<core::ContentRepository> &content_repo) {
+ content_repo_ = content_repo;
+ std::vector<std::pair<std::string, uint64_t>> purgeList;
+ rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions());
+
+ repo_size_ = 0;
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+ std::shared_ptr<FlowFileRecord> eventRead = std::make_shared<FlowFileRecord>(shared_from_this(), content_repo_);
+ std::string key = it->key().ToString();
+ repo_size_ += it->value().size();
+ if (eventRead->DeSerialize(reinterpret_cast<const uint8_t *>(it->value().data()), it->value().size())) {
+ logger_->log_info("Found connection for %s, path %s ", eventRead->getConnectionUuid(), eventRead->getContentFullPath());
+ auto search = connectionMap.find(eventRead->getConnectionUuid());
+ if (search != connectionMap.end()) {
+ // we find the connection for the persistent flowfile, create the flowfile and enqueue that
+ std::shared_ptr<core::FlowFile> flow_file_ref = std::static_pointer_cast<core::FlowFile>(eventRead);
+ eventRead->setStoredToRepository(true);
+ search->second->put(eventRead);
+ } else {
+ logger_->log_info("Could not find connectinon for %s, path %s ", eventRead->getConnectionUuid(), eventRead->getContentFullPath());
+ if (eventRead->getContentFullPath().length() > 0) {
+ if (nullptr != eventRead->getResourceClaim()) {
+ content_repo_->remove(eventRead->getResourceClaim());
+ }
+ }
+ purgeList.push_back(std::make_pair(key, it->value().size()));
+ }
+ } else {
+ purgeList.push_back(std::make_pair(key, it->value().size()));
+ }
+ }
+
+ delete it;
+ for (auto eventId : purgeList) {
+ logger_->log_info("Repository Repo %s Purge %s", name_.c_str(), eventId.first.c_str());
+ if (Delete(eventId.first)) {
+ repo_size_ -= eventId.second;
+ }
+ }
+
+ return;
+}
+
+} /* namespace repository */
+} /* namespace core */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/rocksdb-repos/FlowFileRepository.h
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/FlowFileRepository.h b/extensions/rocksdb-repos/FlowFileRepository.h
new file mode 100644
index 0000000..f473415
--- /dev/null
+++ b/extensions/rocksdb-repos/FlowFileRepository.h
@@ -0,0 +1,168 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_
+#define LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_
+
+#include "rocksdb/db.h"
+#include "rocksdb/options.h"
+#include "rocksdb/slice.h"
+#include "core/Repository.h"
+#include "core/Core.h"
+#include "Connection.h"
+#include "core/logging/LoggerConfiguration.h"
+#include "concurrentqueue.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace core {
+namespace repository {
+
+#define FLOWFILE_REPOSITORY_DIRECTORY "./flowfile_repository"
+#define MAX_FLOWFILE_REPOSITORY_STORAGE_SIZE (10*1024*1024) // 10M
+#define MAX_FLOWFILE_REPOSITORY_ENTRY_LIFE_TIME (600000) // 10 minute
+#define FLOWFILE_REPOSITORY_PURGE_PERIOD (2000) // 2000 msec
+
+/**
+ * Flow File repository
+ * Design: Extends Repository and implements the run function, using rocksdb as the primary substrate.
+ */
+class FlowFileRepository : public core::Repository, public std::enable_shared_from_this<FlowFileRepository> {
+ public:
+ // Constructor
+
+ FlowFileRepository(std::string name, uuid_t uuid)
+ : FlowFileRepository(name){
+
+ }
+
+ FlowFileRepository(const std::string repo_name = "", std::string directory = FLOWFILE_REPOSITORY_DIRECTORY, int64_t maxPartitionMillis = MAX_FLOWFILE_REPOSITORY_ENTRY_LIFE_TIME,
+ int64_t maxPartitionBytes = MAX_FLOWFILE_REPOSITORY_STORAGE_SIZE, uint64_t purgePeriod = FLOWFILE_REPOSITORY_PURGE_PERIOD)
+ : core::SerializableComponent(repo_name,0), Repository(repo_name.length() > 0 ? repo_name : core::getClassName<FlowFileRepository>(), directory, maxPartitionMillis, maxPartitionBytes, purgePeriod),
+ logger_(logging::LoggerFactory<FlowFileRepository>::getLogger()),
+ content_repo_(nullptr) {
+ db_ = NULL;
+ }
+
+ // Destructor
+ ~FlowFileRepository() {
+ if (db_)
+ delete db_;
+ }
+
+ virtual void flush();
+
+ // initialize
+ virtual bool initialize(const std::shared_ptr<Configure> &configure) {
+ std::string value;
+
+ if (configure->get(Configure::nifi_flowfile_repository_directory_default, value)) {
+ directory_ = value;
+ }
+ logger_->log_info("NiFi FlowFile Repository Directory %s", directory_.c_str());
+ if (configure->get(Configure::nifi_flowfile_repository_max_storage_size, value)) {
+ Property::StringToInt(value, max_partition_bytes_);
+ }
+ logger_->log_info("NiFi FlowFile Max Partition Bytes %d", max_partition_bytes_);
+ if (configure->get(Configure::nifi_flowfile_repository_max_storage_time, value)) {
+ TimeUnit unit;
+ if (Property::StringToTime(value, max_partition_millis_, unit) && Property::ConvertTimeUnitToMS(max_partition_millis_, unit, max_partition_millis_)) {
+ }
+ }
+ logger_->log_info("NiFi FlowFile Max Storage Time: [%d] ms", max_partition_millis_);
+ rocksdb::Options options;
+ options.create_if_missing = true;
+ options.use_direct_io_for_flush_and_compaction = true;
+ options.use_direct_reads = true;
+ rocksdb::Status status = rocksdb::DB::Open(options, directory_.c_str(), &db_);
+ if (status.ok()) {
+ logger_->log_info("NiFi FlowFile Repository database open %s success", directory_.c_str());
+ } else {
+ logger_->log_error("NiFi FlowFile Repository database open %s fail", directory_.c_str());
+ return false;
+ }
+ return true;
+ }
+
+ virtual void run();
+
+ virtual bool Put(std::string key, const uint8_t *buf, size_t bufLen) {
+ // persistent to the DB
+ rocksdb::Slice value((const char *) buf, bufLen);
+ rocksdb::Status status;
+ repo_size_ += bufLen;
+ status = db_->Put(rocksdb::WriteOptions(), key, value);
+ if (status.ok())
+ return true;
+ else
+ return false;
+ }
+ /**
+ *
+ * Deletes the key
+ * @return status of the delete operation
+ */
+ virtual bool Delete(std::string key) {
+ keys_to_delete.enqueue(key);
+ return true;
+ }
+ /**
+ * Sets the value from the provided key
+ * @return status of the get operation.
+ */
+ virtual bool Get(const std::string &key, std::string &value) {
+ if (db_ == nullptr)
+ return false;
+ rocksdb::Status status;
+ status = db_->Get(rocksdb::ReadOptions(), key, &value);
+ if (status.ok())
+ return true;
+ else
+ return false;
+ }
+
+ virtual void loadComponent(const std::shared_ptr<core::ContentRepository> &content_repo);
+
+ void start() {
+ if (this->purge_period_ <= 0) {
+ return;
+ }
+ if (running_) {
+ return;
+ }
+ running_ = true;
+ thread_ = std::thread(&FlowFileRepository::run, shared_from_this());
+ logger_->log_info("%s Repository Monitor Thread Start", getName());
+ }
+
+ private:
+ moodycamel::ConcurrentQueue<std::string> keys_to_delete;
+ std::shared_ptr<core::ContentRepository> content_repo_;
+ rocksdb::DB* db_;
+ std::shared_ptr<logging::Logger> logger_;
+};
+
+} /* namespace repository */
+} /* namespace core */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif /* LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_ */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/rocksdb-repos/ProvenanceRepository.cpp
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/ProvenanceRepository.cpp b/extensions/rocksdb-repos/ProvenanceRepository.cpp
new file mode 100644
index 0000000..4540a4a
--- /dev/null
+++ b/extensions/rocksdb-repos/ProvenanceRepository.cpp
@@ -0,0 +1,93 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "ProvenanceRepository.h"
+#include "rocksdb/write_batch.h"
+#include <string>
+#include <vector>
+#include "rocksdb/options.h"
+#include "provenance/Provenance.h"
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace provenance {
+
+void ProvenanceRepository::flush() {
+ rocksdb::WriteBatch batch;
+ std::string key;
+ std::string value;
+ rocksdb::ReadOptions options;
+ uint64_t decrement_total = 0;
+ while (keys_to_delete.size_approx() > 0) {
+ if (keys_to_delete.try_dequeue(key)) {
+ db_->Get(options, key, &value);
+ decrement_total += value.size();
+ batch.Delete(key);
+ logger_->log_info("Removing %s", key);
+ }
+ }
+ if (db_->Write(rocksdb::WriteOptions(), &batch).ok()) {
+ logger_->log_info("Decrementing %u from a repo size of %u", decrement_total, repo_size_.load());
+ if (decrement_total > repo_size_.load()) {
+ repo_size_ = 0;
+ } else {
+ repo_size_ -= decrement_total;
+ }
+ }
+}
+
+void ProvenanceRepository::run() {
+ while (running_) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(purge_period_));
+ uint64_t curTime = getTimeMillis();
+ // threshold for purge
+ uint64_t purgeThreshold = max_partition_bytes_ * 3 / 4;
+
+ uint64_t size = getRepoSize();
+
+ if (size >= purgeThreshold) {
+ rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions());
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+ ProvenanceEventRecord eventRead;
+ std::string key = it->key().ToString();
+ uint64_t eventTime = eventRead.getEventTime(reinterpret_cast<uint8_t*>(const_cast<char*>(it->value().data())), it->value().size());
+ if (eventTime > 0) {
+ if ((curTime - eventTime) > max_partition_millis_)
+ Delete(key);
+ } else {
+ logger_->log_debug("NiFi Provenance retrieve event %s fail", key.c_str());
+ Delete(key);
+ }
+ }
+ delete it;
+ }
+ flush();
+ size = getRepoSize();
+ if (size > max_partition_bytes_)
+ repo_full_ = true;
+ else
+ repo_full_ = false;
+ }
+}
+} /* namespace provenance */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/rocksdb-repos/ProvenanceRepository.h
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/ProvenanceRepository.h b/extensions/rocksdb-repos/ProvenanceRepository.h
new file mode 100644
index 0000000..67072eb
--- /dev/null
+++ b/extensions/rocksdb-repos/ProvenanceRepository.h
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_
+#define LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_
+
+#include "rocksdb/db.h"
+#include "rocksdb/options.h"
+#include "rocksdb/slice.h"
+#include "core/Repository.h"
+#include "core/Core.h"
+#include "provenance/Provenance.h"
+#include "core/logging/LoggerConfiguration.h"
+#include "concurrentqueue.h"
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace provenance {
+
+#define PROVENANCE_DIRECTORY "./provenance_repository"
+#define MAX_PROVENANCE_STORAGE_SIZE (10*1024*1024) // 10M
+#define MAX_PROVENANCE_ENTRY_LIFE_TIME (60000) // 1 minute
+#define PROVENANCE_PURGE_PERIOD (2500) // 2500 msec
+
+class ProvenanceRepository : public core::Repository, public std::enable_shared_from_this<ProvenanceRepository> {
+ public:
+
+ ProvenanceRepository(std::string name, uuid_t uuid)
+ : ProvenanceRepository(name){
+
+ }
+ // Constructor
+ /*!
+ * Create a new provenance repository
+ */
+ ProvenanceRepository(const std::string repo_name = "", std::string directory = PROVENANCE_DIRECTORY, int64_t maxPartitionMillis = MAX_PROVENANCE_ENTRY_LIFE_TIME, int64_t maxPartitionBytes =
+ MAX_PROVENANCE_STORAGE_SIZE,
+ uint64_t purgePeriod = PROVENANCE_PURGE_PERIOD)
+ : core::SerializableComponent(repo_name, 0),
+ Repository(repo_name.length() > 0 ? repo_name : core::getClassName<ProvenanceRepository>(), directory, maxPartitionMillis, maxPartitionBytes, purgePeriod),
+ logger_(logging::LoggerFactory<ProvenanceRepository>::getLogger()) {
+ db_ = NULL;
+ }
+
+ // Destructor
+ virtual ~ProvenanceRepository() {
+ if (db_)
+ delete db_;
+ }
+
+ virtual void flush();
+
+ void start() {
+ if (this->purge_period_ <= 0)
+ return;
+ if (running_)
+ return;
+ running_ = true;
+ thread_ = std::thread(&ProvenanceRepository::run, shared_from_this());
+ logger_->log_info("%s Repository Monitor Thread Start", name_);
+ }
+
+ // initialize
+ virtual bool initialize(const std::shared_ptr<org::apache::nifi::minifi::Configure> &config) {
+ std::string value;
+ if (config->get(Configure::nifi_provenance_repository_directory_default, value)) {
+ directory_ = value;
+ }
+ logger_->log_info("NiFi Provenance Repository Directory %s", directory_.c_str());
+ if (config->get(Configure::nifi_provenance_repository_max_storage_size, value)) {
+ core::Property::StringToInt(value, max_partition_bytes_);
+ }
+ logger_->log_info("NiFi Provenance Max Partition Bytes %d", max_partition_bytes_);
+ if (config->get(Configure::nifi_provenance_repository_max_storage_time, value)) {
+ core::TimeUnit unit;
+ if (core::Property::StringToTime(value, max_partition_millis_, unit) && core::Property::ConvertTimeUnitToMS(max_partition_millis_, unit, max_partition_millis_)) {
+ }
+ }
+ logger_->log_info("NiFi Provenance Max Storage Time: [%d] ms", max_partition_millis_);
+ rocksdb::Options options;
+ options.create_if_missing = true;
+ options.use_direct_io_for_flush_and_compaction = true;
+ options.use_direct_reads = true;
+ rocksdb::Status status = rocksdb::DB::Open(options, directory_.c_str(), &db_);
+ if (status.ok()) {
+ logger_->log_info("NiFi Provenance Repository database open %s success", directory_.c_str());
+ } else {
+ logger_->log_error("NiFi Provenance Repository database open %s fail", directory_.c_str());
+ return false;
+ }
+
+ return true;
+ }
+ // Put
+ virtual bool Put(std::string key, const uint8_t *buf, size_t bufLen) {
+
+ if (repo_full_) {
+ return false;
+ }
+
+ // persist to the DB
+ rocksdb::Slice value((const char *) buf, bufLen);
+ rocksdb::Status status;
+ status = db_->Put(rocksdb::WriteOptions(), key, value);
+ if (status.ok())
+ return true;
+ else
+ return false;
+ }
+ // Delete
+ virtual bool Delete(std::string key) {
+ keys_to_delete.enqueue(key);
+ return true;
+ }
+ // Get
+ virtual bool Get(const std::string &key, std::string &value) {
+ rocksdb::Status status;
+ status = db_->Get(rocksdb::ReadOptions(), key, &value);
+ if (status.ok())
+ return true;
+ else
+ return false;
+ }
+
+ // Remove event
+ void removeEvent(ProvenanceEventRecord *event) {
+ Delete(event->getEventId());
+ }
+
+ virtual bool Serialize(const std::string &key, const uint8_t *buffer, const size_t bufferSize) {
+ return Put(key, buffer, bufferSize);
+ }
+
+ virtual bool get(std::vector<std::shared_ptr<core::CoreComponent>> &store, size_t max_size) {
+ rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions());
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+ std::shared_ptr<ProvenanceEventRecord> eventRead = std::make_shared<ProvenanceEventRecord>();
+ std::string key = it->key().ToString();
+ if (store.size() >= max_size)
+ break;
+ if (eventRead->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) {
+ store.push_back(std::dynamic_pointer_cast<core::CoreComponent>(eventRead));
+ }
+ }
+ delete it;
+ return true;
+ }
+
+ virtual bool DeSerialize(std::vector<std::shared_ptr<core::SerializableComponent>> &records, size_t &max_size, std::function<std::shared_ptr<core::SerializableComponent>()> lambda) {
+ rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions());
+ size_t requested_batch = max_size;
+ max_size = 0;
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+
+ if (max_size >= requested_batch)
+ break;
+ std::shared_ptr<core::SerializableComponent> eventRead = lambda();
+ std::string key = it->key().ToString();
+ if (eventRead->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) {
+ max_size++;
+ records.push_back(eventRead);
+ }
+
+ }
+ delete it;
+
+ if (max_size > 0) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ //! get record
+ void getProvenanceRecord(std::vector<std::shared_ptr<ProvenanceEventRecord>> &records, int maxSize) {
+ rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions());
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+ std::shared_ptr<ProvenanceEventRecord> eventRead = std::make_shared<ProvenanceEventRecord>();
+ std::string key = it->key().ToString();
+ if (records.size() >= maxSize)
+ break;
+ if (eventRead->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) {
+ records.push_back(eventRead);
+ }
+ }
+ delete it;
+ }
+
+ virtual bool DeSerialize(std::vector<std::shared_ptr<core::SerializableComponent>> &store, size_t &max_size) {
+ rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions());
+ max_size = 0;
+ for (it->SeekToFirst(); it->Valid(); it->Next()) {
+ std::shared_ptr<ProvenanceEventRecord> eventRead = std::make_shared<ProvenanceEventRecord>();
+ std::string key = it->key().ToString();
+
+ if (store.at(max_size)->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) {
+ max_size++;
+ }
+ if (store.size() >= max_size)
+ break;
+ }
+ delete it;
+ if (max_size > 0) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ //! purge record
+ void purgeProvenanceRecord(std::vector<std::shared_ptr<ProvenanceEventRecord>> &records) {
+ for (auto record : records) {
+ Delete(record->getEventId());
+ }
+ flush();
+ }
+ // destroy
+ void destroy() {
+ if (db_) {
+ delete db_;
+ db_ = NULL;
+ }
+ }
+ // Run function for the thread
+ void run();
+
+ // Prevent default copy constructor and assignment operation
+ // Only support pass by reference or pointer
+ ProvenanceRepository(const ProvenanceRepository &parent) = delete;
+ ProvenanceRepository &operator=(const ProvenanceRepository &parent) = delete;
+
+ private:
+ moodycamel::ConcurrentQueue<std::string> keys_to_delete;
+ rocksdb::DB* db_;
+ std::shared_ptr<logging::Logger> logger_;
+};
+
+} /* namespace provenance */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+#endif /* LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_ */
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/rocksdb-repos/RocksDBLoader.cpp
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/RocksDBLoader.cpp b/extensions/rocksdb-repos/RocksDBLoader.cpp
new file mode 100644
index 0000000..444c1db
--- /dev/null
+++ b/extensions/rocksdb-repos/RocksDBLoader.cpp
@@ -0,0 +1,29 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "core/FlowConfiguration.h"
+#include "RocksDBLoader.h"
+
+bool RocksDBFactory::added = core::FlowConfiguration::add_static_func("createRocksDBFactory");
+
+extern "C" {
+
+void *createRocksDBFactory(void) {
+ return new RocksDBFactory();
+}
+
+}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/rocksdb-repos/RocksDBLoader.h
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/RocksDBLoader.h b/extensions/rocksdb-repos/RocksDBLoader.h
new file mode 100644
index 0000000..4b14def
--- /dev/null
+++ b/extensions/rocksdb-repos/RocksDBLoader.h
@@ -0,0 +1,80 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef EXTENSIONS_ROCKSDBLOADER_H
+#define EXTENSIONS_ROCKSDBLOADER_H
+
+#include "DatabaseContentRepository.h"
+#include "FlowFileRepository.h"
+#include "ProvenanceRepository.h"
+#include "RocksDbStream.h"
+#include "core/ClassLoader.h"
+
+class __attribute__((visibility("default"))) RocksDBFactory : public core::ObjectFactory {
+ public:
+ RocksDBFactory() {
+
+ }
+
+ /**
+ * Gets the name of the object.
+ * @return class name of processor
+ */
+ virtual std::string getName() {
+ return "RocksDBFactory";
+ }
+
+ virtual std::string getClassName() {
+ return "RocksDBFactory";
+ }
+ /**
+ * Gets the class name for the object
+ * @return class name for the processor.
+ */
+ virtual std::vector<std::string> getClassNames() {
+ std::vector<std::string> class_names;
+ class_names.push_back("DatabaseContentRepository");
+ class_names.push_back("FlowFileRepository");
+ class_names.push_back("ProvenanceRepository");
+ class_names.push_back("databasecontentrepository");
+ class_names.push_back("flowfilerepository");
+ class_names.push_back("provenancerepository");
+ return class_names;
+ }
+
+ virtual std::unique_ptr<ObjectFactory> assign(const std::string &class_name) {
+ std::string name = class_name;
+ std::transform(name.begin(), name.end(), name.begin(), ::tolower);
+ if (name == "databasecontentrepository") {
+ return std::unique_ptr<ObjectFactory>(new core::DefautObjectFactory<core::repository::DatabaseContentRepository>());
+ } else if (name == "flowfilerepository") {
+ return std::unique_ptr<ObjectFactory>(new core::DefautObjectFactory<core::repository::FlowFileRepository>());
+ } else if (name == "provenancerepository") {
+ return std::unique_ptr<ObjectFactory>(new core::DefautObjectFactory<minifi::provenance::ProvenanceRepository>());
+ } else {
+ return nullptr;
+ }
+ }
+
+ static bool added;
+
+};
+
+extern "C" {
+void *createRocksDBFactory(void);
+}
+#endif /* EXTENSIONS_ROCKSDBLOADER_H */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/rocksdb-repos/RocksDbStream.cpp
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/RocksDbStream.cpp b/extensions/rocksdb-repos/RocksDbStream.cpp
new file mode 100644
index 0000000..73f691e
--- /dev/null
+++ b/extensions/rocksdb-repos/RocksDbStream.cpp
@@ -0,0 +1,124 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "RocksDbStream.h"
+#include <fstream>
+#include <vector>
+#include <memory>
+#include <string>
+#include "io/validation.h"
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace io {
+
+RocksDbStream::RocksDbStream(const std::string &path, rocksdb::DB *db, bool write_enable)
+ : BaseStream(),
+ logger_(logging::LoggerFactory<RocksDbStream>::getLogger()),
+ db_(db),
+ path_(path),
+ write_enable_(write_enable) {
+ rocksdb::Status status;
+ status = db_->Get(rocksdb::ReadOptions(), path_, &value_);
+ if (status.ok()) {
+ exists_ = true;
+ } else {
+ exists_ = false;
+ }
+ offset_ = 0;
+ size_ = value_.size();
+}
+
+void RocksDbStream::closeStream() {
+}
+
+void RocksDbStream::seek(uint64_t offset) {
+ // noop
+}
+
+int RocksDbStream::writeData(std::vector<uint8_t> &buf, int buflen) {
+ if (buf.capacity() < buflen) {
+ return -1;
+ }
+ return writeData(reinterpret_cast<uint8_t *>(&buf[0]), buflen);
+}
+
+// data stream overrides
+
+int RocksDbStream::writeData(uint8_t *value, int size) {
+ if (!IsNullOrEmpty(value) && write_enable_) {
+ rocksdb::Slice slice_value((const char *) value, size);
+ rocksdb::Status status;
+ size_ += size;
+ rocksdb::WriteOptions opts;
+ opts.sync = true;
+ db_->Merge(opts, path_, slice_value);
+ if (status.ok()) {
+ return 0;
+ } else {
+ return -1;
+ }
+ } else {
+ return -1;
+ }
+}
+
+template<typename T>
+inline std::vector<uint8_t> RocksDbStream::readBuffer(const T& t) {
+ std::vector<uint8_t> buf;
+ buf.resize(sizeof t);
+ readData(reinterpret_cast<uint8_t *>(&buf[0]), sizeof(t));
+ return buf;
+}
+
+int RocksDbStream::readData(std::vector<uint8_t> &buf, int buflen) {
+ if (buf.capacity() < buflen) {
+ buf.resize(buflen);
+ }
+ int ret = readData(reinterpret_cast<uint8_t*>(&buf[0]), buflen);
+
+ if (ret < buflen) {
+ buf.resize(ret);
+ }
+ return ret;
+}
+
+int RocksDbStream::readData(uint8_t *buf, int buflen) {
+ if (!IsNullOrEmpty(buf) && exists_) {
+ int amtToRead = buflen;
+ if (offset_ >= value_.size()) {
+ return 0;
+ }
+ if (buflen > value_.size() - offset_) {
+ amtToRead = value_.size() - offset_;
+ }
+ std::memcpy(buf, value_.data() + offset_, amtToRead);
+ offset_ += amtToRead;
+ return amtToRead;
+ } else {
+ return -1;
+ }
+}
+
+} /* namespace io */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/extensions/rocksdb-repos/RocksDbStream.h
----------------------------------------------------------------------
diff --git a/extensions/rocksdb-repos/RocksDbStream.h b/extensions/rocksdb-repos/RocksDbStream.h
new file mode 100644
index 0000000..da08899
--- /dev/null
+++ b/extensions/rocksdb-repos/RocksDbStream.h
@@ -0,0 +1,185 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef LIBMINIFI_INCLUDE_IO_TLS_RocksDbStream_H_
+#define LIBMINIFI_INCLUDE_IO_TLS_RocksDbStream_H_
+
+#include "rocksdb/db.h"
+#include <iostream>
+#include <cstdint>
+#include <string>
+#include "io/EndianCheck.h"
+#include "io/BaseStream.h"
+#include "io/Serializable.h"
+#include "core/logging/LoggerConfiguration.h"
+
+namespace org {
+namespace apache {
+namespace nifi {
+namespace minifi {
+namespace io {
+
+/**
+ * Purpose: File Stream Base stream extension. This is intended to be a thread safe access to
+ * read/write to the local file system.
+ *
+ * Design: Simply extends BaseStream and overrides readData/writeData to allow a sink to the
+ * fstream object.
+ */
+class RocksDbStream : public io::BaseStream {
+ public:
+ /**
+ * File Stream constructor that accepts an fstream shared pointer.
+ * It must already be initialized for read and write.
+ */
+ explicit RocksDbStream(const std::string &path, rocksdb::DB *db, bool write_enable = false);
+
+ /**
+ * File Stream constructor that accepts an fstream shared pointer.
+ * It must already be initialized for read and write.
+ */
+ explicit RocksDbStream(const std::string &path);
+
+ virtual ~RocksDbStream() {
+ closeStream();
+ }
+
+ virtual void closeStream();
+ /**
+ * Skip to the specified offset.
+ * @param offset offset to which we will skip
+ */
+ void seek(uint64_t offset);
+
+ const uint32_t getSize() const {
+ return size_;
+ }
+
+ virtual int read(uint16_t &value, bool is_little_endian) {
+ uint8_t buf[2];
+ if (readData(&buf[0], 2) < 0)
+ return -1;
+ if (is_little_endian) {
+ value = (buf[0] << 8) | buf[1];
+ } else {
+ value = buf[0] | buf[1] << 8;
+ }
+ return 2;
+ }
+
+ virtual int read(uint32_t &value, bool is_little_endian) {
+ uint8_t buf[4];
+ if (readData(&buf[0], 4) < 0)
+ return -1;
+
+ if (is_little_endian) {
+ value = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3];
+ } else {
+ value = buf[0] | buf[1] << 8 | buf[2] << 16 | buf[3] << 24;
+ }
+
+ return 4;
+ }
+ virtual int read(uint64_t &value, bool is_little_endian) {
+ uint8_t buf[0];
+ if (readData(&buf[0], 8) < 0)
+ return -1;
+ if (is_little_endian) {
+ value = ((uint64_t) buf[0] << 56) | ((uint64_t) (buf[1] & 255) << 48) | ((uint64_t) (buf[2] & 255) << 40) | ((uint64_t) (buf[3] & 255) << 32) | ((uint64_t) (buf[4] & 255) << 24)
+ | ((uint64_t) (buf[5] & 255) << 16) | ((uint64_t) (buf[6] & 255) << 8) | ((uint64_t) (buf[7] & 255) << 0);
+ } else {
+ value = ((uint64_t) buf[0] << 0) | ((uint64_t) (buf[1] & 255) << 8) | ((uint64_t) (buf[2] & 255) << 16) | ((uint64_t) (buf[3] & 255) << 24) | ((uint64_t) (buf[4] & 255) << 32)
+ | ((uint64_t) (buf[5] & 255) << 40) | ((uint64_t) (buf[6] & 255) << 48) | ((uint64_t) (buf[7] & 255) << 56);
+ }
+ return 8;
+ }
+
+ // data stream extensions
+ /**
+ * Reads data and places it into buf
+ * @param buf buffer in which we extract data
+ * @param buflen
+ */
+ virtual int readData(std::vector<uint8_t> &buf, int buflen);
+ /**
+ * Reads data and places it into buf
+ * @param buf buffer in which we extract data
+ * @param buflen
+ */
+ virtual int readData(uint8_t *buf, int buflen);
+
+ /**
+ * Write value to the stream using std::vector
+ * @param buf incoming buffer
+ * @param buflen buffer to write
+ *
+ */
+ virtual int writeData(std::vector<uint8_t> &buf, int buflen);
+
+ /**
+ * writes value to stream
+ * @param value value to write
+ * @param size size of value
+ */
+ virtual int writeData(uint8_t *value, int size);
+
+ /**
+ * Returns the underlying buffer
+ * @return vector's array
+ **/
+ const uint8_t *getBuffer() const {
+ throw std::runtime_error("Stream does not support this operation");
+ }
+
+ protected:
+
+ /**
+ * Creates a vector and returns the vector using the provided
+ * type name.
+ * @param t incoming object
+ * @returns vector.
+ */
+ template<typename T>
+ std::vector<uint8_t> readBuffer(const T&);
+
+ std::string path_;
+
+ bool write_enable_;
+
+ bool exists_;
+
+ int64_t offset_;
+
+ std::string value_;
+
+ rocksdb::DB *db_;
+
+ size_t size_;
+
+ private:
+
+ std::shared_ptr<logging::Logger> logger_;
+
+};
+
+} /* namespace io */
+} /* namespace minifi */
+} /* namespace nifi */
+} /* namespace apache */
+} /* namespace org */
+
+#endif /* LIBMINIFI_INCLUDE_IO_TLS_RocksDbStream_H_ */
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt
index 22046e3..9bdab2a 100644
--- a/libminifi/CMakeLists.txt
+++ b/libminifi/CMakeLists.txt
@@ -39,8 +39,8 @@ IF (IOS)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fobjc-abi-version=2 -fobjc-arc -std=gnu++11 -stdlib=libc++ -isysroot ${CMAKE_OSX_SYSROOT} -DIOS")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fobjc-abi-version=2 -fobjc-arc -isysroot ${CMAKE_OSX_SYSROOT} -DIOS")
ELSE ()
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLEVELDB_SUPPORT -DOPENSSL_SUPPORT -DYAML_SUPPORT")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DLEVELDB_SUPPORT -DOPENSSL_SUPPORT -DYAML_SUPPORT")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOPENSSL_SUPPORT -DYAML_SUPPORT")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DOPENSSL_SUPPORT -DYAML_SUPPORT")
ENDIF()
@@ -61,6 +61,7 @@ include_directories(../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include)
include_directories(../thirdparty/civetweb-1.9.1/include)
include_directories(../thirdparty/jsoncpp/include)
include_directories(../thirdparty/concurrentqueue/)
+include_directories(../thirdparty/rocksdb/include)
include_directories(include)
file(GLOB SOURCES "src/core/logging/*.cpp" "src/core/state/*.cpp" "src/c2/protocols/*.cpp" "src/c2/*.cpp" "src/io/*.cpp" "src/io/tls/*.cpp" "src/core/controller/*.cpp" "src/controllers/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/core/yaml/*.cpp" "src/core/reporting/*.cpp" "src/provenance/*.cpp" "src/processors/*.cpp" "src/utils/*.cpp" "src/*.cpp")
@@ -90,14 +91,10 @@ find_package(Boost COMPONENTS system filesystem REQUIRED)
target_link_libraries(minifi ${Boost_SYSTEM_LIBRARY})
target_link_libraries(minifi ${Boost_FILESYSTEM_LIBRARY})
-# Include LevelDB
-find_package (Leveldb REQUIRED)
-if (LEVELDB_FOUND)
- include_directories(${LEVELDB_INCLUDE_DIRS})
- target_link_libraries (minifi ${LEVELDB_LIBRARIES})
-else ()
- message( FATAL_ERROR "LevelDB was not found. Please install LevelDB" )
-endif (LEVELDB_FOUND)
+if (CURL_FOUND)
+ include_directories(${CURL_INCLUDE_DIRS})
+ target_link_libraries (minifi ${CURL_LIBRARIES})
+endif(CURL_FOUND)
# Include OpenSSL
find_package (OpenSSL REQUIRED)
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/cmake/FindLeveldb.cmake
----------------------------------------------------------------------
diff --git a/libminifi/cmake/FindLeveldb.cmake b/libminifi/cmake/FindLeveldb.cmake
deleted file mode 100644
index 32adafa..0000000
--- a/libminifi/cmake/FindLeveldb.cmake
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-# Find module for Leveldb library and includes
-# LEVELDB_FOUND - if system found LEVELDB library
-# LEVELDB_INCLUDE_DIRS - The LEVELDB include directories
-# LEVELDB_LIBRARIES - The libraries needed to use LEVELDB
-# LEVELDB_DEFINITIONS - Compiler switches required for using LEVELDB
-
-# For OS X do not attempt to use the OS X application frameworks or bundles.
-set (CMAKE_FIND_FRAMEWORK NEVER)
-set (CMAKE_FIND_APPBUNDLE NEVER)
-
-find_path(LEVELDB_INCLUDE_DIR
- NAMES leveldb/db.h
- PATHS /usr/local/include /usr/include
- DOC "LevelDB include header"
-)
-
-find_library(LEVELDB_LIBRARY
- NAMES libleveldb.dylib libleveldb.so
- PATHS /usr/local/lib /usr/lib/x86_64-linux-gnu
- DOC "LevelDB library"
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(LEVELDB DEFAULT_MSG LEVELDB_INCLUDE_DIR LEVELDB_LIBRARY)
-
-if (LEVELDB_FOUND)
- set(LEVELDB_LIBRARIES ${LEVELDB_LIBRARY} )
- set(LEVELDB_INCLUDE_DIRS ${LEVELDB_INCLUDE_DIR} )
- set(LEVELDB_DEFINITIONS )
-endif()
-
-mark_as_advanced(LEVELDB_ROOT_DIR LEVELDB_INCLUDE_DIR LEVELDB_LIBRARY)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/cmake/FindRocksDB.cmake
----------------------------------------------------------------------
diff --git a/libminifi/cmake/FindRocksDB.cmake b/libminifi/cmake/FindRocksDB.cmake
new file mode 100644
index 0000000..db9c2d1
--- /dev/null
+++ b/libminifi/cmake/FindRocksDB.cmake
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ROCKSDB_FOUND System has RocksDB library/headers.
+# ROCKSDB_LIBRARIES The RocksDB library.
+# ROCKSDB_INCLUDE_DIR The location of RocksDB headers.
+
+find_path(ROCKSDB_ROOT_DIR
+ NAMES include/rocksdb/db.h
+)
+
+find_library(ROCKSDB_LIBRARIES
+ NAMES rocksdb
+ HINTS ${ROCKSDB_ROOT_DIR}/lib
+)
+
+find_path(ROCKSDB_INCLUDE_DIR
+ NAMES rocksdb/db.h
+ HINTS ${ROCKSDB_ROOT_DIR}/include
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(RocksDB DEFAULT_MSG
+ ROCKSDB_LIBRARIES
+ ROCKSDB_INCLUDE_DIR
+)
+
+mark_as_advanced(
+ ROCKSDB_ROOT_DIR
+ ROCKSDB_LIBRARIES
+ ROCKSDB_INCLUDE_DIR
+)
+
+if(ROCKSDB_INCLUDE_DIR AND ROCKSDB_LIBRARIES)
+ set(ROCKSDB_FOUND "YES")
+ message(STATUS "Found RocksDB...${ROCKSDB_LIBRARIES}")
+endif()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/Connection.h
----------------------------------------------------------------------
diff --git a/libminifi/include/Connection.h b/libminifi/include/Connection.h
index c92a626..e88b071 100644
--- a/libminifi/include/Connection.h
+++ b/libminifi/include/Connection.h
@@ -48,7 +48,7 @@ class Connection : public core::Connectable, public std::enable_shared_from_this
* Create a new processor
*/
explicit Connection(const std::shared_ptr<core::Repository> &flow_repository, const std::shared_ptr<core::ContentRepository> &content_repo, std::string name, uuid_t uuid = NULL, uuid_t srcUUID =
- NULL,
+ NULL,
uuid_t destUUID = NULL);
// Destructor
virtual ~Connection() {
@@ -132,6 +132,12 @@ class Connection : public core::Connectable, public std::enable_shared_from_this
uint64_t getQueueDataSize() {
return queued_data_size_;
}
+ void put(std::shared_ptr<core::Connectable> flow) {
+ std::shared_ptr<core::FlowFile> ff = std::static_pointer_cast<core::FlowFile>(flow);
+ if (nullptr != ff) {
+ put(ff);
+ }
+ }
// Put the flow file into queue
void put(std::shared_ptr<core::FlowFile> flow);
// Poll the flow file from queue, the expired flow file record also being returned
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/SchedulingAgent.h
----------------------------------------------------------------------
diff --git a/libminifi/include/SchedulingAgent.h b/libminifi/include/SchedulingAgent.h
index 569c4ee..3fc72c6 100644
--- a/libminifi/include/SchedulingAgent.h
+++ b/libminifi/include/SchedulingAgent.h
@@ -38,7 +38,6 @@
#include "core/ProcessContext.h"
#include "core/controller/ControllerServiceProvider.h"
#include "core/controller/ControllerServiceNode.h"
-#include "provenance/ProvenanceRepository.h"
namespace org {
namespace apache {
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/ClassLoader.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/ClassLoader.h b/libminifi/include/core/ClassLoader.h
index c44cb9a..d16a39b 100644
--- a/libminifi/include/core/ClassLoader.h
+++ b/libminifi/include/core/ClassLoader.h
@@ -21,7 +21,6 @@
#include <mutex>
#include <vector>
#include <map>
-#include "Connectable.h"
#include "utils/StringUtils.h"
#include <dlfcn.h>
#include "core/Core.h"
@@ -55,28 +54,28 @@ class ObjectFactory {
/**
* Create a shared pointer to a new processor.
*/
- virtual std::shared_ptr<Connectable> create(const std::string &name) {
+ virtual std::shared_ptr<CoreComponent> create(const std::string &name) {
return nullptr;
}
/**
* Create a shared pointer to a new processor.
*/
- virtual Connectable *createRaw(const std::string &name) {
+ virtual CoreComponent *createRaw(const std::string &name) {
return nullptr;
}
/**
* Create a shared pointer to a new processor.
*/
- virtual std::shared_ptr<Connectable> create(const std::string &name, uuid_t uuid) {
+ virtual std::shared_ptr<CoreComponent> create(const std::string &name, uuid_t uuid) {
return nullptr;
}
/**
* Create a shared pointer to a new processor.
*/
- virtual Connectable* createRaw(const std::string &name, uuid_t uuid) {
+ virtual CoreComponent* createRaw(const std::string &name, uuid_t uuid) {
return nullptr;
}
@@ -119,33 +118,33 @@ class DefautObjectFactory : public ObjectFactory {
/**
* Create a shared pointer to a new processor.
*/
- virtual std::shared_ptr<Connectable> create(const std::string &name) {
+ virtual std::shared_ptr<CoreComponent> create(const std::string &name) {
std::shared_ptr<T> ptr = std::make_shared<T>(name);
- return std::static_pointer_cast<Connectable>(ptr);
+ return std::static_pointer_cast<CoreComponent>(ptr);
}
/**
* Create a shared pointer to a new processor.
*/
- virtual std::shared_ptr<Connectable> create(const std::string &name, uuid_t uuid) {
+ virtual std::shared_ptr<CoreComponent> create(const std::string &name, uuid_t uuid) {
std::shared_ptr<T> ptr = std::make_shared<T>(name, uuid);
- return std::static_pointer_cast<Connectable>(ptr);
+ return std::static_pointer_cast<CoreComponent>(ptr);
}
/**
* Create a shared pointer to a new processor.
*/
- virtual Connectable* createRaw(const std::string &name) {
+ virtual CoreComponent* createRaw(const std::string &name) {
T *ptr = new T(name);
- return dynamic_cast<Connectable*>(ptr);
+ return dynamic_cast<CoreComponent*>(ptr);
}
/**
* Create a shared pointer to a new processor.
*/
- virtual Connectable* createRaw(const std::string &name, uuid_t uuid) {
+ virtual CoreComponent* createRaw(const std::string &name, uuid_t uuid) {
T *ptr = new T(name, uuid);
- return dynamic_cast<Connectable*>(ptr);
+ return dynamic_cast<CoreComponent*>(ptr);
}
/**
@@ -234,7 +233,7 @@ class ClassLoader {
* @param uuid uuid of object
* @return nullptr or object created from class_name definition.
*/
- template<class T = Connectable>
+ template<class T = CoreComponent>
std::shared_ptr<T> instantiate(const std::string &class_name, const std::string &name);
/**
@@ -243,7 +242,7 @@ class ClassLoader {
* @param uuid uuid of object
* @return nullptr or object created from class_name definition.
*/
- template<class T = Connectable>
+ template<class T = CoreComponent>
std::shared_ptr<T> instantiate(const std::string &class_name, uuid_t uuid);
/**
@@ -252,7 +251,7 @@ class ClassLoader {
* @param uuid uuid of object
* @return nullptr or object created from class_name definition.
*/
- template<class T = Connectable>
+ template<class T = CoreComponent>
T *instantiateRaw(const std::string &class_name, const std::string &name);
/**
@@ -261,7 +260,7 @@ class ClassLoader {
* @param uuid uuid of object
* @return nullptr or object created from class_name definition.
*/
- template<class T = Connectable>
+ template<class T = CoreComponent>
T *instantiateRaw(const std::string &class_name, uuid_t uuid);
protected:
@@ -282,7 +281,7 @@ std::shared_ptr<T> ClassLoader::instantiate(const std::string &class_name, const
auto factory_entry = loaded_factories_.find(class_name);
if (factory_entry != loaded_factories_.end()) {
auto obj = factory_entry->second->create(name);
- return std::static_pointer_cast<T>(obj);
+ return std::dynamic_pointer_cast<T>(obj);
} else {
return nullptr;
}
@@ -294,7 +293,7 @@ std::shared_ptr<T> ClassLoader::instantiate(const std::string &class_name, uuid_
auto factory_entry = loaded_factories_.find(class_name);
if (factory_entry != loaded_factories_.end()) {
auto obj = factory_entry->second->create(class_name, uuid);
- return std::static_pointer_cast<T>(obj);
+ return std::dynamic_pointer_cast<T>(obj);
} else {
return nullptr;
}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/Connectable.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/Connectable.h b/libminifi/include/core/Connectable.h
index 5d27901..588b67a 100644
--- a/libminifi/include/core/Connectable.h
+++ b/libminifi/include/core/Connectable.h
@@ -70,6 +70,10 @@ class __attribute__((visibility("default"))) Connectable : public CoreComponent
*/
std::set<std::shared_ptr<Connectable>> getOutGoingConnections(std::string relationship);
+ void put(std::shared_ptr<Connectable> flow) {
+
+ }
+
/**
* Get next incoming connection
* @return next incoming connection
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/ContentRepository.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/ContentRepository.h b/libminifi/include/core/ContentRepository.h
index 5558b93..54ec8d3 100644
--- a/libminifi/include/core/ContentRepository.h
+++ b/libminifi/include/core/ContentRepository.h
@@ -23,6 +23,7 @@
#include "io/DataStream.h"
#include "io/BaseStream.h"
#include "StreamManager.h"
+#include "core/Connectable.h"
namespace org {
namespace apache {
@@ -35,6 +36,7 @@ namespace core {
*/
class ContentRepository : public StreamManager<minifi::ResourceClaim> {
public:
+
virtual ~ContentRepository() {
}
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/Core.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/Core.h b/libminifi/include/core/Core.h
index 1dc79e7..80b1ca4 100644
--- a/libminifi/include/core/Core.h
+++ b/libminifi/include/core/Core.h
@@ -118,8 +118,12 @@ class CoreComponent {
uuid_copy(uuid_, other.uuid_);
}
+ virtual ~CoreComponent() {
+
+ }
+
// Get component name Name
- std::string getName();
+ virtual std::string getName();
/**
* Set name.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/FlowConfiguration.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/FlowConfiguration.h b/libminifi/include/core/FlowConfiguration.h
index 344a188..6c50772 100644
--- a/libminifi/include/core/FlowConfiguration.h
+++ b/libminifi/include/core/FlowConfiguration.h
@@ -69,9 +69,8 @@ class FlowConfiguration : public CoreComponent {
logger_(logging::LoggerFactory<FlowConfiguration>::getLogger()) {
controller_services_ = std::make_shared<core::controller::ControllerServiceMap>();
service_provider_ = std::make_shared<core::controller::StandardControllerServiceProvider>(controller_services_, nullptr, configuration);
- for(auto sl_func : statics_sl_funcs_){
- registerResource("",sl_func);
- }
+ // it is okay if this has already been called
+ initialize_static_functions();
}
virtual ~FlowConfiguration();
@@ -120,11 +119,19 @@ class FlowConfiguration : public CoreComponent {
return service_provider_;
}
- static bool add_static_func(std::string functor){
+ static bool add_static_func(std::string functor) {
statics_sl_funcs_.push_back(functor);
return true;
}
+ static void initialize_static_functions() {
+ std::lock_guard<std::mutex> lock(atomic_initialization_);
+ for (auto sl_func : statics_sl_funcs_) {
+ core::ClassLoader::getDefaultClassLoader().registerResource("", sl_func);
+ }
+ statics_sl_funcs_.clear();
+ }
+
protected:
void registerResource(const std::string &resource_function) {
@@ -151,6 +158,7 @@ class FlowConfiguration : public CoreComponent {
private:
std::shared_ptr<logging::Logger> logger_;
+ static std::mutex atomic_initialization_;
static std::vector<std::string> statics_sl_funcs_;
};
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/FlowFile.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/FlowFile.h b/libminifi/include/core/FlowFile.h
index 95b901b..8417ac9 100644
--- a/libminifi/include/core/FlowFile.h
+++ b/libminifi/include/core/FlowFile.h
@@ -28,7 +28,7 @@ namespace nifi {
namespace minifi {
namespace core {
-class FlowFile {
+class FlowFile : public core::Connectable {
public:
FlowFile();
~FlowFile();
@@ -179,11 +179,6 @@ class FlowFile {
*/
uint64_t getOffset();
- // Get the UUID as string
- std::string getUUIDStr() {
- return uuid_str_;
- }
-
bool getUUID(uuid_t other) {
uuid_copy(other, uuid_);
return true;
@@ -195,6 +190,27 @@ class FlowFile {
}
/**
+ * Yield
+ */
+ virtual void yield(){
+
+ }
+ /**
+ * Determines if we are connected and operating
+ */
+ virtual bool isRunning(){
+ return true;
+ }
+
+ /**
+ * Determines if work is available by this connectable
+ * @return boolean if work is available.
+ */
+ virtual bool isWorkAvailable(){
+ return true;
+ }
+
+ /**
* Sets the original connection with a shared pointer.
* @param connection shared connection.
*/
@@ -245,7 +261,6 @@ class FlowFile {
// Size in bytes of the data corresponding to this flow file
uint64_t size_;
// A global unique identifier
- uuid_t uuid_;
// A local unique identifier
uint64_t id_;
// Offset to the content
@@ -257,7 +272,7 @@ class FlowFile {
// Pointer to the associated content resource claim
std::shared_ptr<ResourceClaim> claim_;
// UUID string
- std::string uuid_str_;
+ //std::string uuid_str_;
// UUID string for all parents
std::set<std::string> lineage_Identifiers_;
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/ProcessGroup.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/ProcessGroup.h b/libminifi/include/core/ProcessGroup.h
index 907fdfc..93caab9 100644
--- a/libminifi/include/core/ProcessGroup.h
+++ b/libminifi/include/core/ProcessGroup.h
@@ -167,6 +167,8 @@ class ProcessGroup {
void getConnections(std::map<std::string, std::shared_ptr<Connection>> &connectionMap);
+ void getConnections(std::map<std::string, std::shared_ptr<Connectable>> &connectionMap);
+
protected:
// A global unique identifier
uuid_t uuid_;
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/Repository.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/Repository.h b/libminifi/include/core/Repository.h
index cdc81c5..427a46d 100644
--- a/libminifi/include/core/Repository.h
+++ b/libminifi/include/core/Repository.h
@@ -41,6 +41,7 @@
#include "utils/TimeUtil.h"
#include "utils/StringUtils.h"
#include "Core.h"
+#include "core/Connectable.h"
namespace org {
namespace apache {
@@ -53,7 +54,7 @@ namespace core {
#define MAX_REPOSITORY_ENTRY_LIFE_TIME (600000) // 10 minute
#define REPOSITORY_PURGE_PERIOD (2500) // 2500 msec
-class Repository : public core::SerializableComponent {
+class Repository : public virtual core::SerializableComponent {
public:
/*
* Constructor for the repository
@@ -100,6 +101,12 @@ class Repository : public core::SerializableComponent {
}
return found;
}
+
+ void setConnectionMap(std::map<std::string, std::shared_ptr<core::Connectable>> &connectionMap) {
+ this->connectionMap = connectionMap;
+ }
+
+
virtual bool Get(const std::string &key, std::string &value) {
return false;
}
@@ -204,6 +211,7 @@ class Repository : public core::SerializableComponent {
Repository &operator=(const Repository &parent) = delete;
protected:
+ std::map<std::string, std::shared_ptr<core::Connectable>> connectionMap;
// Mutex for protection
std::mutex mutex_;
// repository directory
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/libminifi/include/core/SerializableComponent.h
----------------------------------------------------------------------
diff --git a/libminifi/include/core/SerializableComponent.h b/libminifi/include/core/SerializableComponent.h
index f7f9feb..2cf13b5 100644
--- a/libminifi/include/core/SerializableComponent.h
+++ b/libminifi/include/core/SerializableComponent.h
@@ -19,6 +19,7 @@
#define LIBMINIFI_INCLUDE_CORE_SERIALIZABLECOMPONENT_H_
#include "io/Serializable.h"
+#include "core/Connectable.h"
#include "core/Core.h"
namespace org {
@@ -30,12 +31,12 @@ namespace core {
/**
* Represents a component that is serializable and an extension point of core Component
*/
-class SerializableComponent : public core::CoreComponent, public minifi::io::Serializable {
+class SerializableComponent : public core::Connectable, public minifi::io::Serializable {
public:
SerializableComponent(const std::string name, uuid_t uuid = nullptr)
- : core::CoreComponent(name, uuid) {
+ : core::Connectable(name, uuid) {
}
@@ -76,6 +77,25 @@ class SerializableComponent : public core::CoreComponent, public minifi::io::Ser
return false;
}
+ virtual void yield() {
+
+ }
+
+ /**
+ * Determines if we are connected and operating
+ */
+ virtual bool isRunning() {
+ return true;
+ }
+
+ /**
+ * Determines if work is available by this connectable
+ * @return boolean if work is available.
+ */
+ virtual bool isWorkAvailable() {
+ return true;
+ }
+
};
} /* namespace core */
[13/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/log_reader.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/log_reader.cc b/thirdparty/rocksdb/db/log_reader.cc
new file mode 100644
index 0000000..cae5d8e
--- /dev/null
+++ b/thirdparty/rocksdb/db/log_reader.cc
@@ -0,0 +1,432 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/log_reader.h"
+
+#include <stdio.h>
+#include "rocksdb/env.h"
+#include "util/coding.h"
+#include "util/crc32c.h"
+#include "util/file_reader_writer.h"
+
+namespace rocksdb {
+namespace log {
+
+Reader::Reporter::~Reporter() {
+}
+
+Reader::Reader(std::shared_ptr<Logger> info_log,
+ unique_ptr<SequentialFileReader>&& _file, Reporter* reporter,
+ bool checksum, uint64_t initial_offset, uint64_t log_num)
+ : info_log_(info_log),
+ file_(std::move(_file)),
+ reporter_(reporter),
+ checksum_(checksum),
+ backing_store_(new char[kBlockSize]),
+ buffer_(),
+ eof_(false),
+ read_error_(false),
+ eof_offset_(0),
+ last_record_offset_(0),
+ end_of_buffer_offset_(0),
+ initial_offset_(initial_offset),
+ log_number_(log_num),
+ recycled_(false) {}
+
+Reader::~Reader() {
+ delete[] backing_store_;
+}
+
+bool Reader::SkipToInitialBlock() {
+ size_t initial_offset_in_block = initial_offset_ % kBlockSize;
+ uint64_t block_start_location = initial_offset_ - initial_offset_in_block;
+
+ // Don't search a block if we'd be in the trailer
+ if (initial_offset_in_block > kBlockSize - 6) {
+ block_start_location += kBlockSize;
+ }
+
+ end_of_buffer_offset_ = block_start_location;
+
+ // Skip to start of first block that can contain the initial record
+ if (block_start_location > 0) {
+ Status skip_status = file_->Skip(block_start_location);
+ if (!skip_status.ok()) {
+ ReportDrop(static_cast<size_t>(block_start_location), skip_status);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// For kAbsoluteConsistency, on clean shutdown we don't expect any error
+// in the log files. For other modes, we can ignore only incomplete records
+// in the last log file, which are presumably due to a write in progress
+// during restart (or from log recycling).
+//
+// TODO krad: Evaluate if we need to move to a more strict mode where we
+// restrict the inconsistency to only the last log
+bool Reader::ReadRecord(Slice* record, std::string* scratch,
+ WALRecoveryMode wal_recovery_mode) {
+ if (last_record_offset_ < initial_offset_) {
+ if (!SkipToInitialBlock()) {
+ return false;
+ }
+ }
+
+ scratch->clear();
+ record->clear();
+ bool in_fragmented_record = false;
+ // Record offset of the logical record that we're reading
+ // 0 is a dummy value to make compilers happy
+ uint64_t prospective_record_offset = 0;
+
+ Slice fragment;
+ while (true) {
+ uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
+ size_t drop_size = 0;
+ const unsigned int record_type = ReadPhysicalRecord(&fragment, &drop_size);
+ switch (record_type) {
+ case kFullType:
+ case kRecyclableFullType:
+ if (in_fragmented_record && !scratch->empty()) {
+ // Handle bug in earlier versions of log::Writer where
+ // it could emit an empty kFirstType record at the tail end
+ // of a block followed by a kFullType or kFirstType record
+ // at the beginning of the next block.
+ ReportCorruption(scratch->size(), "partial record without end(1)");
+ }
+ prospective_record_offset = physical_record_offset;
+ scratch->clear();
+ *record = fragment;
+ last_record_offset_ = prospective_record_offset;
+ return true;
+
+ case kFirstType:
+ case kRecyclableFirstType:
+ if (in_fragmented_record && !scratch->empty()) {
+ // Handle bug in earlier versions of log::Writer where
+ // it could emit an empty kFirstType record at the tail end
+ // of a block followed by a kFullType or kFirstType record
+ // at the beginning of the next block.
+ ReportCorruption(scratch->size(), "partial record without end(2)");
+ }
+ prospective_record_offset = physical_record_offset;
+ scratch->assign(fragment.data(), fragment.size());
+ in_fragmented_record = true;
+ break;
+
+ case kMiddleType:
+ case kRecyclableMiddleType:
+ if (!in_fragmented_record) {
+ ReportCorruption(fragment.size(),
+ "missing start of fragmented record(1)");
+ } else {
+ scratch->append(fragment.data(), fragment.size());
+ }
+ break;
+
+ case kLastType:
+ case kRecyclableLastType:
+ if (!in_fragmented_record) {
+ ReportCorruption(fragment.size(),
+ "missing start of fragmented record(2)");
+ } else {
+ scratch->append(fragment.data(), fragment.size());
+ *record = Slice(*scratch);
+ last_record_offset_ = prospective_record_offset;
+ return true;
+ }
+ break;
+
+ case kBadHeader:
+ if (wal_recovery_mode == WALRecoveryMode::kAbsoluteConsistency) {
+ // in clean shutdown we don't expect any error in the log files
+ ReportCorruption(drop_size, "truncated header");
+ }
+ // fall-thru
+
+ case kEof:
+ if (in_fragmented_record) {
+ if (wal_recovery_mode == WALRecoveryMode::kAbsoluteConsistency) {
+ // in clean shutdown we don't expect any error in the log files
+ ReportCorruption(scratch->size(), "error reading trailing data");
+ }
+ // This can be caused by the writer dying immediately after
+ // writing a physical record but before completing the next; don't
+ // treat it as a corruption, just ignore the entire logical record.
+ scratch->clear();
+ }
+ return false;
+
+ case kOldRecord:
+ if (wal_recovery_mode != WALRecoveryMode::kSkipAnyCorruptedRecords) {
+ // Treat a record from a previous instance of the log as EOF.
+ if (in_fragmented_record) {
+ if (wal_recovery_mode == WALRecoveryMode::kAbsoluteConsistency) {
+ // in clean shutdown we don't expect any error in the log files
+ ReportCorruption(scratch->size(), "error reading trailing data");
+ }
+ // This can be caused by the writer dying immediately after
+ // writing a physical record but before completing the next; don't
+ // treat it as a corruption, just ignore the entire logical record.
+ scratch->clear();
+ }
+ return false;
+ }
+ // fall-thru
+
+ case kBadRecord:
+ if (in_fragmented_record) {
+ ReportCorruption(scratch->size(), "error in middle of record");
+ in_fragmented_record = false;
+ scratch->clear();
+ }
+ break;
+
+ case kBadRecordLen:
+ case kBadRecordChecksum:
+ if (recycled_ &&
+ wal_recovery_mode ==
+ WALRecoveryMode::kTolerateCorruptedTailRecords) {
+ scratch->clear();
+ return false;
+ }
+ if (record_type == kBadRecordLen) {
+ ReportCorruption(drop_size, "bad record length");
+ } else {
+ ReportCorruption(drop_size, "checksum mismatch");
+ }
+ if (in_fragmented_record) {
+ ReportCorruption(scratch->size(), "error in middle of record");
+ in_fragmented_record = false;
+ scratch->clear();
+ }
+ break;
+
+ default: {
+ char buf[40];
+ snprintf(buf, sizeof(buf), "unknown record type %u", record_type);
+ ReportCorruption(
+ (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
+ buf);
+ in_fragmented_record = false;
+ scratch->clear();
+ break;
+ }
+ }
+ }
+ return false;
+}
+
+uint64_t Reader::LastRecordOffset() {
+ return last_record_offset_;
+}
+
+void Reader::UnmarkEOF() {
+ if (read_error_) {
+ return;
+ }
+
+ eof_ = false;
+
+ if (eof_offset_ == 0) {
+ return;
+ }
+
+ // If the EOF was in the middle of a block (a partial block was read) we have
+ // to read the rest of the block as ReadPhysicalRecord can only read full
+ // blocks and expects the file position indicator to be aligned to the start
+ // of a block.
+ //
+ // consumed_bytes + buffer_size() + remaining == kBlockSize
+
+ size_t consumed_bytes = eof_offset_ - buffer_.size();
+ size_t remaining = kBlockSize - eof_offset_;
+
+ // backing_store_ is used to concatenate what is left in buffer_ and
+ // the remainder of the block. If buffer_ already uses backing_store_,
+ // we just append the new data.
+ if (buffer_.data() != backing_store_ + consumed_bytes) {
+ // Buffer_ does not use backing_store_ for storage.
+ // Copy what is left in buffer_ to backing_store.
+ memmove(backing_store_ + consumed_bytes, buffer_.data(), buffer_.size());
+ }
+
+ Slice read_buffer;
+ Status status = file_->Read(remaining, &read_buffer,
+ backing_store_ + eof_offset_);
+
+ size_t added = read_buffer.size();
+ end_of_buffer_offset_ += added;
+
+ if (!status.ok()) {
+ if (added > 0) {
+ ReportDrop(added, status);
+ }
+
+ read_error_ = true;
+ return;
+ }
+
+ if (read_buffer.data() != backing_store_ + eof_offset_) {
+ // Read did not write to backing_store_
+ memmove(backing_store_ + eof_offset_, read_buffer.data(),
+ read_buffer.size());
+ }
+
+ buffer_ = Slice(backing_store_ + consumed_bytes,
+ eof_offset_ + added - consumed_bytes);
+
+ if (added < remaining) {
+ eof_ = true;
+ eof_offset_ += added;
+ } else {
+ eof_offset_ = 0;
+ }
+}
+
+void Reader::ReportCorruption(size_t bytes, const char* reason) {
+ ReportDrop(bytes, Status::Corruption(reason));
+}
+
+void Reader::ReportDrop(size_t bytes, const Status& reason) {
+ if (reporter_ != nullptr &&
+ end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) {
+ reporter_->Corruption(bytes, reason);
+ }
+}
+
+bool Reader::ReadMore(size_t* drop_size, int *error) {
+ if (!eof_ && !read_error_) {
+ // Last read was a full read, so this is a trailer to skip
+ buffer_.clear();
+ Status status = file_->Read(kBlockSize, &buffer_, backing_store_);
+ end_of_buffer_offset_ += buffer_.size();
+ if (!status.ok()) {
+ buffer_.clear();
+ ReportDrop(kBlockSize, status);
+ read_error_ = true;
+ *error = kEof;
+ return false;
+ } else if (buffer_.size() < (size_t)kBlockSize) {
+ eof_ = true;
+ eof_offset_ = buffer_.size();
+ }
+ return true;
+ } else {
+ // Note that if buffer_ is non-empty, we have a truncated header at the
+ // end of the file, which can be caused by the writer crashing in the
+ // middle of writing the header. Unless explicitly requested we don't
+ // considering this an error, just report EOF.
+ if (buffer_.size()) {
+ *drop_size = buffer_.size();
+ buffer_.clear();
+ *error = kBadHeader;
+ return false;
+ }
+ buffer_.clear();
+ *error = kEof;
+ return false;
+ }
+}
+
+unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size) {
+ while (true) {
+ // We need at least the minimum header size
+ if (buffer_.size() < (size_t)kHeaderSize) {
+ int r;
+ if (!ReadMore(drop_size, &r)) {
+ return r;
+ }
+ continue;
+ }
+
+ // Parse the header
+ const char* header = buffer_.data();
+ const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
+ const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
+ const unsigned int type = header[6];
+ const uint32_t length = a | (b << 8);
+ int header_size = kHeaderSize;
+ if (type >= kRecyclableFullType && type <= kRecyclableLastType) {
+ if (end_of_buffer_offset_ - buffer_.size() == 0) {
+ recycled_ = true;
+ }
+ header_size = kRecyclableHeaderSize;
+ // We need enough for the larger header
+ if (buffer_.size() < (size_t)kRecyclableHeaderSize) {
+ int r;
+ if (!ReadMore(drop_size, &r)) {
+ return r;
+ }
+ continue;
+ }
+ const uint32_t log_num = DecodeFixed32(header + 7);
+ if (log_num != log_number_) {
+ return kOldRecord;
+ }
+ }
+ if (header_size + length > buffer_.size()) {
+ *drop_size = buffer_.size();
+ buffer_.clear();
+ if (!eof_) {
+ return kBadRecordLen;
+ }
+ // If the end of the file has been reached without reading |length| bytes
+ // of payload, assume the writer died in the middle of writing the record.
+ // Don't report a corruption unless requested.
+ if (*drop_size) {
+ return kBadHeader;
+ }
+ return kEof;
+ }
+
+ if (type == kZeroType && length == 0) {
+ // Skip zero length record without reporting any drops since
+ // such records are produced by the mmap based writing code in
+ // env_posix.cc that preallocates file regions.
+ // NOTE: this should never happen in DB written by new RocksDB versions,
+ // since we turn off mmap writes to manifest and log files
+ buffer_.clear();
+ return kBadRecord;
+ }
+
+ // Check crc
+ if (checksum_) {
+ uint32_t expected_crc = crc32c::Unmask(DecodeFixed32(header));
+ uint32_t actual_crc = crc32c::Value(header + 6, length + header_size - 6);
+ if (actual_crc != expected_crc) {
+ // Drop the rest of the buffer since "length" itself may have
+ // been corrupted and if we trust it, we could find some
+ // fragment of a real log record that just happens to look
+ // like a valid log record.
+ *drop_size = buffer_.size();
+ buffer_.clear();
+ return kBadRecordChecksum;
+ }
+ }
+
+ buffer_.remove_prefix(header_size + length);
+
+ // Skip physical record that started before initial_offset_
+ if (end_of_buffer_offset_ - buffer_.size() - header_size - length <
+ initial_offset_) {
+ result->clear();
+ return kBadRecord;
+ }
+
+ *result = Slice(header + header_size, length);
+ return type;
+ }
+}
+
+} // namespace log
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/log_reader.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/log_reader.h b/thirdparty/rocksdb/db/log_reader.h
new file mode 100644
index 0000000..c6a471c
--- /dev/null
+++ b/thirdparty/rocksdb/db/log_reader.h
@@ -0,0 +1,160 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <memory>
+#include <stdint.h>
+
+#include "db/log_format.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
+#include "rocksdb/options.h"
+
+namespace rocksdb {
+
+class SequentialFileReader;
+class Logger;
+using std::unique_ptr;
+
+namespace log {
+
+/**
+ * Reader is a general purpose log stream reader implementation. The actual job
+ * of reading from the device is implemented by the SequentialFile interface.
+ *
+ * Please see Writer for details on the file and record layout.
+ */
+class Reader {
+ public:
+ // Interface for reporting errors.
+ class Reporter {
+ public:
+ virtual ~Reporter();
+
+ // Some corruption was detected. "size" is the approximate number
+ // of bytes dropped due to the corruption.
+ virtual void Corruption(size_t bytes, const Status& status) = 0;
+ };
+
+ // Create a reader that will return log records from "*file".
+ // "*file" must remain live while this Reader is in use.
+ //
+ // If "reporter" is non-nullptr, it is notified whenever some data is
+ // dropped due to a detected corruption. "*reporter" must remain
+ // live while this Reader is in use.
+ //
+ // If "checksum" is true, verify checksums if available.
+ //
+ // The Reader will start reading at the first record located at physical
+ // position >= initial_offset within the file.
+ Reader(std::shared_ptr<Logger> info_log,
+ unique_ptr<SequentialFileReader>&& file,
+ Reporter* reporter, bool checksum, uint64_t initial_offset,
+ uint64_t log_num);
+
+ ~Reader();
+
+ // Read the next record into *record. Returns true if read
+ // successfully, false if we hit end of the input. May use
+ // "*scratch" as temporary storage. The contents filled in *record
+ // will only be valid until the next mutating operation on this
+ // reader or the next mutation to *scratch.
+ bool ReadRecord(Slice* record, std::string* scratch,
+ WALRecoveryMode wal_recovery_mode =
+ WALRecoveryMode::kTolerateCorruptedTailRecords);
+
+ // Returns the physical offset of the last record returned by ReadRecord.
+ //
+ // Undefined before the first call to ReadRecord.
+ uint64_t LastRecordOffset();
+
+ // returns true if the reader has encountered an eof condition.
+ bool IsEOF() {
+ return eof_;
+ }
+
+ // when we know more data has been written to the file. we can use this
+ // function to force the reader to look again in the file.
+ // Also aligns the file position indicator to the start of the next block
+ // by reading the rest of the data from the EOF position to the end of the
+ // block that was partially read.
+ void UnmarkEOF();
+
+ SequentialFileReader* file() { return file_.get(); }
+
+ private:
+ std::shared_ptr<Logger> info_log_;
+ const unique_ptr<SequentialFileReader> file_;
+ Reporter* const reporter_;
+ bool const checksum_;
+ char* const backing_store_;
+ Slice buffer_;
+ bool eof_; // Last Read() indicated EOF by returning < kBlockSize
+ bool read_error_; // Error occurred while reading from file
+
+ // Offset of the file position indicator within the last block when an
+ // EOF was detected.
+ size_t eof_offset_;
+
+ // Offset of the last record returned by ReadRecord.
+ uint64_t last_record_offset_;
+ // Offset of the first location past the end of buffer_.
+ uint64_t end_of_buffer_offset_;
+
+ // Offset at which to start looking for the first record to return
+ uint64_t const initial_offset_;
+
+ // which log number this is
+ uint64_t const log_number_;
+
+ // Whether this is a recycled log file
+ bool recycled_;
+
+ // Extend record types with the following special values
+ enum {
+ kEof = kMaxRecordType + 1,
+ // Returned whenever we find an invalid physical record.
+ // Currently there are three situations in which this happens:
+ // * The record has an invalid CRC (ReadPhysicalRecord reports a drop)
+ // * The record is a 0-length record (No drop is reported)
+ // * The record is below constructor's initial_offset (No drop is reported)
+ kBadRecord = kMaxRecordType + 2,
+ // Returned when we fail to read a valid header.
+ kBadHeader = kMaxRecordType + 3,
+ // Returned when we read an old record from a previous user of the log.
+ kOldRecord = kMaxRecordType + 4,
+ // Returned when we get a bad record length
+ kBadRecordLen = kMaxRecordType + 5,
+ // Returned when we get a bad record checksum
+ kBadRecordChecksum = kMaxRecordType + 6,
+ };
+
+ // Skips all blocks that are completely before "initial_offset_".
+ //
+ // Returns true on success. Handles reporting.
+ bool SkipToInitialBlock();
+
+ // Return type, or one of the preceding special values
+ unsigned int ReadPhysicalRecord(Slice* result, size_t* drop_size);
+
+ // Read some more
+ bool ReadMore(size_t* drop_size, int *error);
+
+ // Reports dropped bytes to the reporter.
+ // buffer_ must be updated to remove the dropped bytes prior to invocation.
+ void ReportCorruption(size_t bytes, const char* reason);
+ void ReportDrop(size_t bytes, const Status& reason);
+
+ // No copying allowed
+ Reader(const Reader&);
+ void operator=(const Reader&);
+};
+
+} // namespace log
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/log_writer.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/log_writer.cc b/thirdparty/rocksdb/db/log_writer.cc
new file mode 100644
index 0000000..b02eec8
--- /dev/null
+++ b/thirdparty/rocksdb/db/log_writer.cc
@@ -0,0 +1,142 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/log_writer.h"
+
+#include <stdint.h>
+#include "rocksdb/env.h"
+#include "util/coding.h"
+#include "util/crc32c.h"
+#include "util/file_reader_writer.h"
+
+namespace rocksdb {
+namespace log {
+
+Writer::Writer(unique_ptr<WritableFileWriter>&& dest, uint64_t log_number,
+ bool recycle_log_files, bool manual_flush)
+ : dest_(std::move(dest)),
+ block_offset_(0),
+ log_number_(log_number),
+ recycle_log_files_(recycle_log_files),
+ manual_flush_(manual_flush) {
+ for (int i = 0; i <= kMaxRecordType; i++) {
+ char t = static_cast<char>(i);
+ type_crc_[i] = crc32c::Value(&t, 1);
+ }
+}
+
+Writer::~Writer() { WriteBuffer(); }
+
+Status Writer::WriteBuffer() { return dest_->Flush(); }
+
+Status Writer::AddRecord(const Slice& slice) {
+ const char* ptr = slice.data();
+ size_t left = slice.size();
+
+ // Header size varies depending on whether we are recycling or not.
+ const int header_size =
+ recycle_log_files_ ? kRecyclableHeaderSize : kHeaderSize;
+
+ // Fragment the record if necessary and emit it. Note that if slice
+ // is empty, we still want to iterate once to emit a single
+ // zero-length record
+ Status s;
+ bool begin = true;
+ do {
+ const int64_t leftover = kBlockSize - block_offset_;
+ assert(leftover >= 0);
+ if (leftover < header_size) {
+ // Switch to a new block
+ if (leftover > 0) {
+ // Fill the trailer (literal below relies on kHeaderSize and
+ // kRecyclableHeaderSize being <= 11)
+ assert(header_size <= 11);
+ dest_->Append(
+ Slice("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", leftover));
+ }
+ block_offset_ = 0;
+ }
+
+ // Invariant: we never leave < header_size bytes in a block.
+ assert(static_cast<int64_t>(kBlockSize - block_offset_) >= header_size);
+
+ const size_t avail = kBlockSize - block_offset_ - header_size;
+ const size_t fragment_length = (left < avail) ? left : avail;
+
+ RecordType type;
+ const bool end = (left == fragment_length);
+ if (begin && end) {
+ type = recycle_log_files_ ? kRecyclableFullType : kFullType;
+ } else if (begin) {
+ type = recycle_log_files_ ? kRecyclableFirstType : kFirstType;
+ } else if (end) {
+ type = recycle_log_files_ ? kRecyclableLastType : kLastType;
+ } else {
+ type = recycle_log_files_ ? kRecyclableMiddleType : kMiddleType;
+ }
+
+ s = EmitPhysicalRecord(type, ptr, fragment_length);
+ ptr += fragment_length;
+ left -= fragment_length;
+ begin = false;
+ } while (s.ok() && left > 0);
+ return s;
+}
+
+Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
+ assert(n <= 0xffff); // Must fit in two bytes
+
+ size_t header_size;
+ char buf[kRecyclableHeaderSize];
+
+ // Format the header
+ buf[4] = static_cast<char>(n & 0xff);
+ buf[5] = static_cast<char>(n >> 8);
+ buf[6] = static_cast<char>(t);
+
+ uint32_t crc = type_crc_[t];
+ if (t < kRecyclableFullType) {
+ // Legacy record format
+ assert(block_offset_ + kHeaderSize + n <= kBlockSize);
+ header_size = kHeaderSize;
+ } else {
+ // Recyclable record format
+ assert(block_offset_ + kRecyclableHeaderSize + n <= kBlockSize);
+ header_size = kRecyclableHeaderSize;
+
+ // Only encode low 32-bits of the 64-bit log number. This means
+ // we will fail to detect an old record if we recycled a log from
+ // ~4 billion logs ago, but that is effectively impossible, and
+ // even if it were we'dbe far more likely to see a false positive
+ // on the 32-bit CRC.
+ EncodeFixed32(buf + 7, static_cast<uint32_t>(log_number_));
+ crc = crc32c::Extend(crc, buf + 7, 4);
+ }
+
+ // Compute the crc of the record type and the payload.
+ crc = crc32c::Extend(crc, ptr, n);
+ crc = crc32c::Mask(crc); // Adjust for storage
+ EncodeFixed32(buf, crc);
+
+ // Write the header and the payload
+ Status s = dest_->Append(Slice(buf, header_size));
+ if (s.ok()) {
+ s = dest_->Append(Slice(ptr, n));
+ if (s.ok()) {
+ if (!manual_flush_) {
+ s = dest_->Flush();
+ }
+ }
+ }
+ block_offset_ += header_size + n;
+ return s;
+}
+
+} // namespace log
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/log_writer.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/log_writer.h b/thirdparty/rocksdb/db/log_writer.h
new file mode 100644
index 0000000..a3a8799
--- /dev/null
+++ b/thirdparty/rocksdb/db/log_writer.h
@@ -0,0 +1,111 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#pragma once
+
+#include <stdint.h>
+
+#include <memory>
+
+#include "db/log_format.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
+
+namespace rocksdb {
+
+class WritableFileWriter;
+
+using std::unique_ptr;
+
+namespace log {
+
+/**
+ * Writer is a general purpose log stream writer. It provides an append-only
+ * abstraction for writing data. The details of the how the data is written is
+ * handled by the WriteableFile sub-class implementation.
+ *
+ * File format:
+ *
+ * File is broken down into variable sized records. The format of each record
+ * is described below.
+ * +-----+-------------+--+----+----------+------+-- ... ----+
+ * File | r0 | r1 |P | r2 | r3 | r4 | |
+ * +-----+-------------+--+----+----------+------+-- ... ----+
+ * <--- kBlockSize ------>|<-- kBlockSize ------>|
+ * rn = variable size records
+ * P = Padding
+ *
+ * Data is written out in kBlockSize chunks. If next record does not fit
+ * into the space left, the leftover space will be padded with \0.
+ *
+ * Legacy record format:
+ *
+ * +---------+-----------+-----------+--- ... ---+
+ * |CRC (4B) | Size (2B) | Type (1B) | Payload |
+ * +---------+-----------+-----------+--- ... ---+
+ *
+ * CRC = 32bit hash computed over the payload using CRC
+ * Size = Length of the payload data
+ * Type = Type of record
+ * (kZeroType, kFullType, kFirstType, kLastType, kMiddleType )
+ * The type is used to group a bunch of records together to represent
+ * blocks that are larger than kBlockSize
+ * Payload = Byte stream as long as specified by the payload size
+ *
+ * Recyclable record format:
+ *
+ * +---------+-----------+-----------+----------------+--- ... ---+
+ * |CRC (4B) | Size (2B) | Type (1B) | Log number (4B)| Payload |
+ * +---------+-----------+-----------+----------------+--- ... ---+
+ *
+ * Same as above, with the addition of
+ * Log number = 32bit log file number, so that we can distinguish between
+ * records written by the most recent log writer vs a previous one.
+ */
+class Writer {
+ public:
+ // Create a writer that will append data to "*dest".
+ // "*dest" must be initially empty.
+ // "*dest" must remain live while this Writer is in use.
+ explicit Writer(unique_ptr<WritableFileWriter>&& dest, uint64_t log_number,
+ bool recycle_log_files, bool manual_flush = false);
+ ~Writer();
+
+ Status AddRecord(const Slice& slice);
+
+ WritableFileWriter* file() { return dest_.get(); }
+ const WritableFileWriter* file() const { return dest_.get(); }
+
+ uint64_t get_log_number() const { return log_number_; }
+
+ Status WriteBuffer();
+
+ private:
+ unique_ptr<WritableFileWriter> dest_;
+ size_t block_offset_; // Current offset in block
+ uint64_t log_number_;
+ bool recycle_log_files_;
+
+ // crc32c values for all supported record types. These are
+ // pre-computed to reduce the overhead of computing the crc of the
+ // record type stored in the header.
+ uint32_t type_crc_[kMaxRecordType + 1];
+
+ Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
+
+ // If true, it does not flush after each write. Instead it relies on the upper
+ // layer to manually does the flush by calling ::WriteBuffer()
+ bool manual_flush_;
+
+ // No copying allowed
+ Writer(const Writer&);
+ void operator=(const Writer&);
+};
+
+} // namespace log
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/malloc_stats.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/malloc_stats.cc b/thirdparty/rocksdb/db/malloc_stats.cc
new file mode 100644
index 0000000..7acca65
--- /dev/null
+++ b/thirdparty/rocksdb/db/malloc_stats.cc
@@ -0,0 +1,52 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/malloc_stats.h"
+
+#ifndef ROCKSDB_LITE
+#include <memory>
+#include <string.h>
+
+namespace rocksdb {
+
+#ifdef ROCKSDB_JEMALLOC
+#include "jemalloc/jemalloc.h"
+
+typedef struct {
+ char* cur;
+ char* end;
+} MallocStatus;
+
+static void GetJemallocStatus(void* mstat_arg, const char* status) {
+ MallocStatus* mstat = reinterpret_cast<MallocStatus*>(mstat_arg);
+ size_t status_len = status ? strlen(status) : 0;
+ size_t buf_size = (size_t)(mstat->end - mstat->cur);
+ if (!status_len || status_len > buf_size) {
+ return;
+ }
+
+ snprintf(mstat->cur, buf_size, "%s", status);
+ mstat->cur += status_len;
+}
+#endif // ROCKSDB_JEMALLOC
+
+void DumpMallocStats(std::string* stats) {
+#ifdef ROCKSDB_JEMALLOC
+ MallocStatus mstat;
+ const unsigned int kMallocStatusLen = 1000000;
+ std::unique_ptr<char[]> buf{new char[kMallocStatusLen + 1]};
+ mstat.cur = buf.get();
+ mstat.end = buf.get() + kMallocStatusLen;
+ je_malloc_stats_print(GetJemallocStatus, &mstat, "");
+ stats->append(buf.get());
+#endif // ROCKSDB_JEMALLOC
+}
+
+}
+#endif // !ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/malloc_stats.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/malloc_stats.h b/thirdparty/rocksdb/db/malloc_stats.h
new file mode 100644
index 0000000..a2f324f
--- /dev/null
+++ b/thirdparty/rocksdb/db/malloc_stats.h
@@ -0,0 +1,22 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#ifndef ROCKSDB_LITE
+
+#include <string>
+
+namespace rocksdb {
+
+void DumpMallocStats(std::string*);
+
+}
+
+#endif // !ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/managed_iterator.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/managed_iterator.cc b/thirdparty/rocksdb/db/managed_iterator.cc
new file mode 100644
index 0000000..c393eb5
--- /dev/null
+++ b/thirdparty/rocksdb/db/managed_iterator.cc
@@ -0,0 +1,262 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#ifndef ROCKSDB_LITE
+
+#include "db/managed_iterator.h"
+
+#include <limits>
+#include <string>
+#include <utility>
+
+#include "db/column_family.h"
+#include "db/db_impl.h"
+#include "db/db_iter.h"
+#include "db/dbformat.h"
+#include "rocksdb/env.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/slice_transform.h"
+#include "table/merging_iterator.h"
+
+namespace rocksdb {
+
+namespace {
+// Helper class that locks a mutex on construction and unlocks the mutex when
+// the destructor of the MutexLock object is invoked.
+//
+// Typical usage:
+//
+// void MyClass::MyMethod() {
+// MILock l(&mu_); // mu_ is an instance variable
+// ... some complex code, possibly with multiple return paths ...
+// }
+
+class MILock {
+ public:
+ explicit MILock(std::mutex* mu, ManagedIterator* mi) : mu_(mu), mi_(mi) {
+ this->mu_->lock();
+ }
+ ~MILock() {
+ this->mu_->unlock();
+ }
+ ManagedIterator* GetManagedIterator() { return mi_; }
+
+ private:
+ std::mutex* const mu_;
+ ManagedIterator* mi_;
+ // No copying allowed
+ MILock(const MILock&) = delete;
+ void operator=(const MILock&) = delete;
+};
+} // anonymous namespace
+
+//
+// Synchronization between modifiers, releasers, creators
+// If iterator operation, wait till (!in_use), set in_use, do op, reset in_use
+// if modifying mutable_iter, atomically exchange in_use:
+// return if in_use set / otherwise set in use,
+// atomically replace new iter with old , reset in use
+// The releaser is the new operation and it holds a lock for a very short time
+// The existing non-const iterator operations are supposed to be single
+// threaded and hold the lock for the duration of the operation
+// The existing const iterator operations use the cached key/values
+// and don't do any locking.
+ManagedIterator::ManagedIterator(DBImpl* db, const ReadOptions& read_options,
+ ColumnFamilyData* cfd)
+ : db_(db),
+ read_options_(read_options),
+ cfd_(cfd),
+ svnum_(cfd->GetSuperVersionNumber()),
+ mutable_iter_(nullptr),
+ valid_(false),
+ snapshot_created_(false),
+ release_supported_(true) {
+ read_options_.managed = false;
+ if ((!read_options_.tailing) && (read_options_.snapshot == nullptr)) {
+ assert(nullptr != (read_options_.snapshot = db_->GetSnapshot()));
+ snapshot_created_ = true;
+ }
+ cfh_.SetCFD(cfd);
+ mutable_iter_ = unique_ptr<Iterator>(db->NewIterator(read_options_, &cfh_));
+}
+
+ManagedIterator::~ManagedIterator() {
+ Lock();
+ if (snapshot_created_) {
+ db_->ReleaseSnapshot(read_options_.snapshot);
+ snapshot_created_ = false;
+ read_options_.snapshot = nullptr;
+ }
+ UnLock();
+}
+
+bool ManagedIterator::Valid() const { return valid_; }
+
+void ManagedIterator::SeekToLast() {
+ MILock l(&in_use_, this);
+ if (NeedToRebuild()) {
+ RebuildIterator();
+ }
+ assert(mutable_iter_ != nullptr);
+ mutable_iter_->SeekToLast();
+ if (mutable_iter_->status().ok()) {
+ UpdateCurrent();
+ }
+}
+
+void ManagedIterator::SeekToFirst() {
+ MILock l(&in_use_, this);
+ SeekInternal(Slice(), true);
+}
+
+void ManagedIterator::Seek(const Slice& user_key) {
+ MILock l(&in_use_, this);
+ SeekInternal(user_key, false);
+}
+
+void ManagedIterator::SeekForPrev(const Slice& user_key) {
+ MILock l(&in_use_, this);
+ if (NeedToRebuild()) {
+ RebuildIterator();
+ }
+ assert(mutable_iter_ != nullptr);
+ mutable_iter_->SeekForPrev(user_key);
+ UpdateCurrent();
+}
+
+void ManagedIterator::SeekInternal(const Slice& user_key, bool seek_to_first) {
+ if (NeedToRebuild()) {
+ RebuildIterator();
+ }
+ assert(mutable_iter_ != nullptr);
+ if (seek_to_first) {
+ mutable_iter_->SeekToFirst();
+ } else {
+ mutable_iter_->Seek(user_key);
+ }
+ UpdateCurrent();
+}
+
+void ManagedIterator::Prev() {
+ if (!valid_) {
+ status_ = Status::InvalidArgument("Iterator value invalid");
+ return;
+ }
+ MILock l(&in_use_, this);
+ if (NeedToRebuild()) {
+ std::string current_key = key().ToString();
+ Slice old_key(current_key);
+ RebuildIterator();
+ SeekInternal(old_key, false);
+ UpdateCurrent();
+ if (!valid_) {
+ return;
+ }
+ if (key().compare(old_key) != 0) {
+ valid_ = false;
+ status_ = Status::Incomplete("Cannot do Prev now");
+ return;
+ }
+ }
+ mutable_iter_->Prev();
+ if (mutable_iter_->status().ok()) {
+ UpdateCurrent();
+ status_ = Status::OK();
+ } else {
+ status_ = mutable_iter_->status();
+ }
+}
+
+void ManagedIterator::Next() {
+ if (!valid_) {
+ status_ = Status::InvalidArgument("Iterator value invalid");
+ return;
+ }
+ MILock l(&in_use_, this);
+ if (NeedToRebuild()) {
+ std::string current_key = key().ToString();
+ Slice old_key(current_key.data(), cached_key_.Size());
+ RebuildIterator();
+ SeekInternal(old_key, false);
+ UpdateCurrent();
+ if (!valid_) {
+ return;
+ }
+ if (key().compare(old_key) != 0) {
+ valid_ = false;
+ status_ = Status::Incomplete("Cannot do Next now");
+ return;
+ }
+ }
+ mutable_iter_->Next();
+ UpdateCurrent();
+}
+
+Slice ManagedIterator::key() const {
+ assert(valid_);
+ return cached_key_.GetUserKey();
+}
+
+Slice ManagedIterator::value() const {
+ assert(valid_);
+ return cached_value_.GetUserKey();
+}
+
+Status ManagedIterator::status() const { return status_; }
+
+void ManagedIterator::RebuildIterator() {
+ svnum_ = cfd_->GetSuperVersionNumber();
+ mutable_iter_ = unique_ptr<Iterator>(db_->NewIterator(read_options_, &cfh_));
+}
+
+void ManagedIterator::UpdateCurrent() {
+ assert(mutable_iter_ != nullptr);
+
+ valid_ = mutable_iter_->Valid();
+ if (!valid_) {
+ status_ = mutable_iter_->status();
+ return;
+ }
+
+ status_ = Status::OK();
+ cached_key_.SetUserKey(mutable_iter_->key());
+ cached_value_.SetUserKey(mutable_iter_->value());
+}
+
+void ManagedIterator::ReleaseIter(bool only_old) {
+ if ((mutable_iter_ == nullptr) || (!release_supported_)) {
+ return;
+ }
+ if (svnum_ != cfd_->GetSuperVersionNumber() || !only_old) {
+ if (!TryLock()) { // Don't release iter if in use
+ return;
+ }
+ mutable_iter_ = nullptr; // in_use for a very short time
+ UnLock();
+ }
+}
+
+bool ManagedIterator::NeedToRebuild() {
+ if ((mutable_iter_ == nullptr) || (status_.IsIncomplete()) ||
+ (!only_drop_old_ && (svnum_ != cfd_->GetSuperVersionNumber()))) {
+ return true;
+ }
+ return false;
+}
+
+void ManagedIterator::Lock() {
+ in_use_.lock();
+ return;
+}
+
+bool ManagedIterator::TryLock() { return in_use_.try_lock(); }
+
+void ManagedIterator::UnLock() {
+ in_use_.unlock();
+}
+
+} // namespace rocksdb
+
+#endif // ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/managed_iterator.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/managed_iterator.h b/thirdparty/rocksdb/db/managed_iterator.h
new file mode 100644
index 0000000..8e962f7
--- /dev/null
+++ b/thirdparty/rocksdb/db/managed_iterator.h
@@ -0,0 +1,85 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+#pragma once
+
+#ifndef ROCKSDB_LITE
+
+#include <mutex>
+#include <queue>
+#include <string>
+#include <vector>
+
+#include "db/column_family.h"
+#include "rocksdb/db.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/options.h"
+#include "util/arena.h"
+
+namespace rocksdb {
+
+class DBImpl;
+struct SuperVersion;
+class ColumnFamilyData;
+
+/**
+ * ManagedIterator is a special type of iterator that supports freeing the
+ * underlying iterator and still being able to access the current key/value
+ * pair. This is done by copying the key/value pair so that clients can
+ * continue to access the data without getting a SIGSEGV.
+ * The underlying iterator can be freed manually through the call to
+ * ReleaseIter or automatically (as needed on space pressure or age.)
+ * The iterator is recreated using the saved original arguments.
+ */
+class ManagedIterator : public Iterator {
+ public:
+ ManagedIterator(DBImpl* db, const ReadOptions& read_options,
+ ColumnFamilyData* cfd);
+ virtual ~ManagedIterator();
+
+ virtual void SeekToLast() override;
+ virtual void Prev() override;
+ virtual bool Valid() const override;
+ void SeekToFirst() override;
+ virtual void Seek(const Slice& target) override;
+ virtual void SeekForPrev(const Slice& target) override;
+ virtual void Next() override;
+ virtual Slice key() const override;
+ virtual Slice value() const override;
+ virtual Status status() const override;
+ void ReleaseIter(bool only_old);
+ void SetDropOld(bool only_old) {
+ only_drop_old_ = read_options_.tailing || only_old;
+ }
+
+ private:
+ void RebuildIterator();
+ void UpdateCurrent();
+ void SeekInternal(const Slice& user_key, bool seek_to_first);
+ bool NeedToRebuild();
+ void Lock();
+ bool TryLock();
+ void UnLock();
+ DBImpl* const db_;
+ ReadOptions read_options_;
+ ColumnFamilyData* const cfd_;
+ ColumnFamilyHandleInternal cfh_;
+
+ uint64_t svnum_;
+ std::unique_ptr<Iterator> mutable_iter_;
+ // internal iterator status
+ Status status_;
+ bool valid_;
+
+ IterKey cached_key_;
+ IterKey cached_value_;
+
+ bool only_drop_old_ = true;
+ bool snapshot_created_;
+ bool release_supported_;
+ std::mutex in_use_; // is managed iterator in use
+};
+
+} // namespace rocksdb
+#endif // !ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/memtable.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/memtable.cc b/thirdparty/rocksdb/db/memtable.cc
new file mode 100644
index 0000000..efea619
--- /dev/null
+++ b/thirdparty/rocksdb/db/memtable.cc
@@ -0,0 +1,885 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/memtable.h"
+
+#include <memory>
+#include <algorithm>
+
+#include "db/dbformat.h"
+#include "db/merge_context.h"
+#include "db/merge_helper.h"
+#include "db/pinned_iterators_manager.h"
+#include "monitoring/perf_context_imp.h"
+#include "monitoring/statistics.h"
+#include "port/port.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/env.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/slice_transform.h"
+#include "rocksdb/write_buffer_manager.h"
+#include "table/internal_iterator.h"
+#include "table/iterator_wrapper.h"
+#include "table/merging_iterator.h"
+#include "util/arena.h"
+#include "util/autovector.h"
+#include "util/coding.h"
+#include "util/memory_usage.h"
+#include "util/murmurhash.h"
+#include "util/mutexlock.h"
+
+namespace rocksdb {
+
+MemTableOptions::MemTableOptions(const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& mutable_cf_options)
+ : write_buffer_size(mutable_cf_options.write_buffer_size),
+ arena_block_size(mutable_cf_options.arena_block_size),
+ memtable_prefix_bloom_bits(
+ static_cast<uint32_t>(
+ static_cast<double>(mutable_cf_options.write_buffer_size) *
+ mutable_cf_options.memtable_prefix_bloom_size_ratio) *
+ 8u),
+ memtable_huge_page_size(mutable_cf_options.memtable_huge_page_size),
+ inplace_update_support(ioptions.inplace_update_support),
+ inplace_update_num_locks(mutable_cf_options.inplace_update_num_locks),
+ inplace_callback(ioptions.inplace_callback),
+ max_successive_merges(mutable_cf_options.max_successive_merges),
+ statistics(ioptions.statistics),
+ merge_operator(ioptions.merge_operator),
+ info_log(ioptions.info_log) {}
+
+MemTable::MemTable(const InternalKeyComparator& cmp,
+ const ImmutableCFOptions& ioptions,
+ const MutableCFOptions& mutable_cf_options,
+ WriteBufferManager* write_buffer_manager,
+ SequenceNumber latest_seq, uint32_t column_family_id)
+ : comparator_(cmp),
+ moptions_(ioptions, mutable_cf_options),
+ refs_(0),
+ kArenaBlockSize(OptimizeBlockSize(moptions_.arena_block_size)),
+ mem_tracker_(write_buffer_manager),
+ arena_(
+ moptions_.arena_block_size,
+ (write_buffer_manager != nullptr && write_buffer_manager->enabled())
+ ? &mem_tracker_
+ : nullptr,
+ mutable_cf_options.memtable_huge_page_size),
+ table_(ioptions.memtable_factory->CreateMemTableRep(
+ comparator_, &arena_, ioptions.prefix_extractor, ioptions.info_log,
+ column_family_id)),
+ range_del_table_(SkipListFactory().CreateMemTableRep(
+ comparator_, &arena_, nullptr /* transform */, ioptions.info_log,
+ column_family_id)),
+ is_range_del_table_empty_(true),
+ data_size_(0),
+ num_entries_(0),
+ num_deletes_(0),
+ flush_in_progress_(false),
+ flush_completed_(false),
+ file_number_(0),
+ first_seqno_(0),
+ earliest_seqno_(latest_seq),
+ creation_seq_(latest_seq),
+ mem_next_logfile_number_(0),
+ min_prep_log_referenced_(0),
+ locks_(moptions_.inplace_update_support
+ ? moptions_.inplace_update_num_locks
+ : 0),
+ prefix_extractor_(ioptions.prefix_extractor),
+ flush_state_(FLUSH_NOT_REQUESTED),
+ env_(ioptions.env),
+ insert_with_hint_prefix_extractor_(
+ ioptions.memtable_insert_with_hint_prefix_extractor) {
+ UpdateFlushState();
+ // something went wrong if we need to flush before inserting anything
+ assert(!ShouldScheduleFlush());
+
+ if (prefix_extractor_ && moptions_.memtable_prefix_bloom_bits > 0) {
+ prefix_bloom_.reset(new DynamicBloom(
+ &arena_, moptions_.memtable_prefix_bloom_bits, ioptions.bloom_locality,
+ 6 /* hard coded 6 probes */, nullptr, moptions_.memtable_huge_page_size,
+ ioptions.info_log));
+ }
+}
+
+MemTable::~MemTable() {
+ mem_tracker_.FreeMem();
+ assert(refs_ == 0);
+}
+
+size_t MemTable::ApproximateMemoryUsage() {
+ autovector<size_t> usages = {arena_.ApproximateMemoryUsage(),
+ table_->ApproximateMemoryUsage(),
+ range_del_table_->ApproximateMemoryUsage(),
+ rocksdb::ApproximateMemoryUsage(insert_hints_)};
+ size_t total_usage = 0;
+ for (size_t usage : usages) {
+ // If usage + total_usage >= kMaxSizet, return kMaxSizet.
+ // the following variation is to avoid numeric overflow.
+ if (usage >= port::kMaxSizet - total_usage) {
+ return port::kMaxSizet;
+ }
+ total_usage += usage;
+ }
+ // otherwise, return the actual usage
+ return total_usage;
+}
+
+bool MemTable::ShouldFlushNow() const {
+ // In a lot of times, we cannot allocate arena blocks that exactly matches the
+ // buffer size. Thus we have to decide if we should over-allocate or
+ // under-allocate.
+ // This constant variable can be interpreted as: if we still have more than
+ // "kAllowOverAllocationRatio * kArenaBlockSize" space left, we'd try to over
+ // allocate one more block.
+ const double kAllowOverAllocationRatio = 0.6;
+
+ // If arena still have room for new block allocation, we can safely say it
+ // shouldn't flush.
+ auto allocated_memory = table_->ApproximateMemoryUsage() +
+ range_del_table_->ApproximateMemoryUsage() +
+ arena_.MemoryAllocatedBytes();
+
+ // if we can still allocate one more block without exceeding the
+ // over-allocation ratio, then we should not flush.
+ if (allocated_memory + kArenaBlockSize <
+ moptions_.write_buffer_size +
+ kArenaBlockSize * kAllowOverAllocationRatio) {
+ return false;
+ }
+
+ // if user keeps adding entries that exceeds moptions.write_buffer_size,
+ // we need to flush earlier even though we still have much available
+ // memory left.
+ if (allocated_memory > moptions_.write_buffer_size +
+ kArenaBlockSize * kAllowOverAllocationRatio) {
+ return true;
+ }
+
+ // In this code path, Arena has already allocated its "last block", which
+ // means the total allocatedmemory size is either:
+ // (1) "moderately" over allocated the memory (no more than `0.6 * arena
+ // block size`. Or,
+ // (2) the allocated memory is less than write buffer size, but we'll stop
+ // here since if we allocate a new arena block, we'll over allocate too much
+ // more (half of the arena block size) memory.
+ //
+ // In either case, to avoid over-allocate, the last block will stop allocation
+ // when its usage reaches a certain ratio, which we carefully choose "0.75
+ // full" as the stop condition because it addresses the following issue with
+ // great simplicity: What if the next inserted entry's size is
+ // bigger than AllocatedAndUnused()?
+ //
+ // The answer is: if the entry size is also bigger than 0.25 *
+ // kArenaBlockSize, a dedicated block will be allocated for it; otherwise
+ // arena will anyway skip the AllocatedAndUnused() and allocate a new, empty
+ // and regular block. In either case, we *overly* over-allocated.
+ //
+ // Therefore, setting the last block to be at most "0.75 full" avoids both
+ // cases.
+ //
+ // NOTE: the average percentage of waste space of this approach can be counted
+ // as: "arena block size * 0.25 / write buffer size". User who specify a small
+ // write buffer size and/or big arena block size may suffer.
+ return arena_.AllocatedAndUnused() < kArenaBlockSize / 4;
+}
+
+void MemTable::UpdateFlushState() {
+ auto state = flush_state_.load(std::memory_order_relaxed);
+ if (state == FLUSH_NOT_REQUESTED && ShouldFlushNow()) {
+ // ignore CAS failure, because that means somebody else requested
+ // a flush
+ flush_state_.compare_exchange_strong(state, FLUSH_REQUESTED,
+ std::memory_order_relaxed,
+ std::memory_order_relaxed);
+ }
+}
+
+int MemTable::KeyComparator::operator()(const char* prefix_len_key1,
+ const char* prefix_len_key2) const {
+ // Internal keys are encoded as length-prefixed strings.
+ Slice k1 = GetLengthPrefixedSlice(prefix_len_key1);
+ Slice k2 = GetLengthPrefixedSlice(prefix_len_key2);
+ return comparator.Compare(k1, k2);
+}
+
+int MemTable::KeyComparator::operator()(const char* prefix_len_key,
+ const Slice& key)
+ const {
+ // Internal keys are encoded as length-prefixed strings.
+ Slice a = GetLengthPrefixedSlice(prefix_len_key);
+ return comparator.Compare(a, key);
+}
+
+Slice MemTableRep::UserKey(const char* key) const {
+ Slice slice = GetLengthPrefixedSlice(key);
+ return Slice(slice.data(), slice.size() - 8);
+}
+
+KeyHandle MemTableRep::Allocate(const size_t len, char** buf) {
+ *buf = allocator_->Allocate(len);
+ return static_cast<KeyHandle>(*buf);
+}
+
+// Encode a suitable internal key target for "target" and return it.
+// Uses *scratch as scratch space, and the returned pointer will point
+// into this scratch space.
+const char* EncodeKey(std::string* scratch, const Slice& target) {
+ scratch->clear();
+ PutVarint32(scratch, static_cast<uint32_t>(target.size()));
+ scratch->append(target.data(), target.size());
+ return scratch->data();
+}
+
+class MemTableIterator : public InternalIterator {
+ public:
+ MemTableIterator(const MemTable& mem, const ReadOptions& read_options,
+ Arena* arena, bool use_range_del_table = false)
+ : bloom_(nullptr),
+ prefix_extractor_(mem.prefix_extractor_),
+ comparator_(mem.comparator_),
+ valid_(false),
+ arena_mode_(arena != nullptr),
+ value_pinned_(!mem.GetMemTableOptions()->inplace_update_support) {
+ if (use_range_del_table) {
+ iter_ = mem.range_del_table_->GetIterator(arena);
+ } else if (prefix_extractor_ != nullptr && !read_options.total_order_seek) {
+ bloom_ = mem.prefix_bloom_.get();
+ iter_ = mem.table_->GetDynamicPrefixIterator(arena);
+ } else {
+ iter_ = mem.table_->GetIterator(arena);
+ }
+ }
+
+ ~MemTableIterator() {
+#ifndef NDEBUG
+ // Assert that the MemTableIterator is never deleted while
+ // Pinning is Enabled.
+ assert(!pinned_iters_mgr_ ||
+ (pinned_iters_mgr_ && !pinned_iters_mgr_->PinningEnabled()));
+#endif
+ if (arena_mode_) {
+ iter_->~Iterator();
+ } else {
+ delete iter_;
+ }
+ }
+
+#ifndef NDEBUG
+ virtual void SetPinnedItersMgr(
+ PinnedIteratorsManager* pinned_iters_mgr) override {
+ pinned_iters_mgr_ = pinned_iters_mgr;
+ }
+ PinnedIteratorsManager* pinned_iters_mgr_ = nullptr;
+#endif
+
+ virtual bool Valid() const override { return valid_; }
+ virtual void Seek(const Slice& k) override {
+ PERF_TIMER_GUARD(seek_on_memtable_time);
+ PERF_COUNTER_ADD(seek_on_memtable_count, 1);
+ if (bloom_ != nullptr) {
+ if (!bloom_->MayContain(
+ prefix_extractor_->Transform(ExtractUserKey(k)))) {
+ PERF_COUNTER_ADD(bloom_memtable_miss_count, 1);
+ valid_ = false;
+ return;
+ } else {
+ PERF_COUNTER_ADD(bloom_memtable_hit_count, 1);
+ }
+ }
+ iter_->Seek(k, nullptr);
+ valid_ = iter_->Valid();
+ }
+ virtual void SeekForPrev(const Slice& k) override {
+ PERF_TIMER_GUARD(seek_on_memtable_time);
+ PERF_COUNTER_ADD(seek_on_memtable_count, 1);
+ if (bloom_ != nullptr) {
+ if (!bloom_->MayContain(
+ prefix_extractor_->Transform(ExtractUserKey(k)))) {
+ PERF_COUNTER_ADD(bloom_memtable_miss_count, 1);
+ valid_ = false;
+ return;
+ } else {
+ PERF_COUNTER_ADD(bloom_memtable_hit_count, 1);
+ }
+ }
+ iter_->Seek(k, nullptr);
+ valid_ = iter_->Valid();
+ if (!Valid()) {
+ SeekToLast();
+ }
+ while (Valid() && comparator_.comparator.Compare(k, key()) < 0) {
+ Prev();
+ }
+ }
+ virtual void SeekToFirst() override {
+ iter_->SeekToFirst();
+ valid_ = iter_->Valid();
+ }
+ virtual void SeekToLast() override {
+ iter_->SeekToLast();
+ valid_ = iter_->Valid();
+ }
+ virtual void Next() override {
+ PERF_COUNTER_ADD(next_on_memtable_count, 1);
+ assert(Valid());
+ iter_->Next();
+ valid_ = iter_->Valid();
+ }
+ virtual void Prev() override {
+ PERF_COUNTER_ADD(prev_on_memtable_count, 1);
+ assert(Valid());
+ iter_->Prev();
+ valid_ = iter_->Valid();
+ }
+ virtual Slice key() const override {
+ assert(Valid());
+ return GetLengthPrefixedSlice(iter_->key());
+ }
+ virtual Slice value() const override {
+ assert(Valid());
+ Slice key_slice = GetLengthPrefixedSlice(iter_->key());
+ return GetLengthPrefixedSlice(key_slice.data() + key_slice.size());
+ }
+
+ virtual Status status() const override { return Status::OK(); }
+
+ virtual bool IsKeyPinned() const override {
+ // memtable data is always pinned
+ return true;
+ }
+
+ virtual bool IsValuePinned() const override {
+ // memtable value is always pinned, except if we allow inplace update.
+ return value_pinned_;
+ }
+
+ private:
+ DynamicBloom* bloom_;
+ const SliceTransform* const prefix_extractor_;
+ const MemTable::KeyComparator comparator_;
+ MemTableRep::Iterator* iter_;
+ bool valid_;
+ bool arena_mode_;
+ bool value_pinned_;
+
+ // No copying allowed
+ MemTableIterator(const MemTableIterator&);
+ void operator=(const MemTableIterator&);
+};
+
+InternalIterator* MemTable::NewIterator(const ReadOptions& read_options,
+ Arena* arena) {
+ assert(arena != nullptr);
+ auto mem = arena->AllocateAligned(sizeof(MemTableIterator));
+ return new (mem) MemTableIterator(*this, read_options, arena);
+}
+
+InternalIterator* MemTable::NewRangeTombstoneIterator(
+ const ReadOptions& read_options) {
+ if (read_options.ignore_range_deletions || is_range_del_table_empty_) {
+ return nullptr;
+ }
+ return new MemTableIterator(*this, read_options, nullptr /* arena */,
+ true /* use_range_del_table */);
+}
+
+port::RWMutex* MemTable::GetLock(const Slice& key) {
+ static murmur_hash hash;
+ return &locks_[hash(key) % locks_.size()];
+}
+
+MemTable::MemTableStats MemTable::ApproximateStats(const Slice& start_ikey,
+ const Slice& end_ikey) {
+ uint64_t entry_count = table_->ApproximateNumEntries(start_ikey, end_ikey);
+ entry_count += range_del_table_->ApproximateNumEntries(start_ikey, end_ikey);
+ if (entry_count == 0) {
+ return {0, 0};
+ }
+ uint64_t n = num_entries_.load(std::memory_order_relaxed);
+ if (n == 0) {
+ return {0, 0};
+ }
+ if (entry_count > n) {
+ // (range_del_)table_->ApproximateNumEntries() is just an estimate so it can
+ // be larger than actual entries we have. Cap it to entries we have to limit
+ // the inaccuracy.
+ entry_count = n;
+ }
+ uint64_t data_size = data_size_.load(std::memory_order_relaxed);
+ return {entry_count * (data_size / n), entry_count};
+}
+
+void MemTable::Add(SequenceNumber s, ValueType type,
+ const Slice& key, /* user key */
+ const Slice& value, bool allow_concurrent,
+ MemTablePostProcessInfo* post_process_info) {
+ // Format of an entry is concatenation of:
+ // key_size : varint32 of internal_key.size()
+ // key bytes : char[internal_key.size()]
+ // value_size : varint32 of value.size()
+ // value bytes : char[value.size()]
+ uint32_t key_size = static_cast<uint32_t>(key.size());
+ uint32_t val_size = static_cast<uint32_t>(value.size());
+ uint32_t internal_key_size = key_size + 8;
+ const uint32_t encoded_len = VarintLength(internal_key_size) +
+ internal_key_size + VarintLength(val_size) +
+ val_size;
+ char* buf = nullptr;
+ std::unique_ptr<MemTableRep>& table =
+ type == kTypeRangeDeletion ? range_del_table_ : table_;
+ KeyHandle handle = table->Allocate(encoded_len, &buf);
+
+ char* p = EncodeVarint32(buf, internal_key_size);
+ memcpy(p, key.data(), key_size);
+ Slice key_slice(p, key_size);
+ p += key_size;
+ uint64_t packed = PackSequenceAndType(s, type);
+ EncodeFixed64(p, packed);
+ p += 8;
+ p = EncodeVarint32(p, val_size);
+ memcpy(p, value.data(), val_size);
+ assert((unsigned)(p + val_size - buf) == (unsigned)encoded_len);
+ if (!allow_concurrent) {
+ // Extract prefix for insert with hint.
+ if (insert_with_hint_prefix_extractor_ != nullptr &&
+ insert_with_hint_prefix_extractor_->InDomain(key_slice)) {
+ Slice prefix = insert_with_hint_prefix_extractor_->Transform(key_slice);
+ table->InsertWithHint(handle, &insert_hints_[prefix]);
+ } else {
+ table->Insert(handle);
+ }
+
+ // this is a bit ugly, but is the way to avoid locked instructions
+ // when incrementing an atomic
+ num_entries_.store(num_entries_.load(std::memory_order_relaxed) + 1,
+ std::memory_order_relaxed);
+ data_size_.store(data_size_.load(std::memory_order_relaxed) + encoded_len,
+ std::memory_order_relaxed);
+ if (type == kTypeDeletion) {
+ num_deletes_.store(num_deletes_.load(std::memory_order_relaxed) + 1,
+ std::memory_order_relaxed);
+ }
+
+ if (prefix_bloom_) {
+ assert(prefix_extractor_);
+ prefix_bloom_->Add(prefix_extractor_->Transform(key));
+ }
+
+ // The first sequence number inserted into the memtable
+ assert(first_seqno_ == 0 || s > first_seqno_);
+ if (first_seqno_ == 0) {
+ first_seqno_.store(s, std::memory_order_relaxed);
+
+ if (earliest_seqno_ == kMaxSequenceNumber) {
+ earliest_seqno_.store(GetFirstSequenceNumber(),
+ std::memory_order_relaxed);
+ }
+ assert(first_seqno_.load() >= earliest_seqno_.load());
+ }
+ assert(post_process_info == nullptr);
+ UpdateFlushState();
+ } else {
+ table->InsertConcurrently(handle);
+
+ assert(post_process_info != nullptr);
+ post_process_info->num_entries++;
+ post_process_info->data_size += encoded_len;
+ if (type == kTypeDeletion) {
+ post_process_info->num_deletes++;
+ }
+
+ if (prefix_bloom_) {
+ assert(prefix_extractor_);
+ prefix_bloom_->AddConcurrently(prefix_extractor_->Transform(key));
+ }
+
+ // atomically update first_seqno_ and earliest_seqno_.
+ uint64_t cur_seq_num = first_seqno_.load(std::memory_order_relaxed);
+ while ((cur_seq_num == 0 || s < cur_seq_num) &&
+ !first_seqno_.compare_exchange_weak(cur_seq_num, s)) {
+ }
+ uint64_t cur_earliest_seqno =
+ earliest_seqno_.load(std::memory_order_relaxed);
+ while (
+ (cur_earliest_seqno == kMaxSequenceNumber || s < cur_earliest_seqno) &&
+ !first_seqno_.compare_exchange_weak(cur_earliest_seqno, s)) {
+ }
+ }
+ if (is_range_del_table_empty_ && type == kTypeRangeDeletion) {
+ is_range_del_table_empty_ = false;
+ }
+}
+
+// Callback from MemTable::Get()
+namespace {
+
+struct Saver {
+ Status* status;
+ const LookupKey* key;
+ bool* found_final_value; // Is value set correctly? Used by KeyMayExist
+ bool* merge_in_progress;
+ std::string* value;
+ SequenceNumber seq;
+ const MergeOperator* merge_operator;
+ // the merge operations encountered;
+ MergeContext* merge_context;
+ RangeDelAggregator* range_del_agg;
+ MemTable* mem;
+ Logger* logger;
+ Statistics* statistics;
+ bool inplace_update_support;
+ Env* env_;
+};
+} // namespace
+
+static bool SaveValue(void* arg, const char* entry) {
+ Saver* s = reinterpret_cast<Saver*>(arg);
+ MergeContext* merge_context = s->merge_context;
+ RangeDelAggregator* range_del_agg = s->range_del_agg;
+ const MergeOperator* merge_operator = s->merge_operator;
+
+ assert(s != nullptr && merge_context != nullptr && range_del_agg != nullptr);
+
+ // entry format is:
+ // klength varint32
+ // userkey char[klength-8]
+ // tag uint64
+ // vlength varint32
+ // value char[vlength]
+ // Check that it belongs to same user key. We do not check the
+ // sequence number since the Seek() call above should have skipped
+ // all entries with overly large sequence numbers.
+ uint32_t key_length;
+ const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
+ if (s->mem->GetInternalKeyComparator().user_comparator()->Equal(
+ Slice(key_ptr, key_length - 8), s->key->user_key())) {
+ // Correct user key
+ const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
+ ValueType type;
+ UnPackSequenceAndType(tag, &s->seq, &type);
+
+ if ((type == kTypeValue || type == kTypeMerge) &&
+ range_del_agg->ShouldDelete(Slice(key_ptr, key_length))) {
+ type = kTypeRangeDeletion;
+ }
+ switch (type) {
+ case kTypeValue: {
+ if (s->inplace_update_support) {
+ s->mem->GetLock(s->key->user_key())->ReadLock();
+ }
+ Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
+ *(s->status) = Status::OK();
+ if (*(s->merge_in_progress)) {
+ *(s->status) = MergeHelper::TimedFullMerge(
+ merge_operator, s->key->user_key(), &v,
+ merge_context->GetOperands(), s->value, s->logger, s->statistics,
+ s->env_, nullptr /* result_operand */, true);
+ } else if (s->value != nullptr) {
+ s->value->assign(v.data(), v.size());
+ }
+ if (s->inplace_update_support) {
+ s->mem->GetLock(s->key->user_key())->ReadUnlock();
+ }
+ *(s->found_final_value) = true;
+ return false;
+ }
+ case kTypeDeletion:
+ case kTypeSingleDeletion:
+ case kTypeRangeDeletion: {
+ if (*(s->merge_in_progress)) {
+ *(s->status) = MergeHelper::TimedFullMerge(
+ merge_operator, s->key->user_key(), nullptr,
+ merge_context->GetOperands(), s->value, s->logger, s->statistics,
+ s->env_, nullptr /* result_operand */, true);
+ } else {
+ *(s->status) = Status::NotFound();
+ }
+ *(s->found_final_value) = true;
+ return false;
+ }
+ case kTypeMerge: {
+ if (!merge_operator) {
+ *(s->status) = Status::InvalidArgument(
+ "merge_operator is not properly initialized.");
+ // Normally we continue the loop (return true) when we see a merge
+ // operand. But in case of an error, we should stop the loop
+ // immediately and pretend we have found the value to stop further
+ // seek. Otherwise, the later call will override this error status.
+ *(s->found_final_value) = true;
+ return false;
+ }
+ Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
+ *(s->merge_in_progress) = true;
+ merge_context->PushOperand(
+ v, s->inplace_update_support == false /* operand_pinned */);
+ return true;
+ }
+ default:
+ assert(false);
+ return true;
+ }
+ }
+
+ // s->state could be Corrupt, merge or notfound
+ return false;
+}
+
+bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
+ MergeContext* merge_context,
+ RangeDelAggregator* range_del_agg, SequenceNumber* seq,
+ const ReadOptions& read_opts) {
+ // The sequence number is updated synchronously in version_set.h
+ if (IsEmpty()) {
+ // Avoiding recording stats for speed.
+ return false;
+ }
+ PERF_TIMER_GUARD(get_from_memtable_time);
+
+ Slice user_key = key.user_key();
+ bool found_final_value = false;
+ bool merge_in_progress = s->IsMergeInProgress();
+ bool const may_contain =
+ nullptr == prefix_bloom_
+ ? false
+ : prefix_bloom_->MayContain(prefix_extractor_->Transform(user_key));
+ if (prefix_bloom_ && !may_contain) {
+ // iter is null if prefix bloom says the key does not exist
+ PERF_COUNTER_ADD(bloom_memtable_miss_count, 1);
+ *seq = kMaxSequenceNumber;
+ } else {
+ if (prefix_bloom_) {
+ PERF_COUNTER_ADD(bloom_memtable_hit_count, 1);
+ }
+ std::unique_ptr<InternalIterator> range_del_iter(
+ NewRangeTombstoneIterator(read_opts));
+ Status status = range_del_agg->AddTombstones(std::move(range_del_iter));
+ if (!status.ok()) {
+ *s = status;
+ return false;
+ }
+ Saver saver;
+ saver.status = s;
+ saver.found_final_value = &found_final_value;
+ saver.merge_in_progress = &merge_in_progress;
+ saver.key = &key;
+ saver.value = value;
+ saver.seq = kMaxSequenceNumber;
+ saver.mem = this;
+ saver.merge_context = merge_context;
+ saver.range_del_agg = range_del_agg;
+ saver.merge_operator = moptions_.merge_operator;
+ saver.logger = moptions_.info_log;
+ saver.inplace_update_support = moptions_.inplace_update_support;
+ saver.statistics = moptions_.statistics;
+ saver.env_ = env_;
+ table_->Get(key, &saver, SaveValue);
+
+ *seq = saver.seq;
+ }
+
+ // No change to value, since we have not yet found a Put/Delete
+ if (!found_final_value && merge_in_progress) {
+ *s = Status::MergeInProgress();
+ }
+ PERF_COUNTER_ADD(get_from_memtable_count, 1);
+ return found_final_value;
+}
+
+void MemTable::Update(SequenceNumber seq,
+ const Slice& key,
+ const Slice& value) {
+ LookupKey lkey(key, seq);
+ Slice mem_key = lkey.memtable_key();
+
+ std::unique_ptr<MemTableRep::Iterator> iter(
+ table_->GetDynamicPrefixIterator());
+ iter->Seek(lkey.internal_key(), mem_key.data());
+
+ if (iter->Valid()) {
+ // entry format is:
+ // key_length varint32
+ // userkey char[klength-8]
+ // tag uint64
+ // vlength varint32
+ // value char[vlength]
+ // Check that it belongs to same user key. We do not check the
+ // sequence number since the Seek() call above should have skipped
+ // all entries with overly large sequence numbers.
+ const char* entry = iter->key();
+ uint32_t key_length = 0;
+ const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
+ if (comparator_.comparator.user_comparator()->Equal(
+ Slice(key_ptr, key_length - 8), lkey.user_key())) {
+ // Correct user key
+ const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
+ ValueType type;
+ SequenceNumber unused;
+ UnPackSequenceAndType(tag, &unused, &type);
+ if (type == kTypeValue) {
+ Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length);
+ uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
+ uint32_t new_size = static_cast<uint32_t>(value.size());
+
+ // Update value, if new value size <= previous value size
+ if (new_size <= prev_size) {
+ char* p =
+ EncodeVarint32(const_cast<char*>(key_ptr) + key_length, new_size);
+ WriteLock wl(GetLock(lkey.user_key()));
+ memcpy(p, value.data(), value.size());
+ assert((unsigned)((p + value.size()) - entry) ==
+ (unsigned)(VarintLength(key_length) + key_length +
+ VarintLength(value.size()) + value.size()));
+ return;
+ }
+ }
+ }
+ }
+
+ // key doesn't exist
+ Add(seq, kTypeValue, key, value);
+}
+
+bool MemTable::UpdateCallback(SequenceNumber seq,
+ const Slice& key,
+ const Slice& delta) {
+ LookupKey lkey(key, seq);
+ Slice memkey = lkey.memtable_key();
+
+ std::unique_ptr<MemTableRep::Iterator> iter(
+ table_->GetDynamicPrefixIterator());
+ iter->Seek(lkey.internal_key(), memkey.data());
+
+ if (iter->Valid()) {
+ // entry format is:
+ // key_length varint32
+ // userkey char[klength-8]
+ // tag uint64
+ // vlength varint32
+ // value char[vlength]
+ // Check that it belongs to same user key. We do not check the
+ // sequence number since the Seek() call above should have skipped
+ // all entries with overly large sequence numbers.
+ const char* entry = iter->key();
+ uint32_t key_length = 0;
+ const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
+ if (comparator_.comparator.user_comparator()->Equal(
+ Slice(key_ptr, key_length - 8), lkey.user_key())) {
+ // Correct user key
+ const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
+ ValueType type;
+ uint64_t unused;
+ UnPackSequenceAndType(tag, &unused, &type);
+ switch (type) {
+ case kTypeValue: {
+ Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length);
+ uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
+
+ char* prev_buffer = const_cast<char*>(prev_value.data());
+ uint32_t new_prev_size = prev_size;
+
+ std::string str_value;
+ WriteLock wl(GetLock(lkey.user_key()));
+ auto status = moptions_.inplace_callback(prev_buffer, &new_prev_size,
+ delta, &str_value);
+ if (status == UpdateStatus::UPDATED_INPLACE) {
+ // Value already updated by callback.
+ assert(new_prev_size <= prev_size);
+ if (new_prev_size < prev_size) {
+ // overwrite the new prev_size
+ char* p = EncodeVarint32(const_cast<char*>(key_ptr) + key_length,
+ new_prev_size);
+ if (VarintLength(new_prev_size) < VarintLength(prev_size)) {
+ // shift the value buffer as well.
+ memcpy(p, prev_buffer, new_prev_size);
+ }
+ }
+ RecordTick(moptions_.statistics, NUMBER_KEYS_UPDATED);
+ UpdateFlushState();
+ return true;
+ } else if (status == UpdateStatus::UPDATED) {
+ Add(seq, kTypeValue, key, Slice(str_value));
+ RecordTick(moptions_.statistics, NUMBER_KEYS_WRITTEN);
+ UpdateFlushState();
+ return true;
+ } else if (status == UpdateStatus::UPDATE_FAILED) {
+ // No action required. Return.
+ UpdateFlushState();
+ return true;
+ }
+ }
+ default:
+ break;
+ }
+ }
+ }
+ // If the latest value is not kTypeValue
+ // or key doesn't exist
+ return false;
+}
+
+size_t MemTable::CountSuccessiveMergeEntries(const LookupKey& key) {
+ Slice memkey = key.memtable_key();
+
+ // A total ordered iterator is costly for some memtablerep (prefix aware
+ // reps). By passing in the user key, we allow efficient iterator creation.
+ // The iterator only needs to be ordered within the same user key.
+ std::unique_ptr<MemTableRep::Iterator> iter(
+ table_->GetDynamicPrefixIterator());
+ iter->Seek(key.internal_key(), memkey.data());
+
+ size_t num_successive_merges = 0;
+
+ for (; iter->Valid(); iter->Next()) {
+ const char* entry = iter->key();
+ uint32_t key_length = 0;
+ const char* iter_key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
+ if (!comparator_.comparator.user_comparator()->Equal(
+ Slice(iter_key_ptr, key_length - 8), key.user_key())) {
+ break;
+ }
+
+ const uint64_t tag = DecodeFixed64(iter_key_ptr + key_length - 8);
+ ValueType type;
+ uint64_t unused;
+ UnPackSequenceAndType(tag, &unused, &type);
+ if (type != kTypeMerge) {
+ break;
+ }
+
+ ++num_successive_merges;
+ }
+
+ return num_successive_merges;
+}
+
+void MemTableRep::Get(const LookupKey& k, void* callback_args,
+ bool (*callback_func)(void* arg, const char* entry)) {
+ auto iter = GetDynamicPrefixIterator();
+ for (iter->Seek(k.internal_key(), k.memtable_key().data());
+ iter->Valid() && callback_func(callback_args, iter->key());
+ iter->Next()) {
+ }
+}
+
+void MemTable::RefLogContainingPrepSection(uint64_t log) {
+ assert(log > 0);
+ auto cur = min_prep_log_referenced_.load();
+ while ((log < cur || cur == 0) &&
+ !min_prep_log_referenced_.compare_exchange_strong(cur, log)) {
+ cur = min_prep_log_referenced_.load();
+ }
+}
+
+uint64_t MemTable::GetMinLogContainingPrepSection() {
+ return min_prep_log_referenced_.load();
+}
+
+} // namespace rocksdb
[14/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/internal_stats.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/internal_stats.cc b/thirdparty/rocksdb/db/internal_stats.cc
new file mode 100644
index 0000000..54723ea
--- /dev/null
+++ b/thirdparty/rocksdb/db/internal_stats.cc
@@ -0,0 +1,1174 @@
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/internal_stats.h"
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include <string>
+#include <algorithm>
+#include <utility>
+#include <vector>
+#include "db/column_family.h"
+
+#include "db/db_impl.h"
+#include "util/string_util.h"
+
+namespace rocksdb {
+
+#ifndef ROCKSDB_LITE
+
+const std::map<LevelStatType, LevelStat> InternalStats::compaction_level_stats =
+ {
+ {LevelStatType::NUM_FILES, LevelStat{"NumFiles", "Files"}},
+ {LevelStatType::COMPACTED_FILES,
+ LevelStat{"CompactedFiles", "CompactedFiles"}},
+ {LevelStatType::SIZE_BYTES, LevelStat{"SizeBytes", "Size"}},
+ {LevelStatType::SCORE, LevelStat{"Score", "Score"}},
+ {LevelStatType::READ_GB, LevelStat{"ReadGB", "Read(GB)"}},
+ {LevelStatType::RN_GB, LevelStat{"RnGB", "Rn(GB)"}},
+ {LevelStatType::RNP1_GB, LevelStat{"Rnp1GB", "Rnp1(GB)"}},
+ {LevelStatType::WRITE_GB, LevelStat{"WriteGB", "Write(GB)"}},
+ {LevelStatType::W_NEW_GB, LevelStat{"WnewGB", "Wnew(GB)"}},
+ {LevelStatType::MOVED_GB, LevelStat{"MovedGB", "Moved(GB)"}},
+ {LevelStatType::WRITE_AMP, LevelStat{"WriteAmp", "W-Amp"}},
+ {LevelStatType::READ_MBPS, LevelStat{"ReadMBps", "Rd(MB/s)"}},
+ {LevelStatType::WRITE_MBPS, LevelStat{"WriteMBps", "Wr(MB/s)"}},
+ {LevelStatType::COMP_SEC, LevelStat{"CompSec", "Comp(sec)"}},
+ {LevelStatType::COMP_COUNT, LevelStat{"CompCount", "Comp(cnt)"}},
+ {LevelStatType::AVG_SEC, LevelStat{"AvgSec", "Avg(sec)"}},
+ {LevelStatType::KEY_IN, LevelStat{"KeyIn", "KeyIn"}},
+ {LevelStatType::KEY_DROP, LevelStat{"KeyDrop", "KeyDrop"}},
+};
+
+namespace {
+const double kMB = 1048576.0;
+const double kGB = kMB * 1024;
+const double kMicrosInSec = 1000000.0;
+
+void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
+ int written_size =
+ snprintf(buf, len, "\n** Compaction Stats [%s] **\n", cf_name.c_str());
+ auto hdr = [](LevelStatType t) {
+ return InternalStats::compaction_level_stats.at(t).header_name.c_str();
+ };
+ int line_size = snprintf(
+ buf + written_size, len - written_size,
+ "Level %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s\n",
+ // Note that we skip COMPACTED_FILES and merge it with Files column
+ hdr(LevelStatType::NUM_FILES), hdr(LevelStatType::SIZE_BYTES),
+ hdr(LevelStatType::SCORE), hdr(LevelStatType::READ_GB),
+ hdr(LevelStatType::RN_GB), hdr(LevelStatType::RNP1_GB),
+ hdr(LevelStatType::WRITE_GB), hdr(LevelStatType::W_NEW_GB),
+ hdr(LevelStatType::MOVED_GB), hdr(LevelStatType::WRITE_AMP),
+ hdr(LevelStatType::READ_MBPS), hdr(LevelStatType::WRITE_MBPS),
+ hdr(LevelStatType::COMP_SEC), hdr(LevelStatType::COMP_COUNT),
+ hdr(LevelStatType::AVG_SEC), hdr(LevelStatType::KEY_IN),
+ hdr(LevelStatType::KEY_DROP));
+
+ written_size += line_size;
+ snprintf(buf + written_size, len - written_size, "%s\n",
+ std::string(line_size, '-').c_str());
+}
+
+void PrepareLevelStats(std::map<LevelStatType, double>* level_stats,
+ int num_files, int being_compacted,
+ double total_file_size, double score, double w_amp,
+ const InternalStats::CompactionStats& stats) {
+ uint64_t bytes_read =
+ stats.bytes_read_non_output_levels + stats.bytes_read_output_level;
+ int64_t bytes_new =
+ stats.bytes_written - stats.bytes_read_output_level;
+ double elapsed = (stats.micros + 1) / kMicrosInSec;
+
+ (*level_stats)[LevelStatType::NUM_FILES] = num_files;
+ (*level_stats)[LevelStatType::COMPACTED_FILES] = being_compacted;
+ (*level_stats)[LevelStatType::SIZE_BYTES] = total_file_size;
+ (*level_stats)[LevelStatType::SCORE] = score;
+ (*level_stats)[LevelStatType::READ_GB] = bytes_read / kGB;
+ (*level_stats)[LevelStatType::RN_GB] =
+ stats.bytes_read_non_output_levels / kGB;
+ (*level_stats)[LevelStatType::RNP1_GB] = stats.bytes_read_output_level / kGB;
+ (*level_stats)[LevelStatType::WRITE_GB] = stats.bytes_written / kGB;
+ (*level_stats)[LevelStatType::W_NEW_GB] = bytes_new / kGB;
+ (*level_stats)[LevelStatType::MOVED_GB] = stats.bytes_moved / kGB;
+ (*level_stats)[LevelStatType::WRITE_AMP] = w_amp;
+ (*level_stats)[LevelStatType::READ_MBPS] = bytes_read / kMB / elapsed;
+ (*level_stats)[LevelStatType::WRITE_MBPS] =
+ stats.bytes_written / kMB / elapsed;
+ (*level_stats)[LevelStatType::COMP_SEC] = stats.micros / kMicrosInSec;
+ (*level_stats)[LevelStatType::COMP_COUNT] = stats.count;
+ (*level_stats)[LevelStatType::AVG_SEC] =
+ stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count;
+ (*level_stats)[LevelStatType::KEY_IN] =
+ static_cast<double>(stats.num_input_records);
+ (*level_stats)[LevelStatType::KEY_DROP] =
+ static_cast<double>(stats.num_dropped_records);
+}
+
+void PrintLevelStats(char* buf, size_t len, const std::string& name,
+ const std::map<LevelStatType, double>& stat_value) {
+ snprintf(buf, len,
+ "%4s " /* Level */
+ "%6d/%-3d " /* Files */
+ "%8s " /* Size */
+ "%5.1f " /* Score */
+ "%8.1f " /* Read(GB) */
+ "%7.1f " /* Rn(GB) */
+ "%8.1f " /* Rnp1(GB) */
+ "%9.1f " /* Write(GB) */
+ "%8.1f " /* Wnew(GB) */
+ "%9.1f " /* Moved(GB) */
+ "%5.1f " /* W-Amp */
+ "%8.1f " /* Rd(MB/s) */
+ "%8.1f " /* Wr(MB/s) */
+ "%9.0f " /* Comp(sec) */
+ "%9d " /* Comp(cnt) */
+ "%8.3f " /* Avg(sec) */
+ "%7s " /* KeyIn */
+ "%6s\n", /* KeyDrop */
+ name.c_str(),
+ static_cast<int>(stat_value.at(LevelStatType::NUM_FILES)),
+ static_cast<int>(stat_value.at(LevelStatType::COMPACTED_FILES)),
+ BytesToHumanString(
+ static_cast<uint64_t>(stat_value.at(LevelStatType::SIZE_BYTES)))
+ .c_str(),
+ stat_value.at(LevelStatType::SCORE),
+ stat_value.at(LevelStatType::READ_GB),
+ stat_value.at(LevelStatType::RN_GB),
+ stat_value.at(LevelStatType::RNP1_GB),
+ stat_value.at(LevelStatType::WRITE_GB),
+ stat_value.at(LevelStatType::W_NEW_GB),
+ stat_value.at(LevelStatType::MOVED_GB),
+ stat_value.at(LevelStatType::WRITE_AMP),
+ stat_value.at(LevelStatType::READ_MBPS),
+ stat_value.at(LevelStatType::WRITE_MBPS),
+ stat_value.at(LevelStatType::COMP_SEC),
+ static_cast<int>(stat_value.at(LevelStatType::COMP_COUNT)),
+ stat_value.at(LevelStatType::AVG_SEC),
+ NumberToHumanString(
+ static_cast<std::int64_t>(stat_value.at(LevelStatType::KEY_IN)))
+ .c_str(),
+ NumberToHumanString(static_cast<std::int64_t>(
+ stat_value.at(LevelStatType::KEY_DROP)))
+ .c_str());
+}
+
+void PrintLevelStats(char* buf, size_t len, const std::string& name,
+ int num_files, int being_compacted, double total_file_size,
+ double score, double w_amp,
+ const InternalStats::CompactionStats& stats) {
+ std::map<LevelStatType, double> level_stats;
+ PrepareLevelStats(&level_stats, num_files, being_compacted, total_file_size,
+ score, w_amp, stats);
+ PrintLevelStats(buf, len, name, level_stats);
+}
+
+// Assumes that trailing numbers represent an optional argument. This requires
+// property names to not end with numbers.
+std::pair<Slice, Slice> GetPropertyNameAndArg(const Slice& property) {
+ Slice name = property, arg = property;
+ size_t sfx_len = 0;
+ while (sfx_len < property.size() &&
+ isdigit(property[property.size() - sfx_len - 1])) {
+ ++sfx_len;
+ }
+ name.remove_suffix(sfx_len);
+ arg.remove_prefix(property.size() - sfx_len);
+ return {name, arg};
+}
+} // anonymous namespace
+
+static const std::string rocksdb_prefix = "rocksdb.";
+
+static const std::string num_files_at_level_prefix = "num-files-at-level";
+static const std::string compression_ratio_at_level_prefix =
+ "compression-ratio-at-level";
+static const std::string allstats = "stats";
+static const std::string sstables = "sstables";
+static const std::string cfstats = "cfstats";
+static const std::string cfstats_no_file_histogram =
+ "cfstats-no-file-histogram";
+static const std::string cf_file_histogram = "cf-file-histogram";
+static const std::string dbstats = "dbstats";
+static const std::string levelstats = "levelstats";
+static const std::string num_immutable_mem_table = "num-immutable-mem-table";
+static const std::string num_immutable_mem_table_flushed =
+ "num-immutable-mem-table-flushed";
+static const std::string mem_table_flush_pending = "mem-table-flush-pending";
+static const std::string compaction_pending = "compaction-pending";
+static const std::string background_errors = "background-errors";
+static const std::string cur_size_active_mem_table =
+ "cur-size-active-mem-table";
+static const std::string cur_size_all_mem_tables = "cur-size-all-mem-tables";
+static const std::string size_all_mem_tables = "size-all-mem-tables";
+static const std::string num_entries_active_mem_table =
+ "num-entries-active-mem-table";
+static const std::string num_entries_imm_mem_tables =
+ "num-entries-imm-mem-tables";
+static const std::string num_deletes_active_mem_table =
+ "num-deletes-active-mem-table";
+static const std::string num_deletes_imm_mem_tables =
+ "num-deletes-imm-mem-tables";
+static const std::string estimate_num_keys = "estimate-num-keys";
+static const std::string estimate_table_readers_mem =
+ "estimate-table-readers-mem";
+static const std::string is_file_deletions_enabled =
+ "is-file-deletions-enabled";
+static const std::string num_snapshots = "num-snapshots";
+static const std::string oldest_snapshot_time = "oldest-snapshot-time";
+static const std::string num_live_versions = "num-live-versions";
+static const std::string current_version_number =
+ "current-super-version-number";
+static const std::string estimate_live_data_size = "estimate-live-data-size";
+static const std::string min_log_number_to_keep = "min-log-number-to-keep";
+static const std::string base_level = "base-level";
+static const std::string total_sst_files_size = "total-sst-files-size";
+static const std::string estimate_pending_comp_bytes =
+ "estimate-pending-compaction-bytes";
+static const std::string aggregated_table_properties =
+ "aggregated-table-properties";
+static const std::string aggregated_table_properties_at_level =
+ aggregated_table_properties + "-at-level";
+static const std::string num_running_compactions = "num-running-compactions";
+static const std::string num_running_flushes = "num-running-flushes";
+static const std::string actual_delayed_write_rate =
+ "actual-delayed-write-rate";
+static const std::string is_write_stopped = "is-write-stopped";
+
+const std::string DB::Properties::kNumFilesAtLevelPrefix =
+ rocksdb_prefix + num_files_at_level_prefix;
+const std::string DB::Properties::kCompressionRatioAtLevelPrefix =
+ rocksdb_prefix + compression_ratio_at_level_prefix;
+const std::string DB::Properties::kStats = rocksdb_prefix + allstats;
+const std::string DB::Properties::kSSTables = rocksdb_prefix + sstables;
+const std::string DB::Properties::kCFStats = rocksdb_prefix + cfstats;
+const std::string DB::Properties::kCFStatsNoFileHistogram =
+ rocksdb_prefix + cfstats_no_file_histogram;
+const std::string DB::Properties::kCFFileHistogram =
+ rocksdb_prefix + cf_file_histogram;
+const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats;
+const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats;
+const std::string DB::Properties::kNumImmutableMemTable =
+ rocksdb_prefix + num_immutable_mem_table;
+const std::string DB::Properties::kNumImmutableMemTableFlushed =
+ rocksdb_prefix + num_immutable_mem_table_flushed;
+const std::string DB::Properties::kMemTableFlushPending =
+ rocksdb_prefix + mem_table_flush_pending;
+const std::string DB::Properties::kCompactionPending =
+ rocksdb_prefix + compaction_pending;
+const std::string DB::Properties::kNumRunningCompactions =
+ rocksdb_prefix + num_running_compactions;
+const std::string DB::Properties::kNumRunningFlushes =
+ rocksdb_prefix + num_running_flushes;
+const std::string DB::Properties::kBackgroundErrors =
+ rocksdb_prefix + background_errors;
+const std::string DB::Properties::kCurSizeActiveMemTable =
+ rocksdb_prefix + cur_size_active_mem_table;
+const std::string DB::Properties::kCurSizeAllMemTables =
+ rocksdb_prefix + cur_size_all_mem_tables;
+const std::string DB::Properties::kSizeAllMemTables =
+ rocksdb_prefix + size_all_mem_tables;
+const std::string DB::Properties::kNumEntriesActiveMemTable =
+ rocksdb_prefix + num_entries_active_mem_table;
+const std::string DB::Properties::kNumEntriesImmMemTables =
+ rocksdb_prefix + num_entries_imm_mem_tables;
+const std::string DB::Properties::kNumDeletesActiveMemTable =
+ rocksdb_prefix + num_deletes_active_mem_table;
+const std::string DB::Properties::kNumDeletesImmMemTables =
+ rocksdb_prefix + num_deletes_imm_mem_tables;
+const std::string DB::Properties::kEstimateNumKeys =
+ rocksdb_prefix + estimate_num_keys;
+const std::string DB::Properties::kEstimateTableReadersMem =
+ rocksdb_prefix + estimate_table_readers_mem;
+const std::string DB::Properties::kIsFileDeletionsEnabled =
+ rocksdb_prefix + is_file_deletions_enabled;
+const std::string DB::Properties::kNumSnapshots =
+ rocksdb_prefix + num_snapshots;
+const std::string DB::Properties::kOldestSnapshotTime =
+ rocksdb_prefix + oldest_snapshot_time;
+const std::string DB::Properties::kNumLiveVersions =
+ rocksdb_prefix + num_live_versions;
+const std::string DB::Properties::kCurrentSuperVersionNumber =
+ rocksdb_prefix + current_version_number;
+const std::string DB::Properties::kEstimateLiveDataSize =
+ rocksdb_prefix + estimate_live_data_size;
+const std::string DB::Properties::kMinLogNumberToKeep =
+ rocksdb_prefix + min_log_number_to_keep;
+const std::string DB::Properties::kTotalSstFilesSize =
+ rocksdb_prefix + total_sst_files_size;
+const std::string DB::Properties::kBaseLevel = rocksdb_prefix + base_level;
+const std::string DB::Properties::kEstimatePendingCompactionBytes =
+ rocksdb_prefix + estimate_pending_comp_bytes;
+const std::string DB::Properties::kAggregatedTableProperties =
+ rocksdb_prefix + aggregated_table_properties;
+const std::string DB::Properties::kAggregatedTablePropertiesAtLevel =
+ rocksdb_prefix + aggregated_table_properties_at_level;
+const std::string DB::Properties::kActualDelayedWriteRate =
+ rocksdb_prefix + actual_delayed_write_rate;
+const std::string DB::Properties::kIsWriteStopped =
+ rocksdb_prefix + is_write_stopped;
+
+const std::unordered_map<std::string, DBPropertyInfo>
+ InternalStats::ppt_name_to_info = {
+ {DB::Properties::kNumFilesAtLevelPrefix,
+ {false, &InternalStats::HandleNumFilesAtLevel, nullptr, nullptr}},
+ {DB::Properties::kCompressionRatioAtLevelPrefix,
+ {false, &InternalStats::HandleCompressionRatioAtLevelPrefix, nullptr,
+ nullptr}},
+ {DB::Properties::kLevelStats,
+ {false, &InternalStats::HandleLevelStats, nullptr, nullptr}},
+ {DB::Properties::kStats,
+ {false, &InternalStats::HandleStats, nullptr, nullptr}},
+ {DB::Properties::kCFStats,
+ {false, &InternalStats::HandleCFStats, nullptr,
+ &InternalStats::HandleCFMapStats}},
+ {DB::Properties::kCFStatsNoFileHistogram,
+ {false, &InternalStats::HandleCFStatsNoFileHistogram, nullptr,
+ nullptr}},
+ {DB::Properties::kCFFileHistogram,
+ {false, &InternalStats::HandleCFFileHistogram, nullptr, nullptr}},
+ {DB::Properties::kDBStats,
+ {false, &InternalStats::HandleDBStats, nullptr, nullptr}},
+ {DB::Properties::kSSTables,
+ {false, &InternalStats::HandleSsTables, nullptr, nullptr}},
+ {DB::Properties::kAggregatedTableProperties,
+ {false, &InternalStats::HandleAggregatedTableProperties, nullptr,
+ nullptr}},
+ {DB::Properties::kAggregatedTablePropertiesAtLevel,
+ {false, &InternalStats::HandleAggregatedTablePropertiesAtLevel,
+ nullptr, nullptr}},
+ {DB::Properties::kNumImmutableMemTable,
+ {false, nullptr, &InternalStats::HandleNumImmutableMemTable, nullptr}},
+ {DB::Properties::kNumImmutableMemTableFlushed,
+ {false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed,
+ nullptr}},
+ {DB::Properties::kMemTableFlushPending,
+ {false, nullptr, &InternalStats::HandleMemTableFlushPending, nullptr}},
+ {DB::Properties::kCompactionPending,
+ {false, nullptr, &InternalStats::HandleCompactionPending, nullptr}},
+ {DB::Properties::kBackgroundErrors,
+ {false, nullptr, &InternalStats::HandleBackgroundErrors, nullptr}},
+ {DB::Properties::kCurSizeActiveMemTable,
+ {false, nullptr, &InternalStats::HandleCurSizeActiveMemTable,
+ nullptr}},
+ {DB::Properties::kCurSizeAllMemTables,
+ {false, nullptr, &InternalStats::HandleCurSizeAllMemTables, nullptr}},
+ {DB::Properties::kSizeAllMemTables,
+ {false, nullptr, &InternalStats::HandleSizeAllMemTables, nullptr}},
+ {DB::Properties::kNumEntriesActiveMemTable,
+ {false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable,
+ nullptr}},
+ {DB::Properties::kNumEntriesImmMemTables,
+ {false, nullptr, &InternalStats::HandleNumEntriesImmMemTables,
+ nullptr}},
+ {DB::Properties::kNumDeletesActiveMemTable,
+ {false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable,
+ nullptr}},
+ {DB::Properties::kNumDeletesImmMemTables,
+ {false, nullptr, &InternalStats::HandleNumDeletesImmMemTables,
+ nullptr}},
+ {DB::Properties::kEstimateNumKeys,
+ {false, nullptr, &InternalStats::HandleEstimateNumKeys, nullptr}},
+ {DB::Properties::kEstimateTableReadersMem,
+ {true, nullptr, &InternalStats::HandleEstimateTableReadersMem,
+ nullptr}},
+ {DB::Properties::kIsFileDeletionsEnabled,
+ {false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled,
+ nullptr}},
+ {DB::Properties::kNumSnapshots,
+ {false, nullptr, &InternalStats::HandleNumSnapshots, nullptr}},
+ {DB::Properties::kOldestSnapshotTime,
+ {false, nullptr, &InternalStats::HandleOldestSnapshotTime, nullptr}},
+ {DB::Properties::kNumLiveVersions,
+ {false, nullptr, &InternalStats::HandleNumLiveVersions, nullptr}},
+ {DB::Properties::kCurrentSuperVersionNumber,
+ {false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber,
+ nullptr}},
+ {DB::Properties::kEstimateLiveDataSize,
+ {true, nullptr, &InternalStats::HandleEstimateLiveDataSize, nullptr}},
+ {DB::Properties::kMinLogNumberToKeep,
+ {false, nullptr, &InternalStats::HandleMinLogNumberToKeep, nullptr}},
+ {DB::Properties::kBaseLevel,
+ {false, nullptr, &InternalStats::HandleBaseLevel, nullptr}},
+ {DB::Properties::kTotalSstFilesSize,
+ {false, nullptr, &InternalStats::HandleTotalSstFilesSize, nullptr}},
+ {DB::Properties::kEstimatePendingCompactionBytes,
+ {false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes,
+ nullptr}},
+ {DB::Properties::kNumRunningFlushes,
+ {false, nullptr, &InternalStats::HandleNumRunningFlushes, nullptr}},
+ {DB::Properties::kNumRunningCompactions,
+ {false, nullptr, &InternalStats::HandleNumRunningCompactions,
+ nullptr}},
+ {DB::Properties::kActualDelayedWriteRate,
+ {false, nullptr, &InternalStats::HandleActualDelayedWriteRate,
+ nullptr}},
+ {DB::Properties::kIsWriteStopped,
+ {false, nullptr, &InternalStats::HandleIsWriteStopped, nullptr}},
+};
+
+const DBPropertyInfo* GetPropertyInfo(const Slice& property) {
+ std::string ppt_name = GetPropertyNameAndArg(property).first.ToString();
+ auto ppt_info_iter = InternalStats::ppt_name_to_info.find(ppt_name);
+ if (ppt_info_iter == InternalStats::ppt_name_to_info.end()) {
+ return nullptr;
+ }
+ return &ppt_info_iter->second;
+}
+
+bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info,
+ const Slice& property,
+ std::string* value) {
+ assert(value != nullptr);
+ assert(property_info.handle_string != nullptr);
+ Slice arg = GetPropertyNameAndArg(property).second;
+ return (this->*(property_info.handle_string))(value, arg);
+}
+
+bool InternalStats::GetMapProperty(const DBPropertyInfo& property_info,
+ const Slice& property,
+ std::map<std::string, double>* value) {
+ assert(value != nullptr);
+ assert(property_info.handle_map != nullptr);
+ return (this->*(property_info.handle_map))(value);
+}
+
+bool InternalStats::GetIntProperty(const DBPropertyInfo& property_info,
+ uint64_t* value, DBImpl* db) {
+ assert(value != nullptr);
+ assert(property_info.handle_int != nullptr &&
+ !property_info.need_out_of_mutex);
+ db->mutex_.AssertHeld();
+ return (this->*(property_info.handle_int))(value, db, nullptr /* version */);
+}
+
+bool InternalStats::GetIntPropertyOutOfMutex(
+ const DBPropertyInfo& property_info, Version* version, uint64_t* value) {
+ assert(value != nullptr);
+ assert(property_info.handle_int != nullptr &&
+ property_info.need_out_of_mutex);
+ return (this->*(property_info.handle_int))(value, nullptr /* db */, version);
+}
+
+bool InternalStats::HandleNumFilesAtLevel(std::string* value, Slice suffix) {
+ uint64_t level;
+ const auto* vstorage = cfd_->current()->storage_info();
+ bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
+ if (!ok || static_cast<int>(level) >= number_levels_) {
+ return false;
+ } else {
+ char buf[100];
+ snprintf(buf, sizeof(buf), "%d",
+ vstorage->NumLevelFiles(static_cast<int>(level)));
+ *value = buf;
+ return true;
+ }
+}
+
+bool InternalStats::HandleCompressionRatioAtLevelPrefix(std::string* value,
+ Slice suffix) {
+ uint64_t level;
+ const auto* vstorage = cfd_->current()->storage_info();
+ bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
+ if (!ok || level >= static_cast<uint64_t>(number_levels_)) {
+ return false;
+ }
+ *value = ToString(
+ vstorage->GetEstimatedCompressionRatioAtLevel(static_cast<int>(level)));
+ return true;
+}
+
+bool InternalStats::HandleLevelStats(std::string* value, Slice suffix) {
+ char buf[1000];
+ const auto* vstorage = cfd_->current()->storage_info();
+ snprintf(buf, sizeof(buf),
+ "Level Files Size(MB)\n"
+ "--------------------\n");
+ value->append(buf);
+
+ for (int level = 0; level < number_levels_; level++) {
+ snprintf(buf, sizeof(buf), "%3d %8d %8.0f\n", level,
+ vstorage->NumLevelFiles(level),
+ vstorage->NumLevelBytes(level) / kMB);
+ value->append(buf);
+ }
+ return true;
+}
+
+bool InternalStats::HandleStats(std::string* value, Slice suffix) {
+ if (!HandleCFStats(value, suffix)) {
+ return false;
+ }
+ if (!HandleDBStats(value, suffix)) {
+ return false;
+ }
+ return true;
+}
+
+bool InternalStats::HandleCFMapStats(std::map<std::string, double>* cf_stats) {
+ DumpCFMapStats(cf_stats);
+ return true;
+}
+
+bool InternalStats::HandleCFStats(std::string* value, Slice suffix) {
+ DumpCFStats(value);
+ return true;
+}
+
+bool InternalStats::HandleCFStatsNoFileHistogram(std::string* value,
+ Slice suffix) {
+ DumpCFStatsNoFileHistogram(value);
+ return true;
+}
+
+bool InternalStats::HandleCFFileHistogram(std::string* value, Slice suffix) {
+ DumpCFFileHistogram(value);
+ return true;
+}
+
+bool InternalStats::HandleDBStats(std::string* value, Slice suffix) {
+ DumpDBStats(value);
+ return true;
+}
+
+bool InternalStats::HandleSsTables(std::string* value, Slice suffix) {
+ auto* current = cfd_->current();
+ *value = current->DebugString(true, true);
+ return true;
+}
+
+bool InternalStats::HandleAggregatedTableProperties(std::string* value,
+ Slice suffix) {
+ std::shared_ptr<const TableProperties> tp;
+ auto s = cfd_->current()->GetAggregatedTableProperties(&tp);
+ if (!s.ok()) {
+ return false;
+ }
+ *value = tp->ToString();
+ return true;
+}
+
+bool InternalStats::HandleAggregatedTablePropertiesAtLevel(std::string* value,
+ Slice suffix) {
+ uint64_t level;
+ bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
+ if (!ok || static_cast<int>(level) >= number_levels_) {
+ return false;
+ }
+ std::shared_ptr<const TableProperties> tp;
+ auto s = cfd_->current()->GetAggregatedTableProperties(
+ &tp, static_cast<int>(level));
+ if (!s.ok()) {
+ return false;
+ }
+ *value = tp->ToString();
+ return true;
+}
+
+bool InternalStats::HandleNumImmutableMemTable(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = cfd_->imm()->NumNotFlushed();
+ return true;
+}
+
+bool InternalStats::HandleNumImmutableMemTableFlushed(uint64_t* value,
+ DBImpl* db,
+ Version* version) {
+ *value = cfd_->imm()->NumFlushed();
+ return true;
+}
+
+bool InternalStats::HandleMemTableFlushPending(uint64_t* value, DBImpl* db,
+ Version* version) {
+ // Return number of mem tables that are ready to flush (made immutable)
+ *value = (cfd_->imm()->IsFlushPending() ? 1 : 0);
+ return true;
+}
+
+bool InternalStats::HandleNumRunningFlushes(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = db->num_running_flushes();
+ return true;
+}
+
+bool InternalStats::HandleCompactionPending(uint64_t* value, DBImpl* db,
+ Version* version) {
+ // 1 if the system already determines at least one compaction is needed.
+ // 0 otherwise,
+ const auto* vstorage = cfd_->current()->storage_info();
+ *value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0);
+ return true;
+}
+
+bool InternalStats::HandleNumRunningCompactions(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = db->num_running_compactions_;
+ return true;
+}
+
+bool InternalStats::HandleBackgroundErrors(uint64_t* value, DBImpl* db,
+ Version* version) {
+ // Accumulated number of errors in background flushes or compactions.
+ *value = GetBackgroundErrorCount();
+ return true;
+}
+
+bool InternalStats::HandleCurSizeActiveMemTable(uint64_t* value, DBImpl* db,
+ Version* version) {
+ // Current size of the active memtable
+ *value = cfd_->mem()->ApproximateMemoryUsage();
+ return true;
+}
+
+bool InternalStats::HandleCurSizeAllMemTables(uint64_t* value, DBImpl* db,
+ Version* version) {
+ // Current size of the active memtable + immutable memtables
+ *value = cfd_->mem()->ApproximateMemoryUsage() +
+ cfd_->imm()->ApproximateUnflushedMemTablesMemoryUsage();
+ return true;
+}
+
+bool InternalStats::HandleSizeAllMemTables(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = cfd_->mem()->ApproximateMemoryUsage() +
+ cfd_->imm()->ApproximateMemoryUsage();
+ return true;
+}
+
+bool InternalStats::HandleNumEntriesActiveMemTable(uint64_t* value, DBImpl* db,
+ Version* version) {
+ // Current number of entires in the active memtable
+ *value = cfd_->mem()->num_entries();
+ return true;
+}
+
+bool InternalStats::HandleNumEntriesImmMemTables(uint64_t* value, DBImpl* db,
+ Version* version) {
+ // Current number of entries in the immutable memtables
+ *value = cfd_->imm()->current()->GetTotalNumEntries();
+ return true;
+}
+
+bool InternalStats::HandleNumDeletesActiveMemTable(uint64_t* value, DBImpl* db,
+ Version* version) {
+ // Current number of entires in the active memtable
+ *value = cfd_->mem()->num_deletes();
+ return true;
+}
+
+bool InternalStats::HandleNumDeletesImmMemTables(uint64_t* value, DBImpl* db,
+ Version* version) {
+ // Current number of entries in the immutable memtables
+ *value = cfd_->imm()->current()->GetTotalNumDeletes();
+ return true;
+}
+
+bool InternalStats::HandleEstimateNumKeys(uint64_t* value, DBImpl* db,
+ Version* version) {
+ // Estimate number of entries in the column family:
+ // Use estimated entries in tables + total entries in memtables.
+ const auto* vstorage = cfd_->current()->storage_info();
+ uint64_t estimate_keys = cfd_->mem()->num_entries() +
+ cfd_->imm()->current()->GetTotalNumEntries() +
+ vstorage->GetEstimatedActiveKeys();
+ uint64_t estimate_deletes =
+ cfd_->mem()->num_deletes() + cfd_->imm()->current()->GetTotalNumDeletes();
+ *value = estimate_keys > estimate_deletes * 2
+ ? estimate_keys - (estimate_deletes * 2)
+ : 0;
+ return true;
+}
+
+bool InternalStats::HandleNumSnapshots(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = db->snapshots().count();
+ return true;
+}
+
+bool InternalStats::HandleOldestSnapshotTime(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotTime());
+ return true;
+}
+
+bool InternalStats::HandleNumLiveVersions(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = cfd_->GetNumLiveVersions();
+ return true;
+}
+
+bool InternalStats::HandleCurrentSuperVersionNumber(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = cfd_->GetSuperVersionNumber();
+ return true;
+}
+
+bool InternalStats::HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = db->IsFileDeletionsEnabled();
+ return true;
+}
+
+bool InternalStats::HandleBaseLevel(uint64_t* value, DBImpl* db,
+ Version* version) {
+ const auto* vstorage = cfd_->current()->storage_info();
+ *value = vstorage->base_level();
+ return true;
+}
+
+bool InternalStats::HandleTotalSstFilesSize(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = cfd_->GetTotalSstFilesSize();
+ return true;
+}
+
+bool InternalStats::HandleEstimatePendingCompactionBytes(uint64_t* value,
+ DBImpl* db,
+ Version* version) {
+ const auto* vstorage = cfd_->current()->storage_info();
+ *value = vstorage->estimated_compaction_needed_bytes();
+ return true;
+}
+
+bool InternalStats::HandleEstimateTableReadersMem(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = (version == nullptr) ? 0 : version->GetMemoryUsageByTableReaders();
+ return true;
+}
+
+bool InternalStats::HandleEstimateLiveDataSize(uint64_t* value, DBImpl* db,
+ Version* version) {
+ const auto* vstorage = cfd_->current()->storage_info();
+ *value = vstorage->EstimateLiveDataSize();
+ return true;
+}
+
+bool InternalStats::HandleMinLogNumberToKeep(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = db->MinLogNumberToKeep();
+ return true;
+}
+
+bool InternalStats::HandleActualDelayedWriteRate(uint64_t* value, DBImpl* db,
+ Version* version) {
+ const WriteController& wc = db->write_controller();
+ if (!wc.NeedsDelay()) {
+ *value = 0;
+ } else {
+ *value = wc.delayed_write_rate();
+ }
+ return true;
+}
+
+bool InternalStats::HandleIsWriteStopped(uint64_t* value, DBImpl* db,
+ Version* version) {
+ *value = db->write_controller().IsStopped() ? 1 : 0;
+ return true;
+}
+
+void InternalStats::DumpDBStats(std::string* value) {
+ char buf[1000];
+ // DB-level stats, only available from default column family
+ double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec;
+ double interval_seconds_up = seconds_up - db_stats_snapshot_.seconds_up;
+ snprintf(buf, sizeof(buf),
+ "\n** DB Stats **\nUptime(secs): %.1f total, %.1f interval\n",
+ seconds_up, interval_seconds_up);
+ value->append(buf);
+ // Cumulative
+ uint64_t user_bytes_written = GetDBStats(InternalStats::BYTES_WRITTEN);
+ uint64_t num_keys_written = GetDBStats(InternalStats::NUMBER_KEYS_WRITTEN);
+ uint64_t write_other = GetDBStats(InternalStats::WRITE_DONE_BY_OTHER);
+ uint64_t write_self = GetDBStats(InternalStats::WRITE_DONE_BY_SELF);
+ uint64_t wal_bytes = GetDBStats(InternalStats::WAL_FILE_BYTES);
+ uint64_t wal_synced = GetDBStats(InternalStats::WAL_FILE_SYNCED);
+ uint64_t write_with_wal = GetDBStats(InternalStats::WRITE_WITH_WAL);
+ uint64_t write_stall_micros = GetDBStats(InternalStats::WRITE_STALL_MICROS);
+
+ const int kHumanMicrosLen = 32;
+ char human_micros[kHumanMicrosLen];
+
+ // Data
+ // writes: total number of write requests.
+ // keys: total number of key updates issued by all the write requests
+ // commit groups: number of group commits issued to the DB. Each group can
+ // contain one or more writes.
+ // so writes/keys is the average number of put in multi-put or put
+ // writes/groups is the average group commit size.
+ //
+ // The format is the same for interval stats.
+ snprintf(buf, sizeof(buf),
+ "Cumulative writes: %s writes, %s keys, %s commit groups, "
+ "%.1f writes per commit group, ingest: %.2f GB, %.2f MB/s\n",
+ NumberToHumanString(write_other + write_self).c_str(),
+ NumberToHumanString(num_keys_written).c_str(),
+ NumberToHumanString(write_self).c_str(),
+ (write_other + write_self) / static_cast<double>(write_self + 1),
+ user_bytes_written / kGB, user_bytes_written / kMB / seconds_up);
+ value->append(buf);
+ // WAL
+ snprintf(buf, sizeof(buf),
+ "Cumulative WAL: %s writes, %s syncs, "
+ "%.2f writes per sync, written: %.2f GB, %.2f MB/s\n",
+ NumberToHumanString(write_with_wal).c_str(),
+ NumberToHumanString(wal_synced).c_str(),
+ write_with_wal / static_cast<double>(wal_synced + 1),
+ wal_bytes / kGB, wal_bytes / kMB / seconds_up);
+ value->append(buf);
+ // Stall
+ AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true);
+ snprintf(buf, sizeof(buf),
+ "Cumulative stall: %s, %.1f percent\n",
+ human_micros,
+ // 10000 = divide by 1M to get secs, then multiply by 100 for pct
+ write_stall_micros / 10000.0 / std::max(seconds_up, 0.001));
+ value->append(buf);
+
+ // Interval
+ uint64_t interval_write_other = write_other - db_stats_snapshot_.write_other;
+ uint64_t interval_write_self = write_self - db_stats_snapshot_.write_self;
+ uint64_t interval_num_keys_written =
+ num_keys_written - db_stats_snapshot_.num_keys_written;
+ snprintf(buf, sizeof(buf),
+ "Interval writes: %s writes, %s keys, %s commit groups, "
+ "%.1f writes per commit group, ingest: %.2f MB, %.2f MB/s\n",
+ NumberToHumanString(
+ interval_write_other + interval_write_self).c_str(),
+ NumberToHumanString(interval_num_keys_written).c_str(),
+ NumberToHumanString(interval_write_self).c_str(),
+ static_cast<double>(interval_write_other + interval_write_self) /
+ (interval_write_self + 1),
+ (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB,
+ (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB /
+ std::max(interval_seconds_up, 0.001)),
+ value->append(buf);
+
+ uint64_t interval_write_with_wal =
+ write_with_wal - db_stats_snapshot_.write_with_wal;
+ uint64_t interval_wal_synced = wal_synced - db_stats_snapshot_.wal_synced;
+ uint64_t interval_wal_bytes = wal_bytes - db_stats_snapshot_.wal_bytes;
+
+ snprintf(buf, sizeof(buf),
+ "Interval WAL: %s writes, %s syncs, "
+ "%.2f writes per sync, written: %.2f MB, %.2f MB/s\n",
+ NumberToHumanString(interval_write_with_wal).c_str(),
+ NumberToHumanString(interval_wal_synced).c_str(),
+ interval_write_with_wal /
+ static_cast<double>(interval_wal_synced + 1),
+ interval_wal_bytes / kGB,
+ interval_wal_bytes / kMB / std::max(interval_seconds_up, 0.001));
+ value->append(buf);
+
+ // Stall
+ AppendHumanMicros(
+ write_stall_micros - db_stats_snapshot_.write_stall_micros,
+ human_micros, kHumanMicrosLen, true);
+ snprintf(buf, sizeof(buf),
+ "Interval stall: %s, %.1f percent\n",
+ human_micros,
+ // 10000 = divide by 1M to get secs, then multiply by 100 for pct
+ (write_stall_micros - db_stats_snapshot_.write_stall_micros) /
+ 10000.0 / std::max(interval_seconds_up, 0.001));
+ value->append(buf);
+
+ db_stats_snapshot_.seconds_up = seconds_up;
+ db_stats_snapshot_.ingest_bytes = user_bytes_written;
+ db_stats_snapshot_.write_other = write_other;
+ db_stats_snapshot_.write_self = write_self;
+ db_stats_snapshot_.num_keys_written = num_keys_written;
+ db_stats_snapshot_.wal_bytes = wal_bytes;
+ db_stats_snapshot_.wal_synced = wal_synced;
+ db_stats_snapshot_.write_with_wal = write_with_wal;
+ db_stats_snapshot_.write_stall_micros = write_stall_micros;
+}
+
+/**
+ * Dump Compaction Level stats to a map of stat name to value in double.
+ * The level in stat name is represented with a prefix "Lx" where "x"
+ * is the level number. A special level "Sum" represents the sum of a stat
+ * for all levels.
+ */
+void InternalStats::DumpCFMapStats(std::map<std::string, double>* cf_stats) {
+ CompactionStats compaction_stats_sum(0);
+ std::map<int, std::map<LevelStatType, double>> levels_stats;
+ DumpCFMapStats(&levels_stats, &compaction_stats_sum);
+ for (auto const& level_ent : levels_stats) {
+ auto level_str =
+ level_ent.first == -1 ? "Sum" : "L" + ToString(level_ent.first);
+ for (auto const& stat_ent : level_ent.second) {
+ auto stat_type = stat_ent.first;
+ auto key_str =
+ level_str + "." +
+ InternalStats::compaction_level_stats.at(stat_type).property_name;
+ (*cf_stats)[key_str] = stat_ent.second;
+ }
+ }
+}
+
+void InternalStats::DumpCFMapStats(
+ std::map<int, std::map<LevelStatType, double>>* levels_stats,
+ CompactionStats* compaction_stats_sum) {
+ const VersionStorageInfo* vstorage = cfd_->current()->storage_info();
+
+ int num_levels_to_check =
+ (cfd_->ioptions()->compaction_style != kCompactionStyleFIFO)
+ ? vstorage->num_levels() - 1
+ : 1;
+
+ // Compaction scores are sorted based on its value. Restore them to the
+ // level order
+ std::vector<double> compaction_score(number_levels_, 0);
+ for (int i = 0; i < num_levels_to_check; ++i) {
+ compaction_score[vstorage->CompactionScoreLevel(i)] =
+ vstorage->CompactionScore(i);
+ }
+ // Count # of files being compacted for each level
+ std::vector<int> files_being_compacted(number_levels_, 0);
+ for (int level = 0; level < number_levels_; ++level) {
+ for (auto* f : vstorage->LevelFiles(level)) {
+ if (f->being_compacted) {
+ ++files_being_compacted[level];
+ }
+ }
+ }
+
+ int total_files = 0;
+ int total_files_being_compacted = 0;
+ double total_file_size = 0;
+ uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
+ uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
+ uint64_t curr_ingest = flush_ingest + add_file_ingest;
+ for (int level = 0; level < number_levels_; level++) {
+ int files = vstorage->NumLevelFiles(level);
+ total_files += files;
+ total_files_being_compacted += files_being_compacted[level];
+ if (comp_stats_[level].micros > 0 || files > 0) {
+ compaction_stats_sum->Add(comp_stats_[level]);
+ total_file_size += vstorage->NumLevelBytes(level);
+ uint64_t input_bytes;
+ if (level == 0) {
+ input_bytes = curr_ingest;
+ } else {
+ input_bytes = comp_stats_[level].bytes_read_non_output_levels;
+ }
+ double w_amp =
+ (input_bytes == 0)
+ ? 0.0
+ : static_cast<double>(comp_stats_[level].bytes_written) /
+ input_bytes;
+ std::map<LevelStatType, double> level_stats;
+ PrepareLevelStats(&level_stats, files, files_being_compacted[level],
+ static_cast<double>(vstorage->NumLevelBytes(level)),
+ compaction_score[level], w_amp, comp_stats_[level]);
+ (*levels_stats)[level] = level_stats;
+ }
+ }
+ // Cumulative summary
+ double w_amp = compaction_stats_sum->bytes_written /
+ static_cast<double>(curr_ingest + 1);
+ // Stats summary across levels
+ std::map<LevelStatType, double> sum_stats;
+ PrepareLevelStats(&sum_stats, total_files, total_files_being_compacted,
+ total_file_size, 0, w_amp, *compaction_stats_sum);
+ (*levels_stats)[-1] = sum_stats; // -1 is for the Sum level
+}
+
+void InternalStats::DumpCFStats(std::string* value) {
+ DumpCFStatsNoFileHistogram(value);
+ DumpCFFileHistogram(value);
+}
+
+void InternalStats::DumpCFStatsNoFileHistogram(std::string* value) {
+ char buf[2000];
+ // Per-ColumnFamily stats
+ PrintLevelStatsHeader(buf, sizeof(buf), cfd_->GetName());
+ value->append(buf);
+
+ // Print stats for each level
+ std::map<int, std::map<LevelStatType, double>> levels_stats;
+ CompactionStats compaction_stats_sum(0);
+ DumpCFMapStats(&levels_stats, &compaction_stats_sum);
+ for (int l = 0; l < number_levels_; ++l) {
+ if (levels_stats.find(l) != levels_stats.end()) {
+ PrintLevelStats(buf, sizeof(buf), "L" + ToString(l), levels_stats[l]);
+ value->append(buf);
+ }
+ }
+
+ // Print sum of level stats
+ PrintLevelStats(buf, sizeof(buf), "Sum", levels_stats[-1]);
+ value->append(buf);
+
+ uint64_t flush_ingest = cf_stats_value_[BYTES_FLUSHED];
+ uint64_t add_file_ingest = cf_stats_value_[BYTES_INGESTED_ADD_FILE];
+ uint64_t ingest_files_addfile = cf_stats_value_[INGESTED_NUM_FILES_TOTAL];
+ uint64_t ingest_l0_files_addfile =
+ cf_stats_value_[INGESTED_LEVEL0_NUM_FILES_TOTAL];
+ uint64_t ingest_keys_addfile = cf_stats_value_[INGESTED_NUM_KEYS_TOTAL];
+ // Cumulative summary
+ uint64_t total_stall_count =
+ cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] +
+ cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] +
+ cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT] +
+ cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] +
+ cf_stats_count_[MEMTABLE_COMPACTION] + cf_stats_count_[MEMTABLE_SLOWDOWN];
+ // Interval summary
+ uint64_t interval_flush_ingest =
+ flush_ingest - cf_stats_snapshot_.ingest_bytes_flush;
+ uint64_t interval_add_file_inget =
+ add_file_ingest - cf_stats_snapshot_.ingest_bytes_addfile;
+ uint64_t interval_ingest =
+ interval_flush_ingest + interval_add_file_inget + 1;
+ CompactionStats interval_stats(compaction_stats_sum);
+ interval_stats.Subtract(cf_stats_snapshot_.comp_stats);
+ double w_amp =
+ interval_stats.bytes_written / static_cast<double>(interval_ingest);
+ PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0, w_amp, interval_stats);
+ value->append(buf);
+
+ double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec;
+ double interval_seconds_up = seconds_up - cf_stats_snapshot_.seconds_up;
+ snprintf(buf, sizeof(buf), "Uptime(secs): %.1f total, %.1f interval\n",
+ seconds_up, interval_seconds_up);
+ value->append(buf);
+ snprintf(buf, sizeof(buf), "Flush(GB): cumulative %.3f, interval %.3f\n",
+ flush_ingest / kGB, interval_flush_ingest / kGB);
+ value->append(buf);
+ snprintf(buf, sizeof(buf), "AddFile(GB): cumulative %.3f, interval %.3f\n",
+ add_file_ingest / kGB, interval_add_file_inget / kGB);
+ value->append(buf);
+
+ uint64_t interval_ingest_files_addfile =
+ ingest_files_addfile - cf_stats_snapshot_.ingest_files_addfile;
+ snprintf(buf, sizeof(buf), "AddFile(Total Files): cumulative %" PRIu64
+ ", interval %" PRIu64 "\n",
+ ingest_files_addfile, interval_ingest_files_addfile);
+ value->append(buf);
+
+ uint64_t interval_ingest_l0_files_addfile =
+ ingest_l0_files_addfile - cf_stats_snapshot_.ingest_l0_files_addfile;
+ snprintf(buf, sizeof(buf),
+ "AddFile(L0 Files): cumulative %" PRIu64 ", interval %" PRIu64 "\n",
+ ingest_l0_files_addfile, interval_ingest_l0_files_addfile);
+ value->append(buf);
+
+ uint64_t interval_ingest_keys_addfile =
+ ingest_keys_addfile - cf_stats_snapshot_.ingest_keys_addfile;
+ snprintf(buf, sizeof(buf),
+ "AddFile(Keys): cumulative %" PRIu64 ", interval %" PRIu64 "\n",
+ ingest_keys_addfile, interval_ingest_keys_addfile);
+ value->append(buf);
+
+ // Compact
+ uint64_t compact_bytes_read = 0;
+ uint64_t compact_bytes_write = 0;
+ uint64_t compact_micros = 0;
+ for (int level = 0; level < number_levels_; level++) {
+ compact_bytes_read += comp_stats_[level].bytes_read_output_level +
+ comp_stats_[level].bytes_read_non_output_levels;
+ compact_bytes_write += comp_stats_[level].bytes_written;
+ compact_micros += comp_stats_[level].micros;
+ }
+
+ snprintf(buf, sizeof(buf),
+ "Cumulative compaction: %.2f GB write, %.2f MB/s write, "
+ "%.2f GB read, %.2f MB/s read, %.1f seconds\n",
+ compact_bytes_write / kGB, compact_bytes_write / kMB / seconds_up,
+ compact_bytes_read / kGB, compact_bytes_read / kMB / seconds_up,
+ compact_micros / kMicrosInSec);
+ value->append(buf);
+
+ // Compaction interval
+ uint64_t interval_compact_bytes_write =
+ compact_bytes_write - cf_stats_snapshot_.compact_bytes_write;
+ uint64_t interval_compact_bytes_read =
+ compact_bytes_read - cf_stats_snapshot_.compact_bytes_read;
+ uint64_t interval_compact_micros =
+ compact_micros - cf_stats_snapshot_.compact_micros;
+
+ snprintf(
+ buf, sizeof(buf),
+ "Interval compaction: %.2f GB write, %.2f MB/s write, "
+ "%.2f GB read, %.2f MB/s read, %.1f seconds\n",
+ interval_compact_bytes_write / kGB,
+ interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001),
+ interval_compact_bytes_read / kGB,
+ interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001),
+ interval_compact_micros / kMicrosInSec);
+ value->append(buf);
+ cf_stats_snapshot_.compact_bytes_write = compact_bytes_write;
+ cf_stats_snapshot_.compact_bytes_read = compact_bytes_read;
+ cf_stats_snapshot_.compact_micros = compact_micros;
+
+ snprintf(buf, sizeof(buf), "Stalls(count): %" PRIu64
+ " level0_slowdown, "
+ "%" PRIu64
+ " level0_slowdown_with_compaction, "
+ "%" PRIu64
+ " level0_numfiles, "
+ "%" PRIu64
+ " level0_numfiles_with_compaction, "
+ "%" PRIu64
+ " stop for pending_compaction_bytes, "
+ "%" PRIu64
+ " slowdown for pending_compaction_bytes, "
+ "%" PRIu64
+ " memtable_compaction, "
+ "%" PRIu64
+ " memtable_slowdown, "
+ "interval %" PRIu64 " total count\n",
+ cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL],
+ cf_stats_count_[LEVEL0_SLOWDOWN_WITH_COMPACTION],
+ cf_stats_count_[LEVEL0_NUM_FILES_TOTAL],
+ cf_stats_count_[LEVEL0_NUM_FILES_WITH_COMPACTION],
+ cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT],
+ cf_stats_count_[SOFT_PENDING_COMPACTION_BYTES_LIMIT],
+ cf_stats_count_[MEMTABLE_COMPACTION],
+ cf_stats_count_[MEMTABLE_SLOWDOWN],
+ total_stall_count - cf_stats_snapshot_.stall_count);
+ value->append(buf);
+
+ cf_stats_snapshot_.seconds_up = seconds_up;
+ cf_stats_snapshot_.ingest_bytes_flush = flush_ingest;
+ cf_stats_snapshot_.ingest_bytes_addfile = add_file_ingest;
+ cf_stats_snapshot_.ingest_files_addfile = ingest_files_addfile;
+ cf_stats_snapshot_.ingest_l0_files_addfile = ingest_l0_files_addfile;
+ cf_stats_snapshot_.ingest_keys_addfile = ingest_keys_addfile;
+ cf_stats_snapshot_.comp_stats = compaction_stats_sum;
+ cf_stats_snapshot_.stall_count = total_stall_count;
+}
+
+void InternalStats::DumpCFFileHistogram(std::string* value) {
+ char buf[2000];
+ snprintf(buf, sizeof(buf),
+ "\n** File Read Latency Histogram By Level [%s] **\n",
+ cfd_->GetName().c_str());
+ value->append(buf);
+
+ for (int level = 0; level < number_levels_; level++) {
+ if (!file_read_latency_[level].Empty()) {
+ char buf2[5000];
+ snprintf(buf2, sizeof(buf2),
+ "** Level %d read latency histogram (micros):\n%s\n", level,
+ file_read_latency_[level].ToString().c_str());
+ value->append(buf2);
+ }
+ }
+}
+
+#else
+
+const DBPropertyInfo* GetPropertyInfo(const Slice& property) { return nullptr; }
+
+#endif // !ROCKSDB_LITE
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/internal_stats.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/internal_stats.h b/thirdparty/rocksdb/db/internal_stats.h
new file mode 100644
index 0000000..1dd393f
--- /dev/null
+++ b/thirdparty/rocksdb/db/internal_stats.h
@@ -0,0 +1,588 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+
+#pragma once
+#include <map>
+#include <string>
+#include <vector>
+
+#include "db/version_set.h"
+
+class ColumnFamilyData;
+
+namespace rocksdb {
+
+class MemTableList;
+class DBImpl;
+
+// Config for retrieving a property's value.
+struct DBPropertyInfo {
+ bool need_out_of_mutex;
+
+ // gcc had an internal error for initializing union of pointer-to-member-
+ // functions. Workaround is to populate exactly one of the following function
+ // pointers with a non-nullptr value.
+
+ // @param value Value-result argument for storing the property's string value
+ // @param suffix Argument portion of the property. For example, suffix would
+ // be "5" for the property "rocksdb.num-files-at-level5". So far, only
+ // certain string properties take an argument.
+ bool (InternalStats::*handle_string)(std::string* value, Slice suffix);
+
+ // @param value Value-result argument for storing the property's uint64 value
+ // @param db Many of the int properties rely on DBImpl methods.
+ // @param version Version is needed in case the property is retrieved without
+ // holding db mutex, which is only supported for int properties.
+ bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool (InternalStats::*handle_map)(
+ std::map<std::string, double>* compaction_stats);
+};
+
+extern const DBPropertyInfo* GetPropertyInfo(const Slice& property);
+
+#ifndef ROCKSDB_LITE
+#undef SCORE
+enum class LevelStatType {
+ INVALID = 0,
+ NUM_FILES,
+ COMPACTED_FILES,
+ SIZE_BYTES,
+ SCORE,
+ READ_GB,
+ RN_GB,
+ RNP1_GB,
+ WRITE_GB,
+ W_NEW_GB,
+ MOVED_GB,
+ WRITE_AMP,
+ READ_MBPS,
+ WRITE_MBPS,
+ COMP_SEC,
+ COMP_COUNT,
+ AVG_SEC,
+ KEY_IN,
+ KEY_DROP,
+ TOTAL // total number of types
+};
+
+struct LevelStat {
+ // This what will be L?.property_name in the flat map returned to the user
+ std::string property_name;
+ // This will be what we will print in the header in the cli
+ std::string header_name;
+};
+
+class InternalStats {
+ public:
+ static const std::map<LevelStatType, LevelStat> compaction_level_stats;
+
+ enum InternalCFStatsType {
+ LEVEL0_SLOWDOWN_TOTAL,
+ LEVEL0_SLOWDOWN_WITH_COMPACTION,
+ MEMTABLE_COMPACTION,
+ MEMTABLE_SLOWDOWN,
+ LEVEL0_NUM_FILES_TOTAL,
+ LEVEL0_NUM_FILES_WITH_COMPACTION,
+ SOFT_PENDING_COMPACTION_BYTES_LIMIT,
+ HARD_PENDING_COMPACTION_BYTES_LIMIT,
+ WRITE_STALLS_ENUM_MAX,
+ BYTES_FLUSHED,
+ BYTES_INGESTED_ADD_FILE,
+ INGESTED_NUM_FILES_TOTAL,
+ INGESTED_LEVEL0_NUM_FILES_TOTAL,
+ INGESTED_NUM_KEYS_TOTAL,
+ INTERNAL_CF_STATS_ENUM_MAX,
+ };
+
+ enum InternalDBStatsType {
+ WAL_FILE_BYTES,
+ WAL_FILE_SYNCED,
+ BYTES_WRITTEN,
+ NUMBER_KEYS_WRITTEN,
+ WRITE_DONE_BY_OTHER,
+ WRITE_DONE_BY_SELF,
+ WRITE_WITH_WAL,
+ WRITE_STALL_MICROS,
+ INTERNAL_DB_STATS_ENUM_MAX,
+ };
+
+ InternalStats(int num_levels, Env* env, ColumnFamilyData* cfd)
+ : db_stats_{},
+ cf_stats_value_{},
+ cf_stats_count_{},
+ comp_stats_(num_levels),
+ file_read_latency_(num_levels),
+ bg_error_count_(0),
+ number_levels_(num_levels),
+ env_(env),
+ cfd_(cfd),
+ started_at_(env->NowMicros()) {}
+
+ // Per level compaction stats. comp_stats_[level] stores the stats for
+ // compactions that produced data for the specified "level".
+ struct CompactionStats {
+ uint64_t micros;
+
+ // The number of bytes read from all non-output levels
+ uint64_t bytes_read_non_output_levels;
+
+ // The number of bytes read from the compaction output level.
+ uint64_t bytes_read_output_level;
+
+ // Total number of bytes written during compaction
+ uint64_t bytes_written;
+
+ // Total number of bytes moved to the output level
+ uint64_t bytes_moved;
+
+ // The number of compaction input files in all non-output levels.
+ int num_input_files_in_non_output_levels;
+
+ // The number of compaction input files in the output level.
+ int num_input_files_in_output_level;
+
+ // The number of compaction output files.
+ int num_output_files;
+
+ // Total incoming entries during compaction between levels N and N+1
+ uint64_t num_input_records;
+
+ // Accumulated diff number of entries
+ // (num input entries - num output entires) for compaction levels N and N+1
+ uint64_t num_dropped_records;
+
+ // Number of compactions done
+ int count;
+
+ explicit CompactionStats(int _count = 0)
+ : micros(0),
+ bytes_read_non_output_levels(0),
+ bytes_read_output_level(0),
+ bytes_written(0),
+ bytes_moved(0),
+ num_input_files_in_non_output_levels(0),
+ num_input_files_in_output_level(0),
+ num_output_files(0),
+ num_input_records(0),
+ num_dropped_records(0),
+ count(_count) {}
+
+ explicit CompactionStats(const CompactionStats& c)
+ : micros(c.micros),
+ bytes_read_non_output_levels(c.bytes_read_non_output_levels),
+ bytes_read_output_level(c.bytes_read_output_level),
+ bytes_written(c.bytes_written),
+ bytes_moved(c.bytes_moved),
+ num_input_files_in_non_output_levels(
+ c.num_input_files_in_non_output_levels),
+ num_input_files_in_output_level(
+ c.num_input_files_in_output_level),
+ num_output_files(c.num_output_files),
+ num_input_records(c.num_input_records),
+ num_dropped_records(c.num_dropped_records),
+ count(c.count) {}
+
+ void Clear() {
+ this->micros = 0;
+ this->bytes_read_non_output_levels = 0;
+ this->bytes_read_output_level = 0;
+ this->bytes_written = 0;
+ this->bytes_moved = 0;
+ this->num_input_files_in_non_output_levels = 0;
+ this->num_input_files_in_output_level = 0;
+ this->num_output_files = 0;
+ this->num_input_records = 0;
+ this->num_dropped_records = 0;
+ this->count = 0;
+ }
+
+ void Add(const CompactionStats& c) {
+ this->micros += c.micros;
+ this->bytes_read_non_output_levels += c.bytes_read_non_output_levels;
+ this->bytes_read_output_level += c.bytes_read_output_level;
+ this->bytes_written += c.bytes_written;
+ this->bytes_moved += c.bytes_moved;
+ this->num_input_files_in_non_output_levels +=
+ c.num_input_files_in_non_output_levels;
+ this->num_input_files_in_output_level +=
+ c.num_input_files_in_output_level;
+ this->num_output_files += c.num_output_files;
+ this->num_input_records += c.num_input_records;
+ this->num_dropped_records += c.num_dropped_records;
+ this->count += c.count;
+ }
+
+ void Subtract(const CompactionStats& c) {
+ this->micros -= c.micros;
+ this->bytes_read_non_output_levels -= c.bytes_read_non_output_levels;
+ this->bytes_read_output_level -= c.bytes_read_output_level;
+ this->bytes_written -= c.bytes_written;
+ this->bytes_moved -= c.bytes_moved;
+ this->num_input_files_in_non_output_levels -=
+ c.num_input_files_in_non_output_levels;
+ this->num_input_files_in_output_level -=
+ c.num_input_files_in_output_level;
+ this->num_output_files -= c.num_output_files;
+ this->num_input_records -= c.num_input_records;
+ this->num_dropped_records -= c.num_dropped_records;
+ this->count -= c.count;
+ }
+ };
+
+ void Clear() {
+ for (int i = 0; i < INTERNAL_DB_STATS_ENUM_MAX; i++) {
+ db_stats_[i].store(0);
+ }
+ for (int i = 0; i < INTERNAL_CF_STATS_ENUM_MAX; i++) {
+ cf_stats_count_[i] = 0;
+ cf_stats_value_[i] = 0;
+ }
+ for (auto& comp_stat : comp_stats_) {
+ comp_stat.Clear();
+ }
+ for (auto& h : file_read_latency_) {
+ h.Clear();
+ }
+ cf_stats_snapshot_.Clear();
+ db_stats_snapshot_.Clear();
+ bg_error_count_ = 0;
+ started_at_ = env_->NowMicros();
+ }
+
+ void AddCompactionStats(int level, const CompactionStats& stats) {
+ comp_stats_[level].Add(stats);
+ }
+
+ void IncBytesMoved(int level, uint64_t amount) {
+ comp_stats_[level].bytes_moved += amount;
+ }
+
+ void AddCFStats(InternalCFStatsType type, uint64_t value) {
+ cf_stats_value_[type] += value;
+ ++cf_stats_count_[type];
+ }
+
+ void AddDBStats(InternalDBStatsType type, uint64_t value,
+ bool concurrent = false) {
+ auto& v = db_stats_[type];
+ if (concurrent) {
+ v.fetch_add(value, std::memory_order_relaxed);
+ } else {
+ v.store(v.load(std::memory_order_relaxed) + value,
+ std::memory_order_relaxed);
+ }
+ }
+
+ uint64_t GetDBStats(InternalDBStatsType type) {
+ return db_stats_[type].load(std::memory_order_relaxed);
+ }
+
+ HistogramImpl* GetFileReadHist(int level) {
+ return &file_read_latency_[level];
+ }
+
+ uint64_t GetBackgroundErrorCount() const { return bg_error_count_; }
+
+ uint64_t BumpAndGetBackgroundErrorCount() { return ++bg_error_count_; }
+
+ bool GetStringProperty(const DBPropertyInfo& property_info,
+ const Slice& property, std::string* value);
+
+ bool GetMapProperty(const DBPropertyInfo& property_info,
+ const Slice& property,
+ std::map<std::string, double>* value);
+
+ bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
+ DBImpl* db);
+
+ bool GetIntPropertyOutOfMutex(const DBPropertyInfo& property_info,
+ Version* version, uint64_t* value);
+
+ // Store a mapping from the user-facing DB::Properties string to our
+ // DBPropertyInfo struct used internally for retrieving properties.
+ static const std::unordered_map<std::string, DBPropertyInfo> ppt_name_to_info;
+
+ private:
+ void DumpDBStats(std::string* value);
+ void DumpCFMapStats(std::map<std::string, double>* cf_stats);
+ void DumpCFMapStats(
+ std::map<int, std::map<LevelStatType, double>>* level_stats,
+ CompactionStats* compaction_stats_sum);
+ void DumpCFStats(std::string* value);
+ void DumpCFStatsNoFileHistogram(std::string* value);
+ void DumpCFFileHistogram(std::string* value);
+
+ // Per-DB stats
+ std::atomic<uint64_t> db_stats_[INTERNAL_DB_STATS_ENUM_MAX];
+ // Per-ColumnFamily stats
+ uint64_t cf_stats_value_[INTERNAL_CF_STATS_ENUM_MAX];
+ uint64_t cf_stats_count_[INTERNAL_CF_STATS_ENUM_MAX];
+ // Per-ColumnFamily/level compaction stats
+ std::vector<CompactionStats> comp_stats_;
+ std::vector<HistogramImpl> file_read_latency_;
+
+ // Used to compute per-interval statistics
+ struct CFStatsSnapshot {
+ // ColumnFamily-level stats
+ CompactionStats comp_stats;
+ uint64_t ingest_bytes_flush; // Bytes written to L0 (Flush)
+ uint64_t stall_count; // Stall count
+ // Stats from compaction jobs - bytes written, bytes read, duration.
+ uint64_t compact_bytes_write;
+ uint64_t compact_bytes_read;
+ uint64_t compact_micros;
+ double seconds_up;
+
+ // AddFile specific stats
+ uint64_t ingest_bytes_addfile; // Total Bytes ingested
+ uint64_t ingest_files_addfile; // Total number of files ingested
+ uint64_t ingest_l0_files_addfile; // Total number of files ingested to L0
+ uint64_t ingest_keys_addfile; // Total number of keys ingested
+
+ CFStatsSnapshot()
+ : comp_stats(0),
+ ingest_bytes_flush(0),
+ stall_count(0),
+ compact_bytes_write(0),
+ compact_bytes_read(0),
+ compact_micros(0),
+ seconds_up(0),
+ ingest_bytes_addfile(0),
+ ingest_files_addfile(0),
+ ingest_l0_files_addfile(0),
+ ingest_keys_addfile(0) {}
+
+ void Clear() {
+ comp_stats.Clear();
+ ingest_bytes_flush = 0;
+ stall_count = 0;
+ compact_bytes_write = 0;
+ compact_bytes_read = 0;
+ compact_micros = 0;
+ seconds_up = 0;
+ ingest_bytes_addfile = 0;
+ ingest_files_addfile = 0;
+ ingest_l0_files_addfile = 0;
+ ingest_keys_addfile = 0;
+ }
+ } cf_stats_snapshot_;
+
+ struct DBStatsSnapshot {
+ // DB-level stats
+ uint64_t ingest_bytes; // Bytes written by user
+ uint64_t wal_bytes; // Bytes written to WAL
+ uint64_t wal_synced; // Number of times WAL is synced
+ uint64_t write_with_wal; // Number of writes that request WAL
+ // These count the number of writes processed by the calling thread or
+ // another thread.
+ uint64_t write_other;
+ uint64_t write_self;
+ // Total number of keys written. write_self and write_other measure number
+ // of write requests written, Each of the write request can contain updates
+ // to multiple keys. num_keys_written is total number of keys updated by all
+ // those writes.
+ uint64_t num_keys_written;
+ // Total time writes delayed by stalls.
+ uint64_t write_stall_micros;
+ double seconds_up;
+
+ DBStatsSnapshot()
+ : ingest_bytes(0),
+ wal_bytes(0),
+ wal_synced(0),
+ write_with_wal(0),
+ write_other(0),
+ write_self(0),
+ num_keys_written(0),
+ write_stall_micros(0),
+ seconds_up(0) {}
+
+ void Clear() {
+ ingest_bytes = 0;
+ wal_bytes = 0;
+ wal_synced = 0;
+ write_with_wal = 0;
+ write_other = 0;
+ write_self = 0;
+ num_keys_written = 0;
+ write_stall_micros = 0;
+ seconds_up = 0;
+ }
+ } db_stats_snapshot_;
+
+ // Handler functions for getting property values. They use "value" as a value-
+ // result argument, and return true upon successfully setting "value".
+ bool HandleNumFilesAtLevel(std::string* value, Slice suffix);
+ bool HandleCompressionRatioAtLevelPrefix(std::string* value, Slice suffix);
+ bool HandleLevelStats(std::string* value, Slice suffix);
+ bool HandleStats(std::string* value, Slice suffix);
+ bool HandleCFMapStats(std::map<std::string, double>* compaction_stats);
+ bool HandleCFStats(std::string* value, Slice suffix);
+ bool HandleCFStatsNoFileHistogram(std::string* value, Slice suffix);
+ bool HandleCFFileHistogram(std::string* value, Slice suffix);
+ bool HandleDBStats(std::string* value, Slice suffix);
+ bool HandleSsTables(std::string* value, Slice suffix);
+ bool HandleAggregatedTableProperties(std::string* value, Slice suffix);
+ bool HandleAggregatedTablePropertiesAtLevel(std::string* value, Slice suffix);
+ bool HandleNumImmutableMemTable(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleNumImmutableMemTableFlushed(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleMemTableFlushPending(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleNumRunningFlushes(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleCompactionPending(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleNumRunningCompactions(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleBackgroundErrors(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleCurSizeActiveMemTable(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleCurSizeAllMemTables(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleSizeAllMemTables(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleNumEntriesActiveMemTable(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleNumEntriesImmMemTables(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleNumDeletesActiveMemTable(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleNumDeletesImmMemTables(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleEstimateNumKeys(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleNumSnapshots(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleOldestSnapshotTime(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleNumLiveVersions(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleCurrentSuperVersionNumber(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleBaseLevel(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleTotalSstFilesSize(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleEstimatePendingCompactionBytes(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleEstimateTableReadersMem(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleEstimateLiveDataSize(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleMinLogNumberToKeep(uint64_t* value, DBImpl* db, Version* version);
+ bool HandleActualDelayedWriteRate(uint64_t* value, DBImpl* db,
+ Version* version);
+ bool HandleIsWriteStopped(uint64_t* value, DBImpl* db, Version* version);
+
+ // Total number of background errors encountered. Every time a flush task
+ // or compaction task fails, this counter is incremented. The failure can
+ // be caused by any possible reason, including file system errors, out of
+ // resources, or input file corruption. Failing when retrying the same flush
+ // or compaction will cause the counter to increase too.
+ uint64_t bg_error_count_;
+
+ const int number_levels_;
+ Env* env_;
+ ColumnFamilyData* cfd_;
+ uint64_t started_at_;
+};
+
+#else
+
+class InternalStats {
+ public:
+ enum InternalCFStatsType {
+ LEVEL0_SLOWDOWN_TOTAL,
+ LEVEL0_SLOWDOWN_WITH_COMPACTION,
+ MEMTABLE_COMPACTION,
+ MEMTABLE_SLOWDOWN,
+ LEVEL0_NUM_FILES_TOTAL,
+ LEVEL0_NUM_FILES_WITH_COMPACTION,
+ SOFT_PENDING_COMPACTION_BYTES_LIMIT,
+ HARD_PENDING_COMPACTION_BYTES_LIMIT,
+ WRITE_STALLS_ENUM_MAX,
+ BYTES_FLUSHED,
+ BYTES_INGESTED_ADD_FILE,
+ INGESTED_NUM_FILES_TOTAL,
+ INGESTED_LEVEL0_NUM_FILES_TOTAL,
+ INGESTED_NUM_KEYS_TOTAL,
+ INTERNAL_CF_STATS_ENUM_MAX,
+ };
+
+ enum InternalDBStatsType {
+ WAL_FILE_BYTES,
+ WAL_FILE_SYNCED,
+ BYTES_WRITTEN,
+ NUMBER_KEYS_WRITTEN,
+ WRITE_DONE_BY_OTHER,
+ WRITE_DONE_BY_SELF,
+ WRITE_WITH_WAL,
+ WRITE_STALL_MICROS,
+ INTERNAL_DB_STATS_ENUM_MAX,
+ };
+
+ InternalStats(int num_levels, Env* env, ColumnFamilyData* cfd) {}
+
+ struct CompactionStats {
+ uint64_t micros;
+ uint64_t bytes_read_non_output_levels;
+ uint64_t bytes_read_output_level;
+ uint64_t bytes_written;
+ uint64_t bytes_moved;
+ int num_input_files_in_non_output_levels;
+ int num_input_files_in_output_level;
+ int num_output_files;
+ uint64_t num_input_records;
+ uint64_t num_dropped_records;
+ int count;
+
+ explicit CompactionStats(int _count = 0) {}
+
+ explicit CompactionStats(const CompactionStats& c) {}
+
+ void Add(const CompactionStats& c) {}
+
+ void Subtract(const CompactionStats& c) {}
+ };
+
+ void AddCompactionStats(int level, const CompactionStats& stats) {}
+
+ void IncBytesMoved(int level, uint64_t amount) {}
+
+ void AddCFStats(InternalCFStatsType type, uint64_t value) {}
+
+ void AddDBStats(InternalDBStatsType type, uint64_t value,
+ bool concurrent = false) {}
+
+ HistogramImpl* GetFileReadHist(int level) { return nullptr; }
+
+ uint64_t GetBackgroundErrorCount() const { return 0; }
+
+ uint64_t BumpAndGetBackgroundErrorCount() { return 0; }
+
+ bool GetStringProperty(const DBPropertyInfo& property_info,
+ const Slice& property, std::string* value) {
+ return false;
+ }
+
+ bool GetMapProperty(const DBPropertyInfo& property_info,
+ const Slice& property,
+ std::map<std::string, double>* value) {
+ return false;
+ }
+
+ bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
+ DBImpl* db) const {
+ return false;
+ }
+
+ bool GetIntPropertyOutOfMutex(const DBPropertyInfo& property_info,
+ Version* version, uint64_t* value) const {
+ return false;
+ }
+};
+#endif // !ROCKSDB_LITE
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/job_context.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/job_context.h b/thirdparty/rocksdb/db/job_context.h
new file mode 100644
index 0000000..950a3a6
--- /dev/null
+++ b/thirdparty/rocksdb/db/job_context.h
@@ -0,0 +1,129 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "db/log_writer.h"
+
+namespace rocksdb {
+
+class MemTable;
+
+struct JobContext {
+ inline bool HaveSomethingToDelete() const {
+ return full_scan_candidate_files.size() || sst_delete_files.size() ||
+ log_delete_files.size() || manifest_delete_files.size() ||
+ new_superversion != nullptr || superversions_to_free.size() > 0 ||
+ memtables_to_free.size() > 0 || logs_to_free.size() > 0;
+ }
+
+ // Structure to store information for candidate files to delete.
+ struct CandidateFileInfo {
+ std::string file_name;
+ uint32_t path_id;
+ CandidateFileInfo(std::string name, uint32_t path)
+ : file_name(std::move(name)), path_id(path) {}
+ bool operator==(const CandidateFileInfo& other) const {
+ return file_name == other.file_name && path_id == other.path_id;
+ }
+ };
+
+ // Unique job id
+ int job_id;
+
+ // a list of all files that we'll consider deleting
+ // (every once in a while this is filled up with all files
+ // in the DB directory)
+ // (filled only if we're doing full scan)
+ std::vector<CandidateFileInfo> full_scan_candidate_files;
+
+ // the list of all live sst files that cannot be deleted
+ std::vector<FileDescriptor> sst_live;
+
+ // a list of sst files that we need to delete
+ std::vector<FileMetaData*> sst_delete_files;
+
+ // a list of log files that we need to delete
+ std::vector<uint64_t> log_delete_files;
+
+ // a list of log files that we need to preserve during full purge since they
+ // will be reused later
+ std::vector<uint64_t> log_recycle_files;
+
+ // a list of manifest files that we need to delete
+ std::vector<std::string> manifest_delete_files;
+
+ // a list of memtables to be free
+ autovector<MemTable*> memtables_to_free;
+
+ autovector<SuperVersion*> superversions_to_free;
+
+ autovector<log::Writer*> logs_to_free;
+
+ SuperVersion* new_superversion; // if nullptr no new superversion
+
+ // the current manifest_file_number, log_number and prev_log_number
+ // that corresponds to the set of files in 'live'.
+ uint64_t manifest_file_number;
+ uint64_t pending_manifest_file_number;
+ uint64_t log_number;
+ uint64_t prev_log_number;
+
+ uint64_t min_pending_output = 0;
+ uint64_t prev_total_log_size = 0;
+ size_t num_alive_log_files = 0;
+ uint64_t size_log_to_delete = 0;
+
+ explicit JobContext(int _job_id, bool create_superversion = false) {
+ job_id = _job_id;
+ manifest_file_number = 0;
+ pending_manifest_file_number = 0;
+ log_number = 0;
+ prev_log_number = 0;
+ new_superversion = create_superversion ? new SuperVersion() : nullptr;
+ }
+
+ // For non-empty JobContext Clean() has to be called at least once before
+ // before destruction (see asserts in ~JobContext()). Should be called with
+ // unlocked DB mutex. Destructor doesn't call Clean() to avoid accidentally
+ // doing potentially slow Clean() with locked DB mutex.
+ void Clean() {
+ // free pending memtables
+ for (auto m : memtables_to_free) {
+ delete m;
+ }
+ // free superversions
+ for (auto s : superversions_to_free) {
+ delete s;
+ }
+ for (auto l : logs_to_free) {
+ delete l;
+ }
+ // if new_superversion was not used, it will be non-nullptr and needs
+ // to be freed here
+ delete new_superversion;
+
+ memtables_to_free.clear();
+ superversions_to_free.clear();
+ logs_to_free.clear();
+ new_superversion = nullptr;
+ }
+
+ ~JobContext() {
+ assert(memtables_to_free.size() == 0);
+ assert(superversions_to_free.size() == 0);
+ assert(new_superversion == nullptr);
+ assert(logs_to_free.size() == 0);
+ }
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/log_format.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/log_format.h b/thirdparty/rocksdb/db/log_format.h
new file mode 100644
index 0000000..be22201
--- /dev/null
+++ b/thirdparty/rocksdb/db/log_format.h
@@ -0,0 +1,45 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// Log format information shared by reader and writer.
+// See ../doc/log_format.txt for more detail.
+
+#pragma once
+namespace rocksdb {
+namespace log {
+
+enum RecordType {
+ // Zero is reserved for preallocated files
+ kZeroType = 0,
+ kFullType = 1,
+
+ // For fragments
+ kFirstType = 2,
+ kMiddleType = 3,
+ kLastType = 4,
+
+ // For recycled log files
+ kRecyclableFullType = 5,
+ kRecyclableFirstType = 6,
+ kRecyclableMiddleType = 7,
+ kRecyclableLastType = 8,
+};
+static const int kMaxRecordType = kRecyclableLastType;
+
+static const unsigned int kBlockSize = 32768;
+
+// Header is checksum (4 bytes), length (2 bytes), type (1 byte)
+static const int kHeaderSize = 4 + 2 + 1;
+
+// Recyclable header is checksum (4 bytes), type (1 byte), log number
+// (4 bytes), length (2 bytes).
+static const int kRecyclableHeaderSize = 4 + 1 + 4 + 2;
+
+} // namespace log
+} // namespace rocksdb
[06/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/write_thread.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/write_thread.cc b/thirdparty/rocksdb/db/write_thread.cc
new file mode 100644
index 0000000..2d3b346
--- /dev/null
+++ b/thirdparty/rocksdb/db/write_thread.cc
@@ -0,0 +1,656 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "db/write_thread.h"
+#include <chrono>
+#include <thread>
+#include "db/column_family.h"
+#include "port/port.h"
+#include "util/random.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+WriteThread::WriteThread(const ImmutableDBOptions& db_options)
+ : max_yield_usec_(db_options.enable_write_thread_adaptive_yield
+ ? db_options.write_thread_max_yield_usec
+ : 0),
+ slow_yield_usec_(db_options.write_thread_slow_yield_usec),
+ allow_concurrent_memtable_write_(
+ db_options.allow_concurrent_memtable_write),
+ enable_pipelined_write_(db_options.enable_pipelined_write),
+ newest_writer_(nullptr),
+ newest_memtable_writer_(nullptr),
+ last_sequence_(0) {}
+
+uint8_t WriteThread::BlockingAwaitState(Writer* w, uint8_t goal_mask) {
+ // We're going to block. Lazily create the mutex. We guarantee
+ // propagation of this construction to the waker via the
+ // STATE_LOCKED_WAITING state. The waker won't try to touch the mutex
+ // or the condvar unless they CAS away the STATE_LOCKED_WAITING that
+ // we install below.
+ w->CreateMutex();
+
+ auto state = w->state.load(std::memory_order_acquire);
+ assert(state != STATE_LOCKED_WAITING);
+ if ((state & goal_mask) == 0 &&
+ w->state.compare_exchange_strong(state, STATE_LOCKED_WAITING)) {
+ // we have permission (and an obligation) to use StateMutex
+ std::unique_lock<std::mutex> guard(w->StateMutex());
+ w->StateCV().wait(guard, [w] {
+ return w->state.load(std::memory_order_relaxed) != STATE_LOCKED_WAITING;
+ });
+ state = w->state.load(std::memory_order_relaxed);
+ }
+ // else tricky. Goal is met or CAS failed. In the latter case the waker
+ // must have changed the state, and compare_exchange_strong has updated
+ // our local variable with the new one. At the moment WriteThread never
+ // waits for a transition across intermediate states, so we know that
+ // since a state change has occurred the goal must have been met.
+ assert((state & goal_mask) != 0);
+ return state;
+}
+
+uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask,
+ AdaptationContext* ctx) {
+ uint8_t state;
+
+ // 1. Busy loop using "pause" for 1 micro sec
+ // 2. Else SOMETIMES busy loop using "yield" for 100 micro sec (default)
+ // 3. Else blocking wait
+
+ // On a modern Xeon each loop takes about 7 nanoseconds (most of which
+ // is the effect of the pause instruction), so 200 iterations is a bit
+ // more than a microsecond. This is long enough that waits longer than
+ // this can amortize the cost of accessing the clock and yielding.
+ for (uint32_t tries = 0; tries < 200; ++tries) {
+ state = w->state.load(std::memory_order_acquire);
+ if ((state & goal_mask) != 0) {
+ return state;
+ }
+ port::AsmVolatilePause();
+ }
+
+ // If we're only going to end up waiting a short period of time,
+ // it can be a lot more efficient to call std::this_thread::yield()
+ // in a loop than to block in StateMutex(). For reference, on my 4.0
+ // SELinux test server with support for syscall auditing enabled, the
+ // minimum latency between FUTEX_WAKE to returning from FUTEX_WAIT is
+ // 2.7 usec, and the average is more like 10 usec. That can be a big
+ // drag on RockDB's single-writer design. Of course, spinning is a
+ // bad idea if other threads are waiting to run or if we're going to
+ // wait for a long time. How do we decide?
+ //
+ // We break waiting into 3 categories: short-uncontended,
+ // short-contended, and long. If we had an oracle, then we would always
+ // spin for short-uncontended, always block for long, and our choice for
+ // short-contended might depend on whether we were trying to optimize
+ // RocksDB throughput or avoid being greedy with system resources.
+ //
+ // Bucketing into short or long is easy by measuring elapsed time.
+ // Differentiating short-uncontended from short-contended is a bit
+ // trickier, but not too bad. We could look for involuntary context
+ // switches using getrusage(RUSAGE_THREAD, ..), but it's less work
+ // (portability code and CPU) to just look for yield calls that take
+ // longer than we expect. sched_yield() doesn't actually result in any
+ // context switch overhead if there are no other runnable processes
+ // on the current core, in which case it usually takes less than
+ // a microsecond.
+ //
+ // There are two primary tunables here: the threshold between "short"
+ // and "long" waits, and the threshold at which we suspect that a yield
+ // is slow enough to indicate we should probably block. If these
+ // thresholds are chosen well then CPU-bound workloads that don't
+ // have more threads than cores will experience few context switches
+ // (voluntary or involuntary), and the total number of context switches
+ // (voluntary and involuntary) will not be dramatically larger (maybe
+ // 2x) than the number of voluntary context switches that occur when
+ // --max_yield_wait_micros=0.
+ //
+ // There's another constant, which is the number of slow yields we will
+ // tolerate before reversing our previous decision. Solitary slow
+ // yields are pretty common (low-priority small jobs ready to run),
+ // so this should be at least 2. We set this conservatively to 3 so
+ // that we can also immediately schedule a ctx adaptation, rather than
+ // waiting for the next update_ctx.
+
+ const size_t kMaxSlowYieldsWhileSpinning = 3;
+
+ // Whether the yield approach has any credit in this context. The credit is
+ // added by yield being succesfull before timing out, and decreased otherwise.
+ auto& yield_credit = ctx->value;
+ // Update the yield_credit based on sample runs or right after a hard failure
+ bool update_ctx = false;
+ // Should we reinforce the yield credit
+ bool would_spin_again = false;
+ // The samling base for updating the yeild credit. The sampling rate would be
+ // 1/sampling_base.
+ const int sampling_base = 256;
+
+ if (max_yield_usec_ > 0) {
+ update_ctx = Random::GetTLSInstance()->OneIn(sampling_base);
+
+ if (update_ctx || yield_credit.load(std::memory_order_relaxed) >= 0) {
+ // we're updating the adaptation statistics, or spinning has >
+ // 50% chance of being shorter than max_yield_usec_ and causing no
+ // involuntary context switches
+ auto spin_begin = std::chrono::steady_clock::now();
+
+ // this variable doesn't include the final yield (if any) that
+ // causes the goal to be met
+ size_t slow_yield_count = 0;
+
+ auto iter_begin = spin_begin;
+ while ((iter_begin - spin_begin) <=
+ std::chrono::microseconds(max_yield_usec_)) {
+ std::this_thread::yield();
+
+ state = w->state.load(std::memory_order_acquire);
+ if ((state & goal_mask) != 0) {
+ // success
+ would_spin_again = true;
+ break;
+ }
+
+ auto now = std::chrono::steady_clock::now();
+ if (now == iter_begin ||
+ now - iter_begin >= std::chrono::microseconds(slow_yield_usec_)) {
+ // conservatively count it as a slow yield if our clock isn't
+ // accurate enough to measure the yield duration
+ ++slow_yield_count;
+ if (slow_yield_count >= kMaxSlowYieldsWhileSpinning) {
+ // Not just one ivcsw, but several. Immediately update yield_credit
+ // and fall back to blocking
+ update_ctx = true;
+ break;
+ }
+ }
+ iter_begin = now;
+ }
+ }
+ }
+
+ if ((state & goal_mask) == 0) {
+ state = BlockingAwaitState(w, goal_mask);
+ }
+
+ if (update_ctx) {
+ // Since our update is sample based, it is ok if a thread overwrites the
+ // updates by other threads. Thus the update does not have to be atomic.
+ auto v = yield_credit.load(std::memory_order_relaxed);
+ // fixed point exponential decay with decay constant 1/1024, with +1
+ // and -1 scaled to avoid overflow for int32_t
+ //
+ // On each update the positive credit is decayed by a facor of 1/1024 (i.e.,
+ // 0.1%). If the sampled yield was successful, the credit is also increased
+ // by X. Setting X=2^17 ensures that the credit never exceeds
+ // 2^17*2^10=2^27, which is lower than 2^31 the upperbound of int32_t. Same
+ // logic applies to negative credits.
+ v = v - (v / 1024) + (would_spin_again ? 1 : -1) * 131072;
+ yield_credit.store(v, std::memory_order_relaxed);
+ }
+
+ assert((state & goal_mask) != 0);
+ return state;
+}
+
+void WriteThread::SetState(Writer* w, uint8_t new_state) {
+ auto state = w->state.load(std::memory_order_acquire);
+ if (state == STATE_LOCKED_WAITING ||
+ !w->state.compare_exchange_strong(state, new_state)) {
+ assert(state == STATE_LOCKED_WAITING);
+
+ std::lock_guard<std::mutex> guard(w->StateMutex());
+ assert(w->state.load(std::memory_order_relaxed) != new_state);
+ w->state.store(new_state, std::memory_order_relaxed);
+ w->StateCV().notify_one();
+ }
+}
+
+bool WriteThread::LinkOne(Writer* w, std::atomic<Writer*>* newest_writer) {
+ assert(newest_writer != nullptr);
+ assert(w->state == STATE_INIT);
+ Writer* writers = newest_writer->load(std::memory_order_relaxed);
+ while (true) {
+ w->link_older = writers;
+ if (newest_writer->compare_exchange_weak(writers, w)) {
+ return (writers == nullptr);
+ }
+ }
+}
+
+bool WriteThread::LinkGroup(WriteGroup& write_group,
+ std::atomic<Writer*>* newest_writer) {
+ assert(newest_writer != nullptr);
+ Writer* leader = write_group.leader;
+ Writer* last_writer = write_group.last_writer;
+ Writer* w = last_writer;
+ while (true) {
+ // Unset link_newer pointers to make sure when we call
+ // CreateMissingNewerLinks later it create all missing links.
+ w->link_newer = nullptr;
+ w->write_group = nullptr;
+ if (w == leader) {
+ break;
+ }
+ w = w->link_older;
+ }
+ Writer* newest = newest_writer->load(std::memory_order_relaxed);
+ while (true) {
+ leader->link_older = newest;
+ if (newest_writer->compare_exchange_weak(newest, last_writer)) {
+ return (newest == nullptr);
+ }
+ }
+}
+
+void WriteThread::CreateMissingNewerLinks(Writer* head) {
+ while (true) {
+ Writer* next = head->link_older;
+ if (next == nullptr || next->link_newer != nullptr) {
+ assert(next == nullptr || next->link_newer == head);
+ break;
+ }
+ next->link_newer = head;
+ head = next;
+ }
+}
+
+void WriteThread::CompleteLeader(WriteGroup& write_group) {
+ assert(write_group.size > 0);
+ Writer* leader = write_group.leader;
+ if (write_group.size == 1) {
+ write_group.leader = nullptr;
+ write_group.last_writer = nullptr;
+ } else {
+ assert(leader->link_newer != nullptr);
+ leader->link_newer->link_older = nullptr;
+ write_group.leader = leader->link_newer;
+ }
+ write_group.size -= 1;
+ SetState(leader, STATE_COMPLETED);
+}
+
+void WriteThread::CompleteFollower(Writer* w, WriteGroup& write_group) {
+ assert(write_group.size > 1);
+ assert(w != write_group.leader);
+ if (w == write_group.last_writer) {
+ w->link_older->link_newer = nullptr;
+ write_group.last_writer = w->link_older;
+ } else {
+ w->link_older->link_newer = w->link_newer;
+ w->link_newer->link_older = w->link_older;
+ }
+ write_group.size -= 1;
+ SetState(w, STATE_COMPLETED);
+}
+
+static WriteThread::AdaptationContext jbg_ctx("JoinBatchGroup");
+void WriteThread::JoinBatchGroup(Writer* w) {
+ TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:Start", w);
+ assert(w->batch != nullptr);
+
+ bool linked_as_leader = LinkOne(w, &newest_writer_);
+ if (linked_as_leader) {
+ SetState(w, STATE_GROUP_LEADER);
+ }
+
+ TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:Wait", w);
+
+ if (!linked_as_leader) {
+ /**
+ * Wait util:
+ * 1) An existing leader pick us as the new leader when it finishes
+ * 2) An existing leader pick us as its follewer and
+ * 2.1) finishes the memtable writes on our behalf
+ * 2.2) Or tell us to finish the memtable writes in pralallel
+ * 3) (pipelined write) An existing leader pick us as its follower and
+ * finish book-keeping and WAL write for us, enqueue us as pending
+ * memtable writer, and
+ * 3.1) we become memtable writer group leader, or
+ * 3.2) an existing memtable writer group leader tell us to finish memtable
+ * writes in parallel.
+ */
+ AwaitState(w, STATE_GROUP_LEADER | STATE_MEMTABLE_WRITER_LEADER |
+ STATE_PARALLEL_MEMTABLE_WRITER | STATE_COMPLETED,
+ &jbg_ctx);
+ TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:DoneWaiting", w);
+ }
+}
+
+size_t WriteThread::EnterAsBatchGroupLeader(Writer* leader,
+ WriteGroup* write_group) {
+ assert(leader->link_older == nullptr);
+ assert(leader->batch != nullptr);
+ assert(write_group != nullptr);
+
+ size_t size = WriteBatchInternal::ByteSize(leader->batch);
+
+ // Allow the group to grow up to a maximum size, but if the
+ // original write is small, limit the growth so we do not slow
+ // down the small write too much.
+ size_t max_size = 1 << 20;
+ if (size <= (128 << 10)) {
+ max_size = size + (128 << 10);
+ }
+
+ leader->write_group = write_group;
+ write_group->leader = leader;
+ write_group->last_writer = leader;
+ write_group->size = 1;
+ Writer* newest_writer = newest_writer_.load(std::memory_order_acquire);
+
+ // This is safe regardless of any db mutex status of the caller. Previous
+ // calls to ExitAsGroupLeader either didn't call CreateMissingNewerLinks
+ // (they emptied the list and then we added ourself as leader) or had to
+ // explicitly wake us up (the list was non-empty when we added ourself,
+ // so we have already received our MarkJoined).
+ CreateMissingNewerLinks(newest_writer);
+
+ // Tricky. Iteration start (leader) is exclusive and finish
+ // (newest_writer) is inclusive. Iteration goes from old to new.
+ Writer* w = leader;
+ while (w != newest_writer) {
+ w = w->link_newer;
+
+ if (w->sync && !leader->sync) {
+ // Do not include a sync write into a batch handled by a non-sync write.
+ break;
+ }
+
+ if (w->no_slowdown != leader->no_slowdown) {
+ // Do not mix writes that are ok with delays with the ones that
+ // request fail on delays.
+ break;
+ }
+
+ if (!w->disable_wal && leader->disable_wal) {
+ // Do not include a write that needs WAL into a batch that has
+ // WAL disabled.
+ break;
+ }
+
+ if (w->batch == nullptr) {
+ // Do not include those writes with nullptr batch. Those are not writes,
+ // those are something else. They want to be alone
+ break;
+ }
+
+ if (w->callback != nullptr && !w->callback->AllowWriteBatching()) {
+ // dont batch writes that don't want to be batched
+ break;
+ }
+
+ auto batch_size = WriteBatchInternal::ByteSize(w->batch);
+ if (size + batch_size > max_size) {
+ // Do not make batch too big
+ break;
+ }
+
+ w->write_group = write_group;
+ size += batch_size;
+ write_group->last_writer = w;
+ write_group->size++;
+ }
+ return size;
+}
+
+void WriteThread::EnterAsMemTableWriter(Writer* leader,
+ WriteGroup* write_group) {
+ assert(leader != nullptr);
+ assert(leader->link_older == nullptr);
+ assert(leader->batch != nullptr);
+ assert(write_group != nullptr);
+
+ size_t size = WriteBatchInternal::ByteSize(leader->batch);
+
+ // Allow the group to grow up to a maximum size, but if the
+ // original write is small, limit the growth so we do not slow
+ // down the small write too much.
+ size_t max_size = 1 << 20;
+ if (size <= (128 << 10)) {
+ max_size = size + (128 << 10);
+ }
+
+ leader->write_group = write_group;
+ write_group->leader = leader;
+ write_group->size = 1;
+ Writer* last_writer = leader;
+
+ if (!allow_concurrent_memtable_write_ || !leader->batch->HasMerge()) {
+ Writer* newest_writer = newest_memtable_writer_.load();
+ CreateMissingNewerLinks(newest_writer);
+
+ Writer* w = leader;
+ while (w != newest_writer) {
+ w = w->link_newer;
+
+ if (w->batch == nullptr) {
+ break;
+ }
+
+ if (w->batch->HasMerge()) {
+ break;
+ }
+
+ if (!allow_concurrent_memtable_write_) {
+ auto batch_size = WriteBatchInternal::ByteSize(w->batch);
+ if (size + batch_size > max_size) {
+ // Do not make batch too big
+ break;
+ }
+ size += batch_size;
+ }
+
+ w->write_group = write_group;
+ last_writer = w;
+ write_group->size++;
+ }
+ }
+
+ write_group->last_writer = last_writer;
+ write_group->last_sequence =
+ last_writer->sequence + WriteBatchInternal::Count(last_writer->batch) - 1;
+}
+
+void WriteThread::ExitAsMemTableWriter(Writer* self, WriteGroup& write_group) {
+ Writer* leader = write_group.leader;
+ Writer* last_writer = write_group.last_writer;
+
+ Writer* newest_writer = last_writer;
+ if (!newest_memtable_writer_.compare_exchange_strong(newest_writer,
+ nullptr)) {
+ CreateMissingNewerLinks(newest_writer);
+ Writer* next_leader = last_writer->link_newer;
+ assert(next_leader != nullptr);
+ next_leader->link_older = nullptr;
+ SetState(next_leader, STATE_MEMTABLE_WRITER_LEADER);
+ }
+ Writer* w = leader;
+ while (true) {
+ if (!write_group.status.ok()) {
+ w->status = write_group.status;
+ }
+ Writer* next = w->link_newer;
+ if (w != leader) {
+ SetState(w, STATE_COMPLETED);
+ }
+ if (w == last_writer) {
+ break;
+ }
+ w = next;
+ }
+ // Note that leader has to exit last, since it owns the write group.
+ SetState(leader, STATE_COMPLETED);
+}
+
+void WriteThread::LaunchParallelMemTableWriters(WriteGroup* write_group) {
+ assert(write_group != nullptr);
+ write_group->running.store(write_group->size);
+ for (auto w : *write_group) {
+ SetState(w, STATE_PARALLEL_MEMTABLE_WRITER);
+ }
+}
+
+static WriteThread::AdaptationContext cpmtw_ctx("CompleteParallelMemTableWriter");
+// This method is called by both the leader and parallel followers
+bool WriteThread::CompleteParallelMemTableWriter(Writer* w) {
+
+ auto* write_group = w->write_group;
+ if (!w->status.ok()) {
+ std::lock_guard<std::mutex> guard(write_group->leader->StateMutex());
+ write_group->status = w->status;
+ }
+
+ if (write_group->running-- > 1) {
+ // we're not the last one
+ AwaitState(w, STATE_COMPLETED, &cpmtw_ctx);
+ return false;
+ }
+ // else we're the last parallel worker and should perform exit duties.
+ w->status = write_group->status;
+ return true;
+}
+
+void WriteThread::ExitAsBatchGroupFollower(Writer* w) {
+ auto* write_group = w->write_group;
+
+ assert(w->state == STATE_PARALLEL_MEMTABLE_WRITER);
+ assert(write_group->status.ok());
+ ExitAsBatchGroupLeader(*write_group, write_group->status);
+ assert(w->status.ok());
+ assert(w->state == STATE_COMPLETED);
+ SetState(write_group->leader, STATE_COMPLETED);
+}
+
+static WriteThread::AdaptationContext eabgl_ctx("ExitAsBatchGroupLeader");
+void WriteThread::ExitAsBatchGroupLeader(WriteGroup& write_group,
+ Status status) {
+ Writer* leader = write_group.leader;
+ Writer* last_writer = write_group.last_writer;
+ assert(leader->link_older == nullptr);
+
+ if (enable_pipelined_write_) {
+ // Notify writers don't write to memtable to exit.
+ for (Writer* w = last_writer; w != leader;) {
+ Writer* next = w->link_older;
+ w->status = status;
+ if (!w->ShouldWriteToMemtable()) {
+ CompleteFollower(w, write_group);
+ }
+ w = next;
+ }
+ if (!leader->ShouldWriteToMemtable()) {
+ CompleteLeader(write_group);
+ }
+ // Link the ramaining of the group to memtable writer list.
+ if (write_group.size > 0) {
+ if (LinkGroup(write_group, &newest_memtable_writer_)) {
+ // The leader can now be different from current writer.
+ SetState(write_group.leader, STATE_MEMTABLE_WRITER_LEADER);
+ }
+ }
+ // Reset newest_writer_ and wake up the next leader.
+ Writer* newest_writer = last_writer;
+ if (!newest_writer_.compare_exchange_strong(newest_writer, nullptr)) {
+ Writer* next_leader = newest_writer;
+ while (next_leader->link_older != last_writer) {
+ next_leader = next_leader->link_older;
+ assert(next_leader != nullptr);
+ }
+ next_leader->link_older = nullptr;
+ SetState(next_leader, STATE_GROUP_LEADER);
+ }
+ AwaitState(leader, STATE_MEMTABLE_WRITER_LEADER |
+ STATE_PARALLEL_MEMTABLE_WRITER | STATE_COMPLETED,
+ &eabgl_ctx);
+ } else {
+ Writer* head = newest_writer_.load(std::memory_order_acquire);
+ if (head != last_writer ||
+ !newest_writer_.compare_exchange_strong(head, nullptr)) {
+ // Either w wasn't the head during the load(), or it was the head
+ // during the load() but somebody else pushed onto the list before
+ // we did the compare_exchange_strong (causing it to fail). In the
+ // latter case compare_exchange_strong has the effect of re-reading
+ // its first param (head). No need to retry a failing CAS, because
+ // only a departing leader (which we are at the moment) can remove
+ // nodes from the list.
+ assert(head != last_writer);
+
+ // After walking link_older starting from head (if not already done)
+ // we will be able to traverse w->link_newer below. This function
+ // can only be called from an active leader, only a leader can
+ // clear newest_writer_, we didn't, and only a clear newest_writer_
+ // could cause the next leader to start their work without a call
+ // to MarkJoined, so we can definitely conclude that no other leader
+ // work is going on here (with or without db mutex).
+ CreateMissingNewerLinks(head);
+ assert(last_writer->link_newer->link_older == last_writer);
+ last_writer->link_newer->link_older = nullptr;
+
+ // Next leader didn't self-identify, because newest_writer_ wasn't
+ // nullptr when they enqueued (we were definitely enqueued before them
+ // and are still in the list). That means leader handoff occurs when
+ // we call MarkJoined
+ SetState(last_writer->link_newer, STATE_GROUP_LEADER);
+ }
+ // else nobody else was waiting, although there might already be a new
+ // leader now
+
+ while (last_writer != leader) {
+ last_writer->status = status;
+ // we need to read link_older before calling SetState, because as soon
+ // as it is marked committed the other thread's Await may return and
+ // deallocate the Writer.
+ auto next = last_writer->link_older;
+ SetState(last_writer, STATE_COMPLETED);
+
+ last_writer = next;
+ }
+ }
+}
+
+static WriteThread::AdaptationContext eu_ctx("EnterUnbatched");
+void WriteThread::EnterUnbatched(Writer* w, InstrumentedMutex* mu) {
+ assert(w != nullptr && w->batch == nullptr);
+ mu->Unlock();
+ bool linked_as_leader = LinkOne(w, &newest_writer_);
+ if (!linked_as_leader) {
+ TEST_SYNC_POINT("WriteThread::EnterUnbatched:Wait");
+ // Last leader will not pick us as a follower since our batch is nullptr
+ AwaitState(w, STATE_GROUP_LEADER, &eu_ctx);
+ }
+ if (enable_pipelined_write_) {
+ WaitForMemTableWriters();
+ }
+ mu->Lock();
+}
+
+void WriteThread::ExitUnbatched(Writer* w) {
+ assert(w != nullptr);
+ Writer* newest_writer = w;
+ if (!newest_writer_.compare_exchange_strong(newest_writer, nullptr)) {
+ CreateMissingNewerLinks(newest_writer);
+ Writer* next_leader = w->link_newer;
+ assert(next_leader != nullptr);
+ next_leader->link_older = nullptr;
+ SetState(next_leader, STATE_GROUP_LEADER);
+ }
+}
+
+static WriteThread::AdaptationContext wfmw_ctx("WaitForMemTableWriters");
+void WriteThread::WaitForMemTableWriters() {
+ assert(enable_pipelined_write_);
+ if (newest_memtable_writer_.load() == nullptr) {
+ return;
+ }
+ Writer w;
+ if (!LinkOne(&w, &newest_memtable_writer_)) {
+ AwaitState(&w, STATE_MEMTABLE_WRITER_LEADER, &wfmw_ctx);
+ }
+ newest_memtable_writer_.store(nullptr);
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/write_thread.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/write_thread.h b/thirdparty/rocksdb/db/write_thread.h
new file mode 100644
index 0000000..57ce71e
--- /dev/null
+++ b/thirdparty/rocksdb/db/write_thread.h
@@ -0,0 +1,391 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <assert.h>
+#include <stdint.h>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <mutex>
+#include <type_traits>
+#include <vector>
+
+#include "db/write_callback.h"
+#include "monitoring/instrumented_mutex.h"
+#include "rocksdb/options.h"
+#include "rocksdb/status.h"
+#include "rocksdb/types.h"
+#include "rocksdb/write_batch.h"
+#include "util/autovector.h"
+
+namespace rocksdb {
+
+class WriteThread {
+ public:
+ enum State : uint8_t {
+ // The initial state of a writer. This is a Writer that is
+ // waiting in JoinBatchGroup. This state can be left when another
+ // thread informs the waiter that it has become a group leader
+ // (-> STATE_GROUP_LEADER), when a leader that has chosen to be
+ // non-parallel informs a follower that its writes have been committed
+ // (-> STATE_COMPLETED), or when a leader that has chosen to perform
+ // updates in parallel and needs this Writer to apply its batch (->
+ // STATE_PARALLEL_FOLLOWER).
+ STATE_INIT = 1,
+
+ // The state used to inform a waiting Writer that it has become the
+ // leader, and it should now build a write batch group. Tricky:
+ // this state is not used if newest_writer_ is empty when a writer
+ // enqueues itself, because there is no need to wait (or even to
+ // create the mutex and condvar used to wait) in that case. This is
+ // a terminal state unless the leader chooses to make this a parallel
+ // batch, in which case the last parallel worker to finish will move
+ // the leader to STATE_COMPLETED.
+ STATE_GROUP_LEADER = 2,
+
+ // The state used to inform a waiting writer that it has become the
+ // leader of memtable writer group. The leader will either write
+ // memtable for the whole group, or launch a parallel group write
+ // to memtable by calling LaunchParallelMemTableWrite.
+ STATE_MEMTABLE_WRITER_LEADER = 4,
+
+ // The state used to inform a waiting writer that it has become a
+ // parallel memtable writer. It can be the group leader who launch the
+ // parallel writer group, or one of the followers. The writer should then
+ // apply its batch to the memtable concurrently and call
+ // CompleteParallelMemTableWriter.
+ STATE_PARALLEL_MEMTABLE_WRITER = 8,
+
+ // A follower whose writes have been applied, or a parallel leader
+ // whose followers have all finished their work. This is a terminal
+ // state.
+ STATE_COMPLETED = 16,
+
+ // A state indicating that the thread may be waiting using StateMutex()
+ // and StateCondVar()
+ STATE_LOCKED_WAITING = 32,
+ };
+
+ struct Writer;
+
+ struct WriteGroup {
+ Writer* leader = nullptr;
+ Writer* last_writer = nullptr;
+ SequenceNumber last_sequence;
+ // before running goes to zero, status needs leader->StateMutex()
+ Status status;
+ std::atomic<size_t> running;
+ size_t size = 0;
+
+ struct Iterator {
+ Writer* writer;
+ Writer* last_writer;
+
+ explicit Iterator(Writer* w, Writer* last)
+ : writer(w), last_writer(last) {}
+
+ Writer* operator*() const { return writer; }
+
+ Iterator& operator++() {
+ assert(writer != nullptr);
+ if (writer == last_writer) {
+ writer = nullptr;
+ } else {
+ writer = writer->link_newer;
+ }
+ return *this;
+ }
+
+ bool operator!=(const Iterator& other) const {
+ return writer != other.writer;
+ }
+ };
+
+ Iterator begin() const { return Iterator(leader, last_writer); }
+ Iterator end() const { return Iterator(nullptr, nullptr); }
+ };
+
+ // Information kept for every waiting writer.
+ struct Writer {
+ WriteBatch* batch;
+ bool sync;
+ bool no_slowdown;
+ bool disable_wal;
+ bool disable_memtable;
+ uint64_t log_used; // log number that this batch was inserted into
+ uint64_t log_ref; // log number that memtable insert should reference
+ WriteCallback* callback;
+ bool made_waitable; // records lazy construction of mutex and cv
+ std::atomic<uint8_t> state; // write under StateMutex() or pre-link
+ WriteGroup* write_group;
+ SequenceNumber sequence; // the sequence number to use for the first key
+ Status status; // status of memtable inserter
+ Status callback_status; // status returned by callback->Callback()
+ std::aligned_storage<sizeof(std::mutex)>::type state_mutex_bytes;
+ std::aligned_storage<sizeof(std::condition_variable)>::type state_cv_bytes;
+ Writer* link_older; // read/write only before linking, or as leader
+ Writer* link_newer; // lazy, read/write only before linking, or as leader
+
+ Writer()
+ : batch(nullptr),
+ sync(false),
+ no_slowdown(false),
+ disable_wal(false),
+ disable_memtable(false),
+ log_used(0),
+ log_ref(0),
+ callback(nullptr),
+ made_waitable(false),
+ state(STATE_INIT),
+ write_group(nullptr),
+ link_older(nullptr),
+ link_newer(nullptr) {}
+
+ Writer(const WriteOptions& write_options, WriteBatch* _batch,
+ WriteCallback* _callback, uint64_t _log_ref, bool _disable_memtable)
+ : batch(_batch),
+ sync(write_options.sync),
+ no_slowdown(write_options.no_slowdown),
+ disable_wal(write_options.disableWAL),
+ disable_memtable(_disable_memtable),
+ log_used(0),
+ log_ref(_log_ref),
+ callback(_callback),
+ made_waitable(false),
+ state(STATE_INIT),
+ write_group(nullptr),
+ link_older(nullptr),
+ link_newer(nullptr) {}
+
+ ~Writer() {
+ if (made_waitable) {
+ StateMutex().~mutex();
+ StateCV().~condition_variable();
+ }
+ }
+
+ bool CheckCallback(DB* db) {
+ if (callback != nullptr) {
+ callback_status = callback->Callback(db);
+ }
+ return callback_status.ok();
+ }
+
+ void CreateMutex() {
+ if (!made_waitable) {
+ // Note that made_waitable is tracked separately from state
+ // transitions, because we can't atomically create the mutex and
+ // link into the list.
+ made_waitable = true;
+ new (&state_mutex_bytes) std::mutex;
+ new (&state_cv_bytes) std::condition_variable;
+ }
+ }
+
+ // returns the aggregate status of this Writer
+ Status FinalStatus() {
+ if (!status.ok()) {
+ // a non-ok memtable write status takes presidence
+ assert(callback == nullptr || callback_status.ok());
+ return status;
+ } else if (!callback_status.ok()) {
+ // if the callback failed then that is the status we want
+ // because a memtable insert should not have been attempted
+ assert(callback != nullptr);
+ assert(status.ok());
+ return callback_status;
+ } else {
+ // if there is no callback then we only care about
+ // the memtable insert status
+ assert(callback == nullptr || callback_status.ok());
+ return status;
+ }
+ }
+
+ bool CallbackFailed() {
+ return (callback != nullptr) && !callback_status.ok();
+ }
+
+ bool ShouldWriteToMemtable() {
+ return status.ok() && !CallbackFailed() && !disable_memtable;
+ }
+
+ bool ShouldWriteToWAL() {
+ return status.ok() && !CallbackFailed() && !disable_wal;
+ }
+
+ // No other mutexes may be acquired while holding StateMutex(), it is
+ // always last in the order
+ std::mutex& StateMutex() {
+ assert(made_waitable);
+ return *static_cast<std::mutex*>(static_cast<void*>(&state_mutex_bytes));
+ }
+
+ std::condition_variable& StateCV() {
+ assert(made_waitable);
+ return *static_cast<std::condition_variable*>(
+ static_cast<void*>(&state_cv_bytes));
+ }
+ };
+
+ struct AdaptationContext {
+ const char* name;
+ std::atomic<int32_t> value;
+
+ explicit AdaptationContext(const char* name0) : name(name0), value(0) {}
+ };
+
+ explicit WriteThread(const ImmutableDBOptions& db_options);
+
+ virtual ~WriteThread() = default;
+
+ // IMPORTANT: None of the methods in this class rely on the db mutex
+ // for correctness. All of the methods except JoinBatchGroup and
+ // EnterUnbatched may be called either with or without the db mutex held.
+ // Correctness is maintained by ensuring that only a single thread is
+ // a leader at a time.
+
+ // Registers w as ready to become part of a batch group, waits until the
+ // caller should perform some work, and returns the current state of the
+ // writer. If w has become the leader of a write batch group, returns
+ // STATE_GROUP_LEADER. If w has been made part of a sequential batch
+ // group and the leader has performed the write, returns STATE_DONE.
+ // If w has been made part of a parallel batch group and is responsible
+ // for updating the memtable, returns STATE_PARALLEL_FOLLOWER.
+ //
+ // The db mutex SHOULD NOT be held when calling this function, because
+ // it will block.
+ //
+ // Writer* w: Writer to be executed as part of a batch group
+ void JoinBatchGroup(Writer* w);
+
+ // Constructs a write batch group led by leader, which should be a
+ // Writer passed to JoinBatchGroup on the current thread.
+ //
+ // Writer* leader: Writer that is STATE_GROUP_LEADER
+ // WriteGroup* write_group: Out-param of group members
+ // returns: Total batch group byte size
+ size_t EnterAsBatchGroupLeader(Writer* leader, WriteGroup* write_group);
+
+ // Unlinks the Writer-s in a batch group, wakes up the non-leaders,
+ // and wakes up the next leader (if any).
+ //
+ // WriteGroup* write_group: the write group
+ // Status status: Status of write operation
+ void ExitAsBatchGroupLeader(WriteGroup& write_group, Status status);
+
+ // Exit batch group on behalf of batch group leader.
+ void ExitAsBatchGroupFollower(Writer* w);
+
+ // Constructs a write batch group led by leader from newest_memtable_writers_
+ // list. The leader should either write memtable for the whole group and
+ // call ExitAsMemTableWriter, or launch parallel memtable write through
+ // LaunchParallelMemTableWriters.
+ void EnterAsMemTableWriter(Writer* leader, WriteGroup* write_grup);
+
+ // Memtable writer group leader, or the last finished writer in a parallel
+ // write group, exit from the newest_memtable_writers_ list, and wake up
+ // the next leader if needed.
+ void ExitAsMemTableWriter(Writer* self, WriteGroup& write_group);
+
+ // Causes JoinBatchGroup to return STATE_PARALLEL_FOLLOWER for all of the
+ // non-leader members of this write batch group. Sets Writer::sequence
+ // before waking them up.
+ //
+ // WriteGroup* write_group: Extra state used to coordinate the parallel add
+ void LaunchParallelMemTableWriters(WriteGroup* write_group);
+
+ // Reports the completion of w's batch to the parallel group leader, and
+ // waits for the rest of the parallel batch to complete. Returns true
+ // if this thread is the last to complete, and hence should advance
+ // the sequence number and then call EarlyExitParallelGroup, false if
+ // someone else has already taken responsibility for that.
+ bool CompleteParallelMemTableWriter(Writer* w);
+
+ // Waits for all preceding writers (unlocking mu while waiting), then
+ // registers w as the currently proceeding writer.
+ //
+ // Writer* w: A Writer not eligible for batching
+ // InstrumentedMutex* mu: The db mutex, to unlock while waiting
+ // REQUIRES: db mutex held
+ void EnterUnbatched(Writer* w, InstrumentedMutex* mu);
+
+ // Completes a Writer begun with EnterUnbatched, unblocking subsequent
+ // writers.
+ void ExitUnbatched(Writer* w);
+
+ // Wait for all parallel memtable writers to finish, in case pipelined
+ // write is enabled.
+ void WaitForMemTableWriters();
+
+ SequenceNumber UpdateLastSequence(SequenceNumber sequence) {
+ if (sequence > last_sequence_) {
+ last_sequence_ = sequence;
+ }
+ return last_sequence_;
+ }
+
+ private:
+ // See AwaitState.
+ const uint64_t max_yield_usec_;
+ const uint64_t slow_yield_usec_;
+
+ // Allow multiple writers write to memtable concurrently.
+ const bool allow_concurrent_memtable_write_;
+
+ // Enable pipelined write to WAL and memtable.
+ const bool enable_pipelined_write_;
+
+ // Points to the newest pending writer. Only leader can remove
+ // elements, adding can be done lock-free by anybody.
+ std::atomic<Writer*> newest_writer_;
+
+ // Points to the newest pending memtable writer. Used only when pipelined
+ // write is enabled.
+ std::atomic<Writer*> newest_memtable_writer_;
+
+ // The last sequence that have been consumed by a writer. The sequence
+ // is not necessary visible to reads because the writer can be ongoing.
+ SequenceNumber last_sequence_;
+
+ // Waits for w->state & goal_mask using w->StateMutex(). Returns
+ // the state that satisfies goal_mask.
+ uint8_t BlockingAwaitState(Writer* w, uint8_t goal_mask);
+
+ // Blocks until w->state & goal_mask, returning the state value
+ // that satisfied the predicate. Uses ctx to adaptively use
+ // std::this_thread::yield() to avoid mutex overheads. ctx should be
+ // a context-dependent static.
+ uint8_t AwaitState(Writer* w, uint8_t goal_mask, AdaptationContext* ctx);
+
+ // Set writer state and wake the writer up if it is waiting.
+ void SetState(Writer* w, uint8_t new_state);
+
+ // Links w into the newest_writer list. Return true if w was linked directly
+ // into the leader position. Safe to call from multiple threads without
+ // external locking.
+ bool LinkOne(Writer* w, std::atomic<Writer*>* newest_writer);
+
+ // Link write group into the newest_writer list as a whole, while keeping the
+ // order of the writers unchanged. Return true if the group was linked
+ // directly into the leader position.
+ bool LinkGroup(WriteGroup& write_group, std::atomic<Writer*>* newest_writer);
+
+ // Computes any missing link_newer links. Should not be called
+ // concurrently with itself.
+ void CreateMissingNewerLinks(Writer* head);
+
+ // Set the leader in write_group to completed state and remove it from the
+ // write group.
+ void CompleteLeader(WriteGroup& write_group);
+
+ // Set a follower in write_group to completed state and remove it from the
+ // write group.
+ void CompleteFollower(Writer* w, WriteGroup& write_group);
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/.gitignore
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/.gitignore b/thirdparty/rocksdb/docs/.gitignore
new file mode 100644
index 0000000..e48dc98
--- /dev/null
+++ b/thirdparty/rocksdb/docs/.gitignore
@@ -0,0 +1,9 @@
+.DS_STORE
+_site/
+*.swo
+*.swp
+_site
+.sass-cache
+*.psd
+*~
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/CNAME
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/CNAME b/thirdparty/rocksdb/docs/CNAME
new file mode 100644
index 0000000..827d1c0
--- /dev/null
+++ b/thirdparty/rocksdb/docs/CNAME
@@ -0,0 +1 @@
+rocksdb.org
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/CONTRIBUTING.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/CONTRIBUTING.md b/thirdparty/rocksdb/docs/CONTRIBUTING.md
new file mode 100644
index 0000000..2c5842f
--- /dev/null
+++ b/thirdparty/rocksdb/docs/CONTRIBUTING.md
@@ -0,0 +1,115 @@
+This provides guidance on how to contribute various content to `rocksdb.org`.
+
+## Getting started
+
+You should only have to do these one time.
+
+- Rename this file to `CONTRIBUTING.md`.
+- Rename `EXAMPLE-README-FOR-RUNNING-DOCS.md` to `README.md` (replacing the existing `README.md` that came with the template).
+- Rename `EXAMPLE-LICENSE` to `LICENSE`.
+- Review the [template information](./TEMPLATE-INFORMATION.md).
+- Review `./_config.yml`.
+- Make sure you update `title`, `description`, `tagline` and `gacode` (Google Analytics) in `./_config.yml`.
+
+## Basic Structure
+
+Most content is written in markdown. You name the file `something.md`, then have a header that looks like this:
+
+```
+---
+docid: getting-started
+title: Getting started with ProjectName
+layout: docs
+permalink: /docs/getting-started.html
+---
+```
+
+Customize these values for each document, blog post, etc.
+
+> The filename of the `.md` file doesn't actually matter; what is important is the `docid` being unique and the `permalink` correct and unique too).
+
+## Landing page
+
+Modify `index.md` with your new or updated content.
+
+If you want a `GridBlock` as part of your content, you can do so directly with HTML:
+
+```
+<div class="gridBlock">
+ <div class="blockElement twoByGridBlock alignLeft">
+ <div class="blockContent">
+ <h3>Your Features</h3>
+ <ul>
+ <li>The <a href="http://example.org/">Example</a></li>
+ <li><a href="http://example.com">Another Example</a></li>
+ </ul>
+ </div>
+ </div>
+
+ <div class="blockElement twoByGridBlock alignLeft">
+ <div class="blockContent">
+ <h3>More information</h3>
+ <p>
+ Stuff here
+ </p>
+ </div>
+ </div>
+</div>
+```
+
+or with a combination of changing `./_data/features.yml` and adding some Liquid to `index.md`, such as:
+
+```
+{% include content/gridblocks.html data_source=site.data.features imagealign="bottom"%}
+```
+
+## Blog
+
+To modify a blog post, edit the appopriate markdown file in `./_posts/`.
+
+Adding a new blog post is a four-step process.
+
+> Some posts have a `permalink` and `comments` in the blog post YAML header. You will not need these for new blog posts. These are an artifact of migrating the blog from Wordpress to gh-pages.
+
+1. Create your blog post in `./_posts/` in markdown (file extension `.md` or `.markdown`). See current posts in that folder or `./doc-type-examples/2016-04-07-blog-post-example.md` for an example of the YAML format. **If the `./_posts` directory does not exist, create it**.
+ - You can add a `<!--truncate-->` tag in the middle of your post such that you show only the excerpt above that tag in the main `/blog` index on your page.
+1. If you have not authored a blog post before, modify the `./_data/authors.yml` file with the `author` id you used in your blog post, along with your full name and Facebook ID to get your profile picture.
+1. [Run the site locally](./README.md) to test your changes. It will be at `http://127.0.0.1/blog/your-new-blog-post-title.html`
+1. Push your changes to GitHub.
+
+## Docs
+
+To modify docs, edit the appropriate markdown file in `./_docs/`.
+
+To add docs to the site....
+
+1. Add your markdown file to the `./_docs/` folder. See `./doc-type-examples/docs-hello-world.md` for an example of the YAML header format. **If the `./_docs/` directory does not exist, create it**.
+ - You can use folders in the `./_docs/` directory to organize your content if you want.
+1. Update `_data/nav_docs.yml` to add your new document to the navigation bar. Use the `docid` you put in your doc markdown in as the `id` in the `_data/nav_docs.yml` file.
+1. [Run the site locally](./README.md) to test your changes. It will be at `http://127.0.0.1/docs/your-new-doc-permalink.html`
+1. Push your changes to GitHub.
+
+## Header Bar
+
+To modify the header bar, change `./_data/nav.yml`.
+
+## Top Level Page
+
+To modify a top-level page, edit the appropriate markdown file in `./top-level/`
+
+If you want a top-level page (e.g., http://your-site.com/top-level.html) -- not in `/blog/` or `/docs/`....
+
+1. Create a markdown file in the root `./top-level/`. See `./doc-type-examples/top-level-example.md` for more information.
+1. If you want a visible link to that file, update `_data/nav.yml` to add a link to your new top-level document in the header bar.
+
+ > This is not necessary if you just want to have a page that is linked to from another page, but not exposed as direct link to the user.
+
+1. [Run the site locally](./README.md) to test your changes. It will be at `http://127.0.0.1/your-top-level-page-permalink.html`
+1. Push your changes to GitHub.
+
+## Other Changes
+
+- CSS: `./css/main.css` or `./_sass/*.scss`.
+- Images: `./static/images/[docs | posts]/....`
+- Main Blog post HTML: `./_includes/post.html`
+- Main Docs HTML: `./_includes/doc.html`
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/Gemfile
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/Gemfile b/thirdparty/rocksdb/docs/Gemfile
new file mode 100644
index 0000000..93dc8b0
--- /dev/null
+++ b/thirdparty/rocksdb/docs/Gemfile
@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gem 'github-pages', '~> 104'
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/Gemfile.lock
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/Gemfile.lock b/thirdparty/rocksdb/docs/Gemfile.lock
new file mode 100644
index 0000000..346fc6c
--- /dev/null
+++ b/thirdparty/rocksdb/docs/Gemfile.lock
@@ -0,0 +1,145 @@
+GEM
+ remote: https://rubygems.org/
+ specs:
+ activesupport (4.2.7)
+ i18n (~> 0.7)
+ json (~> 1.7, >= 1.7.7)
+ minitest (~> 5.1)
+ thread_safe (~> 0.3, >= 0.3.4)
+ tzinfo (~> 1.1)
+ addressable (2.4.0)
+ coffee-script (2.4.1)
+ coffee-script-source
+ execjs
+ coffee-script-source (1.10.0)
+ colorator (1.1.0)
+ ethon (0.9.1)
+ ffi (>= 1.3.0)
+ execjs (2.7.0)
+ faraday (0.9.2)
+ multipart-post (>= 1.2, < 3)
+ ffi (1.9.14)
+ forwardable-extended (2.6.0)
+ gemoji (2.1.0)
+ github-pages (104)
+ activesupport (= 4.2.7)
+ github-pages-health-check (= 1.2.0)
+ jekyll (= 3.3.0)
+ jekyll-avatar (= 0.4.2)
+ jekyll-coffeescript (= 1.0.1)
+ jekyll-feed (= 0.8.0)
+ jekyll-gist (= 1.4.0)
+ jekyll-github-metadata (= 2.2.0)
+ jekyll-mentions (= 1.2.0)
+ jekyll-paginate (= 1.1.0)
+ jekyll-redirect-from (= 0.11.0)
+ jekyll-sass-converter (= 1.3.0)
+ jekyll-seo-tag (= 2.1.0)
+ jekyll-sitemap (= 0.12.0)
+ jekyll-swiss (= 0.4.0)
+ jemoji (= 0.7.0)
+ kramdown (= 1.11.1)
+ liquid (= 3.0.6)
+ listen (= 3.0.6)
+ mercenary (~> 0.3)
+ minima (= 2.0.0)
+ rouge (= 1.11.1)
+ terminal-table (~> 1.4)
+ github-pages-health-check (1.2.0)
+ addressable (~> 2.3)
+ net-dns (~> 0.8)
+ octokit (~> 4.0)
+ public_suffix (~> 1.4)
+ typhoeus (~> 0.7)
+ html-pipeline (2.4.2)
+ activesupport (>= 2)
+ nokogiri (>= 1.4)
+ i18n (0.7.0)
+ jekyll (3.3.0)
+ addressable (~> 2.4)
+ colorator (~> 1.0)
+ jekyll-sass-converter (~> 1.0)
+ jekyll-watch (~> 1.1)
+ kramdown (~> 1.3)
+ liquid (~> 3.0)
+ mercenary (~> 0.3.3)
+ pathutil (~> 0.9)
+ rouge (~> 1.7)
+ safe_yaml (~> 1.0)
+ jekyll-avatar (0.4.2)
+ jekyll (~> 3.0)
+ jekyll-coffeescript (1.0.1)
+ coffee-script (~> 2.2)
+ jekyll-feed (0.8.0)
+ jekyll (~> 3.3)
+ jekyll-gist (1.4.0)
+ octokit (~> 4.2)
+ jekyll-github-metadata (2.2.0)
+ jekyll (~> 3.1)
+ octokit (~> 4.0, != 4.4.0)
+ jekyll-mentions (1.2.0)
+ activesupport (~> 4.0)
+ html-pipeline (~> 2.3)
+ jekyll (~> 3.0)
+ jekyll-paginate (1.1.0)
+ jekyll-redirect-from (0.11.0)
+ jekyll (>= 2.0)
+ jekyll-sass-converter (1.3.0)
+ sass (~> 3.2)
+ jekyll-seo-tag (2.1.0)
+ jekyll (~> 3.3)
+ jekyll-sitemap (0.12.0)
+ jekyll (~> 3.3)
+ jekyll-swiss (0.4.0)
+ jekyll-watch (1.5.0)
+ listen (~> 3.0, < 3.1)
+ jemoji (0.7.0)
+ activesupport (~> 4.0)
+ gemoji (~> 2.0)
+ html-pipeline (~> 2.2)
+ jekyll (>= 3.0)
+ json (1.8.3)
+ kramdown (1.11.1)
+ liquid (3.0.6)
+ listen (3.0.6)
+ rb-fsevent (>= 0.9.3)
+ rb-inotify (>= 0.9.7)
+ mercenary (0.3.6)
+ mini_portile2 (2.1.0)
+ minima (2.0.0)
+ minitest (5.9.1)
+ multipart-post (2.0.0)
+ net-dns (0.8.0)
+ nokogiri (1.6.8.1)
+ mini_portile2 (~> 2.1.0)
+ octokit (4.4.1)
+ sawyer (~> 0.7.0, >= 0.5.3)
+ pathutil (0.14.0)
+ forwardable-extended (~> 2.6)
+ public_suffix (1.5.3)
+ rb-fsevent (0.9.8)
+ rb-inotify (0.9.7)
+ ffi (>= 0.5.0)
+ rouge (1.11.1)
+ safe_yaml (1.0.4)
+ sass (3.4.22)
+ sawyer (0.7.0)
+ addressable (>= 2.3.5, < 2.5)
+ faraday (~> 0.8, < 0.10)
+ terminal-table (1.7.3)
+ unicode-display_width (~> 1.1.1)
+ thread_safe (0.3.5)
+ typhoeus (0.8.0)
+ ethon (>= 0.8.0)
+ tzinfo (1.2.2)
+ thread_safe (~> 0.1)
+ unicode-display_width (1.1.1)
+
+PLATFORMS
+ ruby
+
+DEPENDENCIES
+ github-pages (~> 104)
+
+BUNDLED WITH
+ 1.13.1
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/LICENSE-DOCUMENTATION
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/LICENSE-DOCUMENTATION b/thirdparty/rocksdb/docs/LICENSE-DOCUMENTATION
new file mode 100644
index 0000000..1f255c9
--- /dev/null
+++ b/thirdparty/rocksdb/docs/LICENSE-DOCUMENTATION
@@ -0,0 +1,385 @@
+Attribution 4.0 International
+
+=======================================================================
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright
+and certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+ Considerations for licensors: Our public licenses are
+ intended for use by those authorized to give the public
+ permission to use material in ways otherwise restricted by
+ copyright and certain other rights. Our licenses are
+ irrevocable. Licensors should read and understand the terms
+ and conditions of the license they choose before applying it.
+ Licensors should also secure all rights necessary before
+ applying our licenses so that the public can reuse the
+ material as expected. Licensors should clearly mark any
+ material not subject to the license. This includes other CC-
+ licensed material, or material used under an exception or
+ limitation to copyright. More considerations for licensors:
+ wiki.creativecommons.org/Considerations_for_licensors
+
+ Considerations for the public: By using one of our public
+ licenses, a licensor grants the public permission to use the
+ licensed material under specified terms and conditions. If
+ the licensor's permission is not necessary for any reason--for
+ example, because of any applicable exception or limitation to
+ copyright--then that use is not regulated by the license. Our
+ licenses grant only permissions under copyright and certain
+ other rights that a licensor has authority to grant. Use of
+ the licensed material may still be restricted for other
+ reasons, including because others have copyright or other
+ rights in the material. A licensor may make special requests,
+ such as asking that all changes be marked or described.
+ Although not required by our licenses, you are encouraged to
+ respect those requests where reasonable. More_considerations
+ for the public:
+ wiki.creativecommons.org/Considerations_for_licensees
+
+=======================================================================
+
+Creative Commons Attribution 4.0 International Public License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution 4.0 International Public License ("Public License"). To the
+extent this Public License may be interpreted as a contract, You are
+granted the Licensed Rights in consideration of Your acceptance of
+these terms and conditions, and the Licensor grants You such rights in
+consideration of benefits the Licensor receives from making the
+Licensed Material available under these terms and conditions.
+
+Section 1 -- Definitions.
+
+ a. Adapted Material means material subject to Copyright and Similar
+ Rights that is derived from or based upon the Licensed Material
+ and in which the Licensed Material is translated, altered,
+ arranged, transformed, or otherwise modified in a manner requiring
+ permission under the Copyright and Similar Rights held by the
+ Licensor. For purposes of this Public License, where the Licensed
+ Material is a musical work, performance, or sound recording,
+ Adapted Material is always produced where the Licensed Material is
+ synched in timed relation with a moving image.
+
+b. Adapter's License means the license You apply to Your Copyright
+ and Similar Rights in Your contributions to Adapted Material in
+ accordance with the terms and conditions of this Public License.
+
+c. Copyright and Similar Rights means copyright and/or similar rights
+ closely related to copyright including, without limitation,
+ performance, broadcast, sound recording, and Sui Generis Database
+ Rights, without regard to how the rights are labeled or
+ categorized. For purposes of this Public License, the rights
+ specified in Section 2(b)(1)-(2) are not Copyright and Similar
+ Rights.
+
+d. Effective Technological Measures means those measures that, in the
+ absence of proper authority, may not be circumvented under laws
+ fulfilling obligations under Article 11 of the WIPO Copyright
+ Treaty adopted on December 20, 1996, and/or similar international
+ agreements.
+
+e. Exceptions and Limitations means fair use, fair dealing, and/or
+ any other exception or limitation to Copyright and Similar Rights
+ that applies to Your use of the Licensed Material.
+
+f. Licensed Material means the artistic or literary work, database,
+ or other material to which the Licensor applied this Public
+ License.
+
+g. Licensed Rights means the rights granted to You subject to the
+ terms and conditions of this Public License, which are limited to
+ all Copyright and Similar Rights that apply to Your use of the
+ Licensed Material and that the Licensor has authority to license.
+
+h. Licensor means the individual(s) or entity(ies) granting rights
+ under this Public License.
+
+i. Share means to provide material to the public by any means or
+ process that requires permission under the Licensed Rights, such
+ as reproduction, public display, public performance, distribution,
+ dissemination, communication, or importation, and to make material
+ available to the public including in ways that members of the
+ public may access the material from a place and at a time
+ individually chosen by them.
+
+j. Sui Generis Database Rights means rights other than copyright
+ resulting from Directive 96/9/EC of the European Parliament and of
+ the Council of 11 March 1996 on the legal protection of databases,
+ as amended and/or succeeded, as well as other essentially
+ equivalent rights anywhere in the world.
+
+k. You means the individual or entity exercising the Licensed Rights
+ under this Public License. Your has a corresponding meaning.
+
+Section 2 -- Scope.
+
+a. License grant.
+
+ 1. Subject to the terms and conditions of this Public License,
+ the Licensor hereby grants You a worldwide, royalty-free,
+ non-sublicensable, non-exclusive, irrevocable license to
+ exercise the Licensed Rights in the Licensed Material to:
+
+ a. reproduce and Share the Licensed Material, in whole or
+ in part; and
+
+ b. produce, reproduce, and Share Adapted Material.
+
+ 2. Exceptions and Limitations. For the avoidance of doubt, where
+ Exceptions and Limitations apply to Your use, this Public
+ License does not apply, and You do not need to comply with
+ its terms and conditions.
+
+ 3. Term. The term of this Public License is specified in Section
+ 6(a).
+
+ 4. Media and formats; technical modifications allowed. The
+ Licensor authorizes You to exercise the Licensed Rights in
+ all media and formats whether now known or hereafter created,
+ and to make technical modifications necessary to do so. The
+ Licensor waives and/or agrees not to assert any right or
+ authority to forbid You from making technical modifications
+ necessary to exercise the Licensed Rights, including
+ technical modifications necessary to circumvent Effective
+ Technological Measures. For purposes of this Public License,
+ simply making modifications authorized by this Section 2(a)
+ (4) never produces Adapted Material.
+
+ 5. Downstream recipients.
+
+ a. Offer from the Licensor -- Licensed Material. Every
+ recipient of the Licensed Material automatically
+ receives an offer from the Licensor to exercise the
+ Licensed Rights under the terms and conditions of this
+ Public License.
+
+ b. No downstream restrictions. You may not offer or impose
+ any additional or different terms or conditions on, or
+ apply any Effective Technological Measures to, the
+ Licensed Material if doing so restricts exercise of the
+ Licensed Rights by any recipient of the Licensed
+ Material.
+
+ 6. No endorsement. Nothing in this Public License constitutes or
+ may be construed as permission to assert or imply that You
+ are, or that Your use of the Licensed Material is, connected
+ with, or sponsored, endorsed, or granted official status by,
+ the Licensor or others designated to receive attribution as
+ provided in Section 3(a)(1)(A)(i).
+
+b. Other rights.
+
+ 1. Moral rights, such as the right of integrity, are not
+ licensed under this Public License, nor are publicity,
+ privacy, and/or other similar personality rights; however, to
+ the extent possible, the Licensor waives and/or agrees not to
+ assert any such rights held by the Licensor to the limited
+ extent necessary to allow You to exercise the Licensed
+ Rights, but not otherwise.
+
+ 2. Patent and trademark rights are not licensed under this
+ Public License.
+
+ 3. To the extent possible, the Licensor waives any right to
+ collect royalties from You for the exercise of the Licensed
+ Rights, whether directly or through a collecting society
+ under any voluntary or waivable statutory or compulsory
+ licensing scheme. In all other cases the Licensor expressly
+ reserves any right to collect such royalties.
+
+Section 3 -- License Conditions.
+
+Your exercise of the Licensed Rights is expressly made subject to the
+following conditions.
+
+a. Attribution.
+
+ 1. If You Share the Licensed Material (including in modified
+ form), You must:
+
+ a. retain the following if it is supplied by the Licensor
+ with the Licensed Material:
+
+ i. identification of the creator(s) of the Licensed
+ Material and any others designated to receive
+ attribution, in any reasonable manner requested by
+ the Licensor (including by pseudonym if
+ designated);
+
+ ii. a copyright notice;
+
+ iii. a notice that refers to this Public License;
+
+ iv. a notice that refers to the disclaimer of
+ warranties;
+
+ v. a URI or hyperlink to the Licensed Material to the
+ extent reasonably practicable;
+
+ b. indicate if You modified the Licensed Material and
+ retain an indication of any previous modifications; and
+
+ c. indicate the Licensed Material is licensed under this
+ Public License, and include the text of, or the URI or
+ hyperlink to, this Public License.
+
+ 2. You may satisfy the conditions in Section 3(a)(1) in any
+ reasonable manner based on the medium, means, and context in
+ which You Share the Licensed Material. For example, it may be
+ reasonable to satisfy the conditions by providing a URI or
+ hyperlink to a resource that includes the required
+ information.
+
+ 3. If requested by the Licensor, You must remove any of the
+ information required by Section 3(a)(1)(A) to the extent
+ reasonably practicable.
+
+ 4. If You Share Adapted Material You produce, the Adapter's
+ License You apply must not prevent recipients of the Adapted
+ Material from complying with this Public License.
+
+Section 4 -- Sui Generis Database Rights.
+
+Where the Licensed Rights include Sui Generis Database Rights that
+apply to Your use of the Licensed Material:
+
+a. for the avoidance of doubt, Section 2(a)(1) grants You the right
+ to extract, reuse, reproduce, and Share all or a substantial
+ portion of the contents of the database;
+
+b. if You include all or a substantial portion of the database
+ contents in a database in which You have Sui Generis Database
+ Rights, then the database in which You have Sui Generis Database
+ Rights (but not its individual contents) is Adapted Material; and
+
+c. You must comply with the conditions in Section 3(a) if You Share
+ all or a substantial portion of the contents of the database.
+
+For the avoidance of doubt, this Section 4 supplements and does not
+replace Your obligations under this Public License where the Licensed
+Rights include other Copyright and Similar Rights.
+
+Section 5 -- Disclaimer of Warranties and Limitation of Liability.
+
+a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
+ EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
+ AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
+ ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
+ IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
+ WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
+ ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
+ KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
+ ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
+
+b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
+ TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
+ NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
+ INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
+ COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
+ USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
+ ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
+ DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
+ IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
+
+c. The disclaimer of warranties and limitation of liability provided
+ above shall be interpreted in a manner that, to the extent
+ possible, most closely approximates an absolute disclaimer and
+ waiver of all liability.
+
+Section 6 -- Term and Termination.
+
+a. This Public License applies for the term of the Copyright and
+ Similar Rights licensed here. However, if You fail to comply with
+ this Public License, then Your rights under this Public License
+ terminate automatically.
+
+b. Where Your right to use the Licensed Material has terminated under
+ Section 6(a), it reinstates:
+
+ 1. automatically as of the date the violation is cured, provided
+ it is cured within 30 days of Your discovery of the
+ violation; or
+
+ 2. upon express reinstatement by the Licensor.
+
+ For the avoidance of doubt, this Section 6(b) does not affect any
+ right the Licensor may have to seek remedies for Your violations
+ of this Public License.
+
+c. For the avoidance of doubt, the Licensor may also offer the
+ Licensed Material under separate terms or conditions or stop
+ distributing the Licensed Material at any time; however, doing so
+ will not terminate this Public License.
+
+d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+ License.
+
+Section 7 -- Other Terms and Conditions.
+
+a. The Licensor shall not be bound by any additional or different
+ terms or conditions communicated by You unless expressly agreed.
+
+b. Any arrangements, understandings, or agreements regarding the
+ Licensed Material not stated herein are separate from and
+ independent of the terms and conditions of this Public License.
+
+Section 8 -- Interpretation.
+
+a. For the avoidance of doubt, this Public License does not, and
+ shall not be interpreted to, reduce, limit, restrict, or impose
+ conditions on any use of the Licensed Material that could lawfully
+ be made without permission under this Public License.
+
+b. To the extent possible, if any provision of this Public License is
+ deemed unenforceable, it shall be automatically reformed to the
+ minimum extent necessary to make it enforceable. If the provision
+ cannot be reformed, it shall be severed from this Public License
+ without affecting the enforceability of the remaining terms and
+ conditions.
+
+c. No term or condition of this Public License will be waived and no
+ failure to comply consented to unless expressly agreed to by the
+ Licensor.
+
+d. Nothing in this Public License constitutes or may be interpreted
+ as a limitation upon, or waiver of, any privileges and immunities
+ that apply to the Licensor or You, including from the legal
+ processes of any jurisdiction or authority.
+
+=======================================================================
+
+Creative Commons is not a party to its public licenses.
+Notwithstanding, Creative Commons may elect to apply one of its public
+licenses to material it publishes and in those instances will be
+considered the "Licensor." Except for the limited purpose of indicating
+that material is shared under a Creative Commons public license or as
+otherwise permitted by the Creative Commons policies published at
+creativecommons.org/policies, Creative Commons does not authorize the
+use of the trademark "Creative Commons" or any other trademark or logo
+of Creative Commons without its prior written consent including,
+without limitation, in connection with any unauthorized modifications
+to any of its public licenses or any other arrangements,
+understandings, or agreements concerning use of licensed material. For
+the avoidance of doubt, this paragraph does not form part of the public
+licenses.
+
+Creative Commons may be contacted at creativecommons.org.
+
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/README.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/README.md b/thirdparty/rocksdb/docs/README.md
new file mode 100644
index 0000000..0ae8978
--- /dev/null
+++ b/thirdparty/rocksdb/docs/README.md
@@ -0,0 +1,80 @@
+## User Documentation for rocksdb.org
+
+This directory will contain the user and feature documentation for RocksDB. The documentation will be hosted on GitHub pages.
+
+### Contributing
+
+See [CONTRIBUTING.md](./CONTRIBUTING.md) for details on how to add or modify content.
+
+### Run the Site Locally
+
+The requirements for running a GitHub pages site locally is described in [GitHub help](https://help.github.com/articles/setting-up-your-github-pages-site-locally-with-jekyll/#requirements). The steps below summarize these steps.
+
+> If you have run the site before, you can start with step 1 and then move on to step 5.
+
+1. Ensure that you are in the `/docs` directory in your local RocksDB clone (i.e., the same directory where this `README.md` exists). The below RubyGems commands, etc. must be run from there.
+
+1. Make sure you have Ruby and [RubyGems](https://rubygems.org/) installed.
+
+ > Ruby >= 2.2 is required for the gems. On the latest versions of Mac OS X, Ruby 2.0 is the
+ > default. Use `brew install ruby` (or your preferred upgrade mechanism) to install a newer
+ > version of Ruby for your Mac OS X system.
+
+1. Make sure you have [Bundler](http://bundler.io/) installed.
+
+ ```
+ # may require sudo
+ gem install bundler
+ ```
+1. Install the project's dependencies
+
+ ```
+ # run this in the 'docs' directory
+ bundle install
+ ```
+
+ > If you get an error when installing `nokogiri`, you may be running into the problem described
+ > in [this nokogiri issue](https://github.com/sparklemotion/nokogiri/issues/1483). You can
+ > either `brew uninstall xz` (and then `brew install xz` after the bundle is installed) or
+ > `xcode-select --install` (although this may not work if you have already installed command
+ > line tools).
+
+1. Run Jekyll's server.
+
+ - On first runs or for structural changes to the documentation (e.g., new sidebar menu item), do a full build.
+
+ ```
+ bundle exec jekyll serve
+ ```
+
+ - For content changes only, you can use `--incremental` for faster builds.
+
+ ```
+ bundle exec jekyll serve --incremental
+ ```
+
+ > We use `bundle exec` instead of running straight `jekyll` because `bundle exec` will always use the version of Jekyll from our `Gemfile`. Just running `jekyll` will use the system version and may not necessarily be compatible.
+
+ - To run using an actual IP address, you can use `--host=0.0.0.0`
+
+ ```
+ bundle exec jekyll serve --host=0.0.0.0
+ ```
+
+ This will allow you to use the IP address associated with your machine in the URL. That way you could share it with other people.
+
+ e.g., on a Mac, you can your IP address with something like `ifconfig | grep "inet " | grep -v 127.0.0.1`.
+
+1. Either of commands in the previous step will serve up the site on your local device at http://127.0.0.1:4000/ or http://localhost:4000.
+
+### Updating the Bundle
+
+The site depends on Github Pages and the installed bundle is based on the `github-pages` gem.
+Occasionally that gem might get updated with new or changed functionality. If that is the case,
+you can run:
+
+```
+bundle update
+```
+
+to get the latest packages for the installation.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/TEMPLATE-INFORMATION.md
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/TEMPLATE-INFORMATION.md b/thirdparty/rocksdb/docs/TEMPLATE-INFORMATION.md
new file mode 100644
index 0000000..9175bc0
--- /dev/null
+++ b/thirdparty/rocksdb/docs/TEMPLATE-INFORMATION.md
@@ -0,0 +1,17 @@
+## Template Details
+
+First, go through `_config.yml` and adjust the available settings to your project's standard. When you make changes here, you'll have to kill the `jekyll serve` instance and restart it to see those changes, but that's only the case with the config file.
+
+Next, update some image assets - you'll want to update `favicon.png`, `logo.svg`, and `og_image.png` (used for Like button stories and Shares on Facbeook) in the `static` folder with your own logos.
+
+Next, if you're going to have docs on your site, keep the `_docs` and `docs` folders, if not, you can safely remove them (or you can safely leave them and not include them in your navigation - Jekyll renders all of this before a client views the site anyway, so there's no performance hit from just leaving it there for a future expansion).
+
+Same thing with a blog section, either keep or delete the `_posts` and `blog` folders.
+
+You can customize your homepage in three parts - the first in the homepage header, which is mostly automatically derived from the elements you insert into your config file. However, you can also specify a series of 'promotional' elements in `_data/promo.yml`. You can read that file for more information.
+
+The second place for your homepage is in `index.md` which contains the bulk of the main content below the header. This is all markdown if you want, but you can use HTML and Jekyll's template tags (called Liquid) in there too. Checkout this folder's index.md for an example of one common template tag that we use on our sites called gridblocks.
+
+The third and last place is in the `_data/powered_by.yml` and `_data/powered_by_highlight.yml` files. Both these files combine to create a section on the homepage that is intended to show a list of companies or apps that are using your project. The `powered_by_highlight` file is a list of curated companies/apps that you want to show as a highlight at the top of this section, including their logos in whatever format you want. The `powered_by` file is a more open list that is just text links to the companies/apps and can be updated via Pull Request by the community. If you don't want these sections on your homepage, just empty out both files and leave them blank.
+
+The last thing you'll want to do is setup your top level navigation bar. You can do this by editing `nav.yml` and keeping the existing title/href/category structure used there. Although the nav is responsive and fairly flexible design-wise, no more than 5 or 6 nav items is recommended.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_config.yml
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_config.yml b/thirdparty/rocksdb/docs/_config.yml
new file mode 100644
index 0000000..2e5cee0
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_config.yml
@@ -0,0 +1,85 @@
+# Site settings
+permalink: /blog/:year/:month/:day/:title.html
+title: RocksDB
+tagline: A persistent key-value store for fast storage environments
+description: >
+ RocksDB is an embeddable persistent key-value store for fast storage.
+fbappid: "1615782811974223"
+gacode: "UA-49459723-1"
+# baseurl determines the subpath of your site. For example if you're using an
+# organisation.github.io/reponame/ basic site URL, then baseurl would be set
+# as "/reponame" but leave blank if you have a top-level domain URL as it is
+# now set to "" by default as discussed in:
+# http://jekyllrb.com/news/2016/10/06/jekyll-3-3-is-here/
+baseurl: ""
+
+# the base hostname & protocol for your site
+# If baseurl is set, then the absolute url for your site would be url/baseurl
+# This was also be set to the right thing automatically for local development
+# https://github.com/blog/2277-what-s-new-in-github-pages-with-jekyll-3-3
+# http://jekyllrb.com/news/2016/10/06/jekyll-3-3-is-here/
+url: "http://rocksdb.org"
+
+# Note: There are new filters in Jekyll 3.3 to help with absolute and relative urls
+# absolute_url
+# relative_url
+# So you will see these used throughout the Jekyll code in this template.
+# no more need for | prepend: site.url | prepend: site.baseurl
+# http://jekyllrb.com/news/2016/10/06/jekyll-3-3-is-here/
+#https://github.com/blog/2277-what-s-new-in-github-pages-with-jekyll-3-3
+
+# The GitHub repo for your project
+ghrepo: "facebook/rocksdb"
+
+# Use these color settings to determine your colour scheme for the site.
+color:
+ # primary should be a vivid color that reflects the project's brand
+ primary: "#2a2a2a"
+ # secondary should be a subtle light or dark color used on page backgrounds
+ secondary: "#f9f9f9"
+ # Use the following to specify whether the previous two colours are 'light'
+ # or 'dark' and therefore what colors can be overlaid on them
+ primary-overlay: "dark"
+ secondary-overlay: "light"
+
+#Uncomment this if you want to enable Algolia doc search with your own values
+#searchconfig:
+# apikey: ""
+# indexname: ""
+
+# Blog posts are builtin to Jekyll by default, with the `_posts` directory.
+# Here you can specify other types of documentation. The names here are `docs`
+# and `top-level`. This means their content will be in `_docs` and `_top-level`.
+# The permalink format is also given.
+# http://ben.balter.com/2015/02/20/jekyll-collections/
+collections:
+ docs:
+ output: true
+ permalink: /docs/:name/
+ top-level:
+ output: true
+ permalink: :name.html
+
+# DO NOT ADJUST BELOW THIS LINE UNLESS YOU KNOW WHAT YOU ARE CHANGING
+
+markdown: kramdown
+kramdown:
+ input: GFM
+ syntax_highlighter: rouge
+
+ syntax_highlighter_opts:
+ css_class: 'rougeHighlight'
+ span:
+ line_numbers: false
+ block:
+ line_numbers: true
+ start_line: 1
+
+sass:
+ style: :compressed
+
+redcarpet:
+ extensions: [with_toc_data]
+
+gems:
+ - jekyll-redirect-from
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_data/authors.yml
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_data/authors.yml b/thirdparty/rocksdb/docs/_data/authors.yml
new file mode 100644
index 0000000..b3ed49e
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_data/authors.yml
@@ -0,0 +1,62 @@
+icanadi:
+ full_name: Igor Canadi
+ fbid: 706165749
+
+xjin:
+ full_name: Xing Jin
+ fbid: 100000739847320
+
+leijin:
+ full_name: Lei Jin
+ fbid: 634570164
+
+yhciang:
+ full_name: Yueh-Hsuan Chiang
+ fbid: 1619020986
+
+radheshyam:
+ full_name: Radheshyam Balasundaram
+ fbid: 800837305
+
+zagfox:
+ full_name: Feng Zhu
+ fbid: 100006493823622
+
+lgalanis:
+ full_name: Leonidas Galanis
+ fbid: 8649950
+
+sdong:
+ full_name: Siying Dong
+ fbid: 9805119
+
+dmitrism:
+ full_name: Dmitri Smirnov
+
+rven2:
+ full_name: Venkatesh Radhakrishnan
+ fbid: 100008352697325
+
+yiwu:
+ full_name: Yi Wu
+ fbid: 100000476362039
+
+maysamyabandeh:
+ full_name: Maysam Yabandeh
+ fbid: 100003482360101
+
+IslamAbdelRahman:
+ full_name: Islam AbdelRahman
+ fbid: 642759407
+
+ajkr:
+ full_name: Andrew Kryczka
+ fbid: 100010829806660
+
+sagar0:
+ full_name: Sagar Vemuri
+ fbid: 2419111
+
+lightmark:
+ full_name: Aaron Gao
+ fbid: 1351549072
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_data/features.yml
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_data/features.yml b/thirdparty/rocksdb/docs/_data/features.yml
new file mode 100644
index 0000000..d692c18
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_data/features.yml
@@ -0,0 +1,19 @@
+- title: High Performance
+ text: |
+ RocksDB uses a log structured database engine, written entirely in C++, for maximum performance. Keys and values are just arbitrarily-sized byte streams.
+ image: images/promo-performance.svg
+
+- title: Optimized for Fast Storage
+ text: |
+ RocksDB is optimized for fast, low latency storage such as flash drives and high-speed disk drives. RocksDB exploits the full potential of high read/write rates offered by flash or RAM.
+ image: images/promo-flash.svg
+
+- title: Adaptable
+ text: |
+ RocksDB is adaptable to different workloads. From database storage engines such as [MyRocks](https://github.com/facebook/mysql-5.6) to [application data caching](http://techblog.netflix.com/2016/05/application-data-caching-using-ssds.html) to embedded workloads, RocksDB can be used for a variety of data needs.
+ image: images/promo-adapt.svg
+
+- title: Basic and Advanced Database Operations
+ text: |
+ RocksDB provides basic operations such as opening and closing a database, reading and writing to more advanced operations such as merging and compaction filters.
+ image: images/promo-operations.svg
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_data/nav.yml
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_data/nav.yml b/thirdparty/rocksdb/docs/_data/nav.yml
new file mode 100644
index 0000000..108de02
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_data/nav.yml
@@ -0,0 +1,30 @@
+- title: Docs
+ href: /docs/
+ category: docs
+
+- title: GitHub
+ href: https://github.com/facebook/rocksdb/
+ category: external
+
+- title: API (C++)
+ href: https://github.com/facebook/rocksdb/tree/master/include/rocksdb
+ category: external
+
+- title: API (Java)
+ href: https://github.com/facebook/rocksdb/tree/master/java/src/main/java/org/rocksdb
+ category: external
+
+- title: Support
+ href: /support.html
+ category: support
+
+- title: Blog
+ href: /blog/
+ category: blog
+
+- title: Facebook
+ href: https://www.facebook.com/groups/rocksdb.dev/
+ category: external
+
+# Use external for external links not associated with the paths of the current site.
+# If a category is external, site urls, for example, are not prepended to the href, etc..
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_data/nav_docs.yml
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_data/nav_docs.yml b/thirdparty/rocksdb/docs/_data/nav_docs.yml
new file mode 100644
index 0000000..8cdfd2d
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_data/nav_docs.yml
@@ -0,0 +1,3 @@
+- title: Quick Start
+ items:
+ - id: getting-started
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_data/powered_by.yml
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_data/powered_by.yml b/thirdparty/rocksdb/docs/_data/powered_by.yml
new file mode 100644
index 0000000..a780cfe
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_data/powered_by.yml
@@ -0,0 +1 @@
+# Fill in later if desired
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_data/powered_by_highlight.yml
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_data/powered_by_highlight.yml b/thirdparty/rocksdb/docs/_data/powered_by_highlight.yml
new file mode 100644
index 0000000..a780cfe
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_data/powered_by_highlight.yml
@@ -0,0 +1 @@
+# Fill in later if desired
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/docs/_data/promo.yml
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/docs/_data/promo.yml b/thirdparty/rocksdb/docs/_data/promo.yml
new file mode 100644
index 0000000..9a72aa8
--- /dev/null
+++ b/thirdparty/rocksdb/docs/_data/promo.yml
@@ -0,0 +1,6 @@
+# This file determines the list of promotional elements added to the header of \
+# your site's homepage. Full list of plugins are shown
+
+- type: button
+ href: docs/getting-started.html
+ text: Get Started
[11/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/range_del_aggregator.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/range_del_aggregator.cc b/thirdparty/rocksdb/db/range_del_aggregator.cc
new file mode 100644
index 0000000..0aa5d22
--- /dev/null
+++ b/thirdparty/rocksdb/db/range_del_aggregator.cc
@@ -0,0 +1,519 @@
+// Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "db/range_del_aggregator.h"
+
+#include <algorithm>
+
+namespace rocksdb {
+
+RangeDelAggregator::RangeDelAggregator(
+ const InternalKeyComparator& icmp,
+ const std::vector<SequenceNumber>& snapshots,
+ bool collapse_deletions /* = true */)
+ : upper_bound_(kMaxSequenceNumber),
+ icmp_(icmp),
+ collapse_deletions_(collapse_deletions) {
+ InitRep(snapshots);
+}
+
+RangeDelAggregator::RangeDelAggregator(const InternalKeyComparator& icmp,
+ SequenceNumber snapshot,
+ bool collapse_deletions /* = false */)
+ : upper_bound_(snapshot),
+ icmp_(icmp),
+ collapse_deletions_(collapse_deletions) {}
+
+void RangeDelAggregator::InitRep(const std::vector<SequenceNumber>& snapshots) {
+ assert(rep_ == nullptr);
+ rep_.reset(new Rep());
+ for (auto snapshot : snapshots) {
+ rep_->stripe_map_.emplace(
+ snapshot,
+ PositionalTombstoneMap(TombstoneMap(
+ stl_wrappers::LessOfComparator(icmp_.user_comparator()))));
+ }
+ // Data newer than any snapshot falls in this catch-all stripe
+ rep_->stripe_map_.emplace(
+ kMaxSequenceNumber,
+ PositionalTombstoneMap(TombstoneMap(
+ stl_wrappers::LessOfComparator(icmp_.user_comparator()))));
+ rep_->pinned_iters_mgr_.StartPinning();
+}
+
+bool RangeDelAggregator::ShouldDelete(
+ const Slice& internal_key, RangeDelAggregator::RangePositioningMode mode) {
+ if (rep_ == nullptr) {
+ return false;
+ }
+ ParsedInternalKey parsed;
+ if (!ParseInternalKey(internal_key, &parsed)) {
+ assert(false);
+ }
+ return ShouldDelete(parsed, mode);
+}
+
+bool RangeDelAggregator::ShouldDelete(
+ const ParsedInternalKey& parsed,
+ RangeDelAggregator::RangePositioningMode mode) {
+ assert(IsValueType(parsed.type));
+ if (rep_ == nullptr) {
+ return false;
+ }
+ auto& positional_tombstone_map = GetPositionalTombstoneMap(parsed.sequence);
+ const auto& tombstone_map = positional_tombstone_map.raw_map;
+ if (tombstone_map.empty()) {
+ return false;
+ }
+ auto& tombstone_map_iter = positional_tombstone_map.iter;
+ if (tombstone_map_iter == tombstone_map.end() &&
+ (mode == kForwardTraversal || mode == kBackwardTraversal)) {
+ // invalid (e.g., if AddTombstones() changed the deletions), so need to
+ // reseek
+ mode = kBinarySearch;
+ }
+ switch (mode) {
+ case kFullScan:
+ assert(!collapse_deletions_);
+ // The maintained state (PositionalTombstoneMap::iter) isn't useful when
+ // we linear scan from the beginning each time, but we maintain it anyways
+ // for consistency.
+ tombstone_map_iter = tombstone_map.begin();
+ while (tombstone_map_iter != tombstone_map.end()) {
+ const auto& tombstone = tombstone_map_iter->second;
+ if (icmp_.user_comparator()->Compare(parsed.user_key,
+ tombstone.start_key_) < 0) {
+ break;
+ }
+ if (parsed.sequence < tombstone.seq_ &&
+ icmp_.user_comparator()->Compare(parsed.user_key,
+ tombstone.end_key_) < 0) {
+ return true;
+ }
+ ++tombstone_map_iter;
+ }
+ return false;
+ case kForwardTraversal:
+ assert(collapse_deletions_ && tombstone_map_iter != tombstone_map.end());
+ if (tombstone_map_iter == tombstone_map.begin() &&
+ icmp_.user_comparator()->Compare(parsed.user_key,
+ tombstone_map_iter->first) < 0) {
+ // before start of deletion intervals
+ return false;
+ }
+ while (std::next(tombstone_map_iter) != tombstone_map.end() &&
+ icmp_.user_comparator()->Compare(
+ std::next(tombstone_map_iter)->first, parsed.user_key) <= 0) {
+ ++tombstone_map_iter;
+ }
+ break;
+ case kBackwardTraversal:
+ assert(collapse_deletions_ && tombstone_map_iter != tombstone_map.end());
+ while (tombstone_map_iter != tombstone_map.begin() &&
+ icmp_.user_comparator()->Compare(parsed.user_key,
+ tombstone_map_iter->first) < 0) {
+ --tombstone_map_iter;
+ }
+ if (tombstone_map_iter == tombstone_map.begin() &&
+ icmp_.user_comparator()->Compare(parsed.user_key,
+ tombstone_map_iter->first) < 0) {
+ // before start of deletion intervals
+ return false;
+ }
+ break;
+ case kBinarySearch:
+ assert(collapse_deletions_);
+ tombstone_map_iter =
+ tombstone_map.upper_bound(parsed.user_key);
+ if (tombstone_map_iter == tombstone_map.begin()) {
+ // before start of deletion intervals
+ return false;
+ }
+ --tombstone_map_iter;
+ break;
+ }
+ assert(mode != kFullScan);
+ assert(tombstone_map_iter != tombstone_map.end() &&
+ icmp_.user_comparator()->Compare(tombstone_map_iter->first,
+ parsed.user_key) <= 0);
+ assert(std::next(tombstone_map_iter) == tombstone_map.end() ||
+ icmp_.user_comparator()->Compare(
+ parsed.user_key, std::next(tombstone_map_iter)->first) < 0);
+ return parsed.sequence < tombstone_map_iter->second.seq_;
+}
+
+bool RangeDelAggregator::ShouldAddTombstones(
+ bool bottommost_level /* = false */) {
+ // TODO(andrewkr): can we just open a file and throw it away if it ends up
+ // empty after AddToBuilder()? This function doesn't take into subcompaction
+ // boundaries so isn't completely accurate.
+ if (rep_ == nullptr) {
+ return false;
+ }
+ auto stripe_map_iter = rep_->stripe_map_.begin();
+ assert(stripe_map_iter != rep_->stripe_map_.end());
+ if (bottommost_level) {
+ // For the bottommost level, keys covered by tombstones in the first
+ // (oldest) stripe have been compacted away, so the tombstones are obsolete.
+ ++stripe_map_iter;
+ }
+ while (stripe_map_iter != rep_->stripe_map_.end()) {
+ if (!stripe_map_iter->second.raw_map.empty()) {
+ return true;
+ }
+ ++stripe_map_iter;
+ }
+ return false;
+}
+
+Status RangeDelAggregator::AddTombstones(
+ std::unique_ptr<InternalIterator> input) {
+ if (input == nullptr) {
+ return Status::OK();
+ }
+ input->SeekToFirst();
+ bool first_iter = true;
+ while (input->Valid()) {
+ if (first_iter) {
+ if (rep_ == nullptr) {
+ InitRep({upper_bound_});
+ } else {
+ InvalidateTombstoneMapPositions();
+ }
+ first_iter = false;
+ }
+ ParsedInternalKey parsed_key;
+ if (!ParseInternalKey(input->key(), &parsed_key)) {
+ return Status::Corruption("Unable to parse range tombstone InternalKey");
+ }
+ RangeTombstone tombstone(parsed_key, input->value());
+ AddTombstone(std::move(tombstone));
+ input->Next();
+ }
+ if (!first_iter) {
+ rep_->pinned_iters_mgr_.PinIterator(input.release(), false /* arena */);
+ }
+ return Status::OK();
+}
+
+void RangeDelAggregator::InvalidateTombstoneMapPositions() {
+ if (rep_ == nullptr) {
+ return;
+ }
+ for (auto stripe_map_iter = rep_->stripe_map_.begin();
+ stripe_map_iter != rep_->stripe_map_.end(); ++stripe_map_iter) {
+ stripe_map_iter->second.iter = stripe_map_iter->second.raw_map.end();
+ }
+}
+
+Status RangeDelAggregator::AddTombstone(RangeTombstone tombstone) {
+ auto& positional_tombstone_map = GetPositionalTombstoneMap(tombstone.seq_);
+ auto& tombstone_map = positional_tombstone_map.raw_map;
+ if (collapse_deletions_) {
+ // In collapsed mode, we only fill the seq_ field in the TombstoneMap's
+ // values. The end_key is unneeded because we assume the tombstone extends
+ // until the next tombstone starts. For gaps between real tombstones and
+ // for the last real tombstone, we denote end keys by inserting fake
+ // tombstones with sequence number zero.
+ std::vector<RangeTombstone> new_range_dels{
+ tombstone, RangeTombstone(tombstone.end_key_, Slice(), 0)};
+ auto new_range_dels_iter = new_range_dels.begin();
+ // Position at the first overlapping existing tombstone; if none exists,
+ // insert until we find an existing one overlapping a new point
+ const Slice* tombstone_map_begin = nullptr;
+ if (!tombstone_map.empty()) {
+ tombstone_map_begin = &tombstone_map.begin()->first;
+ }
+ auto last_range_dels_iter = new_range_dels_iter;
+ while (new_range_dels_iter != new_range_dels.end() &&
+ (tombstone_map_begin == nullptr ||
+ icmp_.user_comparator()->Compare(new_range_dels_iter->start_key_,
+ *tombstone_map_begin) < 0)) {
+ tombstone_map.emplace(
+ new_range_dels_iter->start_key_,
+ RangeTombstone(Slice(), Slice(), new_range_dels_iter->seq_));
+ last_range_dels_iter = new_range_dels_iter;
+ ++new_range_dels_iter;
+ }
+ if (new_range_dels_iter == new_range_dels.end()) {
+ return Status::OK();
+ }
+ // above loop advances one too far
+ new_range_dels_iter = last_range_dels_iter;
+ auto tombstone_map_iter =
+ tombstone_map.upper_bound(new_range_dels_iter->start_key_);
+ // if nothing overlapped we would've already inserted all the new points
+ // and returned early
+ assert(tombstone_map_iter != tombstone_map.begin());
+ tombstone_map_iter--;
+
+ // untermed_seq is non-kMaxSequenceNumber when we covered an existing point
+ // but haven't seen its corresponding endpoint. It's used for (1) deciding
+ // whether to forcibly insert the new interval's endpoint; and (2) possibly
+ // raising the seqnum for the to-be-inserted element (we insert the max
+ // seqnum between the next new interval and the unterminated interval).
+ SequenceNumber untermed_seq = kMaxSequenceNumber;
+ while (tombstone_map_iter != tombstone_map.end() &&
+ new_range_dels_iter != new_range_dels.end()) {
+ const Slice *tombstone_map_iter_end = nullptr,
+ *new_range_dels_iter_end = nullptr;
+ if (tombstone_map_iter != tombstone_map.end()) {
+ auto next_tombstone_map_iter = std::next(tombstone_map_iter);
+ if (next_tombstone_map_iter != tombstone_map.end()) {
+ tombstone_map_iter_end = &next_tombstone_map_iter->first;
+ }
+ }
+ if (new_range_dels_iter != new_range_dels.end()) {
+ auto next_new_range_dels_iter = std::next(new_range_dels_iter);
+ if (next_new_range_dels_iter != new_range_dels.end()) {
+ new_range_dels_iter_end = &next_new_range_dels_iter->start_key_;
+ }
+ }
+
+ // our positions in existing/new tombstone collections should always
+ // overlap. The non-overlapping cases are handled above and below this
+ // loop.
+ assert(new_range_dels_iter_end == nullptr ||
+ icmp_.user_comparator()->Compare(tombstone_map_iter->first,
+ *new_range_dels_iter_end) < 0);
+ assert(tombstone_map_iter_end == nullptr ||
+ icmp_.user_comparator()->Compare(new_range_dels_iter->start_key_,
+ *tombstone_map_iter_end) < 0);
+
+ int new_to_old_start_cmp = icmp_.user_comparator()->Compare(
+ new_range_dels_iter->start_key_, tombstone_map_iter->first);
+ // nullptr end means extends infinitely rightwards, set new_to_old_end_cmp
+ // accordingly so we can use common code paths later.
+ int new_to_old_end_cmp;
+ if (new_range_dels_iter_end == nullptr &&
+ tombstone_map_iter_end == nullptr) {
+ new_to_old_end_cmp = 0;
+ } else if (new_range_dels_iter_end == nullptr) {
+ new_to_old_end_cmp = 1;
+ } else if (tombstone_map_iter_end == nullptr) {
+ new_to_old_end_cmp = -1;
+ } else {
+ new_to_old_end_cmp = icmp_.user_comparator()->Compare(
+ *new_range_dels_iter_end, *tombstone_map_iter_end);
+ }
+
+ if (new_to_old_start_cmp < 0) {
+ // the existing one's left endpoint comes after, so raise/delete it if
+ // it's covered.
+ if (tombstone_map_iter->second.seq_ < new_range_dels_iter->seq_) {
+ untermed_seq = tombstone_map_iter->second.seq_;
+ if (tombstone_map_iter != tombstone_map.begin() &&
+ std::prev(tombstone_map_iter)->second.seq_ ==
+ new_range_dels_iter->seq_) {
+ tombstone_map_iter = tombstone_map.erase(tombstone_map_iter);
+ --tombstone_map_iter;
+ } else {
+ tombstone_map_iter->second.seq_ = new_range_dels_iter->seq_;
+ }
+ }
+ } else if (new_to_old_start_cmp > 0) {
+ if (untermed_seq != kMaxSequenceNumber ||
+ tombstone_map_iter->second.seq_ < new_range_dels_iter->seq_) {
+ auto seq = tombstone_map_iter->second.seq_;
+ // need to adjust this element if not intended to span beyond the new
+ // element (i.e., was_tombstone_map_iter_raised == true), or if it
+ // can be raised
+ tombstone_map_iter = tombstone_map.emplace(
+ new_range_dels_iter->start_key_,
+ RangeTombstone(
+ Slice(), Slice(),
+ std::max(
+ untermed_seq == kMaxSequenceNumber ? 0 : untermed_seq,
+ new_range_dels_iter->seq_)));
+ untermed_seq = seq;
+ }
+ } else {
+ // their left endpoints coincide, so raise the existing one if needed
+ if (tombstone_map_iter->second.seq_ < new_range_dels_iter->seq_) {
+ untermed_seq = tombstone_map_iter->second.seq_;
+ tombstone_map_iter->second.seq_ = new_range_dels_iter->seq_;
+ }
+ }
+
+ // advance whichever one ends earlier, or both if their right endpoints
+ // coincide
+ if (new_to_old_end_cmp < 0) {
+ ++new_range_dels_iter;
+ } else if (new_to_old_end_cmp > 0) {
+ ++tombstone_map_iter;
+ untermed_seq = kMaxSequenceNumber;
+ } else {
+ ++new_range_dels_iter;
+ ++tombstone_map_iter;
+ untermed_seq = kMaxSequenceNumber;
+ }
+ }
+ while (new_range_dels_iter != new_range_dels.end()) {
+ tombstone_map.emplace(
+ new_range_dels_iter->start_key_,
+ RangeTombstone(Slice(), Slice(), new_range_dels_iter->seq_));
+ ++new_range_dels_iter;
+ }
+ } else {
+ tombstone_map.emplace(tombstone.start_key_, std::move(tombstone));
+ }
+ return Status::OK();
+}
+
+RangeDelAggregator::PositionalTombstoneMap&
+RangeDelAggregator::GetPositionalTombstoneMap(SequenceNumber seq) {
+ assert(rep_ != nullptr);
+ // The stripe includes seqnum for the snapshot above and excludes seqnum for
+ // the snapshot below.
+ StripeMap::iterator iter;
+ if (seq > 0) {
+ // upper_bound() checks strict inequality so need to subtract one
+ iter = rep_->stripe_map_.upper_bound(seq - 1);
+ } else {
+ iter = rep_->stripe_map_.begin();
+ }
+ // catch-all stripe justifies this assertion in either of above cases
+ assert(iter != rep_->stripe_map_.end());
+ return iter->second;
+}
+
+// TODO(andrewkr): We should implement an iterator over range tombstones in our
+// map. It'd enable compaction to open tables on-demand, i.e., only once range
+// tombstones are known to be available, without the code duplication we have
+// in ShouldAddTombstones(). It'll also allow us to move the table-modifying
+// code into more coherent places: CompactionJob and BuildTable().
+void RangeDelAggregator::AddToBuilder(
+ TableBuilder* builder, const Slice* lower_bound, const Slice* upper_bound,
+ FileMetaData* meta,
+ CompactionIterationStats* range_del_out_stats /* = nullptr */,
+ bool bottommost_level /* = false */) {
+ if (rep_ == nullptr) {
+ return;
+ }
+ auto stripe_map_iter = rep_->stripe_map_.begin();
+ assert(stripe_map_iter != rep_->stripe_map_.end());
+ if (bottommost_level) {
+ // TODO(andrewkr): these are counted for each compaction output file, so
+ // lots of double-counting.
+ if (!stripe_map_iter->second.raw_map.empty()) {
+ range_del_out_stats->num_range_del_drop_obsolete +=
+ static_cast<int64_t>(stripe_map_iter->second.raw_map.size()) -
+ (collapse_deletions_ ? 1 : 0);
+ range_del_out_stats->num_record_drop_obsolete +=
+ static_cast<int64_t>(stripe_map_iter->second.raw_map.size()) -
+ (collapse_deletions_ ? 1 : 0);
+ }
+ // For the bottommost level, keys covered by tombstones in the first
+ // (oldest) stripe have been compacted away, so the tombstones are obsolete.
+ ++stripe_map_iter;
+ }
+
+ // Note the order in which tombstones are stored is insignificant since we
+ // insert them into a std::map on the read path.
+ bool first_added = false;
+ while (stripe_map_iter != rep_->stripe_map_.end()) {
+ for (auto tombstone_map_iter = stripe_map_iter->second.raw_map.begin();
+ tombstone_map_iter != stripe_map_iter->second.raw_map.end();
+ ++tombstone_map_iter) {
+ RangeTombstone tombstone;
+ if (collapse_deletions_) {
+ auto next_tombstone_map_iter = std::next(tombstone_map_iter);
+ if (next_tombstone_map_iter == stripe_map_iter->second.raw_map.end() ||
+ tombstone_map_iter->second.seq_ == 0) {
+ // it's a sentinel tombstone
+ continue;
+ }
+ tombstone.start_key_ = tombstone_map_iter->first;
+ tombstone.end_key_ = next_tombstone_map_iter->first;
+ tombstone.seq_ = tombstone_map_iter->second.seq_;
+ } else {
+ tombstone = tombstone_map_iter->second;
+ }
+ if (upper_bound != nullptr &&
+ icmp_.user_comparator()->Compare(*upper_bound,
+ tombstone.start_key_) <= 0) {
+ // Tombstones starting at upper_bound or later only need to be included
+ // in the next table. Break because subsequent tombstones will start
+ // even later.
+ break;
+ }
+ if (lower_bound != nullptr &&
+ icmp_.user_comparator()->Compare(tombstone.end_key_,
+ *lower_bound) <= 0) {
+ // Tombstones ending before or at lower_bound only need to be included
+ // in the prev table. Continue because subsequent tombstones may still
+ // overlap [lower_bound, upper_bound).
+ continue;
+ }
+
+ auto ikey_and_end_key = tombstone.Serialize();
+ builder->Add(ikey_and_end_key.first.Encode(), ikey_and_end_key.second);
+ if (!first_added) {
+ first_added = true;
+ InternalKey smallest_candidate = std::move(ikey_and_end_key.first);;
+ if (lower_bound != nullptr &&
+ icmp_.user_comparator()->Compare(smallest_candidate.user_key(),
+ *lower_bound) <= 0) {
+ // Pretend the smallest key has the same user key as lower_bound
+ // (the max key in the previous table or subcompaction) in order for
+ // files to appear key-space partitioned.
+ //
+ // Choose lowest seqnum so this file's smallest internal key comes
+ // after the previous file's/subcompaction's largest. The fake seqnum
+ // is OK because the read path's file-picking code only considers user
+ // key.
+ smallest_candidate = InternalKey(*lower_bound, 0, kTypeRangeDeletion);
+ }
+ if (meta->smallest.size() == 0 ||
+ icmp_.Compare(smallest_candidate, meta->smallest) < 0) {
+ meta->smallest = std::move(smallest_candidate);
+ }
+ }
+ InternalKey largest_candidate = tombstone.SerializeEndKey();
+ if (upper_bound != nullptr &&
+ icmp_.user_comparator()->Compare(*upper_bound,
+ largest_candidate.user_key()) <= 0) {
+ // Pretend the largest key has the same user key as upper_bound (the
+ // min key in the following table or subcompaction) in order for files
+ // to appear key-space partitioned.
+ //
+ // Choose highest seqnum so this file's largest internal key comes
+ // before the next file's/subcompaction's smallest. The fake seqnum is
+ // OK because the read path's file-picking code only considers the user
+ // key portion.
+ //
+ // Note Seek() also creates InternalKey with (user_key,
+ // kMaxSequenceNumber), but with kTypeDeletion (0x7) instead of
+ // kTypeRangeDeletion (0xF), so the range tombstone comes before the
+ // Seek() key in InternalKey's ordering. So Seek() will look in the
+ // next file for the user key.
+ largest_candidate = InternalKey(*upper_bound, kMaxSequenceNumber,
+ kTypeRangeDeletion);
+ }
+ if (meta->largest.size() == 0 ||
+ icmp_.Compare(meta->largest, largest_candidate) < 0) {
+ meta->largest = std::move(largest_candidate);
+ }
+ meta->smallest_seqno = std::min(meta->smallest_seqno, tombstone.seq_);
+ meta->largest_seqno = std::max(meta->largest_seqno, tombstone.seq_);
+ }
+ ++stripe_map_iter;
+ }
+}
+
+bool RangeDelAggregator::IsEmpty() {
+ if (rep_ == nullptr) {
+ return true;
+ }
+ for (auto stripe_map_iter = rep_->stripe_map_.begin();
+ stripe_map_iter != rep_->stripe_map_.end(); ++stripe_map_iter) {
+ if (!stripe_map_iter->second.raw_map.empty()) {
+ return false;
+ }
+ }
+ return true;
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/range_del_aggregator.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/range_del_aggregator.h b/thirdparty/rocksdb/db/range_del_aggregator.h
new file mode 100644
index 0000000..9d4b8ca
--- /dev/null
+++ b/thirdparty/rocksdb/db/range_del_aggregator.h
@@ -0,0 +1,161 @@
+// Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "db/compaction_iteration_stats.h"
+#include "db/dbformat.h"
+#include "db/pinned_iterators_manager.h"
+#include "db/version_edit.h"
+#include "include/rocksdb/comparator.h"
+#include "include/rocksdb/types.h"
+#include "table/internal_iterator.h"
+#include "table/scoped_arena_iterator.h"
+#include "table/table_builder.h"
+#include "util/kv_map.h"
+
+namespace rocksdb {
+
+// A RangeDelAggregator aggregates range deletion tombstones as they are
+// encountered in memtables/SST files. It provides methods that check whether a
+// key is covered by range tombstones or write the relevant tombstones to a new
+// SST file.
+class RangeDelAggregator {
+ public:
+ // @param snapshots These are used to organize the tombstones into snapshot
+ // stripes, which is the seqnum range between consecutive snapshots,
+ // including the higher snapshot and excluding the lower one. Currently,
+ // this is used by ShouldDelete() to prevent deletion of keys that are
+ // covered by range tombstones in other snapshot stripes. This constructor
+ // is used for writes (flush/compaction). All DB snapshots are provided
+ // such that no keys are removed that are uncovered according to any DB
+ // snapshot.
+ // Note this overload does not lazily initialize Rep.
+ RangeDelAggregator(const InternalKeyComparator& icmp,
+ const std::vector<SequenceNumber>& snapshots,
+ bool collapse_deletions = true);
+
+ // @param upper_bound Similar to snapshots above, except with a single
+ // snapshot, which allows us to store the snapshot on the stack and defer
+ // initialization of heap-allocating members (in Rep) until the first range
+ // deletion is encountered. This constructor is used in case of reads (get/
+ // iterator), for which only the user snapshot (upper_bound) is provided
+ // such that the seqnum space is divided into two stripes. Only the older
+ // stripe will be used by ShouldDelete().
+ RangeDelAggregator(const InternalKeyComparator& icmp,
+ SequenceNumber upper_bound,
+ bool collapse_deletions = false);
+
+ // We maintain position in the tombstone map across calls to ShouldDelete. The
+ // caller may wish to specify a mode to optimize positioning the iterator
+ // during the next call to ShouldDelete. The non-kFullScan modes are only
+ // available when deletion collapsing is enabled.
+ //
+ // For example, if we invoke Next() on an iterator, kForwardTraversal should
+ // be specified to advance one-by-one through deletions until one is found
+ // with its interval containing the key. This will typically be faster than
+ // doing a full binary search (kBinarySearch).
+ enum RangePositioningMode {
+ kFullScan, // used iff collapse_deletions_ == false
+ kForwardTraversal,
+ kBackwardTraversal,
+ kBinarySearch,
+ };
+
+ // Returns whether the key should be deleted, which is the case when it is
+ // covered by a range tombstone residing in the same snapshot stripe.
+ // @param mode If collapse_deletions_ is true, this dictates how we will find
+ // the deletion whose interval contains this key. Otherwise, its
+ // value must be kFullScan indicating linear scan from beginning..
+ bool ShouldDelete(const ParsedInternalKey& parsed,
+ RangePositioningMode mode = kFullScan);
+ bool ShouldDelete(const Slice& internal_key,
+ RangePositioningMode mode = kFullScan);
+ bool ShouldAddTombstones(bool bottommost_level = false);
+
+ // Adds tombstones to the tombstone aggregation structure maintained by this
+ // object.
+ // @return non-OK status if any of the tombstone keys are corrupted.
+ Status AddTombstones(std::unique_ptr<InternalIterator> input);
+
+ // Resets iterators maintained across calls to ShouldDelete(). This may be
+ // called when the tombstones change, or the owner may call explicitly, e.g.,
+ // if it's an iterator that just seeked to an arbitrary position. The effect
+ // of invalidation is that the following call to ShouldDelete() will binary
+ // search for its tombstone.
+ void InvalidateTombstoneMapPositions();
+
+ // Writes tombstones covering a range to a table builder.
+ // @param extend_before_min_key If true, the range of tombstones to be added
+ // to the TableBuilder starts from the beginning of the key-range;
+ // otherwise, it starts from meta->smallest.
+ // @param lower_bound/upper_bound Any range deletion with [start_key, end_key)
+ // that overlaps the target range [*lower_bound, *upper_bound) is added to
+ // the builder. If lower_bound is nullptr, the target range extends
+ // infinitely to the left. If upper_bound is nullptr, the target range
+ // extends infinitely to the right. If both are nullptr, the target range
+ // extends infinitely in both directions, i.e., all range deletions are
+ // added to the builder.
+ // @param meta The file's metadata. We modify the begin and end keys according
+ // to the range tombstones added to this file such that the read path does
+ // not miss range tombstones that cover gaps before/after/between files in
+ // a level. lower_bound/upper_bound above constrain how far file boundaries
+ // can be extended.
+ // @param bottommost_level If true, we will filter out any tombstones
+ // belonging to the oldest snapshot stripe, because all keys potentially
+ // covered by this tombstone are guaranteed to have been deleted by
+ // compaction.
+ void AddToBuilder(TableBuilder* builder, const Slice* lower_bound,
+ const Slice* upper_bound, FileMetaData* meta,
+ CompactionIterationStats* range_del_out_stats = nullptr,
+ bool bottommost_level = false);
+ bool IsEmpty();
+
+ private:
+ // Maps tombstone user start key -> tombstone object
+ typedef std::multimap<Slice, RangeTombstone, stl_wrappers::LessOfComparator>
+ TombstoneMap;
+ // Also maintains position in TombstoneMap last seen by ShouldDelete(). The
+ // end iterator indicates invalidation (e.g., if AddTombstones() changes the
+ // underlying map). End iterator cannot be invalidated.
+ struct PositionalTombstoneMap {
+ explicit PositionalTombstoneMap(TombstoneMap _raw_map)
+ : raw_map(std::move(_raw_map)), iter(raw_map.end()) {}
+ PositionalTombstoneMap(const PositionalTombstoneMap&) = delete;
+ PositionalTombstoneMap(PositionalTombstoneMap&& other)
+ : raw_map(std::move(other.raw_map)), iter(raw_map.end()) {}
+
+ TombstoneMap raw_map;
+ TombstoneMap::const_iterator iter;
+ };
+
+ // Maps snapshot seqnum -> map of tombstones that fall in that stripe, i.e.,
+ // their seqnums are greater than the next smaller snapshot's seqnum.
+ typedef std::map<SequenceNumber, PositionalTombstoneMap> StripeMap;
+
+ struct Rep {
+ StripeMap stripe_map_;
+ PinnedIteratorsManager pinned_iters_mgr_;
+ };
+ // Initializes rep_ lazily. This aggregator object is constructed for every
+ // read, so expensive members should only be created when necessary, i.e.,
+ // once the first range deletion is encountered.
+ void InitRep(const std::vector<SequenceNumber>& snapshots);
+
+ PositionalTombstoneMap& GetPositionalTombstoneMap(SequenceNumber seq);
+ Status AddTombstone(RangeTombstone tombstone);
+
+ SequenceNumber upper_bound_;
+ std::unique_ptr<Rep> rep_;
+ const InternalKeyComparator& icmp_;
+ // collapse range deletions so they're binary searchable
+ const bool collapse_deletions_;
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/repair.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/repair.cc b/thirdparty/rocksdb/db/repair.cc
new file mode 100644
index 0000000..9ed3260
--- /dev/null
+++ b/thirdparty/rocksdb/db/repair.cc
@@ -0,0 +1,650 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// Repairer does best effort recovery to recover as much data as possible after
+// a disaster without compromising consistency. It does not guarantee bringing
+// the database to a time consistent state.
+//
+// Repair process is broken into 4 phases:
+// (a) Find files
+// (b) Convert logs to tables
+// (c) Extract metadata
+// (d) Write Descriptor
+//
+// (a) Find files
+//
+// The repairer goes through all the files in the directory, and classifies them
+// based on their file name. Any file that cannot be identified by name will be
+// ignored.
+//
+// (b) Convert logs to table
+//
+// Every log file that is active is replayed. All sections of the file where the
+// checksum does not match is skipped over. We intentionally give preference to
+// data consistency.
+//
+// (c) Extract metadata
+//
+// We scan every table to compute
+// (1) smallest/largest for the table
+// (2) largest sequence number in the table
+//
+// If we are unable to scan the file, then we ignore the table.
+//
+// (d) Write Descriptor
+//
+// We generate descriptor contents:
+// - log number is set to zero
+// - next-file-number is set to 1 + largest file number we found
+// - last-sequence-number is set to largest sequence# found across
+// all tables (see 2c)
+// - compaction pointers are cleared
+// - every table file is added at level 0
+//
+// Possible optimization 1:
+// (a) Compute total size and use to pick appropriate max-level M
+// (b) Sort tables by largest sequence# in the table
+// (c) For each table: if it overlaps earlier table, place in level-0,
+// else place in level-M.
+// (d) We can provide options for time consistent recovery and unsafe recovery
+// (ignore checksum failure when applicable)
+// Possible optimization 2:
+// Store per-table metadata (smallest, largest, largest-seq#, ...)
+// in the table's meta section to speed up ScanTable.
+
+#ifndef ROCKSDB_LITE
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
+#include "db/builder.h"
+#include "db/db_impl.h"
+#include "db/dbformat.h"
+#include "db/log_reader.h"
+#include "db/log_writer.h"
+#include "db/memtable.h"
+#include "db/table_cache.h"
+#include "db/version_edit.h"
+#include "db/write_batch_internal.h"
+#include "options/cf_options.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+#include "rocksdb/write_buffer_manager.h"
+#include "table/scoped_arena_iterator.h"
+#include "util/file_reader_writer.h"
+#include "util/filename.h"
+#include "util/string_util.h"
+
+namespace rocksdb {
+
+namespace {
+
+class Repairer {
+ public:
+ Repairer(const std::string& dbname, const DBOptions& db_options,
+ const std::vector<ColumnFamilyDescriptor>& column_families,
+ const ColumnFamilyOptions& default_cf_opts,
+ const ColumnFamilyOptions& unknown_cf_opts, bool create_unknown_cfs)
+ : dbname_(dbname),
+ env_(db_options.env),
+ env_options_(),
+ db_options_(SanitizeOptions(dbname_, db_options)),
+ immutable_db_options_(db_options_),
+ icmp_(default_cf_opts.comparator),
+ default_cf_opts_(default_cf_opts),
+ default_cf_iopts_(
+ ImmutableCFOptions(immutable_db_options_, default_cf_opts)),
+ unknown_cf_opts_(unknown_cf_opts),
+ create_unknown_cfs_(create_unknown_cfs),
+ raw_table_cache_(
+ // TableCache can be small since we expect each table to be opened
+ // once.
+ NewLRUCache(10, db_options_.table_cache_numshardbits)),
+ table_cache_(new TableCache(default_cf_iopts_, env_options_,
+ raw_table_cache_.get())),
+ wb_(db_options_.db_write_buffer_size),
+ wc_(db_options_.delayed_write_rate),
+ vset_(dbname_, &immutable_db_options_, env_options_,
+ raw_table_cache_.get(), &wb_, &wc_),
+ next_file_number_(1) {
+ for (const auto& cfd : column_families) {
+ cf_name_to_opts_[cfd.name] = cfd.options;
+ }
+ }
+
+ const ColumnFamilyOptions* GetColumnFamilyOptions(
+ const std::string& cf_name) {
+ if (cf_name_to_opts_.find(cf_name) == cf_name_to_opts_.end()) {
+ if (create_unknown_cfs_) {
+ return &unknown_cf_opts_;
+ }
+ return nullptr;
+ }
+ return &cf_name_to_opts_[cf_name];
+ }
+
+ // Adds a column family to the VersionSet with cf_options_ and updates
+ // manifest.
+ Status AddColumnFamily(const std::string& cf_name, uint32_t cf_id) {
+ const auto* cf_opts = GetColumnFamilyOptions(cf_name);
+ if (cf_opts == nullptr) {
+ return Status::Corruption("Encountered unknown column family with name=" +
+ cf_name + ", id=" + ToString(cf_id));
+ }
+ Options opts(db_options_, *cf_opts);
+ MutableCFOptions mut_cf_opts(opts);
+
+ VersionEdit edit;
+ edit.SetComparatorName(opts.comparator->Name());
+ edit.SetLogNumber(0);
+ edit.SetColumnFamily(cf_id);
+ ColumnFamilyData* cfd;
+ cfd = nullptr;
+ edit.AddColumnFamily(cf_name);
+
+ mutex_.Lock();
+ Status status = vset_.LogAndApply(cfd, mut_cf_opts, &edit, &mutex_,
+ nullptr /* db_directory */,
+ false /* new_descriptor_log */, cf_opts);
+ mutex_.Unlock();
+ return status;
+ }
+
+ ~Repairer() {
+ delete table_cache_;
+ }
+
+ Status Run() {
+ Status status = FindFiles();
+ if (status.ok()) {
+ // Discard older manifests and start a fresh one
+ for (size_t i = 0; i < manifests_.size(); i++) {
+ ArchiveFile(dbname_ + "/" + manifests_[i]);
+ }
+ // Just create a DBImpl temporarily so we can reuse NewDB()
+ DBImpl* db_impl = new DBImpl(db_options_, dbname_);
+ status = db_impl->NewDB();
+ delete db_impl;
+ }
+
+ if (status.ok()) {
+ // Recover using the fresh manifest created by NewDB()
+ status =
+ vset_.Recover({{kDefaultColumnFamilyName, default_cf_opts_}}, false);
+ }
+ if (status.ok()) {
+ // Need to scan existing SST files first so the column families are
+ // created before we process WAL files
+ ExtractMetaData();
+
+ // ExtractMetaData() uses table_fds_ to know which SST files' metadata to
+ // extract -- we need to clear it here since metadata for existing SST
+ // files has been extracted already
+ table_fds_.clear();
+ ConvertLogFilesToTables();
+ ExtractMetaData();
+ status = AddTables();
+ }
+ if (status.ok()) {
+ uint64_t bytes = 0;
+ for (size_t i = 0; i < tables_.size(); i++) {
+ bytes += tables_[i].meta.fd.GetFileSize();
+ }
+ ROCKS_LOG_WARN(db_options_.info_log,
+ "**** Repaired rocksdb %s; "
+ "recovered %" ROCKSDB_PRIszt " files; %" PRIu64
+ "bytes. "
+ "Some data may have been lost. "
+ "****",
+ dbname_.c_str(), tables_.size(), bytes);
+ }
+ return status;
+ }
+
+ private:
+ struct TableInfo {
+ FileMetaData meta;
+ uint32_t column_family_id;
+ std::string column_family_name;
+ SequenceNumber min_sequence;
+ SequenceNumber max_sequence;
+ };
+
+ std::string const dbname_;
+ Env* const env_;
+ const EnvOptions env_options_;
+ const DBOptions db_options_;
+ const ImmutableDBOptions immutable_db_options_;
+ const InternalKeyComparator icmp_;
+ const ColumnFamilyOptions default_cf_opts_;
+ const ImmutableCFOptions default_cf_iopts_; // table_cache_ holds reference
+ const ColumnFamilyOptions unknown_cf_opts_;
+ const bool create_unknown_cfs_;
+ std::shared_ptr<Cache> raw_table_cache_;
+ TableCache* table_cache_;
+ WriteBufferManager wb_;
+ WriteController wc_;
+ VersionSet vset_;
+ std::unordered_map<std::string, ColumnFamilyOptions> cf_name_to_opts_;
+ InstrumentedMutex mutex_;
+
+ std::vector<std::string> manifests_;
+ std::vector<FileDescriptor> table_fds_;
+ std::vector<uint64_t> logs_;
+ std::vector<TableInfo> tables_;
+ uint64_t next_file_number_;
+
+ Status FindFiles() {
+ std::vector<std::string> filenames;
+ bool found_file = false;
+ std::vector<std::string> to_search_paths;
+
+ for (size_t path_id = 0; path_id < db_options_.db_paths.size(); path_id++) {
+ to_search_paths.push_back(db_options_.db_paths[path_id].path);
+ }
+
+ // search wal_dir if user uses a customize wal_dir
+ if (!db_options_.wal_dir.empty() &&
+ db_options_.wal_dir != dbname_) {
+ to_search_paths.push_back(db_options_.wal_dir);
+ }
+
+ for (size_t path_id = 0; path_id < to_search_paths.size(); path_id++) {
+ Status status =
+ env_->GetChildren(to_search_paths[path_id], &filenames);
+ if (!status.ok()) {
+ return status;
+ }
+ if (!filenames.empty()) {
+ found_file = true;
+ }
+
+ uint64_t number;
+ FileType type;
+ for (size_t i = 0; i < filenames.size(); i++) {
+ if (ParseFileName(filenames[i], &number, &type)) {
+ if (type == kDescriptorFile) {
+ manifests_.push_back(filenames[i]);
+ } else {
+ if (number + 1 > next_file_number_) {
+ next_file_number_ = number + 1;
+ }
+ if (type == kLogFile) {
+ logs_.push_back(number);
+ } else if (type == kTableFile) {
+ table_fds_.emplace_back(number, static_cast<uint32_t>(path_id),
+ 0);
+ } else {
+ // Ignore other files
+ }
+ }
+ }
+ }
+ }
+ if (!found_file) {
+ return Status::Corruption(dbname_, "repair found no files");
+ }
+ return Status::OK();
+ }
+
+ void ConvertLogFilesToTables() {
+ for (size_t i = 0; i < logs_.size(); i++) {
+ // we should use LogFileName(wal_dir, logs_[i]) here. user might uses wal_dir option.
+ std::string logname = LogFileName(db_options_.wal_dir, logs_[i]);
+ Status status = ConvertLogToTable(logs_[i]);
+ if (!status.ok()) {
+ ROCKS_LOG_WARN(db_options_.info_log,
+ "Log #%" PRIu64 ": ignoring conversion error: %s",
+ logs_[i], status.ToString().c_str());
+ }
+ ArchiveFile(logname);
+ }
+ }
+
+ Status ConvertLogToTable(uint64_t log) {
+ struct LogReporter : public log::Reader::Reporter {
+ Env* env;
+ std::shared_ptr<Logger> info_log;
+ uint64_t lognum;
+ virtual void Corruption(size_t bytes, const Status& s) override {
+ // We print error messages for corruption, but continue repairing.
+ ROCKS_LOG_ERROR(info_log, "Log #%" PRIu64 ": dropping %d bytes; %s",
+ lognum, static_cast<int>(bytes), s.ToString().c_str());
+ }
+ };
+
+ // Open the log file
+ std::string logname = LogFileName(db_options_.wal_dir, log);
+ unique_ptr<SequentialFile> lfile;
+ Status status = env_->NewSequentialFile(
+ logname, &lfile, env_->OptimizeForLogRead(env_options_));
+ if (!status.ok()) {
+ return status;
+ }
+ unique_ptr<SequentialFileReader> lfile_reader(
+ new SequentialFileReader(std::move(lfile)));
+
+ // Create the log reader.
+ LogReporter reporter;
+ reporter.env = env_;
+ reporter.info_log = db_options_.info_log;
+ reporter.lognum = log;
+ // We intentionally make log::Reader do checksumming so that
+ // corruptions cause entire commits to be skipped instead of
+ // propagating bad information (like overly large sequence
+ // numbers).
+ log::Reader reader(db_options_.info_log, std::move(lfile_reader), &reporter,
+ true /*enable checksum*/, 0 /*initial_offset*/, log);
+
+ // Initialize per-column family memtables
+ for (auto* cfd : *vset_.GetColumnFamilySet()) {
+ cfd->CreateNewMemtable(*cfd->GetLatestMutableCFOptions(),
+ kMaxSequenceNumber);
+ }
+ auto cf_mems = new ColumnFamilyMemTablesImpl(vset_.GetColumnFamilySet());
+
+ // Read all the records and add to a memtable
+ std::string scratch;
+ Slice record;
+ WriteBatch batch;
+ int counter = 0;
+ while (reader.ReadRecord(&record, &scratch)) {
+ if (record.size() < WriteBatchInternal::kHeader) {
+ reporter.Corruption(
+ record.size(), Status::Corruption("log record too small"));
+ continue;
+ }
+ WriteBatchInternal::SetContents(&batch, record);
+ status = WriteBatchInternal::InsertInto(&batch, cf_mems, nullptr);
+ if (status.ok()) {
+ counter += WriteBatchInternal::Count(&batch);
+ } else {
+ ROCKS_LOG_WARN(db_options_.info_log, "Log #%" PRIu64 ": ignoring %s",
+ log, status.ToString().c_str());
+ status = Status::OK(); // Keep going with rest of file
+ }
+ }
+
+ // Dump a table for each column family with entries in this log file.
+ for (auto* cfd : *vset_.GetColumnFamilySet()) {
+ // Do not record a version edit for this conversion to a Table
+ // since ExtractMetaData() will also generate edits.
+ MemTable* mem = cfd->mem();
+ if (mem->IsEmpty()) {
+ continue;
+ }
+
+ FileMetaData meta;
+ meta.fd = FileDescriptor(next_file_number_++, 0, 0);
+ ReadOptions ro;
+ ro.total_order_seek = true;
+ Arena arena;
+ ScopedArenaIterator iter(mem->NewIterator(ro, &arena));
+ EnvOptions optimized_env_options =
+ env_->OptimizeForCompactionTableWrite(env_options_, immutable_db_options_);
+
+ int64_t _current_time = 0;
+ status = env_->GetCurrentTime(&_current_time); // ignore error
+ const uint64_t current_time = static_cast<uint64_t>(_current_time);
+
+ status = BuildTable(
+ dbname_, env_, *cfd->ioptions(), *cfd->GetLatestMutableCFOptions(),
+ optimized_env_options, table_cache_, iter.get(),
+ std::unique_ptr<InternalIterator>(mem->NewRangeTombstoneIterator(ro)),
+ &meta, cfd->internal_comparator(),
+ cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
+ {}, kMaxSequenceNumber, kNoCompression, CompressionOptions(), false,
+ nullptr /* internal_stats */, TableFileCreationReason::kRecovery,
+ nullptr /* event_logger */, 0 /* job_id */, Env::IO_HIGH,
+ nullptr /* table_properties */, -1 /* level */, current_time);
+ ROCKS_LOG_INFO(db_options_.info_log,
+ "Log #%" PRIu64 ": %d ops saved to Table #%" PRIu64 " %s",
+ log, counter, meta.fd.GetNumber(),
+ status.ToString().c_str());
+ if (status.ok()) {
+ if (meta.fd.GetFileSize() > 0) {
+ table_fds_.push_back(meta.fd);
+ }
+ } else {
+ break;
+ }
+ }
+ delete cf_mems;
+ return status;
+ }
+
+ void ExtractMetaData() {
+ for (size_t i = 0; i < table_fds_.size(); i++) {
+ TableInfo t;
+ t.meta.fd = table_fds_[i];
+ Status status = ScanTable(&t);
+ if (!status.ok()) {
+ std::string fname = TableFileName(
+ db_options_.db_paths, t.meta.fd.GetNumber(), t.meta.fd.GetPathId());
+ char file_num_buf[kFormatFileNumberBufSize];
+ FormatFileNumber(t.meta.fd.GetNumber(), t.meta.fd.GetPathId(),
+ file_num_buf, sizeof(file_num_buf));
+ ROCKS_LOG_WARN(db_options_.info_log, "Table #%s: ignoring %s",
+ file_num_buf, status.ToString().c_str());
+ ArchiveFile(fname);
+ } else {
+ tables_.push_back(t);
+ }
+ }
+ }
+
+ Status ScanTable(TableInfo* t) {
+ std::string fname = TableFileName(
+ db_options_.db_paths, t->meta.fd.GetNumber(), t->meta.fd.GetPathId());
+ int counter = 0;
+ uint64_t file_size;
+ Status status = env_->GetFileSize(fname, &file_size);
+ t->meta.fd = FileDescriptor(t->meta.fd.GetNumber(), t->meta.fd.GetPathId(),
+ file_size);
+ std::shared_ptr<const TableProperties> props;
+ if (status.ok()) {
+ status = table_cache_->GetTableProperties(env_options_, icmp_, t->meta.fd,
+ &props);
+ }
+ if (status.ok()) {
+ t->column_family_id = static_cast<uint32_t>(props->column_family_id);
+ if (t->column_family_id ==
+ TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) {
+ ROCKS_LOG_WARN(
+ db_options_.info_log,
+ "Table #%" PRIu64
+ ": column family unknown (probably due to legacy format); "
+ "adding to default column family id 0.",
+ t->meta.fd.GetNumber());
+ t->column_family_id = 0;
+ }
+
+ if (vset_.GetColumnFamilySet()->GetColumnFamily(t->column_family_id) ==
+ nullptr) {
+ status =
+ AddColumnFamily(props->column_family_name, t->column_family_id);
+ }
+ }
+ ColumnFamilyData* cfd = nullptr;
+ if (status.ok()) {
+ cfd = vset_.GetColumnFamilySet()->GetColumnFamily(t->column_family_id);
+ if (cfd->GetName() != props->column_family_name) {
+ ROCKS_LOG_ERROR(
+ db_options_.info_log,
+ "Table #%" PRIu64
+ ": inconsistent column family name '%s'; expected '%s' for column "
+ "family id %" PRIu32 ".",
+ t->meta.fd.GetNumber(), props->column_family_name.c_str(),
+ cfd->GetName().c_str(), t->column_family_id);
+ status = Status::Corruption(dbname_, "inconsistent column family name");
+ }
+ }
+ if (status.ok()) {
+ InternalIterator* iter = table_cache_->NewIterator(
+ ReadOptions(), env_options_, cfd->internal_comparator(), t->meta.fd,
+ nullptr /* range_del_agg */);
+ bool empty = true;
+ ParsedInternalKey parsed;
+ t->min_sequence = 0;
+ t->max_sequence = 0;
+ for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+ Slice key = iter->key();
+ if (!ParseInternalKey(key, &parsed)) {
+ ROCKS_LOG_ERROR(db_options_.info_log,
+ "Table #%" PRIu64 ": unparsable key %s",
+ t->meta.fd.GetNumber(), EscapeString(key).c_str());
+ continue;
+ }
+
+ counter++;
+ if (empty) {
+ empty = false;
+ t->meta.smallest.DecodeFrom(key);
+ t->min_sequence = parsed.sequence;
+ }
+ t->meta.largest.DecodeFrom(key);
+ if (parsed.sequence < t->min_sequence) {
+ t->min_sequence = parsed.sequence;
+ }
+ if (parsed.sequence > t->max_sequence) {
+ t->max_sequence = parsed.sequence;
+ }
+ }
+ if (!iter->status().ok()) {
+ status = iter->status();
+ }
+ delete iter;
+
+ ROCKS_LOG_INFO(db_options_.info_log, "Table #%" PRIu64 ": %d entries %s",
+ t->meta.fd.GetNumber(), counter,
+ status.ToString().c_str());
+ }
+ return status;
+ }
+
+ Status AddTables() {
+ std::unordered_map<uint32_t, std::vector<const TableInfo*>> cf_id_to_tables;
+ SequenceNumber max_sequence = 0;
+ for (size_t i = 0; i < tables_.size(); i++) {
+ cf_id_to_tables[tables_[i].column_family_id].push_back(&tables_[i]);
+ if (max_sequence < tables_[i].max_sequence) {
+ max_sequence = tables_[i].max_sequence;
+ }
+ }
+ vset_.SetLastToBeWrittenSequence(max_sequence);
+ vset_.SetLastSequence(max_sequence);
+
+ for (const auto& cf_id_and_tables : cf_id_to_tables) {
+ auto* cfd =
+ vset_.GetColumnFamilySet()->GetColumnFamily(cf_id_and_tables.first);
+ VersionEdit edit;
+ edit.SetComparatorName(cfd->user_comparator()->Name());
+ edit.SetLogNumber(0);
+ edit.SetNextFile(next_file_number_);
+ edit.SetColumnFamily(cfd->GetID());
+
+ // TODO(opt): separate out into multiple levels
+ for (const auto* table : cf_id_and_tables.second) {
+ edit.AddFile(0, table->meta.fd.GetNumber(), table->meta.fd.GetPathId(),
+ table->meta.fd.GetFileSize(), table->meta.smallest,
+ table->meta.largest, table->min_sequence,
+ table->max_sequence, table->meta.marked_for_compaction);
+ }
+ mutex_.Lock();
+ Status status = vset_.LogAndApply(
+ cfd, *cfd->GetLatestMutableCFOptions(), &edit, &mutex_,
+ nullptr /* db_directory */, false /* new_descriptor_log */);
+ mutex_.Unlock();
+ if (!status.ok()) {
+ return status;
+ }
+ }
+ return Status::OK();
+ }
+
+ void ArchiveFile(const std::string& fname) {
+ // Move into another directory. E.g., for
+ // dir/foo
+ // rename to
+ // dir/lost/foo
+ const char* slash = strrchr(fname.c_str(), '/');
+ std::string new_dir;
+ if (slash != nullptr) {
+ new_dir.assign(fname.data(), slash - fname.data());
+ }
+ new_dir.append("/lost");
+ env_->CreateDir(new_dir); // Ignore error
+ std::string new_file = new_dir;
+ new_file.append("/");
+ new_file.append((slash == nullptr) ? fname.c_str() : slash + 1);
+ Status s = env_->RenameFile(fname, new_file);
+ ROCKS_LOG_INFO(db_options_.info_log, "Archiving %s: %s\n", fname.c_str(),
+ s.ToString().c_str());
+ }
+};
+
+Status GetDefaultCFOptions(
+ const std::vector<ColumnFamilyDescriptor>& column_families,
+ ColumnFamilyOptions* res) {
+ assert(res != nullptr);
+ auto iter = std::find_if(column_families.begin(), column_families.end(),
+ [](const ColumnFamilyDescriptor& cfd) {
+ return cfd.name == kDefaultColumnFamilyName;
+ });
+ if (iter == column_families.end()) {
+ return Status::InvalidArgument(
+ "column_families", "Must contain entry for default column family");
+ }
+ *res = iter->options;
+ return Status::OK();
+}
+} // anonymous namespace
+
+Status RepairDB(const std::string& dbname, const DBOptions& db_options,
+ const std::vector<ColumnFamilyDescriptor>& column_families) {
+ ColumnFamilyOptions default_cf_opts;
+ Status status = GetDefaultCFOptions(column_families, &default_cf_opts);
+ if (status.ok()) {
+ Repairer repairer(dbname, db_options, column_families, default_cf_opts,
+ ColumnFamilyOptions() /* unknown_cf_opts */,
+ false /* create_unknown_cfs */);
+ status = repairer.Run();
+ }
+ return status;
+}
+
+Status RepairDB(const std::string& dbname, const DBOptions& db_options,
+ const std::vector<ColumnFamilyDescriptor>& column_families,
+ const ColumnFamilyOptions& unknown_cf_opts) {
+ ColumnFamilyOptions default_cf_opts;
+ Status status = GetDefaultCFOptions(column_families, &default_cf_opts);
+ if (status.ok()) {
+ Repairer repairer(dbname, db_options, column_families, default_cf_opts,
+ unknown_cf_opts, true /* create_unknown_cfs */);
+ status = repairer.Run();
+ }
+ return status;
+}
+
+Status RepairDB(const std::string& dbname, const Options& options) {
+ DBOptions db_options(options);
+ ColumnFamilyOptions cf_options(options);
+ Repairer repairer(dbname, db_options, {}, cf_options /* default_cf_opts */,
+ cf_options /* unknown_cf_opts */,
+ true /* create_unknown_cfs */);
+ return repairer.Run();
+}
+
+} // namespace rocksdb
+
+#endif // ROCKSDB_LITE
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/snapshot_impl.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/snapshot_impl.cc b/thirdparty/rocksdb/db/snapshot_impl.cc
new file mode 100644
index 0000000..032ef39
--- /dev/null
+++ b/thirdparty/rocksdb/db/snapshot_impl.cc
@@ -0,0 +1,26 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "rocksdb/snapshot.h"
+
+#include "rocksdb/db.h"
+
+namespace rocksdb {
+
+ManagedSnapshot::ManagedSnapshot(DB* db) : db_(db),
+ snapshot_(db->GetSnapshot()) {}
+
+ManagedSnapshot::ManagedSnapshot(DB* db, const Snapshot* _snapshot)
+ : db_(db), snapshot_(_snapshot) {}
+
+ManagedSnapshot::~ManagedSnapshot() {
+ if (snapshot_) {
+ db_->ReleaseSnapshot(snapshot_);
+ }
+}
+
+const Snapshot* ManagedSnapshot::snapshot() { return snapshot_;}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/snapshot_impl.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/snapshot_impl.h b/thirdparty/rocksdb/db/snapshot_impl.h
new file mode 100644
index 0000000..b94602f
--- /dev/null
+++ b/thirdparty/rocksdb/db/snapshot_impl.h
@@ -0,0 +1,130 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <vector>
+
+#include "rocksdb/db.h"
+
+namespace rocksdb {
+
+class SnapshotList;
+
+// Snapshots are kept in a doubly-linked list in the DB.
+// Each SnapshotImpl corresponds to a particular sequence number.
+class SnapshotImpl : public Snapshot {
+ public:
+ SequenceNumber number_; // const after creation
+
+ virtual SequenceNumber GetSequenceNumber() const override { return number_; }
+
+ private:
+ friend class SnapshotList;
+
+ // SnapshotImpl is kept in a doubly-linked circular list
+ SnapshotImpl* prev_;
+ SnapshotImpl* next_;
+
+ SnapshotList* list_; // just for sanity checks
+
+ int64_t unix_time_;
+
+ // Will this snapshot be used by a Transaction to do write-conflict checking?
+ bool is_write_conflict_boundary_;
+};
+
+class SnapshotList {
+ public:
+ SnapshotList() {
+ list_.prev_ = &list_;
+ list_.next_ = &list_;
+ list_.number_ = 0xFFFFFFFFL; // placeholder marker, for debugging
+ count_ = 0;
+ }
+
+ bool empty() const { return list_.next_ == &list_; }
+ SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; }
+ SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; }
+
+ const SnapshotImpl* New(SnapshotImpl* s, SequenceNumber seq,
+ uint64_t unix_time, bool is_write_conflict_boundary) {
+ s->number_ = seq;
+ s->unix_time_ = unix_time;
+ s->is_write_conflict_boundary_ = is_write_conflict_boundary;
+ s->list_ = this;
+ s->next_ = &list_;
+ s->prev_ = list_.prev_;
+ s->prev_->next_ = s;
+ s->next_->prev_ = s;
+ count_++;
+ return s;
+ }
+
+ // Do not responsible to free the object.
+ void Delete(const SnapshotImpl* s) {
+ assert(s->list_ == this);
+ s->prev_->next_ = s->next_;
+ s->next_->prev_ = s->prev_;
+ count_--;
+ }
+
+ // retrieve all snapshot numbers. They are sorted in ascending order.
+ std::vector<SequenceNumber> GetAll(
+ SequenceNumber* oldest_write_conflict_snapshot = nullptr) {
+ std::vector<SequenceNumber> ret;
+
+ if (oldest_write_conflict_snapshot != nullptr) {
+ *oldest_write_conflict_snapshot = kMaxSequenceNumber;
+ }
+
+ if (empty()) {
+ return ret;
+ }
+ SnapshotImpl* s = &list_;
+ while (s->next_ != &list_) {
+ ret.push_back(s->next_->number_);
+
+ if (oldest_write_conflict_snapshot != nullptr &&
+ *oldest_write_conflict_snapshot == kMaxSequenceNumber &&
+ s->next_->is_write_conflict_boundary_) {
+ // If this is the first write-conflict boundary snapshot in the list,
+ // it is the oldest
+ *oldest_write_conflict_snapshot = s->next_->number_;
+ }
+
+ s = s->next_;
+ }
+ return ret;
+ }
+
+ // get the sequence number of the most recent snapshot
+ SequenceNumber GetNewest() {
+ if (empty()) {
+ return 0;
+ }
+ return newest()->number_;
+ }
+
+ int64_t GetOldestSnapshotTime() const {
+ if (empty()) {
+ return 0;
+ } else {
+ return oldest()->unix_time_;
+ }
+ }
+
+ uint64_t count() const { return count_; }
+
+ private:
+ // Dummy head of doubly-linked list of snapshots
+ SnapshotImpl list_;
+ uint64_t count_;
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/table_cache.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/table_cache.cc b/thirdparty/rocksdb/db/table_cache.cc
new file mode 100644
index 0000000..b4d5cc1
--- /dev/null
+++ b/thirdparty/rocksdb/db/table_cache.cc
@@ -0,0 +1,472 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/table_cache.h"
+
+#include "db/dbformat.h"
+#include "db/version_edit.h"
+#include "util/filename.h"
+
+#include "monitoring/perf_context_imp.h"
+#include "rocksdb/statistics.h"
+#include "table/get_context.h"
+#include "table/internal_iterator.h"
+#include "table/iterator_wrapper.h"
+#include "table/table_builder.h"
+#include "table/table_reader.h"
+#include "util/coding.h"
+#include "util/file_reader_writer.h"
+#include "util/stop_watch.h"
+#include "util/sync_point.h"
+
+namespace rocksdb {
+
+namespace {
+
+template <class T>
+static void DeleteEntry(const Slice& key, void* value) {
+ T* typed_value = reinterpret_cast<T*>(value);
+ delete typed_value;
+}
+
+static void UnrefEntry(void* arg1, void* arg2) {
+ Cache* cache = reinterpret_cast<Cache*>(arg1);
+ Cache::Handle* h = reinterpret_cast<Cache::Handle*>(arg2);
+ cache->Release(h);
+}
+
+static void DeleteTableReader(void* arg1, void* arg2) {
+ TableReader* table_reader = reinterpret_cast<TableReader*>(arg1);
+ delete table_reader;
+}
+
+static Slice GetSliceForFileNumber(const uint64_t* file_number) {
+ return Slice(reinterpret_cast<const char*>(file_number),
+ sizeof(*file_number));
+}
+
+#ifndef ROCKSDB_LITE
+
+void AppendVarint64(IterKey* key, uint64_t v) {
+ char buf[10];
+ auto ptr = EncodeVarint64(buf, v);
+ key->TrimAppend(key->Size(), buf, ptr - buf);
+}
+
+#endif // ROCKSDB_LITE
+
+} // namespace
+
+TableCache::TableCache(const ImmutableCFOptions& ioptions,
+ const EnvOptions& env_options, Cache* const cache)
+ : ioptions_(ioptions), env_options_(env_options), cache_(cache) {
+ if (ioptions_.row_cache) {
+ // If the same cache is shared by multiple instances, we need to
+ // disambiguate its entries.
+ PutVarint64(&row_cache_id_, ioptions_.row_cache->NewId());
+ }
+}
+
+TableCache::~TableCache() {
+}
+
+TableReader* TableCache::GetTableReaderFromHandle(Cache::Handle* handle) {
+ return reinterpret_cast<TableReader*>(cache_->Value(handle));
+}
+
+void TableCache::ReleaseHandle(Cache::Handle* handle) {
+ cache_->Release(handle);
+}
+
+Status TableCache::GetTableReader(
+ const EnvOptions& env_options,
+ const InternalKeyComparator& internal_comparator, const FileDescriptor& fd,
+ bool sequential_mode, size_t readahead, bool record_read_stats,
+ HistogramImpl* file_read_hist, unique_ptr<TableReader>* table_reader,
+ bool skip_filters, int level, bool prefetch_index_and_filter_in_cache,
+ bool for_compaction) {
+ std::string fname =
+ TableFileName(ioptions_.db_paths, fd.GetNumber(), fd.GetPathId());
+ unique_ptr<RandomAccessFile> file;
+ Status s = ioptions_.env->NewRandomAccessFile(fname, &file, env_options);
+
+ RecordTick(ioptions_.statistics, NO_FILE_OPENS);
+ if (s.ok()) {
+ if (readahead > 0) {
+ file = NewReadaheadRandomAccessFile(std::move(file), readahead);
+ }
+ if (!sequential_mode && ioptions_.advise_random_on_open) {
+ file->Hint(RandomAccessFile::RANDOM);
+ }
+ StopWatch sw(ioptions_.env, ioptions_.statistics, TABLE_OPEN_IO_MICROS);
+ std::unique_ptr<RandomAccessFileReader> file_reader(
+ new RandomAccessFileReader(
+ std::move(file), fname, ioptions_.env,
+ record_read_stats ? ioptions_.statistics : nullptr, SST_READ_MICROS,
+ file_read_hist, ioptions_.rate_limiter, for_compaction));
+ s = ioptions_.table_factory->NewTableReader(
+ TableReaderOptions(ioptions_, env_options, internal_comparator,
+ skip_filters, level),
+ std::move(file_reader), fd.GetFileSize(), table_reader,
+ prefetch_index_and_filter_in_cache);
+ TEST_SYNC_POINT("TableCache::GetTableReader:0");
+ }
+ return s;
+}
+
+void TableCache::EraseHandle(const FileDescriptor& fd, Cache::Handle* handle) {
+ ReleaseHandle(handle);
+ uint64_t number = fd.GetNumber();
+ Slice key = GetSliceForFileNumber(&number);
+ cache_->Erase(key);
+}
+
+Status TableCache::FindTable(const EnvOptions& env_options,
+ const InternalKeyComparator& internal_comparator,
+ const FileDescriptor& fd, Cache::Handle** handle,
+ const bool no_io, bool record_read_stats,
+ HistogramImpl* file_read_hist, bool skip_filters,
+ int level,
+ bool prefetch_index_and_filter_in_cache) {
+ PERF_TIMER_GUARD(find_table_nanos);
+ Status s;
+ uint64_t number = fd.GetNumber();
+ Slice key = GetSliceForFileNumber(&number);
+ *handle = cache_->Lookup(key);
+ TEST_SYNC_POINT_CALLBACK("TableCache::FindTable:0",
+ const_cast<bool*>(&no_io));
+
+ if (*handle == nullptr) {
+ if (no_io) { // Don't do IO and return a not-found status
+ return Status::Incomplete("Table not found in table_cache, no_io is set");
+ }
+ unique_ptr<TableReader> table_reader;
+ s = GetTableReader(env_options, internal_comparator, fd,
+ false /* sequential mode */, 0 /* readahead */,
+ record_read_stats, file_read_hist, &table_reader,
+ skip_filters, level, prefetch_index_and_filter_in_cache);
+ if (!s.ok()) {
+ assert(table_reader == nullptr);
+ RecordTick(ioptions_.statistics, NO_FILE_ERRORS);
+ // We do not cache error results so that if the error is transient,
+ // or somebody repairs the file, we recover automatically.
+ } else {
+ s = cache_->Insert(key, table_reader.get(), 1, &DeleteEntry<TableReader>,
+ handle);
+ if (s.ok()) {
+ // Release ownership of table reader.
+ table_reader.release();
+ }
+ }
+ }
+ return s;
+}
+
+InternalIterator* TableCache::NewIterator(
+ const ReadOptions& options, const EnvOptions& env_options,
+ const InternalKeyComparator& icomparator, const FileDescriptor& fd,
+ RangeDelAggregator* range_del_agg, TableReader** table_reader_ptr,
+ HistogramImpl* file_read_hist, bool for_compaction, Arena* arena,
+ bool skip_filters, int level) {
+ PERF_TIMER_GUARD(new_table_iterator_nanos);
+
+ Status s;
+ bool create_new_table_reader = false;
+ TableReader* table_reader = nullptr;
+ Cache::Handle* handle = nullptr;
+ if (s.ok()) {
+ if (table_reader_ptr != nullptr) {
+ *table_reader_ptr = nullptr;
+ }
+ size_t readahead = 0;
+ if (for_compaction) {
+#ifndef NDEBUG
+ bool use_direct_reads_for_compaction = env_options.use_direct_reads;
+ TEST_SYNC_POINT_CALLBACK("TableCache::NewIterator:for_compaction",
+ &use_direct_reads_for_compaction);
+#endif // !NDEBUG
+ if (ioptions_.new_table_reader_for_compaction_inputs) {
+ readahead = ioptions_.compaction_readahead_size;
+ create_new_table_reader = true;
+ }
+ } else {
+ readahead = options.readahead_size;
+ create_new_table_reader = readahead > 0;
+ }
+
+ if (create_new_table_reader) {
+ unique_ptr<TableReader> table_reader_unique_ptr;
+ s = GetTableReader(
+ env_options, icomparator, fd, true /* sequential_mode */, readahead,
+ !for_compaction /* record stats */, nullptr, &table_reader_unique_ptr,
+ false /* skip_filters */, level,
+ true /* prefetch_index_and_filter_in_cache */, for_compaction);
+ if (s.ok()) {
+ table_reader = table_reader_unique_ptr.release();
+ }
+ } else {
+ table_reader = fd.table_reader;
+ if (table_reader == nullptr) {
+ s = FindTable(env_options, icomparator, fd, &handle,
+ options.read_tier == kBlockCacheTier /* no_io */,
+ !for_compaction /* record read_stats */, file_read_hist,
+ skip_filters, level);
+ if (s.ok()) {
+ table_reader = GetTableReaderFromHandle(handle);
+ }
+ }
+ }
+ }
+ InternalIterator* result = nullptr;
+ if (s.ok()) {
+ result = table_reader->NewIterator(options, arena, skip_filters);
+ if (create_new_table_reader) {
+ assert(handle == nullptr);
+ result->RegisterCleanup(&DeleteTableReader, table_reader, nullptr);
+ } else if (handle != nullptr) {
+ result->RegisterCleanup(&UnrefEntry, cache_, handle);
+ handle = nullptr; // prevent from releasing below
+ }
+
+ if (for_compaction) {
+ table_reader->SetupForCompaction();
+ }
+ if (table_reader_ptr != nullptr) {
+ *table_reader_ptr = table_reader;
+ }
+ }
+ if (s.ok() && range_del_agg != nullptr && !options.ignore_range_deletions) {
+ std::unique_ptr<InternalIterator> range_del_iter(
+ table_reader->NewRangeTombstoneIterator(options));
+ if (range_del_iter != nullptr) {
+ s = range_del_iter->status();
+ }
+ if (s.ok()) {
+ s = range_del_agg->AddTombstones(std::move(range_del_iter));
+ }
+ }
+
+ if (handle != nullptr) {
+ ReleaseHandle(handle);
+ }
+ if (!s.ok()) {
+ assert(result == nullptr);
+ result = NewErrorInternalIterator(s, arena);
+ }
+ return result;
+}
+
+InternalIterator* TableCache::NewRangeTombstoneIterator(
+ const ReadOptions& options, const EnvOptions& env_options,
+ const InternalKeyComparator& icomparator, const FileDescriptor& fd,
+ HistogramImpl* file_read_hist, bool skip_filters, int level) {
+ Status s;
+ TableReader* table_reader = nullptr;
+ Cache::Handle* handle = nullptr;
+ table_reader = fd.table_reader;
+ if (table_reader == nullptr) {
+ s = FindTable(env_options, icomparator, fd, &handle,
+ options.read_tier == kBlockCacheTier /* no_io */,
+ true /* record read_stats */, file_read_hist, skip_filters,
+ level);
+ if (s.ok()) {
+ table_reader = GetTableReaderFromHandle(handle);
+ }
+ }
+ InternalIterator* result = nullptr;
+ if (s.ok()) {
+ result = table_reader->NewRangeTombstoneIterator(options);
+ if (result != nullptr) {
+ if (handle != nullptr) {
+ result->RegisterCleanup(&UnrefEntry, cache_, handle);
+ }
+ }
+ }
+ if (result == nullptr && handle != nullptr) {
+ // the range deletion block didn't exist, or there was a failure between
+ // getting handle and getting iterator.
+ ReleaseHandle(handle);
+ }
+ if (!s.ok()) {
+ assert(result == nullptr);
+ result = NewErrorInternalIterator(s);
+ }
+ return result;
+}
+
+Status TableCache::Get(const ReadOptions& options,
+ const InternalKeyComparator& internal_comparator,
+ const FileDescriptor& fd, const Slice& k,
+ GetContext* get_context, HistogramImpl* file_read_hist,
+ bool skip_filters, int level) {
+ std::string* row_cache_entry = nullptr;
+ bool done = false;
+#ifndef ROCKSDB_LITE
+ IterKey row_cache_key;
+ std::string row_cache_entry_buffer;
+
+ // Check row cache if enabled. Since row cache does not currently store
+ // sequence numbers, we cannot use it if we need to fetch the sequence.
+ if (ioptions_.row_cache && !get_context->NeedToReadSequence()) {
+ uint64_t fd_number = fd.GetNumber();
+ auto user_key = ExtractUserKey(k);
+ // We use the user key as cache key instead of the internal key,
+ // otherwise the whole cache would be invalidated every time the
+ // sequence key increases. However, to support caching snapshot
+ // reads, we append the sequence number (incremented by 1 to
+ // distinguish from 0) only in this case.
+ uint64_t seq_no =
+ options.snapshot == nullptr ? 0 : 1 + GetInternalKeySeqno(k);
+
+ // Compute row cache key.
+ row_cache_key.TrimAppend(row_cache_key.Size(), row_cache_id_.data(),
+ row_cache_id_.size());
+ AppendVarint64(&row_cache_key, fd_number);
+ AppendVarint64(&row_cache_key, seq_no);
+ row_cache_key.TrimAppend(row_cache_key.Size(), user_key.data(),
+ user_key.size());
+
+ if (auto row_handle =
+ ioptions_.row_cache->Lookup(row_cache_key.GetUserKey())) {
+ // Cleanable routine to release the cache entry
+ Cleanable value_pinner;
+ auto release_cache_entry_func = [](void* cache_to_clean,
+ void* cache_handle) {
+ ((Cache*)cache_to_clean)->Release((Cache::Handle*)cache_handle);
+ };
+ auto found_row_cache_entry = static_cast<const std::string*>(
+ ioptions_.row_cache->Value(row_handle));
+ // If it comes here value is located on the cache.
+ // found_row_cache_entry points to the value on cache,
+ // and value_pinner has cleanup procedure for the cached entry.
+ // After replayGetContextLog() returns, get_context.pinnable_slice_
+ // will point to cache entry buffer (or a copy based on that) and
+ // cleanup routine under value_pinner will be delegated to
+ // get_context.pinnable_slice_. Cache entry is released when
+ // get_context.pinnable_slice_ is reset.
+ value_pinner.RegisterCleanup(release_cache_entry_func,
+ ioptions_.row_cache.get(), row_handle);
+ replayGetContextLog(*found_row_cache_entry, user_key, get_context,
+ &value_pinner);
+ RecordTick(ioptions_.statistics, ROW_CACHE_HIT);
+ done = true;
+ } else {
+ // Not found, setting up the replay log.
+ RecordTick(ioptions_.statistics, ROW_CACHE_MISS);
+ row_cache_entry = &row_cache_entry_buffer;
+ }
+ }
+#endif // ROCKSDB_LITE
+ Status s;
+ TableReader* t = fd.table_reader;
+ Cache::Handle* handle = nullptr;
+ if (!done && s.ok()) {
+ if (t == nullptr) {
+ s = FindTable(env_options_, internal_comparator, fd, &handle,
+ options.read_tier == kBlockCacheTier /* no_io */,
+ true /* record_read_stats */, file_read_hist, skip_filters,
+ level);
+ if (s.ok()) {
+ t = GetTableReaderFromHandle(handle);
+ }
+ }
+ if (s.ok() && get_context->range_del_agg() != nullptr &&
+ !options.ignore_range_deletions) {
+ std::unique_ptr<InternalIterator> range_del_iter(
+ t->NewRangeTombstoneIterator(options));
+ if (range_del_iter != nullptr) {
+ s = range_del_iter->status();
+ }
+ if (s.ok()) {
+ s = get_context->range_del_agg()->AddTombstones(
+ std::move(range_del_iter));
+ }
+ }
+ if (s.ok()) {
+ get_context->SetReplayLog(row_cache_entry); // nullptr if no cache.
+ s = t->Get(options, k, get_context, skip_filters);
+ get_context->SetReplayLog(nullptr);
+ } else if (options.read_tier == kBlockCacheTier && s.IsIncomplete()) {
+ // Couldn't find Table in cache but treat as kFound if no_io set
+ get_context->MarkKeyMayExist();
+ s = Status::OK();
+ done = true;
+ }
+ }
+
+#ifndef ROCKSDB_LITE
+ // Put the replay log in row cache only if something was found.
+ if (!done && s.ok() && row_cache_entry && !row_cache_entry->empty()) {
+ size_t charge =
+ row_cache_key.Size() + row_cache_entry->size() + sizeof(std::string);
+ void* row_ptr = new std::string(std::move(*row_cache_entry));
+ ioptions_.row_cache->Insert(row_cache_key.GetUserKey(), row_ptr, charge,
+ &DeleteEntry<std::string>);
+ }
+#endif // ROCKSDB_LITE
+
+ if (handle != nullptr) {
+ ReleaseHandle(handle);
+ }
+ return s;
+}
+
+Status TableCache::GetTableProperties(
+ const EnvOptions& env_options,
+ const InternalKeyComparator& internal_comparator, const FileDescriptor& fd,
+ std::shared_ptr<const TableProperties>* properties, bool no_io) {
+ Status s;
+ auto table_reader = fd.table_reader;
+ // table already been pre-loaded?
+ if (table_reader) {
+ *properties = table_reader->GetTableProperties();
+
+ return s;
+ }
+
+ Cache::Handle* table_handle = nullptr;
+ s = FindTable(env_options, internal_comparator, fd, &table_handle, no_io);
+ if (!s.ok()) {
+ return s;
+ }
+ assert(table_handle);
+ auto table = GetTableReaderFromHandle(table_handle);
+ *properties = table->GetTableProperties();
+ ReleaseHandle(table_handle);
+ return s;
+}
+
+size_t TableCache::GetMemoryUsageByTableReader(
+ const EnvOptions& env_options,
+ const InternalKeyComparator& internal_comparator,
+ const FileDescriptor& fd) {
+ Status s;
+ auto table_reader = fd.table_reader;
+ // table already been pre-loaded?
+ if (table_reader) {
+ return table_reader->ApproximateMemoryUsage();
+ }
+
+ Cache::Handle* table_handle = nullptr;
+ s = FindTable(env_options, internal_comparator, fd, &table_handle, true);
+ if (!s.ok()) {
+ return 0;
+ }
+ assert(table_handle);
+ auto table = GetTableReaderFromHandle(table_handle);
+ auto ret = table->ApproximateMemoryUsage();
+ ReleaseHandle(table_handle);
+ return ret;
+}
+
+void TableCache::Evict(Cache* cache, uint64_t file_number) {
+ cache->Erase(GetSliceForFileNumber(&file_number));
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/table_cache.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/table_cache.h b/thirdparty/rocksdb/db/table_cache.h
new file mode 100644
index 0000000..8b65baf
--- /dev/null
+++ b/thirdparty/rocksdb/db/table_cache.h
@@ -0,0 +1,146 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// Thread-safe (provides internal synchronization)
+
+#pragma once
+#include <string>
+#include <vector>
+#include <stdint.h>
+
+#include "db/dbformat.h"
+#include "db/range_del_aggregator.h"
+#include "options/cf_options.h"
+#include "port/port.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+#include "rocksdb/table.h"
+#include "table/table_reader.h"
+
+namespace rocksdb {
+
+class Env;
+class Arena;
+struct FileDescriptor;
+class GetContext;
+class HistogramImpl;
+class InternalIterator;
+
+class TableCache {
+ public:
+ TableCache(const ImmutableCFOptions& ioptions,
+ const EnvOptions& storage_options, Cache* cache);
+ ~TableCache();
+
+ // Return an iterator for the specified file number (the corresponding
+ // file length must be exactly "file_size" bytes). If "tableptr" is
+ // non-nullptr, also sets "*tableptr" to point to the Table object
+ // underlying the returned iterator, or nullptr if no Table object underlies
+ // the returned iterator. The returned "*tableptr" object is owned by
+ // the cache and should not be deleted, and is valid for as long as the
+ // returned iterator is live.
+ // @param range_del_agg If non-nullptr, adds range deletions to the
+ // aggregator. If an error occurs, returns it in a NewErrorInternalIterator
+ // @param skip_filters Disables loading/accessing the filter block
+ // @param level The level this table is at, -1 for "not set / don't know"
+ InternalIterator* NewIterator(
+ const ReadOptions& options, const EnvOptions& toptions,
+ const InternalKeyComparator& internal_comparator,
+ const FileDescriptor& file_fd, RangeDelAggregator* range_del_agg,
+ TableReader** table_reader_ptr = nullptr,
+ HistogramImpl* file_read_hist = nullptr, bool for_compaction = false,
+ Arena* arena = nullptr, bool skip_filters = false, int level = -1);
+
+ InternalIterator* NewRangeTombstoneIterator(
+ const ReadOptions& options, const EnvOptions& toptions,
+ const InternalKeyComparator& internal_comparator,
+ const FileDescriptor& file_fd, HistogramImpl* file_read_hist,
+ bool skip_filters, int level);
+
+ // If a seek to internal key "k" in specified file finds an entry,
+ // call (*handle_result)(arg, found_key, found_value) repeatedly until
+ // it returns false.
+ // @param get_context State for get operation. If its range_del_agg() returns
+ // non-nullptr, adds range deletions to the aggregator. If an error occurs,
+ // returns non-ok status.
+ // @param skip_filters Disables loading/accessing the filter block
+ // @param level The level this table is at, -1 for "not set / don't know"
+ Status Get(const ReadOptions& options,
+ const InternalKeyComparator& internal_comparator,
+ const FileDescriptor& file_fd, const Slice& k,
+ GetContext* get_context, HistogramImpl* file_read_hist = nullptr,
+ bool skip_filters = false, int level = -1);
+
+ // Evict any entry for the specified file number
+ static void Evict(Cache* cache, uint64_t file_number);
+
+ // Clean table handle and erase it from the table cache
+ // Used in DB close, or the file is not live anymore.
+ void EraseHandle(const FileDescriptor& fd, Cache::Handle* handle);
+
+ // Find table reader
+ // @param skip_filters Disables loading/accessing the filter block
+ // @param level == -1 means not specified
+ Status FindTable(const EnvOptions& toptions,
+ const InternalKeyComparator& internal_comparator,
+ const FileDescriptor& file_fd, Cache::Handle**,
+ const bool no_io = false, bool record_read_stats = true,
+ HistogramImpl* file_read_hist = nullptr,
+ bool skip_filters = false, int level = -1,
+ bool prefetch_index_and_filter_in_cache = true);
+
+ // Get TableReader from a cache handle.
+ TableReader* GetTableReaderFromHandle(Cache::Handle* handle);
+
+ // Get the table properties of a given table.
+ // @no_io: indicates if we should load table to the cache if it is not present
+ // in table cache yet.
+ // @returns: `properties` will be reset on success. Please note that we will
+ // return Status::Incomplete() if table is not present in cache and
+ // we set `no_io` to be true.
+ Status GetTableProperties(const EnvOptions& toptions,
+ const InternalKeyComparator& internal_comparator,
+ const FileDescriptor& file_meta,
+ std::shared_ptr<const TableProperties>* properties,
+ bool no_io = false);
+
+ // Return total memory usage of the table reader of the file.
+ // 0 if table reader of the file is not loaded.
+ size_t GetMemoryUsageByTableReader(
+ const EnvOptions& toptions,
+ const InternalKeyComparator& internal_comparator,
+ const FileDescriptor& fd);
+
+ // Release the handle from a cache
+ void ReleaseHandle(Cache::Handle* handle);
+
+ // Capacity of the backing Cache that indicates inifinite TableCache capacity.
+ // For example when max_open_files is -1 we set the backing Cache to this.
+ static const int kInfiniteCapacity = 0x400000;
+
+ private:
+ // Build a table reader
+ Status GetTableReader(const EnvOptions& env_options,
+ const InternalKeyComparator& internal_comparator,
+ const FileDescriptor& fd, bool sequential_mode,
+ size_t readahead, bool record_read_stats,
+ HistogramImpl* file_read_hist,
+ unique_ptr<TableReader>* table_reader,
+ bool skip_filters = false, int level = -1,
+ bool prefetch_index_and_filter_in_cache = true,
+ bool for_compaction = false);
+
+ const ImmutableCFOptions& ioptions_;
+ const EnvOptions& env_options_;
+ Cache* const cache_;
+ std::string row_cache_id_;
+};
+
+} // namespace rocksdb
[07/51] [partial] nifi-minifi-cpp git commit: MINIFI-372: Replace
leveldb with RocksDB
Posted by je...@apache.org.
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/write_batch.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/write_batch.cc b/thirdparty/rocksdb/db/write_batch.cc
new file mode 100644
index 0000000..91be9a0
--- /dev/null
+++ b/thirdparty/rocksdb/db/write_batch.cc
@@ -0,0 +1,1396 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// WriteBatch::rep_ :=
+// sequence: fixed64
+// count: fixed32
+// data: record[count]
+// record :=
+// kTypeValue varstring varstring
+// kTypeDeletion varstring
+// kTypeSingleDeletion varstring
+// kTypeMerge varstring varstring
+// kTypeColumnFamilyValue varint32 varstring varstring
+// kTypeColumnFamilyDeletion varint32 varstring varstring
+// kTypeColumnFamilySingleDeletion varint32 varstring varstring
+// kTypeColumnFamilyMerge varint32 varstring varstring
+// kTypeBeginPrepareXID varstring
+// kTypeEndPrepareXID
+// kTypeCommitXID varstring
+// kTypeRollbackXID varstring
+// kTypeNoop
+// varstring :=
+// len: varint32
+// data: uint8[len]
+
+#include "rocksdb/write_batch.h"
+
+#include <map>
+#include <stack>
+#include <stdexcept>
+#include <type_traits>
+#include <vector>
+
+#include "db/column_family.h"
+#include "db/db_impl.h"
+#include "db/dbformat.h"
+#include "db/flush_scheduler.h"
+#include "db/memtable.h"
+#include "db/merge_context.h"
+#include "db/snapshot_impl.h"
+#include "db/write_batch_internal.h"
+#include "monitoring/perf_context_imp.h"
+#include "monitoring/statistics.h"
+#include "rocksdb/merge_operator.h"
+#include "util/coding.h"
+#include "util/string_util.h"
+
+namespace rocksdb {
+
+// anon namespace for file-local types
+namespace {
+
+enum ContentFlags : uint32_t {
+ DEFERRED = 1 << 0,
+ HAS_PUT = 1 << 1,
+ HAS_DELETE = 1 << 2,
+ HAS_SINGLE_DELETE = 1 << 3,
+ HAS_MERGE = 1 << 4,
+ HAS_BEGIN_PREPARE = 1 << 5,
+ HAS_END_PREPARE = 1 << 6,
+ HAS_COMMIT = 1 << 7,
+ HAS_ROLLBACK = 1 << 8,
+ HAS_DELETE_RANGE = 1 << 9,
+};
+
+struct BatchContentClassifier : public WriteBatch::Handler {
+ uint32_t content_flags = 0;
+
+ Status PutCF(uint32_t, const Slice&, const Slice&) override {
+ content_flags |= ContentFlags::HAS_PUT;
+ return Status::OK();
+ }
+
+ Status DeleteCF(uint32_t, const Slice&) override {
+ content_flags |= ContentFlags::HAS_DELETE;
+ return Status::OK();
+ }
+
+ Status SingleDeleteCF(uint32_t, const Slice&) override {
+ content_flags |= ContentFlags::HAS_SINGLE_DELETE;
+ return Status::OK();
+ }
+
+ Status DeleteRangeCF(uint32_t, const Slice&, const Slice&) override {
+ content_flags |= ContentFlags::HAS_DELETE_RANGE;
+ return Status::OK();
+ }
+
+ Status MergeCF(uint32_t, const Slice&, const Slice&) override {
+ content_flags |= ContentFlags::HAS_MERGE;
+ return Status::OK();
+ }
+
+ Status MarkBeginPrepare() override {
+ content_flags |= ContentFlags::HAS_BEGIN_PREPARE;
+ return Status::OK();
+ }
+
+ Status MarkEndPrepare(const Slice&) override {
+ content_flags |= ContentFlags::HAS_END_PREPARE;
+ return Status::OK();
+ }
+
+ Status MarkCommit(const Slice&) override {
+ content_flags |= ContentFlags::HAS_COMMIT;
+ return Status::OK();
+ }
+
+ Status MarkRollback(const Slice&) override {
+ content_flags |= ContentFlags::HAS_ROLLBACK;
+ return Status::OK();
+ }
+};
+
+} // anon namespace
+
+struct SavePoints {
+ std::stack<SavePoint> stack;
+};
+
+WriteBatch::WriteBatch(size_t reserved_bytes, size_t max_bytes)
+ : save_points_(nullptr), content_flags_(0), max_bytes_(max_bytes), rep_() {
+ rep_.reserve((reserved_bytes > WriteBatchInternal::kHeader) ?
+ reserved_bytes : WriteBatchInternal::kHeader);
+ rep_.resize(WriteBatchInternal::kHeader);
+}
+
+WriteBatch::WriteBatch(const std::string& rep)
+ : save_points_(nullptr),
+ content_flags_(ContentFlags::DEFERRED),
+ max_bytes_(0),
+ rep_(rep) {}
+
+WriteBatch::WriteBatch(const WriteBatch& src)
+ : save_points_(src.save_points_),
+ wal_term_point_(src.wal_term_point_),
+ content_flags_(src.content_flags_.load(std::memory_order_relaxed)),
+ max_bytes_(src.max_bytes_),
+ rep_(src.rep_) {}
+
+WriteBatch::WriteBatch(WriteBatch&& src)
+ : save_points_(std::move(src.save_points_)),
+ wal_term_point_(std::move(src.wal_term_point_)),
+ content_flags_(src.content_flags_.load(std::memory_order_relaxed)),
+ max_bytes_(src.max_bytes_),
+ rep_(std::move(src.rep_)) {}
+
+WriteBatch& WriteBatch::operator=(const WriteBatch& src) {
+ if (&src != this) {
+ this->~WriteBatch();
+ new (this) WriteBatch(src);
+ }
+ return *this;
+}
+
+WriteBatch& WriteBatch::operator=(WriteBatch&& src) {
+ if (&src != this) {
+ this->~WriteBatch();
+ new (this) WriteBatch(std::move(src));
+ }
+ return *this;
+}
+
+WriteBatch::~WriteBatch() { delete save_points_; }
+
+WriteBatch::Handler::~Handler() { }
+
+void WriteBatch::Handler::LogData(const Slice& blob) {
+ // If the user has not specified something to do with blobs, then we ignore
+ // them.
+}
+
+bool WriteBatch::Handler::Continue() {
+ return true;
+}
+
+void WriteBatch::Clear() {
+ rep_.clear();
+ rep_.resize(WriteBatchInternal::kHeader);
+
+ content_flags_.store(0, std::memory_order_relaxed);
+
+ if (save_points_ != nullptr) {
+ while (!save_points_->stack.empty()) {
+ save_points_->stack.pop();
+ }
+ }
+
+ wal_term_point_.clear();
+}
+
+int WriteBatch::Count() const {
+ return WriteBatchInternal::Count(this);
+}
+
+uint32_t WriteBatch::ComputeContentFlags() const {
+ auto rv = content_flags_.load(std::memory_order_relaxed);
+ if ((rv & ContentFlags::DEFERRED) != 0) {
+ BatchContentClassifier classifier;
+ Iterate(&classifier);
+ rv = classifier.content_flags;
+
+ // this method is conceptually const, because it is performing a lazy
+ // computation that doesn't affect the abstract state of the batch.
+ // content_flags_ is marked mutable so that we can perform the
+ // following assignment
+ content_flags_.store(rv, std::memory_order_relaxed);
+ }
+ return rv;
+}
+
+void WriteBatch::MarkWalTerminationPoint() {
+ wal_term_point_.size = GetDataSize();
+ wal_term_point_.count = Count();
+ wal_term_point_.content_flags = content_flags_;
+}
+
+bool WriteBatch::HasPut() const {
+ return (ComputeContentFlags() & ContentFlags::HAS_PUT) != 0;
+}
+
+bool WriteBatch::HasDelete() const {
+ return (ComputeContentFlags() & ContentFlags::HAS_DELETE) != 0;
+}
+
+bool WriteBatch::HasSingleDelete() const {
+ return (ComputeContentFlags() & ContentFlags::HAS_SINGLE_DELETE) != 0;
+}
+
+bool WriteBatch::HasDeleteRange() const {
+ return (ComputeContentFlags() & ContentFlags::HAS_DELETE_RANGE) != 0;
+}
+
+bool WriteBatch::HasMerge() const {
+ return (ComputeContentFlags() & ContentFlags::HAS_MERGE) != 0;
+}
+
+bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key, bool cf_record) {
+ assert(input != nullptr && key != nullptr);
+ // Skip tag byte
+ input->remove_prefix(1);
+
+ if (cf_record) {
+ // Skip column_family bytes
+ uint32_t cf;
+ if (!GetVarint32(input, &cf)) {
+ return false;
+ }
+ }
+
+ // Extract key
+ return GetLengthPrefixedSlice(input, key);
+}
+
+bool WriteBatch::HasBeginPrepare() const {
+ return (ComputeContentFlags() & ContentFlags::HAS_BEGIN_PREPARE) != 0;
+}
+
+bool WriteBatch::HasEndPrepare() const {
+ return (ComputeContentFlags() & ContentFlags::HAS_END_PREPARE) != 0;
+}
+
+bool WriteBatch::HasCommit() const {
+ return (ComputeContentFlags() & ContentFlags::HAS_COMMIT) != 0;
+}
+
+bool WriteBatch::HasRollback() const {
+ return (ComputeContentFlags() & ContentFlags::HAS_ROLLBACK) != 0;
+}
+
+Status ReadRecordFromWriteBatch(Slice* input, char* tag,
+ uint32_t* column_family, Slice* key,
+ Slice* value, Slice* blob, Slice* xid) {
+ assert(key != nullptr && value != nullptr);
+ *tag = (*input)[0];
+ input->remove_prefix(1);
+ *column_family = 0; // default
+ switch (*tag) {
+ case kTypeColumnFamilyValue:
+ if (!GetVarint32(input, column_family)) {
+ return Status::Corruption("bad WriteBatch Put");
+ }
+ // intentional fallthrough
+ case kTypeValue:
+ if (!GetLengthPrefixedSlice(input, key) ||
+ !GetLengthPrefixedSlice(input, value)) {
+ return Status::Corruption("bad WriteBatch Put");
+ }
+ break;
+ case kTypeColumnFamilyDeletion:
+ case kTypeColumnFamilySingleDeletion:
+ if (!GetVarint32(input, column_family)) {
+ return Status::Corruption("bad WriteBatch Delete");
+ }
+ // intentional fallthrough
+ case kTypeDeletion:
+ case kTypeSingleDeletion:
+ if (!GetLengthPrefixedSlice(input, key)) {
+ return Status::Corruption("bad WriteBatch Delete");
+ }
+ break;
+ case kTypeColumnFamilyRangeDeletion:
+ if (!GetVarint32(input, column_family)) {
+ return Status::Corruption("bad WriteBatch DeleteRange");
+ }
+ // intentional fallthrough
+ case kTypeRangeDeletion:
+ // for range delete, "key" is begin_key, "value" is end_key
+ if (!GetLengthPrefixedSlice(input, key) ||
+ !GetLengthPrefixedSlice(input, value)) {
+ return Status::Corruption("bad WriteBatch DeleteRange");
+ }
+ break;
+ case kTypeColumnFamilyMerge:
+ if (!GetVarint32(input, column_family)) {
+ return Status::Corruption("bad WriteBatch Merge");
+ }
+ // intentional fallthrough
+ case kTypeMerge:
+ if (!GetLengthPrefixedSlice(input, key) ||
+ !GetLengthPrefixedSlice(input, value)) {
+ return Status::Corruption("bad WriteBatch Merge");
+ }
+ break;
+ case kTypeLogData:
+ assert(blob != nullptr);
+ if (!GetLengthPrefixedSlice(input, blob)) {
+ return Status::Corruption("bad WriteBatch Blob");
+ }
+ break;
+ case kTypeNoop:
+ case kTypeBeginPrepareXID:
+ break;
+ case kTypeEndPrepareXID:
+ if (!GetLengthPrefixedSlice(input, xid)) {
+ return Status::Corruption("bad EndPrepare XID");
+ }
+ break;
+ case kTypeCommitXID:
+ if (!GetLengthPrefixedSlice(input, xid)) {
+ return Status::Corruption("bad Commit XID");
+ }
+ break;
+ case kTypeRollbackXID:
+ if (!GetLengthPrefixedSlice(input, xid)) {
+ return Status::Corruption("bad Rollback XID");
+ }
+ break;
+ default:
+ return Status::Corruption("unknown WriteBatch tag");
+ }
+ return Status::OK();
+}
+
+Status WriteBatch::Iterate(Handler* handler) const {
+ Slice input(rep_);
+ if (input.size() < WriteBatchInternal::kHeader) {
+ return Status::Corruption("malformed WriteBatch (too small)");
+ }
+
+ input.remove_prefix(WriteBatchInternal::kHeader);
+ Slice key, value, blob, xid;
+ int found = 0;
+ Status s;
+ while (s.ok() && !input.empty() && handler->Continue()) {
+ char tag = 0;
+ uint32_t column_family = 0; // default
+
+ s = ReadRecordFromWriteBatch(&input, &tag, &column_family, &key, &value,
+ &blob, &xid);
+ if (!s.ok()) {
+ return s;
+ }
+
+ switch (tag) {
+ case kTypeColumnFamilyValue:
+ case kTypeValue:
+ assert(content_flags_.load(std::memory_order_relaxed) &
+ (ContentFlags::DEFERRED | ContentFlags::HAS_PUT));
+ s = handler->PutCF(column_family, key, value);
+ found++;
+ break;
+ case kTypeColumnFamilyDeletion:
+ case kTypeDeletion:
+ assert(content_flags_.load(std::memory_order_relaxed) &
+ (ContentFlags::DEFERRED | ContentFlags::HAS_DELETE));
+ s = handler->DeleteCF(column_family, key);
+ found++;
+ break;
+ case kTypeColumnFamilySingleDeletion:
+ case kTypeSingleDeletion:
+ assert(content_flags_.load(std::memory_order_relaxed) &
+ (ContentFlags::DEFERRED | ContentFlags::HAS_SINGLE_DELETE));
+ s = handler->SingleDeleteCF(column_family, key);
+ found++;
+ break;
+ case kTypeColumnFamilyRangeDeletion:
+ case kTypeRangeDeletion:
+ assert(content_flags_.load(std::memory_order_relaxed) &
+ (ContentFlags::DEFERRED | ContentFlags::HAS_DELETE_RANGE));
+ s = handler->DeleteRangeCF(column_family, key, value);
+ found++;
+ break;
+ case kTypeColumnFamilyMerge:
+ case kTypeMerge:
+ assert(content_flags_.load(std::memory_order_relaxed) &
+ (ContentFlags::DEFERRED | ContentFlags::HAS_MERGE));
+ s = handler->MergeCF(column_family, key, value);
+ found++;
+ break;
+ case kTypeLogData:
+ handler->LogData(blob);
+ break;
+ case kTypeBeginPrepareXID:
+ assert(content_flags_.load(std::memory_order_relaxed) &
+ (ContentFlags::DEFERRED | ContentFlags::HAS_BEGIN_PREPARE));
+ handler->MarkBeginPrepare();
+ break;
+ case kTypeEndPrepareXID:
+ assert(content_flags_.load(std::memory_order_relaxed) &
+ (ContentFlags::DEFERRED | ContentFlags::HAS_END_PREPARE));
+ handler->MarkEndPrepare(xid);
+ break;
+ case kTypeCommitXID:
+ assert(content_flags_.load(std::memory_order_relaxed) &
+ (ContentFlags::DEFERRED | ContentFlags::HAS_COMMIT));
+ handler->MarkCommit(xid);
+ break;
+ case kTypeRollbackXID:
+ assert(content_flags_.load(std::memory_order_relaxed) &
+ (ContentFlags::DEFERRED | ContentFlags::HAS_ROLLBACK));
+ handler->MarkRollback(xid);
+ break;
+ case kTypeNoop:
+ break;
+ default:
+ return Status::Corruption("unknown WriteBatch tag");
+ }
+ }
+ if (!s.ok()) {
+ return s;
+ }
+ if (found != WriteBatchInternal::Count(this)) {
+ return Status::Corruption("WriteBatch has wrong count");
+ } else {
+ return Status::OK();
+ }
+}
+
+int WriteBatchInternal::Count(const WriteBatch* b) {
+ return DecodeFixed32(b->rep_.data() + 8);
+}
+
+void WriteBatchInternal::SetCount(WriteBatch* b, int n) {
+ EncodeFixed32(&b->rep_[8], n);
+}
+
+SequenceNumber WriteBatchInternal::Sequence(const WriteBatch* b) {
+ return SequenceNumber(DecodeFixed64(b->rep_.data()));
+}
+
+void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) {
+ EncodeFixed64(&b->rep_[0], seq);
+}
+
+size_t WriteBatchInternal::GetFirstOffset(WriteBatch* b) {
+ return WriteBatchInternal::kHeader;
+}
+
+Status WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id,
+ const Slice& key, const Slice& value) {
+ LocalSavePoint save(b);
+ WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+ if (column_family_id == 0) {
+ b->rep_.push_back(static_cast<char>(kTypeValue));
+ } else {
+ b->rep_.push_back(static_cast<char>(kTypeColumnFamilyValue));
+ PutVarint32(&b->rep_, column_family_id);
+ }
+ PutLengthPrefixedSlice(&b->rep_, key);
+ PutLengthPrefixedSlice(&b->rep_, value);
+ b->content_flags_.store(
+ b->content_flags_.load(std::memory_order_relaxed) | ContentFlags::HAS_PUT,
+ std::memory_order_relaxed);
+ return save.commit();
+}
+
+Status WriteBatch::Put(ColumnFamilyHandle* column_family, const Slice& key,
+ const Slice& value) {
+ return WriteBatchInternal::Put(this, GetColumnFamilyID(column_family), key,
+ value);
+}
+
+Status WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id,
+ const SliceParts& key, const SliceParts& value) {
+ LocalSavePoint save(b);
+ WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+ if (column_family_id == 0) {
+ b->rep_.push_back(static_cast<char>(kTypeValue));
+ } else {
+ b->rep_.push_back(static_cast<char>(kTypeColumnFamilyValue));
+ PutVarint32(&b->rep_, column_family_id);
+ }
+ PutLengthPrefixedSliceParts(&b->rep_, key);
+ PutLengthPrefixedSliceParts(&b->rep_, value);
+ b->content_flags_.store(
+ b->content_flags_.load(std::memory_order_relaxed) | ContentFlags::HAS_PUT,
+ std::memory_order_relaxed);
+ return save.commit();
+}
+
+Status WriteBatch::Put(ColumnFamilyHandle* column_family, const SliceParts& key,
+ const SliceParts& value) {
+ return WriteBatchInternal::Put(this, GetColumnFamilyID(column_family), key,
+ value);
+}
+
+Status WriteBatchInternal::InsertNoop(WriteBatch* b) {
+ b->rep_.push_back(static_cast<char>(kTypeNoop));
+ return Status::OK();
+}
+
+Status WriteBatchInternal::MarkEndPrepare(WriteBatch* b, const Slice& xid) {
+ // a manually constructed batch can only contain one prepare section
+ assert(b->rep_[12] == static_cast<char>(kTypeNoop));
+
+ // all savepoints up to this point are cleared
+ if (b->save_points_ != nullptr) {
+ while (!b->save_points_->stack.empty()) {
+ b->save_points_->stack.pop();
+ }
+ }
+
+ // rewrite noop as begin marker
+ b->rep_[12] = static_cast<char>(kTypeBeginPrepareXID);
+ b->rep_.push_back(static_cast<char>(kTypeEndPrepareXID));
+ PutLengthPrefixedSlice(&b->rep_, xid);
+ b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+ ContentFlags::HAS_END_PREPARE |
+ ContentFlags::HAS_BEGIN_PREPARE,
+ std::memory_order_relaxed);
+ return Status::OK();
+}
+
+Status WriteBatchInternal::MarkCommit(WriteBatch* b, const Slice& xid) {
+ b->rep_.push_back(static_cast<char>(kTypeCommitXID));
+ PutLengthPrefixedSlice(&b->rep_, xid);
+ b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+ ContentFlags::HAS_COMMIT,
+ std::memory_order_relaxed);
+ return Status::OK();
+}
+
+Status WriteBatchInternal::MarkRollback(WriteBatch* b, const Slice& xid) {
+ b->rep_.push_back(static_cast<char>(kTypeRollbackXID));
+ PutLengthPrefixedSlice(&b->rep_, xid);
+ b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+ ContentFlags::HAS_ROLLBACK,
+ std::memory_order_relaxed);
+ return Status::OK();
+}
+
+Status WriteBatchInternal::Delete(WriteBatch* b, uint32_t column_family_id,
+ const Slice& key) {
+ LocalSavePoint save(b);
+ WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+ if (column_family_id == 0) {
+ b->rep_.push_back(static_cast<char>(kTypeDeletion));
+ } else {
+ b->rep_.push_back(static_cast<char>(kTypeColumnFamilyDeletion));
+ PutVarint32(&b->rep_, column_family_id);
+ }
+ PutLengthPrefixedSlice(&b->rep_, key);
+ b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+ ContentFlags::HAS_DELETE,
+ std::memory_order_relaxed);
+ return save.commit();
+}
+
+Status WriteBatch::Delete(ColumnFamilyHandle* column_family, const Slice& key) {
+ return WriteBatchInternal::Delete(this, GetColumnFamilyID(column_family),
+ key);
+}
+
+Status WriteBatchInternal::Delete(WriteBatch* b, uint32_t column_family_id,
+ const SliceParts& key) {
+ LocalSavePoint save(b);
+ WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+ if (column_family_id == 0) {
+ b->rep_.push_back(static_cast<char>(kTypeDeletion));
+ } else {
+ b->rep_.push_back(static_cast<char>(kTypeColumnFamilyDeletion));
+ PutVarint32(&b->rep_, column_family_id);
+ }
+ PutLengthPrefixedSliceParts(&b->rep_, key);
+ b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+ ContentFlags::HAS_DELETE,
+ std::memory_order_relaxed);
+ return save.commit();
+}
+
+Status WriteBatch::Delete(ColumnFamilyHandle* column_family,
+ const SliceParts& key) {
+ return WriteBatchInternal::Delete(this, GetColumnFamilyID(column_family),
+ key);
+}
+
+Status WriteBatchInternal::SingleDelete(WriteBatch* b,
+ uint32_t column_family_id,
+ const Slice& key) {
+ LocalSavePoint save(b);
+ WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+ if (column_family_id == 0) {
+ b->rep_.push_back(static_cast<char>(kTypeSingleDeletion));
+ } else {
+ b->rep_.push_back(static_cast<char>(kTypeColumnFamilySingleDeletion));
+ PutVarint32(&b->rep_, column_family_id);
+ }
+ PutLengthPrefixedSlice(&b->rep_, key);
+ b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+ ContentFlags::HAS_SINGLE_DELETE,
+ std::memory_order_relaxed);
+ return save.commit();
+}
+
+Status WriteBatch::SingleDelete(ColumnFamilyHandle* column_family,
+ const Slice& key) {
+ return WriteBatchInternal::SingleDelete(
+ this, GetColumnFamilyID(column_family), key);
+}
+
+Status WriteBatchInternal::SingleDelete(WriteBatch* b,
+ uint32_t column_family_id,
+ const SliceParts& key) {
+ LocalSavePoint save(b);
+ WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+ if (column_family_id == 0) {
+ b->rep_.push_back(static_cast<char>(kTypeSingleDeletion));
+ } else {
+ b->rep_.push_back(static_cast<char>(kTypeColumnFamilySingleDeletion));
+ PutVarint32(&b->rep_, column_family_id);
+ }
+ PutLengthPrefixedSliceParts(&b->rep_, key);
+ b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+ ContentFlags::HAS_SINGLE_DELETE,
+ std::memory_order_relaxed);
+ return save.commit();
+}
+
+Status WriteBatch::SingleDelete(ColumnFamilyHandle* column_family,
+ const SliceParts& key) {
+ return WriteBatchInternal::SingleDelete(
+ this, GetColumnFamilyID(column_family), key);
+}
+
+Status WriteBatchInternal::DeleteRange(WriteBatch* b, uint32_t column_family_id,
+ const Slice& begin_key,
+ const Slice& end_key) {
+ LocalSavePoint save(b);
+ WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+ if (column_family_id == 0) {
+ b->rep_.push_back(static_cast<char>(kTypeRangeDeletion));
+ } else {
+ b->rep_.push_back(static_cast<char>(kTypeColumnFamilyRangeDeletion));
+ PutVarint32(&b->rep_, column_family_id);
+ }
+ PutLengthPrefixedSlice(&b->rep_, begin_key);
+ PutLengthPrefixedSlice(&b->rep_, end_key);
+ b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+ ContentFlags::HAS_DELETE_RANGE,
+ std::memory_order_relaxed);
+ return save.commit();
+}
+
+Status WriteBatch::DeleteRange(ColumnFamilyHandle* column_family,
+ const Slice& begin_key, const Slice& end_key) {
+ return WriteBatchInternal::DeleteRange(this, GetColumnFamilyID(column_family),
+ begin_key, end_key);
+}
+
+Status WriteBatchInternal::DeleteRange(WriteBatch* b, uint32_t column_family_id,
+ const SliceParts& begin_key,
+ const SliceParts& end_key) {
+ LocalSavePoint save(b);
+ WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+ if (column_family_id == 0) {
+ b->rep_.push_back(static_cast<char>(kTypeRangeDeletion));
+ } else {
+ b->rep_.push_back(static_cast<char>(kTypeColumnFamilyRangeDeletion));
+ PutVarint32(&b->rep_, column_family_id);
+ }
+ PutLengthPrefixedSliceParts(&b->rep_, begin_key);
+ PutLengthPrefixedSliceParts(&b->rep_, end_key);
+ b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+ ContentFlags::HAS_DELETE_RANGE,
+ std::memory_order_relaxed);
+ return save.commit();
+}
+
+Status WriteBatch::DeleteRange(ColumnFamilyHandle* column_family,
+ const SliceParts& begin_key,
+ const SliceParts& end_key) {
+ return WriteBatchInternal::DeleteRange(this, GetColumnFamilyID(column_family),
+ begin_key, end_key);
+}
+
+Status WriteBatchInternal::Merge(WriteBatch* b, uint32_t column_family_id,
+ const Slice& key, const Slice& value) {
+ LocalSavePoint save(b);
+ WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+ if (column_family_id == 0) {
+ b->rep_.push_back(static_cast<char>(kTypeMerge));
+ } else {
+ b->rep_.push_back(static_cast<char>(kTypeColumnFamilyMerge));
+ PutVarint32(&b->rep_, column_family_id);
+ }
+ PutLengthPrefixedSlice(&b->rep_, key);
+ PutLengthPrefixedSlice(&b->rep_, value);
+ b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+ ContentFlags::HAS_MERGE,
+ std::memory_order_relaxed);
+ return save.commit();
+}
+
+Status WriteBatch::Merge(ColumnFamilyHandle* column_family, const Slice& key,
+ const Slice& value) {
+ return WriteBatchInternal::Merge(this, GetColumnFamilyID(column_family), key,
+ value);
+}
+
+Status WriteBatchInternal::Merge(WriteBatch* b, uint32_t column_family_id,
+ const SliceParts& key,
+ const SliceParts& value) {
+ LocalSavePoint save(b);
+ WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+ if (column_family_id == 0) {
+ b->rep_.push_back(static_cast<char>(kTypeMerge));
+ } else {
+ b->rep_.push_back(static_cast<char>(kTypeColumnFamilyMerge));
+ PutVarint32(&b->rep_, column_family_id);
+ }
+ PutLengthPrefixedSliceParts(&b->rep_, key);
+ PutLengthPrefixedSliceParts(&b->rep_, value);
+ b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
+ ContentFlags::HAS_MERGE,
+ std::memory_order_relaxed);
+ return save.commit();
+}
+
+Status WriteBatch::Merge(ColumnFamilyHandle* column_family,
+ const SliceParts& key, const SliceParts& value) {
+ return WriteBatchInternal::Merge(this, GetColumnFamilyID(column_family), key,
+ value);
+}
+
+Status WriteBatch::PutLogData(const Slice& blob) {
+ LocalSavePoint save(this);
+ rep_.push_back(static_cast<char>(kTypeLogData));
+ PutLengthPrefixedSlice(&rep_, blob);
+ return save.commit();
+}
+
+void WriteBatch::SetSavePoint() {
+ if (save_points_ == nullptr) {
+ save_points_ = new SavePoints();
+ }
+ // Record length and count of current batch of writes.
+ save_points_->stack.push(SavePoint(
+ GetDataSize(), Count(), content_flags_.load(std::memory_order_relaxed)));
+}
+
+Status WriteBatch::RollbackToSavePoint() {
+ if (save_points_ == nullptr || save_points_->stack.size() == 0) {
+ return Status::NotFound();
+ }
+
+ // Pop the most recent savepoint off the stack
+ SavePoint savepoint = save_points_->stack.top();
+ save_points_->stack.pop();
+
+ assert(savepoint.size <= rep_.size());
+ assert(savepoint.count <= Count());
+
+ if (savepoint.size == rep_.size()) {
+ // No changes to rollback
+ } else if (savepoint.size == 0) {
+ // Rollback everything
+ Clear();
+ } else {
+ rep_.resize(savepoint.size);
+ WriteBatchInternal::SetCount(this, savepoint.count);
+ content_flags_.store(savepoint.content_flags, std::memory_order_relaxed);
+ }
+
+ return Status::OK();
+}
+
+Status WriteBatch::PopSavePoint() {
+ if (save_points_ == nullptr || save_points_->stack.size() == 0) {
+ return Status::NotFound();
+ }
+
+ // Pop the most recent savepoint off the stack
+ save_points_->stack.pop();
+
+ return Status::OK();
+}
+
+class MemTableInserter : public WriteBatch::Handler {
+
+ SequenceNumber sequence_;
+ ColumnFamilyMemTables* const cf_mems_;
+ FlushScheduler* const flush_scheduler_;
+ const bool ignore_missing_column_families_;
+ const uint64_t recovering_log_number_;
+ // log number that all Memtables inserted into should reference
+ uint64_t log_number_ref_;
+ DBImpl* db_;
+ const bool concurrent_memtable_writes_;
+ bool post_info_created_;
+
+ bool* has_valid_writes_;
+ // On some (!) platforms just default creating
+ // a map is too expensive in the Write() path as they
+ // cause memory allocations though unused.
+ // Make creation optional but do not incur
+ // unique_ptr additional allocation
+ using
+ MemPostInfoMap = std::map<MemTable*, MemTablePostProcessInfo>;
+ using
+ PostMapType = std::aligned_storage<sizeof(MemPostInfoMap)>::type;
+ PostMapType mem_post_info_map_;
+ // current recovered transaction we are rebuilding (recovery)
+ WriteBatch* rebuilding_trx_;
+
+ MemPostInfoMap& GetPostMap() {
+ assert(concurrent_memtable_writes_);
+ if(!post_info_created_) {
+ new (&mem_post_info_map_) MemPostInfoMap();
+ post_info_created_ = true;
+ }
+ return *reinterpret_cast<MemPostInfoMap*>(&mem_post_info_map_);
+ }
+
+public:
+ // cf_mems should not be shared with concurrent inserters
+ MemTableInserter(SequenceNumber _sequence, ColumnFamilyMemTables* cf_mems,
+ FlushScheduler* flush_scheduler,
+ bool ignore_missing_column_families,
+ uint64_t recovering_log_number, DB* db,
+ bool concurrent_memtable_writes,
+ bool* has_valid_writes = nullptr)
+ : sequence_(_sequence),
+ cf_mems_(cf_mems),
+ flush_scheduler_(flush_scheduler),
+ ignore_missing_column_families_(ignore_missing_column_families),
+ recovering_log_number_(recovering_log_number),
+ log_number_ref_(0),
+ db_(reinterpret_cast<DBImpl*>(db)),
+ concurrent_memtable_writes_(concurrent_memtable_writes),
+ post_info_created_(false),
+ has_valid_writes_(has_valid_writes),
+ rebuilding_trx_(nullptr) {
+ assert(cf_mems_);
+ }
+
+ ~MemTableInserter() {
+ if (post_info_created_) {
+ reinterpret_cast<MemPostInfoMap*>
+ (&mem_post_info_map_)->~MemPostInfoMap();
+ }
+ }
+
+ MemTableInserter(const MemTableInserter&) = delete;
+ MemTableInserter& operator=(const MemTableInserter&) = delete;
+
+ void set_log_number_ref(uint64_t log) { log_number_ref_ = log; }
+
+ SequenceNumber sequence() const { return sequence_; }
+
+ void PostProcess() {
+ assert(concurrent_memtable_writes_);
+ // If post info was not created there is nothing
+ // to process and no need to create on demand
+ if(post_info_created_) {
+ for (auto& pair : GetPostMap()) {
+ pair.first->BatchPostProcess(pair.second);
+ }
+ }
+ }
+
+ bool SeekToColumnFamily(uint32_t column_family_id, Status* s) {
+ // If we are in a concurrent mode, it is the caller's responsibility
+ // to clone the original ColumnFamilyMemTables so that each thread
+ // has its own instance. Otherwise, it must be guaranteed that there
+ // is no concurrent access
+ bool found = cf_mems_->Seek(column_family_id);
+ if (!found) {
+ if (ignore_missing_column_families_) {
+ *s = Status::OK();
+ } else {
+ *s = Status::InvalidArgument(
+ "Invalid column family specified in write batch");
+ }
+ return false;
+ }
+ if (recovering_log_number_ != 0 &&
+ recovering_log_number_ < cf_mems_->GetLogNumber()) {
+ // This is true only in recovery environment (recovering_log_number_ is
+ // always 0 in
+ // non-recovery, regular write code-path)
+ // * If recovering_log_number_ < cf_mems_->GetLogNumber(), this means that
+ // column
+ // family already contains updates from this log. We can't apply updates
+ // twice because of update-in-place or merge workloads -- ignore the
+ // update
+ *s = Status::OK();
+ return false;
+ }
+
+ if (has_valid_writes_ != nullptr) {
+ *has_valid_writes_ = true;
+ }
+
+ if (log_number_ref_ > 0) {
+ cf_mems_->GetMemTable()->RefLogContainingPrepSection(log_number_ref_);
+ }
+
+ return true;
+ }
+
+ virtual Status PutCF(uint32_t column_family_id, const Slice& key,
+ const Slice& value) override {
+ if (rebuilding_trx_ != nullptr) {
+ WriteBatchInternal::Put(rebuilding_trx_, column_family_id, key, value);
+ return Status::OK();
+ }
+
+ Status seek_status;
+ if (!SeekToColumnFamily(column_family_id, &seek_status)) {
+ ++sequence_;
+ return seek_status;
+ }
+
+ MemTable* mem = cf_mems_->GetMemTable();
+ auto* moptions = mem->GetMemTableOptions();
+ if (!moptions->inplace_update_support) {
+ mem->Add(sequence_, kTypeValue, key, value, concurrent_memtable_writes_,
+ get_post_process_info(mem));
+ } else if (moptions->inplace_callback == nullptr) {
+ assert(!concurrent_memtable_writes_);
+ mem->Update(sequence_, key, value);
+ RecordTick(moptions->statistics, NUMBER_KEYS_UPDATED);
+ } else {
+ assert(!concurrent_memtable_writes_);
+ if (mem->UpdateCallback(sequence_, key, value)) {
+ } else {
+ // key not found in memtable. Do sst get, update, add
+ SnapshotImpl read_from_snapshot;
+ read_from_snapshot.number_ = sequence_;
+ ReadOptions ropts;
+ ropts.snapshot = &read_from_snapshot;
+
+ std::string prev_value;
+ std::string merged_value;
+
+ auto cf_handle = cf_mems_->GetColumnFamilyHandle();
+ Status s = Status::NotSupported();
+ if (db_ != nullptr && recovering_log_number_ == 0) {
+ if (cf_handle == nullptr) {
+ cf_handle = db_->DefaultColumnFamily();
+ }
+ s = db_->Get(ropts, cf_handle, key, &prev_value);
+ }
+
+ char* prev_buffer = const_cast<char*>(prev_value.c_str());
+ uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
+ auto status = moptions->inplace_callback(s.ok() ? prev_buffer : nullptr,
+ s.ok() ? &prev_size : nullptr,
+ value, &merged_value);
+ if (status == UpdateStatus::UPDATED_INPLACE) {
+ // prev_value is updated in-place with final value.
+ mem->Add(sequence_, kTypeValue, key, Slice(prev_buffer, prev_size));
+ RecordTick(moptions->statistics, NUMBER_KEYS_WRITTEN);
+ } else if (status == UpdateStatus::UPDATED) {
+ // merged_value contains the final value.
+ mem->Add(sequence_, kTypeValue, key, Slice(merged_value));
+ RecordTick(moptions->statistics, NUMBER_KEYS_WRITTEN);
+ }
+ }
+ }
+ // Since all Puts are logged in trasaction logs (if enabled), always bump
+ // sequence number. Even if the update eventually fails and does not result
+ // in memtable add/update.
+ sequence_++;
+ CheckMemtableFull();
+ return Status::OK();
+ }
+
+ Status DeleteImpl(uint32_t column_family_id, const Slice& key,
+ const Slice& value, ValueType delete_type) {
+ MemTable* mem = cf_mems_->GetMemTable();
+ mem->Add(sequence_, delete_type, key, value, concurrent_memtable_writes_,
+ get_post_process_info(mem));
+ sequence_++;
+ CheckMemtableFull();
+ return Status::OK();
+ }
+
+ virtual Status DeleteCF(uint32_t column_family_id,
+ const Slice& key) override {
+ if (rebuilding_trx_ != nullptr) {
+ WriteBatchInternal::Delete(rebuilding_trx_, column_family_id, key);
+ return Status::OK();
+ }
+
+ Status seek_status;
+ if (!SeekToColumnFamily(column_family_id, &seek_status)) {
+ ++sequence_;
+ return seek_status;
+ }
+
+ return DeleteImpl(column_family_id, key, Slice(), kTypeDeletion);
+ }
+
+ virtual Status SingleDeleteCF(uint32_t column_family_id,
+ const Slice& key) override {
+ if (rebuilding_trx_ != nullptr) {
+ WriteBatchInternal::SingleDelete(rebuilding_trx_, column_family_id, key);
+ return Status::OK();
+ }
+
+ Status seek_status;
+ if (!SeekToColumnFamily(column_family_id, &seek_status)) {
+ ++sequence_;
+ return seek_status;
+ }
+
+ return DeleteImpl(column_family_id, key, Slice(), kTypeSingleDeletion);
+ }
+
+ virtual Status DeleteRangeCF(uint32_t column_family_id,
+ const Slice& begin_key,
+ const Slice& end_key) override {
+ if (rebuilding_trx_ != nullptr) {
+ WriteBatchInternal::DeleteRange(rebuilding_trx_, column_family_id,
+ begin_key, end_key);
+ return Status::OK();
+ }
+
+ Status seek_status;
+ if (!SeekToColumnFamily(column_family_id, &seek_status)) {
+ ++sequence_;
+ return seek_status;
+ }
+ if (db_ != nullptr) {
+ auto cf_handle = cf_mems_->GetColumnFamilyHandle();
+ if (cf_handle == nullptr) {
+ cf_handle = db_->DefaultColumnFamily();
+ }
+ auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cf_handle)->cfd();
+ if (!cfd->is_delete_range_supported()) {
+ return Status::NotSupported(
+ std::string("DeleteRange not supported for table type ") +
+ cfd->ioptions()->table_factory->Name() + " in CF " +
+ cfd->GetName());
+ }
+ }
+
+ return DeleteImpl(column_family_id, begin_key, end_key, kTypeRangeDeletion);
+ }
+
+ virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
+ const Slice& value) override {
+ assert(!concurrent_memtable_writes_);
+ if (rebuilding_trx_ != nullptr) {
+ WriteBatchInternal::Merge(rebuilding_trx_, column_family_id, key, value);
+ return Status::OK();
+ }
+
+ Status seek_status;
+ if (!SeekToColumnFamily(column_family_id, &seek_status)) {
+ ++sequence_;
+ return seek_status;
+ }
+
+ MemTable* mem = cf_mems_->GetMemTable();
+ auto* moptions = mem->GetMemTableOptions();
+ bool perform_merge = false;
+
+ // If we pass DB through and options.max_successive_merges is hit
+ // during recovery, Get() will be issued which will try to acquire
+ // DB mutex and cause deadlock, as DB mutex is already held.
+ // So we disable merge in recovery
+ if (moptions->max_successive_merges > 0 && db_ != nullptr &&
+ recovering_log_number_ == 0) {
+ LookupKey lkey(key, sequence_);
+
+ // Count the number of successive merges at the head
+ // of the key in the memtable
+ size_t num_merges = mem->CountSuccessiveMergeEntries(lkey);
+
+ if (num_merges >= moptions->max_successive_merges) {
+ perform_merge = true;
+ }
+ }
+
+ if (perform_merge) {
+ // 1) Get the existing value
+ std::string get_value;
+
+ // Pass in the sequence number so that we also include previous merge
+ // operations in the same batch.
+ SnapshotImpl read_from_snapshot;
+ read_from_snapshot.number_ = sequence_;
+ ReadOptions read_options;
+ read_options.snapshot = &read_from_snapshot;
+
+ auto cf_handle = cf_mems_->GetColumnFamilyHandle();
+ if (cf_handle == nullptr) {
+ cf_handle = db_->DefaultColumnFamily();
+ }
+ db_->Get(read_options, cf_handle, key, &get_value);
+ Slice get_value_slice = Slice(get_value);
+
+ // 2) Apply this merge
+ auto merge_operator = moptions->merge_operator;
+ assert(merge_operator);
+
+ std::string new_value;
+
+ Status merge_status = MergeHelper::TimedFullMerge(
+ merge_operator, key, &get_value_slice, {value}, &new_value,
+ moptions->info_log, moptions->statistics, Env::Default());
+
+ if (!merge_status.ok()) {
+ // Failed to merge!
+ // Store the delta in memtable
+ perform_merge = false;
+ } else {
+ // 3) Add value to memtable
+ mem->Add(sequence_, kTypeValue, key, new_value);
+ }
+ }
+
+ if (!perform_merge) {
+ // Add merge operator to memtable
+ mem->Add(sequence_, kTypeMerge, key, value);
+ }
+
+ sequence_++;
+ CheckMemtableFull();
+ return Status::OK();
+ }
+
+ void CheckMemtableFull() {
+ if (flush_scheduler_ != nullptr) {
+ auto* cfd = cf_mems_->current();
+ assert(cfd != nullptr);
+ if (cfd->mem()->ShouldScheduleFlush() &&
+ cfd->mem()->MarkFlushScheduled()) {
+ // MarkFlushScheduled only returns true if we are the one that
+ // should take action, so no need to dedup further
+ flush_scheduler_->ScheduleFlush(cfd);
+ }
+ }
+ }
+
+ Status MarkBeginPrepare() override {
+ assert(rebuilding_trx_ == nullptr);
+ assert(db_);
+
+ if (recovering_log_number_ != 0) {
+ // during recovery we rebuild a hollow transaction
+ // from all encountered prepare sections of the wal
+ if (db_->allow_2pc() == false) {
+ return Status::NotSupported(
+ "WAL contains prepared transactions. Open with "
+ "TransactionDB::Open().");
+ }
+
+ // we are now iterating through a prepared section
+ rebuilding_trx_ = new WriteBatch();
+ if (has_valid_writes_ != nullptr) {
+ *has_valid_writes_ = true;
+ }
+ } else {
+ // in non-recovery we ignore prepare markers
+ // and insert the values directly. making sure we have a
+ // log for each insertion to reference.
+ assert(log_number_ref_ > 0);
+ }
+
+ return Status::OK();
+ }
+
+ Status MarkEndPrepare(const Slice& name) override {
+ assert(db_);
+ assert((rebuilding_trx_ != nullptr) == (recovering_log_number_ != 0));
+
+ if (recovering_log_number_ != 0) {
+ assert(db_->allow_2pc());
+ db_->InsertRecoveredTransaction(recovering_log_number_, name.ToString(),
+ rebuilding_trx_);
+ rebuilding_trx_ = nullptr;
+ } else {
+ assert(rebuilding_trx_ == nullptr);
+ assert(log_number_ref_ > 0);
+ }
+
+ return Status::OK();
+ }
+
+ Status MarkCommit(const Slice& name) override {
+ assert(db_);
+
+ Status s;
+
+ if (recovering_log_number_ != 0) {
+ // in recovery when we encounter a commit marker
+ // we lookup this transaction in our set of rebuilt transactions
+ // and commit.
+ auto trx = db_->GetRecoveredTransaction(name.ToString());
+
+ // the log contaiting the prepared section may have
+ // been released in the last incarnation because the
+ // data was flushed to L0
+ if (trx != nullptr) {
+ // at this point individual CF lognumbers will prevent
+ // duplicate re-insertion of values.
+ assert(log_number_ref_ == 0);
+ // all insertes must reference this trx log number
+ log_number_ref_ = trx->log_number_;
+ s = trx->batch_->Iterate(this);
+ log_number_ref_ = 0;
+
+ if (s.ok()) {
+ db_->DeleteRecoveredTransaction(name.ToString());
+ }
+ if (has_valid_writes_ != nullptr) {
+ *has_valid_writes_ = true;
+ }
+ }
+ } else {
+ // in non recovery we simply ignore this tag
+ }
+
+ return s;
+ }
+
+ Status MarkRollback(const Slice& name) override {
+ assert(db_);
+
+ if (recovering_log_number_ != 0) {
+ auto trx = db_->GetRecoveredTransaction(name.ToString());
+
+ // the log containing the transactions prep section
+ // may have been released in the previous incarnation
+ // because we knew it had been rolled back
+ if (trx != nullptr) {
+ db_->DeleteRecoveredTransaction(name.ToString());
+ }
+ } else {
+ // in non recovery we simply ignore this tag
+ }
+
+ return Status::OK();
+ }
+
+ private:
+ MemTablePostProcessInfo* get_post_process_info(MemTable* mem) {
+ if (!concurrent_memtable_writes_) {
+ // No need to batch counters locally if we don't use concurrent mode.
+ return nullptr;
+ }
+ return &GetPostMap()[mem];
+ }
+};
+
+// This function can only be called in these conditions:
+// 1) During Recovery()
+// 2) During Write(), in a single-threaded write thread
+// 3) During Write(), in a concurrent context where memtables has been cloned
+// The reason is that it calls memtables->Seek(), which has a stateful cache
+Status WriteBatchInternal::InsertInto(WriteThread::WriteGroup& write_group,
+ SequenceNumber sequence,
+ ColumnFamilyMemTables* memtables,
+ FlushScheduler* flush_scheduler,
+ bool ignore_missing_column_families,
+ uint64_t recovery_log_number, DB* db,
+ bool concurrent_memtable_writes) {
+ MemTableInserter inserter(sequence, memtables, flush_scheduler,
+ ignore_missing_column_families, recovery_log_number,
+ db, concurrent_memtable_writes);
+ for (auto w : write_group) {
+ if (!w->ShouldWriteToMemtable()) {
+ continue;
+ }
+ SetSequence(w->batch, inserter.sequence());
+ inserter.set_log_number_ref(w->log_ref);
+ w->status = w->batch->Iterate(&inserter);
+ if (!w->status.ok()) {
+ return w->status;
+ }
+ }
+ return Status::OK();
+}
+
+Status WriteBatchInternal::InsertInto(WriteThread::Writer* writer,
+ SequenceNumber sequence,
+ ColumnFamilyMemTables* memtables,
+ FlushScheduler* flush_scheduler,
+ bool ignore_missing_column_families,
+ uint64_t log_number, DB* db,
+ bool concurrent_memtable_writes) {
+ assert(writer->ShouldWriteToMemtable());
+ MemTableInserter inserter(sequence, memtables, flush_scheduler,
+ ignore_missing_column_families, log_number, db,
+ concurrent_memtable_writes);
+ SetSequence(writer->batch, sequence);
+ inserter.set_log_number_ref(writer->log_ref);
+ Status s = writer->batch->Iterate(&inserter);
+ if (concurrent_memtable_writes) {
+ inserter.PostProcess();
+ }
+ return s;
+}
+
+Status WriteBatchInternal::InsertInto(
+ const WriteBatch* batch, ColumnFamilyMemTables* memtables,
+ FlushScheduler* flush_scheduler, bool ignore_missing_column_families,
+ uint64_t log_number, DB* db, bool concurrent_memtable_writes,
+ SequenceNumber* last_seq_used, bool* has_valid_writes) {
+ MemTableInserter inserter(Sequence(batch), memtables, flush_scheduler,
+ ignore_missing_column_families, log_number, db,
+ concurrent_memtable_writes, has_valid_writes);
+ Status s = batch->Iterate(&inserter);
+ if (last_seq_used != nullptr) {
+ *last_seq_used = inserter.sequence();
+ }
+ if (concurrent_memtable_writes) {
+ inserter.PostProcess();
+ }
+ return s;
+}
+
+Status WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) {
+ assert(contents.size() >= WriteBatchInternal::kHeader);
+ b->rep_.assign(contents.data(), contents.size());
+ b->content_flags_.store(ContentFlags::DEFERRED, std::memory_order_relaxed);
+ return Status::OK();
+}
+
+Status WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src,
+ const bool wal_only) {
+ size_t src_len;
+ int src_count;
+ uint32_t src_flags;
+
+ const SavePoint& batch_end = src->GetWalTerminationPoint();
+
+ if (wal_only && !batch_end.is_cleared()) {
+ src_len = batch_end.size - WriteBatchInternal::kHeader;
+ src_count = batch_end.count;
+ src_flags = batch_end.content_flags;
+ } else {
+ src_len = src->rep_.size() - WriteBatchInternal::kHeader;
+ src_count = Count(src);
+ src_flags = src->content_flags_.load(std::memory_order_relaxed);
+ }
+
+ SetCount(dst, Count(dst) + src_count);
+ assert(src->rep_.size() >= WriteBatchInternal::kHeader);
+ dst->rep_.append(src->rep_.data() + WriteBatchInternal::kHeader, src_len);
+ dst->content_flags_.store(
+ dst->content_flags_.load(std::memory_order_relaxed) | src_flags,
+ std::memory_order_relaxed);
+ return Status::OK();
+}
+
+size_t WriteBatchInternal::AppendedByteSize(size_t leftByteSize,
+ size_t rightByteSize) {
+ if (leftByteSize == 0 || rightByteSize == 0) {
+ return leftByteSize + rightByteSize;
+ } else {
+ return leftByteSize + rightByteSize - WriteBatchInternal::kHeader;
+ }
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/write_batch_base.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/write_batch_base.cc b/thirdparty/rocksdb/db/write_batch_base.cc
new file mode 100644
index 0000000..5522c1f
--- /dev/null
+++ b/thirdparty/rocksdb/db/write_batch_base.cc
@@ -0,0 +1,94 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "rocksdb/write_batch_base.h"
+
+#include <string>
+
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
+
+namespace rocksdb {
+
+// Simple implementation of SlicePart variants of Put(). Child classes
+// can override these method with more performant solutions if they choose.
+Status WriteBatchBase::Put(ColumnFamilyHandle* column_family,
+ const SliceParts& key, const SliceParts& value) {
+ std::string key_buf, value_buf;
+ Slice key_slice(key, &key_buf);
+ Slice value_slice(value, &value_buf);
+
+ return Put(column_family, key_slice, value_slice);
+}
+
+Status WriteBatchBase::Put(const SliceParts& key, const SliceParts& value) {
+ std::string key_buf, value_buf;
+ Slice key_slice(key, &key_buf);
+ Slice value_slice(value, &value_buf);
+
+ return Put(key_slice, value_slice);
+}
+
+Status WriteBatchBase::Delete(ColumnFamilyHandle* column_family,
+ const SliceParts& key) {
+ std::string key_buf;
+ Slice key_slice(key, &key_buf);
+ return Delete(column_family, key_slice);
+}
+
+Status WriteBatchBase::Delete(const SliceParts& key) {
+ std::string key_buf;
+ Slice key_slice(key, &key_buf);
+ return Delete(key_slice);
+}
+
+Status WriteBatchBase::SingleDelete(ColumnFamilyHandle* column_family,
+ const SliceParts& key) {
+ std::string key_buf;
+ Slice key_slice(key, &key_buf);
+ return SingleDelete(column_family, key_slice);
+}
+
+Status WriteBatchBase::SingleDelete(const SliceParts& key) {
+ std::string key_buf;
+ Slice key_slice(key, &key_buf);
+ return SingleDelete(key_slice);
+}
+
+Status WriteBatchBase::DeleteRange(ColumnFamilyHandle* column_family,
+ const SliceParts& begin_key,
+ const SliceParts& end_key) {
+ std::string begin_key_buf, end_key_buf;
+ Slice begin_key_slice(begin_key, &begin_key_buf);
+ Slice end_key_slice(end_key, &end_key_buf);
+ return DeleteRange(column_family, begin_key_slice, end_key_slice);
+}
+
+Status WriteBatchBase::DeleteRange(const SliceParts& begin_key,
+ const SliceParts& end_key) {
+ std::string begin_key_buf, end_key_buf;
+ Slice begin_key_slice(begin_key, &begin_key_buf);
+ Slice end_key_slice(end_key, &end_key_buf);
+ return DeleteRange(begin_key_slice, end_key_slice);
+}
+
+Status WriteBatchBase::Merge(ColumnFamilyHandle* column_family,
+ const SliceParts& key, const SliceParts& value) {
+ std::string key_buf, value_buf;
+ Slice key_slice(key, &key_buf);
+ Slice value_slice(value, &value_buf);
+
+ return Merge(column_family, key_slice, value_slice);
+}
+
+Status WriteBatchBase::Merge(const SliceParts& key, const SliceParts& value) {
+ std::string key_buf, value_buf;
+ Slice key_slice(key, &key_buf);
+ Slice value_slice(value, &value_buf);
+
+ return Merge(key_slice, value_slice);
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/write_batch_internal.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/write_batch_internal.h b/thirdparty/rocksdb/db/write_batch_internal.h
new file mode 100644
index 0000000..48a417c
--- /dev/null
+++ b/thirdparty/rocksdb/db/write_batch_internal.h
@@ -0,0 +1,227 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <vector>
+#include "db/write_thread.h"
+#include "rocksdb/types.h"
+#include "rocksdb/write_batch.h"
+#include "rocksdb/db.h"
+#include "rocksdb/options.h"
+#include "util/autovector.h"
+
+namespace rocksdb {
+
+class MemTable;
+class FlushScheduler;
+class ColumnFamilyData;
+
+class ColumnFamilyMemTables {
+ public:
+ virtual ~ColumnFamilyMemTables() {}
+ virtual bool Seek(uint32_t column_family_id) = 0;
+ // returns true if the update to memtable should be ignored
+ // (useful when recovering from log whose updates have already
+ // been processed)
+ virtual uint64_t GetLogNumber() const = 0;
+ virtual MemTable* GetMemTable() const = 0;
+ virtual ColumnFamilyHandle* GetColumnFamilyHandle() = 0;
+ virtual ColumnFamilyData* current() { return nullptr; }
+};
+
+class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
+ public:
+ explicit ColumnFamilyMemTablesDefault(MemTable* mem)
+ : ok_(false), mem_(mem) {}
+
+ bool Seek(uint32_t column_family_id) override {
+ ok_ = (column_family_id == 0);
+ return ok_;
+ }
+
+ uint64_t GetLogNumber() const override { return 0; }
+
+ MemTable* GetMemTable() const override {
+ assert(ok_);
+ return mem_;
+ }
+
+ ColumnFamilyHandle* GetColumnFamilyHandle() override { return nullptr; }
+
+ private:
+ bool ok_;
+ MemTable* mem_;
+};
+
+// WriteBatchInternal provides static methods for manipulating a
+// WriteBatch that we don't want in the public WriteBatch interface.
+class WriteBatchInternal {
+ public:
+
+ // WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
+ static const size_t kHeader = 12;
+
+ // WriteBatch methods with column_family_id instead of ColumnFamilyHandle*
+ static Status Put(WriteBatch* batch, uint32_t column_family_id,
+ const Slice& key, const Slice& value);
+
+ static Status Put(WriteBatch* batch, uint32_t column_family_id,
+ const SliceParts& key, const SliceParts& value);
+
+ static Status Delete(WriteBatch* batch, uint32_t column_family_id,
+ const SliceParts& key);
+
+ static Status Delete(WriteBatch* batch, uint32_t column_family_id,
+ const Slice& key);
+
+ static Status SingleDelete(WriteBatch* batch, uint32_t column_family_id,
+ const SliceParts& key);
+
+ static Status SingleDelete(WriteBatch* batch, uint32_t column_family_id,
+ const Slice& key);
+
+ static Status DeleteRange(WriteBatch* b, uint32_t column_family_id,
+ const Slice& begin_key, const Slice& end_key);
+
+ static Status DeleteRange(WriteBatch* b, uint32_t column_family_id,
+ const SliceParts& begin_key,
+ const SliceParts& end_key);
+
+ static Status Merge(WriteBatch* batch, uint32_t column_family_id,
+ const Slice& key, const Slice& value);
+
+ static Status Merge(WriteBatch* batch, uint32_t column_family_id,
+ const SliceParts& key, const SliceParts& value);
+
+ static Status MarkEndPrepare(WriteBatch* batch, const Slice& xid);
+
+ static Status MarkRollback(WriteBatch* batch, const Slice& xid);
+
+ static Status MarkCommit(WriteBatch* batch, const Slice& xid);
+
+ static Status InsertNoop(WriteBatch* batch);
+
+ // Return the number of entries in the batch.
+ static int Count(const WriteBatch* batch);
+
+ // Set the count for the number of entries in the batch.
+ static void SetCount(WriteBatch* batch, int n);
+
+ // Return the seqeunce number for the start of this batch.
+ static SequenceNumber Sequence(const WriteBatch* batch);
+
+ // Store the specified number as the seqeunce number for the start of
+ // this batch.
+ static void SetSequence(WriteBatch* batch, SequenceNumber seq);
+
+ // Returns the offset of the first entry in the batch.
+ // This offset is only valid if the batch is not empty.
+ static size_t GetFirstOffset(WriteBatch* batch);
+
+ static Slice Contents(const WriteBatch* batch) {
+ return Slice(batch->rep_);
+ }
+
+ static size_t ByteSize(const WriteBatch* batch) {
+ return batch->rep_.size();
+ }
+
+ static Status SetContents(WriteBatch* batch, const Slice& contents);
+
+ // Inserts batches[i] into memtable, for i in 0..num_batches-1 inclusive.
+ //
+ // If ignore_missing_column_families == true. WriteBatch
+ // referencing non-existing column family will be ignored.
+ // If ignore_missing_column_families == false, processing of the
+ // batches will be stopped if a reference is found to a non-existing
+ // column family and InvalidArgument() will be returned. The writes
+ // in batches may be only partially applied at that point.
+ //
+ // If log_number is non-zero, the memtable will be updated only if
+ // memtables->GetLogNumber() >= log_number.
+ //
+ // If flush_scheduler is non-null, it will be invoked if the memtable
+ // should be flushed.
+ //
+ // Under concurrent use, the caller is responsible for making sure that
+ // the memtables object itself is thread-local.
+ static Status InsertInto(WriteThread::WriteGroup& write_group,
+ SequenceNumber sequence,
+ ColumnFamilyMemTables* memtables,
+ FlushScheduler* flush_scheduler,
+ bool ignore_missing_column_families = false,
+ uint64_t log_number = 0, DB* db = nullptr,
+ bool concurrent_memtable_writes = false);
+
+ // Convenience form of InsertInto when you have only one batch
+ // last_seq_used returns the last sequnce number used in a MemTable insert
+ static Status InsertInto(const WriteBatch* batch,
+ ColumnFamilyMemTables* memtables,
+ FlushScheduler* flush_scheduler,
+ bool ignore_missing_column_families = false,
+ uint64_t log_number = 0, DB* db = nullptr,
+ bool concurrent_memtable_writes = false,
+ SequenceNumber* last_seq_used = nullptr,
+ bool* has_valid_writes = nullptr);
+
+ static Status InsertInto(WriteThread::Writer* writer, SequenceNumber sequence,
+ ColumnFamilyMemTables* memtables,
+ FlushScheduler* flush_scheduler,
+ bool ignore_missing_column_families = false,
+ uint64_t log_number = 0, DB* db = nullptr,
+ bool concurrent_memtable_writes = false);
+
+ static Status Append(WriteBatch* dst, const WriteBatch* src,
+ const bool WAL_only = false);
+
+ // Returns the byte size of appending a WriteBatch with ByteSize
+ // leftByteSize and a WriteBatch with ByteSize rightByteSize
+ static size_t AppendedByteSize(size_t leftByteSize, size_t rightByteSize);
+};
+
+// LocalSavePoint is similar to a scope guard
+class LocalSavePoint {
+ public:
+ explicit LocalSavePoint(WriteBatch* batch)
+ : batch_(batch),
+ savepoint_(batch->GetDataSize(), batch->Count(),
+ batch->content_flags_.load(std::memory_order_relaxed))
+#ifndef NDEBUG
+ ,
+ committed_(false)
+#endif
+ {
+ }
+
+#ifndef NDEBUG
+ ~LocalSavePoint() { assert(committed_); }
+#endif
+ Status commit() {
+#ifndef NDEBUG
+ committed_ = true;
+#endif
+ if (batch_->max_bytes_ && batch_->rep_.size() > batch_->max_bytes_) {
+ batch_->rep_.resize(savepoint_.size);
+ WriteBatchInternal::SetCount(batch_, savepoint_.count);
+ batch_->content_flags_.store(savepoint_.content_flags,
+ std::memory_order_relaxed);
+ return Status::MemoryLimit();
+ }
+ return Status::OK();
+ }
+
+ private:
+ WriteBatch* batch_;
+ SavePoint savepoint_;
+#ifndef NDEBUG
+ bool committed_;
+#endif
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/write_callback.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/write_callback.h b/thirdparty/rocksdb/db/write_callback.h
new file mode 100644
index 0000000..6517a7c
--- /dev/null
+++ b/thirdparty/rocksdb/db/write_callback.h
@@ -0,0 +1,27 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include "rocksdb/status.h"
+
+namespace rocksdb {
+
+class DB;
+
+class WriteCallback {
+ public:
+ virtual ~WriteCallback() {}
+
+ // Will be called while on the write thread before the write executes. If
+ // this function returns a non-OK status, the write will be aborted and this
+ // status will be returned to the caller of DB::Write().
+ virtual Status Callback(DB* db) = 0;
+
+ // return true if writes with this callback can be batched with other writes
+ virtual bool AllowWriteBatching() = 0;
+};
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/write_controller.cc
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/write_controller.cc b/thirdparty/rocksdb/db/write_controller.cc
new file mode 100644
index 0000000..558aa72
--- /dev/null
+++ b/thirdparty/rocksdb/db/write_controller.cc
@@ -0,0 +1,128 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "db/write_controller.h"
+
+#include <atomic>
+#include <cassert>
+#include <ratio>
+#include "rocksdb/env.h"
+
+namespace rocksdb {
+
+std::unique_ptr<WriteControllerToken> WriteController::GetStopToken() {
+ ++total_stopped_;
+ return std::unique_ptr<WriteControllerToken>(new StopWriteToken(this));
+}
+
+std::unique_ptr<WriteControllerToken> WriteController::GetDelayToken(
+ uint64_t write_rate) {
+ total_delayed_++;
+ // Reset counters.
+ last_refill_time_ = 0;
+ bytes_left_ = 0;
+ set_delayed_write_rate(write_rate);
+ return std::unique_ptr<WriteControllerToken>(new DelayWriteToken(this));
+}
+
+std::unique_ptr<WriteControllerToken>
+WriteController::GetCompactionPressureToken() {
+ ++total_compaction_pressure_;
+ return std::unique_ptr<WriteControllerToken>(
+ new CompactionPressureToken(this));
+}
+
+bool WriteController::IsStopped() const {
+ return total_stopped_.load(std::memory_order_relaxed) > 0;
+}
+// This is inside DB mutex, so we can't sleep and need to minimize
+// frequency to get time.
+// If it turns out to be a performance issue, we can redesign the thread
+// synchronization model here.
+// The function trust caller will sleep micros returned.
+uint64_t WriteController::GetDelay(Env* env, uint64_t num_bytes) {
+ if (total_stopped_.load(std::memory_order_relaxed) > 0) {
+ return 0;
+ }
+ if (total_delayed_.load(std::memory_order_relaxed) == 0) {
+ return 0;
+ }
+
+ const uint64_t kMicrosPerSecond = 1000000;
+ const uint64_t kRefillInterval = 1024U;
+
+ if (bytes_left_ >= num_bytes) {
+ bytes_left_ -= num_bytes;
+ return 0;
+ }
+ // The frequency to get time inside DB mutex is less than one per refill
+ // interval.
+ auto time_now = NowMicrosMonotonic(env);
+
+ uint64_t sleep_debt = 0;
+ uint64_t time_since_last_refill = 0;
+ if (last_refill_time_ != 0) {
+ if (last_refill_time_ > time_now) {
+ sleep_debt = last_refill_time_ - time_now;
+ } else {
+ time_since_last_refill = time_now - last_refill_time_;
+ bytes_left_ +=
+ static_cast<uint64_t>(static_cast<double>(time_since_last_refill) /
+ kMicrosPerSecond * delayed_write_rate_);
+ if (time_since_last_refill >= kRefillInterval &&
+ bytes_left_ > num_bytes) {
+ // If refill interval already passed and we have enough bytes
+ // return without extra sleeping.
+ last_refill_time_ = time_now;
+ bytes_left_ -= num_bytes;
+ return 0;
+ }
+ }
+ }
+
+ uint64_t single_refill_amount =
+ delayed_write_rate_ * kRefillInterval / kMicrosPerSecond;
+ if (bytes_left_ + single_refill_amount >= num_bytes) {
+ // Wait until a refill interval
+ // Never trigger expire for less than one refill interval to avoid to get
+ // time.
+ bytes_left_ = bytes_left_ + single_refill_amount - num_bytes;
+ last_refill_time_ = time_now + kRefillInterval;
+ return kRefillInterval + sleep_debt;
+ }
+
+ // Need to refill more than one interval. Need to sleep longer. Check
+ // whether expiration will hit
+
+ // Sleep just until `num_bytes` is allowed.
+ uint64_t sleep_amount =
+ static_cast<uint64_t>(num_bytes /
+ static_cast<long double>(delayed_write_rate_) *
+ kMicrosPerSecond) +
+ sleep_debt;
+ last_refill_time_ = time_now + sleep_amount;
+ return sleep_amount;
+}
+
+uint64_t WriteController::NowMicrosMonotonic(Env* env) {
+ return env->NowNanos() / std::milli::den;
+}
+
+StopWriteToken::~StopWriteToken() {
+ assert(controller_->total_stopped_ >= 1);
+ --controller_->total_stopped_;
+}
+
+DelayWriteToken::~DelayWriteToken() {
+ controller_->total_delayed_--;
+ assert(controller_->total_delayed_.load() >= 0);
+}
+
+CompactionPressureToken::~CompactionPressureToken() {
+ controller_->total_compaction_pressure_--;
+ assert(controller_->total_compaction_pressure_ >= 0);
+}
+
+} // namespace rocksdb
http://git-wip-us.apache.org/repos/asf/nifi-minifi-cpp/blob/48867732/thirdparty/rocksdb/db/write_controller.h
----------------------------------------------------------------------
diff --git a/thirdparty/rocksdb/db/write_controller.h b/thirdparty/rocksdb/db/write_controller.h
new file mode 100644
index 0000000..7c301ce
--- /dev/null
+++ b/thirdparty/rocksdb/db/write_controller.h
@@ -0,0 +1,144 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <stdint.h>
+
+#include <atomic>
+#include <memory>
+#include "rocksdb/rate_limiter.h"
+
+namespace rocksdb {
+
+class Env;
+class WriteControllerToken;
+
+// WriteController is controlling write stalls in our write code-path. Write
+// stalls happen when compaction can't keep up with write rate.
+// All of the methods here (including WriteControllerToken's destructors) need
+// to be called while holding DB mutex
+class WriteController {
+ public:
+ explicit WriteController(uint64_t _delayed_write_rate = 1024u * 1024u * 32u,
+ int64_t low_pri_rate_bytes_per_sec = 1024 * 1024)
+ : total_stopped_(0),
+ total_delayed_(0),
+ total_compaction_pressure_(0),
+ bytes_left_(0),
+ last_refill_time_(0),
+ low_pri_rate_limiter_(
+ NewGenericRateLimiter(low_pri_rate_bytes_per_sec)) {
+ set_max_delayed_write_rate(_delayed_write_rate);
+ }
+ ~WriteController() = default;
+
+ // When an actor (column family) requests a stop token, all writes will be
+ // stopped until the stop token is released (deleted)
+ std::unique_ptr<WriteControllerToken> GetStopToken();
+ // When an actor (column family) requests a delay token, total delay for all
+ // writes to the DB will be controlled under the delayed write rate. Every
+ // write needs to call GetDelay() with number of bytes writing to the DB,
+ // which returns number of microseconds to sleep.
+ std::unique_ptr<WriteControllerToken> GetDelayToken(
+ uint64_t delayed_write_rate);
+ // When an actor (column family) requests a moderate token, compaction
+ // threads will be increased
+ std::unique_ptr<WriteControllerToken> GetCompactionPressureToken();
+
+ // these three metods are querying the state of the WriteController
+ bool IsStopped() const;
+ bool NeedsDelay() const { return total_delayed_.load() > 0; }
+ bool NeedSpeedupCompaction() const {
+ return IsStopped() || NeedsDelay() || total_compaction_pressure_ > 0;
+ }
+ // return how many microseconds the caller needs to sleep after the call
+ // num_bytes: how many number of bytes to put into the DB.
+ // Prerequisite: DB mutex held.
+ uint64_t GetDelay(Env* env, uint64_t num_bytes);
+ void set_delayed_write_rate(uint64_t write_rate) {
+ // avoid divide 0
+ if (write_rate == 0) {
+ write_rate = 1u;
+ } else if (write_rate > max_delayed_write_rate()) {
+ write_rate = max_delayed_write_rate();
+ }
+ delayed_write_rate_ = write_rate;
+ }
+
+ void set_max_delayed_write_rate(uint64_t write_rate) {
+ // avoid divide 0
+ if (write_rate == 0) {
+ write_rate = 1u;
+ }
+ max_delayed_write_rate_ = write_rate;
+ // update delayed_write_rate_ as well
+ delayed_write_rate_ = write_rate;
+ }
+
+ uint64_t delayed_write_rate() const { return delayed_write_rate_; }
+
+ uint64_t max_delayed_write_rate() const { return max_delayed_write_rate_; }
+
+ RateLimiter* low_pri_rate_limiter() { return low_pri_rate_limiter_.get(); }
+
+ private:
+ uint64_t NowMicrosMonotonic(Env* env);
+
+ friend class WriteControllerToken;
+ friend class StopWriteToken;
+ friend class DelayWriteToken;
+ friend class CompactionPressureToken;
+
+ std::atomic<int> total_stopped_;
+ std::atomic<int> total_delayed_;
+ std::atomic<int> total_compaction_pressure_;
+ uint64_t bytes_left_;
+ uint64_t last_refill_time_;
+ // write rate set when initialization or by `DBImpl::SetDBOptions`
+ uint64_t max_delayed_write_rate_;
+ // current write rate
+ uint64_t delayed_write_rate_;
+
+ std::unique_ptr<RateLimiter> low_pri_rate_limiter_;
+};
+
+class WriteControllerToken {
+ public:
+ explicit WriteControllerToken(WriteController* controller)
+ : controller_(controller) {}
+ virtual ~WriteControllerToken() {}
+
+ protected:
+ WriteController* controller_;
+
+ private:
+ // no copying allowed
+ WriteControllerToken(const WriteControllerToken&) = delete;
+ void operator=(const WriteControllerToken&) = delete;
+};
+
+class StopWriteToken : public WriteControllerToken {
+ public:
+ explicit StopWriteToken(WriteController* controller)
+ : WriteControllerToken(controller) {}
+ virtual ~StopWriteToken();
+};
+
+class DelayWriteToken : public WriteControllerToken {
+ public:
+ explicit DelayWriteToken(WriteController* controller)
+ : WriteControllerToken(controller) {}
+ virtual ~DelayWriteToken();
+};
+
+class CompactionPressureToken : public WriteControllerToken {
+ public:
+ explicit CompactionPressureToken(WriteController* controller)
+ : WriteControllerToken(controller) {}
+ virtual ~CompactionPressureToken();
+};
+
+} // namespace rocksdb