You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wh...@apache.org on 2016/01/05 20:52:28 UTC
[28/50] [abbrv] hadoop git commit: [partial-ns] Import HDFSDB.
[partial-ns] Import HDFSDB.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a6419f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a6419f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a6419f4
Branch: refs/heads/feature-HDFS-8286
Commit: 4a6419f4ac5e84bcafe6bfaa6cc5663cfc8283ad
Parents: 36cdcd7
Author: Haohui Mai <wh...@apache.org>
Authored: Thu Sep 18 15:41:58 2014 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Fri Jun 12 13:56:59 2015 -0700
----------------------------------------------------------------------
.../src/main/native/hdfsdb/AUTHORS | 12 +
.../src/main/native/hdfsdb/LICENSE | 27 +
.../src/main/native/hdfsdb/Makefile | 215 ++
.../hadoop-hdfsdb/src/main/native/hdfsdb/NEWS | 17 +
.../hadoop-hdfsdb/src/main/native/hdfsdb/README | 51 +
.../src/main/native/hdfsdb/README.hdfsdb | 6 +
.../src/main/native/hdfsdb/README.hdfsdb~ | 1 +
.../hadoop-hdfsdb/src/main/native/hdfsdb/TODO | 14 +
.../main/native/hdfsdb/build_detect_platform | 221 ++
.../main/native/hdfsdb/db/autocompact_test.cc | 118 +
.../src/main/native/hdfsdb/db/builder.cc | 88 +
.../src/main/native/hdfsdb/db/builder.h | 34 +
.../src/main/native/hdfsdb/db/c.cc | 595 +++++
.../src/main/native/hdfsdb/db/c_test.c | 390 ++++
.../main/native/hdfsdb/db/corruption_test.cc | 374 +++
.../src/main/native/hdfsdb/db/db_bench.cc | 979 ++++++++
.../src/main/native/hdfsdb/db/db_impl.cc | 1513 +++++++++++++
.../src/main/native/hdfsdb/db/db_impl.h | 211 ++
.../src/main/native/hdfsdb/db/db_iter.cc | 317 +++
.../src/main/native/hdfsdb/db/db_iter.h | 28 +
.../src/main/native/hdfsdb/db/db_test.cc | 2128 ++++++++++++++++++
.../src/main/native/hdfsdb/db/dbformat.cc | 140 ++
.../src/main/native/hdfsdb/db/dbformat.h | 230 ++
.../src/main/native/hdfsdb/db/dbformat_test.cc | 112 +
.../src/main/native/hdfsdb/db/filename.cc | 144 ++
.../src/main/native/hdfsdb/db/filename.h | 85 +
.../src/main/native/hdfsdb/db/filename_test.cc | 123 +
.../src/main/native/hdfsdb/db/leveldb_main.cc | 238 ++
.../src/main/native/hdfsdb/db/log_format.h | 35 +
.../src/main/native/hdfsdb/db/log_reader.cc | 266 +++
.../src/main/native/hdfsdb/db/log_reader.h | 108 +
.../src/main/native/hdfsdb/db/log_test.cc | 530 +++++
.../src/main/native/hdfsdb/db/log_writer.cc | 103 +
.../src/main/native/hdfsdb/db/log_writer.h | 48 +
.../src/main/native/hdfsdb/db/memtable.cc | 145 ++
.../src/main/native/hdfsdb/db/memtable.h | 91 +
.../src/main/native/hdfsdb/db/repair.cc | 461 ++++
.../src/main/native/hdfsdb/db/skiplist.h | 379 ++++
.../src/main/native/hdfsdb/db/skiplist_test.cc | 378 ++++
.../src/main/native/hdfsdb/db/snapshot.h | 66 +
.../src/main/native/hdfsdb/db/table_cache.cc | 127 ++
.../src/main/native/hdfsdb/db/table_cache.h | 61 +
.../src/main/native/hdfsdb/db/version_edit.cc | 266 +++
.../src/main/native/hdfsdb/db/version_edit.h | 107 +
.../main/native/hdfsdb/db/version_edit_test.cc | 46 +
.../src/main/native/hdfsdb/db/version_set.cc | 1484 ++++++++++++
.../src/main/native/hdfsdb/db/version_set.h | 396 ++++
.../main/native/hdfsdb/db/version_set_test.cc | 179 ++
.../src/main/native/hdfsdb/db/write_batch.cc | 147 ++
.../native/hdfsdb/db/write_batch_internal.h | 49 +
.../main/native/hdfsdb/db/write_batch_test.cc | 120 +
.../native/hdfsdb/doc/bench/db_bench_sqlite3.cc | 718 ++++++
.../native/hdfsdb/doc/bench/db_bench_tree_db.cc | 528 +++++
.../src/main/native/hdfsdb/doc/benchmark.html | 459 ++++
.../src/main/native/hdfsdb/doc/doc.css | 89 +
.../src/main/native/hdfsdb/doc/impl.html | 213 ++
.../src/main/native/hdfsdb/doc/index.html | 549 +++++
.../src/main/native/hdfsdb/doc/log_format.txt | 75 +
.../src/main/native/hdfsdb/doc/table_format.txt | 104 +
.../main/native/hdfsdb/helpers/memenv/memenv.cc | 384 ++++
.../main/native/hdfsdb/helpers/memenv/memenv.h | 20 +
.../native/hdfsdb/helpers/memenv/memenv_test.cc | 232 ++
.../src/main/native/hdfsdb/include/leveldb/c.h | 290 +++
.../main/native/hdfsdb/include/leveldb/cache.h | 99 +
.../native/hdfsdb/include/leveldb/comparator.h | 63 +
.../src/main/native/hdfsdb/include/leveldb/db.h | 161 ++
.../main/native/hdfsdb/include/leveldb/env.h | 333 +++
.../hdfsdb/include/leveldb/filter_policy.h | 70 +
.../native/hdfsdb/include/leveldb/iterator.h | 100 +
.../native/hdfsdb/include/leveldb/options.h | 195 ++
.../main/native/hdfsdb/include/leveldb/slice.h | 109 +
.../main/native/hdfsdb/include/leveldb/status.h | 106 +
.../main/native/hdfsdb/include/leveldb/table.h | 85 +
.../hdfsdb/include/leveldb/table_builder.h | 92 +
.../native/hdfsdb/include/leveldb/write_batch.h | 64 +
.../main/native/hdfsdb/issues/issue178_test.cc | 92 +
.../main/native/hdfsdb/issues/issue200_test.cc | 59 +
.../src/main/native/hdfsdb/port/README | 10 +
.../main/native/hdfsdb/port/atomic_pointer.h | 224 ++
.../src/main/native/hdfsdb/port/port.h | 19 +
.../src/main/native/hdfsdb/port/port_example.h | 135 ++
.../src/main/native/hdfsdb/port/port_posix.cc | 54 +
.../src/main/native/hdfsdb/port/port_posix.h | 157 ++
.../native/hdfsdb/port/thread_annotations.h | 59 +
.../src/main/native/hdfsdb/port/win/stdint.h | 24 +
.../src/main/native/hdfsdb/table/block.cc | 268 +++
.../src/main/native/hdfsdb/table/block.h | 44 +
.../main/native/hdfsdb/table/block_builder.cc | 109 +
.../main/native/hdfsdb/table/block_builder.h | 57 +
.../main/native/hdfsdb/table/filter_block.cc | 111 +
.../src/main/native/hdfsdb/table/filter_block.h | 68 +
.../native/hdfsdb/table/filter_block_test.cc | 128 ++
.../src/main/native/hdfsdb/table/format.cc | 145 ++
.../src/main/native/hdfsdb/table/format.h | 108 +
.../src/main/native/hdfsdb/table/iterator.cc | 67 +
.../main/native/hdfsdb/table/iterator_wrapper.h | 63 +
.../src/main/native/hdfsdb/table/merger.cc | 197 ++
.../src/main/native/hdfsdb/table/merger.h | 26 +
.../src/main/native/hdfsdb/table/table.cc | 275 +++
.../main/native/hdfsdb/table/table_builder.cc | 270 +++
.../src/main/native/hdfsdb/table/table_test.cc | 868 +++++++
.../native/hdfsdb/table/two_level_iterator.cc | 182 ++
.../native/hdfsdb/table/two_level_iterator.h | 34 +
.../src/main/native/hdfsdb/util/arena.cc | 68 +
.../src/main/native/hdfsdb/util/arena.h | 68 +
.../src/main/native/hdfsdb/util/arena_test.cc | 68 +
.../src/main/native/hdfsdb/util/bloom.cc | 95 +
.../src/main/native/hdfsdb/util/bloom_test.cc | 161 ++
.../src/main/native/hdfsdb/util/cache.cc | 325 +++
.../src/main/native/hdfsdb/util/cache_test.cc | 186 ++
.../src/main/native/hdfsdb/util/coding.cc | 194 ++
.../src/main/native/hdfsdb/util/coding.h | 104 +
.../src/main/native/hdfsdb/util/coding_test.cc | 196 ++
.../src/main/native/hdfsdb/util/comparator.cc | 81 +
.../src/main/native/hdfsdb/util/crc32c.cc | 332 +++
.../src/main/native/hdfsdb/util/crc32c.h | 45 +
.../src/main/native/hdfsdb/util/crc32c_test.cc | 72 +
.../src/main/native/hdfsdb/util/env.cc | 96 +
.../src/main/native/hdfsdb/util/env_posix.cc | 607 +++++
.../src/main/native/hdfsdb/util/env_test.cc | 104 +
.../main/native/hdfsdb/util/filter_policy.cc | 11 +
.../src/main/native/hdfsdb/util/hash.cc | 52 +
.../src/main/native/hdfsdb/util/hash.h | 19 +
.../src/main/native/hdfsdb/util/histogram.cc | 139 ++
.../src/main/native/hdfsdb/util/histogram.h | 42 +
.../src/main/native/hdfsdb/util/logging.cc | 81 +
.../src/main/native/hdfsdb/util/logging.h | 47 +
.../src/main/native/hdfsdb/util/mutexlock.h | 41 +
.../src/main/native/hdfsdb/util/options.cc | 29 +
.../src/main/native/hdfsdb/util/posix_logger.h | 98 +
.../src/main/native/hdfsdb/util/random.h | 64 +
.../src/main/native/hdfsdb/util/status.cc | 75 +
.../src/main/native/hdfsdb/util/testharness.cc | 77 +
.../src/main/native/hdfsdb/util/testharness.h | 138 ++
.../src/main/native/hdfsdb/util/testutil.cc | 51 +
.../src/main/native/hdfsdb/util/testutil.h | 53 +
136 files changed, 26813 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/AUTHORS
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/AUTHORS b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/AUTHORS
new file mode 100644
index 0000000..2439d7a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/AUTHORS
@@ -0,0 +1,12 @@
+# Names should be added to this file like so:
+# Name or Organization <email address>
+
+Google Inc.
+
+# Initial version authors:
+Jeffrey Dean <je...@google.com>
+Sanjay Ghemawat <sa...@google.com>
+
+# Partial list of contributors:
+Kevin Regan <ke...@gmail.com>
+Johan Bilien <jo...@litl.com>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/LICENSE
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/LICENSE b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/LICENSE
new file mode 100644
index 0000000..8e80208
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/LICENSE
@@ -0,0 +1,27 @@
+Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/Makefile
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/Makefile b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/Makefile
new file mode 100644
index 0000000..f8903b6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/Makefile
@@ -0,0 +1,215 @@
+# Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#-----------------------------------------------
+# Uncomment exactly one of the lines labelled (A), (B), and (C) below
+# to switch between compilation modes.
+
+OPT ?= -O2 -DNDEBUG # (A) Production use (optimized mode)
+# OPT ?= -g2 # (B) Debug mode, w/ full line-level debugging symbols
+# OPT ?= -O2 -g2 -DNDEBUG # (C) Profiling mode: opt, but w/debugging symbols
+#-----------------------------------------------
+
+# detect what platform we're building on
+$(shell CC="$(CC)" CXX="$(CXX)" TARGET_OS="$(TARGET_OS)" \
+ ./build_detect_platform build_config.mk ./)
+# this file is generated by the previous line to set build flags and sources
+include build_config.mk
+
+CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
+CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT)
+
+LDFLAGS += $(PLATFORM_LDFLAGS)
+LIBS += $(PLATFORM_LIBS)
+
+LIBOBJECTS = $(SOURCES:.cc=.o)
+MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
+
+TESTUTIL = ./util/testutil.o
+TESTHARNESS = ./util/testharness.o $(TESTUTIL)
+
+TESTS = \
+ arena_test \
+ autocompact_test \
+ bloom_test \
+ c_test \
+ cache_test \
+ coding_test \
+ corruption_test \
+ crc32c_test \
+ db_test \
+ dbformat_test \
+ env_test \
+ filename_test \
+ filter_block_test \
+ issue178_test \
+ issue200_test \
+ log_test \
+ memenv_test \
+ skiplist_test \
+ table_test \
+ version_edit_test \
+ version_set_test \
+ write_batch_test
+
+PROGRAMS = db_bench leveldbutil $(TESTS)
+BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
+
+LIBRARY = libleveldb.a
+MEMENVLIBRARY = libmemenv.a
+
+default: all
+
+# Should we build shared libraries?
+ifneq ($(PLATFORM_SHARED_EXT),)
+
+ifneq ($(PLATFORM_SHARED_VERSIONED),true)
+SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
+SHARED2 = $(SHARED1)
+SHARED3 = $(SHARED1)
+SHARED = $(SHARED1)
+else
+# Update db.h if you change these.
+SHARED_MAJOR = 1
+SHARED_MINOR = 17
+SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
+SHARED2 = $(SHARED1).$(SHARED_MAJOR)
+SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
+SHARED = $(SHARED1) $(SHARED2) $(SHARED3)
+$(SHARED1): $(SHARED3)
+ ln -fs $(SHARED3) $(SHARED1)
+$(SHARED2): $(SHARED3)
+ ln -fs $(SHARED3) $(SHARED2)
+endif
+
+$(SHARED3):
+ $(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3) $(LIBS)
+
+endif # PLATFORM_SHARED_EXT
+
+all: $(SHARED) $(LIBRARY)
+
+check: all $(PROGRAMS) $(TESTS)
+ for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
+
+clean:
+ -rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o build_config.mk
+ -rm -rf ios-x86/* ios-arm/*
+
+$(LIBRARY): $(LIBOBJECTS)
+ rm -f $@
+ $(AR) -rs $@ $(LIBOBJECTS)
+
+db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL)
+ $(CXX) $(LDFLAGS) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS)
+
+db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL)
+ $(CXX) $(LDFLAGS) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS)
+
+db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL)
+ $(CXX) $(LDFLAGS) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS)
+
+leveldbutil: db/leveldb_main.o $(LIBOBJECTS)
+ $(CXX) $(LDFLAGS) db/leveldb_main.o $(LIBOBJECTS) -o $@ $(LIBS)
+
+arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+autocompact_test: db/autocompact_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) db/autocompact_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+corruption_test: db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+issue178_test: issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+issue200_test: issues/issue200_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) issues/issue200_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(MEMENVLIBRARY) : $(MEMENVOBJECTS)
+ rm -f $@
+ $(AR) -rs $@ $(MEMENVOBJECTS)
+
+memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS)
+ $(CXX) $(LDFLAGS) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LIBS)
+
+ifeq ($(PLATFORM), IOS)
+# For iOS, create universal object files to be used on both the simulator and
+# a device.
+PLATFORMSROOT=/Applications/Xcode.app/Contents/Developer/Platforms
+SIMULATORROOT=$(PLATFORMSROOT)/iPhoneSimulator.platform/Developer
+DEVICEROOT=$(PLATFORMSROOT)/iPhoneOS.platform/Developer
+IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBundleShortVersionString)
+IOSARCH=-arch armv6 -arch armv7 -arch armv7s -arch arm64
+
+.cc.o:
+ mkdir -p ios-x86/$(dir $@)
+ $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@
+ mkdir -p ios-arm/$(dir $@)
+ xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@
+ lipo ios-x86/$@ ios-arm/$@ -create -output $@
+
+.c.o:
+ mkdir -p ios-x86/$(dir $@)
+ $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@
+ mkdir -p ios-arm/$(dir $@)
+ xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@
+ lipo ios-x86/$@ ios-arm/$@ -create -output $@
+
+else
+.cc.o:
+ $(CXX) $(CXXFLAGS) -c $< -o $@
+
+.c.o:
+ $(CC) $(CFLAGS) -c $< -o $@
+endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/NEWS
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/NEWS b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/NEWS
new file mode 100644
index 0000000..3fd9924
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/NEWS
@@ -0,0 +1,17 @@
+Release 1.2 2011-05-16
+----------------------
+
+Fixes for larger databases (tested up to one billion 100-byte entries,
+i.e., ~100GB).
+
+(1) Place hard limit on number of level-0 files. This fixes errors
+of the form "too many open files".
+
+(2) Fixed memtable management. Before the fix, a heavy write burst
+could cause unbounded memory usage.
+
+A fix for a logging bug where the reader would incorrectly complain
+about corruption.
+
+Allow public access to WriteBatch contents so that users can easily
+wrap a DB.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README
new file mode 100644
index 0000000..3618ade
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README
@@ -0,0 +1,51 @@
+leveldb: A key-value store
+Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
+
+The code under this directory implements a system for maintaining a
+persistent key/value store.
+
+See doc/index.html for more explanation.
+See doc/impl.html for a brief overview of the implementation.
+
+The public interface is in include/*.h. Callers should not include or
+rely on the details of any other header files in this package. Those
+internal APIs may be changed without warning.
+
+Guide to header files:
+
+include/db.h
+ Main interface to the DB: Start here
+
+include/options.h
+ Control over the behavior of an entire database, and also
+ control over the behavior of individual reads and writes.
+
+include/comparator.h
+ Abstraction for user-specified comparison function. If you want
+ just bytewise comparison of keys, you can use the default comparator,
+ but clients can write their own comparator implementations if they
+ want custom ordering (e.g. to handle different character
+ encodings, etc.)
+
+include/iterator.h
+ Interface for iterating over data. You can get an iterator
+ from a DB object.
+
+include/write_batch.h
+ Interface for atomically applying multiple updates to a database.
+
+include/slice.h
+ A simple module for maintaining a pointer and a length into some
+ other byte array.
+
+include/status.h
+ Status is returned from many of the public interfaces and is used
+ to report success and various kinds of errors.
+
+include/env.h
+ Abstraction of the OS environment. A posix implementation of
+ this interface is in util/env_posix.cc
+
+include/table.h
+include/table_builder.h
+ Lower-level modules that most clients probably won't use directly
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README.hdfsdb
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README.hdfsdb b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README.hdfsdb
new file mode 100644
index 0000000..ae9a68d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README.hdfsdb
@@ -0,0 +1,6 @@
+HDFSDB is a key-value store derived from Google's LevelDB in order to
+meet the specific needs of persisting the namespace of HDFS.
+
+HDFS is released under the same license of the LevelDB (i.e., the
+3-clause BSD license). Please see the original LICENSE for more
+details.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README.hdfsdb~
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README.hdfsdb~ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README.hdfsdb~
new file mode 100644
index 0000000..f51157b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/README.hdfsdb~
@@ -0,0 +1 @@
+HDFSDB is derived from Google's LevelDB to meet
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/TODO
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/TODO b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/TODO
new file mode 100644
index 0000000..e603c07
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/TODO
@@ -0,0 +1,14 @@
+ss
+- Stats
+
+db
+- Maybe implement DB::BulkDeleteForRange(start_key, end_key)
+ that would blow away files whose ranges are entirely contained
+ within [start_key..end_key]? For Chrome, deletion of obsolete
+ object stores, etc. can be done in the background anyway, so
+ probably not that important.
+- There have been requests for MultiGet.
+
+After a range is completely deleted, what gets rid of the
+corresponding files if we do no future changes to that range. Make
+the conditions for triggering compactions fire in more situations?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/build_detect_platform
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/build_detect_platform b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/build_detect_platform
new file mode 100755
index 0000000..6e59c6f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/build_detect_platform
@@ -0,0 +1,221 @@
+#!/bin/sh
+#
+# Detects OS we're compiling on and outputs a file specified by the first
+# argument, which in turn gets read while processing Makefile.
+#
+# The output will set the following variables:
+# CC C Compiler path
+# CXX C++ Compiler path
+# PLATFORM_LDFLAGS Linker flags
+# PLATFORM_LIBS Libraries flags
+# PLATFORM_SHARED_EXT Extension for shared libraries
+# PLATFORM_SHARED_LDFLAGS Flags for building shared library
+# This flag is embedded just before the name
+# of the shared library without intervening spaces
+# PLATFORM_SHARED_CFLAGS Flags for compiling objects for shared library
+# PLATFORM_CCFLAGS C compiler flags
+# PLATFORM_CXXFLAGS C++ compiler flags. Will contain:
+# PLATFORM_SHARED_VERSIONED Set to 'true' if platform supports versioned
+# shared libraries, empty otherwise.
+#
+# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
+#
+# -DLEVELDB_CSTDATOMIC_PRESENT if <cstdatomic> is present
+# -DLEVELDB_PLATFORM_POSIX for Posix-based platforms
+# -DSNAPPY if the Snappy library is present
+#
+
+OUTPUT=$1
+PREFIX=$2
+if test -z "$OUTPUT" || test -z "$PREFIX"; then
+ echo "usage: $0 <output-filename> <directory_prefix>" >&2
+ exit 1
+fi
+
+# Delete existing output, if it exists
+rm -f $OUTPUT
+touch $OUTPUT
+
+if test -z "$CC"; then
+ CC=cc
+fi
+
+if test -z "$CXX"; then
+ CXX=g++
+fi
+
+if test -z "$TMPDIR"; then
+ TMPDIR=/tmp
+fi
+
+# Detect OS
+if test -z "$TARGET_OS"; then
+ TARGET_OS=`uname -s`
+fi
+
+COMMON_FLAGS=
+CROSS_COMPILE=
+PLATFORM_CCFLAGS=
+PLATFORM_CXXFLAGS=
+PLATFORM_LDFLAGS=
+PLATFORM_LIBS=
+PLATFORM_SHARED_EXT="so"
+PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl,"
+PLATFORM_SHARED_CFLAGS="-fPIC"
+PLATFORM_SHARED_VERSIONED=true
+
+MEMCMP_FLAG=
+if [ "$CXX" = "g++" ]; then
+ # Use libc's memcmp instead of GCC's memcmp. This results in ~40%
+ # performance improvement on readrandom under gcc 4.4.3 on Linux/x86.
+ MEMCMP_FLAG="-fno-builtin-memcmp"
+fi
+
+case "$TARGET_OS" in
+ Darwin)
+ PLATFORM=OS_MACOSX
+ COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
+ PLATFORM_SHARED_EXT=dylib
+ [ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd`
+ PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name $INSTALL_PATH/"
+ PORT_FILE=port/port_posix.cc
+ ;;
+ Linux)
+ PLATFORM=OS_LINUX
+ COMMON_FLAGS="$MEMCMP_FLAG -pthread -DOS_LINUX"
+ PLATFORM_LDFLAGS="-pthread"
+ PORT_FILE=port/port_posix.cc
+ ;;
+ SunOS)
+ PLATFORM=OS_SOLARIS
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_SOLARIS"
+ PLATFORM_LIBS="-lpthread -lrt"
+ PORT_FILE=port/port_posix.cc
+ ;;
+ FreeBSD)
+ PLATFORM=OS_FREEBSD
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_FREEBSD"
+ PLATFORM_LIBS="-lpthread"
+ PORT_FILE=port/port_posix.cc
+ ;;
+ NetBSD)
+ PLATFORM=OS_NETBSD
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_NETBSD"
+ PLATFORM_LIBS="-lpthread -lgcc_s"
+ PORT_FILE=port/port_posix.cc
+ ;;
+ OpenBSD)
+ PLATFORM=OS_OPENBSD
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_OPENBSD"
+ PLATFORM_LDFLAGS="-pthread"
+ PORT_FILE=port/port_posix.cc
+ ;;
+ DragonFly)
+ PLATFORM=OS_DRAGONFLYBSD
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_DRAGONFLYBSD"
+ PLATFORM_LIBS="-lpthread"
+ PORT_FILE=port/port_posix.cc
+ ;;
+ OS_ANDROID_CROSSCOMPILE)
+ PLATFORM=OS_ANDROID
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
+ PLATFORM_LDFLAGS="" # All pthread features are in the Android C library
+ PORT_FILE=port/port_posix.cc
+ CROSS_COMPILE=true
+ ;;
+ HP-UX)
+ PLATFORM=OS_HPUX
+ COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_HPUX"
+ PLATFORM_LDFLAGS="-pthread"
+ PORT_FILE=port/port_posix.cc
+ # man ld: +h internal_name
+ PLATFORM_SHARED_LDFLAGS="-shared -Wl,+h -Wl,"
+ ;;
+ IOS)
+ PLATFORM=IOS
+ COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
+ [ -z "$INSTALL_PATH" ] && INSTALL_PATH=`pwd`
+ PORT_FILE=port/port_posix.cc
+ PLATFORM_SHARED_EXT=
+ PLATFORM_SHARED_LDFLAGS=
+ PLATFORM_SHARED_CFLAGS=
+ PLATFORM_SHARED_VERSIONED=
+ ;;
+ *)
+ echo "Unknown platform!" >&2
+ exit 1
+esac
+
+# We want to make a list of all cc files within util, db, table, and helpers
+# except for the test and benchmark files. By default, find will output a list
+# of all files matching either rule, so we need to append -print to make the
+# prune take effect.
+DIRS="$PREFIX/db $PREFIX/util $PREFIX/table"
+
+set -f # temporarily disable globbing so that our patterns aren't expanded
+PRUNE_TEST="-name *test*.cc -prune"
+PRUNE_BENCH="-name *_bench.cc -prune"
+PRUNE_TOOL="-name leveldb_main.cc -prune"
+PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`
+
+set +f # re-enable globbing
+
+# The sources consist of the portable files, plus the platform-specific port
+# file.
+echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT
+echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT
+
+if [ "$CROSS_COMPILE" = "true" ]; then
+ # Cross-compiling; do not try any compilation tests.
+ true
+else
+ CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$"
+
+ # If -std=c++0x works, use <cstdatomic>. Otherwise use port_posix.h.
+ $CXX $CXXFLAGS -std=c++0x -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
+ #include <cstdatomic>
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_CSTDATOMIC_PRESENT"
+ PLATFORM_CXXFLAGS="-std=c++0x"
+ else
+ COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
+ fi
+
+ # Test whether Snappy library is installed
+ # http://code.google.com/p/snappy/
+ $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
+ #include <snappy.h>
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
+ PLATFORM_LIBS="$PLATFORM_LIBS -lsnappy"
+ fi
+
+ # Test whether tcmalloc is available
+ $CXX $CXXFLAGS -x c++ - -o $CXXOUTPUT -ltcmalloc 2>/dev/null <<EOF
+ int main() {}
+EOF
+ if [ "$?" = 0 ]; then
+ PLATFORM_LIBS="$PLATFORM_LIBS -ltcmalloc"
+ fi
+
+ rm -f $CXXOUTPUT 2>/dev/null
+fi
+
+PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
+PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
+
+echo "CC=$CC" >> $OUTPUT
+echo "CXX=$CXX" >> $OUTPUT
+echo "PLATFORM=$PLATFORM" >> $OUTPUT
+echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT
+echo "PLATFORM_LIBS=$PLATFORM_LIBS" >> $OUTPUT
+echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT
+echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT
+echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
+echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT
+echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT
+echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/autocompact_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/autocompact_test.cc b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/autocompact_test.cc
new file mode 100644
index 0000000..d20a236
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/autocompact_test.cc
@@ -0,0 +1,118 @@
+// Copyright (c) 2013 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "leveldb/db.h"
+#include "db/db_impl.h"
+#include "leveldb/cache.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+
+namespace leveldb {
+
+class AutoCompactTest {
+ public:
+ std::string dbname_;
+ Cache* tiny_cache_;
+ Options options_;
+ DB* db_;
+
+ AutoCompactTest() {
+ dbname_ = test::TmpDir() + "/autocompact_test";
+ tiny_cache_ = NewLRUCache(100);
+ options_.block_cache = tiny_cache_;
+ DestroyDB(dbname_, options_);
+ options_.create_if_missing = true;
+ options_.compression = kNoCompression;
+ ASSERT_OK(DB::Open(options_, dbname_, &db_));
+ }
+
+ ~AutoCompactTest() {
+ delete db_;
+ DestroyDB(dbname_, Options());
+ delete tiny_cache_;
+ }
+
+ std::string Key(int i) {
+ char buf[100];
+ snprintf(buf, sizeof(buf), "key%06d", i);
+ return std::string(buf);
+ }
+
+ uint64_t Size(const Slice& start, const Slice& limit) {
+ Range r(start, limit);
+ uint64_t size;
+ db_->GetApproximateSizes(&r, 1, &size);
+ return size;
+ }
+
+ void DoReads(int n);
+};
+
+static const int kValueSize = 200 * 1024;
+static const int kTotalSize = 100 * 1024 * 1024;
+static const int kCount = kTotalSize / kValueSize;
+
+// Read through the first n keys repeatedly and check that they get
+// compacted (verified by checking the size of the key space).
+void AutoCompactTest::DoReads(int n) {
+ std::string value(kValueSize, 'x');
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
+
+ // Fill database
+ for (int i = 0; i < kCount; i++) {
+ ASSERT_OK(db_->Put(WriteOptions(), Key(i), value));
+ }
+ ASSERT_OK(dbi->TEST_CompactMemTable());
+
+ // Delete everything
+ for (int i = 0; i < kCount; i++) {
+ ASSERT_OK(db_->Delete(WriteOptions(), Key(i)));
+ }
+ ASSERT_OK(dbi->TEST_CompactMemTable());
+
+ // Get initial measurement of the space we will be reading.
+ const int64_t initial_size = Size(Key(0), Key(n));
+ const int64_t initial_other_size = Size(Key(n), Key(kCount));
+
+ // Read until size drops significantly.
+ std::string limit_key = Key(n);
+ for (int read = 0; true; read++) {
+ ASSERT_LT(read, 100) << "Taking too long to compact";
+ Iterator* iter = db_->NewIterator(ReadOptions());
+ for (iter->SeekToFirst();
+ iter->Valid() && iter->key().ToString() < limit_key;
+ iter->Next()) {
+ // Drop data
+ }
+ delete iter;
+ // Wait a little bit to allow any triggered compactions to complete.
+ Env::Default()->SleepForMicroseconds(1000000);
+ uint64_t size = Size(Key(0), Key(n));
+ fprintf(stderr, "iter %3d => %7.3f MB [other %7.3f MB]\n",
+ read+1, size/1048576.0, Size(Key(n), Key(kCount))/1048576.0);
+ if (size <= initial_size/10) {
+ break;
+ }
+ }
+
+ // Verify that the size of the key space not touched by the reads
+ // is pretty much unchanged.
+ const int64_t final_other_size = Size(Key(n), Key(kCount));
+ ASSERT_LE(final_other_size, initial_other_size + 1048576);
+ ASSERT_GE(final_other_size, initial_other_size/5 - 1048576);
+}
+
+TEST(AutoCompactTest, ReadAll) {
+ DoReads(kCount);
+}
+
+TEST(AutoCompactTest, ReadHalf) {
+ DoReads(kCount/2);
+}
+
+} // namespace leveldb
+
+int main(int argc, char** argv) {
+ return leveldb::test::RunAllTests();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/builder.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/builder.cc b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/builder.cc
new file mode 100644
index 0000000..f419882
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/builder.cc
@@ -0,0 +1,88 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/builder.h"
+
+#include "db/filename.h"
+#include "db/dbformat.h"
+#include "db/table_cache.h"
+#include "db/version_edit.h"
+#include "leveldb/db.h"
+#include "leveldb/env.h"
+#include "leveldb/iterator.h"
+
+namespace leveldb {
+
+Status BuildTable(const std::string& dbname,
+ Env* env,
+ const Options& options,
+ TableCache* table_cache,
+ Iterator* iter,
+ FileMetaData* meta) {
+ Status s;
+ meta->file_size = 0;
+ iter->SeekToFirst();
+
+ std::string fname = TableFileName(dbname, meta->number);
+ if (iter->Valid()) {
+ WritableFile* file;
+ s = env->NewWritableFile(fname, &file);
+ if (!s.ok()) {
+ return s;
+ }
+
+ TableBuilder* builder = new TableBuilder(options, file);
+ meta->smallest.DecodeFrom(iter->key());
+ for (; iter->Valid(); iter->Next()) {
+ Slice key = iter->key();
+ meta->largest.DecodeFrom(key);
+ builder->Add(key, iter->value());
+ }
+
+ // Finish and check for builder errors
+ if (s.ok()) {
+ s = builder->Finish();
+ if (s.ok()) {
+ meta->file_size = builder->FileSize();
+ assert(meta->file_size > 0);
+ }
+ } else {
+ builder->Abandon();
+ }
+ delete builder;
+
+ // Finish and check for file errors
+ if (s.ok()) {
+ s = file->Sync();
+ }
+ if (s.ok()) {
+ s = file->Close();
+ }
+ delete file;
+ file = NULL;
+
+ if (s.ok()) {
+ // Verify that the table is usable
+ Iterator* it = table_cache->NewIterator(ReadOptions(),
+ meta->number,
+ meta->file_size);
+ s = it->status();
+ delete it;
+ }
+ }
+
+ // Check for input iterator errors
+ if (!iter->status().ok()) {
+ s = iter->status();
+ }
+
+ if (s.ok() && meta->file_size > 0) {
+ // Keep it
+ } else {
+ env->DeleteFile(fname);
+ }
+ return s;
+}
+
+} // namespace leveldb
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/builder.h
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/builder.h b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/builder.h
new file mode 100644
index 0000000..62431fc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/builder.h
@@ -0,0 +1,34 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef STORAGE_LEVELDB_DB_BUILDER_H_
+#define STORAGE_LEVELDB_DB_BUILDER_H_
+
+#include "leveldb/status.h"
+
+namespace leveldb {
+
+struct Options;
+struct FileMetaData;
+
+class Env;
+class Iterator;
+class TableCache;
+class VersionEdit;
+
+// Build a Table file from the contents of *iter. The generated file
+// will be named according to meta->number. On success, the rest of
+// *meta will be filled with metadata about the generated table.
+// If no data is present in *iter, meta->file_size will be set to
+// zero, and no Table file will be produced.
+extern Status BuildTable(const std::string& dbname,
+ Env* env,
+ const Options& options,
+ TableCache* table_cache,
+ Iterator* iter,
+ FileMetaData* meta);
+
+} // namespace leveldb
+
+#endif // STORAGE_LEVELDB_DB_BUILDER_H_
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/c.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/c.cc b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/c.cc
new file mode 100644
index 0000000..08ff0ad
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/c.cc
@@ -0,0 +1,595 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "leveldb/c.h"
+
+#include <stdlib.h>
+#include <unistd.h>
+#include "leveldb/cache.h"
+#include "leveldb/comparator.h"
+#include "leveldb/db.h"
+#include "leveldb/env.h"
+#include "leveldb/filter_policy.h"
+#include "leveldb/iterator.h"
+#include "leveldb/options.h"
+#include "leveldb/status.h"
+#include "leveldb/write_batch.h"
+
+using leveldb::Cache;
+using leveldb::Comparator;
+using leveldb::CompressionType;
+using leveldb::DB;
+using leveldb::Env;
+using leveldb::FileLock;
+using leveldb::FilterPolicy;
+using leveldb::Iterator;
+using leveldb::kMajorVersion;
+using leveldb::kMinorVersion;
+using leveldb::Logger;
+using leveldb::NewBloomFilterPolicy;
+using leveldb::NewLRUCache;
+using leveldb::Options;
+using leveldb::RandomAccessFile;
+using leveldb::Range;
+using leveldb::ReadOptions;
+using leveldb::SequentialFile;
+using leveldb::Slice;
+using leveldb::Snapshot;
+using leveldb::Status;
+using leveldb::WritableFile;
+using leveldb::WriteBatch;
+using leveldb::WriteOptions;
+
+extern "C" {
+
+struct leveldb_t { DB* rep; };
+struct leveldb_iterator_t { Iterator* rep; };
+struct leveldb_writebatch_t { WriteBatch rep; };
+struct leveldb_snapshot_t { const Snapshot* rep; };
+struct leveldb_readoptions_t { ReadOptions rep; };
+struct leveldb_writeoptions_t { WriteOptions rep; };
+struct leveldb_options_t { Options rep; };
+struct leveldb_cache_t { Cache* rep; };
+struct leveldb_seqfile_t { SequentialFile* rep; };
+struct leveldb_randomfile_t { RandomAccessFile* rep; };
+struct leveldb_writablefile_t { WritableFile* rep; };
+struct leveldb_logger_t { Logger* rep; };
+struct leveldb_filelock_t { FileLock* rep; };
+
+struct leveldb_comparator_t : public Comparator {
+ void* state_;
+ void (*destructor_)(void*);
+ int (*compare_)(
+ void*,
+ const char* a, size_t alen,
+ const char* b, size_t blen);
+ const char* (*name_)(void*);
+
+ virtual ~leveldb_comparator_t() {
+ (*destructor_)(state_);
+ }
+
+ virtual int Compare(const Slice& a, const Slice& b) const {
+ return (*compare_)(state_, a.data(), a.size(), b.data(), b.size());
+ }
+
+ virtual const char* Name() const {
+ return (*name_)(state_);
+ }
+
+ // No-ops since the C binding does not support key shortening methods.
+ virtual void FindShortestSeparator(std::string*, const Slice&) const { }
+ virtual void FindShortSuccessor(std::string* key) const { }
+};
+
+struct leveldb_filterpolicy_t : public FilterPolicy {
+ void* state_;
+ void (*destructor_)(void*);
+ const char* (*name_)(void*);
+ char* (*create_)(
+ void*,
+ const char* const* key_array, const size_t* key_length_array,
+ int num_keys,
+ size_t* filter_length);
+ unsigned char (*key_match_)(
+ void*,
+ const char* key, size_t length,
+ const char* filter, size_t filter_length);
+
+ virtual ~leveldb_filterpolicy_t() {
+ (*destructor_)(state_);
+ }
+
+ virtual const char* Name() const {
+ return (*name_)(state_);
+ }
+
+ virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {
+ std::vector<const char*> key_pointers(n);
+ std::vector<size_t> key_sizes(n);
+ for (int i = 0; i < n; i++) {
+ key_pointers[i] = keys[i].data();
+ key_sizes[i] = keys[i].size();
+ }
+ size_t len;
+ char* filter = (*create_)(state_, &key_pointers[0], &key_sizes[0], n, &len);
+ dst->append(filter, len);
+ free(filter);
+ }
+
+ virtual bool KeyMayMatch(const Slice& key, const Slice& filter) const {
+ return (*key_match_)(state_, key.data(), key.size(),
+ filter.data(), filter.size());
+ }
+};
+
+struct leveldb_env_t {
+ Env* rep;
+ bool is_default;
+};
+
+static bool SaveError(char** errptr, const Status& s) {
+ assert(errptr != NULL);
+ if (s.ok()) {
+ return false;
+ } else if (*errptr == NULL) {
+ *errptr = strdup(s.ToString().c_str());
+ } else {
+ // TODO(sanjay): Merge with existing error?
+ free(*errptr);
+ *errptr = strdup(s.ToString().c_str());
+ }
+ return true;
+}
+
+static char* CopyString(const std::string& str) {
+ char* result = reinterpret_cast<char*>(malloc(sizeof(char) * str.size()));
+ memcpy(result, str.data(), sizeof(char) * str.size());
+ return result;
+}
+
+leveldb_t* leveldb_open(
+ const leveldb_options_t* options,
+ const char* name,
+ char** errptr) {
+ DB* db;
+ if (SaveError(errptr, DB::Open(options->rep, std::string(name), &db))) {
+ return NULL;
+ }
+ leveldb_t* result = new leveldb_t;
+ result->rep = db;
+ return result;
+}
+
+void leveldb_close(leveldb_t* db) {
+ delete db->rep;
+ delete db;
+}
+
+void leveldb_put(
+ leveldb_t* db,
+ const leveldb_writeoptions_t* options,
+ const char* key, size_t keylen,
+ const char* val, size_t vallen,
+ char** errptr) {
+ SaveError(errptr,
+ db->rep->Put(options->rep, Slice(key, keylen), Slice(val, vallen)));
+}
+
+void leveldb_delete(
+ leveldb_t* db,
+ const leveldb_writeoptions_t* options,
+ const char* key, size_t keylen,
+ char** errptr) {
+ SaveError(errptr, db->rep->Delete(options->rep, Slice(key, keylen)));
+}
+
+
+void leveldb_write(
+ leveldb_t* db,
+ const leveldb_writeoptions_t* options,
+ leveldb_writebatch_t* batch,
+ char** errptr) {
+ SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
+}
+
+char* leveldb_get(
+ leveldb_t* db,
+ const leveldb_readoptions_t* options,
+ const char* key, size_t keylen,
+ size_t* vallen,
+ char** errptr) {
+ char* result = NULL;
+ std::string tmp;
+ Status s = db->rep->Get(options->rep, Slice(key, keylen), &tmp);
+ if (s.ok()) {
+ *vallen = tmp.size();
+ result = CopyString(tmp);
+ } else {
+ *vallen = 0;
+ if (!s.IsNotFound()) {
+ SaveError(errptr, s);
+ }
+ }
+ return result;
+}
+
+leveldb_iterator_t* leveldb_create_iterator(
+ leveldb_t* db,
+ const leveldb_readoptions_t* options) {
+ leveldb_iterator_t* result = new leveldb_iterator_t;
+ result->rep = db->rep->NewIterator(options->rep);
+ return result;
+}
+
+const leveldb_snapshot_t* leveldb_create_snapshot(
+ leveldb_t* db) {
+ leveldb_snapshot_t* result = new leveldb_snapshot_t;
+ result->rep = db->rep->GetSnapshot();
+ return result;
+}
+
+void leveldb_release_snapshot(
+ leveldb_t* db,
+ const leveldb_snapshot_t* snapshot) {
+ db->rep->ReleaseSnapshot(snapshot->rep);
+ delete snapshot;
+}
+
+char* leveldb_property_value(
+ leveldb_t* db,
+ const char* propname) {
+ std::string tmp;
+ if (db->rep->GetProperty(Slice(propname), &tmp)) {
+ // We use strdup() since we expect human readable output.
+ return strdup(tmp.c_str());
+ } else {
+ return NULL;
+ }
+}
+
+void leveldb_approximate_sizes(
+ leveldb_t* db,
+ int num_ranges,
+ const char* const* range_start_key, const size_t* range_start_key_len,
+ const char* const* range_limit_key, const size_t* range_limit_key_len,
+ uint64_t* sizes) {
+ Range* ranges = new Range[num_ranges];
+ for (int i = 0; i < num_ranges; i++) {
+ ranges[i].start = Slice(range_start_key[i], range_start_key_len[i]);
+ ranges[i].limit = Slice(range_limit_key[i], range_limit_key_len[i]);
+ }
+ db->rep->GetApproximateSizes(ranges, num_ranges, sizes);
+ delete[] ranges;
+}
+
+void leveldb_compact_range(
+ leveldb_t* db,
+ const char* start_key, size_t start_key_len,
+ const char* limit_key, size_t limit_key_len) {
+ Slice a, b;
+ db->rep->CompactRange(
+ // Pass NULL Slice if corresponding "const char*" is NULL
+ (start_key ? (a = Slice(start_key, start_key_len), &a) : NULL),
+ (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : NULL));
+}
+
+void leveldb_destroy_db(
+ const leveldb_options_t* options,
+ const char* name,
+ char** errptr) {
+ SaveError(errptr, DestroyDB(name, options->rep));
+}
+
+void leveldb_repair_db(
+ const leveldb_options_t* options,
+ const char* name,
+ char** errptr) {
+ SaveError(errptr, RepairDB(name, options->rep));
+}
+
+void leveldb_iter_destroy(leveldb_iterator_t* iter) {
+ delete iter->rep;
+ delete iter;
+}
+
+unsigned char leveldb_iter_valid(const leveldb_iterator_t* iter) {
+ return iter->rep->Valid();
+}
+
+void leveldb_iter_seek_to_first(leveldb_iterator_t* iter) {
+ iter->rep->SeekToFirst();
+}
+
+void leveldb_iter_seek_to_last(leveldb_iterator_t* iter) {
+ iter->rep->SeekToLast();
+}
+
+void leveldb_iter_seek(leveldb_iterator_t* iter, const char* k, size_t klen) {
+ iter->rep->Seek(Slice(k, klen));
+}
+
+void leveldb_iter_next(leveldb_iterator_t* iter) {
+ iter->rep->Next();
+}
+
+void leveldb_iter_prev(leveldb_iterator_t* iter) {
+ iter->rep->Prev();
+}
+
+const char* leveldb_iter_key(const leveldb_iterator_t* iter, size_t* klen) {
+ Slice s = iter->rep->key();
+ *klen = s.size();
+ return s.data();
+}
+
+const char* leveldb_iter_value(const leveldb_iterator_t* iter, size_t* vlen) {
+ Slice s = iter->rep->value();
+ *vlen = s.size();
+ return s.data();
+}
+
+void leveldb_iter_get_error(const leveldb_iterator_t* iter, char** errptr) {
+ SaveError(errptr, iter->rep->status());
+}
+
+leveldb_writebatch_t* leveldb_writebatch_create() {
+ return new leveldb_writebatch_t;
+}
+
+void leveldb_writebatch_destroy(leveldb_writebatch_t* b) {
+ delete b;
+}
+
+void leveldb_writebatch_clear(leveldb_writebatch_t* b) {
+ b->rep.Clear();
+}
+
+void leveldb_writebatch_put(
+ leveldb_writebatch_t* b,
+ const char* key, size_t klen,
+ const char* val, size_t vlen) {
+ b->rep.Put(Slice(key, klen), Slice(val, vlen));
+}
+
+void leveldb_writebatch_delete(
+ leveldb_writebatch_t* b,
+ const char* key, size_t klen) {
+ b->rep.Delete(Slice(key, klen));
+}
+
+void leveldb_writebatch_iterate(
+ leveldb_writebatch_t* b,
+ void* state,
+ void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
+ void (*deleted)(void*, const char* k, size_t klen)) {
+ class H : public WriteBatch::Handler {
+ public:
+ void* state_;
+ void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+ void (*deleted_)(void*, const char* k, size_t klen);
+ virtual void Put(const Slice& key, const Slice& value) {
+ (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+ }
+ virtual void Delete(const Slice& key) {
+ (*deleted_)(state_, key.data(), key.size());
+ }
+ };
+ H handler;
+ handler.state_ = state;
+ handler.put_ = put;
+ handler.deleted_ = deleted;
+ b->rep.Iterate(&handler);
+}
+
+leveldb_options_t* leveldb_options_create() {
+ return new leveldb_options_t;
+}
+
+void leveldb_options_destroy(leveldb_options_t* options) {
+ delete options;
+}
+
+void leveldb_options_set_comparator(
+ leveldb_options_t* opt,
+ leveldb_comparator_t* cmp) {
+ opt->rep.comparator = cmp;
+}
+
+void leveldb_options_set_filter_policy(
+ leveldb_options_t* opt,
+ leveldb_filterpolicy_t* policy) {
+ opt->rep.filter_policy = policy;
+}
+
+void leveldb_options_set_create_if_missing(
+ leveldb_options_t* opt, unsigned char v) {
+ opt->rep.create_if_missing = v;
+}
+
+void leveldb_options_set_error_if_exists(
+ leveldb_options_t* opt, unsigned char v) {
+ opt->rep.error_if_exists = v;
+}
+
+void leveldb_options_set_paranoid_checks(
+ leveldb_options_t* opt, unsigned char v) {
+ opt->rep.paranoid_checks = v;
+}
+
+void leveldb_options_set_env(leveldb_options_t* opt, leveldb_env_t* env) {
+ opt->rep.env = (env ? env->rep : NULL);
+}
+
+void leveldb_options_set_info_log(leveldb_options_t* opt, leveldb_logger_t* l) {
+ opt->rep.info_log = (l ? l->rep : NULL);
+}
+
+void leveldb_options_set_write_buffer_size(leveldb_options_t* opt, size_t s) {
+ opt->rep.write_buffer_size = s;
+}
+
+void leveldb_options_set_max_open_files(leveldb_options_t* opt, int n) {
+ opt->rep.max_open_files = n;
+}
+
+void leveldb_options_set_cache(leveldb_options_t* opt, leveldb_cache_t* c) {
+ opt->rep.block_cache = c->rep;
+}
+
+void leveldb_options_set_block_size(leveldb_options_t* opt, size_t s) {
+ opt->rep.block_size = s;
+}
+
+void leveldb_options_set_block_restart_interval(leveldb_options_t* opt, int n) {
+ opt->rep.block_restart_interval = n;
+}
+
+void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
+ opt->rep.compression = static_cast<CompressionType>(t);
+}
+
+leveldb_comparator_t* leveldb_comparator_create(
+ void* state,
+ void (*destructor)(void*),
+ int (*compare)(
+ void*,
+ const char* a, size_t alen,
+ const char* b, size_t blen),
+ const char* (*name)(void*)) {
+ leveldb_comparator_t* result = new leveldb_comparator_t;
+ result->state_ = state;
+ result->destructor_ = destructor;
+ result->compare_ = compare;
+ result->name_ = name;
+ return result;
+}
+
+void leveldb_comparator_destroy(leveldb_comparator_t* cmp) {
+ delete cmp;
+}
+
+leveldb_filterpolicy_t* leveldb_filterpolicy_create(
+ void* state,
+ void (*destructor)(void*),
+ char* (*create_filter)(
+ void*,
+ const char* const* key_array, const size_t* key_length_array,
+ int num_keys,
+ size_t* filter_length),
+ unsigned char (*key_may_match)(
+ void*,
+ const char* key, size_t length,
+ const char* filter, size_t filter_length),
+ const char* (*name)(void*)) {
+ leveldb_filterpolicy_t* result = new leveldb_filterpolicy_t;
+ result->state_ = state;
+ result->destructor_ = destructor;
+ result->create_ = create_filter;
+ result->key_match_ = key_may_match;
+ result->name_ = name;
+ return result;
+}
+
+void leveldb_filterpolicy_destroy(leveldb_filterpolicy_t* filter) {
+ delete filter;
+}
+
+leveldb_filterpolicy_t* leveldb_filterpolicy_create_bloom(int bits_per_key) {
+ // Make a leveldb_filterpolicy_t, but override all of its methods so
+ // they delegate to a NewBloomFilterPolicy() instead of user
+ // supplied C functions.
+ struct Wrapper : public leveldb_filterpolicy_t {
+ const FilterPolicy* rep_;
+ ~Wrapper() { delete rep_; }
+ const char* Name() const { return rep_->Name(); }
+ void CreateFilter(const Slice* keys, int n, std::string* dst) const {
+ return rep_->CreateFilter(keys, n, dst);
+ }
+ bool KeyMayMatch(const Slice& key, const Slice& filter) const {
+ return rep_->KeyMayMatch(key, filter);
+ }
+ static void DoNothing(void*) { }
+ };
+ Wrapper* wrapper = new Wrapper;
+ wrapper->rep_ = NewBloomFilterPolicy(bits_per_key);
+ wrapper->state_ = NULL;
+ wrapper->destructor_ = &Wrapper::DoNothing;
+ return wrapper;
+}
+
+leveldb_readoptions_t* leveldb_readoptions_create() {
+ return new leveldb_readoptions_t;
+}
+
+void leveldb_readoptions_destroy(leveldb_readoptions_t* opt) {
+ delete opt;
+}
+
+void leveldb_readoptions_set_verify_checksums(
+ leveldb_readoptions_t* opt,
+ unsigned char v) {
+ opt->rep.verify_checksums = v;
+}
+
+void leveldb_readoptions_set_fill_cache(
+ leveldb_readoptions_t* opt, unsigned char v) {
+ opt->rep.fill_cache = v;
+}
+
+void leveldb_readoptions_set_snapshot(
+ leveldb_readoptions_t* opt,
+ const leveldb_snapshot_t* snap) {
+ opt->rep.snapshot = (snap ? snap->rep : NULL);
+}
+
+leveldb_writeoptions_t* leveldb_writeoptions_create() {
+ return new leveldb_writeoptions_t;
+}
+
+void leveldb_writeoptions_destroy(leveldb_writeoptions_t* opt) {
+ delete opt;
+}
+
+void leveldb_writeoptions_set_sync(
+ leveldb_writeoptions_t* opt, unsigned char v) {
+ opt->rep.sync = v;
+}
+
+leveldb_cache_t* leveldb_cache_create_lru(size_t capacity) {
+ leveldb_cache_t* c = new leveldb_cache_t;
+ c->rep = NewLRUCache(capacity);
+ return c;
+}
+
+void leveldb_cache_destroy(leveldb_cache_t* cache) {
+ delete cache->rep;
+ delete cache;
+}
+
+leveldb_env_t* leveldb_create_default_env() {
+ leveldb_env_t* result = new leveldb_env_t;
+ result->rep = Env::Default();
+ result->is_default = true;
+ return result;
+}
+
+void leveldb_env_destroy(leveldb_env_t* env) {
+ if (!env->is_default) delete env->rep;
+ delete env;
+}
+
+void leveldb_free(void* ptr) {
+ free(ptr);
+}
+
+int leveldb_major_version() {
+ return kMajorVersion;
+}
+
+int leveldb_minor_version() {
+ return kMinorVersion;
+}
+
+} // end extern "C"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/c_test.c
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/c_test.c b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/c_test.c
new file mode 100644
index 0000000..7cd5ee0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/c_test.c
@@ -0,0 +1,390 @@
+/* Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+ Use of this source code is governed by a BSD-style license that can be
+ found in the LICENSE file. See the AUTHORS file for names of contributors. */
+
+#include "leveldb/c.h"
+
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+const char* phase = "";
+static char dbname[200];
+
+static void StartPhase(const char* name) {
+ fprintf(stderr, "=== Test %s\n", name);
+ phase = name;
+}
+
+static const char* GetTempDir(void) {
+ const char* ret = getenv("TEST_TMPDIR");
+ if (ret == NULL || ret[0] == '\0')
+ ret = "/tmp";
+ return ret;
+}
+
+#define CheckNoError(err) \
+ if ((err) != NULL) { \
+ fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, (err)); \
+ abort(); \
+ }
+
+#define CheckCondition(cond) \
+ if (!(cond)) { \
+ fprintf(stderr, "%s:%d: %s: %s\n", __FILE__, __LINE__, phase, #cond); \
+ abort(); \
+ }
+
+static void CheckEqual(const char* expected, const char* v, size_t n) {
+ if (expected == NULL && v == NULL) {
+ // ok
+ } else if (expected != NULL && v != NULL && n == strlen(expected) &&
+ memcmp(expected, v, n) == 0) {
+ // ok
+ return;
+ } else {
+ fprintf(stderr, "%s: expected '%s', got '%s'\n",
+ phase,
+ (expected ? expected : "(null)"),
+ (v ? v : "(null"));
+ abort();
+ }
+}
+
+static void Free(char** ptr) {
+ if (*ptr) {
+ free(*ptr);
+ *ptr = NULL;
+ }
+}
+
+static void CheckGet(
+ leveldb_t* db,
+ const leveldb_readoptions_t* options,
+ const char* key,
+ const char* expected) {
+ char* err = NULL;
+ size_t val_len;
+ char* val;
+ val = leveldb_get(db, options, key, strlen(key), &val_len, &err);
+ CheckNoError(err);
+ CheckEqual(expected, val, val_len);
+ Free(&val);
+}
+
+static void CheckIter(leveldb_iterator_t* iter,
+ const char* key, const char* val) {
+ size_t len;
+ const char* str;
+ str = leveldb_iter_key(iter, &len);
+ CheckEqual(key, str, len);
+ str = leveldb_iter_value(iter, &len);
+ CheckEqual(val, str, len);
+}
+
+// Callback from leveldb_writebatch_iterate()
+static void CheckPut(void* ptr,
+ const char* k, size_t klen,
+ const char* v, size_t vlen) {
+ int* state = (int*) ptr;
+ CheckCondition(*state < 2);
+ switch (*state) {
+ case 0:
+ CheckEqual("bar", k, klen);
+ CheckEqual("b", v, vlen);
+ break;
+ case 1:
+ CheckEqual("box", k, klen);
+ CheckEqual("c", v, vlen);
+ break;
+ }
+ (*state)++;
+}
+
+// Callback from leveldb_writebatch_iterate()
+static void CheckDel(void* ptr, const char* k, size_t klen) {
+ int* state = (int*) ptr;
+ CheckCondition(*state == 2);
+ CheckEqual("bar", k, klen);
+ (*state)++;
+}
+
+static void CmpDestroy(void* arg) { }
+
+static int CmpCompare(void* arg, const char* a, size_t alen,
+ const char* b, size_t blen) {
+ int n = (alen < blen) ? alen : blen;
+ int r = memcmp(a, b, n);
+ if (r == 0) {
+ if (alen < blen) r = -1;
+ else if (alen > blen) r = +1;
+ }
+ return r;
+}
+
+static const char* CmpName(void* arg) {
+ return "foo";
+}
+
+// Custom filter policy
+static unsigned char fake_filter_result = 1;
+static void FilterDestroy(void* arg) { }
+static const char* FilterName(void* arg) {
+ return "TestFilter";
+}
+static char* FilterCreate(
+ void* arg,
+ const char* const* key_array, const size_t* key_length_array,
+ int num_keys,
+ size_t* filter_length) {
+ *filter_length = 4;
+ char* result = malloc(4);
+ memcpy(result, "fake", 4);
+ return result;
+}
+unsigned char FilterKeyMatch(
+ void* arg,
+ const char* key, size_t length,
+ const char* filter, size_t filter_length) {
+ CheckCondition(filter_length == 4);
+ CheckCondition(memcmp(filter, "fake", 4) == 0);
+ return fake_filter_result;
+}
+
+int main(int argc, char** argv) {
+ leveldb_t* db;
+ leveldb_comparator_t* cmp;
+ leveldb_cache_t* cache;
+ leveldb_env_t* env;
+ leveldb_options_t* options;
+ leveldb_readoptions_t* roptions;
+ leveldb_writeoptions_t* woptions;
+ char* err = NULL;
+ int run = -1;
+
+ CheckCondition(leveldb_major_version() >= 1);
+ CheckCondition(leveldb_minor_version() >= 1);
+
+ snprintf(dbname, sizeof(dbname),
+ "%s/leveldb_c_test-%d",
+ GetTempDir(),
+ ((int) geteuid()));
+
+ StartPhase("create_objects");
+ cmp = leveldb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
+ env = leveldb_create_default_env();
+ cache = leveldb_cache_create_lru(100000);
+
+ options = leveldb_options_create();
+ leveldb_options_set_comparator(options, cmp);
+ leveldb_options_set_error_if_exists(options, 1);
+ leveldb_options_set_cache(options, cache);
+ leveldb_options_set_env(options, env);
+ leveldb_options_set_info_log(options, NULL);
+ leveldb_options_set_write_buffer_size(options, 100000);
+ leveldb_options_set_paranoid_checks(options, 1);
+ leveldb_options_set_max_open_files(options, 10);
+ leveldb_options_set_block_size(options, 1024);
+ leveldb_options_set_block_restart_interval(options, 8);
+ leveldb_options_set_compression(options, leveldb_no_compression);
+
+ roptions = leveldb_readoptions_create();
+ leveldb_readoptions_set_verify_checksums(roptions, 1);
+ leveldb_readoptions_set_fill_cache(roptions, 0);
+
+ woptions = leveldb_writeoptions_create();
+ leveldb_writeoptions_set_sync(woptions, 1);
+
+ StartPhase("destroy");
+ leveldb_destroy_db(options, dbname, &err);
+ Free(&err);
+
+ StartPhase("open_error");
+ db = leveldb_open(options, dbname, &err);
+ CheckCondition(err != NULL);
+ Free(&err);
+
+ StartPhase("leveldb_free");
+ db = leveldb_open(options, dbname, &err);
+ CheckCondition(err != NULL);
+ leveldb_free(err);
+ err = NULL;
+
+ StartPhase("open");
+ leveldb_options_set_create_if_missing(options, 1);
+ db = leveldb_open(options, dbname, &err);
+ CheckNoError(err);
+ CheckGet(db, roptions, "foo", NULL);
+
+ StartPhase("put");
+ leveldb_put(db, woptions, "foo", 3, "hello", 5, &err);
+ CheckNoError(err);
+ CheckGet(db, roptions, "foo", "hello");
+
+ StartPhase("compactall");
+ leveldb_compact_range(db, NULL, 0, NULL, 0);
+ CheckGet(db, roptions, "foo", "hello");
+
+ StartPhase("compactrange");
+ leveldb_compact_range(db, "a", 1, "z", 1);
+ CheckGet(db, roptions, "foo", "hello");
+
+ StartPhase("writebatch");
+ {
+ leveldb_writebatch_t* wb = leveldb_writebatch_create();
+ leveldb_writebatch_put(wb, "foo", 3, "a", 1);
+ leveldb_writebatch_clear(wb);
+ leveldb_writebatch_put(wb, "bar", 3, "b", 1);
+ leveldb_writebatch_put(wb, "box", 3, "c", 1);
+ leveldb_writebatch_delete(wb, "bar", 3);
+ leveldb_write(db, woptions, wb, &err);
+ CheckNoError(err);
+ CheckGet(db, roptions, "foo", "hello");
+ CheckGet(db, roptions, "bar", NULL);
+ CheckGet(db, roptions, "box", "c");
+ int pos = 0;
+ leveldb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
+ CheckCondition(pos == 3);
+ leveldb_writebatch_destroy(wb);
+ }
+
+ StartPhase("iter");
+ {
+ leveldb_iterator_t* iter = leveldb_create_iterator(db, roptions);
+ CheckCondition(!leveldb_iter_valid(iter));
+ leveldb_iter_seek_to_first(iter);
+ CheckCondition(leveldb_iter_valid(iter));
+ CheckIter(iter, "box", "c");
+ leveldb_iter_next(iter);
+ CheckIter(iter, "foo", "hello");
+ leveldb_iter_prev(iter);
+ CheckIter(iter, "box", "c");
+ leveldb_iter_prev(iter);
+ CheckCondition(!leveldb_iter_valid(iter));
+ leveldb_iter_seek_to_last(iter);
+ CheckIter(iter, "foo", "hello");
+ leveldb_iter_seek(iter, "b", 1);
+ CheckIter(iter, "box", "c");
+ leveldb_iter_get_error(iter, &err);
+ CheckNoError(err);
+ leveldb_iter_destroy(iter);
+ }
+
+ StartPhase("approximate_sizes");
+ {
+ int i;
+ int n = 20000;
+ char keybuf[100];
+ char valbuf[100];
+ uint64_t sizes[2];
+ const char* start[2] = { "a", "k00000000000000010000" };
+ size_t start_len[2] = { 1, 21 };
+ const char* limit[2] = { "k00000000000000010000", "z" };
+ size_t limit_len[2] = { 21, 1 };
+ leveldb_writeoptions_set_sync(woptions, 0);
+ for (i = 0; i < n; i++) {
+ snprintf(keybuf, sizeof(keybuf), "k%020d", i);
+ snprintf(valbuf, sizeof(valbuf), "v%020d", i);
+ leveldb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
+ &err);
+ CheckNoError(err);
+ }
+ leveldb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
+ CheckCondition(sizes[0] > 0);
+ CheckCondition(sizes[1] > 0);
+ }
+
+ StartPhase("property");
+ {
+ char* prop = leveldb_property_value(db, "nosuchprop");
+ CheckCondition(prop == NULL);
+ prop = leveldb_property_value(db, "leveldb.stats");
+ CheckCondition(prop != NULL);
+ Free(&prop);
+ }
+
+ StartPhase("snapshot");
+ {
+ const leveldb_snapshot_t* snap;
+ snap = leveldb_create_snapshot(db);
+ leveldb_delete(db, woptions, "foo", 3, &err);
+ CheckNoError(err);
+ leveldb_readoptions_set_snapshot(roptions, snap);
+ CheckGet(db, roptions, "foo", "hello");
+ leveldb_readoptions_set_snapshot(roptions, NULL);
+ CheckGet(db, roptions, "foo", NULL);
+ leveldb_release_snapshot(db, snap);
+ }
+
+ StartPhase("repair");
+ {
+ leveldb_close(db);
+ leveldb_options_set_create_if_missing(options, 0);
+ leveldb_options_set_error_if_exists(options, 0);
+ leveldb_repair_db(options, dbname, &err);
+ CheckNoError(err);
+ db = leveldb_open(options, dbname, &err);
+ CheckNoError(err);
+ CheckGet(db, roptions, "foo", NULL);
+ CheckGet(db, roptions, "bar", NULL);
+ CheckGet(db, roptions, "box", "c");
+ leveldb_options_set_create_if_missing(options, 1);
+ leveldb_options_set_error_if_exists(options, 1);
+ }
+
+ StartPhase("filter");
+ for (run = 0; run < 2; run++) {
+ // First run uses custom filter, second run uses bloom filter
+ CheckNoError(err);
+ leveldb_filterpolicy_t* policy;
+ if (run == 0) {
+ policy = leveldb_filterpolicy_create(
+ NULL, FilterDestroy, FilterCreate, FilterKeyMatch, FilterName);
+ } else {
+ policy = leveldb_filterpolicy_create_bloom(10);
+ }
+
+ // Create new database
+ leveldb_close(db);
+ leveldb_destroy_db(options, dbname, &err);
+ leveldb_options_set_filter_policy(options, policy);
+ db = leveldb_open(options, dbname, &err);
+ CheckNoError(err);
+ leveldb_put(db, woptions, "foo", 3, "foovalue", 8, &err);
+ CheckNoError(err);
+ leveldb_put(db, woptions, "bar", 3, "barvalue", 8, &err);
+ CheckNoError(err);
+ leveldb_compact_range(db, NULL, 0, NULL, 0);
+
+ fake_filter_result = 1;
+ CheckGet(db, roptions, "foo", "foovalue");
+ CheckGet(db, roptions, "bar", "barvalue");
+ if (phase == 0) {
+ // Must not find value when custom filter returns false
+ fake_filter_result = 0;
+ CheckGet(db, roptions, "foo", NULL);
+ CheckGet(db, roptions, "bar", NULL);
+ fake_filter_result = 1;
+
+ CheckGet(db, roptions, "foo", "foovalue");
+ CheckGet(db, roptions, "bar", "barvalue");
+ }
+ leveldb_options_set_filter_policy(options, NULL);
+ leveldb_filterpolicy_destroy(policy);
+ }
+
+ StartPhase("cleanup");
+ leveldb_close(db);
+ leveldb_options_destroy(options);
+ leveldb_readoptions_destroy(roptions);
+ leveldb_writeoptions_destroy(woptions);
+ leveldb_cache_destroy(cache);
+ leveldb_comparator_destroy(cmp);
+ leveldb_env_destroy(env);
+
+ fprintf(stderr, "PASS\n");
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6419f4/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/corruption_test.cc
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/corruption_test.cc b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/corruption_test.cc
new file mode 100644
index 0000000..96afc68
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfsdb/src/main/native/hdfsdb/db/corruption_test.cc
@@ -0,0 +1,374 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "leveldb/db.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include "leveldb/cache.h"
+#include "leveldb/env.h"
+#include "leveldb/table.h"
+#include "leveldb/write_batch.h"
+#include "db/db_impl.h"
+#include "db/filename.h"
+#include "db/log_format.h"
+#include "db/version_set.h"
+#include "util/logging.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+
+namespace leveldb {
+
+static const int kValueSize = 1000;
+
+class CorruptionTest {
+ public:
+ test::ErrorEnv env_;
+ std::string dbname_;
+ Cache* tiny_cache_;
+ Options options_;
+ DB* db_;
+
+ CorruptionTest() {
+ tiny_cache_ = NewLRUCache(100);
+ options_.env = &env_;
+ options_.block_cache = tiny_cache_;
+ dbname_ = test::TmpDir() + "/db_test";
+ DestroyDB(dbname_, options_);
+
+ db_ = NULL;
+ options_.create_if_missing = true;
+ Reopen();
+ options_.create_if_missing = false;
+ }
+
+ ~CorruptionTest() {
+ delete db_;
+ DestroyDB(dbname_, Options());
+ delete tiny_cache_;
+ }
+
+ Status TryReopen() {
+ delete db_;
+ db_ = NULL;
+ return DB::Open(options_, dbname_, &db_);
+ }
+
+ void Reopen() {
+ ASSERT_OK(TryReopen());
+ }
+
+ void RepairDB() {
+ delete db_;
+ db_ = NULL;
+ ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
+ }
+
+ void Build(int n) {
+ std::string key_space, value_space;
+ WriteBatch batch;
+ for (int i = 0; i < n; i++) {
+ //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
+ Slice key = Key(i, &key_space);
+ batch.Clear();
+ batch.Put(key, Value(i, &value_space));
+ WriteOptions options;
+ // Corrupt() doesn't work without this sync on windows; stat reports 0 for
+ // the file size.
+ if (i == n - 1) {
+ options.sync = true;
+ }
+ ASSERT_OK(db_->Write(options, &batch));
+ }
+ }
+
+ void Check(int min_expected, int max_expected) {
+ int next_expected = 0;
+ int missed = 0;
+ int bad_keys = 0;
+ int bad_values = 0;
+ int correct = 0;
+ std::string value_space;
+ Iterator* iter = db_->NewIterator(ReadOptions());
+ for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+ uint64_t key;
+ Slice in(iter->key());
+ if (in == "" || in == "~") {
+ // Ignore boundary keys.
+ continue;
+ }
+ if (!ConsumeDecimalNumber(&in, &key) ||
+ !in.empty() ||
+ key < next_expected) {
+ bad_keys++;
+ continue;
+ }
+ missed += (key - next_expected);
+ next_expected = key + 1;
+ if (iter->value() != Value(key, &value_space)) {
+ bad_values++;
+ } else {
+ correct++;
+ }
+ }
+ delete iter;
+
+ fprintf(stderr,
+ "expected=%d..%d; got=%d; bad_keys=%d; bad_values=%d; missed=%d\n",
+ min_expected, max_expected, correct, bad_keys, bad_values, missed);
+ ASSERT_LE(min_expected, correct);
+ ASSERT_GE(max_expected, correct);
+ }
+
+ void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
+ // Pick file to corrupt
+ std::vector<std::string> filenames;
+ ASSERT_OK(env_.GetChildren(dbname_, &filenames));
+ uint64_t number;
+ FileType type;
+ std::string fname;
+ int picked_number = -1;
+ for (size_t i = 0; i < filenames.size(); i++) {
+ if (ParseFileName(filenames[i], &number, &type) &&
+ type == filetype &&
+ int(number) > picked_number) { // Pick latest file
+ fname = dbname_ + "/" + filenames[i];
+ picked_number = number;
+ }
+ }
+ ASSERT_TRUE(!fname.empty()) << filetype;
+
+ struct stat sbuf;
+ if (stat(fname.c_str(), &sbuf) != 0) {
+ const char* msg = strerror(errno);
+ ASSERT_TRUE(false) << fname << ": " << msg;
+ }
+
+ if (offset < 0) {
+ // Relative to end of file; make it absolute
+ if (-offset > sbuf.st_size) {
+ offset = 0;
+ } else {
+ offset = sbuf.st_size + offset;
+ }
+ }
+ if (offset > sbuf.st_size) {
+ offset = sbuf.st_size;
+ }
+ if (offset + bytes_to_corrupt > sbuf.st_size) {
+ bytes_to_corrupt = sbuf.st_size - offset;
+ }
+
+ // Do it
+ std::string contents;
+ Status s = ReadFileToString(Env::Default(), fname, &contents);
+ ASSERT_TRUE(s.ok()) << s.ToString();
+ for (int i = 0; i < bytes_to_corrupt; i++) {
+ contents[i + offset] ^= 0x80;
+ }
+ s = WriteStringToFile(Env::Default(), contents, fname);
+ ASSERT_TRUE(s.ok()) << s.ToString();
+ }
+
+ int Property(const std::string& name) {
+ std::string property;
+ int result;
+ if (db_->GetProperty(name, &property) &&
+ sscanf(property.c_str(), "%d", &result) == 1) {
+ return result;
+ } else {
+ return -1;
+ }
+ }
+
+ // Return the ith key
+ Slice Key(int i, std::string* storage) {
+ char buf[100];
+ snprintf(buf, sizeof(buf), "%016d", i);
+ storage->assign(buf, strlen(buf));
+ return Slice(*storage);
+ }
+
+ // Return the value to associate with the specified key
+ Slice Value(int k, std::string* storage) {
+ Random r(k);
+ return test::RandomString(&r, kValueSize, storage);
+ }
+};
+
+TEST(CorruptionTest, Recovery) {
+ Build(100);
+ Check(100, 100);
+ Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record
+ Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block
+ Reopen();
+
+ // The 64 records in the first two log blocks are completely lost.
+ Check(36, 36);
+}
+
+TEST(CorruptionTest, RecoverWriteError) {
+ env_.writable_file_error_ = true;
+ Status s = TryReopen();
+ ASSERT_TRUE(!s.ok());
+}
+
+TEST(CorruptionTest, NewFileErrorDuringWrite) {
+ // Do enough writing to force minor compaction
+ env_.writable_file_error_ = true;
+ const int num = 3 + (Options().write_buffer_size / kValueSize);
+ std::string value_storage;
+ Status s;
+ for (int i = 0; s.ok() && i < num; i++) {
+ WriteBatch batch;
+ batch.Put("a", Value(100, &value_storage));
+ s = db_->Write(WriteOptions(), &batch);
+ }
+ ASSERT_TRUE(!s.ok());
+ ASSERT_GE(env_.num_writable_file_errors_, 1);
+ env_.writable_file_error_ = false;
+ Reopen();
+}
+
+TEST(CorruptionTest, TableFile) {
+ Build(100);
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
+ dbi->TEST_CompactMemTable();
+ dbi->TEST_CompactRange(0, NULL, NULL);
+ dbi->TEST_CompactRange(1, NULL, NULL);
+
+ Corrupt(kTableFile, 100, 1);
+ Check(90, 99);
+}
+
+TEST(CorruptionTest, TableFileRepair) {
+ options_.block_size = 2 * kValueSize; // Limit scope of corruption
+ options_.paranoid_checks = true;
+ Reopen();
+ Build(100);
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
+ dbi->TEST_CompactMemTable();
+ dbi->TEST_CompactRange(0, NULL, NULL);
+ dbi->TEST_CompactRange(1, NULL, NULL);
+
+ Corrupt(kTableFile, 100, 1);
+ RepairDB();
+ Reopen();
+ Check(95, 99);
+}
+
+TEST(CorruptionTest, TableFileIndexData) {
+ Build(10000); // Enough to build multiple Tables
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
+ dbi->TEST_CompactMemTable();
+
+ Corrupt(kTableFile, -2000, 500);
+ Reopen();
+ Check(5000, 9999);
+}
+
+TEST(CorruptionTest, MissingDescriptor) {
+ Build(1000);
+ RepairDB();
+ Reopen();
+ Check(1000, 1000);
+}
+
+TEST(CorruptionTest, SequenceNumberRecovery) {
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v3"));
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v4"));
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v5"));
+ RepairDB();
+ Reopen();
+ std::string v;
+ ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
+ ASSERT_EQ("v5", v);
+ // Write something. If sequence number was not recovered properly,
+ // it will be hidden by an earlier write.
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "v6"));
+ ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
+ ASSERT_EQ("v6", v);
+ Reopen();
+ ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
+ ASSERT_EQ("v6", v);
+}
+
+TEST(CorruptionTest, CorruptedDescriptor) {
+ ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
+ dbi->TEST_CompactMemTable();
+ dbi->TEST_CompactRange(0, NULL, NULL);
+
+ Corrupt(kDescriptorFile, 0, 1000);
+ Status s = TryReopen();
+ ASSERT_TRUE(!s.ok());
+
+ RepairDB();
+ Reopen();
+ std::string v;
+ ASSERT_OK(db_->Get(ReadOptions(), "foo", &v));
+ ASSERT_EQ("hello", v);
+}
+
+TEST(CorruptionTest, CompactionInputError) {
+ Build(10);
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
+ dbi->TEST_CompactMemTable();
+ const int last = config::kMaxMemCompactLevel;
+ ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
+
+ Corrupt(kTableFile, 100, 1);
+ Check(5, 9);
+
+ // Force compactions by writing lots of values
+ Build(10000);
+ Check(10000, 10000);
+}
+
+TEST(CorruptionTest, CompactionInputErrorParanoid) {
+ options_.paranoid_checks = true;
+ options_.write_buffer_size = 512 << 10;
+ Reopen();
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
+
+ // Make multiple inputs so we need to compact.
+ for (int i = 0; i < 2; i++) {
+ Build(10);
+ dbi->TEST_CompactMemTable();
+ Corrupt(kTableFile, 100, 1);
+ env_.SleepForMicroseconds(100000);
+ }
+ dbi->CompactRange(NULL, NULL);
+
+ // Write must fail because of corrupted table
+ std::string tmp1, tmp2;
+ Status s = db_->Put(WriteOptions(), Key(5, &tmp1), Value(5, &tmp2));
+ ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
+}
+
+TEST(CorruptionTest, UnrelatedKeys) {
+ Build(10);
+ DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
+ dbi->TEST_CompactMemTable();
+ Corrupt(kTableFile, 100, 1);
+
+ std::string tmp1, tmp2;
+ ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
+ std::string v;
+ ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
+ ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
+ dbi->TEST_CompactMemTable();
+ ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
+ ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
+}
+
+} // namespace leveldb
+
+int main(int argc, char** argv) {
+ return leveldb::test::RunAllTests();
+}