You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by si...@apache.org on 2018/06/06 06:53:38 UTC
[bookkeeper] branch master updated: Upgrade RocksDB to 5.13.1
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 7bbd3de Upgrade RocksDB to 5.13.1
7bbd3de is described below
commit 7bbd3dee4e060046fcdf06294bb9d18bb22b41b9
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Tue Jun 5 23:53:27 2018 -0700
Upgrade RocksDB to 5.13.1
Upgrade RocksDB version to include a fix for empty SSTs written by flushing with deleteRange() operations that cause assertion failures on DB open.
Description can be found at : facebook/rocksdb#2717
Author: Matteo Merli <mm...@apache.org>
Reviewers: Jia Zhai <None>, Sijie Guo <si...@apache.org>
This closes #1466 from merlimat/upgrade-rocksdb
---
.../src/main/resources/LICENSE-all.bin.txt | 6 +--
.../src/main/resources/LICENSE-server.bin.txt | 6 +--
.../bookie/storage/ldb/KeyValueStorage.java | 6 +--
.../bookie/storage/ldb/KeyValueStorageRocksDB.java | 24 +++++++++---
pom.xml | 2 +-
.../statelib/impl/kv/RocksdbKVStore.java | 43 ++++++++++++++--------
.../statelib/impl/mvcc/MVCCStoreImpl.java | 36 ++++++++++++------
7 files changed, 81 insertions(+), 42 deletions(-)
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
index e0c19cc..f1bcc84 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
@@ -269,7 +269,7 @@ Apache Software License, Version 2.
- lib/org.eclipse.jetty-jetty-server-9.4.5.v20170502.jar [34]
- lib/org.eclipse.jetty-jetty-servlet-9.4.5.v20170502.jar [34]
- lib/org.eclipse.jetty-jetty-util-9.4.5.v20170502.jar [34]
-- lib/org.rocksdb-rocksdbjni-5.8.6.jar [35]
+- lib/org.rocksdb-rocksdbjni-5.13.1.jar [35]
- lib/com.beust-jcommander-1.48.jar [36]
- lib/com.yahoo.datasketches-memory-0.8.3.jar [37]
- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [37]
@@ -332,7 +332,7 @@ Apache Software License, Version 2.
[32] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=3ad2e8
[33] Source available at https://github.com/apache/zookeeper/tree/release-3.5.3
[34] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.5.v20170502
-[35] Source available at https://github.com/facebook/rocksdb/tree/v5.8.6
+[35] Source available at https://github.com/facebook/rocksdb/tree/v5.13.1
[36] Source available at https://github.com/cbeust/jcommander/tree/jcommander-1.48
[37] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
[38] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
@@ -453,7 +453,7 @@ interchange format, which can be obtained at:
* https://github.com/google/protobuf
------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-5.8.6.jar is derived from leveldb, which is under the following license.
+lib/org.rocksdb-rocksdbjni-5.13.1.jar is derived from leveldb, which is under the following license.
Copyright (c) 2011 The LevelDB Authors. All rights reserved.
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
index 22abe3a..d59d322 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
@@ -234,7 +234,7 @@ Apache Software License, Version 2.
- lib/org.eclipse.jetty-jetty-server-9.4.5.v20170502.jar [21]
- lib/org.eclipse.jetty-jetty-servlet-9.4.5.v20170502.jar [21]
- lib/org.eclipse.jetty-jetty-util-9.4.5.v20170502.jar [21]
-- lib/org.rocksdb-rocksdbjni-5.8.6.jar [22]
+- lib/org.rocksdb-rocksdbjni-5.13.1.jar [22]
- lib/com.beust-jcommander-1.48.jar [23]
- lib/com.yahoo.datasketches-memory-0.8.3.jar [24]
- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [24]
@@ -285,7 +285,7 @@ Apache Software License, Version 2.
[19] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=3ad2e8
[20] Source available at https://github.com/apache/zookeeper/tree/release-3.5.3
[21] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.5.v20170502
-[22] Source available at https://github.com/facebook/rocksdb/tree/v5.8.6
+[22] Source available at https://github.com/facebook/rocksdb/tree/v5.13.1
[23] Source available at https://github.com/cbeust/jcommander/tree/jcommander-1.48
[24] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
[25] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
@@ -371,7 +371,7 @@ interchange format, which can be obtained at:
* https://github.com/google/protobuf
------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-5.8.6.jar is derived from leveldb, which is under the following license.
+lib/org.rocksdb-rocksdbjni-5.13.1.jar is derived from leveldb, which is under the following license.
Copyright (c) 2011 The LevelDB Authors. All rights reserved.
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorage.java
index b9bbb2a..aa0119a 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorage.java
@@ -149,11 +149,11 @@ public interface KeyValueStorage extends Closeable {
* Interface for a batch to be written in the storage.
*/
public interface Batch extends Closeable {
- void put(byte[] key, byte[] value);
+ void put(byte[] key, byte[] value) throws IOException;
- void remove(byte[] key);
+ void remove(byte[] key) throws IOException;
- void deleteRange(byte[] beginKey, byte[] endKey);
+ void deleteRange(byte[] beginKey, byte[] endKey) throws IOException;
void clear();
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
index 1ecc4ba..a343b59 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
@@ -391,13 +391,21 @@ public class KeyValueStorageRocksDB implements KeyValueStorage {
}
@Override
- public void put(byte[] key, byte[] value) {
- writeBatch.put(key, value);
+ public void put(byte[] key, byte[] value) throws IOException {
+ try {
+ writeBatch.put(key, value);
+ } catch (RocksDBException e) {
+ throw new IOException("Failed to flush RocksDB batch", e);
+ }
}
@Override
- public void remove(byte[] key) {
- writeBatch.remove(key);
+ public void remove(byte[] key) throws IOException {
+ try {
+ writeBatch.delete(key);
+ } catch (RocksDBException e) {
+ throw new IOException("Failed to flush RocksDB batch", e);
+ }
}
@Override
@@ -406,8 +414,12 @@ public class KeyValueStorageRocksDB implements KeyValueStorage {
}
@Override
- public void deleteRange(byte[] beginKey, byte[] endKey) {
- writeBatch.deleteRange(beginKey, endKey);
+ public void deleteRange(byte[] beginKey, byte[] endKey) throws IOException {
+ try {
+ writeBatch.deleteRange(beginKey, endKey);
+ } catch (RocksDBException e) {
+ throw new IOException("Failed to flush RocksDB batch", e);
+ }
}
@Override
diff --git a/pom.xml b/pom.xml
index 2e5d0c4..8b326b0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -147,7 +147,7 @@
<datasketches.version>0.8.3</datasketches.version>
<protobuf.version>3.5.1</protobuf.version>
<protoc-gen-grpc-java.version>1.12.0</protoc-gen-grpc-java.version>
- <rocksdb.version>5.8.6</rocksdb.version>
+ <rocksdb.version>5.13.1</rocksdb.version>
<shrinkwrap.version>3.0.1</shrinkwrap.version>
<slf4j.version>1.7.25</slf4j.version>
<spotbugs-annotations.version>3.1.1</spotbugs-annotations.version>
diff --git a/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java b/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java
index 7d51453..4e72e1f 100644
--- a/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java
+++ b/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java
@@ -453,19 +453,20 @@ public class RocksdbKVStore<K, V> implements KVStore<K, V> {
}
private void putRaw(K key, byte[] keyBytes, V value, long revision) {
- WriteBatch batch = new WriteBatch();
- if (revision > 0) {
- // last revision has been set to revision bytes
- batch.put(metaCfHandle, LAST_REVISION, lastRevisionBytes);
- }
- if (null == value) {
- // delete a key if value is null
- batch.remove(dataCfHandle, keyBytes);
- } else {
- byte[] valBytes = valCoder.encode(value);
- batch.put(dataCfHandle, keyBytes, valBytes);
- }
try {
+ WriteBatch batch = new WriteBatch();
+ if (revision > 0) {
+ // last revision has been set to revision bytes
+ batch.put(metaCfHandle, LAST_REVISION, lastRevisionBytes);
+ }
+ if (null == value) {
+ // delete a key if value is null
+ batch.delete(dataCfHandle, keyBytes);
+ } else {
+ byte[] valBytes = valCoder.encode(value);
+ batch.put(dataCfHandle, keyBytes, valBytes);
+ }
+
db.write(writeOpts, batch);
} catch (RocksDBException e) {
throw new StateStoreRuntimeException("Error while updating key " + key
@@ -555,7 +556,11 @@ public class RocksdbKVStore<K, V> implements KVStore<K, V> {
}
private void putRaw(byte[] keyBytes, V value) {
- batch.put(dataCfHandle, keyBytes, valCoder.encode(value));
+ try {
+ batch.put(dataCfHandle, keyBytes, valCoder.encode(value));
+ } catch (RocksDBException e) {
+ throw new StateStoreRuntimeException(e);
+ }
}
@Override
@@ -568,7 +573,11 @@ public class RocksdbKVStore<K, V> implements KVStore<K, V> {
}
private void deleteRaw(byte[] keyBytes) {
- batch.remove(dataCfHandle, keyBytes);
+ try {
+ batch.delete(dataCfHandle, keyBytes);
+ } catch (RocksDBException e) {
+ throw new StateStoreRuntimeException(e);
+ }
}
@Override
@@ -579,7 +588,11 @@ public class RocksdbKVStore<K, V> implements KVStore<K, V> {
byte[] fromBytes = keyCoder.encode(from);
byte[] toBytes = keyCoder.encode(to);
- batch.deleteRange(dataCfHandle, fromBytes, toBytes);
+ try {
+ batch.deleteRange(dataCfHandle, fromBytes, toBytes);
+ } catch (RocksDBException e) {
+ throw new StateStoreRuntimeException(e);
+ }
}
@Override
diff --git a/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/mvcc/MVCCStoreImpl.java b/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/mvcc/MVCCStoreImpl.java
index 4368f66..e7f1150 100644
--- a/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/mvcc/MVCCStoreImpl.java
+++ b/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/mvcc/MVCCStoreImpl.java
@@ -427,6 +427,9 @@ class MVCCStoreImpl<K, V> extends RocksdbKVStore<K, V> implements MVCCStore<K, V
result.totalAmount(newAmount);
}
return result;
+ } catch (RocksDBException rde) {
+ result.close();
+ throw new StateStoreRuntimeException(rde);
} catch (StateStoreRuntimeException e) {
result.close();
throw e;
@@ -524,15 +527,18 @@ class MVCCStoreImpl<K, V> extends RocksdbKVStore<K, V> implements MVCCStore<K, V
result.code(Code.OK);
if (null != oldRecord) {
KeyValueImpl<K, V> prevKV = oldRecord.asKVRecord(
- recordFactory,
- key,
- valCoder);
+ recordFactory,
+ key,
+ valCoder);
result.prevKv(prevKV);
}
return result;
} catch (StateStoreRuntimeException e) {
result.close();
throw e;
+ } catch (RocksDBException e) {
+ result.close();
+ throw new StateStoreRuntimeException(e);
} finally {
if (null != record) {
record.recycle();
@@ -632,13 +638,17 @@ class MVCCStoreImpl<K, V> extends RocksdbKVStore<K, V> implements MVCCStore<K, V
void deleteBlind(WriteBatch batch,
byte[] key,
@Nullable byte[] endKey) {
- if (null == endKey) {
- batch.remove(key);
- } else {
- Pair<byte[], byte[]> realRange = getRealRange(key, endKey);
- endKey = realRange.getRight();
- ++endKey[endKey.length - 1];
- batch.deleteRange(realRange.getLeft(), endKey);
+ try {
+ if (null == endKey) {
+ batch.delete(key);
+ } else {
+ Pair<byte[], byte[]> realRange = getRealRange(key, endKey);
+ endKey = realRange.getRight();
+ ++endKey[endKey.length - 1];
+ batch.deleteRange(realRange.getLeft(), endKey);
+ }
+ } catch (RocksDBException e) {
+ throw new StateStoreRuntimeException(e);
}
}
@@ -660,7 +670,11 @@ class MVCCStoreImpl<K, V> extends RocksdbKVStore<K, V> implements MVCCStore<K, V
record.recycle();
}
numKvs.add(1L);
- batch.remove(rawKey);
+ try {
+ batch.delete(rawKey);
+ } catch (RocksDBException e) {
+ throw new StateStoreRuntimeException(e);
+ }
}
} else {
Pair<byte[], byte[]> realRange = getRealRange(rawKey, rawEndKey);
--
To stop receiving notification emails like this one, please contact
sijie@apache.org.