You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@bookkeeper.apache.org by GitBox <gi...@apache.org> on 2018/06/06 06:53:39 UTC
[GitHub] sijie closed pull request #1466: Upgrade RocksDB to 5.13.1
sijie closed pull request #1466: Upgrade RocksDB to 5.13.1
URL: https://github.com/apache/bookkeeper/pull/1466
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
index 98159e78b..d636cab3c 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-all.bin.txt
@@ -269,7 +269,7 @@ Apache Software License, Version 2.
- lib/org.eclipse.jetty-jetty-server-9.4.5.v20170502.jar [34]
- lib/org.eclipse.jetty-jetty-servlet-9.4.5.v20170502.jar [34]
- lib/org.eclipse.jetty-jetty-util-9.4.5.v20170502.jar [34]
-- lib/org.rocksdb-rocksdbjni-5.8.6.jar [35]
+- lib/org.rocksdb-rocksdbjni-5.13.1.jar [35]
- lib/com.beust-jcommander-1.48.jar [36]
- lib/com.yahoo.datasketches-memory-0.8.3.jar [37]
- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [37]
@@ -332,7 +332,7 @@ Apache Software License, Version 2.
[32] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=3ad2e8
[33] Source available at https://github.com/apache/zookeeper/tree/release-3.5.3
[34] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.5.v20170502
-[35] Source available at https://github.com/facebook/rocksdb/tree/v5.8.6
+[35] Source available at https://github.com/facebook/rocksdb/tree/v5.13.1
[36] Source available at https://github.com/cbeust/jcommander/tree/jcommander-1.48
[37] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
[38] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
@@ -453,7 +453,7 @@ interchange format, which can be obtained at:
* https://github.com/google/protobuf
------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-5.8.6.jar is derived from leveldb, which is under the following license.
+lib/org.rocksdb-rocksdbjni-5.13.1.jar is derived from leveldb, which is under the following license.
Copyright (c) 2011 The LevelDB Authors. All rights reserved.
diff --git a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
index e97057728..a012f6455 100644
--- a/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
+++ b/bookkeeper-dist/src/main/resources/LICENSE-server.bin.txt
@@ -234,7 +234,7 @@ Apache Software License, Version 2.
- lib/org.eclipse.jetty-jetty-server-9.4.5.v20170502.jar [21]
- lib/org.eclipse.jetty-jetty-servlet-9.4.5.v20170502.jar [21]
- lib/org.eclipse.jetty-jetty-util-9.4.5.v20170502.jar [21]
-- lib/org.rocksdb-rocksdbjni-5.8.6.jar [22]
+- lib/org.rocksdb-rocksdbjni-5.13.1.jar [22]
- lib/com.beust-jcommander-1.48.jar [23]
- lib/com.yahoo.datasketches-memory-0.8.3.jar [24]
- lib/com.yahoo.datasketches-sketches-core-0.8.3.jar [24]
@@ -285,7 +285,7 @@ Apache Software License, Version 2.
[19] Source available at https://git-wip-us.apache.org/repos/asf?p=commons-lang.git;a=tag;h=3ad2e8
[20] Source available at https://github.com/apache/zookeeper/tree/release-3.5.3
[21] Source available at https://github.com/eclipse/jetty.project/tree/jetty-9.4.5.v20170502
-[22] Source available at https://github.com/facebook/rocksdb/tree/v5.8.6
+[22] Source available at https://github.com/facebook/rocksdb/tree/v5.13.1
[23] Source available at https://github.com/cbeust/jcommander/tree/jcommander-1.48
[24] Source available at https://github.com/DataSketches/sketches-core/tree/sketches-0.8.3
[25] Source available at https://github.com/lz4/lz4-java/tree/1.3.0
@@ -371,7 +371,7 @@ interchange format, which can be obtained at:
* https://github.com/google/protobuf
------------------------------------------------------------------------------------
-lib/org.rocksdb-rocksdbjni-5.8.6.jar is derived from leveldb, which is under the following license.
+lib/org.rocksdb-rocksdbjni-5.13.1.jar is derived from leveldb, which is under the following license.
Copyright (c) 2011 The LevelDB Authors. All rights reserved.
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorage.java
index b9bbb2a6a..aa0119ad6 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorage.java
@@ -149,11 +149,11 @@
* Interface for a batch to be written in the storage.
*/
public interface Batch extends Closeable {
- void put(byte[] key, byte[] value);
+ void put(byte[] key, byte[] value) throws IOException;
- void remove(byte[] key);
+ void remove(byte[] key) throws IOException;
- void deleteRange(byte[] beginKey, byte[] endKey);
+ void deleteRange(byte[] beginKey, byte[] endKey) throws IOException;
void clear();
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
index 1ecc4ba68..a343b59a8 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
@@ -391,13 +391,21 @@ public void close() {
}
@Override
- public void put(byte[] key, byte[] value) {
- writeBatch.put(key, value);
+ public void put(byte[] key, byte[] value) throws IOException {
+ try {
+ writeBatch.put(key, value);
+ } catch (RocksDBException e) {
+ throw new IOException("Failed to flush RocksDB batch", e);
+ }
}
@Override
- public void remove(byte[] key) {
- writeBatch.remove(key);
+ public void remove(byte[] key) throws IOException {
+ try {
+ writeBatch.delete(key);
+ } catch (RocksDBException e) {
+ throw new IOException("Failed to flush RocksDB batch", e);
+ }
}
@Override
@@ -406,8 +414,12 @@ public void clear() {
}
@Override
- public void deleteRange(byte[] beginKey, byte[] endKey) {
- writeBatch.deleteRange(beginKey, endKey);
+ public void deleteRange(byte[] beginKey, byte[] endKey) throws IOException {
+ try {
+ writeBatch.deleteRange(beginKey, endKey);
+ } catch (RocksDBException e) {
+ throw new IOException("Failed to flush RocksDB batch", e);
+ }
}
@Override
diff --git a/pom.xml b/pom.xml
index bc9d35214..5881f12b2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -147,7 +147,7 @@
<datasketches.version>0.8.3</datasketches.version>
<protobuf.version>3.5.1</protobuf.version>
<protoc-gen-grpc-java.version>1.12.0</protoc-gen-grpc-java.version>
- <rocksdb.version>5.8.6</rocksdb.version>
+ <rocksdb.version>5.13.1</rocksdb.version>
<shrinkwrap.version>3.0.1</shrinkwrap.version>
<slf4j.version>1.7.25</slf4j.version>
<spotbugs-annotations.version>3.1.1</spotbugs-annotations.version>
diff --git a/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java b/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java
index 7d514533e..4e72e1f5c 100644
--- a/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java
+++ b/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java
@@ -453,19 +453,20 @@ synchronized void put(K key, V value, long revision) {
}
private void putRaw(K key, byte[] keyBytes, V value, long revision) {
- WriteBatch batch = new WriteBatch();
- if (revision > 0) {
- // last revision has been set to revision bytes
- batch.put(metaCfHandle, LAST_REVISION, lastRevisionBytes);
- }
- if (null == value) {
- // delete a key if value is null
- batch.remove(dataCfHandle, keyBytes);
- } else {
- byte[] valBytes = valCoder.encode(value);
- batch.put(dataCfHandle, keyBytes, valBytes);
- }
try {
+ WriteBatch batch = new WriteBatch();
+ if (revision > 0) {
+ // last revision has been set to revision bytes
+ batch.put(metaCfHandle, LAST_REVISION, lastRevisionBytes);
+ }
+ if (null == value) {
+ // delete a key if value is null
+ batch.delete(dataCfHandle, keyBytes);
+ } else {
+ byte[] valBytes = valCoder.encode(value);
+ batch.put(dataCfHandle, keyBytes, valBytes);
+ }
+
db.write(writeOpts, batch);
} catch (RocksDBException e) {
throw new StateStoreRuntimeException("Error while updating key " + key
@@ -555,7 +556,11 @@ public void put(K key, V value) {
}
private void putRaw(byte[] keyBytes, V value) {
- batch.put(dataCfHandle, keyBytes, valCoder.encode(value));
+ try {
+ batch.put(dataCfHandle, keyBytes, valCoder.encode(value));
+ } catch (RocksDBException e) {
+ throw new StateStoreRuntimeException(e);
+ }
}
@Override
@@ -568,7 +573,11 @@ public void delete(K key) {
}
private void deleteRaw(byte[] keyBytes) {
- batch.remove(dataCfHandle, keyBytes);
+ try {
+ batch.delete(dataCfHandle, keyBytes);
+ } catch (RocksDBException e) {
+ throw new StateStoreRuntimeException(e);
+ }
}
@Override
@@ -579,7 +588,11 @@ public void deleteRange(K from, K to) {
byte[] fromBytes = keyCoder.encode(from);
byte[] toBytes = keyCoder.encode(to);
- batch.deleteRange(dataCfHandle, fromBytes, toBytes);
+ try {
+ batch.deleteRange(dataCfHandle, fromBytes, toBytes);
+ } catch (RocksDBException e) {
+ throw new StateStoreRuntimeException(e);
+ }
}
@Override
diff --git a/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/mvcc/MVCCStoreImpl.java b/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/mvcc/MVCCStoreImpl.java
index 4368f66da..e7f115067 100644
--- a/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/mvcc/MVCCStoreImpl.java
+++ b/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/mvcc/MVCCStoreImpl.java
@@ -427,6 +427,9 @@ private void executeBatch(WriteBatch batch) {
result.totalAmount(newAmount);
}
return result;
+ } catch (RocksDBException rde) {
+ result.close();
+ throw new StateStoreRuntimeException(rde);
} catch (StateStoreRuntimeException e) {
result.close();
throw e;
@@ -524,15 +527,18 @@ private void executeBatch(WriteBatch batch) {
result.code(Code.OK);
if (null != oldRecord) {
KeyValueImpl<K, V> prevKV = oldRecord.asKVRecord(
- recordFactory,
- key,
- valCoder);
+ recordFactory,
+ key,
+ valCoder);
result.prevKv(prevKV);
}
return result;
} catch (StateStoreRuntimeException e) {
result.close();
throw e;
+ } catch (RocksDBException e) {
+ result.close();
+ throw new StateStoreRuntimeException(e);
} finally {
if (null != record) {
record.recycle();
@@ -632,13 +638,17 @@ private void executeBatch(WriteBatch batch) {
void deleteBlind(WriteBatch batch,
byte[] key,
@Nullable byte[] endKey) {
- if (null == endKey) {
- batch.remove(key);
- } else {
- Pair<byte[], byte[]> realRange = getRealRange(key, endKey);
- endKey = realRange.getRight();
- ++endKey[endKey.length - 1];
- batch.deleteRange(realRange.getLeft(), endKey);
+ try {
+ if (null == endKey) {
+ batch.delete(key);
+ } else {
+ Pair<byte[], byte[]> realRange = getRealRange(key, endKey);
+ endKey = realRange.getRight();
+ ++endKey[endKey.length - 1];
+ batch.deleteRange(realRange.getLeft(), endKey);
+ }
+ } catch (RocksDBException e) {
+ throw new StateStoreRuntimeException(e);
}
}
@@ -660,7 +670,11 @@ long deleteUsingIter(WriteBatch batch,
record.recycle();
}
numKvs.add(1L);
- batch.remove(rawKey);
+ try {
+ batch.delete(rawKey);
+ } catch (RocksDBException e) {
+ throw new StateStoreRuntimeException(e);
+ }
}
} else {
Pair<byte[], byte[]> realRange = getRealRange(rawKey, rawEndKey);
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services