You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by si...@apache.org on 2020/09/11 05:01:57 UTC
[bookkeeper] branch master updated: Updated netty,
netty-boringssl and rocksdb
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 7850554 Updated netty,netty-boringssl and rocksdb
7850554 is described below
commit 7850554a8349cedf1d9236a242e2f7739b471bc9
Author: odidev <od...@puresoftware.com>
AuthorDate: Fri Sep 11 10:31:46 2020 +0530
Updated netty,netty-boringssl and rocksdb
### Descriptions of the changes in this PR:
-Updated netty,netty-boringssl and rocksdb to latest version for aarch64 support
-Updated deprecated methods of rocksdb to remove compilation warning and resolve build failure of bookkeeper-server on amd64 and aarch64 platforms as Werror flag is enabled.
### Motivation
Build of the bookkeeper-server package was failing on amd64 and aarch64 platforms.
### Changes
There are many methods that are marked deprecated in rocksdb, but in use by bookkeeper package.
So updated those methods in bookkeeper package according to current rocksdb implementation.
Also updated netty,netty-boringssl and rocksdb to latest version as they are having aarch64 support.
Master Issue: #2378
Reviewers: Enrico Olivelli <eo...@gmail.com>, Sijie Guo <None>
This closes #2379 from odidev/bookkeeper_aarch
---
.../bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java | 10 ++++++----
pom.xml | 8 ++++----
.../org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java | 5 ++++-
3 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
index 88411ce..c274791 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
@@ -37,9 +37,11 @@ import org.apache.bookkeeper.bookie.storage.ldb.KeyValueStorageFactory.DbConfigT
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.BloomFilter;
+import org.rocksdb.Cache;
import org.rocksdb.ChecksumType;
import org.rocksdb.CompressionType;
import org.rocksdb.InfoLogLevel;
+import org.rocksdb.LRUCache;
import org.rocksdb.Options;
import org.rocksdb.ReadOptions;
import org.rocksdb.RocksDB;
@@ -125,21 +127,21 @@ public class KeyValueStorageRocksDB implements KeyValueStorage {
}
options.setLevelZeroFileNumCompactionTrigger(numFilesInLevel0);
options.setMaxBytesForLevelBase(maxSizeInLevel1MB * 1024 * 1024);
- options.setMaxBackgroundCompactions(16);
- options.setMaxBackgroundFlushes(16);
+ options.setMaxBackgroundJobs(32);
options.setIncreaseParallelism(32);
options.setMaxTotalWalSize(512 * 1024 * 1024);
options.setMaxOpenFiles(-1);
options.setTargetFileSizeBase(sstSizeMB * 1024 * 1024);
options.setDeleteObsoleteFilesPeriodMicros(TimeUnit.HOURS.toMicros(1));
+ final Cache cache = new LRUCache(blockCacheSize);
BlockBasedTableConfig tableOptions = new BlockBasedTableConfig();
tableOptions.setBlockSize(blockSize);
- tableOptions.setBlockCacheSize(blockCacheSize);
+ tableOptions.setBlockCache(cache);
tableOptions.setFormatVersion(2);
tableOptions.setChecksumType(ChecksumType.kxxHash);
if (bloomFilterBitsPerKey > 0) {
- tableOptions.setFilter(new BloomFilter(bloomFilterBitsPerKey, false));
+ tableOptions.setFilterPolicy(new BloomFilter(bloomFilterBitsPerKey, false));
}
// Options best suited for HDDs
diff --git a/pom.xml b/pom.xml
index e76e883..5302235 100644
--- a/pom.xml
+++ b/pom.xml
@@ -150,8 +150,8 @@
<lombok.version>1.18.10</lombok.version>
<lz4.version>1.3.0</lz4.version>
<mockito.version>3.0.0</mockito.version>
- <netty.version>4.1.32.Final</netty.version>
- <netty-boringssl.version>2.0.20.Final</netty-boringssl.version>
+ <netty.version>4.1.50.Final</netty.version>
+ <netty-boringssl.version>2.0.31.Final</netty-boringssl.version>
<ostrich.version>9.1.3</ostrich.version>
<powermock.version>2.0.2</powermock.version>
<prometheus.version>0.8.1</prometheus.version>
@@ -161,7 +161,7 @@
<protoc3.version>3.5.1-1</protoc3.version>
<protoc-gen-grpc-java.version>1.12.0</protoc-gen-grpc-java.version>
<reflections.version>0.9.11</reflections.version>
- <rocksdb.version>5.13.1</rocksdb.version>
+ <rocksdb.version>6.10.2</rocksdb.version>
<shrinkwrap.version>3.0.1</shrinkwrap.version>
<slf4j.version>1.7.25</slf4j.version>
<snakeyaml.version>1.19</snakeyaml.version>
@@ -191,7 +191,7 @@
<maven-javadoc-plugin.version>3.1.1</maven-javadoc-plugin.version>
<maven-shade-plugin.version>3.2.0</maven-shade-plugin.version>
<maven-source-plugin.version>2.2.1</maven-source-plugin.version>
- <maven-surefire-plugin.version>2.21.0</maven-surefire-plugin.version>
+ <maven-surefire-plugin.version>3.0.0-M5</maven-surefire-plugin.version>
<nar-maven-plugin.version>3.5.2</nar-maven-plugin.version>
<os-maven-plugin.version>1.4.1.Final</os-maven-plugin.version>
<protobuf-maven-plugin.version>0.5.0</protobuf-maven-plugin.version>
diff --git a/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java b/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java
index 64ff0b1..0c4b95f 100644
--- a/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java
+++ b/stream/statelib/src/main/java/org/apache/bookkeeper/statelib/impl/kv/RocksdbKVStore.java
@@ -66,11 +66,13 @@ import org.apache.bookkeeper.statelib.impl.rocksdb.RocksUtils;
import org.apache.bookkeeper.statelib.impl.rocksdb.checkpoint.RocksCheckpointer;
import org.apache.commons.lang3.tuple.Pair;
import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.Cache;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.ColumnFamilyOptions;
import org.rocksdb.DBOptions;
import org.rocksdb.FlushOptions;
+import org.rocksdb.LRUCache;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.rocksdb.RocksIterator;
@@ -254,7 +256,8 @@ public class RocksdbKVStore<K, V> implements KVStore<K, V> {
// initialize the db options
final BlockBasedTableConfig tableConfig = new BlockBasedTableConfig();
- tableConfig.setBlockCacheSize(BLOCK_CACHE_SIZE);
+ final Cache cache = new LRUCache(BLOCK_CACHE_SIZE);
+ tableConfig.setBlockCache(cache);
tableConfig.setBlockSize(BLOCK_SIZE);
tableConfig.setChecksumType(DEFAULT_CHECKSUM_TYPE);