You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by si...@apache.org on 2018/11/30 08:27:27 UTC
[bookkeeper] branch master updated: Set default sizes of
DbLedgerStorage read and write cache to be proportional to JVM direct
memory
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 1285d20 Set default sizes of DbLedgerStorage read and write cache to be proportional to JVM direct memory
1285d20 is described below
commit 1285d20f5200a1a1e4e5ed7de4c712559c6123b4
Author: Matteo Merli <mm...@apache.org>
AuthorDate: Fri Nov 30 00:27:23 2018 -0800
Set default sizes of DbLedgerStorage read and write cache to be proportional to JVM direct memory
### Motivation
To simplify Bookie configuration when using `DbLedgerStorage`, set the memory size defaults for WriteCache, ReadCache and RocksDB block cache to be pegged to the available direct memory configured in the JVM.
User can always configure specific values and override this behavior.
Reviewers: Enrico Olivelli <eo...@gmail.com>, Sijie Guo <si...@apache.org>
This closes #1813 from merlimat/auto-conf-db-storage-mem
---
.../bookkeeper/bookie/storage/ldb/DbLedgerStorage.java | 17 ++++++++++++-----
.../bookie/storage/ldb/KeyValueStorageRocksDB.java | 9 ++++++++-
.../bookie/storage/ldb/DbLedgerStorageTest.java | 2 ++
.../org/apache/bookkeeper/conf/TestBKConfiguration.java | 3 +++
4 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
index 831de53..aad42ad 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorage.java
@@ -28,6 +28,9 @@ import com.google.common.collect.Lists;
import io.netty.buffer.ByteBuf;
import io.netty.util.concurrent.DefaultThreadFactory;
+//CHECKSTYLE.OFF: IllegalImport
+import io.netty.util.internal.PlatformDependent;
+//CHECKSTYLE.ON: IllegalImport
import java.io.File;
import java.io.IOException;
@@ -66,17 +69,17 @@ import org.apache.bookkeeper.util.DiskChecker;
@Slf4j
public class DbLedgerStorage implements LedgerStorage {
- static final String WRITE_CACHE_MAX_SIZE_MB = "dbStorage_writeCacheMaxSizeMb";
+ public static final String WRITE_CACHE_MAX_SIZE_MB = "dbStorage_writeCacheMaxSizeMb";
- static final String READ_AHEAD_CACHE_MAX_SIZE_MB = "dbStorage_readAheadCacheMaxSizeMb";
+ public static final String READ_AHEAD_CACHE_MAX_SIZE_MB = "dbStorage_readAheadCacheMaxSizeMb";
static final String MAX_THROTTLE_TIME_MILLIS = "dbStorage_maxThrottleTimeMs";
- private static final long DEFAULT_WRITE_CACHE_MAX_SIZE_MB = 16;
- private static final long DEFAULT_READ_CACHE_MAX_SIZE_MB = 16;
-
private static final int MB = 1024 * 1024;
+ private static final long DEFAULT_WRITE_CACHE_MAX_SIZE_MB = (long) (0.25 * PlatformDependent.maxDirectMemory())
+ / MB;
+ private static final long DEFAULT_READ_CACHE_MAX_SIZE_MB = (long) (0.25 * PlatformDependent.maxDirectMemory()) / MB;
private int numberOfDirs;
private List<SingleDirectoryDbLedgerStorage> ledgerStorageList;
@@ -97,6 +100,10 @@ public class DbLedgerStorage implements LedgerStorage {
log.info(" - Write cache size: {} MB", writeCacheMaxSize / MB);
log.info(" - Read Cache: {} MB", readCacheMaxSize / MB);
+ if (readCacheMaxSize + writeCacheMaxSize > PlatformDependent.maxDirectMemory()) {
+ throw new IOException("Read and write cache sizes exceed the configured max direct memory size");
+ }
+
long perDirectoryWriteCacheSize = writeCacheMaxSize / numberOfDirs;
long perDirectoryReadCacheSize = readCacheMaxSize / numberOfDirs;
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
index a343b59..eff40e8 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDB.java
@@ -24,6 +24,10 @@ import static com.google.common.base.Preconditions.checkState;
import com.google.common.primitives.UnsignedBytes;
+//CHECKSTYLE.OFF: IllegalImport
+import io.netty.util.internal.PlatformDependent;
+//CHECKSTYLE.ON: IllegalImport
+
import java.io.IOException;
import java.util.Comparator;
import java.util.Map.Entry;
@@ -97,13 +101,16 @@ public class KeyValueStorageRocksDB implements KeyValueStorage {
options.setCreateIfMissing(true);
if (dbConfigType == DbConfigType.Huge) {
+ // Set default RocksDB block-cache size to 10% of direct mem, unless override
+ long defaultRocksDBBlockCacheSizeBytes = PlatformDependent.maxDirectMemory() / 10;
+
long writeBufferSizeMB = conf.getInt(ROCKSDB_WRITE_BUFFER_SIZE_MB, 64);
long sstSizeMB = conf.getInt(ROCKSDB_SST_SIZE_MB, 64);
int numLevels = conf.getInt(ROCKSDB_NUM_LEVELS, -1);
int numFilesInLevel0 = conf.getInt(ROCKSDB_NUM_FILES_IN_LEVEL0, 4);
long maxSizeInLevel1MB = conf.getLong(ROCKSDB_MAX_SIZE_IN_LEVEL1_MB, 256);
int blockSize = conf.getInt(ROCKSDB_BLOCK_SIZE, 64 * 1024);
- long blockCacheSize = conf.getLong(ROCKSDB_BLOCK_CACHE_SIZE, 256 * 1024 * 1024);
+ long blockCacheSize = conf.getLong(ROCKSDB_BLOCK_CACHE_SIZE, defaultRocksDBBlockCacheSizeBytes);
int bloomFilterBitsPerKey = conf.getInt(ROCKSDB_BLOOM_FILTERS_BITS_PER_KEY, 10);
boolean lz4CompressionEnabled = conf.getBoolean(ROCKSDB_LZ4_COMPRESSION_ENABLED, true);
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java
index fb0a97a..3a3e982 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageTest.java
@@ -245,6 +245,8 @@ public class DbLedgerStorageTest {
File secondDir = new File(tmpDir, "dir2");
ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
conf.setGcWaitTime(gcWaitTime);
+ conf.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 4);
+ conf.setProperty(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB, 4);
conf.setLedgerStorageClass(DbLedgerStorage.class.getName());
conf.setLedgerDirNames(new String[] { firstDir.getCanonicalPath(), secondDir.getCanonicalPath() });
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java
index aba12a7..9073b01 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/conf/TestBKConfiguration.java
@@ -26,6 +26,7 @@ import java.net.SocketException;
import java.util.Collections;
import java.util.Enumeration;
+import org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,6 +54,8 @@ public class TestBKConfiguration {
confReturn.setGcWaitTime(1000);
confReturn.setDiskUsageThreshold(0.999f);
confReturn.setDiskUsageWarnThreshold(0.99f);
+ confReturn.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 4);
+ confReturn.setProperty(DbLedgerStorage.READ_AHEAD_CACHE_MAX_SIZE_MB, 4);
setLoopbackInterfaceAndAllowLoopback(confReturn);
return confReturn;
}