You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by ch...@apache.org on 2023/03/14 06:54:25 UTC
[bookkeeper] branch master updated: change rocksDB config level_compaction_dynamic_level_bytes to CFOptions (#3860)
This is an automated email from the ASF dual-hosted git repository.
chenhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 5542afe6b6 change rocksDB config level_compaction_dynamic_level_bytes to CFOptions (#3860)
5542afe6b6 is described below
commit 5542afe6b642e4af8896a6219814533221ee09b9
Author: lixinyang <84...@users.noreply.github.com>
AuthorDate: Tue Mar 14 14:54:17 2023 +0800
change rocksDB config level_compaction_dynamic_level_bytes to CFOptions (#3860)
### Motivation
After PR #3056 , Bookkeeper set `level_compaction_dynamic_level_bytes=true` as `TableOptions` in `entry_location_rocksdb.conf.default` , which will cause `level_compaction_dynamic_level_bytes` lose efficacy and will cause rocksDB .sst file compact sort chaos when update bookie release.
As RocksDB conf, `level_compaction_dynamic_level_bytes` need set as `CFOptions` https://github.com/facebook/rocksdb/blob/master/examples/rocksdb_option_file_example.ini
<img width="703" alt="image" src="https://user-images.githubusercontent.com/84127069/224640399-d5481fe5-7b75-4229-ac06-3d280aa9ae6d.png">
<img width="240" alt="image" src="https://user-images.githubusercontent.com/84127069/224640621-737d0a42-4e01-4f38-bd5a-862a93bc4b32.png">
### Changes
1. Change `level_compaction_dynamic_level_bytes=true` from `TableOptions` to `CFOptions` in `entry_location_rocksdb.conf.default` ;
---
.../bookie/storage/ldb/KeyValueStorageRocksDBTest.java | 18 ++++++++++++++++++
.../test/resources/conf/entry_location_rocksdb.conf | 6 +++---
.../test/resources/test_entry_location_rocksdb.conf | 6 +++---
conf/entry_location_rocksdb.conf.default | 6 +++---
4 files changed, 27 insertions(+), 9 deletions(-)
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java
index 6a0e00b0c1..2ef3e010f8 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java
@@ -54,6 +54,7 @@ public class KeyValueStorageRocksDBTest {
assertEquals(64 * 1024 * 1024, options.writeBufferSize());
assertEquals(4, options.maxWriteBufferNumber());
assertEquals(256 * 1024 * 1024, options.maxBytesForLevelBase());
+ assertEquals(true, options.levelCompactionDynamicLevelBytes());
rocksDB.close();
}
@@ -78,6 +79,7 @@ public class KeyValueStorageRocksDBTest {
assertEquals(CompressionType.LZ4_COMPRESSION, familyOptions.compressionType());
assertEquals(1024, familyOptions.writeBufferSize());
assertEquals(1, familyOptions.maxWriteBufferNumber());
+ assertEquals(true, familyOptions.levelCompactionDynamicLevelBytes());
rocksDB.close();
}
@@ -113,4 +115,20 @@ public class KeyValueStorageRocksDBTest {
// After the PR: https://github.com/facebook/rocksdb/pull/10826 merge, we can turn on this test.
assertEquals(ChecksumType.kxxHash, ((BlockBasedTableConfig) familyOptions.tableFormatConfig()).checksumType());
}
+
+ @Test
+ public void testLevelCompactionDynamicLevelBytesFromConfigurationFile() throws Exception {
+ ServerConfiguration configuration = new ServerConfiguration();
+ URL url = getClass().getClassLoader().getResource("conf/entry_location_rocksdb.conf");
+ configuration.setEntryLocationRocksdbConf(url.getPath());
+ File tmpDir = Files.createTempDirectory("bk-kv-rocksdbtest-file").toFile();
+ Files.createDirectory(Paths.get(tmpDir.toString(), "subDir"));
+ KeyValueStorageRocksDB rocksDB = new KeyValueStorageRocksDB(tmpDir.toString(), "subDir",
+ KeyValueStorageFactory.DbConfigType.EntryLocation, configuration);
+ assertNotNull(rocksDB.getColumnFamilyDescriptors());
+
+ List<ColumnFamilyDescriptor> columnFamilyDescriptorList = rocksDB.getColumnFamilyDescriptors();
+ ColumnFamilyOptions familyOptions = columnFamilyDescriptorList.get(0).getOptions();
+ assertEquals(true, familyOptions.levelCompactionDynamicLevelBytes());
+ }
}
diff --git a/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf
index df3ac9fc2e..6f6c1b4d05 100644
--- a/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf
+++ b/bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf
@@ -51,6 +51,8 @@
max_bytes_for_level_base=268435456
# set by jni: options.setTargetFileSizeBase
target_file_size_base=67108864
+ # set by jni: options.setLevelCompactionDynamicLevelBytes
+ level_compaction_dynamic_level_bytes=true
[TableOptions/BlockBasedTable "default"]
# set by jni: tableOptions.setBlockSize
@@ -64,6 +66,4 @@
# set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder]
filter_policy=rocksdb.BloomFilter:10:false
# set by jni: tableOptions.setCacheIndexAndFilterBlocks
- cache_index_and_filter_blocks=true
- # set by jni: options.setLevelCompactionDynamicLevelBytes
- level_compaction_dynamic_level_bytes=true
\ No newline at end of file
+ cache_index_and_filter_blocks=true
\ No newline at end of file
diff --git a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf
index fa3cf9c8d9..9d1c3e08c4 100644
--- a/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf
+++ b/bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf
@@ -31,6 +31,8 @@
write_buffer_size=1024
# set by jni: options.setMaxWriteBufferNumber
max_write_buffer_number=1
+ # set by jni: options.setLevelCompactionDynamicLevelBytes
+ level_compaction_dynamic_level_bytes=true
[TableOptions/BlockBasedTable "default"]
# set by jni: tableOptions.setBlockSize
@@ -44,6 +46,4 @@
# set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder]
filter_policy=rocksdb.BloomFilter:10:false
# set by jni: tableOptions.setCacheIndexAndFilterBlocks
- cache_index_and_filter_blocks=true
- # set by jni: options.setLevelCompactionDynamicLevelBytes
- level_compaction_dynamic_level_bytes=true
\ No newline at end of file
+ cache_index_and_filter_blocks=true
\ No newline at end of file
diff --git a/conf/entry_location_rocksdb.conf.default b/conf/entry_location_rocksdb.conf.default
index df3ac9fc2e..6f6c1b4d05 100644
--- a/conf/entry_location_rocksdb.conf.default
+++ b/conf/entry_location_rocksdb.conf.default
@@ -51,6 +51,8 @@
max_bytes_for_level_base=268435456
# set by jni: options.setTargetFileSizeBase
target_file_size_base=67108864
+ # set by jni: options.setLevelCompactionDynamicLevelBytes
+ level_compaction_dynamic_level_bytes=true
[TableOptions/BlockBasedTable "default"]
# set by jni: tableOptions.setBlockSize
@@ -64,6 +66,4 @@
# set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder]
filter_policy=rocksdb.BloomFilter:10:false
# set by jni: tableOptions.setCacheIndexAndFilterBlocks
- cache_index_and_filter_blocks=true
- # set by jni: options.setLevelCompactionDynamicLevelBytes
- level_compaction_dynamic_level_bytes=true
\ No newline at end of file
+ cache_index_and_filter_blocks=true
\ No newline at end of file