You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by sh...@apache.org on 2022/07/11 22:58:44 UTC
[bookkeeper] branch master updated: stage 2.3 ~ 2.4: (#3397)
This is an automated email from the ASF dual-hosted git repository.
shoothzj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 1fcf74c1dc stage 2.3 ~ 2.4: (#3397)
1fcf74c1dc is described below
commit 1fcf74c1dc18cfc96960ebf41170177f5a98e2d6
Author: StevenLuMT <42...@users.noreply.github.com>
AuthorDate: Tue Jul 12 06:58:38 2022 +0800
stage 2.3 ~ 2.4: (#3397)
Descriptions of the changes in this PR:
### Motivation
planning for index dir: [mail talking](https://lists.apache.org/thread/r657jf55khl59bbqltj2s95107lbkr0w)
stage 3~4 :
1) disk monitor(DiskChecker/LedgerDirsMonitor) have checked indexDir,it's good;
2) add indexDirs in statsLogger, and add indexDirs's Listener.
### Changes
1. add indexDirs in statsLogger, and add indexDirs's Listener.
2. update index's testcase for SingleDirectoryDbLedgerStorage
---
.../ldb/SingleDirectoryDbLedgerStorage.java | 31 ++++++++++++++--------
.../storage/ldb/DbLedgerStorageIndexDirTest.java | 9 ++++++-
2 files changed, 28 insertions(+), 12 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
index 9ac56b9649..a1131df815 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/storage/ldb/SingleDirectoryDbLedgerStorage.java
@@ -80,8 +80,8 @@ import org.apache.bookkeeper.stats.StatsLogger;
import org.apache.bookkeeper.stats.ThreadRegistry;
import org.apache.bookkeeper.util.MathUtils;
import org.apache.bookkeeper.util.collections.ConcurrentLongHashMap;
+import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang.mutable.MutableLong;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -152,17 +152,21 @@ public class SingleDirectoryDbLedgerStorage implements CompactableLedgerStorage
checkArgument(ledgerDirsManager.getAllLedgerDirs().size() == 1,
"Db implementation only allows for one storage dir");
- String indexBaseDir = indexDirsManager.getAllLedgerDirs().get(0).toString();
- if (StringUtils.isBlank(indexBaseDir)) {
- indexBaseDir = ledgerDirsManager.getAllLedgerDirs().get(0).toString();
+ String ledgerBaseDir = ledgerDirsManager.getAllLedgerDirs().get(0).getPath();
+ // indexBaseDir default use ledgerBaseDir
+ String indexBaseDir = ledgerBaseDir;
+ if (CollectionUtils.isEmpty(indexDirsManager.getAllLedgerDirs())) {
log.info("indexDir is not specified, use default, creating single directory db ledger storage on {}",
indexBaseDir);
} else {
+ // if indexDir is specified, set new value
+ indexBaseDir = indexDirsManager.getAllLedgerDirs().get(0).getPath();
log.info("indexDir is specified, creating single directory db ledger storage on {}", indexBaseDir);
}
- StatsLogger ledgerDirStatsLogger = statsLogger.scopeLabel("ledgerDir",
- ledgerDirsManager.getAllLedgerDirs().get(0).getPath());
+ StatsLogger ledgerIndexDirStatsLogger = statsLogger
+ .scopeLabel("ledgerDir", ledgerBaseDir)
+ .scopeLabel("indexDir", indexBaseDir);
this.writeCacheMaxSize = writeCacheSize;
this.writeCache = new WriteCache(allocator, writeCacheMaxSize / 2);
@@ -180,9 +184,10 @@ public class SingleDirectoryDbLedgerStorage implements CompactableLedgerStorage
readCache = new ReadCache(allocator, readCacheMaxSize);
- ledgerIndex = new LedgerMetadataIndex(conf, KeyValueStorageRocksDB.factory, indexBaseDir, ledgerDirStatsLogger);
+ ledgerIndex = new LedgerMetadataIndex(conf,
+ KeyValueStorageRocksDB.factory, indexBaseDir, ledgerIndexDirStatsLogger);
entryLocationIndex = new EntryLocationIndex(conf,
- KeyValueStorageRocksDB.factory, indexBaseDir, ledgerDirStatsLogger);
+ KeyValueStorageRocksDB.factory, indexBaseDir, ledgerIndexDirStatsLogger);
transientLedgerInfoCache = ConcurrentLongHashMap.<TransientLedgerInfo>newBuilder()
.expectedItems(16 * 1024)
@@ -193,17 +198,18 @@ public class SingleDirectoryDbLedgerStorage implements CompactableLedgerStorage
TransientLedgerInfo.LEDGER_INFO_CACHING_TIME_MINUTES, TimeUnit.MINUTES);
this.entryLogger = entryLogger;
- gcThread = new GarbageCollectorThread(conf, ledgerManager, ledgerDirsManager, this, entryLogger, statsLogger);
+ gcThread = new GarbageCollectorThread(conf,
+ ledgerManager, ledgerDirsManager, this, entryLogger, ledgerIndexDirStatsLogger);
dbLedgerStorageStats = new DbLedgerStorageStats(
- ledgerDirStatsLogger,
+ ledgerIndexDirStatsLogger,
() -> writeCache.size() + writeCacheBeingFlushed.size(),
() -> writeCache.count() + writeCacheBeingFlushed.count(),
() -> readCache.size(),
() -> readCache.count()
);
- flushExecutorTime = ledgerDirStatsLogger.getThreadScopedCounter("db-storage-thread-time");
+ flushExecutorTime = ledgerIndexDirStatsLogger.getThreadScopedCounter("db-storage-thread-time");
executor.submit(() -> {
ThreadRegistry.register(dbStoragerExecutorName, 0);
@@ -213,6 +219,9 @@ public class SingleDirectoryDbLedgerStorage implements CompactableLedgerStorage
});
ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener());
+ if (!ledgerBaseDir.equals(indexBaseDir)) {
+ indexDirsManager.addLedgerDirsListener(getLedgerDirsListener());
+ }
}
@Override
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java
index 1f8e64c4cb..961f340c81 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/DbLedgerStorageIndexDirTest.java
@@ -69,6 +69,7 @@ public class DbLedgerStorageIndexDirTest {
int gcWaitTime = 1000;
ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
conf.setGcWaitTime(gcWaitTime);
+ /** the testcase cover specify indexDir for the class {@link SingleDirectoryDbLedgerStorage} */
conf.setLedgerStorageClass(DbLedgerStorage.class.getName());
conf.setProperty(DbLedgerStorage.WRITE_CACHE_MAX_SIZE_MB, 1);
conf.setProperty(DbLedgerStorage.MAX_THROTTLE_TIME_MILLIS, 1000);
@@ -124,8 +125,14 @@ public class DbLedgerStorageIndexDirTest {
}
@Test
- public void checkIndexDirectoryStructure() {
+ public void checkIndexNotExistsInLedgerDirStructure() {
+ // old logic bugfix
assertEquals(false, hasIndexStructure(tmpLedgerDir));
+ }
+
+ @Test
+ public void checkIndexDirectoryStructure() {
+ // index new logic
assertEquals(true, hasIndexStructure(tmpIndexDir));
}