You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by mm...@apache.org on 2022/07/22 16:59:47 UTC
[bookkeeper] branch master updated: Added flag to control whether to transition to read-only mode when any disks full (#3212)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 34746858b4 Added flag to control whether to transition to read-only mode when any disks full (#3212)
34746858b4 is described below
commit 34746858b47e3962914f646803e930afe77859a3
Author: Hang Chen <ch...@apache.org>
AuthorDate: Sat Jul 23 00:59:42 2022 +0800
Added flag to control whether to transition to read-only mode when any disks full (#3212)
* Added flag to control whether to transition to read-only mode when any disk is full
* add unit test
* address comments
* add docs and config in conf/bk_server.conf
---
.../org/apache/bookkeeper/bookie/BookieImpl.java | 21 +++++
.../bookkeeper/bookie/LedgerDirsManager.java | 13 ++++
.../bookkeeper/bookie/LedgerDirsMonitor.java | 26 ++++++-
.../bookkeeper/conf/ServerConfiguration.java | 22 ++++++
.../bookkeeper/bookie/LedgerDirsManagerTest.java | 90 ++++++++++++++++++++++
conf/bk_server.conf | 9 ++-
site3/website/docs/reference/config.md | 6 +-
7 files changed, 181 insertions(+), 6 deletions(-)
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
index 9bc084ee92..4dc717e9bb 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java
@@ -741,6 +741,9 @@ public class BookieImpl extends BookieCriticalThread implements Bookie {
@Override
public void diskWritable(File disk) {
+ if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) {
+ return;
+ }
// Transition to writable mode when a disk becomes writable again.
stateManager.setHighPriorityWritesAvailability(true);
stateManager.transitionToWritableMode();
@@ -748,6 +751,24 @@ public class BookieImpl extends BookieCriticalThread implements Bookie {
@Override
public void diskJustWritable(File disk) {
+ if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) {
+ return;
+ }
+ // Transition to writable mode when a disk becomes writable again.
+ stateManager.setHighPriorityWritesAvailability(true);
+ stateManager.transitionToWritableMode();
+ }
+
+ @Override
+ public void anyDiskFull(boolean highPriorityWritesAllowed) {
+ if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) {
+ stateManager.setHighPriorityWritesAvailability(highPriorityWritesAllowed);
+ stateManager.transitionToReadOnlyMode();
+ }
+ }
+
+ @Override
+ public void allDisksWritable() {
// Transition to writable mode when a disk becomes writable again.
stateManager.setHighPriorityWritesAvailability(true);
stateManager.transitionToWritableMode();
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
index a9d438e4b7..b9934a2ba7 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
@@ -434,6 +434,19 @@ public class LedgerDirsManager {
*/
default void allDisksFull(boolean highPriorityWritesAllowed) {}
+ /**
+ * This will be notified whenever all disks are detected as not full.
+ *
+ */
+ default void allDisksWritable() {}
+
+ /**
+ * This will be notified whenever any disks are detected as full.
+ *
+ * @param highPriorityWritesAllowed the parameter indicates we are still have disk spaces for high priority
+ * * writes even disks are detected as "full"
+ */
+ default void anyDiskFull(boolean highPriorityWritesAllowed) {}
/**
* This will notify the fatal errors.
*/
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java
index 2b7c90152d..f7c2dc31ba 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java
@@ -52,7 +52,7 @@ class LedgerDirsMonitor {
private final ServerConfiguration conf;
private final DiskChecker diskChecker;
private final List<LedgerDirsManager> dirsManagers;
- private long minUsableSizeForHighPriorityWrites;
+ private final long minUsableSizeForHighPriorityWrites;
private ScheduledExecutorService executor;
private ScheduledFuture<?> checkTask;
@@ -68,6 +68,10 @@ class LedgerDirsMonitor {
private void check(final LedgerDirsManager ldm) {
final ConcurrentMap<File, Float> diskUsages = ldm.getDiskUsages();
+ boolean someDiskFulled = false;
+ boolean highPriorityWritesAllowed = true;
+ boolean someDiskRecovered = false;
+
try {
List<File> writableDirs = ldm.getWritableLedgerDirs();
// Check all writable dirs disk space usage.
@@ -99,6 +103,7 @@ class LedgerDirsMonitor {
});
// Notify disk full to all listeners
ldm.addToFilledDirs(dir);
+ someDiskFulled = true;
}
}
// Let's get NoWritableLedgerDirException without waiting for the next iteration
@@ -108,7 +113,6 @@ class LedgerDirsMonitor {
ldm.getWritableLedgerDirs();
} catch (NoWritableLedgerDirException e) {
LOG.warn("LedgerDirsMonitor check process: All ledger directories are non writable");
- boolean highPriorityWritesAllowed = true;
try {
// disk check can be frequent, so disable 'loggingNoWritable' to avoid log flooding.
ldm.getDirsAboveUsableThresholdSize(minUsableSizeForHighPriorityWrites, false);
@@ -146,6 +150,7 @@ class LedgerDirsMonitor {
if (makeWritable) {
ldm.addToWritableDirs(dir, true);
}
+ someDiskRecovered = true;
} catch (DiskErrorException e) {
// Notify disk failure to all the listeners
for (LedgerDirsListener listener : ldm.getListeners()) {
@@ -157,6 +162,7 @@ class LedgerDirsMonitor {
if (makeWritable) {
ldm.addToWritableDirs(dir, false);
}
+ someDiskRecovered = true;
} catch (DiskOutOfSpaceException e) {
// the full-filled dir is still full-filled
diskUsages.put(dir, e.getUsage());
@@ -168,6 +174,22 @@ class LedgerDirsMonitor {
listener.fatalError();
}
}
+
+ if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) {
+ if (someDiskFulled && !ldm.getFullFilledLedgerDirs().isEmpty()) {
+ // notify any disk full.
+ for (LedgerDirsListener listener : ldm.getListeners()) {
+ listener.anyDiskFull(highPriorityWritesAllowed);
+ }
+ }
+
+ if (someDiskRecovered && ldm.getFullFilledLedgerDirs().isEmpty()) {
+ // notify all disk recovered.
+ for (LedgerDirsListener listener : ldm.getListeners()) {
+ listener.allDisksWritable();
+ }
+ }
+ }
}
private void check() {
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
index fa9d58b1da..5518bf0362 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
@@ -187,6 +187,7 @@ public class ServerConfiguration extends AbstractConfiguration<ServerConfigurati
protected static final String LOCK_RELEASE_OF_FAILED_LEDGER_GRACE_PERIOD = "lockReleaseOfFailedLedgerGracePeriod";
//ReadOnly mode support on all disk full
protected static final String READ_ONLY_MODE_ENABLED = "readOnlyModeEnabled";
+ protected static final String READ_ONLY_MODE_ON_ANY_DISK_FULL_ENABLED = "readOnlyModeOnAnyDiskFullEnabled";
//Whether the bookie is force started in ReadOnly mode
protected static final String FORCE_READ_ONLY_BOOKIE = "forceReadOnlyBookie";
//Whether to persist the bookie status
@@ -2395,6 +2396,27 @@ public class ServerConfiguration extends AbstractConfiguration<ServerConfigurati
return getBoolean(READ_ONLY_MODE_ENABLED, true);
}
+ /**
+ * Set whether the bookie is able to go into read-only mode when any disk is full.
+ * If this set to false, it will behave to READ_ONLY_MODE_ENABLED flag.
+ *
+ * @param enabled whether to enable read-only mode when any disk is full.
+ * @return
+ */
+ public ServerConfiguration setReadOnlyModeOnAnyDiskFullEnabled(boolean enabled) {
+ setProperty(READ_ONLY_MODE_ON_ANY_DISK_FULL_ENABLED, enabled);
+ return this;
+ }
+
+ /**
+ * Get whether read-only mode is enable when any disk is full. The default is false.
+ *
+ * @return boolean
+ */
+ public boolean isReadOnlyModeOnAnyDiskFullEnabled() {
+ return getBoolean(READ_ONLY_MODE_ON_ANY_DISK_FULL_ENABLED, false);
+ }
+
/**
* Set the warning threshold for disk usage.
*
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerDirsManagerTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerDirsManagerTest.java
index 31b6da37a3..91d3ba2b34 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerDirsManagerTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerDirsManagerTest.java
@@ -240,6 +240,76 @@ public class LedgerDirsManagerTest {
assertTrue(mockLedgerDirsListener.highPriorityWritesAllowed);
}
+ @Test
+ public void testIsReadOnlyModeOnAnyDiskFullEnabled() throws Exception {
+ testAnyLedgerFullTransitToReadOnly(true);
+ testAnyLedgerFullTransitToReadOnly(false);
+ }
+
+ public void testAnyLedgerFullTransitToReadOnly(boolean isReadOnlyModeOnAnyDiskFullEnabled) throws Exception {
+ ledgerMonitor.shutdown();
+
+ final float nospace = 0.90f;
+ final float lwm = 0.80f;
+ HashMap<File, Float> usageMap;
+
+ File tmpDir1 = createTempDir("bkTest", ".dir");
+ File curDir1 = BookieImpl.getCurrentDirectory(tmpDir1);
+ BookieImpl.checkDirectoryStructure(curDir1);
+
+ File tmpDir2 = createTempDir("bkTest", ".dir");
+ File curDir2 = BookieImpl.getCurrentDirectory(tmpDir2);
+ BookieImpl.checkDirectoryStructure(curDir2);
+
+ conf.setDiskUsageThreshold(nospace);
+ conf.setDiskLowWaterMarkUsageThreshold(lwm);
+ conf.setDiskUsageWarnThreshold(nospace);
+ conf.setReadOnlyModeOnAnyDiskFullEnabled(isReadOnlyModeOnAnyDiskFullEnabled);
+ conf.setLedgerDirNames(new String[] { tmpDir1.toString(), tmpDir2.toString() });
+
+ mockDiskChecker = new MockDiskChecker(nospace, warnThreshold);
+ dirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(),
+ new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()), statsLogger);
+ ledgerMonitor = new LedgerDirsMonitor(conf, mockDiskChecker, Collections.singletonList(dirsManager));
+ usageMap = new HashMap<>();
+ usageMap.put(curDir1, 0.1f);
+ usageMap.put(curDir2, 0.1f);
+ mockDiskChecker.setUsageMap(usageMap);
+ ledgerMonitor.init();
+ final MockLedgerDirsListener mockLedgerDirsListener = new MockLedgerDirsListener();
+ dirsManager.addLedgerDirsListener(mockLedgerDirsListener);
+ ledgerMonitor.start();
+
+ Thread.sleep((diskCheckInterval * 2) + 100);
+ assertFalse(mockLedgerDirsListener.readOnly);
+
+ if (isReadOnlyModeOnAnyDiskFullEnabled) {
+ setUsageAndThenVerify(curDir1, 0.1f, curDir2, nospace + 0.05f, mockDiskChecker,
+ mockLedgerDirsListener, true);
+ setUsageAndThenVerify(curDir1, nospace + 0.05f, curDir2, 0.1f, mockDiskChecker,
+ mockLedgerDirsListener, true);
+ setUsageAndThenVerify(curDir1, nospace + 0.05f, curDir2, nospace + 0.05f, mockDiskChecker,
+ mockLedgerDirsListener, true);
+ setUsageAndThenVerify(curDir1, nospace - 0.30f, curDir2, nospace + 0.05f, mockDiskChecker,
+ mockLedgerDirsListener, true);
+ setUsageAndThenVerify(curDir1, nospace - 0.20f, curDir2, nospace - 0.20f, mockDiskChecker,
+ mockLedgerDirsListener, false);
+ } else {
+ setUsageAndThenVerify(curDir1, 0.1f, curDir2, 0.1f, mockDiskChecker,
+ mockLedgerDirsListener, false);
+ setUsageAndThenVerify(curDir1, 0.1f, curDir2, nospace + 0.05f, mockDiskChecker,
+ mockLedgerDirsListener, false);
+ setUsageAndThenVerify(curDir1, nospace + 0.05f, curDir2, 0.1f, mockDiskChecker,
+ mockLedgerDirsListener, false);
+ setUsageAndThenVerify(curDir1, nospace + 0.05f, curDir2, nospace + 0.05f, mockDiskChecker,
+ mockLedgerDirsListener, true);
+ setUsageAndThenVerify(curDir1, nospace - 0.30f, curDir2, nospace + 0.05f, mockDiskChecker,
+ mockLedgerDirsListener, false);
+ setUsageAndThenVerify(curDir1, nospace - 0.20f, curDir2, nospace - 0.20f, mockDiskChecker,
+ mockLedgerDirsListener, false);
+ }
+ }
+
@Test
public void testLedgerDirsMonitorHandlingLowWaterMark() throws Exception {
ledgerMonitor.shutdown();
@@ -517,12 +587,18 @@ public class LedgerDirsManagerTest {
@Override
public void diskWritable(File disk) {
+ if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) {
+ return;
+ }
readOnly = false;
highPriorityWritesAllowed = true;
}
@Override
public void diskJustWritable(File disk) {
+ if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) {
+ return;
+ }
readOnly = false;
highPriorityWritesAllowed = true;
}
@@ -533,6 +609,20 @@ public class LedgerDirsManagerTest {
this.highPriorityWritesAllowed = highPriorityWritesAllowed;
}
+ @Override
+ public void anyDiskFull(boolean highPriorityWritesAllowed) {
+ if (conf.isReadOnlyModeOnAnyDiskFullEnabled()) {
+ this.readOnly = true;
+ this.highPriorityWritesAllowed = highPriorityWritesAllowed;
+ }
+ }
+
+ @Override
+ public void allDisksWritable() {
+ this.readOnly = false;
+ this.highPriorityWritesAllowed = true;
+ }
+
public void reset() {
readOnly = false;
highPriorityWritesAllowed = true;
diff --git a/conf/bk_server.conf b/conf/bk_server.conf
index ac6fc8aa47..9890674ca4 100755
--- a/conf/bk_server.conf
+++ b/conf/bk_server.conf
@@ -170,7 +170,7 @@ extraServerComponents=
# If all ledger directories configured are full, then support only read requests for clients.
# If "readOnlyModeEnabled=true" then on all ledger disks full, bookie will be converted
# to read-only mode and serve only read requests. Otherwise the bookie will be shutdown.
-# By default this will be disabled.
+# By default this will be enabled.
# readOnlyModeEnabled=true
# Whether the bookie is force started in read only mode or not
@@ -180,6 +180,13 @@ extraServerComponents=
# @Since 4.6
# persistBookieStatusEnabled=false
+# If any ledger directories configured are full, then support only read requests for clients.
+# If "readOnlyModeOnAnyDiskFullEnabled=true" then on any ledger disks full, bookie will be converted
+# to read-only mode and serve only read requests. When all disks recovered,
+# the bookie will be converted to read-write mode.Otherwise it will obey the `readOnlyModeEnabled` behavior.
+# By default this will be disabled.
+# readOnlyModeOnAnyDiskFullEnabled=false
+
#############################################################################
## Netty server settings
#############################################################################
diff --git a/site3/website/docs/reference/config.md b/site3/website/docs/reference/config.md
index 891b447421..57d9abd89b 100644
--- a/site3/website/docs/reference/config.md
+++ b/site3/website/docs/reference/config.md
@@ -51,11 +51,11 @@ The table below lists parameters that you can set to configure bookies. All conf
## Read-only mode support
| Parameter | Description | Default
-| --------- | ----------- | ------- |
-| readOnlyModeEnabled | If all ledger directories configured are full, then support only read requests for clients. If "readOnlyModeEnabled=true" then on all ledger disks full, bookie will be converted to read-only mode and serve only read requests. Otherwise the bookie will be shutdown. By default this will be disabled. | true |
+| -- | ----------- | ------- |
+| readOnlyModeEnabled | If all ledger directories configured are full, then support only read requests for clients. If "readOnlyModeEnabled=true" then on all ledger disks full, bookie will be converted to read-only mode and serve only read requests. Otherwise the bookie will be shutdown. By default, this will be enabled. | true |
| forceReadOnlyBookie | Whether the bookie is force started in read only mode or not. | false |
| persistBookieStatusEnabled | Persist the bookie status locally on the disks. So the bookies can keep their status upon restarts. | false |
-
+| readOnlyModeOnAnyDiskFullEnabled | If any ledger directories configured are full, then support only read requests for clients. If "readOnlyModeOnAnyDiskFullEnabled=true" then on any ledger disks full, bookie will be converted to read-only mode and serve only read requests. When all disks recovered, the bookie will be converted to read-write mode.Otherwise it will obey the `readOnlyModeEnabled` behavior. By default, this will be disabled. | false |
## Netty server settings