You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by si...@apache.org on 2016/03/08 07:14:14 UTC
bookkeeper git commit: Bookie should return to read-write mode once
the disk usage drops before the threshold
Repository: bookkeeper
Updated Branches:
refs/heads/master c8255f8c5 -> bfa74f736
Bookie should return to read-write mode once the disk usage drops before the threshold
…o read only mode
Author: Siddharth Boobna <sb...@yahoo-inc.com>
Reviewers: Sijie Guo <si...@apache.org>
Closes #18 from sboobna/BOOKKEEPER-899
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/bfa74f73
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/bfa74f73
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/bfa74f73
Branch: refs/heads/master
Commit: bfa74f736ba270cc1a0bd0f7a1a6c643a00e385d
Parents: c8255f8
Author: Siddharth Boobna <sb...@yahoo-inc.com>
Authored: Mon Mar 7 22:14:10 2016 -0800
Committer: Sijie Guo <si...@apache.org>
Committed: Mon Mar 7 22:14:10 2016 -0800
----------------------------------------------------------------------
.../bookkeeper/bookie/LedgerDirsManager.java | 64 ++++++-----
.../org/apache/bookkeeper/util/DiskChecker.java | 8 ++
.../bookie/TestLedgerDirsManager.java | 115 ++++++++++++++++++-
3 files changed, 151 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bfa74f73/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
index 935befb..7c95b85 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
@@ -66,13 +66,18 @@ public class LedgerDirsManager {
}
LedgerDirsManager(ServerConfiguration conf, File[] dirs, StatsLogger statsLogger) {
+ this(conf, dirs, statsLogger, new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()));
+ }
+
+ @VisibleForTesting
+ LedgerDirsManager(ServerConfiguration conf, File[] dirs, StatsLogger statsLogger, DiskChecker diskChecker) {
this.ledgerDirectories = Arrays.asList(Bookie
.getCurrentDirectories(dirs));
this.writableLedgerDirectories = new ArrayList<File>(ledgerDirectories);
this.filledDirs = new ArrayList<File>();
- listeners = new ArrayList<LedgerDirsListener>();
- diskChecker = new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold());
- monitor = new LedgerDirsMonitor(conf.getDiskCheckInterval());
+ this.listeners = new ArrayList<LedgerDirsListener>();
+ this.diskChecker = diskChecker;
+ this.monitor = new LedgerDirsMonitor(conf.getDiskCheckInterval());
for (File dir : dirs) {
diskUsages.put(dir, 0f);
String statName = "dir_" + dir.getPath().replace('/', '_') + "_usage";
@@ -274,38 +279,37 @@ public class LedgerDirsManager {
@Override
public void run() {
while (true) {
- List<File> writableDirs;
try {
- writableDirs = getWritableLedgerDirs();
+ List<File> writableDirs = getWritableLedgerDirs();
+ // Check all writable dirs disk space usage.
+ for (File dir : writableDirs) {
+ try {
+ diskUsages.put(dir, diskChecker.checkDir(dir));
+ } catch (DiskErrorException e) {
+ LOG.error("Ledger directory {} failed on disk checking : ", dir, e);
+ // Notify disk failure to all listeners
+ for (LedgerDirsListener listener : listeners) {
+ listener.diskFailed(dir);
+ }
+ } catch (DiskWarnThresholdException e) {
+ LOG.warn("Ledger directory {} is almost full.", dir);
+ diskUsages.put(dir, e.getUsage());
+ for (LedgerDirsListener listener : listeners) {
+ listener.diskAlmostFull(dir);
+ }
+ } catch (DiskOutOfSpaceException e) {
+ LOG.error("Ledger directory {} is out-of-space.", dir);
+ diskUsages.put(dir, e.getUsage());
+ // Notify disk full to all listeners
+ addToFilledDirs(dir);
+ }
+ }
} catch (NoWritableLedgerDirException e) {
for (LedgerDirsListener listener : listeners) {
listener.allDisksFull();
}
- break;
- }
- // Check all writable dirs disk space usage.
- for (File dir : writableDirs) {
- try {
- diskUsages.put(dir, diskChecker.checkDir(dir));
- } catch (DiskErrorException e) {
- LOG.error("Ledger directory {} failed on disk checking : ", dir, e);
- // Notify disk failure to all listeners
- for (LedgerDirsListener listener : listeners) {
- listener.diskFailed(dir);
- }
- } catch (DiskWarnThresholdException e) {
- LOG.warn("Ledger directory {} is almost full.", dir);
- diskUsages.put(dir, e.getUsage());
- for (LedgerDirsListener listener : listeners) {
- listener.diskAlmostFull(dir);
- }
- } catch (DiskOutOfSpaceException e) {
- LOG.error("Ledger directory {} is out-of-space.", dir);
- diskUsages.put(dir, e.getUsage());
- // Notify disk full to all listeners
- addToFilledDirs(dir);
- }
}
+
List<File> fullfilledDirs = new ArrayList<File>(getFullFilledLedgerDirs());
// Check all full-filled disk space usage
for (File dir : fullfilledDirs) {
@@ -313,7 +317,7 @@ public class LedgerDirsManager {
diskUsages.put(dir, diskChecker.checkDir(dir));
addToWritableDirs(dir, true);
} catch (DiskErrorException e) {
- //Notify disk failure to all the listeners
+ // Notify disk failure to all the listeners
for (LedgerDirsListener listener : listeners) {
listener.diskFailed(dir);
}
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bfa74f73/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java
index 75902c9..b1251a6 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java
@@ -205,4 +205,12 @@ public class DiskChecker {
+ " are not valid. Should be > 0 and < 1 and diskSpaceThreshold >= diskSpaceWarnThreshold");
}
}
+
+ public float getDiskUsageThreshold() {
+ return diskUsageThreshold;
+ }
+
+ public float getDiskUsageWarnThreshold() {
+ return diskUsageWarnThreshold;
+ }
}
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bfa74f73/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java
index 416062d..4d8d547 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestLedgerDirsManager.java
@@ -20,30 +20,40 @@
*/
package org.apache.bookkeeper.bookie;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.io.FileUtils;
+import org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener;
import org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.conf.TestBKConfiguration;
+import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.util.DiskChecker;
import org.apache.bookkeeper.util.IOUtils;
-import org.junit.Before;
+import org.apache.commons.io.FileUtils;
import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.junit.Assert.*;
-
public class TestLedgerDirsManager {
private final static Logger LOG = LoggerFactory.getLogger(TestLedgerDirsManager.class);
ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
File curDir;
LedgerDirsManager dirsManager;
+ MockDiskChecker mockDiskChecker;
+ int diskCheckInterval = 1000;
+ float threshold = 0.5f;
+ float warnThreshold = 0.5f;
final List<File> tempDirs = new ArrayList<File>();
@@ -60,13 +70,17 @@ public class TestLedgerDirsManager {
Bookie.checkDirectoryStructure(curDir);
ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
- conf.setLedgerDirNames(new String[] {tmpDir.toString()});
+ conf.setLedgerDirNames(new String[] { tmpDir.toString() });
+ conf.setDiskCheckInterval(diskCheckInterval);
- dirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs());
+ mockDiskChecker = new MockDiskChecker(threshold, warnThreshold);
+ dirsManager = new LedgerDirsManager(conf, conf.getLedgerDirs(), NullStatsLogger.INSTANCE, mockDiskChecker);
+ dirsManager.init();
}
@After
public void tearDown() throws Exception {
+ dirsManager.shutdown();
for (File dir : tempDirs) {
FileUtils.deleteDirectory(dir);
}
@@ -97,4 +111,93 @@ public class TestLedgerDirsManager {
}
}
+ @Test(timeout=60000)
+ public void testLedgerDirsMonitorDuringTransition() throws Exception {
+
+ MockLedgerDirsListener mockLedgerDirsListener = new MockLedgerDirsListener();
+ dirsManager.addLedgerDirsListener(mockLedgerDirsListener);
+ dirsManager.start();
+
+ assertFalse(mockLedgerDirsListener.readOnly);
+ mockDiskChecker.setUsage(threshold + 0.05f);
+
+ Thread.sleep((diskCheckInterval * 2) + 100);
+
+ assertTrue(mockLedgerDirsListener.readOnly);
+ mockDiskChecker.setUsage(threshold - 0.05f);
+
+ Thread.sleep(diskCheckInterval + 100);
+
+ assertFalse(mockLedgerDirsListener.readOnly);
+ }
+
+ private class MockDiskChecker extends DiskChecker {
+
+ private float used;
+
+ public MockDiskChecker(float threshold, float warnThreshold) {
+ super(threshold, warnThreshold);
+ used = 0f;
+ }
+
+ @Override
+ public float checkDir(File dir) throws DiskErrorException, DiskOutOfSpaceException, DiskWarnThresholdException {
+ if (used > getDiskUsageThreshold()) {
+ throw new DiskOutOfSpaceException("", used);
+ }
+ if (used > getDiskUsageWarnThreshold()) {
+ throw new DiskWarnThresholdException("", used);
+ }
+ return used;
+ }
+
+ public void setUsage(float usage) {
+ this.used = usage;
+ }
+ }
+
+ private class MockLedgerDirsListener implements LedgerDirsListener {
+
+ public boolean readOnly;
+
+ public MockLedgerDirsListener() {
+ reset();
+ }
+
+ @Override
+ public void diskFailed(File disk) {
+ }
+
+ @Override
+ public void diskAlmostFull(File disk) {
+ }
+
+ @Override
+ public void diskFull(File disk) {
+ }
+
+ @Override
+ public void diskWritable(File disk) {
+ readOnly = false;
+ }
+
+ @Override
+ public void diskJustWritable(File disk) {
+ readOnly = false;
+ }
+
+ @Override
+ public void allDisksFull() {
+ readOnly = true;
+ }
+
+ @Override
+ public void fatalError() {
+ }
+
+ public void reset() {
+ readOnly = false;
+ }
+
+ }
}