You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by si...@apache.org on 2013/10/02 07:01:52 UTC
svn commit: r1528307 - in /zookeeper/bookkeeper/trunk: ./
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/
bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/
Author: sijie
Date: Wed Oct 2 05:01:52 2013
New Revision: 1528307
URL: http://svn.apache.org/r1528307
Log:
BOOKKEEPER-686: Bookie startup will fail if one of the configured ledgerDir is full and the same is used for replaying the journal (Rakesh via sijie)
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1528307&r1=1528306&r2=1528307&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Wed Oct 2 05:01:52 2013
@@ -46,6 +46,8 @@ Trunk (unreleased changes)
BOOKKEEPER-653: Timeout option is missing in few testcases (Rakesh via ivank)
+ BOOKKEEPER-686: Bookie startup will fail if one of the configured ledgerDir is full and the same is used for replaying the journal (Rakesh via sijie)
+
bookkeeper-server:
BOOKKEEPER-567: ReadOnlyBookieTest hangs on shutdown (sijie via ivank)
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java?rev=1528307&r1=1528306&r2=1528307&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java Wed Oct 2 05:01:52 2013
@@ -413,6 +413,10 @@ public class Bookie extends Thread {
LOG.info("instantiate ledger manager {}", ledgerManagerFactory.getClass().getName());
ledgerManager = ledgerManagerFactory.newLedgerManager();
+ // Initialise ledgerDirManager. This would look through all the
+ // configured directories. When disk errors or all the ledger
+ // directories are full, would throws exception and fail bookie startup.
+ this.ledgerDirsManager.init();
// instantiate the journal
journal = new Journal(conf, ledgerDirsManager);
ledgerStorage = new InterleavedLedgerStorage(conf, ledgerManager,
@@ -486,6 +490,8 @@ public class Bookie extends Thread {
synchronized public void start() {
setDaemon(true);
LOG.debug("I'm starting a bookie with journal directory {}", journalDirectory.getName());
+ //Start DiskChecker thread
+ ledgerDirsManager.start();
// replay journals
try {
readJournal();
@@ -502,9 +508,9 @@ public class Bookie extends Thread {
// start bookie thread
super.start();
+ // After successful bookie startup, register listener for disk
+ // error/full notifications.
ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener());
- //Start DiskChecker thread
- ledgerDirsManager.start();
ledgerStorage.start();
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java?rev=1528307&r1=1528306&r2=1528307&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java Wed Oct 2 05:01:52 2013
@@ -153,6 +153,19 @@ public class LedgerDirsManager {
}
}
+ /**
+ * Sweep through all the directories to check disk errors or disk full.
+ *
+ * @throws DiskErrorException
+ * If disk having errors
+ * @throws NoWritableLedgerDirException
+ * If all the configured ledger directories are full or having
+ * less space than threshold
+ */
+ public void init() throws DiskErrorException, NoWritableLedgerDirException {
+ monitor.checkDirs(writableLedgerDirectories);
+ }
+
// start the daemon for disk monitoring
public void start() {
monitor.setDaemon(true);
@@ -201,6 +214,7 @@ public class LedgerDirsManager {
} catch (DiskErrorException e) {
// Notify disk failure to all listeners
for (LedgerDirsListener listener : listeners) {
+ LOG.warn("{} has errors.", dir, e);
listener.diskFailed(dir);
}
} catch (DiskOutOfSpaceException e) {
@@ -224,6 +238,18 @@ public class LedgerDirsManager {
}
LOG.info("LedgerDirsMonitorThread exited!");
}
+
+ private void checkDirs(List<File> writableDirs)
+ throws DiskErrorException, NoWritableLedgerDirException {
+ for (File dir : writableDirs) {
+ try {
+ diskChecker.checkDir(dir);
+ } catch (DiskOutOfSpaceException e) {
+ addToFilledDirs(dir);
+ }
+ }
+ getWritableLedgerDirs();
+ }
}
/**
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java?rev=1528307&r1=1528306&r2=1528307&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java Wed Oct 2 05:01:52 2013
@@ -34,11 +34,13 @@ import junit.framework.Assert;
import org.apache.bookkeeper.bookie.Bookie;
import org.apache.bookkeeper.bookie.BookieException;
+import org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException;
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.client.BookKeeperAdmin;
import org.apache.bookkeeper.proto.BookieServer;
import org.apache.bookkeeper.test.ZooKeeperUtil;
+import org.apache.bookkeeper.util.DiskChecker.DiskErrorException;
import org.apache.commons.io.FileUtils;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
@@ -332,6 +334,52 @@ public class BookieInitializationTest {
}
}
+ /**
+ * Check disk full. Expected to throw NoWritableLedgerDirException
+ * during bookie initialisation.
+ */
+ @Test(timeout = 30000, expected = NoWritableLedgerDirException.class)
+ public void testWithDiskFull() throws Exception {
+ File tempDir = File.createTempFile("DiskCheck", "test");
+ tempDir.delete();
+ tempDir.mkdir();
+ long usableSpace = tempDir.getUsableSpace();
+ long totalSpace = tempDir.getTotalSpace();
+ final ServerConfiguration conf = new ServerConfiguration()
+ .setZkServers(zkutil.getZooKeeperConnectString())
+ .setZkTimeout(5000).setJournalDirName(tempDir.getPath())
+ .setLedgerDirNames(new String[] { tempDir.getPath() });
+ conf.setDiskUsageThreshold((1f - ((float) usableSpace / (float) totalSpace)) - 0.05f);
+ try {
+ new Bookie(conf);
+ } finally {
+ FileUtils.deleteDirectory(tempDir);
+ }
+ }
+
+ /**
+ * Check disk error for file. Expected to throw DiskErrorException.
+ */
+ @Test(timeout = 30000, expected = DiskErrorException.class)
+ public void testWithDiskError() throws Exception {
+ File parent = File.createTempFile("DiskCheck", "test");
+ parent.delete();
+ parent.mkdir();
+ File child = File.createTempFile("DiskCheck", "test", parent);
+ final ServerConfiguration conf = new ServerConfiguration()
+ .setZkServers(zkutil.getZooKeeperConnectString())
+ .setZkTimeout(5000).setJournalDirName(child.getPath())
+ .setLedgerDirNames(new String[] { child.getPath() });
+ try {
+ // LedgerDirsManager#init() is used in Bookie instantiation.
+ // Simulating disk errors by directly calling #init
+ LedgerDirsManager ldm = new LedgerDirsManager(conf);
+ ldm.init();
+ } finally {
+ FileUtils.deleteDirectory(parent);
+ }
+ }
+
private void createNewZKClient() throws Exception {
// create a zookeeper client
LOG.debug("Instantiate ZK Client");