You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by iv...@apache.org on 2014/01/15 15:58:00 UTC
svn commit: r1558410 - in /zookeeper/bookkeeper/trunk: ./
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/
bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/
bookkeeper-server/src/main/java/org/apache/bookkeeper/util/ bookkeeper-s...
Author: ivank
Date: Wed Jan 15 14:58:00 2014
New Revision: 1558410
URL: http://svn.apache.org/r1558410
Log:
BOOKKEEPER-662: Major GC should kick in immediately if remaining space reaches a warning threshold (sijie via ivank)
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/IndexPersistenceMgr.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestSyncThread.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/TestDiskChecker.java
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Wed Jan 15 14:58:00 2014
@@ -134,6 +134,8 @@ Trunk (unreleased changes)
BOOKKEEPER-714: Logging channel exceptions in PerChannelBookieClient (sijie)
+ BOOKKEEPER-662: Major GC should kick in immediately if remaining space reaches a warning threshold (sijie via ivank)
+
hedwig-server:
BOOKKEEPER-601: readahead cache size isn't updated correctly (sijie via fpj)
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java Wed Jan 15 14:58:00 2014
@@ -561,6 +561,11 @@ public class Bookie extends BookieCritic
}
@Override
+ public void diskAlmostFull(File disk) {
+ // Nothing needs to be handled here.
+ }
+
+ @Override
public void diskFailed(File disk) {
// Shutdown the bookie on disk failure.
triggerBookieShutdown(ExitCode.BOOKIE_EXCEPTION);
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java Wed Jan 15 14:58:00 2014
@@ -211,6 +211,15 @@ public class EntryLogger {
}
@Override
+ public void diskAlmostFull(File disk) {
+ // If the current entry log disk is almost full, then create new entry
+ // log.
+ if (currentDir != null && currentDir.equals(disk)) {
+ shouldCreateNewEntryLog.set(true);
+ }
+ }
+
+ @Override
public void diskFailed(File disk) {
// Nothing to handle here. Will be handled in Bookie
}
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java Wed Jan 15 14:58:00 2014
@@ -89,6 +89,9 @@ public class GarbageCollectorThread exte
// track the last scanned successfully log id
long scannedLogId = 0;
+ // Boolean to trigger a forced GC.
+ final AtomicBoolean forceGarbageCollection = new AtomicBoolean(false);
+
final GarbageCollector garbageCollector;
final GarbageCleaner garbageCleaner;
@@ -273,6 +276,13 @@ public class GarbageCollectorThread exte
lastMinorCompactionTime = lastMajorCompactionTime = MathUtils.now();
}
+ synchronized void forceGC() {
+ if (forceGarbageCollection.compareAndSet(false, true)) {
+ LOG.info("Forced garbage collection triggered by thread: {}", Thread.currentThread().getName());
+ notify();
+ }
+ }
+
@Override
public void run() {
while (running) {
@@ -284,6 +294,10 @@ public class GarbageCollectorThread exte
continue;
}
}
+ boolean force = forceGarbageCollection.get();
+ if (force) {
+ LOG.info("Garbage collector thread forced to perform GC before expiry of wait time.");
+ }
// Extract all of the ledger ID's that comprise all of the entry logs
// (except for the current new one which is still being written to).
@@ -296,8 +310,8 @@ public class GarbageCollectorThread exte
doGcEntryLogs();
long curTime = MathUtils.now();
- if (enableMajorCompaction &&
- curTime - lastMajorCompactionTime > majorCompactionInterval) {
+ if (force || (enableMajorCompaction &&
+ curTime - lastMajorCompactionTime > majorCompactionInterval)) {
// enter major compaction
LOG.info("Enter major compaction");
doCompactEntryLogs(majorCompactionThreshold);
@@ -307,13 +321,14 @@ public class GarbageCollectorThread exte
continue;
}
- if (enableMinorCompaction &&
- curTime - lastMinorCompactionTime > minorCompactionInterval) {
+ if (force || (enableMinorCompaction &&
+ curTime - lastMinorCompactionTime > minorCompactionInterval)) {
// enter minor compaction
LOG.info("Enter minor compaction");
doCompactEntryLogs(minorCompactionThreshold);
lastMinorCompactionTime = MathUtils.now();
}
+ forceGarbageCollection.set(false);
}
LOG.info("GarbageCollectorThread exited loop!");
}
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/IndexPersistenceMgr.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/IndexPersistenceMgr.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/IndexPersistenceMgr.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/IndexPersistenceMgr.java Wed Jan 15 14:58:00 2014
@@ -349,6 +349,11 @@ public class IndexPersistenceMgr {
}
@Override
+ public void diskAlmostFull(File disk) {
+ // Nothing to handle here. Will be handled in Bookie
+ }
+
+ @Override
public void diskFailed(File disk) {
// Nothing to handle here. Will be handled in Bookie
}
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorage.java Wed Jan 15 14:58:00 2014
@@ -21,11 +21,14 @@
package org.apache.bookkeeper.bookie;
+import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.apache.bookkeeper.bookie.CheckpointSource.Checkpoint;
import org.apache.bookkeeper.bookie.EntryLogger.EntryLogListener;
+import org.apache.bookkeeper.bookie.LedgerDirsManager;
+import org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.jmx.BKMBeanInfo;
import org.apache.bookkeeper.meta.LedgerManager;
@@ -96,6 +99,36 @@ class InterleavedLedgerStorage implement
null == indexDirsManager ? ledgerDirsManager : indexDirsManager);
gcThread = new GarbageCollectorThread(conf, ledgerCache, entryLogger,
activeLedgers, ledgerManager);
+ ledgerDirsManager.addLedgerDirsListener(getLedgerDirsListener());
+ }
+
+ private LedgerDirsListener getLedgerDirsListener() {
+ return new LedgerDirsListener() {
+ @Override
+ public void diskFailed(File disk) {
+ // do nothing.
+ }
+
+ @Override
+ public void diskAlmostFull(File disk) {
+ gcThread.forceGC();
+ }
+
+ @Override
+ public void diskFull(File disk) {
+ gcThread.forceGC();
+ }
+
+ @Override
+ public void allDisksFull() {
+ gcThread.forceGC();
+ }
+
+ @Override
+ public void fatalError() {
+ // do nothing.
+ }
+ };
}
@Override
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java Wed Jan 15 14:58:00 2014
@@ -27,14 +27,16 @@ import java.util.Arrays;
import java.util.List;
import java.util.Random;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.util.DiskChecker;
import org.apache.bookkeeper.util.DiskChecker.DiskErrorException;
import org.apache.bookkeeper.util.DiskChecker.DiskOutOfSpaceException;
+import org.apache.bookkeeper.util.DiskChecker.DiskWarnThresholdException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* This class manages ledger directories used by the bookie.
*/
@@ -45,9 +47,9 @@ public class LedgerDirsManager {
private volatile List<File> filledDirs;
private final List<File> ledgerDirectories;
private volatile List<File> writableLedgerDirectories;
- private DiskChecker diskChecker;
- private List<LedgerDirsListener> listeners;
- private LedgerDirsMonitor monitor;
+ private final DiskChecker diskChecker;
+ private final List<LedgerDirsListener> listeners;
+ private final LedgerDirsMonitor monitor;
private final Random rand = new Random();
public LedgerDirsManager(ServerConfiguration conf, File[] dirs) {
@@ -56,7 +58,7 @@ public class LedgerDirsManager {
this.writableLedgerDirectories = new ArrayList<File>(ledgerDirectories);
this.filledDirs = new ArrayList<File>();
listeners = new ArrayList<LedgerDirsManager.LedgerDirsListener>();
- diskChecker = new DiskChecker(conf.getDiskUsageThreshold());
+ diskChecker = new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold());
monitor = new LedgerDirsMonitor(conf.getDiskCheckInterval());
}
@@ -217,6 +219,10 @@ public class LedgerDirsManager {
LOG.warn("{} has errors.", dir, e);
listener.diskFailed(dir);
}
+ } catch (DiskWarnThresholdException e) {
+ for (LedgerDirsListener listener : listeners) {
+ listener.diskAlmostFull(dir);
+ }
} catch (DiskOutOfSpaceException e) {
// Notify disk full to all listeners
addToFilledDirs(dir);
@@ -244,6 +250,8 @@ public class LedgerDirsManager {
for (File dir : writableDirs) {
try {
diskChecker.checkDir(dir);
+ } catch (DiskWarnThresholdException e) {
+ // nop
} catch (DiskOutOfSpaceException e) {
addToFilledDirs(dir);
}
@@ -277,6 +285,13 @@ public class LedgerDirsManager {
void diskFailed(File disk);
/**
+ * Notified when the disk usage warn threshold is exceeded on
+ * the drive.
+ * @param disk
+ */
+ void diskAlmostFull(File disk);
+
+ /**
* This will be notified on disk detected as full
*
* @param disk
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java Wed Jan 15 14:58:00 2014
@@ -80,6 +80,7 @@ public class ServerConfiguration extends
protected final static String READ_ONLY_MODE_ENABLED = "readOnlyModeEnabled";
//Disk utilization
protected final static String DISK_USAGE_THRESHOLD = "diskUsageThreshold";
+ protected final static String DISK_USAGE_WARN_THRESHOLD = "diskUsageWarnThreshold";
protected final static String DISK_CHECK_INTERVAL = "diskCheckInterval";
protected final static String AUDITOR_PERIODIC_CHECK_INTERVAL = "auditorPeriodicCheckInterval";
protected final static String AUTO_RECOVERY_DAEMON_ENABLED = "autoRecoveryDaemonEnabled";
@@ -857,6 +858,27 @@ public class ServerConfiguration extends
}
/**
+ * Set the warning threshold for disk usage.
+ *
+ * @param threshold warning threshold to force gc.
+ *
+ * @return ServerConfiguration
+ */
+ public ServerConfiguration setDiskUsageWarnThreshold(float threshold) {
+ setProperty(DISK_USAGE_WARN_THRESHOLD, threshold);
+ return this;
+ }
+
+ /**
+ * Returns the warning threshold for disk usage. If disk usage
+ * goes beyond this, a garbage collection cycle will be forced.
+ * @return
+ */
+ public float getDiskUsageWarnThreshold() {
+ return getFloat(DISK_USAGE_WARN_THRESHOLD, 0.90f);
+ }
+
+ /**
* Set the Disk free space threshold as a fraction of the total
* after which disk will be considered as full during disk check.
*
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/DiskChecker.java Wed Jan 15 14:58:00 2014
@@ -28,8 +28,15 @@ import com.google.common.annotations.Vis
*/
public class DiskChecker {
private float diskUsageThreshold;
+ private float diskUsageWarnThreshold;
- public static class DiskErrorException extends IOException {
+ public abstract static class DiskException extends IOException {
+ public DiskException(String msg) {
+ super(msg);
+ }
+ }
+
+ public static class DiskErrorException extends DiskException {
private static final long serialVersionUID = 9091606022449761729L;
public DiskErrorException(String msg) {
@@ -37,7 +44,7 @@ public class DiskChecker {
}
}
- public static class DiskOutOfSpaceException extends IOException {
+ public static class DiskOutOfSpaceException extends DiskException {
private static final long serialVersionUID = 160898797915906860L;
public DiskOutOfSpaceException(String msg) {
@@ -45,9 +52,18 @@ public class DiskChecker {
}
}
- public DiskChecker(float threshold) {
- validateThreshold(threshold);
+ public static class DiskWarnThresholdException extends DiskException {
+ private static final long serialVersionUID = -1629284987500841657L;
+
+ public DiskWarnThresholdException(String msg) {
+ super(msg);
+ }
+ }
+
+ public DiskChecker(float threshold, float warnThreshold) {
+ validateThreshold(threshold, warnThreshold);
this.diskUsageThreshold = threshold;
+ this.diskUsageWarnThreshold = warnThreshold;
}
/**
@@ -60,7 +76,7 @@ public class DiskChecker {
* non-existent directory, then we signal an error; Sun's mkdir would signal
* an error (return false) if a directory it is attempting to create already
* exists or the mkdir fails.
- *
+ *
* @param dir
* @return true on success, false on failure
*/
@@ -82,7 +98,7 @@ public class DiskChecker {
/**
* Checks the disk space available.
- *
+ *
* @param dir
* Directory to check for the disk space
* @throws DiskOutOfSpaceException
@@ -90,7 +106,7 @@ public class DiskChecker {
* less than threshhold.
*/
@VisibleForTesting
- void checkDiskFull(File dir) throws DiskOutOfSpaceException {
+ void checkDiskFull(File dir) throws DiskOutOfSpaceException, DiskWarnThresholdException {
if (null == dir) {
return;
}
@@ -101,7 +117,12 @@ public class DiskChecker {
float used = 1f - free;
if (used > diskUsageThreshold) {
throw new DiskOutOfSpaceException("Space left on device "
- + usableSpace + " < threshhold " + diskUsageThreshold);
+ + usableSpace + " Used space fraction:" + used + " < threshhold " + diskUsageThreshold);
+ }
+ // Warn should be triggered only if disk usage threshold doesn't trigger first.
+ if (used > diskUsageWarnThreshold) {
+ throw new DiskWarnThresholdException("Space left on device:"
+ + usableSpace + " Used space fraction:" + used +" < WarnThreshold:" + diskUsageWarnThreshold);
}
} else {
checkDiskFull(dir.getParentFile());
@@ -110,16 +131,18 @@ public class DiskChecker {
/**
* Create the directory if it doesn't exist and
- *
+ *
* @param dir
* Directory to check for the disk error/full.
* @throws DiskErrorException
* If disk having errors
+ * @throws DiskWarnThresholdException
+ * If disk has less than configured amount of free space.
* @throws DiskOutOfSpaceException
* If disk is full or having less space than threshhold
*/
public void checkDir(File dir) throws DiskErrorException,
- DiskOutOfSpaceException {
+ DiskOutOfSpaceException, DiskWarnThresholdException {
checkDiskFull(dir);
if (!mkdirsWithExistsCheck(dir))
throw new DiskErrorException("can not create directory: "
@@ -138,31 +161,22 @@ public class DiskChecker {
}
/**
- * Returns the disk space threshold.
- *
- * @return
- */
- @VisibleForTesting
- float getDiskSpaceThreshold() {
- return diskUsageThreshold;
- }
-
- /**
* Set the disk space threshold
- *
+ *
* @param diskSpaceThreshold
*/
@VisibleForTesting
- void setDiskSpaceThreshold(float diskSpaceThreshold) {
- validateThreshold(diskSpaceThreshold);
+ void setDiskSpaceThreshold(float diskSpaceThreshold, float diskUsageWarnThreshold) {
+ validateThreshold(diskSpaceThreshold, diskSpaceThreshold);
this.diskUsageThreshold = diskSpaceThreshold;
+ this.diskUsageWarnThreshold = diskUsageWarnThreshold;
}
- private void validateThreshold(float diskSpaceThreshold) {
- if (diskSpaceThreshold <= 0 || diskSpaceThreshold >= 1) {
- throw new IllegalArgumentException("Disk space threashold "
- + diskSpaceThreshold
- + " is not valid. Should be > 0 and < 1 ");
+ private void validateThreshold(float diskSpaceThreshold, float diskSpaceWarnThreshold) {
+ if (diskSpaceThreshold <= 0 || diskSpaceThreshold >= 1 || diskSpaceWarnThreshold - diskSpaceThreshold > 1e-6) {
+ throw new IllegalArgumentException("Disk space threashold: "
+ + diskSpaceThreshold + " and warn threshold: " + diskSpaceWarnThreshold
+ + " are not valid. Should be > 0 and < 1 and diskSpaceThreshold >= diskSpaceWarnThreshold");
}
}
}
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java Wed Jan 15 14:58:00 2014
@@ -351,6 +351,7 @@ public class BookieInitializationTest {
.setZkTimeout(5000).setJournalDirName(tempDir.getPath())
.setLedgerDirNames(new String[] { tempDir.getPath() });
conf.setDiskUsageThreshold((1f - ((float) usableSpace / (float) totalSpace)) - 0.05f);
+ conf.setDiskUsageWarnThreshold((1f - ((float) usableSpace / (float) totalSpace)) - 0.25f);
try {
new Bookie(conf);
} finally {
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/CompactionTest.java Wed Jan 15 14:58:00 2014
@@ -1,5 +1,3 @@
-package org.apache.bookkeeper.bookie;
-
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -20,6 +18,8 @@ package org.apache.bookkeeper.bookie;
* under the License.
*
*/
+package org.apache.bookkeeper.bookie;
+
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -30,25 +30,25 @@ import java.util.concurrent.ConcurrentHa
import java.util.Collections;
import java.util.Enumeration;
-import org.apache.bookkeeper.meta.LedgerManager;
-import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.client.LedgerEntry;
import org.apache.bookkeeper.client.LedgerHandle;
-import org.apache.bookkeeper.client.BookKeeper.DigestType;
-import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
-import org.apache.bookkeeper.util.TestUtils;
-
-import org.apache.zookeeper.AsyncCallback;
import org.apache.bookkeeper.client.LedgerMetadata;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerManagerFactory;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback;
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.Processor;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.util.MathUtils;
+import org.apache.bookkeeper.util.TestUtils;
import org.apache.bookkeeper.versioning.Version;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.zookeeper.AsyncCallback;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* This class tests the entry log compaction functionality.
@@ -175,6 +175,46 @@ public class CompactionTest extends Book
}
@Test(timeout=60000)
+ public void testForceGarbageCollection() throws Exception {
+ ServerConfiguration conf = newServerConfiguration();
+ conf.setGcWaitTime(60000);
+ conf.setMinorCompactionInterval(120000);
+ conf.setMajorCompactionInterval(240000);
+ LedgerDirsManager dirManager = new LedgerDirsManager(conf, conf.getLedgerDirs());
+ CheckpointSource cp = new CheckpointSource() {
+ @Override
+ public Checkpoint newCheckpoint() {
+ // Do nothing.
+ return null;
+ }
+
+ @Override
+ public void checkpointComplete(Checkpoint checkPoint, boolean compact)
+ throws IOException {
+ // Do nothing.
+ }
+ };
+ Bookie.checkDirectoryStructure(conf.getJournalDir());
+ for (File dir : dirManager.getAllLedgerDirs()) {
+ Bookie.checkDirectoryStructure(dir);
+ }
+ InterleavedLedgerStorage storage = new InterleavedLedgerStorage(conf,
+ LedgerManagerFactory.newLedgerManagerFactory(conf, zkc).newLedgerManager(),
+ dirManager, cp);
+ storage.start();
+ long startTime = MathUtils.now();
+ Thread.sleep(2000);
+ storage.gcThread.forceGC();
+ Thread.sleep(1000);
+ // Minor and Major compaction times should be larger than when we started
+ // this test.
+ assertTrue("Minor or major compaction did not trigger even on forcing.",
+ storage.gcThread.lastMajorCompactionTime > startTime &&
+ storage.gcThread.lastMinorCompactionTime > startTime);
+ storage.shutdown();
+ }
+
+ @Test(timeout=60000)
public void testMinorCompaction() throws Exception {
// prepare data
LedgerHandle[] lhs = prepareData(3, false);
@@ -199,7 +239,7 @@ public class CompactionTest extends Book
// entry logs ([0,1,2].log) should be compacted.
for (File ledgerDirectory : tmpDirs) {
- assertFalse("Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: "
+ assertFalse("Found entry log file ([0,1,2].log that should have not been compacted in ledgerDirectory: "
+ ledgerDirectory, TestUtils.hasLogFiles(ledgerDirectory, true, 0, 1, 2));
}
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestSyncThread.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestSyncThread.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestSyncThread.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/TestSyncThread.java Wed Jan 15 14:58:00 2014
@@ -326,6 +326,10 @@ public class TestSyncThread {
}
@Override
+ public void diskAlmostFull(File disk) {
+ }
+
+ @Override
public void diskFull(File disk) {
}
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/TestDiskChecker.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/TestDiskChecker.java?rev=1558410&r1=1558409&r2=1558410&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/TestDiskChecker.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/util/TestDiskChecker.java Wed Jan 15 14:58:00 2014
@@ -24,12 +24,13 @@ import java.io.IOException;
import org.apache.bookkeeper.util.DiskChecker.DiskErrorException;
import org.apache.bookkeeper.util.DiskChecker.DiskOutOfSpaceException;
+import org.apache.bookkeeper.util.DiskChecker.DiskWarnThresholdException;
import org.junit.Before;
import org.junit.Test;
/**
* Test to verify {@link DiskChecker}
- *
+ *
*/
public class TestDiskChecker {
@@ -37,7 +38,7 @@ public class TestDiskChecker {
@Before
public void setup() {
- diskChecker = new DiskChecker(0.95f);
+ diskChecker = new DiskChecker(0.95f, 0.95f);
}
/**
@@ -48,8 +49,22 @@ public class TestDiskChecker {
File file = File.createTempFile("DiskCheck", "test");
long usableSpace = file.getUsableSpace();
long totalSpace = file.getTotalSpace();
- diskChecker
- .setDiskSpaceThreshold((1f - ((float) usableSpace / (float) totalSpace)) - 0.05f);
+ float threshold =
+ (1f - ((float) usableSpace / (float) totalSpace)) - 0.05f;
+ diskChecker.setDiskSpaceThreshold(threshold, threshold);
+ diskChecker.checkDiskFull(file);
+ }
+
+ @Test(expected = DiskWarnThresholdException.class)
+ public void testDiskWarnThresholdException() throws IOException {
+ File file = File.createTempFile("DiskCheck", "test");
+ long usableSpace = file.getUsableSpace();
+ long totalSpace = file.getTotalSpace();
+ float diskSpaceThreshold =
+ (1f - ((float) usableSpace / (float) totalSpace)) + 0.01f;
+ float diskWarnThreshold =
+ (1f - ((float) usableSpace / (float) totalSpace)) - 0.05f;
+ diskChecker.setDiskSpaceThreshold(diskSpaceThreshold, diskWarnThreshold);
diskChecker.checkDiskFull(file);
}
@@ -62,8 +77,8 @@ public class TestDiskChecker {
File file = File.createTempFile("DiskCheck", "test");
long usableSpace = file.getUsableSpace();
long totalSpace = file.getTotalSpace();
- diskChecker
- .setDiskSpaceThreshold((1f - ((float) usableSpace / (float) totalSpace)) - 0.05f);
+ float threshold = (1f - ((float) usableSpace / (float) totalSpace)) - 0.05f;
+ diskChecker.setDiskSpaceThreshold(threshold, threshold);
assertTrue(file.delete());
diskChecker.checkDiskFull(file);
}