You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by si...@apache.org on 2017/01/31 01:11:38 UTC
bookkeeper git commit: BOOKKEEPER-968: Entry log flushes at
configurable chunks
Repository: bookkeeper
Updated Branches:
refs/heads/master ba5dadcb3 -> 26b09abb4
BOOKKEEPER-968: Entry log flushes at configurable chunks
With this patch one can configure interval (in bytes) for entry log to flush writes to the disk.
Author: Andrey Yegorov <ay...@salesforce.com>
Reviewers: Sijie Guo <si...@apache.org>
Closes #77 from dlg99/task/entry-log-flush
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/26b09abb
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/26b09abb
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/26b09abb
Branch: refs/heads/master
Commit: 26b09abb4202362ca37d6944ce75eb2a3309dc3c
Parents: ba5dadc
Author: Andrey Yegorov <ay...@salesforce.com>
Authored: Mon Jan 30 17:11:29 2017 -0800
Committer: Sijie Guo <si...@apache.org>
Committed: Mon Jan 30 17:11:29 2017 -0800
----------------------------------------------------------------------
bookkeeper-server/conf/bk_server.conf | 7 +++++
.../apache/bookkeeper/bookie/EntryLogger.java | 23 ++++++++++++++++
.../bookkeeper/conf/ServerConfiguration.java | 28 ++++++++++++++++++++
3 files changed, 58 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/26b09abb/bookkeeper-server/conf/bk_server.conf
----------------------------------------------------------------------
diff --git a/bookkeeper-server/conf/bk_server.conf b/bookkeeper-server/conf/bk_server.conf
index 14c6068..e2a2be6 100644
--- a/bookkeeper-server/conf/bk_server.conf
+++ b/bookkeeper-server/conf/bk_server.conf
@@ -274,6 +274,13 @@ zkTimeout=10000
# The interval is specified in seconds.
#auditorPeriodicBookieCheckInterval=86400
+# Entry log flush interval in bytes.
+# Default is 0. 0 or less disables this feature and effectively flush
+# happens on log rotation.
+# Flushing in smaller chunks but more frequently reduces spikes in disk
+# I/O. Flushing too frequently may also affect performance negatively.
+#flushEntrylogBytes=0
+
# How long to wait, in seconds, before starting auto recovery of a lost bookie
#lostBookieRecoveryDelay=0
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/26b09abb/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
index 08ad1be..3314903 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogger.java
@@ -171,6 +171,10 @@ public class EntryLogger {
final static int MIN_SANE_ENTRY_SIZE = 8 + 8;
final static long MB = 1024 * 1024;
+ private final long flushIntervalInBytes;
+ private final boolean doRegularFlushes;
+ private long bytesWrittenSinceLastFlush = 0;
+
final ServerConfiguration conf;
/**
* Scan entries in a entry log file.
@@ -253,6 +257,9 @@ public class EntryLogger {
this.leastUnflushedLogId = logId + 1;
this.entryLoggerAllocator = new EntryLoggerAllocator(logId);
this.conf = conf;
+ flushIntervalInBytes = conf.getFlushIntervalInBytes();
+ doRegularFlushes = flushIntervalInBytes > 0;
+
initialize();
}
@@ -737,6 +744,7 @@ public class EntryLogger {
synchronized void flushCurrentLog() throws IOException {
if (logChannel != null) {
logChannel.flush(true);
+ bytesWrittenSinceLastFlush = 0;
LOG.debug("Flush and sync current entry logger {}.", logChannel.getLogId());
}
}
@@ -752,6 +760,9 @@ public class EntryLogger {
// Create new log if logSizeLimit reached or current disk is full
boolean createNewLog = shouldCreateNewEntryLog.get();
if (createNewLog || reachEntryLogLimit) {
+ if (doRegularFlushes) {
+ flushCurrentLog();
+ }
createNewLog();
// Reset the flag
if (createNewLog) {
@@ -766,10 +777,22 @@ public class EntryLogger {
long pos = logChannel.position();
logChannel.write(entry);
logChannel.registerWrittenEntry(ledger, entrySize);
+
+ incrementBytesWrittenAndMaybeFlush(4L + entrySize);
return (logChannel.getLogId() << 32L) | pos;
}
+ private void incrementBytesWrittenAndMaybeFlush(long bytesWritten) throws IOException {
+ if (!doRegularFlushes) {
+ return;
+ }
+ bytesWrittenSinceLastFlush += bytesWritten;
+ if (bytesWrittenSinceLastFlush > flushIntervalInBytes) {
+ flushCurrentLog();
+ }
+ }
+
static long logIdForOffset(long offset) {
return offset >> 32L;
}
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/26b09abb/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
index 6bc118a..8ce6908 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
@@ -56,6 +56,7 @@ public class ServerConfiguration extends AbstractConfiguration {
protected final static String GC_OVERREPLICATED_LEDGER_WAIT_TIME = "gcOverreplicatedLedgerWaitTime";
// Sync Parameters
protected final static String FLUSH_INTERVAL = "flushInterval";
+ protected final static String FLUSH_ENTRYLOG_INTERVAL_BYTES = "flushEntrylogBytes";
// Bookie death watch interval
protected final static String DEATH_WATCH_INTERVAL = "bookieDeathWatchInterval";
// Ledger Cache Parameters
@@ -266,6 +267,33 @@ public class ServerConfiguration extends AbstractConfiguration {
}
/**
+ * Set entry log flush interval in bytes.
+ *
+ * Default is 0. 0 or less disables this feature and effectively flush
+ * happens on log rotation.
+ *
+ * Flushing in smaller chunks but more frequently reduces spikes in disk
+ * I/O. Flushing too frequently may also affect performance negatively.
+ *
+ * @return Entry log flush interval in bytes
+ */
+ public long getFlushIntervalInBytes() {
+ return this.getLong(FLUSH_ENTRYLOG_INTERVAL_BYTES, 0);
+ }
+
+ /**
+ * Set entry log flush interval in bytes
+ *
+ * @param flushInterval in bytes
+ * @return server configuration
+ */
+ public ServerConfiguration setFlushIntervalInBytes(long flushInterval) {
+ this.setProperty(FLUSH_ENTRYLOG_INTERVAL_BYTES, Long.toString(flushInterval));
+ return this;
+ }
+
+
+ /**
* Get bookie death watch interval
*
* @return watch interval