You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2016/03/29 22:04:29 UTC
accumulo git commit: ACCUMULO-4004: Add new property for WALog max
age, close log when age is reached.
Repository: accumulo
Updated Branches:
refs/heads/ACCUMULO-4004 [created] f7d22c395
ACCUMULO-4004: Add new property for WALog max age, close log when age is reached.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f7d22c39
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f7d22c39
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f7d22c39
Branch: refs/heads/ACCUMULO-4004
Commit: f7d22c395a27ced9fa382b44162371d8b464988d
Parents: 1d49030
Author: Dave Marion <dl...@apache.org>
Authored: Tue Mar 29 16:02:38 2016 -0400
Committer: Dave Marion <dl...@apache.org>
Committed: Tue Mar 29 16:02:38 2016 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/core/conf/Property.java | 1 +
.../java/org/apache/accumulo/tserver/TabletServer.java | 3 ++-
.../apache/accumulo/tserver/log/TabletServerLogger.java | 11 ++++++++---
3 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7d22c39/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index b8311a9..e8537ac 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -254,6 +254,7 @@ public enum Property {
+ "must be made, which is slower. However opening too many files at once can cause problems."),
TSERV_WALOG_MAX_SIZE("tserver.walog.max.size", "1G", PropertyType.MEMORY,
"The maximum size for each write-ahead log. See comment for property tserver.memory.maps.max"),
+ TSERV_WALOG_MAX_AGE("tserver.walog.max.age", "24h", PropertyType.TIMEDURATION, "The maximum age for each write-ahead log."),
TSERV_WALOG_TOLERATED_CREATION_FAILURES("tserver.walog.tolerated.creation.failures", "50", PropertyType.COUNT,
"The maximum number of failures tolerated when creating a new WAL file within the period specified by tserver.walog.failures.period."
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7d22c39/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index d19dfa9..be52d5b 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -354,6 +354,7 @@ public class TabletServer extends AccumuloServerContext implements Runnable {
}, 5000, 5000);
final long walogMaxSize = aconf.getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE);
+ final long walogMaxAge = aconf.getTimeInMillis(Property.TSERV_WALOG_MAX_AGE);
final long minBlockSize = CachedConfiguration.getInstance().getLong("dfs.namenode.fs-limits.min-block-size", 0);
if (minBlockSize != 0 && minBlockSize > walogMaxSize)
throw new RuntimeException("Unable to start TabletServer. Logger is set to use blocksize " + walogMaxSize + " but hdfs minimum block size is "
@@ -367,7 +368,7 @@ public class TabletServer extends AccumuloServerContext implements Runnable {
final RetryFactory walCreationRetryFactory = new RetryFactory(toleratedWalCreationFailures, walCreationFailureRetryIncrement,
walCreationFailureRetryIncrement, walCreationFailureRetryMax);
- logger = new TabletServerLogger(this, walogMaxSize, syncCounter, flushCounter, walCreationRetryFactory);
+ logger = new TabletServerLogger(this, walogMaxSize, syncCounter, flushCounter, walCreationRetryFactory, walogMaxAge);
this.resourceManager = new TabletServerResourceManager(this, fs);
this.security = AuditedSecurityOperation.getInstance(this);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f7d22c39/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
index b76b9cd..7b52857 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
@@ -74,6 +74,7 @@ public class TabletServerLogger {
private final AtomicLong logSizeEstimate = new AtomicLong();
private final long maxSize;
+ private final long maxAge;
private final TabletServer tserver;
@@ -102,6 +103,8 @@ public class TabletServerLogger {
private final RetryFactory retryFactory;
private Retry retry = null;
+ private long createTime = 0;
+
static private abstract class TestCallWithWriteLock {
abstract boolean test();
@@ -145,13 +148,14 @@ public class TabletServerLogger {
}
}
- public TabletServerLogger(TabletServer tserver, long maxSize, AtomicLong syncCounter, AtomicLong flushCounter, RetryFactory retryFactory) {
+ public TabletServerLogger(TabletServer tserver, long maxSize, AtomicLong syncCounter, AtomicLong flushCounter, RetryFactory retryFactory, long maxAge) {
this.tserver = tserver;
this.maxSize = maxSize;
this.syncCounter = syncCounter;
this.flushCounter = flushCounter;
this.retryFactory = retryFactory;
this.retry = null;
+ this.maxAge = maxAge;
}
private DfsLogger initializeLoggers(final AtomicInteger logIdOut) throws IOException {
@@ -224,6 +228,7 @@ public class TabletServerLogger {
retry = null;
}
+ this.createTime = System.currentTimeMillis();
return;
} else {
throw new RuntimeException("Error: unexpected type seen: " + next);
@@ -434,12 +439,12 @@ public class TabletServerLogger {
});
}
}
- // if the log gets too big, reset it .. grab the write lock first
+ // if the log gets too big or too old, reset it .. grab the write lock first
logSizeEstimate.addAndGet(4 * 3); // event, tid, seq overhead
testLockAndRun(logIdLock, new TestCallWithWriteLock() {
@Override
boolean test() {
- return logSizeEstimate.get() > maxSize;
+ return (logSizeEstimate.get() > maxSize) || ((System.currentTimeMillis() - createTime) > maxAge);
}
@Override