You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2016/04/12 15:58:21 UTC
[04/39] accumulo git commit: ACCUMULO-4004: Added new property to
close walogs after some time period.
ACCUMULO-4004: Added new property to close walogs after some time period.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0206d784
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0206d784
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0206d784
Branch: refs/heads/ACCUMULO-4173
Commit: 0206d7845bf8bba08e18373f1dbb8e9aaf6d6d60
Parents: f6bfe90
Author: Dave Marion <dl...@apache.org>
Authored: Thu Mar 31 15:53:04 2016 -0400
Committer: Dave Marion <dl...@apache.org>
Committed: Thu Mar 31 15:53:04 2016 -0400
----------------------------------------------------------------------
.../java/org/apache/accumulo/core/conf/Property.java | 1 +
.../java/org/apache/accumulo/tserver/TabletServer.java | 3 ++-
.../apache/accumulo/tserver/log/TabletServerLogger.java | 11 ++++++++---
3 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0206d784/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9243494..2149ad9 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -209,6 +209,7 @@ public enum Property {
+ "must be made, which is slower. However opening too many files at once can cause problems."),
TSERV_WALOG_MAX_SIZE("tserver.walog.max.size", "1G", PropertyType.MEMORY,
"The maximum size for each write-ahead log. See comment for property tserver.memory.maps.max"),
+ TSERV_WALOG_MAX_AGE("tserver.walog.max.age", "24h", PropertyType.TIMEDURATION, "The maximum age for each write-ahead log."),
TSERV_MAJC_DELAY("tserver.compaction.major.delay", "30s", PropertyType.TIMEDURATION,
"Time a tablet server will sleep between checking which tablets need compaction."),
TSERV_MAJC_THREAD_MAXOPEN("tserver.compaction.major.thread.files.open.max", "10", PropertyType.COUNT,
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0206d784/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index ac8f2ec..38bd8ac 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -3567,11 +3567,12 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
security = AuditedSecurityOperation.getInstance();
clientAddress = HostAndPort.fromParts(hostname, 0);
long walogMaxSize = getSystemConfiguration().getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE);
+ long walogMaxAge = getSystemConfiguration().getTimeInMillis(Property.TSERV_WALOG_MAX_AGE);
long minBlockSize = CachedConfiguration.getInstance().getLong("dfs.namenode.fs-limits.min-block-size", 0);
if (minBlockSize != 0 && minBlockSize > walogMaxSize)
throw new RuntimeException("Unable to start TabletServer. Logger is set to use blocksize " + walogMaxSize + " but hdfs minimum block size is "
+ minBlockSize + ". Either increase the " + Property.TSERV_WALOG_MAX_SIZE + " or decrease dfs.namenode.fs-limits.min-block-size in hdfs-site.xml.");
- logger = new TabletServerLogger(this, walogMaxSize);
+ logger = new TabletServerLogger(this, walogMaxSize, walogMaxAge);
try {
AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig(new Iterable<Entry<String,String>>() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/0206d784/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
index c864c78..158fdbd 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/TabletServerLogger.java
@@ -56,6 +56,7 @@ public class TabletServerLogger {
private final AtomicLong logSizeEstimate = new AtomicLong();
private final long maxSize;
+ private final long maxAge;
private final TabletServer tserver;
@@ -76,6 +77,8 @@ public class TabletServerLogger {
private final AtomicInteger seqGen = new AtomicInteger();
+ private long createTime = 0;
+
private static boolean enabled(Tablet tablet) {
return tablet.getTableConfiguration().getBoolean(Property.TABLE_WALOG_ENABLED);
}
@@ -127,9 +130,10 @@ public class TabletServerLogger {
}
}
- public TabletServerLogger(TabletServer tserver, long maxSize) {
+ public TabletServerLogger(TabletServer tserver, long maxSize, long maxAge) {
this.tserver = tserver;
this.maxSize = maxSize;
+ this.maxAge = maxAge;
}
private int initializeLoggers(final List<DfsLogger> copy) throws IOException {
@@ -185,6 +189,7 @@ public class TabletServerLogger {
alog.open(tserver.getClientAddressString());
loggers.add(alog);
logSetId.incrementAndGet();
+ this.createTime = System.currentTimeMillis();
return;
} catch (Exception t) {
throw new RuntimeException(t);
@@ -312,11 +317,11 @@ public class TabletServerLogger {
});
}
}
- // if the log gets too big, reset it .. grab the write lock first
+ // if the log gets too big or too old, reset it .. grab the write lock first
logSizeEstimate.addAndGet(4 * 3); // event, tid, seq overhead
testLockAndRun(logSetLock, new TestCallWithWriteLock() {
boolean test() {
- return logSizeEstimate.get() > maxSize;
+ return (logSizeEstimate.get() > maxSize) || ((System.currentTimeMillis() - createTime) > maxAge);
}
void withWriteLock() throws IOException {