You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2014/12/02 18:30:39 UTC
hbase git commit: HBASE-12576 Add metrics for rolling the HLog if
there are too few DN's in the write pipeline
Repository: hbase
Updated Branches:
refs/heads/branch-1 be29fb4c0 -> 091142f47
HBASE-12576 Add metrics for rolling the HLog if there are too few DN's in the write pipeline
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/091142f4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/091142f4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/091142f4
Branch: refs/heads/branch-1
Commit: 091142f47aa4bd540f6498def942944d206b3643
Parents: be29fb4
Author: Elliott Clark <ec...@apache.org>
Authored: Wed Nov 26 09:53:11 2014 -0800
Committer: Elliott Clark <ec...@apache.org>
Committed: Tue Dec 2 09:30:29 2014 -0800
----------------------------------------------------------------------
.../regionserver/wal/MetricsWALSource.java | 9 ++++
.../regionserver/wal/MetricsWALSourceImpl.java | 19 ++++++-
.../hadoop/hbase/regionserver/LogRoller.java | 2 +-
.../hadoop/hbase/regionserver/wal/FSHLog.java | 10 +++-
.../hbase/regionserver/wal/MetricsWAL.java | 19 +++++--
.../regionserver/wal/WALActionsListener.java | 4 +-
.../hadoop/hbase/wal/DisabledWALProvider.java | 2 +-
.../hbase/regionserver/wal/TestFSHLog.java | 7 ---
.../hbase/regionserver/wal/TestLogRolling.java | 20 +++++--
.../hbase/regionserver/wal/TestMetricsWAL.java | 55 ++++++++++++++++++++
10 files changed, 126 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/091142f4/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java
index ba0df80..f8c746f 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java
@@ -57,6 +57,11 @@ public interface MetricsWALSource extends BaseSource {
String SLOW_APPEND_COUNT_DESC = "Number of appends that were slow.";
String SYNC_TIME = "syncTime";
String SYNC_TIME_DESC = "The time it took to sync the WAL to HDFS.";
+ String ROLL_REQUESTED = "rollRequest";
+ String ROLL_REQUESTED_DESC = "How many times a log roll has been requested total";
+ String LOW_REPLICA_ROLL_REQUESTED = "lowReplicaRollRequest";
+ String LOW_REPLICA_ROLL_REQUESTED_DESC =
+ "How many times a log roll was requested due to too few DN's in the write pipeline.";
/**
* Add the append size.
@@ -83,4 +88,8 @@ public interface MetricsWALSource extends BaseSource {
*/
void incrementSyncTime(long time);
+ void incrementLogRollRequested();
+
+ void incrementLowReplicationLogRoll();
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/091142f4/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
index d602d2f..36c3571 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
@@ -38,6 +38,8 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo
private final MetricHistogram syncTimeHisto;
private final MutableCounterLong appendCount;
private final MutableCounterLong slowAppendCount;
+ private final MutableCounterLong logRollRequested;
+ private final MutableCounterLong lowReplicationLogRollRequested;
public MetricsWALSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT);
@@ -53,8 +55,13 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo
appendTimeHisto = this.getMetricsRegistry().newHistogram(APPEND_TIME, APPEND_TIME_DESC);
appendSizeHisto = this.getMetricsRegistry().newHistogram(APPEND_SIZE, APPEND_SIZE_DESC);
appendCount = this.getMetricsRegistry().newCounter(APPEND_COUNT, APPEND_COUNT_DESC, 0l);
- slowAppendCount = this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l);
+ slowAppendCount =
+ this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l);
syncTimeHisto = this.getMetricsRegistry().newHistogram(SYNC_TIME, SYNC_TIME_DESC);
+ logRollRequested =
+ this.getMetricsRegistry().newCounter(ROLL_REQUESTED, ROLL_REQUESTED_DESC, 0L);
+ lowReplicationLogRollRequested = this.getMetricsRegistry()
+ .newCounter(LOW_REPLICA_ROLL_REQUESTED, LOW_REPLICA_ROLL_REQUESTED_DESC, 0L);
}
@Override
@@ -81,4 +88,14 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo
public void incrementSyncTime(long time) {
syncTimeHisto.add(time);
}
+
+ @Override
+ public void incrementLogRollRequested() {
+ logRollRequested.incr();
+ }
+
+ @Override
+ public void incrementLowReplicationLogRoll() {
+ lowReplicationLogRollRequested.incr();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/091142f4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index 821756d..aa5998b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -67,7 +67,7 @@ class LogRoller extends HasThread {
if (null == walNeedsRoll.putIfAbsent(wal, Boolean.FALSE)) {
wal.registerWALActionsListener(new WALActionsListener.Base() {
@Override
- public void logRollRequested() {
+ public void logRollRequested(boolean lowReplicas) {
walNeedsRoll.put(wal, Boolean.TRUE);
// TODO logs will contend with each other here, replace with e.g. DelayedQueue
synchronized(rollLog) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/091142f4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index 8fd0ba7..43072ce 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -1344,7 +1344,9 @@ public class FSHLog implements WAL {
rollWriterLock.unlock();
}
try {
- if (lowReplication || writer != null && writer.getLength() > logrollsize) requestLogRoll();
+ if (lowReplication || writer != null && writer.getLength() > logrollsize) {
+ requestLogRoll(lowReplication);
+ }
} catch (IOException e) {
LOG.warn("Writer.getLength() failed; continuing", e);
}
@@ -1560,9 +1562,13 @@ public class FSHLog implements WAL {
// public only until class moves to o.a.h.h.wal
public void requestLogRoll() {
+ requestLogRoll(false);
+ }
+
+ private void requestLogRoll(boolean tooFewReplicas) {
if (!this.listeners.isEmpty()) {
for (WALActionsListener i: this.listeners) {
- i.logRollRequested();
+ i.logRollRequested(tooFewReplicas);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/091142f4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java
index ad549f0..4d465e4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java
@@ -19,13 +19,13 @@
package org.apache.hadoop.hbase.regionserver.wal;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
import org.apache.hadoop.util.StringUtils;
-
/**
* Class used to push numbers about the WAL into the metrics subsystem. This will take a
* single function call and turn it into multiple manipulations of the hadoop metrics system.
@@ -37,12 +37,17 @@ public class MetricsWAL extends WALActionsListener.Base {
private final MetricsWALSource source;
public MetricsWAL() {
- source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
+ this(CompatibilitySingletonFactory.getInstance(MetricsWALSource.class));
+ }
+
+ @VisibleForTesting
+ MetricsWAL(MetricsWALSource s) {
+ this.source = s;
}
@Override
public void postSync(final long timeInNanos, final int handlerSyncs) {
- source.incrementSyncTime(timeInNanos/1000000l);
+ source.incrementSyncTime(timeInNanos/1000000L);
}
@Override
@@ -59,4 +64,12 @@ public class MetricsWAL extends WALActionsListener.Base {
StringUtils.humanReadableInt(size)));
}
}
+
+ @Override
+ public void logRollRequested(boolean underReplicated) {
+ source.incrementLogRollRequested();
+ if (underReplicated) {
+ source.incrementLowReplicationLogRoll();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/091142f4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
index 2194ce9..457d859 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
@@ -67,7 +67,7 @@ public interface WALActionsListener {
/**
* A request was made that the WAL be rolled.
*/
- void logRollRequested();
+ void logRollRequested(boolean tooFewReplicas);
/**
* The WAL is about to close.
@@ -127,7 +127,7 @@ public interface WALActionsListener {
public void postLogArchive(Path oldPath, Path newPath) throws IOException {}
@Override
- public void logRollRequested() {}
+ public void logRollRequested(boolean tooFewReplicas) {}
@Override
public void logCloseRequested() {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/091142f4/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
index ac23916..70bfff1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DisabledWALProvider.java
@@ -110,7 +110,7 @@ class DisabledWALProvider implements WALProvider {
public byte[][] rollWriter() {
if (!listeners.isEmpty()) {
for (WALActionsListener listener : listeners) {
- listener.logRollRequested();
+ listener.logRollRequested(false);
}
for (WALActionsListener listener : listeners) {
try {
http://git-wip-us.apache.org/repos/asf/hbase/blob/091142f4/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
index 6a7b7fa..c9a3c91 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java
@@ -85,13 +85,6 @@ import org.junit.rules.TestName;
@Category(MediumTests.class)
public class TestFSHLog {
protected static final Log LOG = LogFactory.getLog(TestFSHLog.class);
- {
- ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
- ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
- ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem"))
- .getLogger().setLevel(Level.ALL);
- ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
- }
protected static Configuration conf;
protected static FileSystem fs;
http://git-wip-us.apache.org/repos/asf/hbase/blob/091142f4/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
index 09b66bb..d0d3691 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
@@ -25,19 +25,16 @@ import static org.junit.Assert.assertTrue;
import java.io.EOFException;
import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -318,6 +315,16 @@ public class TestLogRolling {
server = TEST_UTIL.getRSForFirstRegionInTable(desc.getTableName());
final FSHLog log = (FSHLog) server.getWAL(null);
+ final AtomicBoolean lowReplicationHookCalled = new AtomicBoolean(false);
+
+ log.registerWALActionsListener(new WALActionsListener.Base() {
+ @Override
+ public void logRollRequested(boolean lowReplication) {
+ if (lowReplication) {
+ lowReplicationHookCalled.lazySet(true);
+ }
+ }
+ });
// don't run this test without append support (HDFS-200 & HDFS-142)
assertTrue("Need append support for this test", FSUtils
@@ -369,6 +376,9 @@ public class TestLogRolling {
assertTrue("Missing datanode should've triggered a log roll",
newFilenum > oldFilenum && newFilenum > curTime);
+ assertTrue("The log rolling hook should have been called with the low replication flag",
+ lowReplicationHookCalled.get());
+
// write some more log data (this should use a new hdfs_out)
writeData(table, 3);
assertTrue("The log should not roll again.",
@@ -424,8 +434,10 @@ public class TestLogRolling {
final WAL log = server.getWAL(null);
final List<Path> paths = new ArrayList<Path>();
final List<Integer> preLogRolledCalled = new ArrayList<Integer>();
+
paths.add(DefaultWALProvider.getCurrentFileName(log));
log.registerWALActionsListener(new WALActionsListener.Base() {
+
@Override
public void preLogRoll(Path oldFile, Path newFile) {
LOG.debug("preLogRoll: oldFile="+oldFile+" newFile="+newFile);
http://git-wip-us.apache.org/repos/asf/hbase/blob/091142f4/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java
new file mode 100644
index 0000000..2c7b02e
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java
@@ -0,0 +1,55 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+@Category(SmallTests.class)
+public class TestMetricsWAL {
+ @Test
+ public void testLogRollRequested() throws Exception {
+ MetricsWALSource source = mock(MetricsWALSourceImpl.class);
+ MetricsWAL metricsWAL = new MetricsWAL(source);
+ metricsWAL.logRollRequested(false);
+ metricsWAL.logRollRequested(true);
+
+ // Log roll was requested twice
+ verify(source, times(2)).incrementLogRollRequested();
+ // One was because of low replication on the hlog.
+ verify(source, times(1)).incrementLowReplicationLogRoll();
+ }
+
+ @Test
+ public void testPostSync() throws Exception {
+ long nanos = TimeUnit.MILLISECONDS.toNanos(145);
+ MetricsWALSource source = mock(MetricsWALSourceImpl.class);
+ MetricsWAL metricsWAL = new MetricsWAL(source);
+ metricsWAL.postSync(nanos, 1);
+ verify(source, times(1)).incrementSyncTime(145);
+ }
+}
\ No newline at end of file