You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2014/12/11 03:54:37 UTC
[1/2] hbase git commit: HBASE-12631 Backport HBASE-12576 (Add metrics
for rolling the HLog if there are too few DNs in the write pipeline) to 0.98
Repository: hbase
Updated Branches:
refs/heads/0.98 06c4d8a8e -> bc594c079
HBASE-12631 Backport HBASE-12576 (Add metrics for rolling the HLog if there are too few DNs in the write pipeline) to 0.98
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bc594c07
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bc594c07
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bc594c07
Branch: refs/heads/0.98
Commit: bc594c0793b5cbeedccf7633d96e901626685ec9
Parents: 4c1a31f
Author: Andrew Purtell <ap...@apache.org>
Authored: Wed Dec 10 17:23:06 2014 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Dec 10 17:23:06 2014 -0800
----------------------------------------------------------------------
.../regionserver/wal/MetricsWALSource.java | 9 ++++
.../regionserver/wal/MetricsWALSourceImpl.java | 21 ++++++--
.../regionserver/wal/MetricsWALSourceImpl.java | 19 ++++++-
.../hadoop/hbase/regionserver/LogRoller.java | 2 +-
.../hadoop/hbase/regionserver/wal/FSHLog.java | 16 +++---
.../hbase/regionserver/wal/MetricsWAL.java | 52 ++++++++++++++++++-
.../regionserver/wal/WALActionsListener.java | 3 +-
.../replication/regionserver/Replication.java | 2 +-
.../hbase/mapreduce/TestImportExport.java | 2 +-
.../hadoop/hbase/regionserver/wal/TestHLog.java | 2 +-
.../regionserver/wal/TestLogRollPeriod.java | 2 +-
.../hbase/regionserver/wal/TestLogRolling.java | 54 ++++++++++++++------
.../hbase/regionserver/wal/TestMetricsWAL.java | 44 ++++++++++++++++
.../wal/TestWALActionsListener.java | 2 +-
.../TestReplicationHLogReaderManager.java | 2 +-
15 files changed, 195 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java
index 1c59f65..b642a49 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSource.java
@@ -57,6 +57,11 @@ public interface MetricsWALSource extends BaseSource {
String SLOW_APPEND_COUNT_DESC = "Number of appends that were slow.";
String SYNC_TIME = "syncTime";
String SYNC_TIME_DESC = "The time it took to sync the HLog to HDFS.";
+ String ROLL_REQUESTED = "rollRequest";
+ String ROLL_REQUESTED_DESC = "How many times a log roll has been requested total";
+ String LOW_REPLICA_ROLL_REQUESTED = "lowReplicaRollRequest";
+ String LOW_REPLICA_ROLL_REQUESTED_DESC =
+ "How many times a log roll was requested due to too few DN's in the write pipeline.";
/**
* Add the append size.
@@ -83,4 +88,8 @@ public interface MetricsWALSource extends BaseSource {
*/
void incrementSyncTime(long time);
+ void incrementLogRollRequested();
+
+ void incrementLowReplicationLogRoll();
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
index d1a318e..505e989 100644
--- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
+++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.lib.MetricMutableCounterLong;
-
/**
* Class that transitions metrics from HLog's MetricsWAL into the metrics subsystem.
*
@@ -35,7 +34,8 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo
private final MetricMutableCounterLong appendCount;
private final MetricMutableCounterLong slowAppendCount;
private final MetricHistogram syncTimeHisto;
-
+ private final MetricMutableCounterLong logRollRequested;
+ private final MetricMutableCounterLong lowReplicationLogRollRequested;
public MetricsWALSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT);
@@ -51,8 +51,13 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo
appendTimeHisto = this.getMetricsRegistry().newHistogram(APPEND_TIME, APPEND_TIME_DESC);
appendSizeHisto = this.getMetricsRegistry().newHistogram(APPEND_SIZE, APPEND_SIZE_DESC);
appendCount = this.getMetricsRegistry().newCounter(APPEND_COUNT, APPEND_COUNT_DESC, 0l);
- slowAppendCount = this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l);
+ slowAppendCount =
+ this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l);
syncTimeHisto = this.getMetricsRegistry().newHistogram(SYNC_TIME, SYNC_TIME_DESC);
+ logRollRequested =
+ this.getMetricsRegistry().newCounter(ROLL_REQUESTED, ROLL_REQUESTED_DESC, 0L);
+ lowReplicationLogRollRequested = this.getMetricsRegistry()
+ .newCounter(LOW_REPLICA_ROLL_REQUESTED, LOW_REPLICA_ROLL_REQUESTED_DESC, 0L);
}
@Override
@@ -79,4 +84,14 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo
public void incrementSyncTime(long time) {
syncTimeHisto.add(time);
}
+
+ @Override
+ public void incrementLogRollRequested() {
+ logRollRequested.incr();
+ }
+
+ @Override
+ public void incrementLowReplicationLogRoll() {
+ lowReplicationLogRollRequested.incr();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
index ad8f24c..f6f12b8 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWALSourceImpl.java
@@ -37,6 +37,8 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo
private final MetricHistogram syncTimeHisto;
private final MutableCounterLong appendCount;
private final MutableCounterLong slowAppendCount;
+ private final MutableCounterLong logRollRequested;
+ private final MutableCounterLong lowReplicationLogRollRequested;
public MetricsWALSourceImpl() {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT);
@@ -52,8 +54,13 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo
appendTimeHisto = this.getMetricsRegistry().newHistogram(APPEND_TIME, APPEND_TIME_DESC);
appendSizeHisto = this.getMetricsRegistry().newHistogram(APPEND_SIZE, APPEND_SIZE_DESC);
appendCount = this.getMetricsRegistry().newCounter(APPEND_COUNT, APPEND_COUNT_DESC, 0l);
- slowAppendCount = this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l);
+ slowAppendCount =
+ this.getMetricsRegistry().newCounter(SLOW_APPEND_COUNT, SLOW_APPEND_COUNT_DESC, 0l);
syncTimeHisto = this.getMetricsRegistry().newHistogram(SYNC_TIME, SYNC_TIME_DESC);
+ logRollRequested =
+ this.getMetricsRegistry().newCounter(ROLL_REQUESTED, ROLL_REQUESTED_DESC, 0L);
+ lowReplicationLogRollRequested = this.getMetricsRegistry()
+ .newCounter(LOW_REPLICA_ROLL_REQUESTED, LOW_REPLICA_ROLL_REQUESTED_DESC, 0L);
}
@Override
@@ -80,4 +87,14 @@ public class MetricsWALSourceImpl extends BaseSourceImpl implements MetricsWALSo
public void incrementSyncTime(long time) {
syncTimeHisto.add(time);
}
+
+ @Override
+ public void incrementLogRollRequested() {
+ logRollRequested.incr();
+ }
+
+ @Override
+ public void incrementLowReplicationLogRoll() {
+ lowReplicationLogRollRequested.incr();
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
index 0e98143..45722e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
@@ -141,7 +141,7 @@ class LogRoller extends HasThread implements WALActionsListener {
}
}
- public void logRollRequested() {
+ public void logRollRequested(boolean lowReplicas) {
synchronized (rollLog) {
rollLog.set(true);
rollLog.notifyAll();
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index f96e645..b528871 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -36,7 +36,6 @@ import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
@@ -439,6 +438,7 @@ class FSHLog implements HLog, Syncable {
coprocessorHost = new WALCoprocessorHost(this, conf);
this.metrics = new MetricsWAL();
+ registerWALActionsListener(metrics);
}
/**
@@ -1258,16 +1258,16 @@ class FSHLog implements HLog, Syncable {
asyncNotifier.setFlushedTxid(this.lastSyncedTxid);
// 4. check and do logRoll if needed
- boolean logRollNeeded = false;
+ boolean lowReplication = false;
if (rollWriterLock.tryLock()) {
try {
- logRollNeeded = checkLowReplication();
+ lowReplication = checkLowReplication();
} finally {
rollWriterLock.unlock();
}
try {
- if (logRollNeeded || writer != null && writer.getLength() > logrollsize) {
- requestLogRoll();
+ if (lowReplication || writer != null && writer.getLength() > logrollsize) {
+ requestLogRoll(lowReplication);
}
} catch (IOException e) {
LOG.warn("writer.getLength() failed,this failure won't block here");
@@ -1467,9 +1467,13 @@ class FSHLog implements HLog, Syncable {
}
private void requestLogRoll() {
+ requestLogRoll(false);
+ }
+
+ private void requestLogRoll(boolean tooFewReplicas) {
if (!this.listeners.isEmpty()) {
for (WALActionsListener i: this.listeners) {
- i.logRollRequested();
+ i.logRollRequested(tooFewReplicas);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java
index cf4b7a6..ec1e89f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/MetricsWAL.java
@@ -19,25 +19,37 @@
package org.apache.hadoop.hbase.regionserver.wal;
+import java.io.IOException;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Class used to push numbers about the WAL into the metrics subsystem. This will take a
* single function call and turn it into multiple manipulations of the hadoop metrics system.
*/
@InterfaceAudience.Private
-public class MetricsWAL {
+public class MetricsWAL implements WALActionsListener {
static final Log LOG = LogFactory.getLog(MetricsWAL.class);
private final MetricsWALSource source;
public MetricsWAL() {
- source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
+ this(CompatibilitySingletonFactory.getInstance(MetricsWALSource.class));
+ }
+
+ @VisibleForTesting
+ MetricsWAL(MetricsWALSource s) {
+ this.source = s;
}
public void finishSync(long time) {
@@ -58,4 +70,40 @@ public class MetricsWAL {
StringUtils.humanReadableInt(size)));
}
}
+
+ @Override
+ public void logRollRequested(boolean underReplicated) {
+ source.incrementLogRollRequested();
+ if (underReplicated) {
+ source.incrementLowReplicationLogRoll();
+ }
+ }
+
+ @Override
+ public void preLogRoll(Path oldPath, Path newPath) throws IOException {
+ }
+
+ @Override
+ public void postLogRoll(Path oldPath, Path newPath) throws IOException {
+ }
+
+ @Override
+ public void preLogArchive(Path oldPath, Path newPath) throws IOException {
+ }
+
+ @Override
+ public void postLogArchive(Path oldPath, Path newPath) throws IOException {
+ }
+
+ @Override
+ public void logCloseRequested() {
+ }
+
+ @Override
+ public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit) {
+ }
+
+ @Override
+ public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) {
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
index 84552a7..dee042d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.java
@@ -64,8 +64,9 @@ public interface WALActionsListener {
/**
* A request was made that the WAL be rolled.
+ * @param tooFewReplicas roll requested because of too few replicas if true
*/
- void logRollRequested();
+ void logRollRequested(boolean tooFewReplicas);
/**
* The WAL is about to close.
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index 15ccffe..b665d70 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -298,7 +298,7 @@ public class Replication implements WALActionsListener,
}
@Override
- public void logRollRequested() {
+ public void logRollRequested(boolean tooFewReplicas) {
// Not interested
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
index 3e6934d..80f6ac5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
@@ -705,7 +705,7 @@ public class TestImportExport {
}
@Override
- public void logRollRequested() {
+ public void logRollRequested(boolean tooFewReplicas) {
// Not interested in this method.
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index ce9a147..9bb03be 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -1222,7 +1222,7 @@ public class TestHLog {
}
@Override
- public void logRollRequested() {
+ public void logRollRequested(boolean tooFewReplicas) {
// TODO Auto-generated method stub
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java
index 30c3be8..a102103 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollPeriod.java
@@ -149,7 +149,7 @@ public class TestLogRollPeriod {
@Override
public void postLogArchive(Path oldFile, Path newFile) {}
@Override
- public void logRollRequested() {}
+ public void logRollRequested(boolean tooFewReplicas) {}
@Override
public void logCloseRequested() {}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
index 4e50519..d69d095 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
@@ -32,10 +32,10 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -63,12 +63,10 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
import org.apache.hadoop.hbase.util.Threads;
-import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
-import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
-import org.apache.log4j.Level;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -92,18 +90,6 @@ public class TestLogRolling {
private MiniHBaseCluster cluster;
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
- // verbose logging on classes that are touched in these tests
- {
- ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
- ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
- ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hdfs.server.namenode.FSNamesystem"))
- .getLogger().setLevel(Level.ALL);
- ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.ALL);
- ((Log4JLogger)HRegionServer.LOG).getLogger().setLevel(Level.ALL);
- ((Log4JLogger)HRegion.LOG).getLogger().setLevel(Level.ALL);
- ((Log4JLogger)HLog.LOG).getLogger().setLevel(Level.ALL);
- }
-
/**
* constructor
* @throws Exception
@@ -341,6 +327,37 @@ public class TestLogRolling {
server = TEST_UTIL.getRSForFirstRegionInTable(Bytes.toBytes(tableName));
this.log = server.getWAL();
+ final AtomicBoolean lowReplicationHookCalled = new AtomicBoolean(false);
+
+ log.registerWALActionsListener(new WALActionsListener() {
+ @Override
+ public void logRollRequested(boolean lowReplication) {
+ if (lowReplication) {
+ lowReplicationHookCalled.lazySet(true);
+ }
+ }
+
+ @Override
+ public void preLogRoll(Path oldPath, Path newPath) throws IOException { }
+
+ @Override
+ public void postLogRoll(Path oldPath, Path newPath) throws IOException { }
+
+ @Override
+ public void preLogArchive(Path oldPath, Path newPath) throws IOException { }
+
+ @Override
+ public void postLogArchive(Path oldPath, Path newPath) throws IOException { }
+
+ @Override
+ public void logCloseRequested() { }
+
+ @Override
+ public void visitLogEntryBeforeWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit) { }
+
+ @Override
+ public void visitLogEntryBeforeWrite(HTableDescriptor htd, HLogKey logKey, WALEdit logEdit) { }
+ });
assertTrue("Need HDFS-826 for this test", ((FSHLog) log).canGetCurReplicas());
// don't run this test without append support (HDFS-200 & HDFS-142)
@@ -389,6 +406,9 @@ public class TestLogRolling {
assertTrue("Missing datanode should've triggered a log roll",
newFilenum > oldFilenum && newFilenum > curTime);
+ assertTrue("The log rolling hook should have been called with the low replication flag",
+ lowReplicationHookCalled.get());
+
// write some more log data (this should use a new hdfs_out)
writeData(table, 3);
assertTrue("The log should not roll again.",
@@ -461,7 +481,7 @@ public class TestLogRolling {
@Override
public void postLogArchive(Path oldFile, Path newFile) {}
@Override
- public void logRollRequested() {}
+ public void logRollRequested(boolean tooFewReplicas) {}
@Override
public void logCloseRequested() {}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java
new file mode 100644
index 0000000..719b80e
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestMetricsWAL.java
@@ -0,0 +1,44 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+@Category(SmallTests.class)
+public class TestMetricsWAL {
+ @Test
+ public void testLogRollRequested() throws Exception {
+ MetricsWALSource source = mock(MetricsWALSourceImpl.class);
+ MetricsWAL metricsWAL = new MetricsWAL(source);
+ metricsWAL.logRollRequested(false);
+ metricsWAL.logRollRequested(true);
+
+ // Log roll was requested twice
+ verify(source, times(2)).incrementLogRollRequested();
+ // One was because of low replication on the hlog.
+ verify(source, times(1)).incrementLowReplicationLogRoll();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
index 85b1dc3..4d50fbb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALActionsListener.java
@@ -152,7 +152,7 @@ public class TestWALActionsListener {
}
@Override
- public void logRollRequested() {
+ public void logRollRequested(boolean tooFewReplicas) {
// Not interested
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bc594c07/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationHLogReaderManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationHLogReaderManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationHLogReaderManager.java
index cdaeb58..2fd16d0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationHLogReaderManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationHLogReaderManager.java
@@ -225,7 +225,7 @@ public class TestReplicationHLogReaderManager {
public void postLogArchive(Path oldPath, Path newPath) throws IOException {}
@Override
- public void logRollRequested() {}
+ public void logRollRequested(boolean tooFewReplicas) {}
@Override
public void logCloseRequested() {}
[2/2] hbase git commit: HBASE-12604 Backport HBASE-12128 (Cache
configuration and RpcController selection for Table in Connection) to 0.98
Posted by ap...@apache.org.
HBASE-12604 Backport HBASE-12128 (Cache configuration and RpcController selection for Table in Connection) to 0.98
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4c1a31f2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4c1a31f2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4c1a31f2
Branch: refs/heads/0.98
Commit: 4c1a31f289ca76a3196e5456108858e519e9c53e
Parents: 06c4d8a
Author: Andrew Purtell <ap...@apache.org>
Authored: Wed Dec 10 17:23:06 2014 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Dec 10 17:23:06 2014 -0800
----------------------------------------------------------------------
.../hadoop/hbase/client/HConnectionManager.java | 72 ++++++++++--
.../org/apache/hadoop/hbase/client/HTable.java | 64 +++++++----
.../hadoop/hbase/client/TableConfiguration.java | 112 +++++++++++++++++++
3 files changed, 222 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4c1a31f2/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
index 2c29800..da7a122 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
@@ -77,45 +77,89 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AssignRegionResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateNamespaceRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateNamespaceResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.CreateTableResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteColumnResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteNamespaceRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteNamespaceResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteSnapshotResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DeleteTableResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DisableTableResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.DispatchMergingRegionsResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableTableResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ExecProcedureResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetClusterStatusResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetCompletedSnapshotsResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetNamespaceDescriptorRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetNamespaceDescriptorResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetSchemaAlterStatusResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableNamesResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListNamespaceDescriptorsRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListNamespaceDescriptorsResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableDescriptorsByNamespaceRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableDescriptorsByNamespaceResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ListTableNamesByNamespaceResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyColumnResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningResponse;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SnapshotResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.StopMasterResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.TruncateTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.TruncateTableResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.UnassignRegionResponse;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.AddColumnRequest;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.BalanceRequest;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.*;
import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider;
@@ -583,6 +627,10 @@ public class HConnectionManager {
private final Configuration conf;
+ // cache the configuration value for tables so that we can avoid calling
+ // the expensive Configuration to fetch the value multiple times.
+ private final TableConfiguration tableConfig;
+
// Client rpc instance.
private RpcClient rpcClient;
@@ -612,6 +660,10 @@ public class HConnectionManager {
private User user;
+ private RpcRetryingCallerFactory rpcCallerFactory;
+
+ private RpcControllerFactory rpcControllerFactory;
+
/**
* Cluster registry of basic info such as clusterid and meta region location.
*/
@@ -667,6 +719,9 @@ public class HConnectionManager {
}, conf, listenerClass);
}
}
+
+ this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(conf);
+ this.rpcControllerFactory = RpcControllerFactory.instantiate(conf);
}
/** Dummy nonce generator for disabled nonces. */
@@ -686,11 +741,11 @@ public class HConnectionManager {
*/
protected HConnectionImplementation(Configuration conf) {
this.conf = conf;
+ this.tableConfig = new TableConfiguration(conf);
this.closed = false;
this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
- this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
- HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
+ this.numTries = tableConfig.getRetriesNumber();
this.rpcTimeout = conf.getInt(
HConstants.HBASE_RPC_TIMEOUT_KEY,
HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
@@ -710,6 +765,8 @@ public class HConnectionManager {
this.prefetchRegionLimit = conf.getInt(
HConstants.HBASE_CLIENT_PREFETCH_LIMIT,
HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT);
+ this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(conf);
+ this.rpcControllerFactory = RpcControllerFactory.instantiate(conf);
}
@Override
@@ -742,7 +799,8 @@ public class HConnectionManager {
if (managed) {
throw new IOException("The connection has to be unmanaged.");
}
- return new HTable(tableName, this, pool);
+ return new HTable(tableName, this, tableConfig, rpcCallerFactory, rpcControllerFactory,
+ pool);
}
private ExecutorService getBatchPool() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/4c1a31f2/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
index 687a1c9..1d8a037 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
@@ -127,13 +127,13 @@ public class HTable implements HTableInterface {
protected HConnection connection;
private final TableName tableName;
private volatile Configuration configuration;
+ private TableConfiguration tableConfiguration;
protected List<Row> writeAsyncBuffer = new LinkedList<Row>();
private long writeBufferSize;
private boolean clearBufferOnFail;
private boolean autoFlush;
protected long currentWriteBufferSize;
protected int scannerCaching;
- private int maxKeyValueSize;
private ExecutorService pool; // For Multi
private boolean closed;
private int operationTimeout;
@@ -308,15 +308,41 @@ public class HTable implements HTableInterface {
*/
public HTable(TableName tableName, final HConnection connection,
final ExecutorService pool) throws IOException {
+ this(tableName, connection, null, null, null, pool);
+ }
+
+ /**
+ * Creates an object to access a HBase table.
+ * Shares zookeeper connection and other resources with other HTable instances
+ * created with the same <code>connection</code> instance.
+ * Use this constructor when the ExecutorService and HConnection instance are
+ * externally managed.
+ * @param tableName Name of the table.
+ * @param connection HConnection to be used.
+ * @param tableConfig table configuration
+ * @param rpcCallerFactory RPC caller factory
+ * @param rpcControllerFactory RPC controller factory
+ * @param pool ExecutorService to be used.
+ * @throws IOException if a remote or network exception occurs
+ */
+ public HTable(TableName tableName, final HConnection connection,
+ final TableConfiguration tableConfig,
+ final RpcRetryingCallerFactory rpcCallerFactory,
+ final RpcControllerFactory rpcControllerFactory,
+ final ExecutorService pool) throws IOException {
if (connection == null || connection.isClosed()) {
throw new IllegalArgumentException("Connection is null or closed.");
}
this.tableName = tableName;
- this.cleanupPoolOnClose = this.cleanupConnectionOnClose = false;
this.connection = connection;
this.configuration = connection.getConfiguration();
+ this.tableConfiguration = tableConfig;
+ this.cleanupPoolOnClose = this.cleanupConnectionOnClose = false;
this.pool = pool;
+ this.rpcCallerFactory = rpcCallerFactory;
+ this.rpcControllerFactory = rpcControllerFactory;
+
this.finishSetup();
}
@@ -325,6 +351,7 @@ public class HTable implements HTableInterface {
*/
protected HTable(){
tableName = null;
+ tableConfiguration = new TableConfiguration();
cleanupPoolOnClose = false;
cleanupConnectionOnClose = false;
}
@@ -340,31 +367,30 @@ public class HTable implements HTableInterface {
* setup this HTable's parameter based on the passed configuration
*/
private void finishSetup() throws IOException {
+ if (tableConfiguration == null) {
+ tableConfiguration = new TableConfiguration(configuration);
+ }
this.operationTimeout = tableName.isSystemTable() ?
- this.configuration.getInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT,
- HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT):
- this.configuration.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
- HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
- this.writeBufferSize = this.configuration.getLong(
- "hbase.client.write.buffer", 2097152);
+ tableConfiguration.getMetaOperationTimeout() : tableConfiguration.getOperationTimeout();
+ this.writeBufferSize = tableConfiguration.getWriteBufferSize();
this.clearBufferOnFail = true;
this.autoFlush = true;
this.currentWriteBufferSize = 0;
- this.scannerCaching = this.configuration.getInt(
- HConstants.HBASE_CLIENT_SCANNER_CACHING,
- HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
+ this.scannerCaching = tableConfiguration.getScannerCaching();
- this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(configuration);
- this.rpcControllerFactory = RpcControllerFactory.instantiate(configuration);
- ap = new AsyncProcess<Object>(connection, tableName, pool, null,
- configuration, rpcCallerFactory, rpcControllerFactory);
+ if (this.rpcCallerFactory == null) {
+ this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(configuration);
+ }
+ if (this.rpcControllerFactory == null) {
+ this.rpcControllerFactory = RpcControllerFactory.instantiate(configuration);
+ }
+
+ ap = new AsyncProcess<Object>(connection, tableName, pool, null, configuration,
+ rpcCallerFactory, rpcControllerFactory);
- this.maxKeyValueSize = getMaxKeyValueSize(this.configuration);
this.closed = false;
}
-
-
/**
* {@inheritDoc}
*/
@@ -1335,7 +1361,7 @@ public class HTable implements HTableInterface {
// validate for well-formedness
public void validatePut(final Put put) throws IllegalArgumentException {
- validatePut(put, maxKeyValueSize);
+ validatePut(put, tableConfiguration.getMaxKeyValueSize());
}
// validate for well-formedness
http://git-wip-us.apache.org/repos/asf/hbase/blob/4c1a31f2/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableConfiguration.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableConfiguration.java
new file mode 100644
index 0000000..11d56de
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/TableConfiguration.java
@@ -0,0 +1,112 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ *
+ * Configuration is a heavy weight registry that does a lot of string operations and regex matching.
+ * Method calls into Configuration account for high CPU usage and have huge performance impact.
+ * This class caches the value in the TableConfiguration object to improve performance.
+ * see HBASE-12128
+ *
+ */
+@InterfaceAudience.Private
+public class TableConfiguration {
+
+ private final long writeBufferSize;
+
+ private final int metaOperationTimeout;
+
+ private final int operationTimeout;
+
+ private final int scannerCaching;
+
+ private final int retries;
+
+ private final int maxKeyValueSize;
+
+ /**
+ * Constructor
+ * @param conf Configuration object
+ */
+ TableConfiguration(Configuration conf) {
+ this.writeBufferSize = conf.getLong("hbase.client.write.buffer", 2097152);
+
+ this.metaOperationTimeout = conf.getInt(
+ HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT,
+ HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
+
+ this.operationTimeout = conf.getInt(
+ HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
+
+ this.scannerCaching = conf.getInt(
+ HConstants.HBASE_CLIENT_SCANNER_CACHING, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
+
+ this.retries = conf.getInt(
+ HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
+
+ this.maxKeyValueSize = conf.getInt("hbase.client.keyvalue.maxsize", -1);
+ }
+
+ /**
+ * Constructor
+ * This is for internal testing purpose (using the default value).
+ * In real usage, we should read the configuration from the Configuration object.
+ */
+ @VisibleForTesting
+ protected TableConfiguration() {
+ this.writeBufferSize = 2097152;
+ this.metaOperationTimeout = HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT;
+ this.operationTimeout = HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT;
+ this.scannerCaching = HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING;
+ this.retries = HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER;
+ this.maxKeyValueSize = -1;
+ }
+
+ public long getWriteBufferSize() {
+ return writeBufferSize;
+ }
+
+ public int getMetaOperationTimeout() {
+ return metaOperationTimeout;
+ }
+
+ public int getOperationTimeout() {
+ return operationTimeout;
+ }
+
+ public int getScannerCaching() {
+ return scannerCaching;
+ }
+
+ public int getRetriesNumber() {
+ return retries;
+ }
+
+ public int getMaxKeyValueSize() {
+ return maxKeyValueSize;
+ }
+}
\ No newline at end of file