You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/11/05 20:18:27 UTC
svn commit: r1539105 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
test/java/org/apache/hadoop/hbase/client/TestHTableMultiplexer.java
Author: liyin
Date: Tue Nov 5 19:18:26 2013
New Revision: 1539105
URL: http://svn.apache.org/r1539105
Log:
[HBASE-9891] Add more counters to HTableMultiplexer
Author: gauravm
Summary: Adding the number of succeeded puts and retries in HTableMultiplexer.
Test Plan: Unit Test
Reviewers: liyintang, aaiyer, manukranthk, rshroff, adela, fan
Reviewed By: liyintang
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D1041574
Task ID: 3081647
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTableMultiplexer.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java?rev=1539105&r1=1539104&r2=1539105&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java Tue Nov 5 19:18:26 2013
@@ -19,6 +19,14 @@
*/
package org.apache.hadoop.hbase.client;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.ipc.HBaseRPCOptions;
+
import java.io.IOException;
import java.util.AbstractMap.SimpleEntry;
import java.util.ArrayList;
@@ -34,14 +42,6 @@ import java.util.concurrent.LinkedBlocki
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HServerAddress;
-import org.apache.hadoop.hbase.ipc.HBaseRPCOptions;
-
/**
* HTableMultiplexer provides a thread-safe non blocking PUT API across all the tables.
* Each put will be sharded into different buffer queues based on its destination region server.
@@ -264,15 +264,42 @@ public class HTableMultiplexer {
/**
* HTableMultiplexerStatus keeps track of the current status of the HTableMultiplexer.
- * report the number of buffered requests and the number of the failed (dropped) requests
+ * report the number of buffered requests, the number of the failed (dropped)
+ * requests, the number of succeeded and the number of retried requests
* in total or on per region server basis.
+ *
+ * Some notes regarding the usage:
+ * 1. The number of buffered requests reported by HTableMultiplexerStatus
+ * can be > than the number of puts to HTableMultiplexer, because it reports
+ * the number of puts in the queue + the number of puts being processed. If
+ * some of the puts being processed need to be retried later, they are added
+ * to the queue, before the processing ends. And hence, some puts might be
+ * counted twice temporarily. But, eventually it will settle down to the
+ * number of puts in the queue.
+ *
+ * 2. The number of retried puts, is the total number of times we retried
+ * puts which failed. So, if a put has to be retried 3 times before it
+ * succeeds, it will increase the retry count by 3.
+ *
+ * 3. The counters in the HTableMultiplexer object are calculated upon
+ * instantiation, or when recalculateCounters() is called. Whenever you want
+ * to get the latest numbers, you can either create a new one (by calling
+ * getHTableMultiplexerStatus() on your HTableMultiplexer instance) or simply
+ * do a .recalculateCounters() on the same status instance.
+ *
+ * 4. Some of the metrics, such as max latency are reset every time the
+ * counters are calculated.
*/
public static class HTableMultiplexerStatus {
private long totalFailedPutCounter;
+ private long totalSucceededPutCounter;
+ private long totalRetriedPutCounter;
private long totalBufferedPutCounter;
private long maxLatency;
private long overallAverageLatency;
private Map<String, Long> serverToFailedCounterMap;
+ private Map<String, Long> serverToSucceededCounterMap;
+ private Map<String, Long> serverToRetriedCounterMap;
private Map<String, Long> serverToBufferedCounterMap;
private Map<String, Long> serverToAverageLatencyMap;
private Map<String, Long> serverToMaxLatencyMap;
@@ -283,10 +310,14 @@ public class HTableMultiplexer {
public HTableMultiplexerStatus(Map<HServerAddress, HTableFlushWorker> serverToFlushWorkerMap) {
this.totalBufferedPutCounter = 0;
this.totalFailedPutCounter = 0;
+ this.totalSucceededPutCounter = 0;
+ this.totalRetriedPutCounter = 0;
this.maxLatency = 0;
this.overallAverageLatency = 0;
this.serverToBufferedCounterMap = new HashMap<String, Long>();
this.serverToFailedCounterMap = new HashMap<String, Long>();
+ this.serverToSucceededCounterMap = new HashMap<String, Long>();
+ this.serverToRetriedCounterMap = new HashMap<String, Long>();
this.serverToAverageLatencyMap = new HashMap<String, Long>();
this.serverToMaxLatencyMap = new HashMap<String, Long>();
this.serverToFlushWorkerMap = serverToFlushWorkerMap;
@@ -308,7 +339,9 @@ public class HTableMultiplexer {
long bufferedCounter = worker.getTotalBufferedCount();
long failedCounter = worker.getTotalFailedCount();
- long serverMaxLatency = worker.getMaxLatency();
+ long succeededCounter = worker.getTotalSucceededCount();
+ long retriedCounter = worker.getTotalRetriedCount();
+ long serverMaxLatency = worker.getAndResetMaxLatency();
AtomicAverageCounter averageCounter = worker.getAverageLatencyCounter();
// Get sum and count pieces separately to compute overall average
SimpleEntry<Long, Integer> averageComponents =
@@ -317,6 +350,8 @@ public class HTableMultiplexer {
this.totalBufferedPutCounter += bufferedCounter;
this.totalFailedPutCounter += failedCounter;
+ this.totalSucceededPutCounter += succeededCounter;
+ this.totalRetriedPutCounter += retriedCounter;
if (serverMaxLatency > this.maxLatency) {
this.maxLatency = serverMaxLatency;
}
@@ -326,6 +361,8 @@ public class HTableMultiplexer {
String hostnameWithPort = addr.getHostNameWithPort();
this.serverToBufferedCounterMap.put(hostnameWithPort, bufferedCounter);
this.serverToFailedCounterMap.put(hostnameWithPort, failedCounter);
+ this.serverToSucceededCounterMap.put(hostnameWithPort, succeededCounter);
+ this.serverToRetriedCounterMap.put(hostnameWithPort, retriedCounter);
this.serverToAverageLatencyMap.put(hostnameWithPort, serverAvgLatency);
this.serverToMaxLatencyMap.put(hostnameWithPort, serverMaxLatency);
this.metrics.resetMultiPutMetrics(hostnameWithPort, worker);
@@ -340,6 +377,31 @@ public class HTableMultiplexer {
: 0;
}
+ private void resetCounters() {
+ this.totalBufferedPutCounter = 0;
+ this.totalFailedPutCounter = 0;
+ this.totalSucceededPutCounter = 0;
+ this.totalRetriedPutCounter = 0;
+ this.maxLatency = 0;
+ this.serverToBufferedCounterMap.clear();
+ this.serverToFailedCounterMap.clear();
+ this.serverToSucceededCounterMap.clear();
+ this.serverToAverageLatencyMap.clear();
+ this.serverToMaxLatencyMap.clear();
+ this.overallAverageLatency = 0;
+ this.overallAvgMultiPutSize = 0;
+ }
+
+ /**
+ * Recalculate all the counters, since the state of the HTM might have
+ * changed since we last calculated them, if we are interested in the latest
+ * values.
+ */
+ public void recalculateCounters() {
+ resetCounters();
+ initialize();
+ }
+
public long getTotalBufferedCounter() {
return this.totalBufferedPutCounter;
}
@@ -348,6 +410,14 @@ public class HTableMultiplexer {
return this.totalFailedPutCounter;
}
+ public long getTotalSucccededPutCounter() {
+ return this.totalSucceededPutCounter;
+ }
+
+ public long getTotalRetriedPutCounter() {
+ return this.totalRetriedPutCounter;
+ }
+
public long getMaxLatency() {
return this.maxLatency;
}
@@ -392,14 +462,14 @@ public class HTableMultiplexer {
private static class PutStatus {
private final HRegionInfo regionInfo;
private final Put put;
- private final int retryCount;
+ private final int maxRetryCount;
private final HBaseRPCOptions options;
public PutStatus(final HRegionInfo regionInfo, final Put put,
- final int retryCount, final HBaseRPCOptions options) {
+ final int maxRetryCount, final HBaseRPCOptions options) {
this.regionInfo = regionInfo;
this.put = put;
- this.retryCount = retryCount;
+ this.maxRetryCount = maxRetryCount;
this.options = options;
}
@@ -411,8 +481,17 @@ public class HTableMultiplexer {
return put;
}
+ /**
+ * @deprecated Use {@link #getMaxRetryCount()} instead.
+ * @return
+ */
+ @Deprecated
public int getRetryCount() {
- return retryCount;
+ return getMaxRetryCount();
+ }
+
+ public int getMaxRetryCount() {
+ return maxRetryCount;
}
public HBaseRPCOptions getOptions () {
@@ -468,6 +547,8 @@ public class HTableMultiplexer {
private HConnection connection;
private HTableMultiplexer htableMultiplexer;
private AtomicLong totalFailedPutCount;
+ private AtomicLong totalSucceededPutCount;
+ private AtomicLong totalRetriedPutCount;
private AtomicInteger currentProcessingPutCount;
private AtomicInteger minProcessingPutCount;
private AtomicInteger maxProcessingPutCount;
@@ -484,6 +565,8 @@ public class HTableMultiplexer {
this.htableMultiplexer = htableMultiplexer;
this.queue = queue;
this.totalFailedPutCount = new AtomicLong(0);
+ this.totalSucceededPutCount = new AtomicLong(0);
+ this.totalRetriedPutCount = new AtomicLong(0);
this.currentProcessingPutCount = new AtomicInteger(0);
this.minProcessingPutCount = new AtomicInteger(0);
this.maxProcessingPutCount = new AtomicInteger(0);
@@ -496,6 +579,14 @@ public class HTableMultiplexer {
return totalFailedPutCount.get();
}
+ public long getTotalSucceededCount() {
+ return totalSucceededPutCount.get();
+ }
+
+ public long getTotalRetriedCount() {
+ return totalRetriedPutCount.get();
+ }
+
public long getTotalBufferedCount() {
return queue.size() + currentProcessingPutCount.get();
}
@@ -504,23 +595,32 @@ public class HTableMultiplexer {
return this.averageLatency;
}
+ /**
+ * @deprecated Use {@link #getAndResetMaxLatency()} instead.
+ * @return
+ */
+ @Deprecated
public long getMaxLatency() {
return this.maxLatency.getAndSet(0);
}
+ public long getAndResetMaxLatency() {
+ return this.maxLatency.getAndSet(0);
+ }
+
private boolean resubmitFailedPut(PutStatus failedPutStatus, HServerAddress oldLoc) throws IOException{
Put failedPut = failedPutStatus.getPut();
// The currentPut is failed. So get the table name for the currentPut.
byte[] tableName = failedPutStatus.getRegionInfo().getTableDesc().getName();
// Decrease the retry count
- int retryCount = failedPutStatus.getRetryCount() - 1;
-
+ int retryCount = failedPutStatus.getMaxRetryCount() - 1;
+
if (retryCount <= 0) {
// Update the failed counter and no retry any more.
return false;
} else {
// Retry one more time
- HBaseRPCOptions options = failedPutStatus.getOptions ();
+ HBaseRPCOptions options = failedPutStatus.getOptions();
return this.htableMultiplexer.put(tableName, failedPut, retryCount, options);
}
}
@@ -602,6 +702,7 @@ public class HTableMultiplexer {
}
}
+ long putsToRetry = 0;
if (failed != null) {
if (failed.size() == processingList.size()) {
// All the puts for this region server are failed. Going to retry it later
@@ -619,9 +720,15 @@ public class HTableMultiplexer {
}
}
}
+ putsToRetry = failed.size() - failedCount;
}
// Update the totalFailedCount
this.totalFailedPutCount.addAndGet(failedCount);
+ // Update the totalSucceededPutCount
+ this.totalSucceededPutCount.addAndGet(processingList.size() -
+ failedCount - putsToRetry);
+ // Updated the total retried put counts.
+ this.totalRetriedPutCount.addAndGet(putsToRetry);
elapsed = System.currentTimeMillis() - start;
// Update latency counters
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTableMultiplexer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTableMultiplexer.java?rev=1539105&r1=1539104&r2=1539105&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTableMultiplexer.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHTableMultiplexer.java Tue Nov 5 19:18:26 2013
@@ -19,13 +19,10 @@
*/
package org.apache.hadoop.hbase.client;
-import java.util.ArrayList;
-import java.util.List;
-
import junit.framework.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.HTableMultiplexer.HTableMultiplexerStatus;
import org.apache.hadoop.hbase.ipc.HBaseRPCOptions;
@@ -34,6 +31,9 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.util.ArrayList;
+import java.util.List;
+
public class TestHTableMultiplexer {
final Log LOG = LogFactory.getLog(getClass());
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@@ -128,9 +128,77 @@ public class TestHTableMultiplexer {
tries++;
} catch (InterruptedException e) {
} // ignore
+ status.recalculateCounters();
} while (status.getTotalBufferedCounter() != 0 && tries != retries);
Assert.assertEquals("There are still some buffered puts left in the queue",
0, status.getTotalBufferedCounter());
}
+
+
+ /**
+ * This test is to verify that the buffered and succeeded counters work as
+ * expected.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testCounters() throws Exception {
+ byte[] TABLE = Bytes.toBytes("testCounters");
+ Configuration conf = TEST_UTIL.getConfiguration();
+ conf.setLong("hbase.htablemultiplexer.flush.frequency.ms", 100);
+
+ HTableMultiplexer multiplexer =
+ new HTableMultiplexer(TEST_UTIL.getConfiguration(),
+ 100);
+ HTableMultiplexerStatus status = multiplexer.getHTableMultiplexerStatus();
+
+ HTable ht = TEST_UTIL.createTable(TABLE, new byte[][] { FAMILY });
+
+ int numPuts = 100;
+ boolean bufferWasNonEmptyAtSomePoint = false;
+ for (int i = 1; i <= numPuts; i++) {
+ byte[] row = Bytes.toBytes("Row" + i);
+ byte[] qualifier = Bytes.toBytes("Qualifier" + i);
+ byte[] value = Bytes.toBytes("Value" + i);
+ Put put = new Put(row);
+ put.add(FAMILY, qualifier, value);
+
+ multiplexer.put(TABLE, put, HBaseRPCOptions.DEFAULT);
+ status.recalculateCounters();
+ if (status.getTotalBufferedCounter() > 0) {
+ bufferWasNonEmptyAtSomePoint = true;
+ }
+
+ try {
+ Thread.sleep(5);
+ } catch (Exception e) {}
+ }
+
+ // Verify that all the puts got flushed.
+ verifyAllBufferedPutsHasFlushed(status);
+
+ // Recalculate the counters, and ensure that:
+ // 1. The number of buffered puts was non-zero at some point.
+ // 2. The number of buffered puts is zero now.
+ // 3. The total number of succeeded puts is equal to the number of puts.
+ status.recalculateCounters();
+ Assert.assertTrue(bufferWasNonEmptyAtSomePoint);
+ Assert.assertEquals(0, status.getTotalBufferedCounter());
+ Assert.assertEquals(numPuts, status.getTotalSucccededPutCounter());
+
+ // Check if all the puts went through correctly.
+ for (int i = 1; i <= numPuts; i++) {
+ byte[] row = Bytes.toBytes("Row" + i);
+ byte[] qualifier = Bytes.toBytes("Qualifier" + i);
+ byte[] value = Bytes.toBytes("Value" + i);
+
+ // verify that the Get returns the correct result
+ Get get = new Get(row);
+ get.addColumn(FAMILY, qualifier);
+ Result r = ht.get(get);
+ Assert.assertEquals(0,
+ Bytes.compareTo(value, r.getValue(FAMILY, qualifier)));
+ }
+ }
}