You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 19:44:36 UTC
svn commit: r1181965 - in
/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver:
HRegion.java StoreScanner.java
Author: nspiegelberg
Date: Tue Oct 11 17:44:36 2011
New Revision: 1181965
URL: http://svn.apache.org/viewvc?rev=1181965&view=rev
Log:
Adding a metric for showing the overall size of the result set of get operations
Summary: This metric will basically monitor (sadly, in an ever increasing
fashion) the overall size of the list of KVs passed back as a result of a get
operation.
Test Plan:
Did a load test on the cluster and got appropriate output:
"hadoop.regionserver_cf.actions.getsize": 127235584,
"hadoop.regionserver_cf.info.getsize": 17846,
Currently it will report in bytes, so not sure if we want to compact the
overall reported size to print out in KB or something, but probably not, as
that's a rather useless truncation...
Reviewers: kannan, kranganathan
Reviewed By: kannan
CC: hbase@lists, kannan, bogdan
Differential Revision: 313197
Task ID: 688264
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1181965&r1=1181964&r2=1181965&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Oct 11 17:44:36 2011
@@ -273,7 +273,7 @@ public class HRegion implements HeapSize
* the ordered set of column families
* @return a string to print out in metrics
*/
- private String createMutationSignature(Set<byte[]> families) {
+ public static String createMutationSignature(Set<byte[]> families) {
int limit = families.size();
if (1 == limit) {
return "cf." + Bytes.toString(families.iterator().next());
@@ -308,7 +308,7 @@ public class HRegion implements HeapSize
* the family to convert
* @return the string to print out in metrics
*/
- private String createMutationSignature(byte[] family) {
+ public static String createMutationSignature(byte[] family) {
return "cf." + Bytes.toString(family);
}
@@ -1611,7 +1611,7 @@ public class HRegion implements HeapSize
// do after lock
long after = EnvironmentEdgeManager.currentTimeMillis();
- String signature = this.createMutationSignature(familyMap.keySet());
+ String signature = HRegion.createMutationSignature(familyMap.keySet());
HRegion.incrTimeVaryingMetric(signature + ".delete_", after - now);
if (flush) {
@@ -1805,10 +1805,11 @@ public class HRegion implements HeapSize
// else, if all have been consistent so far, check if it still holds
// all else, designate failure signature and mark as unclear
if (null == signature) {
- signature = this.createMutationSignature(put.getFamilyMap().keySet());
+ signature = HRegion.createMutationSignature(put.getFamilyMap()
+ .keySet());
} else {
if (isSignatureClear) {
- if (!signature.equals(this.createMutationSignature(put
+ if (!signature.equals(HRegion.createMutationSignature(put
.getFamilyMap().keySet()))) {
isSignatureClear = false;
signature = "cf.__unknown";
@@ -2094,7 +2095,7 @@ public class HRegion implements HeapSize
// do after lock
long after = EnvironmentEdgeManager.currentTimeMillis();
- String signature = this.createMutationSignature(familyMap.keySet());
+ String signature = HRegion.createMutationSignature(familyMap.keySet());
HRegion.incrTimeVaryingMetric(signature + ".put_", after - now);
if (flush) {
@@ -3370,7 +3371,7 @@ public class HRegion implements HeapSize
// do after lock
long after = EnvironmentEdgeManager.currentTimeMillis();
- String signature = this.createMutationSignature(get.familySet());
+ String signature = HRegion.createMutationSignature(get.familySet());
HRegion.incrTimeVaryingMetric(signature + ".get_", after - now);
return results;
@@ -3443,7 +3444,7 @@ public class HRegion implements HeapSize
// do after lock
long after = EnvironmentEdgeManager.currentTimeMillis();
- String signature = this.createMutationSignature(family);
+ String signature = HRegion.createMutationSignature(family);
HRegion.incrTimeVaryingMetric(signature + ".increment_", after - before);
if (flush) {
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=1181965&r1=1181964&r2=1181965&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java Tue Oct 11 17:44:36 2011
@@ -20,16 +20,17 @@
package org.apache.hadoop.hbase.regionserver;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableSet;
+
import org.apache.commons.lang.NotImplementedException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Scan;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NavigableSet;
+import org.apache.hadoop.hbase.util.Bytes;
/**
* Scanner scans both the memstore and the HStore. Coalesce KeyValue stream
@@ -43,7 +44,7 @@ class StoreScanner implements KeyValueSc
private boolean cacheBlocks;
private int countPerRow = 0;
private int storeLimit;
-
+ private String metricNameGetsize;
// Used to indicate that the scanner has closed (see HBASE-1107)
// Doesnt need to be volatile because it's always accessed via synchronized methods
private boolean closing = false;
@@ -63,6 +64,8 @@ class StoreScanner implements KeyValueSc
StoreScanner(Store store, Scan scan, final NavigableSet<byte[]> columns)
throws IOException {
this.store = store;
+ initializeMetricNames();
+
this.cacheBlocks = scan.getCacheBlocks();
matcher = new ScanQueryMatcher(scan, store.getFamily().getName(),
columns, store.ttl, store.comparator.getRawComparator(),
@@ -104,6 +107,8 @@ class StoreScanner implements KeyValueSc
boolean retainDeletesInOutput)
throws IOException {
this.store = store;
+ this.initializeMetricNames();
+
this.cacheBlocks = false;
this.isGet = false;
matcher = new ScanQueryMatcher(scan, store.getFamily().getName(),
@@ -127,6 +132,8 @@ class StoreScanner implements KeyValueSc
final List<KeyValueScanner> scanners)
throws IOException {
this.store = null;
+ this.initializeMetricNames();
+
this.isGet = false;
this.cacheBlocks = scan.getCacheBlocks();
this.matcher = new ScanQueryMatcher(scan, colFamily, columns, ttl,
@@ -140,6 +147,23 @@ class StoreScanner implements KeyValueSc
heap = new KeyValueHeap(scanners, comparator);
}
+ /**
+ * Method used internally to initialize metric names throughout the
+ * constructors.
+ *
+ * To be called after the store variable has been initialized!
+ */
+ private void initializeMetricNames() {
+ byte[] family;
+ if (null != this.store) {
+ family = this.store.getFamily().getName();
+ } else {
+ family = Bytes.toBytes("__unknown");
+ }
+ String mutationSignature = HRegion.createMutationSignature(family);
+ this.metricNameGetsize = mutationSignature + ".getsize";
+ }
+
/*
* @return List of scanners ordered properly.
*/
@@ -186,6 +210,7 @@ class StoreScanner implements KeyValueSc
return scanners;
}
+ @Override
public synchronized KeyValue peek() {
if (this.heap == null) {
return this.lastTop;
@@ -193,11 +218,13 @@ class StoreScanner implements KeyValueSc
return this.heap.peek();
}
+ @Override
public KeyValue next() {
// throw runtime exception perhaps?
throw new RuntimeException("Never call StoreScanner.next()");
}
+ @Override
public synchronized void close() {
if (this.closing) return;
this.closing = true;
@@ -210,6 +237,7 @@ class StoreScanner implements KeyValueSc
this.lastTop = null; // If both are null, we are closed.
}
+ @Override
public synchronized boolean seek(KeyValue key) throws IOException {
if (this.heap == null) {
@@ -227,6 +255,7 @@ class StoreScanner implements KeyValueSc
* @param limit
* @return true if there are more rows, false if scanner is done
*/
+ @Override
public synchronized boolean next(List<KeyValue> outResult, int limit) throws IOException {
//DebugPrint.println("SS.next");
@@ -269,6 +298,7 @@ class StoreScanner implements KeyValueSc
break LOOP;
}
+ HRegion.incrNumericMetric(this.metricNameGetsize, copyKv.getLength());
results.add(copyKv);
this.heap.next();
if (limit > 0 && (results.size() == limit)) {
@@ -333,11 +363,13 @@ class StoreScanner implements KeyValueSc
return false;
}
+ @Override
public synchronized boolean next(List<KeyValue> outResult) throws IOException {
return next(outResult, -1);
}
// Implementation of ChangedReadersObserver
+ @Override
public synchronized void updateReaders() throws IOException {
if (this.closing) return;