You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/04/02 03:26:01 UTC
svn commit: r1087953 - in
/hbase/trunk/src/main/java/org/apache/hadoop/hbase: HServerInfo.java
HServerLoad.java regionserver/HRegion.java regionserver/HRegionServer.java
regionserver/SplitTransaction.java
regionserver/metrics/RegionServerMetrics.java
Author: stack
Date: Sat Apr 2 01:26:00 2011
New Revision: 1087953
URL: http://svn.apache.org/viewvc?rev=1087953&view=rev
Log:
HBASE-3647 Distinguish read and write request count in region -- reversed the patch because missing a file
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerInfo.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerInfo.java?rev=1087953&r1=1087952&r2=1087953&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerInfo.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerInfo.java Sat Apr 2 01:26:00 2011
@@ -27,6 +27,7 @@ import java.util.Comparator;
import java.util.Set;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.io.VersionedWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
@@ -43,7 +44,10 @@ import org.apache.hadoop.io.WritableComp
* by. In subsequent communications, the regionserver will pass a HServerInfo
* with the master-supplied address.
*/
-public class HServerInfo implements WritableComparable<HServerInfo> {
+public class HServerInfo extends VersionedWritable
+ implements WritableComparable<HServerInfo> {
+ private static final byte VERSION = 0;
+
/*
* This character is used as separator between server hostname and port and
* its startcode. Servername is formatted as
@@ -61,6 +65,11 @@ public class HServerInfo implements Writ
private String hostname;
private String cachedHostnamePort = null;
+ /** @return the object version number */
+ public byte getVersion() {
+ return VERSION;
+ }
+
public HServerInfo() {
this(new HServerAddress(), 0, HConstants.DEFAULT_REGIONSERVER_INFOPORT,
"default name");
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java?rev=1087953&r1=1087952&r2=1087953&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java Sat Apr 2 01:26:00 2011
@@ -28,13 +28,17 @@ import java.util.TreeMap;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Strings;
+import org.apache.hadoop.io.VersionedWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
/**
* This class encapsulates metrics for determining the load on a HRegionServer
*/
-public class HServerLoad implements WritableComparable<HServerLoad> {
+public class HServerLoad extends VersionedWritable
+ implements WritableComparable<HServerLoad> {
+ private static final byte VERSION = 0;
+
/** number of regions */
// could just use regionLoad.size() but master.RegionManager likes to play
// around with this value while passing HServerLoad objects around during
@@ -49,6 +53,11 @@ public class HServerLoad implements Writ
/** per-region load metrics */
private Map<byte[], RegionLoad> regionLoad = new TreeMap<byte[], RegionLoad>(Bytes.BYTES_COMPARATOR);
+ /** @return the object version number */
+ public byte getVersion() {
+ return VERSION;
+ }
+
/**
* Encapsulates per-region loading metrics.
*/
@@ -65,8 +74,10 @@ public class HServerLoad implements Writ
private int memstoreSizeMB;
/** the current total size of storefile indexes for the region, in MB */
private int storefileIndexSizeMB;
- /** the current total request made to region */
- private long requestsCount;
+ /** the current total read requests made to region */
+ private int readRequestsCount;
+ /** the current total write requests made to region */
+ private int writeRequestsCount;
/**
* Constructor, for Writable
@@ -82,18 +93,21 @@ public class HServerLoad implements Writ
* @param storefileSizeMB
* @param memstoreSizeMB
* @param storefileIndexSizeMB
- * @param requestsCount
+ * @param readRequestsCount
+ * @param writeRequestsCount
*/
public RegionLoad(final byte[] name, final int stores,
final int storefiles, final int storefileSizeMB,
- final int memstoreSizeMB, final int storefileIndexSizeMB,final long requestsCount) {
+ final int memstoreSizeMB, final int storefileIndexSizeMB,
+ final int readRequestsCount, final int writeRequestsCount) {
this.name = name;
this.stores = stores;
this.storefiles = storefiles;
this.storefileSizeMB = storefileSizeMB;
this.memstoreSizeMB = memstoreSizeMB;
this.storefileIndexSizeMB = storefileIndexSizeMB;
- this.requestsCount = requestsCount;
+ this.readRequestsCount = readRequestsCount;
+ this.writeRequestsCount = writeRequestsCount;
}
// Getters
@@ -146,12 +160,26 @@ public class HServerLoad implements Writ
public int getStorefileIndexSizeMB() {
return storefileIndexSizeMB;
}
-
+
/**
* @return the number of requests made to region
*/
public long getRequestsCount() {
- return requestsCount;
+ return readRequestsCount + writeRequestsCount;
+ }
+
+ /**
+ * @return the number of read requests made to region
+ */
+ public long getReadRequestsCount() {
+ return readRequestsCount;
+ }
+
+ /**
+ * @return the number of read requests made to region
+ */
+ public long getWriteRequestsCount() {
+ return writeRequestsCount;
}
// Setters
@@ -193,10 +221,17 @@ public class HServerLoad implements Writ
}
/**
- * @param requestsCount the number of requests to region
+ * @param requestsCount the number of read requests to region
+ */
+ public void setReadRequestsCount(int requestsCount) {
+ this.readRequestsCount = requestsCount;
+ }
+
+ /**
+ * @param requestsCount the number of write requests to region
*/
- public void setRequestsCount(long requestsCount) {
- this.requestsCount = requestsCount;
+ public void setWriteRequestsCount(int requestsCount) {
+ this.writeRequestsCount = requestsCount;
}
// Writable
@@ -209,7 +244,8 @@ public class HServerLoad implements Writ
this.storefileSizeMB = in.readInt();
this.memstoreSizeMB = in.readInt();
this.storefileIndexSizeMB = in.readInt();
- this.requestsCount = in.readLong();
+ this.readRequestsCount = in.readInt();
+ this.writeRequestsCount = in.readInt();
}
public void write(DataOutput out) throws IOException {
@@ -220,7 +256,8 @@ public class HServerLoad implements Writ
out.writeInt(storefileSizeMB);
out.writeInt(memstoreSizeMB);
out.writeInt(storefileIndexSizeMB);
- out.writeLong(requestsCount);
+ out.writeInt(readRequestsCount);
+ out.writeInt(writeRequestsCount);
}
/**
@@ -238,8 +275,10 @@ public class HServerLoad implements Writ
Integer.valueOf(this.memstoreSizeMB));
sb = Strings.appendKeyValue(sb, "storefileIndexSizeMB",
Integer.valueOf(this.storefileIndexSizeMB));
- sb = Strings.appendKeyValue(sb, "requestsCount",
- Long.valueOf(this.requestsCount));
+ sb = Strings.appendKeyValue(sb, "readRequestsCount",
+ Long.valueOf(this.readRequestsCount));
+ sb = Strings.appendKeyValue(sb, "writeRequestsCount",
+ Long.valueOf(this.writeRequestsCount));
return sb.toString();
}
}
@@ -483,9 +522,9 @@ public class HServerLoad implements Writ
public void addRegionInfo(final byte[] name, final int stores,
final int storefiles, final int storefileSizeMB,
final int memstoreSizeMB, final int storefileIndexSizeMB,
- final long requestsCount) {
+ final int readRequestsCount, final int writeRequestsCount) {
this.regionLoad.put(name, new HServerLoad.RegionLoad(name, stores, storefiles,
- storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, requestsCount));
+ storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, readRequestsCount, writeRequestsCount));
}
// Writable
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1087953&r1=1087952&r2=1087953&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Sat Apr 2 01:26:00 2011
@@ -177,7 +177,8 @@ public class HRegion implements HeapSize
final AtomicLong memstoreSize = new AtomicLong(0);
- final Counter requestsCount = new Counter();
+ final Counter readRequestsCount = new Counter();
+ final Counter writeRequestsCount = new Counter();
/**
* The directory for the table this region is part of.
@@ -462,7 +463,17 @@ public class HRegion implements HeapSize
/** @return requestsCount for this region */
public long getRequestsCount() {
- return this.requestsCount.get();
+ return this.readRequestsCount.get() + this.writeRequestsCount.get();
+ }
+
+ /** @return readRequestsCount for this region */
+ public long getReadRequestsCount() {
+ return this.readRequestsCount.get();
+ }
+
+ /** @return writeRequestsCount for this region */
+ public long getWriteRequestsCount() {
+ return this.writeRequestsCount.get();
}
/** @return true if region is closed */
@@ -1131,6 +1142,7 @@ public class HRegion implements HeapSize
// closest key is across all column families, since the data may be sparse
checkRow(row);
startRegionOperation();
+ this.readRequestsCount.increment();
try {
Store store = getStore(family);
KeyValue kv = new KeyValue(row, HConstants.LATEST_TIMESTAMP);
@@ -1167,6 +1179,7 @@ public class HRegion implements HeapSize
protected InternalScanner getScanner(Scan scan, List<KeyValueScanner> additionalScanners) throws IOException {
startRegionOperation();
+ this.readRequestsCount.increment();
try {
// Verify families are all valid
if(scan.hasFamilies()) {
@@ -1235,6 +1248,7 @@ public class HRegion implements HeapSize
checkResources();
Integer lid = null;
startRegionOperation();
+ this.writeRequestsCount.increment();
try {
byte [] row = delete.getRow();
// If we did not pass an existing row lock, obtain a new one
@@ -1388,6 +1402,7 @@ public class HRegion implements HeapSize
// will be extremely rare; we'll deal with it when it happens.
checkResources();
startRegionOperation();
+ this.writeRequestsCount.increment();
try {
// We obtain a per-row lock, so other clients will block while one client
// performs an update. The read lock is released by the client calling
@@ -1460,6 +1475,7 @@ public class HRegion implements HeapSize
long newSize;
startRegionOperation();
+ this.writeRequestsCount.increment();
try {
long addedSize = doMiniBatchPut(batchOp);
newSize = memstoreSize.addAndGet(addedSize);
@@ -1661,6 +1677,7 @@ public class HRegion implements HeapSize
}
startRegionOperation();
+ this.writeRequestsCount.increment();
try {
RowLock lock = isPut ? ((Put)w).getRowLock() : ((Delete)w).getRowLock();
Get get = new Get(row, lock);
@@ -2229,6 +2246,7 @@ public class HRegion implements HeapSize
*/
public Integer obtainRowLock(final byte [] row) throws IOException {
startRegionOperation();
+ this.writeRequestsCount.increment();
try {
return internalObtainRowLock(row, true);
} finally {
@@ -2237,21 +2255,6 @@ public class HRegion implements HeapSize
}
/**
- * Tries to obtain a row lock on the given row, but does not block if the
- * row lock is not available. If the lock is not available, returns false.
- * Otherwise behaves the same as the above method.
- * @see HRegion#obtainRowLock(byte[])
- */
- public Integer tryObtainRowLock(final byte[] row) throws IOException {
- startRegionOperation();
- try {
- return internalObtainRowLock(row, false);
- } finally {
- closeRegionOperation();
- }
- }
-
- /**
* Obtains or tries to obtain the given row lock.
* @param waitForLock if true, will block until the lock is available.
* Otherwise, just tries to obtain the lock and returns
@@ -2362,6 +2365,7 @@ public class HRegion implements HeapSize
public void bulkLoadHFile(String hfilePath, byte[] familyName)
throws IOException {
startRegionOperation();
+ this.writeRequestsCount.increment();
try {
Store store = getStore(familyName);
if (store == null) {
@@ -2467,6 +2471,7 @@ public class HRegion implements HeapSize
"or a lengthy garbage collection");
}
startRegionOperation();
+ readRequestsCount.increment();
try {
// This could be a new thread from the last time we called next().
@@ -2973,7 +2978,8 @@ public class HRegion implements HeapSize
listPaths(fs, newRegionDir);
}
HRegion dstRegion = HRegion.newHRegion(tableDir, log, fs, conf, newRegionInfo, null);
- dstRegion.requestsCount.set(a.requestsCount.get() + b.requestsCount.get());
+ dstRegion.readRequestsCount.set(a.readRequestsCount.get() + b.readRequestsCount.get());
+ dstRegion.writeRequestsCount.set(a.writeRequestsCount.get() + b.writeRequestsCount.get());
dstRegion.initialize();
dstRegion.compactStores();
if (LOG.isDebugEnabled()) {
@@ -3219,6 +3225,7 @@ public class HRegion implements HeapSize
// Lock row
startRegionOperation();
+ this.writeRequestsCount.increment();
try {
Integer lid = getLock(lockid, row, true);
this.updatesLock.readLock().lock();
@@ -3309,6 +3316,7 @@ public class HRegion implements HeapSize
// Lock row
long result = amount;
startRegionOperation();
+ this.writeRequestsCount.increment();
try {
Integer lid = obtainRowLock(row);
this.updatesLock.readLock().lock();
@@ -3383,7 +3391,7 @@ public class HRegion implements HeapSize
public static final long FIXED_OVERHEAD = ClassSize.align(
(4 * Bytes.SIZEOF_LONG) + ClassSize.ARRAY +
- ClassSize.align(25 * ClassSize.REFERENCE) + ClassSize.OBJECT +
+ ClassSize.align(26 * ClassSize.REFERENCE) + ClassSize.OBJECT +
ClassSize.align(Bytes.SIZEOF_INT));
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
@@ -3563,7 +3571,6 @@ public class HRegion implements HeapSize
} finally {
scanner.close();
}
- // System.out.println(region.getClosestRowBefore(Bytes.toBytes("GeneratedCSVContent2,E3652782193BC8D66A0BA1629D0FAAAB,9993372036854775807")));
}
} finally {
region.close();
@@ -3647,7 +3654,6 @@ public class HRegion implements HeapSize
throw new NotServingRegionException(regionInfo.getRegionNameAsString() +
" is closed");
}
- this.requestsCount.increment();
}
/**
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1087953&r1=1087952&r2=1087953&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Sat Apr 2 01:26:00 2011
@@ -59,7 +59,6 @@ import org.apache.hadoop.hbase.ClockOutO
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
import org.apache.hadoop.hbase.HMsg;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
@@ -74,6 +73,7 @@ import org.apache.hadoop.hbase.Stoppable
import org.apache.hadoop.hbase.UnknownRowLockException;
import org.apache.hadoop.hbase.UnknownScannerException;
import org.apache.hadoop.hbase.YouAreDeadException;
+import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.catalog.MetaEditor;
import org.apache.hadoop.hbase.catalog.RootLocationEditor;
@@ -897,7 +897,6 @@ public class HRegionServer implements HR
int storefileSizeMB = 0;
int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024);
int storefileIndexSizeMB = 0;
- long requestsCount = r.requestsCount.get();
synchronized (r.stores) {
stores += r.stores.size();
for (Store store : r.stores.values()) {
@@ -907,7 +906,8 @@ public class HRegionServer implements HR
}
}
return new HServerLoad.RegionLoad(name,stores, storefiles,
- storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB, requestsCount);
+ storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB,
+ (int) r.readRequestsCount.get(), (int) r.writeRequestsCount.get());
}
/**
@@ -1142,12 +1142,14 @@ public class HRegionServer implements HR
int stores = 0;
int storefiles = 0;
long memstoreSize = 0;
- long requestsCount = 0;
+ int readRequestsCount = 0;
+ int writeRequestsCount = 0;
long storefileIndexSize = 0;
for (Map.Entry<String, HRegion> e : this.onlineRegions.entrySet()) {
HRegion r = e.getValue();
memstoreSize += r.memstoreSize.get();
- requestsCount += r.requestsCount.get();
+ readRequestsCount += r.readRequestsCount.get();
+ writeRequestsCount += r.writeRequestsCount.get();
synchronized (r.stores) {
stores += r.stores.size();
for (Map.Entry<byte[], Store> ee : r.stores.entrySet()) {
@@ -1160,7 +1162,8 @@ public class HRegionServer implements HR
this.metrics.stores.set(stores);
this.metrics.storefiles.set(storefiles);
this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024)));
- this.metrics.requestsCount.set(requestsCount);
+ this.metrics.readRequestsCount.set(readRequestsCount);
+ this.metrics.writeRequestsCount.set(writeRequestsCount);
this.metrics.storefileIndexSizeMB
.set((int) (storefileIndexSize / (1024 * 1024)));
this.metrics.compactionQueueSize.set(compactSplitThread
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java?rev=1087953&r1=1087952&r2=1087953&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java Sat Apr 2 01:26:00 2011
@@ -543,7 +543,8 @@ public class SplitTransaction {
HRegion r = HRegion.newHRegion(this.parent.getTableDir(),
this.parent.getLog(), fs, this.parent.getConf(),
hri, rsServices);
- r.requestsCount.set(this.parent.getRequestsCount() / 2);
+ r.readRequestsCount.set(this.parent.getReadRequestsCount() / 2);
+ r.writeRequestsCount.set(this.parent.getWriteRequestsCount() / 2);
HRegion.moveInitialFilesIntoPlace(fs, regionDir, r.getRegionDir());
return r;
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1087953&r1=1087952&r2=1087953&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java Sat Apr 2 01:26:00 2011
@@ -128,9 +128,14 @@ public class RegionServerMetrics impleme
public final MetricsIntValue storefiles = new MetricsIntValue("storefiles", registry);
/**
- * Count of requests
+ * Count of read requests
*/
- public final MetricsLongValue requestsCount = new MetricsLongValue("requestsCount", registry);
+ public final MetricsLongValue readRequestsCount = new MetricsLongValue("readRequestsCount", registry);
+
+ /**
+ * Count of write requests
+ */
+ public final MetricsLongValue writeRequestsCount = new MetricsLongValue("writeRequestsCount", registry);
/**
* Sum of all the storefile index sizes in this regionserver in MB
@@ -248,7 +253,8 @@ public class RegionServerMetrics impleme
this.storefiles.pushMetric(this.metricsRecord);
this.storefileIndexSizeMB.pushMetric(this.metricsRecord);
this.memstoreSizeMB.pushMetric(this.metricsRecord);
- this.requestsCount.pushMetric(this.metricsRecord);
+ this.readRequestsCount.pushMetric(this.metricsRecord);
+ this.writeRequestsCount.pushMetric(this.metricsRecord);
this.regions.pushMetric(this.metricsRecord);
this.requests.pushMetric(this.metricsRecord);
this.compactionQueueSize.pushMetric(this.metricsRecord);
@@ -351,8 +357,10 @@ public class RegionServerMetrics impleme
Integer.valueOf(this.storefileIndexSizeMB.get()));
sb = Strings.appendKeyValue(sb, "memstoreSize",
Integer.valueOf(this.memstoreSizeMB.get()));
- sb = Strings.appendKeyValue(sb, "requestsCount",
- Long.valueOf(this.requestsCount.get()));
+ sb = Strings.appendKeyValue(sb, "readRequestsCount",
+ Long.valueOf(this.readRequestsCount.get()));
+ sb = Strings.appendKeyValue(sb, "writeRequestsCount",
+ Long.valueOf(this.writeRequestsCount.get()));
sb = Strings.appendKeyValue(sb, "compactionQueueSize",
Integer.valueOf(this.compactionQueueSize.get()));
sb = Strings.appendKeyValue(sb, "flushQueueSize",