You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 04:08:04 UTC
svn commit: r1181433 - in
/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase:
io/hfile/HFile.java regionserver/metrics/RegionServerMetrics.java
regionserver/wal/HLog.java
Author: nspiegelberg
Date: Tue Oct 11 02:08:04 2011
New Revision: 1181433
URL: http://svn.apache.org/viewvc?rev=1181433&view=rev
Log:
Add writeSize metric
Summary:
Add writeSize variable to JMX so we can track average write sizes and see how
closely they correlate with put latency
Test Plan:
tested on cluster
DiffCamp Revision: 176576
Reviewed By: kannan
CC: kannan
Revert Plan:
OK
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1181433&r1=1181432&r2=1181433&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Tue Oct 11 02:08:04 2011
@@ -170,13 +170,13 @@ public class HFile {
DEFAULT_COMPRESSION_ALGORITHM.getName();
// For measuring latency of "typical" reads and writes
- private static volatile long readOps;
+ private static volatile int readOps;
private static volatile long readTime;
- private static volatile long writeOps;
+ private static volatile int writeOps;
private static volatile long writeTime;
- public static final long getReadOps() {
- long ret = readOps;
+ public static final int getReadOps() {
+ int ret = readOps;
readOps = 0;
return ret;
}
@@ -187,8 +187,8 @@ public class HFile {
return ret;
}
- public static final long getWriteOps() {
- long ret = writeOps;
+ public static final int getWriteOps() {
+ int ret = writeOps;
writeOps = 0;
return ret;
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1181433&r1=1181432&r2=1181433&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java Tue Oct 11 02:08:04 2011
@@ -137,6 +137,12 @@ public class RegionServerMetrics impleme
new MetricsTimeVaryingRate("fsWriteLatency", registry);
/**
+ * size (in bytes) of data in HLog append calls
+ */
+ public final MetricsTimeVaryingRate fsWriteSize =
+ new MetricsTimeVaryingRate("fsWriteSize", registry);
+
+ /**
* filesystem sync latency
*/
public final MetricsTimeVaryingRate fsSyncLatency =
@@ -222,7 +228,6 @@ public class RegionServerMetrics impleme
this.blockCacheCount.pushMetric(this.metricsRecord);
this.blockCacheHitRatio.pushMetric(this.metricsRecord);
- // Mix in HFile and HLog metrics
// Be careful. Here is code for MTVR from up in hadoop:
// public synchronized void inc(final int numOps, final long time) {
// currentData.numOperations += numOps;
@@ -231,15 +236,23 @@ public class RegionServerMetrics impleme
// minMax.update(timePerOps);
// }
// Means you can't pass a numOps of zero or get a ArithmeticException / by zero.
- int ops = (int)HFile.getReadOps();
- if (ops != 0) this.fsReadLatency.inc(ops, HFile.getReadTime());
- ops = (int)HFile.getWriteOps();
- if (ops != 0) this.fsWriteLatency.inc(ops, HFile.getWriteTime());
- // mix in HLog metrics
- ops = (int)HLog.getWriteOps();
- if (ops != 0) this.fsWriteLatency.inc(ops, HLog.getWriteTime());
- ops = (int)HLog.getSyncOps();
+ // HLog metrics
+ int ops = HLog.getWriteOps();
+ if (ops != 0) {
+ this.fsWriteLatency.inc(ops, HLog.getWriteTime());
+ this.fsWriteSize.inc(ops, HLog.getWriteSize());
+ }
+ ops = HLog.getSyncOps();
if (ops != 0) this.fsSyncLatency.inc(ops, HLog.getSyncTime());
+ // HFile metrics
+ ops = HFile.getReadOps();
+ if (ops != 0) this.fsReadLatency.inc(ops, HFile.getReadTime());
+ /* NOTE: removed HFile write latency. 2 reasons:
+ * 1) Mixing HLog latencies are far higher priority since they're
+ * on-demand and HFile is used in background (compact/flush)
+ * 2) HFile metrics are being handled at a higher level
+ * by compaction & flush metrics.
+ */
// push the result
this.fsReadLatency.pushMetric(this.metricsRecord);
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1181433&r1=1181432&r2=1181433&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Tue Oct 11 02:08:04 2011
@@ -249,17 +249,18 @@ public class HLog implements Syncable {
}
// For measuring latency of writes
- private static volatile long writeOps;
+ private static volatile int writeOps;
private static volatile long writeTime;
+ private static volatile long writeSize;
// For measuring latency of syncs
- private static volatile long syncOps;
+ private static volatile int syncOps;
private static volatile long syncTime;
public static volatile long lastSplitTime = 0;
public static volatile long lastSplitSize = 0;
- public static long getWriteOps() {
- long ret = writeOps;
+ public static int getWriteOps() {
+ int ret = writeOps;
writeOps = 0;
return ret;
}
@@ -270,8 +271,14 @@ public class HLog implements Syncable {
return ret;
}
- public static long getSyncOps() {
- long ret = syncOps;
+ public static long getWriteSize() {
+ long ret = writeSize;
+ writeSize = 0;
+ return ret;
+ }
+
+ public static int getSyncOps() {
+ int ret = syncOps;
syncOps = 0;
return ret;
}
@@ -1056,11 +1063,12 @@ public class HLog implements Syncable {
long took = System.currentTimeMillis() - now;
writeTime += took;
writeOps++;
+ long len = 0;
+ for(KeyValue kv : logEdit.getKeyValues()) {
+ len += kv.getLength();
+ }
+ writeSize += len;
if (took > 1000) {
- long len = 0;
- for(KeyValue kv : logEdit.getKeyValues()) {
- len += kv.getLength();
- }
LOG.warn(String.format(
"%s took %d ms appending an edit to hlog; editcount=%d, len~=%s",
Thread.currentThread().getName(), took, this.numEntries.get(),
@@ -1134,6 +1142,11 @@ public class HLog implements Syncable {
this.writer.append(new Entry(key, edit));
writeTime += System.currentTimeMillis() - now;
writeOps++;
+ long len = 0;
+ for(KeyValue kv : edit.getKeyValues()) {
+ len += kv.getLength();
+ }
+ writeSize += len;
this.numEntries.incrementAndGet();
Long seq = this.lastSeqWritten.get(regionName);
if (seq != null && logSeqId >= seq.longValue()) {