svn commit: r1422051 - in /hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver: HRegionServer.java metrics/RegionServerMetrics.java

[li...@apache.org]

Author: liyin
Date: Fri Dec 14 19:22:03 2012
New Revision: 1422051

URL: http://svn.apache.org/viewvc?rev=1422051&view=rev
Log:
[HBASE-7348] Add some statistics from DFSClient to RegionServerMetrics

Author: liyintang

Summary: DFSClient actually collected a number of useful statistics such as bytesLocalRead, bytesLocalRackRead and so on. So this diff is going to merge these metrics into the RegionServerMetrics.

Test Plan: Going to test it on the titan shadow cluster in order to investigate the non-local read.

Reviewers: hkuang, kannan, adela, sdong

Reviewed By: sdong

CC: hbase-eng@

Differential Revision: https://phabricator.fb.com/D657336

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1422051&r1=1422050&r2=1422051&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Fri Dec 14 19:22:03 2012
@@ -71,6 +71,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystem.Statistics;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.Chore;
@@ -1463,6 +1464,35 @@ public class HRegionServer implements HR
       int percent = (int) (ratio * 100);
       this.metrics.blockCacheHitRatio.set(percent);
     }
+    
+    long bytesRead = 0;
+    long bytesLocalRead = 0;
+    long bytesRackLocalRead = 0;
+    long bytesWritten = 0;
+    long filesCreated = 0;
+    long filesRead = 0;
+    long cntWriteException = 0;
+    long cntReadException = 0;
+    
+    for (Statistics fsStatistic : FileSystem.getAllStatistics()) {
+      bytesRead += fsStatistic.getBytesRead();
+      bytesLocalRead += fsStatistic.getLocalBytesRead();
+      bytesRackLocalRead += fsStatistic.getLocalBytesRead();
+      bytesWritten += fsStatistic.getBytesWritten();
+      filesCreated += fsStatistic.getFilesCreated();
+      filesRead += fsStatistic.getFilesRead();
+      cntWriteException += fsStatistic.getCntWriteException();
+      cntReadException += fsStatistic.getCntReadException();
+    }
+    
+    this.metrics.bytesRead.set(bytesRead);
+    this.metrics.bytesLocalRead.set(bytesLocalRead);
+    this.metrics.bytesRackLocalRead.set(bytesRackLocalRead);
+    this.metrics.bytesWritten.set(bytesWritten);
+    this.metrics.filesCreated.set(filesCreated);
+    this.metrics.filesRead.set(filesRead);
+    this.metrics.cntWriteException.set(cntWriteException);
+    this.metrics.cntReadException.set(cntReadException);
   }
 
   /**

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java?rev=1422051&r1=1422050&r2=1422051&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java Fri Dec 14 19:22:03 2012
@@ -228,6 +228,26 @@ public class RegionServerMetrics impleme
 
   protected final PersistentMetricsTimeVaryingRate flushSize =
     new PersistentMetricsTimeVaryingRate("flushSize", registry);
+  
+  /**
+   * DFSClient metrics
+   */
+  public final MetricsLongValue bytesRead = 
+      new MetricsLongValue("dfsBytesRead", registry);
+  public final MetricsLongValue bytesLocalRead = 
+      new MetricsLongValue("dfsBytesLocalRead", registry);
+  public final MetricsLongValue bytesRackLocalRead = 
+      new MetricsLongValue("dfsBytesRackLocalRead", registry);
+  public final MetricsLongValue bytesWritten = 
+      new MetricsLongValue("dfsBytesWritten", registry);
+  public final MetricsLongValue filesCreated = 
+      new MetricsLongValue("dfsFilesCreated", registry);
+  public final MetricsLongValue filesRead = 
+      new MetricsLongValue("dfsFilesRead", registry);
+  public final MetricsLongValue cntWriteException = 
+      new MetricsLongValue("dfsCntWriteException", registry);
+  public final MetricsLongValue cntReadException = 
+      new MetricsLongValue("dfsCntReadException", registry);
 
   public RegionServerMetrics() {
     MetricsContext context = MetricsUtil.getContext("hbase");
@@ -347,6 +367,15 @@ public class RegionServerMetrics impleme
       this.compactionSize.pushMetric(this.metricsRecord);
       this.flushTime.pushMetric(this.metricsRecord);
       this.flushSize.pushMetric(this.metricsRecord);
+      
+      this.bytesRead.pushMetric(this.metricsRecord);
+      this.bytesLocalRead.pushMetric(this.metricsRecord);
+      this.bytesRackLocalRead.pushMetric(this.metricsRecord);
+      this.bytesWritten.pushMetric(this.metricsRecord);
+      this.filesCreated.pushMetric(this.metricsRecord);
+      this.filesRead.pushMetric(this.metricsRecord);
+      this.cntWriteException.pushMetric(this.metricsRecord);
+      this.cntReadException.pushMetric(this.metricsRecord);
     }
     this.metricsRecord.update();
   }
@@ -479,6 +508,24 @@ public class RegionServerMetrics impleme
         Long.valueOf(this.blockCacheEvictedMemoryCount.get()));
     sb = Strings.appendKeyValue(sb, this.blockCacheHitRatio.getName(),
         Long.valueOf(this.blockCacheHitRatio.get()));
+    
+    sb = Strings.appendKeyValue(sb, this.bytesRead.getName(),
+        Long.valueOf(this.bytesRead.get()));
+    sb = Strings.appendKeyValue(sb, this.bytesLocalRead.getName(),
+        Long.valueOf(this.bytesLocalRead.get()));
+    sb = Strings.appendKeyValue(sb, this.bytesRackLocalRead.getName(),
+        Long.valueOf(this.bytesRackLocalRead.get()));
+    sb = Strings.appendKeyValue(sb, this.bytesWritten.getName(),
+        Long.valueOf(this.bytesWritten.get()));
+    sb = Strings.appendKeyValue(sb, this.filesCreated.getName(),
+        Long.valueOf(this.filesCreated.get()));
+    sb = Strings.appendKeyValue(sb, this.filesRead.getName(),
+        Long.valueOf(this.filesRead.get()));
+    sb = Strings.appendKeyValue(sb, this.cntWriteException.getName(),
+        Long.valueOf(this.cntWriteException.get()));
+    sb = Strings.appendKeyValue(sb, this.cntReadException.getName(),
+        Long.valueOf(this.cntReadException.get()));
+    
     return sb.toString();
   }
 }