You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by en...@apache.org on 2016/09/30 18:31:07 UTC
[4/5] hbase git commit: HBASE-16678 MapReduce jobs do not update
counters from ScanMetrics
HBASE-16678 MapReduce jobs do not update counters from ScanMetrics
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2e381ee2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2e381ee2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2e381ee2
Branch: refs/heads/branch-1.2
Commit: 2e381ee2d29b69fa3b47e02176a53c029f7ddcb0
Parents: cf374af
Author: Enis Soztutar <en...@apache.org>
Authored: Thu Sep 29 15:34:03 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Thu Sep 29 19:00:27 2016 -0700
----------------------------------------------------------------------
.../hbase/mapreduce/TableRecordReaderImpl.java | 8 +++++---
.../hbase/mapreduce/TestTableMapReduce.java | 19 +++++++++++++++++++
2 files changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e381ee2/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 23f59e6..dcc4943 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Iterate over an HBase table data, return (ImmutableBytesWritable, Result)
* pairs.
@@ -53,8 +55,8 @@ public class TableRecordReaderImpl {
private static final Log LOG = LogFactory.getLog(TableRecordReaderImpl.class);
// HBASE_COUNTER_GROUP_NAME is the name of mapreduce counter group for HBase
- private static final String HBASE_COUNTER_GROUP_NAME =
- "HBase Counters";
+ @VisibleForTesting
+ static final String HBASE_COUNTER_GROUP_NAME = "HBase Counters";
private ResultScanner scanner = null;
private Scan scan = null;
private Scan currentScan = null;
@@ -269,7 +271,7 @@ public class TableRecordReaderImpl {
* @throws IOException
*/
private void updateCounters() throws IOException {
- ScanMetrics scanMetrics = this.scan.getScanMetrics();
+ ScanMetrics scanMetrics = currentScan.getScanMetrics();
if (scanMetrics == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e381ee2/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
index fde1cd9..f5e8b12 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.mapreduce;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -38,6 +39,8 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.junit.experimental.categories.Category;
@@ -114,6 +117,8 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
// verify map-reduce results
verify(table.getName());
+
+ verifyJobCountersAreEmitted(job);
} catch (InterruptedException e) {
throw new IOException(e);
} catch (ClassNotFoundException e) {
@@ -126,4 +131,18 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
}
}
}
+
+ /**
+ * Verify scan counters are emitted from the job
+ * @param job
+ * @throws IOException
+ */
+ private void verifyJobCountersAreEmitted(Job job) throws IOException {
+ Counters counters = job.getCounters();
+ Counter counter
+ = counters.findCounter(TableRecordReaderImpl.HBASE_COUNTER_GROUP_NAME, "RPC_CALLS");
+ assertNotNull("Unable to find Job counter for HBase scan metrics, RPC_CALLS", counter);
+ assertTrue("Counter value for RPC_CALLS should be larger than 0", counter.getValue() > 0);
+ }
+
}