You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by en...@apache.org on 2016/09/30 18:31:04 UTC
[1/5] hbase git commit: HBASE-16678 MapReduce jobs do not update
counters from ScanMetrics
Repository: hbase
Updated Branches:
refs/heads/branch-1 bf5a7aba5 -> 911f9b9eb
refs/heads/branch-1.1 06c3dec2d -> 88bf5b3b1
refs/heads/branch-1.2 cf374af10 -> 2e381ee2d
refs/heads/branch-1.3 f77f1530d -> 9f364084a
refs/heads/master 3757da643 -> c3c82f355
HBASE-16678 MapReduce jobs do not update counters from ScanMetrics
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c3c82f35
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c3c82f35
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c3c82f35
Branch: refs/heads/master
Commit: c3c82f3558b80b23c8f997d5bacfa78de384208a
Parents: 3757da6
Author: Enis Soztutar <en...@apache.org>
Authored: Thu Sep 29 15:34:03 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Thu Sep 29 15:34:03 2016 -0700
----------------------------------------------------------------------
.../hbase/mapreduce/TableRecordReaderImpl.java | 8 +++++---
.../hbase/mapreduce/TestTableMapReduce.java | 18 ++++++++++++++++++
2 files changed, 23 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c3c82f35/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 23f59e6..dcc4943 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Iterate over an HBase table data, return (ImmutableBytesWritable, Result)
* pairs.
@@ -53,8 +55,8 @@ public class TableRecordReaderImpl {
private static final Log LOG = LogFactory.getLog(TableRecordReaderImpl.class);
// HBASE_COUNTER_GROUP_NAME is the name of mapreduce counter group for HBase
- private static final String HBASE_COUNTER_GROUP_NAME =
- "HBase Counters";
+ @VisibleForTesting
+ static final String HBASE_COUNTER_GROUP_NAME = "HBase Counters";
private ResultScanner scanner = null;
private Scan scan = null;
private Scan currentScan = null;
@@ -269,7 +271,7 @@ public class TableRecordReaderImpl {
* @throws IOException
*/
private void updateCounters() throws IOException {
- ScanMetrics scanMetrics = this.scan.getScanMetrics();
+ ScanMetrics scanMetrics = currentScan.getScanMetrics();
if (scanMetrics == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c3c82f35/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
index 690e776..d702e0d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
@@ -18,6 +18,7 @@
*/
package org.apache.hadoop.hbase.mapreduce;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -42,6 +43,8 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.junit.Test;
@@ -121,6 +124,8 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
// verify map-reduce results
verify(table.getName());
+
+ verifyJobCountersAreEmitted(job);
} catch (InterruptedException e) {
throw new IOException(e);
} catch (ClassNotFoundException e) {
@@ -134,6 +139,19 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
}
}
+ /**
+ * Verify scan counters are emitted from the job
+ * @param job
+ * @throws IOException
+ */
+ private void verifyJobCountersAreEmitted(Job job) throws IOException {
+ Counters counters = job.getCounters();
+ Counter counter
+ = counters.findCounter(TableRecordReaderImpl.HBASE_COUNTER_GROUP_NAME, "RPC_CALLS");
+ assertNotNull("Unable to find Job counter for HBase scan metrics, RPC_CALLS", counter);
+ assertTrue("Counter value for RPC_CALLS should be larger than 0", counter.getValue() > 0);
+ }
+
@Test(expected = TableNotEnabledException.class)
public void testWritingToDisabledTable() throws IOException {
[3/5] hbase git commit: HBASE-16678 MapReduce jobs do not update
counters from ScanMetrics
Posted by en...@apache.org.
HBASE-16678 MapReduce jobs do not update counters from ScanMetrics
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9f364084
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9f364084
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9f364084
Branch: refs/heads/branch-1.3
Commit: 9f364084a2a69800b4a4658cb80e3315b881ad8e
Parents: f77f153
Author: Enis Soztutar <en...@apache.org>
Authored: Thu Sep 29 15:34:03 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Thu Sep 29 18:56:42 2016 -0700
----------------------------------------------------------------------
.../hbase/mapreduce/TableRecordReaderImpl.java | 8 +++++---
.../hbase/mapreduce/TestTableMapReduce.java | 19 +++++++++++++++++++
2 files changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9f364084/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 23f59e6..dcc4943 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Iterate over an HBase table data, return (ImmutableBytesWritable, Result)
* pairs.
@@ -53,8 +55,8 @@ public class TableRecordReaderImpl {
private static final Log LOG = LogFactory.getLog(TableRecordReaderImpl.class);
// HBASE_COUNTER_GROUP_NAME is the name of mapreduce counter group for HBase
- private static final String HBASE_COUNTER_GROUP_NAME =
- "HBase Counters";
+ @VisibleForTesting
+ static final String HBASE_COUNTER_GROUP_NAME = "HBase Counters";
private ResultScanner scanner = null;
private Scan scan = null;
private Scan currentScan = null;
@@ -269,7 +271,7 @@ public class TableRecordReaderImpl {
* @throws IOException
*/
private void updateCounters() throws IOException {
- ScanMetrics scanMetrics = this.scan.getScanMetrics();
+ ScanMetrics scanMetrics = currentScan.getScanMetrics();
if (scanMetrics == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/9f364084/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
index fde1cd9..f5e8b12 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.mapreduce;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -38,6 +39,8 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.junit.experimental.categories.Category;
@@ -114,6 +117,8 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
// verify map-reduce results
verify(table.getName());
+
+ verifyJobCountersAreEmitted(job);
} catch (InterruptedException e) {
throw new IOException(e);
} catch (ClassNotFoundException e) {
@@ -126,4 +131,18 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
}
}
}
+
+ /**
+ * Verify scan counters are emitted from the job
+ * @param job
+ * @throws IOException
+ */
+ private void verifyJobCountersAreEmitted(Job job) throws IOException {
+ Counters counters = job.getCounters();
+ Counter counter
+ = counters.findCounter(TableRecordReaderImpl.HBASE_COUNTER_GROUP_NAME, "RPC_CALLS");
+ assertNotNull("Unable to find Job counter for HBase scan metrics, RPC_CALLS", counter);
+ assertTrue("Counter value for RPC_CALLS should be larger than 0", counter.getValue() > 0);
+ }
+
}
[5/5] hbase git commit: HBASE-16678 MapReduce jobs do not update
counters from ScanMetrics
Posted by en...@apache.org.
HBASE-16678 MapReduce jobs do not update counters from ScanMetrics
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/88bf5b3b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/88bf5b3b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/88bf5b3b
Branch: refs/heads/branch-1.1
Commit: 88bf5b3b1d2ffd11a30c06905c1e51e9d6b2a65b
Parents: 06c3dec
Author: Enis Soztutar <en...@apache.org>
Authored: Thu Sep 29 15:34:03 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Thu Sep 29 19:03:45 2016 -0700
----------------------------------------------------------------------
.../hbase/mapreduce/TableRecordReaderImpl.java | 8 +++++---
.../hbase/mapreduce/TestTableMapReduce.java | 19 +++++++++++++++++++
2 files changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/88bf5b3b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 4349537..06ccc72 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Iterate over an HBase table data, return (ImmutableBytesWritable, Result)
* pairs.
@@ -53,8 +55,8 @@ public class TableRecordReaderImpl {
static final Log LOG = LogFactory.getLog(TableRecordReaderImpl.class);
// HBASE_COUNTER_GROUP_NAME is the name of mapreduce counter group for HBase
- private static final String HBASE_COUNTER_GROUP_NAME =
- "HBase Counters";
+ @VisibleForTesting
+ static final String HBASE_COUNTER_GROUP_NAME = "HBase Counters";
private ResultScanner scanner = null;
private Scan scan = null;
private Scan currentScan = null;
@@ -268,7 +270,7 @@ public class TableRecordReaderImpl {
* @throws IOException
*/
private void updateCounters() throws IOException {
- ScanMetrics scanMetrics = this.scan.getScanMetrics();
+ ScanMetrics scanMetrics = currentScan.getScanMetrics();
if (scanMetrics == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/88bf5b3b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
index fde1cd9..f5e8b12 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.mapreduce;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -38,6 +39,8 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.junit.experimental.categories.Category;
@@ -114,6 +117,8 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
// verify map-reduce results
verify(table.getName());
+
+ verifyJobCountersAreEmitted(job);
} catch (InterruptedException e) {
throw new IOException(e);
} catch (ClassNotFoundException e) {
@@ -126,4 +131,18 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
}
}
}
+
+ /**
+ * Verify scan counters are emitted from the job
+ * @param job
+ * @throws IOException
+ */
+ private void verifyJobCountersAreEmitted(Job job) throws IOException {
+ Counters counters = job.getCounters();
+ Counter counter
+ = counters.findCounter(TableRecordReaderImpl.HBASE_COUNTER_GROUP_NAME, "RPC_CALLS");
+ assertNotNull("Unable to find Job counter for HBase scan metrics, RPC_CALLS", counter);
+ assertTrue("Counter value for RPC_CALLS should be larger than 0", counter.getValue() > 0);
+ }
+
}
[4/5] hbase git commit: HBASE-16678 MapReduce jobs do not update
counters from ScanMetrics
Posted by en...@apache.org.
HBASE-16678 MapReduce jobs do not update counters from ScanMetrics
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2e381ee2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2e381ee2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2e381ee2
Branch: refs/heads/branch-1.2
Commit: 2e381ee2d29b69fa3b47e02176a53c029f7ddcb0
Parents: cf374af
Author: Enis Soztutar <en...@apache.org>
Authored: Thu Sep 29 15:34:03 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Thu Sep 29 19:00:27 2016 -0700
----------------------------------------------------------------------
.../hbase/mapreduce/TableRecordReaderImpl.java | 8 +++++---
.../hbase/mapreduce/TestTableMapReduce.java | 19 +++++++++++++++++++
2 files changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e381ee2/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 23f59e6..dcc4943 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Iterate over an HBase table data, return (ImmutableBytesWritable, Result)
* pairs.
@@ -53,8 +55,8 @@ public class TableRecordReaderImpl {
private static final Log LOG = LogFactory.getLog(TableRecordReaderImpl.class);
// HBASE_COUNTER_GROUP_NAME is the name of mapreduce counter group for HBase
- private static final String HBASE_COUNTER_GROUP_NAME =
- "HBase Counters";
+ @VisibleForTesting
+ static final String HBASE_COUNTER_GROUP_NAME = "HBase Counters";
private ResultScanner scanner = null;
private Scan scan = null;
private Scan currentScan = null;
@@ -269,7 +271,7 @@ public class TableRecordReaderImpl {
* @throws IOException
*/
private void updateCounters() throws IOException {
- ScanMetrics scanMetrics = this.scan.getScanMetrics();
+ ScanMetrics scanMetrics = currentScan.getScanMetrics();
if (scanMetrics == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2e381ee2/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
index fde1cd9..f5e8b12 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.mapreduce;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -38,6 +39,8 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.junit.experimental.categories.Category;
@@ -114,6 +117,8 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
// verify map-reduce results
verify(table.getName());
+
+ verifyJobCountersAreEmitted(job);
} catch (InterruptedException e) {
throw new IOException(e);
} catch (ClassNotFoundException e) {
@@ -126,4 +131,18 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
}
}
}
+
+ /**
+ * Verify scan counters are emitted from the job
+ * @param job
+ * @throws IOException
+ */
+ private void verifyJobCountersAreEmitted(Job job) throws IOException {
+ Counters counters = job.getCounters();
+ Counter counter
+ = counters.findCounter(TableRecordReaderImpl.HBASE_COUNTER_GROUP_NAME, "RPC_CALLS");
+ assertNotNull("Unable to find Job counter for HBase scan metrics, RPC_CALLS", counter);
+ assertTrue("Counter value for RPC_CALLS should be larger than 0", counter.getValue() > 0);
+ }
+
}
[2/5] hbase git commit: HBASE-16678 MapReduce jobs do not update
counters from ScanMetrics
Posted by en...@apache.org.
HBASE-16678 MapReduce jobs do not update counters from ScanMetrics
Conflicts:
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/911f9b9e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/911f9b9e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/911f9b9e
Branch: refs/heads/branch-1
Commit: 911f9b9eb7dc59abe3c01aa75ede88ccdede7a08
Parents: bf5a7ab
Author: Enis Soztutar <en...@apache.org>
Authored: Thu Sep 29 15:34:03 2016 -0700
Committer: Enis Soztutar <en...@apache.org>
Committed: Thu Sep 29 17:14:55 2016 -0700
----------------------------------------------------------------------
.../hbase/mapreduce/TableRecordReaderImpl.java | 8 +++++---
.../hbase/mapreduce/TestTableMapReduce.java | 18 ++++++++++++++++++
2 files changed, 23 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/911f9b9e/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
index 23f59e6..dcc4943 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableRecordReaderImpl.java
@@ -40,6 +40,8 @@ import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Iterate over an HBase table data, return (ImmutableBytesWritable, Result)
* pairs.
@@ -53,8 +55,8 @@ public class TableRecordReaderImpl {
private static final Log LOG = LogFactory.getLog(TableRecordReaderImpl.class);
// HBASE_COUNTER_GROUP_NAME is the name of mapreduce counter group for HBase
- private static final String HBASE_COUNTER_GROUP_NAME =
- "HBase Counters";
+ @VisibleForTesting
+ static final String HBASE_COUNTER_GROUP_NAME = "HBase Counters";
private ResultScanner scanner = null;
private Scan scan = null;
private Scan currentScan = null;
@@ -269,7 +271,7 @@ public class TableRecordReaderImpl {
* @throws IOException
*/
private void updateCounters() throws IOException {
- ScanMetrics scanMetrics = this.scan.getScanMetrics();
+ ScanMetrics scanMetrics = currentScan.getScanMetrics();
if (scanMetrics == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/911f9b9e/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
index 299e226..19e0ab0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduce.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.mapreduce;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -42,6 +43,8 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.junit.Test;
@@ -119,6 +122,8 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
// verify map-reduce results
verify(table.getName());
+
+ verifyJobCountersAreEmitted(job);
} catch (InterruptedException e) {
throw new IOException(e);
} catch (ClassNotFoundException e) {
@@ -132,6 +137,19 @@ public class TestTableMapReduce extends TestTableMapReduceBase {
}
}
+ /**
+ * Verify scan counters are emitted from the job
+ * @param job
+ * @throws IOException
+ */
+ private void verifyJobCountersAreEmitted(Job job) throws IOException {
+ Counters counters = job.getCounters();
+ Counter counter
+ = counters.findCounter(TableRecordReaderImpl.HBASE_COUNTER_GROUP_NAME, "RPC_CALLS");
+ assertNotNull("Unable to find Job counter for HBase scan metrics, RPC_CALLS", counter);
+ assertTrue("Counter value for RPC_CALLS should be larger than 0", counter.getValue() > 0);
+ }
+
@Test(expected = TableNotEnabledException.class)
public void testWritingToDisabledTable() throws IOException {