You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2009/04/23 07:29:11 UTC
svn commit: r767802 - in /hadoop/hbase/trunk: CHANGES.txt
src/java/org/apache/hadoop/hbase/mapred/RowCounter.java
Author: stack
Date: Thu Apr 23 05:29:10 2009
New Revision: 767802
URL: http://svn.apache.org/viewvc?rev=767802&view=rev
Log:
HBASE-1333 RowCounter updates
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=767802&r1=767801&r2=767802&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Thu Apr 23 05:29:10 2009
@@ -151,6 +151,7 @@
HBASE-1331 Lower the default scanner caching value
HBASE-1235 Add table enabled status to shell and UI
(Lars George via Stack)
+ HBASE-1333 RowCounter updates
Release 0.19.0 - 01/21/2009
INCOMPATIBLE CHANGES
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java?rev=767802&r1=767801&r2=767802&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java Thu Apr 23 05:29:10 2009
@@ -22,18 +22,15 @@
import java.io.IOException;
import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.io.HbaseMapWritable;
import org.apache.hadoop.hbase.io.Cell;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.RowResult;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.lib.IdentityReducer;
@@ -45,36 +42,41 @@
* Map outputs table rows IF the input row has columns that have content.
* Uses an {@link IdentityReducer}
*/
-public class RowCounter
-extends MapReduceBase
-implements TableMap<ImmutableBytesWritable, RowResult>, Tool {
- /* Name of this 'program'
- */
+public class RowCounter extends Configured implements Tool {
+ // Name of this 'program'
static final String NAME = "rowcounter";
-
- private Configuration conf;
- private final RowResult EMPTY_RESULT_VALUE =
- new RowResult(Bytes.toBytes("dummy"),new HbaseMapWritable<byte [], Cell>());
- private static enum Counters {ROWS}
-
- public void map(ImmutableBytesWritable row, RowResult value,
- OutputCollector<ImmutableBytesWritable, RowResult> output,
- Reporter reporter)
- throws IOException {
- boolean content = false;
- for (Map.Entry<byte [], Cell> e: value.entrySet()) {
- Cell cell = e.getValue();
- if (cell != null && cell.getValue().length > 0) {
- content = true;
- break;
+
+ static class RowCounterMapper
+ implements TableMap<ImmutableBytesWritable, RowResult> {
+ private static enum Counters {ROWS}
+
+ public void map(ImmutableBytesWritable row, RowResult value,
+ OutputCollector<ImmutableBytesWritable, RowResult> output,
+ Reporter reporter)
+ throws IOException {
+ boolean content = false;
+ for (Map.Entry<byte [], Cell> e: value.entrySet()) {
+ Cell cell = e.getValue();
+ if (cell != null && cell.getValue().length > 0) {
+ content = true;
+ break;
+ }
+ }
+ if (!content) {
+ // Don't count rows that are all empty values.
+ return;
}
+ // Give out same value every time. We're only interested in the row/key
+ reporter.incrCounter(Counters.ROWS, 1);
+ }
+
+ public void configure(JobConf jc) {
+ // Nothing to do.
}
- if (!content) {
- return;
+
+ public void close() throws IOException {
+ // Nothing to do.
}
- // Give out same value every time. We're only interested in the row/key
- reporter.incrCounter(Counters.ROWS, 1);
- output.collect(row, EMPTY_RESULT_VALUE);
}
/**
@@ -83,7 +85,7 @@
* @throws IOException
*/
public JobConf createSubmittableJob(String[] args) throws IOException {
- JobConf c = new JobConf(getConf(), RowCounter.class);
+ JobConf c = new JobConf(getConf(), getClass());
c.setJobName(NAME);
// Columns are space delimited
StringBuilder sb = new StringBuilder();
@@ -95,9 +97,9 @@
sb.append(args[i]);
}
// Second argument is the table name.
- TableMapReduceUtil.initTableMapJob(args[1], sb.toString(), this.getClass(),
- ImmutableBytesWritable.class, RowResult.class, c);
- c.setReducerClass(IdentityReducer.class);
+ TableMapReduceUtil.initTableMapJob(args[1], sb.toString(),
+ RowCounterMapper.class, ImmutableBytesWritable.class, RowResult.class, c);
+ c.setNumReduceTasks(0);
// First arg is the output directory.
FileOutputFormat.setOutputPath(c, new Path(args[0]));
return c;
@@ -119,14 +121,6 @@
return 0;
}
- public Configuration getConf() {
- return this.conf;
- }
-
- public void setConf(final Configuration c) {
- this.conf = c;
- }
-
/**
* @param args
* @throws Exception
@@ -136,4 +130,4 @@
int errCode = ToolRunner.run(c, new RowCounter(), args);
System.exit(errCode);
}
-}
+}
\ No newline at end of file