You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2009/04/25 00:43:04 UTC
svn commit: r768431 - in /hadoop/hbase/trunk: CHANGES.txt
src/java/org/apache/hadoop/hbase/mapred/RowCounter.java
src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
Author: stack
Date: Fri Apr 24 22:43:03 2009
New Revision: 768431
URL: http://svn.apache.org/viewvc?rev=768431&view=rev
Log:
HBASE-1287 Partitioner class not used in TableMapReduceUtil.initTableReduceJob()
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=768431&r1=768430&r2=768431&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Fri Apr 24 22:43:03 2009
@@ -87,6 +87,8 @@
HBASE-1292 php thrift's getRow() would throw an exception if the row does
not exist (Rong-en Fan via Stack)
HBASE-1340 Fix new javadoc warnings (Evgeny Ryabitskiy via Stack)
+ HBASE-1287 Partitioner class not used in TableMapReduceUtil.initTableReduceJob()
+ (Lars George and Billy Pearson via Stack)
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java?rev=768431&r1=768430&r2=768431&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java Fri Apr 24 22:43:03 2009
@@ -46,6 +46,9 @@
// Name of this 'program'
static final String NAME = "rowcounter";
+ /**
+ * Mapper that runs the count.
+ */
static class RowCounterMapper
implements TableMap<ImmutableBytesWritable, RowResult> {
private static enum Counters {ROWS}
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java?rev=768431&r1=768430&r2=768431&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java Fri Apr 24 22:43:03 2009
@@ -35,16 +35,17 @@
*/
@SuppressWarnings("unchecked")
public class TableMapReduceUtil {
+
/**
* Use this before submitting a TableMap job. It will
* appropriately set up the JobConf.
*
- * @param table table name
- * @param columns columns to scan
- * @param mapper mapper class
- * @param outputKeyClass
- * @param outputValueClass
- * @param job job configuration
+ * @param table The table name to read from.
+ * @param columns The columns to scan.
+ * @param mapper The mapper class to use.
+ * @param outputKeyClass The class of the output key.
+ * @param outputValueClass The class of the output value.
+ * @param job The current job configuration to adjust.
*/
public static void initTableMapJob(String table, String columns,
Class<? extends TableMap> mapper,
@@ -63,10 +64,10 @@
* Use this before submitting a TableReduce job. It will
* appropriately set up the JobConf.
*
- * @param table
- * @param reducer
- * @param job
- * @throws IOException
+ * @param table The output table.
+ * @param reducer The reducer class to use.
+ * @param job The current job configuration to adjust.
+ * @throws IOException When determining the region count fails.
*/
public static void initTableReduceJob(String table,
Class<? extends TableReduce> reducer, JobConf job)
@@ -78,12 +79,12 @@
* Use this before submitting a TableReduce job. It will
* appropriately set up the JobConf.
*
- * @param table
- * @param reducer
- * @param job
- * @param partitioner Partitioner to use. Pass null to use default
- * partitioner.
- * @throws IOException
+ * @param table The output table.
+ * @param reducer The reducer class to use.
+ * @param job The current job configuration to adjust.
+ * @param partitioner Partitioner to use. Pass <code>null</code> to use
+ * default partitioner.
+ * @throws IOException When determining the region count fails.
*/
public static void initTableReduceJob(String table,
Class<? extends TableReduce> reducer, JobConf job, Class partitioner)
@@ -93,13 +94,78 @@
job.set(TableOutputFormat.OUTPUT_TABLE, table);
job.setOutputKeyClass(ImmutableBytesWritable.class);
job.setOutputValueClass(BatchUpdate.class);
- if (partitioner != null) {
+ if (partitioner == HRegionPartitioner.class) {
job.setPartitionerClass(HRegionPartitioner.class);
HTable outputTable = new HTable(new HBaseConfiguration(job), table);
int regions = outputTable.getRegionsInfo().size();
- if (job.getNumReduceTasks() > regions){
- job.setNumReduceTasks(outputTable.getRegionsInfo().size());
+ if (job.getNumReduceTasks() > regions) {
+ job.setNumReduceTasks(outputTable.getRegionsInfo().size());
}
+ } else if (partitioner != null) {
+ job.setPartitionerClass(partitioner);
}
}
+
+ /**
+ * Ensures that the given number of reduce tasks for the given job
+ * configuration does not exceed the number of regions for the given table.
+ *
+ * @param table The table to get the region count for.
+ * @param job The current job configuration to adjust.
+ * @throws IOException When retrieving the table details fails.
+ */
+ public void limitNumReduceTasks(String table, JobConf job)
+ throws IOException {
+ HTable outputTable = new HTable(new HBaseConfiguration(job), table);
+ int regions = outputTable.getRegionsInfo().size();
+ if (job.getNumReduceTasks() > regions)
+ job.setNumReduceTasks(regions);
+ }
+
+ /**
+ * Ensures that the given number of map tasks for the given job
+ * configuration does not exceed the number of regions for the given table.
+ *
+ * @param table The table to get the region count for.
+ * @param job The current job configuration to adjust.
+ * @throws IOException When retrieving the table details fails.
+ */
+ public void limitNumMapTasks(String table, JobConf job)
+ throws IOException {
+ HTable outputTable = new HTable(new HBaseConfiguration(job), table);
+ int regions = outputTable.getRegionsInfo().size();
+ if (job.getNumMapTasks() > regions)
+ job.setNumMapTasks(regions);
+ }
+
+ /**
+ * Sets the number of reduce tasks for the given job configuration to the
+ * number of regions the given table has.
+ *
+ * @param table The table to get the region count for.
+ * @param job The current job configuration to adjust.
+ * @throws IOException When retrieving the table details fails.
+ */
+ public void setNumReduceTasks(String table, JobConf job)
+ throws IOException {
+ HTable outputTable = new HTable(new HBaseConfiguration(job), table);
+ int regions = outputTable.getRegionsInfo().size();
+ job.setNumReduceTasks(regions);
+ }
+
+ /**
+ * Sets the number of map tasks for the given job configuration to the
+ * number of regions the given table has.
+ *
+ * @param table The table to get the region count for.
+ * @param job The current job configuration to adjust.
+ * @throws IOException When retrieving the table details fails.
+ */
+ public void setNumMapTasks(String table, JobConf job)
+ throws IOException {
+ HTable outputTable = new HTable(new HBaseConfiguration(job), table);
+ int regions = outputTable.getRegionsInfo().size();
+ job.setNumMapTasks(regions);
+ }
+
}
\ No newline at end of file