You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2009/06/16 03:51:05 UTC
svn commit: r785051 - in /hadoop/hbase/trunk/src:
java/org/apache/hadoop/hbase/mapred/ test/org/apache/hadoop/hbase/mapred/
Author: apurtell
Date: Tue Jun 16 01:51:05 2009
New Revision: 785051
URL: http://svn.apache.org/viewvc?rev=785051&view=rev
Log:
HBASE-1526 mapreduce fixup; put back test, deprecate mapred package
Added:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
- copied, changed from r784993, hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/DisabledTestTableMapReduce.java
Removed:
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/DisabledTestTableMapReduce.java
Modified:
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/BuildTableIndex.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/Driver.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexConfiguration.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexOutputFormat.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexTableReduce.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/LuceneDocumentWrapper.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMap.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableReduce.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableSplit.java
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/BuildTableIndex.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/BuildTableIndex.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/BuildTableIndex.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/BuildTableIndex.java Tue Jun 16 01:51:05 2009
@@ -50,6 +50,7 @@
* column content, etc.</li>
* </ul>
*/
+@Deprecated
public class BuildTableIndex {
private static final String USAGE = "Usage: BuildTableIndex " +
"-m <numMapTasks> -r <numReduceTasks>\n -indexConf <iconfFile> " +
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/Driver.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/Driver.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/Driver.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/Driver.java Tue Jun 16 01:51:05 2009
@@ -25,6 +25,7 @@
* Driver for hbase mapreduce jobs. Select which to run by passing
* name of job to this main.
*/
+@Deprecated
public class Driver {
/**
* @param args
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/GroupingTableMap.java Tue Jun 16 01:51:05 2009
@@ -38,6 +38,7 @@
/**
* Extract grouping columns from input record
*/
+@Deprecated
public class GroupingTableMap
extends MapReduceBase
implements TableMap<ImmutableBytesWritable,RowResult> {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java Tue Jun 16 01:51:05 2009
@@ -39,6 +39,7 @@
* @param <K2>
* @param <V2>
*/
+@Deprecated
public class HRegionPartitioner<K2,V2>
implements Partitioner<ImmutableBytesWritable, V2> {
private final Log LOG = LogFactory.getLog(TableInputFormat.class);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableMap.java Tue Jun 16 01:51:05 2009
@@ -31,6 +31,7 @@
/**
* Pass the given key and record as-is to reduce
*/
+@Deprecated
public class IdentityTableMap
extends MapReduceBase
implements TableMap<ImmutableBytesWritable, RowResult> {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IdentityTableReduce.java Tue Jun 16 01:51:05 2009
@@ -33,6 +33,7 @@
/**
* Write to table each key, record pair
*/
+@Deprecated
public class IdentityTableReduce
extends MapReduceBase
implements TableReduce<ImmutableBytesWritable, BatchUpdate> {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexConfiguration.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexConfiguration.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexConfiguration.java Tue Jun 16 01:51:05 2009
@@ -46,6 +46,7 @@
/**
* Configuration parameters for building a Lucene index
*/
+@Deprecated
public class IndexConfiguration extends Configuration {
private static final Log LOG = LogFactory.getLog(IndexConfiguration.class);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexOutputFormat.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexOutputFormat.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexOutputFormat.java Tue Jun 16 01:51:05 2009
@@ -41,6 +41,7 @@
* Create a local index, unwrap Lucene documents created by reduce, add them to
* the index, and copy the index to the destination.
*/
+@Deprecated
public class IndexOutputFormat extends
FileOutputFormat<ImmutableBytesWritable, LuceneDocumentWrapper> {
static final Log LOG = LogFactory.getLog(IndexOutputFormat.class);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexTableReduce.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexTableReduce.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexTableReduce.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/IndexTableReduce.java Tue Jun 16 01:51:05 2009
@@ -41,6 +41,7 @@
* Construct a Lucene document per row, which is consumed by IndexOutputFormat
* to build a Lucene index
*/
+@Deprecated
public class IndexTableReduce extends MapReduceBase implements
Reducer<ImmutableBytesWritable, RowResult, ImmutableBytesWritable, LuceneDocumentWrapper> {
private static final Log LOG = LogFactory.getLog(IndexTableReduce.class);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/LuceneDocumentWrapper.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/LuceneDocumentWrapper.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/LuceneDocumentWrapper.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/LuceneDocumentWrapper.java Tue Jun 16 01:51:05 2009
@@ -28,6 +28,7 @@
* A utility class used to pass a lucene document from reduce to OutputFormat.
* It doesn't really serialize/deserialize a lucene document.
*/
+@Deprecated
public class LuceneDocumentWrapper implements Writable {
protected Document doc;
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/RowCounter.java Tue Jun 16 01:51:05 2009
@@ -42,6 +42,7 @@
* Map outputs table rows IF the input row has columns that have content.
* Uses an {@link IdentityReducer}
*/
+@Deprecated
public class RowCounter extends Configured implements Tool {
// Name of this 'program'
static final String NAME = "rowcounter";
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableInputFormat.java Tue Jun 16 01:51:05 2009
@@ -35,6 +35,7 @@
/**
* Convert HBase tabular data into a format that is consumable by Map/Reduce.
*/
+@Deprecated
public class TableInputFormat extends TableInputFormatBase implements
JobConfigurable {
private final Log LOG = LogFactory.getLog(TableInputFormat.class);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableInputFormatBase.java Tue Jun 16 01:51:05 2009
@@ -74,6 +74,8 @@
* }
* </pre>
*/
+
+@Deprecated
public abstract class TableInputFormatBase
implements InputFormat<ImmutableBytesWritable, RowResult> {
final Log LOG = LogFactory.getLog(TableInputFormatBase.class);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMap.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMap.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMap.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMap.java Tue Jun 16 01:51:05 2009
@@ -32,6 +32,7 @@
* @param <K> WritableComparable key class
* @param <V> Writable value class
*/
+@Deprecated
public interface TableMap<K extends WritableComparable<K>, V extends Writable>
extends Mapper<ImmutableBytesWritable, RowResult, K, V> {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java Tue Jun 16 01:51:05 2009
@@ -33,6 +33,7 @@
/**
* Utility for {@link TableMap} and {@link TableReduce}
*/
+@Deprecated
@SuppressWarnings("unchecked")
public class TableMapReduceUtil {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableOutputFormat.java Tue Jun 16 01:51:05 2009
@@ -39,6 +39,7 @@
/**
* Convert Map/Reduce output and write it to an HBase table
*/
+@Deprecated
public class TableOutputFormat extends
FileOutputFormat<ImmutableBytesWritable, BatchUpdate> {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableReduce.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableReduce.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableReduce.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableReduce.java Tue Jun 16 01:51:05 2009
@@ -31,6 +31,7 @@
* @param <K> key class
* @param <V> value class
*/
+@Deprecated
@SuppressWarnings("unchecked")
public interface TableReduce<K extends WritableComparable, V extends Writable>
extends Reducer<K, V, ImmutableBytesWritable, BatchUpdate> {
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableSplit.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableSplit.java?rev=785051&r1=785050&r2=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableSplit.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/mapred/TableSplit.java Tue Jun 16 01:51:05 2009
@@ -30,6 +30,7 @@
/**
* A table split corresponds to a key range [low, high)
*/
+@Deprecated
public class TableSplit implements InputSplit, Comparable<TableSplit> {
private byte [] m_tableName;
private byte [] m_startRow;
Copied: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java (from r784993, hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/DisabledTestTableMapReduce.java)
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java?p2=hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java&p1=hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/DisabledTestTableMapReduce.java&r1=784993&r2=785051&rev=785051&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/DisabledTestTableMapReduce.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java Tue Jun 16 01:51:05 2009
@@ -51,9 +51,9 @@
* on our tables is simple - take every row in the table, reverse the value of
* a particular cell, and write it back to the table.
*/
-public class DisabledTestTableMapReduce extends MultiRegionTable {
+public class TestTableMapReduce extends MultiRegionTable {
private static final Log LOG =
- LogFactory.getLog(DisabledTestTableMapReduce.class.getName());
+ LogFactory.getLog(TestTableMapReduce.class.getName());
static final String MULTI_REGION_TABLE_NAME = "mrtest";
static final String INPUT_COLUMN = "contents:";
@@ -65,7 +65,7 @@
};
/** constructor */
- public DisabledTestTableMapReduce() {
+ public TestTableMapReduce() {
super(INPUT_COLUMN);
desc = new HTableDescriptor(MULTI_REGION_TABLE_NAME);
desc.addFamily(new HColumnDescriptor(INPUT_COLUMN));
@@ -93,10 +93,10 @@
if (value.size() != 1) {
throw new IOException("There should only be one input column");
}
- byte [][] keys = value.keySet().toArray(new byte [value.size()][]);
+ byte [][] keys = value.keySet().toArray(new byte[value.size()][]);
if(!Bytes.equals(keys[0], Bytes.toBytes(INPUT_COLUMN))) {
- throw new IOException("Wrong input column. Expected: " + INPUT_COLUMN
- + " but got: " + keys[0]);
+ throw new IOException("Wrong input column. Expected: '" + INPUT_COLUMN
+ + "' but got: '" + Bytes.toString(keys[0]) + "'");
}
// Get the original value and reverse it
@@ -130,7 +130,7 @@
JobConf jobConf = null;
try {
LOG.info("Before map/reduce startup");
- jobConf = new JobConf(conf, DisabledTestTableMapReduce.class);
+ jobConf = new JobConf(conf, TestTableMapReduce.class);
jobConf.setJobName("process column contents");
jobConf.setNumReduceTasks(1);
TableMapReduceUtil.initTableMapJob(Bytes.toString(table.getTableName()),