You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2009/09/17 05:20:58 UTC
svn commit: r816035 - in /hadoop/hbase/branches/0.20: ./
src/java/org/apache/hadoop/hbase/mapreduce/
Author: stack
Date: Thu Sep 17 03:20:57 2009
New Revision: 816035
URL: http://svn.apache.org/viewvc?rev=816035&view=rev
Log:
HBASE-1684 Backup (Export/Import) contrib tool for 0.20
Modified:
hadoop/hbase/branches/0.20/CHANGES.txt
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/Driver.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
Modified: hadoop/hbase/branches/0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/CHANGES.txt?rev=816035&r1=816034&r2=816035&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20/CHANGES.txt Thu Sep 17 03:20:57 2009
@@ -30,6 +30,7 @@
HBASE-1827 Add disabling block cache scanner flag to the shell
HBASE-1574 Client and server APIs to do batch deletes
HBASE-1833 hfile.main fixes
+ HBASE-1684 Backup (Export/Import) contrib tool for 0.20
Release 0.20.0 - Tue Sep 8 12:48:41 PDT 2009
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/Driver.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/Driver.java?rev=816035&r1=816034&r2=816035&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/Driver.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/Driver.java Thu Sep 17 03:20:57 2009
@@ -19,7 +19,6 @@
*/
package org.apache.hadoop.hbase.mapreduce;
-import org.apache.hadoop.hbase.migration.nineteen.HStoreFileToStoreFile;
import org.apache.hadoop.util.ProgramDriver;
/**
@@ -35,9 +34,8 @@
ProgramDriver pgd = new ProgramDriver();
pgd.addClass(RowCounter.NAME, RowCounter.class,
"Count rows in HBase table");
- pgd.addClass(HStoreFileToStoreFile.JOBNAME,
- HStoreFileToStoreFile.class,
- "Bulk convert 0.19 HStoreFiles to 0.20 StoreFiles");
+ pgd.addClass(Export.NAME, Export.class, "Write table data to HDFS.");
+ pgd.addClass(Import.NAME, Import.class, "Import data written by Export.");
pgd.driver(args);
}
}
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java?rev=816035&r1=816034&r2=816035&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java Thu Sep 17 03:20:57 2009
@@ -22,14 +22,13 @@
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
/**
@@ -37,7 +36,6 @@
* input row has columns that have content.
*/
public class RowCounter {
-
/** Name of this 'program'. */
static final String NAME = "rowcounter";
@@ -48,7 +46,7 @@
extends TableMapper<ImmutableBytesWritable, Result> {
/** Counter enumeration to count the actual rows. */
- private static enum Counters { ROWS }
+ private static enum Counters {ROWS}
/**
* Maps the data.
@@ -84,11 +82,12 @@
*/
public static Job createSubmittableJob(Configuration conf, String[] args)
throws IOException {
- Job job = new Job(conf, NAME);
+ String tableName = args[0];
+ Job job = new Job(conf, NAME + "_" + tableName);
job.setJarByClass(RowCounter.class);
// Columns are space delimited
StringBuilder sb = new StringBuilder();
- final int columnoffset = 2;
+ final int columnoffset = 1;
for (int i = columnoffset; i < args.length; i++) {
if (i > columnoffset) {
sb.append(" ");
@@ -96,13 +95,14 @@
sb.append(args[i]);
}
Scan scan = new Scan();
- scan.addColumns(sb.toString());
+ if (sb.length() > 0) scan.addColumns(sb.toString());
// Second argument is the table name.
- TableMapReduceUtil.initTableMapperJob(args[1], scan,
+ TableMapReduceUtil.initTableMapperJob(tableName, scan,
RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, job);
+ job.setOutputFormatClass(NullOutputFormat.class);
job.setNumReduceTasks(0);
// first argument is the output directory.
- FileOutputFormat.setOutputPath(job, new Path(args[0]));
+ // FileOutputFormat.setOutputPath(job, new Path(args[0]));
return job;
}
@@ -115,13 +115,12 @@
public static void main(String[] args) throws Exception {
HBaseConfiguration conf = new HBaseConfiguration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- if (otherArgs.length < 3) {
+ if (otherArgs.length < 1) {
System.err.println("ERROR: Wrong number of parameters: " + args.length);
- System.err.println("Usage: " + NAME +
- " <outputdir> <tablename> <column1> [<column2>...]");
+ System.err.println("Usage: RowCounter <tablename> [<column1> <column2>...]");
System.exit(-1);
}
Job job = createSubmittableJob(conf, otherArgs);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
-}
\ No newline at end of file
+}
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java?rev=816035&r1=816034&r2=816035&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableInputFormatBase.java Thu Sep 17 03:20:57 2009
@@ -21,17 +21,18 @@
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
@@ -249,7 +250,7 @@
if (trr == null) {
trr = new TableRecordReader();
}
- Scan sc = new Scan(scan);
+ Scan sc = new Scan(this.scan);
sc.setStartRow(tSplit.getStartRow());
sc.setStopRow(tSplit.getEndRow());
trr.setScan(sc);
@@ -277,9 +278,6 @@
if (table == null) {
throw new IOException("No table was provided.");
}
- if (!scan.hasFamilies()) {
- throw new IOException("Expecting at least one column.");
- }
int realNumSplits = startKeys.length;
InputSplit[] splits = new InputSplit[realNumSplits];
int middle = startKeys.length / realNumSplits;
@@ -320,7 +318,7 @@
* @return The internal scan instance.
*/
public Scan getScan() {
- if (scan == null) scan = new Scan();
+ if (this.scan == null) this.scan = new Scan();
return scan;
}
Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java?rev=816035&r1=816034&r2=816035&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java Thu Sep 17 03:20:57 2009
@@ -58,11 +58,11 @@
Class<? extends WritableComparable> outputKeyClass,
Class<? extends Writable> outputValueClass, Job job) throws IOException {
job.setInputFormatClass(TableInputFormat.class);
- job.setMapOutputValueClass(outputValueClass);
- job.setMapOutputKeyClass(outputKeyClass);
+ if (outputValueClass != null) job.setMapOutputValueClass(outputValueClass);
+ if (outputKeyClass != null) job.setMapOutputKeyClass(outputKeyClass);
job.setMapperClass(mapper);
job.getConfiguration().set(TableInputFormat.INPUT_TABLE, table);
- job.getConfiguration().set(TableInputFormat.SCAN,
+ job.getConfiguration().set(TableInputFormat.SCAN,
convertScanToString(scan));
}
@@ -125,7 +125,7 @@
Class<? extends TableReducer> reducer, Job job, Class partitioner)
throws IOException {
job.setOutputFormatClass(TableOutputFormat.class);
- job.setReducerClass(reducer);
+ if (reducer != null) job.setReducerClass(reducer);
job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, table);
job.setOutputKeyClass(ImmutableBytesWritable.class);
job.setOutputValueClass(Put.class);