You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/07/29 01:39:56 UTC
git commit: HBASE-3135 Make our MR jobs implement Tool and use
ToolRunner so can do -D trickery, etc. (Talat Uyarer)
Repository: hbase
Updated Branches:
refs/heads/master c71d1f6a0 -> ff655e04d
HBASE-3135 Make our MR jobs implement Tool and use ToolRunner so can do -D trickery, etc. (Talat Uyarer)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ff655e04
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ff655e04
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ff655e04
Branch: refs/heads/master
Commit: ff655e04d17051012a1645a82df42b5aec94af92
Parents: c71d1f6
Author: stack <st...@apache.org>
Authored: Mon Jul 28 16:39:49 2014 -0700
Committer: stack <st...@apache.org>
Committed: Mon Jul 28 16:39:49 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/mapreduce/CellCounter.java | 36 +++++++++++-------
.../hadoop/hbase/mapreduce/CopyTable.java | 12 +++---
.../apache/hadoop/hbase/mapreduce/Export.java | 33 +++++++++-------
.../apache/hadoop/hbase/mapreduce/Import.java | 40 ++++++++++++--------
.../hadoop/hbase/mapreduce/ImportTsv.java | 2 +-
.../hadoop/hbase/mapreduce/RowCounter.java | 38 +++++++++++--------
.../hadoop/hbase/mapreduce/WALPlayer.java | 2 +-
7 files changed, 97 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
index 2661bed..f4916ae 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CellCounter.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
@@ -45,6 +46,8 @@ import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import com.google.common.base.Preconditions;
@@ -68,7 +71,7 @@ import com.google.common.base.Preconditions;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class CellCounter {
+public class CellCounter extends Configured implements Tool {
private static final Log LOG =
LogFactory.getLog(CellCounter.class.getName());
@@ -185,7 +188,7 @@ public class CellCounter {
Path outputDir = new Path(args[1]);
String reportSeparatorString = (args.length > 2) ? args[2]: ":";
conf.set("ReportSeparator", reportSeparatorString);
- Job job = new Job(conf, NAME + "_" + tableName);
+ Job job = Job.getInstance(conf, NAME + "_" + tableName);
job.setJarByClass(CellCounter.class);
Scan scan = getConfiguredScanForJob(conf, args);
TableMapReduceUtil.initTableMapperJob(tableName, scan,
@@ -233,15 +236,9 @@ public class CellCounter {
return rowFilter;
}
- /**
- * Main entry point.
- *
- * @param args The command line parameters.
- * @throws Exception When running the job fails.
- */
- public static void main(String[] args) throws Exception {
- Configuration conf = HBaseConfiguration.create();
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ @Override
+ public int run(String[] args) throws Exception {
+ String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
if (otherArgs.length < 1) {
System.err.println("ERROR: Wrong number of parameters: " + args.length);
System.err.println("Usage: CellCounter <tablename> <outputDir> <reportSeparator> " +
@@ -254,9 +251,20 @@ public class CellCounter {
"string : used to separate the rowId/column family name and qualifier name.");
System.err.println(" [^[regex pattern] or [Prefix] parameter can be used to limit the cell counter count " +
"operation to a limited subset of rows from the table based on regex or prefix pattern.");
- System.exit(-1);
+ return -1;
}
- Job job = createSubmittableJob(conf, otherArgs);
- System.exit(job.waitForCompletion(true) ? 0 : 1);
+ Job job = createSubmittableJob(getConf(), otherArgs);
+ return (job.waitForCompletion(true) ? 0 : 1);
+ }
+
+ /**
+ * Main entry point.
+ * @param args The command line parameters.
+ * @throws Exception When running the job fails.
+ */
+ public static void main(String[] args) throws Exception {
+ int errCode = ToolRunner.run(HBaseConfiguration.create(), new CellCounter(), args);
+ System.exit(errCode);
}
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
index 984f3b5..decb6a7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -18,23 +18,23 @@
*/
package org.apache.hadoop.hbase.mapreduce;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
/**
* Tool used to copy a table to another one which can be on a different setup.
* It is also configurable with a start and time as well as a specification
@@ -72,7 +72,7 @@ public class CopyTable extends Configured implements Tool {
if (!doCommandLine(args)) {
return null;
}
- Job job = new Job(conf, NAME + "_" + tableName);
+ Job job = Job.getInstance(conf, NAME + "_" + tableName);
job.setJarByClass(CopyTable.class);
Scan scan = new Scan();
scan.setCacheBlocks(false);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
index a69d4e4..fdff068 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
@@ -25,7 +25,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Result;
@@ -42,6 +42,8 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
/**
* Export an HBase table.
@@ -50,7 +52,7 @@ import org.apache.hadoop.util.GenericOptionsParser;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class Export {
+public class Export extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(Export.class);
final static String NAME = "export";
final static String RAW_SCAN = "hbase.mapreduce.include.deleted.rows";
@@ -68,7 +70,7 @@ public class Export {
throws IOException {
String tableName = args[0];
Path outputDir = new Path(args[1]);
- Job job = new Job(conf, NAME + "_" + tableName);
+ Job job = Job.getInstance(conf, NAME + "_" + tableName);
job.setJobName(NAME + "_" + tableName);
job.setJarByClass(Export.class);
// Set optional scan parameters
@@ -172,20 +174,25 @@ public class Export {
+ " -D" + EXPORT_BATCHING + "=10");
}
+
+ @Override
+ public int run(String[] args) throws Exception {
+ String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
+ if (otherArgs.length < 2) {
+ usage("Wrong number of arguments: " + otherArgs.length);
+ return -1;
+ }
+ Job job = createSubmittableJob(getConf(), otherArgs);
+ return (job.waitForCompletion(true) ? 0 : 1);
+ }
+
/**
* Main entry point.
- *
- * @param args The command line parameters.
+ * @param args The command line parameters.
* @throws Exception When running the job fails.
*/
public static void main(String[] args) throws Exception {
- Configuration conf = HBaseConfiguration.create();
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
- if (otherArgs.length < 2) {
- usage("Wrong number of arguments: " + otherArgs.length);
- System.exit(-1);
- }
- Job job = createSubmittableJob(conf, otherArgs);
- System.exit(job.waitForCompletion(true)? 0 : 1);
+ int errCode = ToolRunner.run(HBaseConfiguration.create(), new Export(), args);
+ System.exit(errCode);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
index 6e783e9..7751c11 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
@@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
@@ -57,6 +58,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.zookeeper.KeeperException;
@@ -65,7 +68,7 @@ import org.apache.zookeeper.KeeperException;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class Import {
+public class Import extends Configured implements Tool {
private static final Log LOG = LogFactory.getLog(Import.class);
final static String NAME = "import";
public final static String CF_RENAME_PROP = "HBASE_IMPORTER_RENAME_CFS";
@@ -399,7 +402,7 @@ public class Import {
String tableName = args[0];
conf.set(TABLE_NAME, tableName);
Path inputDir = new Path(args[1]);
- Job job = new Job(conf, NAME + "_" + tableName);
+ Job job = Job.getInstance(conf, NAME + "_" + tableName);
job.setJarByClass(Importer.class);
FileInputFormat.setInputPaths(job, inputDir);
job.setInputFormatClass(SequenceFileInputFormat.class);
@@ -491,29 +494,34 @@ public class Import {
}
}
- /**
- * Main entry point.
- *
- * @param args The command line parameters.
- * @throws Exception When running the job fails.
- */
- public static void main(String[] args) throws Exception {
- Configuration conf = HBaseConfiguration.create();
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ @Override
+ public int run(String[] args) throws Exception {
+ String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
if (otherArgs.length < 2) {
usage("Wrong number of arguments: " + otherArgs.length);
- System.exit(-1);
+ return -1;
}
String inputVersionString = System.getProperty(ResultSerialization.IMPORT_FORMAT_VER);
if (inputVersionString != null) {
- conf.set(ResultSerialization.IMPORT_FORMAT_VER, inputVersionString);
+ getConf().set(ResultSerialization.IMPORT_FORMAT_VER, inputVersionString);
}
- Job job = createSubmittableJob(conf, otherArgs);
+ Job job = createSubmittableJob(getConf(), otherArgs);
boolean isJobSuccessful = job.waitForCompletion(true);
if(isJobSuccessful){
// Flush all the regions of the table
- flushRegionsIfNecessary(conf);
+ flushRegionsIfNecessary(getConf());
}
- System.exit(job.waitForCompletion(true) ? 0 : 1);
+ return (isJobSuccessful ? 0 : 1);
+ }
+
+ /**
+ * Main entry point.
+ * @param args The command line parameters.
+ * @throws Exception When running the job fails.
+ */
+ public static void main(String[] args) throws Exception {
+ int errCode = ToolRunner.run(HBaseConfiguration.create(), new Import(), args);
+ System.exit(errCode);
}
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
index 7bbf71f..0748481 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
@@ -416,7 +416,7 @@ public class ImportTsv extends Configured implements Tool {
String tableName = args[0];
Path inputDir = new Path(args[1]);
String jobName = conf.get(JOB_NAME_CONF_KEY,NAME + "_" + tableName);
- Job job = new Job(conf, jobName);
+ Job job = Job.getInstance(conf, NAME + "_" + tableName);
job.setJarByClass(mapperClass);
FileInputFormat.setInputPaths(job, inputDir);
job.setInputFormatClass(TextInputFormat.class);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
index 0851415..c0cf32c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
@@ -26,6 +26,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
@@ -36,6 +37,8 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
/**
* A job with a just a map phase to count rows. Map outputs table rows IF the
@@ -43,7 +46,7 @@ import org.apache.hadoop.util.GenericOptionsParser;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class RowCounter {
+public class RowCounter extends Configured implements Tool {
/** Name of this 'program'. */
static final String NAME = "rowcounter";
@@ -112,7 +115,7 @@ public class RowCounter {
}
}
- Job job = new Job(conf, NAME + "_" + tableName);
+ Job job = Job.getInstance(conf, NAME + "_" + tableName);
job.setJarByClass(RowCounter.class);
Scan scan = new Scan();
scan.setCacheBlocks(false);
@@ -170,23 +173,28 @@ public class RowCounter {
+ "-Dmapreduce.map.speculative=false");
}
- /**
- * Main entry point.
- *
- * @param args The command line parameters.
- * @throws Exception When running the job fails.
- */
- public static void main(String[] args) throws Exception {
- Configuration conf = HBaseConfiguration.create();
- String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+ @Override
+ public int run(String[] args) throws Exception {
+ String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs();
if (otherArgs.length < 1) {
printUsage("Wrong number of parameters: " + args.length);
- System.exit(-1);
+ return -1;
}
- Job job = createSubmittableJob(conf, otherArgs);
+ Job job = createSubmittableJob(getConf(), otherArgs);
if (job == null) {
- System.exit(-1);
+ return -1;
}
- System.exit(job.waitForCompletion(true) ? 0 : 1);
+ return (job.waitForCompletion(true) ? 0 : 1);
}
+
+ /**
+ * Main entry point.
+ * @param args The command line parameters.
+ * @throws Exception When running the job fails.
+ */
+ public static void main(String[] args) throws Exception {
+ int errCode = ToolRunner.run(HBaseConfiguration.create(), new RowCounter(), args);
+ System.exit(errCode);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff655e04/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
index 456d04f..3f0cde4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java
@@ -231,7 +231,7 @@ public class WALPlayer extends Configured implements Tool {
}
conf.setStrings(TABLES_KEY, tables);
conf.setStrings(TABLE_MAP_KEY, tableMap);
- Job job = new Job(conf, NAME + "_" + inputDir);
+ Job job = Job.getInstance(conf, NAME + "_" + inputDir);
job.setJarByClass(WALPlayer.class);
FileInputFormat.setInputPaths(job, inputDir);
job.setInputFormatClass(HLogInputFormat.class);