You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2012/03/22 23:37:51 UTC
svn commit: r1304114 - in
/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce:
CopyTable.java Export.java Import.java ImportTsv.java RowCounter.java
Author: larsh
Date: Thu Mar 22 22:37:51 2012
New Revision: 1304114
URL: http://svn.apache.org/viewvc?rev=1304114&view=rev
Log:
HBASE-4657 Improve the efficiency of our MR jobs with a few configurations
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java?rev=1304114&r1=1304113&r2=1304114&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java Thu Mar 22 22:37:51 2012
@@ -66,6 +66,7 @@ public class CopyTable {
Job job = new Job(conf, NAME + "_" + tableName);
job.setJarByClass(CopyTable.class);
Scan scan = new Scan();
+ scan.setCacheBlocks(false);
if (startTime != 0) {
scan.setTimeRange(startTime,
endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
@@ -111,7 +112,7 @@ public class CopyTable {
if (errorMsg != null && errorMsg.length() > 0) {
System.err.println("ERROR: " + errorMsg);
}
- System.err.println("Usage: CopyTable [--rs.class=CLASS] " +
+ System.err.println("Usage: CopyTable [general options] [--rs.class=CLASS] " +
"[--rs.impl=IMPL] [--starttime=X] [--endtime=Y] " +
"[--new.name=NEW] [--peer.adr=ADR] <tablename>");
System.err.println();
@@ -140,6 +141,9 @@ public class CopyTable {
"org.apache.hadoop.hbase.mapreduce.CopyTable --rs.class=org.apache.hadoop.hbase.ipc.ReplicationRegionInterface " +
"--rs.impl=org.apache.hadoop.hbase.regionserver.replication.ReplicationRegionServer --starttime=1265875194289 --endtime=1265878794289 " +
"--peer.adr=server1,server2,server3:2181:/hbase --families=myOldCf:myNewCf,cf2,cf3 TestTable ");
+ System.err.println("For performance consider the following general options:\n"
+ + "-Dhbase.client.scanner.caching=100\n"
+ + "-Dmapred.map.tasks.speculative.execution=false");
}
private static boolean doCommandLine(final String[] args) {
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java?rev=1304114&r1=1304113&r2=1304114&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java Thu Mar 22 22:37:51 2012
@@ -166,6 +166,10 @@ public class Export {
System.err.println(" to control/limit what is exported..");
System.err.println(" -D " + TableInputFormat.SCAN_COLUMN_FAMILY + "=<familyName>");
System.err.println(" -D " + RAW_SCAN + "=true");
+ System.err.println("For performance consider the following properties:\n"
+ + " -Dhbase.client.scanner.caching=100\n"
+ + " -Dmapred.map.tasks.speculative.execution=false\n"
+ + " -Dmapred.reduce.tasks.speculative.execution=false");
}
/**
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java?rev=1304114&r1=1304113&r2=1304114&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/Import.java Thu Mar 22 22:37:51 2012
@@ -271,6 +271,9 @@ public class Import {
System.err.println("By default Import will load data directly into HBase. To instead generate");
System.err.println("HFiles of data to prepare for a bulk data load, pass the option:");
System.err.println(" -D" + BULK_OUTPUT_CONF_KEY + "=/path/for/output");
+ System.err.println("For performance consider the following options:\n"
+ + " -Dmapred.map.tasks.speculative.execution=false\n"
+ + " -Dmapred.reduce.tasks.speculative.execution=false");
}
/**
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java?rev=1304114&r1=1304113&r2=1304114&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/ImportTsv.java Thu Mar 22 22:37:51 2012
@@ -266,7 +266,10 @@ public class ImportTsv {
" -D" + SKIP_LINES_CONF_KEY + "=false - fail if encountering an invalid line\n" +
" '-D" + SEPARATOR_CONF_KEY + "=|' - eg separate on pipes instead of tabs\n" +
" -D" + TIMESTAMP_CONF_KEY + "=currentTimeAsLong - use the specified timestamp for the import\n" +
- " -D" + MAPPER_CONF_KEY + "=my.Mapper - A user-defined Mapper to use instead of " + DEFAULT_MAPPER.getName() + "\n";
+ " -D" + MAPPER_CONF_KEY + "=my.Mapper - A user-defined Mapper to use instead of " + DEFAULT_MAPPER.getName() + "\n" +
+ "For performance consider the following options:\n" +
+ " -Dmapred.map.tasks.speculative.execution=false\n" +
+ " -Dmapred.reduce.tasks.speculative.execution=false";
System.err.println(usage);
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java?rev=1304114&r1=1304113&r2=1304114&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/mapreduce/RowCounter.java Thu Mar 22 22:37:51 2012
@@ -109,6 +109,7 @@ public class RowCounter {
Job job = new Job(conf, NAME + "_" + tableName);
job.setJarByClass(RowCounter.class);
Scan scan = new Scan();
+ scan.setCacheBlocks(false);
if (startKey != null && !startKey.equals("")) {
scan.setStartRow(Bytes.toBytes(startKey));
}
@@ -145,8 +146,11 @@ public class RowCounter {
* Prints usage without error message
*/
private static void printUsage() {
- System.err.println("Usage: RowCounter <tablename> " +
+ System.err.println("Usage: RowCounter [options] <tablename> " +
"[--range=[startKey],[endKey]] [<column1> <column2>...]");
+ System.err.println("For performance consider the following options:\n"
+ + "-Dhbase.client.scanner.caching=100\n"
+ + "-Dmapred.map.tasks.speculative.execution=false");
}
/**