svn commit: r1377212 - in /hbase/trunk/hbase-server/src: main/java/org/apache/hadoop/hbase/mapreduce/Export.java test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java

[st...@apache.org]

Author: stack
Date: Sat Aug 25 04:51:41 2012
New Revision: 1377212

URL: http://svn.apache.org/viewvc?rev=1377212&view=rev
Log:
HBASE-6372 Add scanner batching to Export job

Modified:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java?rev=1377212&r1=1377211&r2=1377212&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/Export.java Sat Aug 25 04:51:41 2012
@@ -52,7 +52,8 @@ import org.apache.commons.logging.LogFac
 public class Export {
   private static final Log LOG = LogFactory.getLog(Export.class);
   final static String NAME = "export";
-  final static String RAW_SCAN="hbase.mapreduce.include.deleted.rows";
+  final static String RAW_SCAN = "hbase.mapreduce.include.deleted.rows";
+  final static String EXPORT_BATCHING = "hbase.export.scanner.batch";
 
   /**
    * Mapper.
@@ -134,6 +135,15 @@ public class Export {
         LOG.info("Setting Scan Filter for Export.");
       s.setFilter(exportFilter);
     }
+
+    int batching = conf.getInt(EXPORT_BATCHING, -1);
+    if (batching !=  -1){
+      try{
+        s.setBatch(batching);
+	} catch (RuntimeException e) {
+	    LOG.error("Batching could not be set", e);
+      }
+    }
     LOG.info("versions=" + versions + ", starttime=" + startTime +
       ", endtime=" + endTime + ", keepDeletedCells=" + raw);
     return s;
@@ -174,6 +184,8 @@ public class Export {
         + "   -Dhbase.client.scanner.caching=100\n"
         + "   -Dmapred.map.tasks.speculative.execution=false\n"
         + "   -Dmapred.reduce.tasks.speculative.execution=false");
+    System.err.println("For tables with very wide rows consider setting the batch size as below:\n"
+        + "   -D" + EXPORT_BATCHING + "=10");
   }
 
   /**
@@ -192,4 +204,4 @@ public class Export {
     Job job = createSubmittableJob(conf, otherArgs);
     System.exit(job.waitForCompletion(true)? 0 : 1);
   }
-}
\ No newline at end of file
+}

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java?rev=1377212&r1=1377211&r2=1377212&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java Sat Aug 25 04:51:41 2012
@@ -57,6 +57,7 @@ public class TestImportExport {
   private static final byte[] FAMILYB = Bytes.toBytes(FAMILYB_STRING);
   private static final byte[] QUAL = Bytes.toBytes("q");
   private static final String OUTPUT_DIR = "outputdir";
+  private static final String EXPORT_BATCHING = "100";
 
   private static MiniHBaseCluster cluster;
   private static long now = System.currentTimeMillis();
@@ -102,6 +103,7 @@ public class TestImportExport {
     String[] args = new String[] {
         EXPORT_TABLE,
         OUTPUT_DIR,
+	EXPORT_BATCHING,
         "1000"
     };
 
@@ -191,6 +193,7 @@ public class TestImportExport {
         "-D" + Export.RAW_SCAN + "=true",
         EXPORT_TABLE,
         OUTPUT_DIR,
+	EXPORT_BATCHING,
         "1000"
     };