You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2011/07/12 07:15:58 UTC
svn commit: r1145459 - in /hbase/trunk: CHANGES.txt
src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
Author: tedyu
Date: Tue Jul 12 05:15:58 2011
New Revision: 1145459
URL: http://svn.apache.org/viewvc?rev=1145459&view=rev
Log:
HBASE-3871 Speedup LoadIncrementalHFiles by parallelizing HFile splitting
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1145459&r1=1145458&r2=1145459&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Tue Jul 12 05:15:58 2011
@@ -313,6 +313,7 @@ Release 0.91.0 - Unreleased
(Aaron T. Myers via todd)
HBASE-4054 Usability improvement to HTablePool (Daniel Iancu)
HBASE-4079 HTableUtil - helper class for loading data (Doug Meil via Ted Yu)
+ HBASE-3871 Speedup LoadIncrementalHFiles by parallelizing HFile splitting
TASKS
HBASE-3559 Move report of split to master OFF the heartbeat channel
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java?rev=1145459&r1=1145458&r2=1145459&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java Tue Jul 12 05:15:58 2011
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Deque;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
import java.util.Set;
@@ -36,6 +37,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log;
@@ -85,6 +87,7 @@ public class LoadIncrementalHFiles exten
private HBaseAdmin hbAdmin;
private Configuration cfg;
private Set<Future> futures = new HashSet<Future>();
+ private Set<Future> futuresForSplittingHFile = new HashSet<Future>();
public static String NAME = "completebulkload";
@@ -188,13 +191,30 @@ public class LoadIncrementalHFiles exten
try {
queue = discoverLoadQueue(hfofDir);
// outer loop picks up LoadQueueItem due to HFile split
- while (!queue.isEmpty()) {
+ while (!queue.isEmpty() || futuresForSplittingHFile.size() > 0) {
Pair<byte[][],byte[][]> startEndKeys = table.getStartEndKeys();
// inner loop groups callables
while (!queue.isEmpty()) {
LoadQueueItem item = queue.remove();
tryLoad(item, conn, table, queue, startEndKeys, pool);
}
+ Iterator<Future> iter = futuresForSplittingHFile.iterator();
+ while (iter.hasNext()) {
+ boolean timeoutSeen = false;
+ Future future = iter.next();
+ try {
+ future.get(20, TimeUnit.MILLISECONDS);
+ break; // we have at least two new HFiles to process
+ } catch (ExecutionException ee) {
+ LOG.error(ee);
+ } catch (InterruptedException ie) {
+ LOG.error(ie);
+ } catch (TimeoutException te) {
+ timeoutSeen = true;
+ } finally {
+ if (!timeoutSeen) iter.remove();
+ }
+ }
}
for (Future<Void> future : futures) {
try {
@@ -247,8 +267,10 @@ public class LoadIncrementalHFiles exten
// Add these back at the *front* of the queue, so there's a lower
// chance that the region will just split again before we get there.
- queue.addFirst(new LoadQueueItem(item.family, botOut));
- queue.addFirst(new LoadQueueItem(item.family, topOut));
+ synchronized (queue) {
+ queue.addFirst(new LoadQueueItem(item.family, botOut));
+ queue.addFirst(new LoadQueueItem(item.family, topOut));
+ }
LOG.info("Successfully split into new HFiles " + botOut + " and " + topOut);
}
@@ -260,7 +282,8 @@ public class LoadIncrementalHFiles exten
*/
private boolean tryLoad(final LoadQueueItem item,
final HConnection conn, final HTable table,
- final Deque<LoadQueueItem> queue, Pair<byte[][],byte[][]> startEndKeys,
+ final Deque<LoadQueueItem> queue,
+ final Pair<byte[][],byte[][]> startEndKeys,
ExecutorService pool)
throws IOException {
final Path hfilePath = item.hfilePath;
@@ -292,12 +315,21 @@ public class LoadIncrementalHFiles exten
if (idx < 0) {
idx = -(idx+1)-1;
}
+ final int indexForCallable = idx;
boolean lastKeyInRange =
Bytes.compareTo(last, startEndKeys.getSecond()[idx]) < 0 ||
Bytes.equals(startEndKeys.getSecond()[idx], HConstants.EMPTY_BYTE_ARRAY);
if (!lastKeyInRange) {
- splitStoreFileAndRequeue(item, queue, table,
- startEndKeys.getFirst()[idx], startEndKeys.getSecond()[idx]);
+ Callable<Void> callable = new Callable<Void>() {
+ public Void call() throws Exception {
+ splitStoreFileAndRequeue(item, queue, table,
+ startEndKeys.getFirst()[indexForCallable],
+ startEndKeys.getSecond()[indexForCallable]);
+ return (Void)null;
+ }
+ };
+ futuresForSplittingHFile.add(pool.submit(callable));
+
return true;
}