You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2014/08/01 00:08:05 UTC

[29/50] [abbrv] git commit: [HBASE-11233] Asynchronous multiputs

[HBASE-11233] Asynchronous multiputs

Summary:
Idea is to split the multiput pipeline into 2 parts :
1. Before the wall edit.
2. After the wall edit.
This diff attempts to perform actions after the wal edit in an asynchronous manner as a piggy back to the wal edit event. We can configure where the edit actually happens.

Test Plan: Representation diff. Still need to work on the tests. This diff tries to present the idea that I was pursueing. Extension to https://phabricator.fb.com/D1316543

Reviewers: aaiyer, liyintang, daviddeng, gauravm, adela, fan, elliott, arjen, rshroff

Reviewed By: elliott

Subscribers: mycnyc, hbase-eng@

Differential Revision: https://phabricator.fb.com/D1333824

git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@43142 e7acf4d4-3532-417f-9e73-7a9ae25a1f51


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c22acc7f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c22acc7f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c22acc7f

Branch: refs/heads/0.89-fb
Commit: c22acc7fa16028ff2fdcb7fcc9a2ac91221cb22f
Parents: 75ed75d
Author: manukranthk <ma...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Tue Jun 24 23:25:12 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:23 2014 -0700

----------------------------------------------------------------------
 .../hbase/master/TimeToLiveLogCleaner.java      |   5 +-
 .../hadoop/hbase/regionserver/HRegion.java      | 266 +++++++++++++------
 .../hbase/regionserver/HRegionServer.java       | 102 ++++---
 .../MultiVersionConsistencyControl.java         |   3 -
 .../hadoop/hbase/regionserver/wal/HLog.java     | 223 ++++++++++------
 .../hadoop/hbase/HBaseTestingUtility.java       |   4 +
 .../hadoop/hbase/client/TestAsyncMultiputs.java | 185 +++++++++++++
 .../org/apache/hadoop/hbase/client/TestHCM.java |   9 +-
 .../hadoop/hbase/master/TestOldLogsCleaner.java |   5 +-
 .../hadoop/hbase/regionserver/TestHRegion.java  |   9 +-
 .../wal/TestAsyncMultiputsFailingWal.java       |  58 ++++
 .../hadoop/hbase/regionserver/wal/TestHLog.java |  52 +++-
 .../hbase/regionserver/wal/TestHLogSplit.java   |   6 +-
 .../wal/TestLogActionsListener.java             |   2 +-
 .../hbase/regionserver/wal/TestWALReplay.java   |   9 +-
 15 files changed, 708 insertions(+), 230 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java b/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
index cc9e571..fce153e 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/TimeToLiveLogCleaner.java
@@ -48,8 +48,9 @@ public class TimeToLiveLogCleaner implements LogCleanerDelegate {
     long currentTime = System.currentTimeMillis();
     try {
       // If the path name is in hourly format, skip getting modification time
-      if (OldLogsCleaner.isOldLogsArchivedToHourlyDir() && OldLogsCleaner.isMatchDatePattern(filePath)) {
-        time = HLog.DATE_FORMAT.parse(filePath.getName()).getTime();
+      if (OldLogsCleaner.isOldLogsArchivedToHourlyDir()
+          && OldLogsCleaner.isMatchDatePattern(filePath)) {
+        time = HLog.getDateFormat().parse(filePath.getName()).getTime();
       } else {
         FileStatus fStat = filePath.getFileSystem(conf).getFileStatus(filePath);
         time = fStat.getModificationTime();

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index a68f1c3..de8ba0b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -56,6 +56,8 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import javax.annotation.Nullable;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -106,9 +108,14 @@ import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.StringUtils;
 
+import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.AsyncFunction;
+import com.google.common.util.concurrent.CheckedFuture;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
 
 /**
  * HRegion stores data for a certain region of a table.  It stores all columns
@@ -1974,7 +1981,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
         WALEdit walEdit = new WALEdit();
         addFamilyMapToWALEdit(familyMap, walEdit);
         seqNum = this.log.append(regionInfo,
-                regionInfo.getTableDesc().getName(), walEdit, now);
+            regionInfo.getTableDesc().getName(), walEdit, now).checkedGet();
       }
 
       // Now make changes to the memstore.
@@ -2055,7 +2062,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
 
       try {
         // All edits for the given row (across all column families) must happen atomically.
-        put(put.getFamilyMap(), writeToWAL);
+        put(put.getFamilyMap(), writeToWAL).checkedGet();
       } finally {
         if (lockid == null) releaseRowLock(lid);
       }
@@ -2090,14 +2097,28 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
    * Perform a batch put with no pre-specified locks
    * @see HRegion#put(Pair[])
    */
-  public OperationStatusCode[] put(Put[] puts) throws IOException {
+  public CheckedFuture<OperationStatusCode[], IOException> put(Put[] puts) throws IOException {
     @SuppressWarnings("unchecked")
     Pair<Mutation, Integer> putsAndLocks[] = new Pair[puts.length];
 
     for (int i = 0; i < puts.length; i++) {
       putsAndLocks[i] = new Pair<Mutation, Integer>(puts[i], null);
     }
-    return batchMutateWithLocks(putsAndLocks, "multiput_");
+    return Futures.makeChecked(batchMutateWithLocks(putsAndLocks, "multiput_"),
+        getExceptionMapperFunction());
+  }
+
+  public Function<Exception, IOException> getExceptionMapperFunction() {
+    return new Function<Exception, IOException>() {
+      @Override
+      @Nullable
+      public IOException apply(@Nullable Exception e) {
+        if (e instanceof ExecutionException) {
+          return new IOException(e.getCause());
+        }
+        return new IOException(e);
+      }
+    };
   }
 
   /**
@@ -2106,33 +2127,55 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
    * @param methodName "multiput_/multidelete_" to update metrics correctly.
    * @throws IOException
    */
-  public OperationStatusCode[] batchMutateWithLocks(Pair<Mutation, Integer>[] putsAndLocks,
-      String methodName) throws IOException {
+  public ListenableFuture<OperationStatusCode[]> batchMutateWithLocks(
+      Pair<Mutation, Integer>[] putsAndLocks, String methodName)
+          throws IOException {
     BatchOperationInProgress<Pair<Mutation, Integer>> batchOp =
       new BatchOperationInProgress<Pair<Mutation,Integer>>(putsAndLocks);
+    return batchMutateWithLazyRetry(batchOp, methodName);
+  }
 
-    while (!batchOp.isDone()) {
+  public ListenableFuture<OperationStatusCode[]> batchMutateWithLazyRetry(
+      final BatchOperationInProgress<Pair<Mutation, Integer>> batchOp,
+      final String methodName) throws IOException {
+
+    if (!batchOp.isDone()) {
       checkReadOnly();
       checkResources();
 
-      long newSize;
       splitsAndClosesLock.readLock().lock();
       try {
         doPreMutationHook(batchOp);
-        long addedSize = doMiniBatchOp(batchOp, methodName);
-        newSize = this.incMemoryUsage(addedSize);
+        return Futures.transform(doMiniBatchOp(batchOp, methodName),
+            new AsyncFunction<Long, OperationStatusCode[]>() {
+          @Override
+          public ListenableFuture<OperationStatusCode[]> apply(Long addedSize)
+              throws Exception {
+            splitsAndClosesLock.readLock().lock();
+            try {
+              long newSize = incMemoryUsage(addedSize);
+              if (isFlushSize(newSize)) {
+                requestFlush();
+              }
+            } finally {
+              splitsAndClosesLock.readLock().unlock();
+            }
+            if (batchOp.isDone()) {
+              HRegion.writeOps.incrementAndGet();
+              return Futures.immediateFuture(batchOp.retCodes);
+            }
+            return batchMutateWithLazyRetry(batchOp, methodName);
+          }
+        });
+      } catch (Exception e) {
+        return Futures.immediateFuture(batchOp.retCodes);
       } finally {
         splitsAndClosesLock.readLock().unlock();
       }
-      if (isFlushSize(newSize)) {
-        requestFlush();
-      }
     }
-    HRegion.writeOps.incrementAndGet();
-    return batchOp.retCodes;
+    return Futures.immediateFuture(batchOp.retCodes);
   }
 
-
   /**
    * Execute coprocessor hooks for mutations
    * TODO: adela currently can work only with prePut and we need to implement the rest..
@@ -2165,23 +2208,26 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
     }
   }
 
-  private long doMiniBatchOp(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp,
-      String methodNameForMetricsUpdate) throws IOException {
+  private ListenableFuture<Long> doMiniBatchOp(
+      final BatchOperationInProgress <Pair<Mutation, Integer>> batchOp,
+      final String methodNameForMetricsUpdate)
+          throws IOException {
+    boolean preWallEditSuccessfull = false;
     String signature = null;
     // variable to note if all Put items are for the same CF -- metrics related
     boolean isSignatureClear = true;
 
-    long now = EnvironmentEdgeManager.currentTimeMillis();
+    final long now = EnvironmentEdgeManager.currentTimeMillis();
 
     byte[] byteNow = Bytes.toBytes(now);
     boolean locked = false;
 
     /** Keep track of the locks we hold so we can release them in finally clause */
-    List<Integer> acquiredLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
+    final List<Integer> acquiredLocks =
+        Lists.newArrayListWithCapacity(batchOp.operations.length);
     // We try to set up a batch in the range [firstIndex,lastIndexExclusive)
-    int firstIndex = batchOp.nextIndexToProcess;
+    final int firstIndex = batchOp.nextIndexToProcess;
     int lastIndexExclusive = firstIndex;
-    boolean success = false;
     try {
       // ------------------------------------
       // STEP 1. Try to acquire as many locks as we can, and ensure
@@ -2193,7 +2239,8 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
       Integer currentLockID = null;
 
       while (lastIndexExclusive < batchOp.operations.length) {
-        Pair<Mutation, Integer> nextPair = batchOp.operations[lastIndexExclusive];
+        Pair<Mutation, Integer> nextPair =
+            batchOp.operations[lastIndexExclusive];
         Mutation op = nextPair.getFirst();
         Integer providedLockId = nextPair.getSecond();
 
@@ -2244,6 +2291,9 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
           }
         }
       }
+
+      final int finalLastIndexExclusive = lastIndexExclusive;
+
       // We've now grabbed as many puts off the list as we can
       assert numReadyToWrite > 0;
 
@@ -2253,7 +2303,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
       // ------------------------------------
       // STEP 2. Update any LATEST_TIMESTAMP timestamps
       // ----------------------------------
-      for (int i = firstIndex; i < lastIndexExclusive; i++) {
+      for (int i = firstIndex; i < finalLastIndexExclusive; i++) {
         Mutation op = batchOp.operations[i].getFirst();
 
         if (op instanceof Put) {
@@ -2267,13 +2317,14 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
         }
       }
 
-      long seqNum = -1;
+      ListenableFuture<Long> wallEditFuture =
+          Futures.immediateFuture(-1L);
       // ------------------------------------
       // STEP 3. Write to WAL
       // ----------------------------------
       if (!this.disableWAL) {
         WALEdit walEdit = new WALEdit();
-        for (int i = firstIndex; i < lastIndexExclusive; i++) {
+        for (int i = firstIndex; i < finalLastIndexExclusive; i++) {
           // Skip puts that were determined to be invalid during preprocessing
           if (batchOp.retCodes[i] != OperationStatusCode.NOT_RUN) continue;
 
@@ -2283,48 +2334,80 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
         }
 
         // Append the edit to WAL
-        seqNum = this.log.append(regionInfo,
-                regionInfo.getTableDesc().getName(),
-                walEdit, now);
-      }
-
-      // ------------------------------------
-      // STEP 4. Write back to memstore
-      // ----------------------------------
-
-      long addedSize = 0;
-      for (int i = firstIndex; i < lastIndexExclusive; i++) {
-        if (batchOp.retCodes[i] != OperationStatusCode.NOT_RUN) continue;
-
-        Mutation op = batchOp.operations[i].getFirst();
-        addedSize += applyFamilyMapToMemstore(op.getFamilyMap(), seqNum);
-        batchOp.retCodes[i] = OperationStatusCode.SUCCESS;
+        wallEditFuture = this.log.append(regionInfo,
+                  regionInfo.getTableDesc().getName(),
+                  walEdit, now);
       }
-      success = true;
-      return addedSize;
+      final String finalSignature = signature;
+
+      preWallEditSuccessfull = true;
+
+      return Futures.transform(wallEditFuture, new AsyncFunction<Long, Long>() {
+        @Override
+        public ListenableFuture<Long> apply(Long input) throws Exception {
+          boolean memstoreUpdatesSuccessful = false;
+          updatesLock.readLock().lock();
+          try  {
+            // ------------------------------------
+            // STEP 4. Write back to memstore
+            // ----------------------------------
+
+            long addedSize = 0;
+            for (int i = firstIndex; i < finalLastIndexExclusive; i++) {
+              if (batchOp.retCodes[i] != OperationStatusCode.NOT_RUN) continue;
+
+              Mutation op = batchOp.operations[i].getFirst();
+              addedSize += applyFamilyMapToMemstore(op.getFamilyMap(), input);
+              batchOp.retCodes[i] = OperationStatusCode.SUCCESS;
+            }
+            memstoreUpdatesSuccessful = true;
+            return Futures.immediateFuture(addedSize);
+          } finally {
+            releaseLocksAndUpdateMetrics(updatesLock, acquiredLocks,
+                finalSignature, methodNameForMetricsUpdate, firstIndex,
+                finalLastIndexExclusive, batchOp, true,
+                memstoreUpdatesSuccessful, now);
+          }
+        }
+      });
+    } catch (Throwable t) {
+      return Futures.immediateFuture(null);
     } finally {
-      if (locked) {
-        this.updatesLock.readLock().unlock();
-      }
-
-      releaseRowLocks(acquiredLocks);
+      releaseLocksAndUpdateMetrics(updatesLock,
+          Collections.<Integer> emptyList(), signature,
+          methodNameForMetricsUpdate, firstIndex, lastIndexExclusive, batchOp,
+          locked, preWallEditSuccessfull, now);
+    }
+  }
 
-      // do after lock
-      long after = EnvironmentEdgeManager.currentTimeMillis();
-      if (null == signature) {
-        signature = SchemaMetrics.CF_BAD_FAMILY_PREFIX;
-      }
-      HRegion.incrTimeVaryingMetric(signature + methodNameForMetricsUpdate, after - now);
+  /**
+   * This method was intended for code de-duplication and nothing else.
+   */
+  private void releaseLocksAndUpdateMetrics(ReentrantReadWriteLock updatedLock,
+      List<Integer> acquiredLocks, String signature,
+      String methodNameForMetricsUpdate, int firstIndex, int lastIndexExclusive,
+      BatchOperationInProgress<Pair<Mutation, Integer>> batchOp, boolean locked,
+      boolean preWallEditSuccessfull, long now) {
+    if (locked) {
+      this.updatesLock.readLock().unlock();
+    }
+    releaseRowLocks(acquiredLocks);
+    // do after lock
+    long after = EnvironmentEdgeManager.currentTimeMillis();
+    if (null == signature) {
+      signature = SchemaMetrics.CF_BAD_FAMILY_PREFIX;
+    }
+    HRegion.incrTimeVaryingMetric(
+        signature + methodNameForMetricsUpdate, after - now);
 
-      if (!success) {
-        for (int i = firstIndex; i < lastIndexExclusive; i++) {
-          if (batchOp.retCodes[i] == OperationStatusCode.NOT_RUN) {
-            batchOp.retCodes[i] = OperationStatusCode.FAILURE;
-          }
+    if (!preWallEditSuccessfull) {
+      for (int i = firstIndex; i < lastIndexExclusive; i++) {
+        if (batchOp.retCodes[i] == OperationStatusCode.NOT_RUN) {
+          batchOp.retCodes[i] = OperationStatusCode.FAILURE;
         }
       }
-      batchOp.nextIndexToProcess = lastIndexExclusive;
     }
+    batchOp.nextIndexToProcess = lastIndexExclusive;
   }
 
   //TODO, Think that gets/puts and deletes should be refactored a bit so that
@@ -2385,7 +2468,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
         if (matches) {
           // All edits for the given row (across all column families) must happen atomically.
           if (isPut) {
-            put(((Put)w).getFamilyMap(), writeToWAL);
+            put(((Put)w).getFamilyMap(), writeToWAL).checkedGet();
           } else {
             Delete d = (Delete)w;
             prepareDeleteFamilyMap(d);
@@ -2481,7 +2564,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
   throws IOException {
     Map<byte[], List<KeyValue>> familyMap = new HashMap<byte[], List<KeyValue>>();
     familyMap.put(family, edits);
-    this.put(familyMap, true);
+    this.put(familyMap, true).checkedGet();
   }
 
   /**
@@ -2491,11 +2574,12 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
    * @param writeToWAL if true, then we should write to the log
    * @throws IOException
    */
-  private void put(final Map<byte [], List<KeyValue>> familyMap,
+  private CheckedFuture<Void, IOException> put(final Map<byte [], List<KeyValue>> familyMap,
       boolean writeToWAL) throws IOException {
-    long now = EnvironmentEdgeManager.currentTimeMillis();
+    final long now = EnvironmentEdgeManager.currentTimeMillis();
     byte[] byteNow = Bytes.toBytes(now);
-    boolean flush = false;
+    ListenableFuture<Long> wallEditSyncFuture =
+        Futures.immediateFuture(new Long(-1));
     this.updatesLock.readLock().lock();
     try {
       checkFamilies(familyMap.keySet());
@@ -2506,30 +2590,43 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
       // If order is reversed, i.e. we write to memstore first, and
       // for some reason fail to write/sync to commit log, the memstore
       // will contain uncommitted transactions.
-      long seqNum = -1;
       if (!this.disableWAL && writeToWAL) {
         WALEdit walEdit = new WALEdit();
         addFamilyMapToWALEdit(familyMap, walEdit);
-        seqNum = this.log.append(regionInfo,
+        wallEditSyncFuture = this.log.append(regionInfo,
                 regionInfo.getTableDesc().getName(), walEdit, now);
       }
+      return Futures.makeChecked(Futures.transform(wallEditSyncFuture,
+          new AsyncFunction<Long, Void>() {
+            @Override
+            public ListenableFuture<Void> apply(Long input) throws Exception {
+              boolean flush = false;
+              updatesLock.readLock().lock();
+              try {
+                long addedSize = applyFamilyMapToMemstore(familyMap, input);
+                flush = isFlushSize(incMemoryUsage(addedSize));
+              } finally {
+                updatesLock.readLock().unlock();
+              }
 
-      long addedSize = applyFamilyMapToMemstore(familyMap, seqNum);
-      flush = isFlushSize(this.incMemoryUsage(addedSize));
+              // do after lock
+              long after = EnvironmentEdgeManager.currentTimeMillis();
+              String signature = SchemaMetrics.generateSchemaMetricsPrefix(
+                  getTableDesc().getNameAsString(), familyMap.keySet());
+              HRegion.incrTimeVaryingMetric(signature + "put_", after - now);
+
+              if (flush) {
+                // Request a cache flush. Do it outside update lock.
+                requestFlush();
+              }
+              return Futures.immediateFuture(null);
+            }
+          }), this.getExceptionMapperFunction());
+    } catch (Exception e) {
+      return Futures.immediateFailedCheckedFuture(new IOException(e));
     } finally {
       this.updatesLock.readLock().unlock();
     }
-
-    // do after lock
-    long after = EnvironmentEdgeManager.currentTimeMillis();
-    String signature = SchemaMetrics.generateSchemaMetricsPrefix(
-        this.getTableDesc().getNameAsString(), familyMap.keySet());
-    HRegion.incrTimeVaryingMetric(signature + "put_", after - now);
-
-    if (flush) {
-      // Request a cache flush.  Do it outside update lock.
-      requestFlush();
-    }
   }
 
   /**
@@ -3788,8 +3885,9 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
 
         // 6. append/sync all edits at once
         // TODO: Do batching as in doMiniBatchPut
-        long seqNum = this.log.append(regionInfo, this.getTableDesc().getName(),
-                walEdit, now);
+        long seqNum;
+        seqNum = this.log.append(regionInfo, this.getTableDesc().getName(),
+            walEdit, now).checkedGet();
 
         // 7. apply to memstore
         long addedSize = 0;
@@ -3875,7 +3973,7 @@ public class HRegion implements HeapSize, ConfigurationObserver, HRegionIf {
         WALEdit walEdit = new WALEdit();
         walEdit.add(newKv);
         seqNum = this.log.append(regionInfo, regionInfo.getTableDesc().getName(),
-          walEdit, now);
+            walEdit, now).checkedGet();
       }
 
       // Now request the ICV to the store, this will set the timestamp

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 7a5911b..3f3947d 100755
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -64,6 +64,8 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import javax.annotation.Nullable;
+
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.Options;
@@ -174,7 +176,12 @@ import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.Watcher.Event.EventType;
 
+import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.AsyncFunction;
+import com.google.common.util.concurrent.CheckedFuture;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
 
 /**
  * HRegionServer makes a set of HRegions available to clients.  It checks in with
@@ -2064,6 +2071,13 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
     return this.hlogs[i];
   }
 
+  /**
+   * Return number of logs present in the regionserver
+   */
+  public int getLogCount() {
+    return this.hlogs.length;
+  }
+
   public int getTotalHLogCnt() {
     return this.hlogs.length;
   }
@@ -2917,13 +2931,18 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
     }
   }
 
-  @Override
-  public int put(final byte[] regionName, final List<Put> puts)
-  throws IOException {
+  public CheckedFuture<Integer, IOException> putAsync(final byte[] regionName,
+      final List<Put> puts) throws IOException {
     return applyMutations(regionName, puts, "multiput_");
   }
 
-  private int applyMutations(final byte[] regionName,
+  @Override
+  public int put(final byte[] regionName,
+      final List<Put> puts) throws IOException {
+    return applyMutations(regionName, puts, "multiput_").checkedGet();
+  }
+
+  private CheckedFuture<Integer, IOException> applyMutations(final byte[] regionName,
       final List<? extends Mutation> mutations,
       String methodName)
   throws IOException {
@@ -2944,18 +2963,45 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
         opWithLocks[i++] = new Pair<Mutation, Integer>(p, lock);
       }
 
-      OperationStatusCode[] codes = region.batchMutateWithLocks(opWithLocks,
+      ListenableFuture<OperationStatusCode[]> batchMutateFuture =
+          region.batchMutateWithLocks(opWithLocks,
           methodName);
-      for (i = 0; i < codes.length; i++) {
-        if (codes[i] != OperationStatusCode.SUCCESS)
-          return i;
-      }
-      return HConstants.MULTIPUT_SUCCESS;
+      return Futures.makeChecked(Futures.transform(batchMutateFuture,
+            new AsyncFunction<OperationStatusCode[], Integer>() {
+
+          @Override
+          public ListenableFuture<Integer> apply(OperationStatusCode[] codes)
+              throws Exception {
+            try {
+              for (int i = 0; i < codes.length; i++) {
+                if (codes[i] != OperationStatusCode.SUCCESS)
+                  return Futures.immediateFuture(i);
+              }
+              return Futures.immediateFuture(HConstants.MULTIPUT_SUCCESS);
+            } catch (Throwable t) {
+              return Futures.immediateFailedCheckedFuture(
+                  convertThrowableToIOE(cleanup(t)));
+            }
+          }
+        }), getConvertThrowableToIOEFunction());
     } catch (Throwable t) {
       throw convertThrowableToIOE(cleanup(t));
     }
   }
 
+  private Function<Exception, IOException> getConvertThrowableToIOEFunction() {
+    return new Function<Exception, IOException>() {
+      @Override
+      @Nullable
+      public IOException apply(@Nullable Exception e) {
+        if (e instanceof ExecutionException) {
+          return convertThrowableToIOE(cleanup(e.getCause()));
+        }
+        return convertThrowableToIOE(cleanup(e));
+      }
+    };
+  }
+
   private boolean checkAndMutate(final byte[] regionName, final byte [] row,
       final byte [] family, final byte [] qualifier, final byte [] value,
       final Writable w, Integer lock) throws IOException {
@@ -3218,7 +3264,7 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
   public int delete(final byte[] regionName,
                                   final List<Delete> deletes)
   throws IOException {
-    return applyMutations(regionName, deletes, "multidelete_");
+    return applyMutations(regionName, deletes, "multidelete_").checkedGet();
   }
 
   @Override
@@ -3867,32 +3913,18 @@ public class HRegionServer implements HRegionServerIf, HBaseRPCErrorHandler,
   public MultiPutResponse multiPut(MultiPut puts) throws IOException {
     MultiPutResponse resp = new MultiPutResponse();
 
-    // do each region as it's own.
-    int size = puts.puts.size();
-    int index = 0;
+    Map<byte[], CheckedFuture<Integer, IOException>> futuresMap =
+        new TreeMap<>(Bytes.BYTES_COMPARATOR);
     for( Map.Entry<byte[], List<Put>> e: puts.puts.entrySet()) {
-      int result = put(e.getKey(), e.getValue());
-      resp.addResult(e.getKey(), result);
-
-      index++;
-      if (index < size) {
-        // remove the reference to the region list of Puts to save RAM except
-        // for the last one. We will lose the reference to the last one pretty
-        // soon anyway; keep it for a little more, until we get back
-        // to HBaseServer level, where we might need to pretty print
-        // the MultiPut request for debugging slow/large puts.
-        // Note: A single row "put" from client also end up in server
-        // as a multiPut().
-        // We set the value to an empty array so taskmonitor can either get
-        //  the old value or an empty array. If we call clear() on the array,
-        //  then we might have a race condition where we're iterating over the
-        //  array at the same time we clear it (which throws an exception)
-        // This relies on the fact that Map.Entry.setValue() boils down to a
-        //   simple reference assignment, which is atomic
-        e.setValue(emptyPutArray); // clear some RAM
-      }
+      futuresMap.put(e.getKey(), putAsync(e.getKey(), e.getValue()));
+    }
+    for (Entry<byte[], CheckedFuture<Integer, IOException>> e :
+        futuresMap.entrySet()) {
+      CheckedFuture<Integer, IOException> f = e.getValue();
+      byte[] row = e.getKey();
+      int ret = f.checkedGet();
+      resp.addResult(row, ret);
     }
-
     return resp;
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java b/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
index 6b28f03..53e9757 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/MultiVersionConsistencyControl.java
@@ -24,9 +24,6 @@ import java.util.LinkedList;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.Log;
-
 /**
  * Manages the read/write consistency within memstore. This provides
  * an interface for readers to determine what entries to ignore, and

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
index 044fef4..7e2036e 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
@@ -63,6 +63,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import javax.annotation.Nullable;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -80,13 +82,10 @@ import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.Stoppable;
-import org.apache.hadoop.hbase.ipc.HBaseServer.Call;
-import org.apache.hadoop.hbase.ipc.ProfilingData;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionSeqidTransition;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -99,7 +98,13 @@ import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.util.StringUtils;
 
+import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.AsyncFunction;
+import com.google.common.util.concurrent.CheckedFuture;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 /**
@@ -146,7 +151,9 @@ public class HLog implements Syncable {
   public static final byte [] METAFAMILY = Bytes.toBytes("METAFAMILY");
   public static final byte [] METAROW = Bytes.toBytes("METAROW");
   public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;
-  public static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd-HH");
+  public static SimpleDateFormat getDateFormat() {
+    return new SimpleDateFormat("yyyy-MM-dd-HH");
+  }
 
   /*
    * Name of directory that holds recovered edits written by the wal log
@@ -181,8 +188,8 @@ public class HLog implements Syncable {
       Collections.synchronizedList(new ArrayList<LogActionsListener>());
 
 
-  static Class<? extends Writer> logWriterClass;
-  static Class<? extends Reader> logReaderClass;
+  volatile static Class<? extends Writer> logWriterClass;
+  volatile static Class<? extends Reader> logReaderClass;
 
   private OutputStream hdfs_out;     // OutputStream associated with the current SequenceFile.writer
   private int initialReplication;    // initial replication factor of SequenceFile.writer
@@ -284,9 +291,6 @@ public class HLog implements Syncable {
   private final List<LogEntryVisitor> logEntryVisitors =
       new CopyOnWriteArrayList<LogEntryVisitor>();
 
-  private volatile long lastLogRollStartTimeMillis = 0;
-  private volatile long lastLogRollDurationMillis = 0;
-
   /**
    * Pattern used to validate a HLog file name
    */
@@ -305,9 +309,9 @@ public class HLog implements Syncable {
       synchronized (this) {
         min = Math.min(min, val);
         max = Math.max(max, val);
+        total += val;
+        ++count;
       }
-      total += val;
-      ++count;
     }
 
     Metric get() {
@@ -359,14 +363,17 @@ public class HLog implements Syncable {
    */
   private class DoubleListBuffer {
     private LinkedList<Entry> currentList = new LinkedList<Entry>();
+    private SettableFuture<Void> currentFuture = SettableFuture.create();;
     private LinkedList<Entry> syncList = new LinkedList<Entry>();
 
     /**
      * Append a log entry into the buffer
      * @param entry log entry
      */
-    synchronized private void appendToBuffer(Entry entry) {
+    synchronized private ListenableFuture<Void> appendToBuffer(Entry entry) {
+      Preconditions.checkNotNull(this.currentFuture);
       currentList.add(entry);
+      return this.currentFuture;
     }
 
     /**
@@ -375,6 +382,8 @@ public class HLog implements Syncable {
      * @return number of log entries synced
      */
     private int appendAndSync() throws IOException {
+      // The future returned to the append operation.
+      SettableFuture<Void> future;
       synchronized (this) {
         if (currentList.isEmpty()) { // no thing to sync
           return 0;
@@ -385,26 +394,49 @@ public class HLog implements Syncable {
         LinkedList<Entry> tmp = syncList;
         syncList = currentList;
         currentList = tmp;
+        future = currentFuture;
+        this.currentFuture = SettableFuture.create();
       }
 
-      // append entries to writer
-      int syncedEntries = syncList.size();
-      while (!syncList.isEmpty()) {
-        Entry entry = syncList.remove();
-        writer.append(entry);
-      }
+      try {
+        // append entries to writer
+        int syncedEntries = syncList.size();
+        while (!syncList.isEmpty()) {
+          Entry entry = syncList.remove();
+          writer.append(entry);
+        }
 
-      // sync the data
-      long now = System.currentTimeMillis();
-      writer.sync();
-      syncTime.inc(System.currentTimeMillis() - now);
-      return syncedEntries;
+        // sync the data
+        long now = System.currentTimeMillis();
+        writer.sync();
+        syncTime.inc(System.currentTimeMillis() - now);
+        future.set(null);
+        return syncedEntries;
+      } catch (Exception e) {
+        future.setException(e);
+        return 0;
+      }
     }
   }
 
   private DoubleListBuffer logBuffer = new DoubleListBuffer();
 
   /**
+   * Minimal constructor to get going in tests
+   */
+  public HLog(final Configuration conf, Path basePath) throws IOException {
+    this(FileSystem.get(conf), new Path(basePath, ".logs"),
+        new Path(basePath, ".oldlogs"), conf, null);
+  }
+
+  /**
+   * Minimal constructor to get going in tests
+   */
+  public HLog(final Configuration conf) throws IOException {
+    this(conf, new Path("/tmp"));
+  }
+
+  /**
    * HLog creating with a null actions listener.
    *
    * @param fs filesystem handle
@@ -651,8 +683,6 @@ public class HLog implements Syncable {
         this.numEntries.set(0);
 
         t1 = EnvironmentEdgeManager.currentTimeMillis();
-        lastLogRollStartTimeMillis = t0;
-        lastLogRollDurationMillis = (t1 - t0);
       }
 
       Path oldFile = null;
@@ -699,7 +729,7 @@ public class HLog implements Syncable {
   throws IOException {
     try {
       if (logReaderClass == null) {
-        logReaderClass =conf.getClass("hbase.regionserver.hlog.reader.impl",
+        logReaderClass = conf.getClass("hbase.regionserver.hlog.reader.impl",
                 SequenceFileLogReader.class, Reader.class);
       }
 
@@ -708,8 +738,7 @@ public class HLog implements Syncable {
       return reader;
     } catch (IOException e) {
       throw e;
-    }
-    catch (Exception e) {
+    } catch (Exception e) {
       throw new IOException("Cannot get log reader", e);
     }
   }
@@ -1052,10 +1081,10 @@ public class HLog implements Syncable {
    *          of the memstore.
    * @throws IOException
    */
-  public long append(HRegionInfo info, byte [] tableName, WALEdit edits,
+  public CheckedFuture<Long, IOException> append(HRegionInfo info, byte[] tableName, WALEdit edits,
     final long now) throws IOException {
     if (!this.enabled || edits.isEmpty()) {
-      return -1L;
+      return Futures.immediateCheckedFuture(-1L);
     }
     if (logSyncerThread.syncerShuttingDown) {
       // can't acquire lock for the duration of append()
@@ -1063,13 +1092,13 @@ public class HLog implements Syncable {
       throw new IOException("Cannot append; logSyncer shutting down");
     }
 
-    long len = edits.getTotalKeyValueLength();
-    long txid = 0;
-    long seqNum;
+    final long len = edits.getTotalKeyValueLength();
+    final long txid;
+    final long seqNum;
 
-    long start = System.currentTimeMillis();
     byte[] regionName = info.getRegionName();
 
+    ListenableFuture<Void> logSyncFuture;
     synchronized (this.appendLock) {
       // The 'lastSeqWritten' map holds the sequence number of the oldest
       // write for each region (i.e. the first edit added to the particular
@@ -1080,51 +1109,54 @@ public class HLog implements Syncable {
       this.firstSeqWrittenInCurrentMemstore.putIfAbsent(regionName, seqNum);
       HLogKey logKey = makeKey(regionName, tableName, seqNum, now);
 
-      doWrite(info, logKey, edits);
+      logSyncFuture = doWrite(info, logKey, edits);
       // Only count 1 row as an unflushed entry.
       txid = this.unflushedEntries.incrementAndGet();
     }
+    if (info.isMetaRegion()) forceSync();
+    final long start = System.currentTimeMillis();
 
-    // Update the metrics
-    this.numEntries.incrementAndGet();
-    writeSize.inc(len);
-
-    // sync txn to file system
-    start = System.nanoTime();
-    this.sync(info.isMetaRegion(), txid);
-
-    // Update the metrics and log down the outliers
-    long end = System.nanoTime();
-    long syncTime = end - start;
-    gsyncTime.inc(syncTime);
-    if (syncTime > SECOND_IN_NS) {
-      LOG.warn(String.format(
-        "%s took %d ns appending an edit to hlog; editcount=%d, len~=%s",
-        Thread.currentThread().getName(), syncTime, this.numEntries.get(),
-        StringUtils.humanReadableInt(len)));
-      slowSyncs.incrementAndGet();
-    }
-
-    if (slowSyncs.get() >= slowBeforeRoll) {
-      this.requestLogRoll();
-      this.slowSyncs.set(0);
-    }
+    return Futures.makeChecked(Futures.transform(logSyncFuture,
+      new AsyncFunction<Void, Long>() {
+        @Override
+        public ListenableFuture<Long> apply(Void input) throws Exception {
+          Preconditions.checkArgument(txid >= getCurSyncPoint());
+          // Update the metrics
+          numEntries.incrementAndGet();
+          writeSize.inc(len);
+
+          // Update the metrics and log down the outliers
+          long end = System.currentTimeMillis();
+          long syncTime = end - start;
+          gsyncTime.inc(syncTime);
+
+          if (syncTime > SECOND_IN_NS) {
+            LOG.warn(String.format(
+              "%s took %d ns appending an edit to hlog; editcount=%d, len~=%s",
+              Thread.currentThread().getName(), syncTime, numEntries.get(),
+              StringUtils.humanReadableInt(len)));
+            slowSyncs.incrementAndGet();
+          }
+          if (slowSyncs.get() >= slowBeforeRoll) {
+            requestLogRoll();
+            slowSyncs.set(0);
+          }
+          return Futures.immediateFuture(new Long(seqNum));
+        }
+      }), this.getExceptionMapperFunction());
+  }
 
-    // Update the per-request profiling data
-    Call call = HRegionServer.callContext.get();
-    ProfilingData pData = call == null ? null : call.getProfilingData();
-    if (pData != null) {
-      if (this.lastLogRollStartTimeMillis > start
-          && end > this.lastLogRollStartTimeMillis) {
-        // We also had a log roll in between
-        pData.addLong(ProfilingData.HLOG_ROLL_TIME_MS, this.lastLogRollDurationMillis);
-        // Do not account for this as the sync time.
-        syncTime = syncTime - this.lastLogRollDurationMillis;
+  public Function<Exception, IOException> getExceptionMapperFunction() {
+    return new Function<Exception, IOException>() {
+      @Override
+      @Nullable
+      public IOException apply(@Nullable Exception e) {
+        if (e instanceof ExecutionException) {
+          return new IOException(e.getCause());
+        }
+        return new IOException(e);
       }
-      // update sync time
-      pData.addLong(ProfilingData.HLOG_SYNC_TIME_MS, syncTime);
-    }
-    return seqNum;
+    };
   }
 
   /**
@@ -1167,7 +1199,7 @@ public class HLog implements Syncable {
           forceSync = false;
           // wake up every 100ms to check if logsyncer has to shut down
           queueEmpty.awaitNanos(100*1000000);
-          if (unflushedEntries.get() == syncTillHere) {
+          if (unflushedEntries.get() >= syncTillHere) {
             // call hflush() if we haven't flushed for a while now
             // This force-sync is just a safety mechanism - we being
             // paranoid. If there hasn't been any syncing activity for
@@ -1210,6 +1242,18 @@ public class HLog implements Syncable {
     }
 
     /**
+     * This method needs to be called to indicate that we need to force the
+     * log syncer thread to sync.
+     * @param force
+     */
+    public void setForceSync(boolean force) {
+      forceSync = false;
+    }
+
+    public long getCurSyncPoint() {
+      return syncTillHere;
+    }
+    /**
      * This method first signals the thread that there's a sync needed
      * and then waits for it to happen before returning.
      * @throws IOException
@@ -1276,6 +1320,14 @@ public class HLog implements Syncable {
     logSyncerThread.addToSyncQueue(force, txid);
   }
 
+  public void forceSync() {
+    logSyncerThread.setForceSync(true);
+  }
+
+  public long getCurSyncPoint() {
+    return logSyncerThread.getCurSyncPoint();
+  }
+
   private void hflush() {
     synchronized (this.updateLock) {
       if (this.closed) {
@@ -1379,9 +1431,9 @@ public class HLog implements Syncable {
     }
   }
 
-  protected void doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit)
+  protected ListenableFuture<Void> doWrite(HRegionInfo info, HLogKey logKey, WALEdit logEdit)
   throws IOException {
-    this.logBuffer.appendToBuffer(new Entry(logKey, logEdit));
+    return this.logBuffer.appendToBuffer(new Entry(logKey, logEdit));
   }
 
   /** @return How many items have been added to the log */
@@ -1890,7 +1942,8 @@ public class HLog implements Syncable {
     String subDirectoryName;
     if (ARCHIVE_TO_HOURLY_DIR) {
       // Group into hourly sub-directory
-      subDirectoryName = DATE_FORMAT.format(Calendar.getInstance().getTime());
+      subDirectoryName =
+          getDateFormat().format(Calendar.getInstance().getTime());
     } else {
       // since the filename is a valid name, we know there
       // is a last '.' (won't return -1)
@@ -2388,11 +2441,25 @@ public class HLog implements Syncable {
     walEdit.add(kvLast);
     walEdit.add(kvNext);
     walEdit.add(kvRegionInfo);
-    long newSeqid = append(regionInfo, regionInfo.getTableDesc().getName(), walEdit, now);
+    long newSeqid;
+    try {
+      newSeqid = append(regionInfo, regionInfo.getTableDesc()
+              .getName(), walEdit, now).get();
+    } catch (InterruptedException | ExecutionException e) {
+      throw new IOException(e);
+    }
     LOG.info("Region " + regionInfo.getRegionNameAsString() +
                " sequence id transition " +
                seqidTransition.getLastSeqid() + "->" +
                seqidTransition.getNextSeqid() + " appended to HLog at seqid=" + newSeqid);
     return true;
   }
+
+  /**
+   * Returns the underlying configuration object.
+   * @return
+   */
+  public Configuration getConfiguration() {
+    return this.conf;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
index 649512f..a237776 100644
--- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
+++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
@@ -2086,4 +2086,8 @@ public class HBaseTestingUtility {
     getHBaseAdmin().moveRegion(regionInfo.getRegionName(),
         regionServer.toString());
   }
+
+  public HConnection getConnection() {
+    return HConnectionManager.getConnection(this.conf);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java b/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java
new file mode 100644
index 0000000..3b0ef2b
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestAsyncMultiputs.java
@@ -0,0 +1,185 @@
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.StringBytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestAsyncMultiputs {
+  private static final Log LOG = LogFactory.getLog(TestAsyncMultiputs.class);
+  private static final HBaseTestingUtility util = new HBaseTestingUtility();
+  private static int NUM_REGIONSERVERS = 1;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    util.startMiniCluster(NUM_REGIONSERVERS);
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    util.shutdownMiniCluster();
+  }
+
+  /**
+   * Test intended to check if the wal gives the control
+   * back when writers are calling append.
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testMultithreadedCallbackFromHLog() throws IOException, InterruptedException {
+    final HLog log = new HLog(util.getConfiguration(),
+        new Path("/tmp/testMultithreadedCallback"));
+    int numThreads = 100;
+    ExecutorService executor = Executors.newCachedThreadPool();
+    for (int i = 0; i < numThreads; i++) {
+      final int threadId = i;
+      executor.submit(new Callable<Void>() {
+        @Override
+        public Void call() throws Exception {
+          HTableDescriptor desc =
+              new HTableDescriptor(Bytes.toBytes("table" + threadId));
+          HRegionInfo info = new HRegionInfo(desc, new byte[0], new byte[0]);
+          WALEdit edits = new WALEdit();
+          for (byte[] row : getRandomRows(1, 10)) {
+            edits.add(new KeyValue(row));
+          }
+          try {
+            log.append(info, desc.getName(), edits,
+                System.currentTimeMillis()).get(1, TimeUnit.MINUTES);
+          } catch(Exception e) {
+            fail();
+          }
+          return null;
+        }
+      });
+    }
+    executor.shutdown();
+    executor.awaitTermination(1, TimeUnit.MINUTES);
+  }
+
+  @Test
+  public void testMultiputSucceeds() throws IOException {
+    String tableName = "testMultiputSucceeds";
+    byte[] tableNameBytes = Bytes.toBytes(tableName);
+    String cf = "cf";
+    byte[][] families = new byte[][] { Bytes.toBytes(cf) };
+    LOG.debug("Creating the table " + tableName);
+    int numRegions = 100;
+    HRegionServer server = createTableAndGiveMeTheRegionServer(tableNameBytes,
+        families, numRegions);
+    MultiPut multiput = getRandomMultiPut(tableNameBytes, 1000, families[0],
+        server.getServerInfo().getServerAddress());
+    server.multiPut(multiput);
+    for (HRegion region : server.getOnlineRegionsAsArray()) {
+      server.flushRegion(region.getRegionName());
+    }
+    for (Entry<byte[], List<Put>> entry : multiput.getPuts().entrySet()) {
+      byte[] regionName = entry.getKey();
+      for (Put p : entry.getValue()) {
+        Get g = new Get(p.getRow());
+        Result r = server.get(regionName, g);
+        assertNotNull(g);
+        assertTrue(Bytes.equals(r.getValue(families[0], null), p.getRow()));
+        assertTrue(Bytes.equals(r.getRow(), p.getRow()));
+      }
+    }
+  }
+
+  public HRegionServer createTableAndGiveMeTheRegionServer(
+      byte[] tableName, byte[][] families, int numregions) throws IOException {
+    LOG.debug("Creating table " + Bytes.toString(tableName) + " with "
+        + numregions + " regions");
+    util.createTable(tableName, families, 1, Bytes.toBytes("aaa"),
+        Bytes.toBytes("zzz"), numregions);
+    assertTrue(NUM_REGIONSERVERS == 1);
+    HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
+    LOG.debug("Found the regionserver. Returning" + rs);
+    return rs;
+  }
+
+  public static MultiPut getRandomMultiPut(byte[] tableName, int numPuts, byte[] cf,
+      HServerAddress addr) throws IOException {
+    return getRandomMultiPut(tableName, numPuts, cf, addr, util);
+  }
+
+  /**
+   * Creates random puts with rows between aaa.. to zzz..
+   * @param tableName
+   * @param numPuts
+   * @param cf
+   * @param addr
+   * @param util : The hbase testing utility to use.
+   * @return
+   * @throws IOException
+   */
+  public static MultiPut getRandomMultiPut(byte[] tableName, int numPuts, byte[] cf,
+      HServerAddress addr, HBaseTestingUtility util) throws IOException {
+    MultiPut multiput = new MultiPut(addr);
+    List<Put> puts = getRandomPuts(numPuts, cf);
+    LOG.debug("Creating " + numPuts + " puts to insert into the multiput");
+    HConnection conn = util.getConnection();
+    for (Put put : puts) {
+      byte[] regionName = conn.getRegionLocation(new StringBytes(tableName),
+          put.getRow(), false).getRegionInfo().getRegionName();
+      multiput.add(regionName, put);
+    }
+    return multiput;
+  }
+
+  /**
+   * Creates puts with rows between aaa... and zzz..
+   * @param numPuts
+   * @param cf
+   * @return
+   */
+  public static List<Put> getRandomPuts(int numPuts, byte[] cf) {
+    List<Put> ret = new ArrayList<Put> (numPuts);
+    List<byte[]> rows = getRandomRows(numPuts, 3);
+    for (byte[] row : rows) {
+      Put p = new Put(row);
+      p.add(cf, null, row);
+      ret.add(p);
+    }
+    return ret;
+  }
+
+  public static List<byte[]> getRandomRows(int numRows, int rowlen) {
+    Random rand = new Random();
+    List<byte[]> ret = new ArrayList<byte[]>(numRows);
+    for (int i = 0; i < numRows; i++) {
+      byte[] row = new byte[rowlen];
+      for (int j = 0; j < rowlen; j++) {
+        row[j] = (byte) ('a' + (byte) rand.nextInt('z' - 'a'));
+      }
+      ret.add(row);
+    }
+    return ret;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index f22f256..3f0c5bb 100644
--- a/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -50,6 +50,9 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
 /**
  * This class is for testing HCM features
  */
@@ -275,9 +278,11 @@ public class TestHCM {
     }
 
     @Override
-    public OperationStatusCode[] batchMutateWithLocks(Pair<Mutation, Integer>[] putsAndLocks,
+    public ListenableFuture<OperationStatusCode[]> batchMutateWithLocks(
+        Pair<Mutation, Integer>[] putsAndLocks,
         String methodName) throws IOException {
-      throw new IOException("Test induced failure");
+      return Futures.immediateFailedFuture(
+          new IOException("Test induced failure"));
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java b/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
index 42b7880..758ae4e 100644
--- a/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
+++ b/src/test/java/org/apache/hadoop/hbase/master/TestOldLogsCleaner.java
@@ -144,10 +144,11 @@ public class TestOldLogsCleaner {
     fs.mkdirs(legacyDir);
     fs.createNewFile(new Path(legacyDir, "123.456"));
     Calendar cal = Calendar.getInstance();
-    System.out.println("Now is: " + HLog.DATE_FORMAT.format(cal.getTime()));
+    System.out.println("Now is: " + HLog.getDateFormat().format(cal.getTime()));
     for (int i = 0; i < 10; i++) {
       cal.add(Calendar.HOUR, -1);
-      Path hourDir = new Path(oldLogDir, HLog.DATE_FORMAT.format(cal.getTime()));
+      Path hourDir = new Path(oldLogDir,
+          HLog.getDateFormat().format(cal.getTime()));
       fs.mkdirs(hourDir);
       fs.createNewFile(new Path(hourDir, new Path(fakeMachineName + "." + i)));
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 437c19c..cb4a9ec 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -372,7 +372,7 @@ public class TestHRegion extends HBaseTestCase {
       puts[i].add(cf, qual, val);
     }
 
-    OperationStatusCode[] codes = this.region.put(puts);
+    OperationStatusCode[] codes = this.region.put(puts).get();
     assertEquals(10, codes.length);
     for (int i = 0; i < 10; i++) {
       assertEquals(OperationStatusCode.SUCCESS, codes[i]);
@@ -382,7 +382,7 @@ public class TestHRegion extends HBaseTestCase {
     LOG.info("Next a batch put with one invalid family");
     puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
     try {
-      codes = this.region.put(puts);
+      codes = this.region.put(puts).get();
 
       // put should throw an exception
       assertTrue(false);
@@ -403,7 +403,7 @@ public class TestHRegion extends HBaseTestCase {
     TestThread putter = new TestThread(ctx) {
       @Override
       public void doWork() throws IOException {
-        retFromThread.set(region.put(puts));
+        retFromThread.set(region.put(puts).checkedGet());
       }
     };
     LOG.info("...starting put thread while holding lock");
@@ -439,7 +439,8 @@ public class TestHRegion extends HBaseTestCase {
       putsAndLocks.add(pair);
     }
 
-    codes = region.batchMutateWithLocks(putsAndLocks.toArray(new Pair[0]), "multiput_");
+    codes = region.batchMutateWithLocks(putsAndLocks.toArray(
+        new Pair[0]), "multiput_").get();
     LOG.info("...performed put");
     for (int i = 0; i < 10; i++) {
       assertEquals(OperationStatusCode.SUCCESS, codes[i]);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java
new file mode 100644
index 0000000..20150c0
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncMultiputsFailingWal.java
@@ -0,0 +1,58 @@
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import static org.junit.Assert.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.client.MultiPut;
+import org.apache.hadoop.hbase.client.TestAsyncMultiputs;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.wal.TestHLog.FailingSequenceFileLogWriter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+
+public class TestAsyncMultiputsFailingWal {
+  private final Log LOG = LogFactory.getLog(TestAsyncMultiputsFailingWal.class);
+
+  @Test
+  public void testFailedMultiput() throws Exception {
+    String tableName = "testFailedMultiput";
+    byte[] tableNameBytes = Bytes.toBytes(tableName);
+    String cf = "cf";
+    byte[][] families = new byte[][] { Bytes.toBytes(cf) };
+    LOG.debug("Creating the table " + tableName);
+    int numRegions = 3;
+    HBaseTestingUtility localutil = new HBaseTestingUtility();
+    localutil.startMiniCluster();
+
+    LOG.debug("Creating table " + tableName + " with "
+        + numRegions + " regions");
+    localutil.createTable(tableNameBytes, families, 1, Bytes.toBytes("aaa"),
+        Bytes.toBytes("zzz"), numRegions);
+    HRegionServer rs = localutil.getRSForFirstRegionInTable(tableNameBytes);
+    LOG.debug("Found the regionserver. Returning" + rs);
+
+    HLog.logWriterClass = null;
+    assertTrue(rs.getLogCount() == 1);
+    HLog log = rs.getLog(0);
+    log.getConfiguration().set("hbase.regionserver.hlog.writer.impl",
+            FailingSequenceFileLogWriter.class.getName());
+    log.rollWriter();
+    MultiPut multiput = TestAsyncMultiputs.getRandomMultiPut(
+        tableNameBytes, 1000, families[0],
+        rs.getServerInfo().getServerAddress(), localutil);
+
+    try {
+      FailingSequenceFileLogWriter.setFailureMode(true);
+      rs.multiPut(multiput);
+      FailingSequenceFileLogWriter.setFailureMode(false);
+    } catch (Exception e) {
+      LOG.debug("Excepted to see an exception", e);
+      return;
+    }
+    fail();
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
index b7d644e..59e666a 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLog.java
@@ -29,7 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
-
+import java.util.concurrent.ExecutionException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
@@ -139,9 +139,12 @@ public class TestHLog  {
    * Just write multiple logs then split.  Before fix for HADOOP-2283, this
    * would fail.
    * @throws IOException
+   * @throws ExecutionException
+   * @throws InterruptedException
    */
   @Test
-  public void testSplit() throws IOException {
+  public void testSplit() throws IOException,
+      InterruptedException, ExecutionException {
 
     final byte [] tableName = Bytes.toBytes(TestHLog.class.getSimpleName());
     final byte [] rowName = tableName;
@@ -167,7 +170,7 @@ public class TestHLog  {
                 System.currentTimeMillis(), column));
             LOG.info("Region " + i + ": " + edit);
             log.append(infos[i], tableName, edit,
-              System.currentTimeMillis());
+              System.currentTimeMillis()).get();
           }
         }
         log.rollWriter();
@@ -216,7 +219,7 @@ public class TestHLog  {
     for (int i = 0; i < total; i++) {
       WALEdit kvs = new WALEdit();
       kvs.add(new KeyValue(Bytes.toBytes(i), bytes, bytes));
-      wal.append(info, bytes, kvs, System.currentTimeMillis());
+      wal.append(info, bytes, kvs, System.currentTimeMillis()).get();
     }
     // Now call sync and try reading.  Opening a Reader before you sync just
     // gives you EOFE.
@@ -234,7 +237,7 @@ public class TestHLog  {
     for (int i = 0; i < total; i++) {
       WALEdit kvs = new WALEdit();
       kvs.add(new KeyValue(Bytes.toBytes(i), bytes, bytes));
-      wal.append(info, bytes, kvs, System.currentTimeMillis());
+      wal.append(info, bytes, kvs, System.currentTimeMillis()).get();
     }
     reader = HLog.getReader(fs, walPath, conf);
     count = 0;
@@ -253,7 +256,7 @@ public class TestHLog  {
     for (int i = 0; i < total; i++) {
       WALEdit kvs = new WALEdit();
       kvs.add(new KeyValue(Bytes.toBytes(i), bytes, value));
-      wal.append(info, bytes, kvs, System.currentTimeMillis());
+      wal.append(info, bytes, kvs, System.currentTimeMillis()).get();
     }
     // Now I should have written out lots of blocks.  Sync then read.
     wal.sync();
@@ -350,7 +353,7 @@ public class TestHLog  {
     for (int i = 0; i < total; i++) {
       WALEdit kvs = new WALEdit();
       kvs.add(new KeyValue(Bytes.toBytes(i), tableName, tableName));
-      wal.append(regioninfo, tableName, kvs, System.currentTimeMillis());
+      wal.append(regioninfo, tableName, kvs, System.currentTimeMillis()).get();
     }
     // Now call sync to send the data to HDFS datanodes
     wal.sync(true);
@@ -448,9 +451,12 @@ public class TestHLog  {
   /**
    * Tests that we can write out an edit, close, and then read it back in again.
    * @throws IOException
+   * @throws ExecutionException
+   * @throws InterruptedException
    */
   @Test
-  public void testEditAdd() throws IOException {
+  public void testEditAdd() throws IOException,
+      InterruptedException, ExecutionException {
     final int COL_COUNT = 10;
     final byte [] tableName = Bytes.toBytes("tablename");
     final byte [] row = Bytes.toBytes("row");
@@ -469,7 +475,7 @@ public class TestHLog  {
       HRegionInfo info = new HRegionInfo(new HTableDescriptor(tableName),
         row,Bytes.toBytes(Bytes.toString(row) + "1"), false);
       final byte [] regionName = info.getRegionName();
-      log.append(info, tableName, cols, System.currentTimeMillis());
+      log.append(info, tableName, cols, System.currentTimeMillis()).get();
       long logSeqId = log.startCacheFlush(info.getRegionName());
       log.completeCacheFlush(regionName, tableName, logSeqId, info.isMetaRegion());
       log.close();
@@ -541,9 +547,12 @@ public class TestHLog  {
 
   /**
    * @throws IOException
+   * @throws ExecutionException
+   * @throws InterruptedException
    */
   @Test
-  public void testAppend() throws IOException {
+  public void testAppend() throws IOException,
+      InterruptedException, ExecutionException {
     final int COL_COUNT = 10;
     final byte [] tableName = Bytes.toBytes("tablename");
     final byte [] row = Bytes.toBytes("row");
@@ -561,7 +570,7 @@ public class TestHLog  {
       }
       HRegionInfo hri = new HRegionInfo(new HTableDescriptor(tableName),
           HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
-      log.append(hri, tableName, cols, System.currentTimeMillis());
+      log.append(hri, tableName, cols, System.currentTimeMillis()).get();
       long logSeqId = log.startCacheFlush(hri.getRegionName());
       log.completeCacheFlush(hri.getRegionName(), tableName, logSeqId, false);
       log.close();
@@ -641,13 +650,13 @@ public class TestHLog  {
   }
 
   private void addEdits(HLog log, HRegionInfo hri, byte [] tableName,
-                        int times) throws IOException {
+                        int times) throws IOException, InterruptedException, ExecutionException {
     final byte [] row = Bytes.toBytes("row");
     for (int i = 0; i < times; i++) {
       long timestamp = System.currentTimeMillis();
       WALEdit cols = new WALEdit();
       cols.add(new KeyValue(row, row, row, timestamp, row));
-      log.append(hri, tableName, cols, timestamp);
+      log.append(hri, tableName, cols, timestamp).checkedGet();
     }
   }
 
@@ -664,6 +673,23 @@ public class TestHLog  {
     }
   }
 
+  public static class FailingSequenceFileLogWriter
+    extends SequenceFileLogWriter implements HLog.Writer {
+    static boolean enabled = false;
+    public synchronized static void setFailureMode(boolean enabled) {
+      FailingSequenceFileLogWriter.enabled = enabled;
+    }
+
+    @Override
+    public void sync() throws IOException {
+      if (FailingSequenceFileLogWriter.enabled) {
+        throw new IOException(
+          "Failing here to see if the error propogates upstream");
+      }
+      super.sync();
+    }
+  }
+
   static class DumbLogEntriesVisitor implements LogEntryVisitor {
 
     int increments = 0;

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
index 6a5e4a2..8d7b3b1 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java
@@ -52,6 +52,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -540,7 +541,8 @@ public class TestHLogSplit {
    * safely commence on the master.
    * */
   @Test
-  public void testLogRollAfterSplitStart() throws IOException {
+  public void testLogRollAfterSplitStart() throws IOException,
+      InterruptedException, ExecutionException {
     // set flush interval to a large number so it doesn't interrupt us
     final String F_INTERVAL = "hbase.regionserver.optionallogflushinterval";
     long oldFlushInterval = conf.getLong(F_INTERVAL, 1000);
@@ -561,7 +563,7 @@ public class TestHLogSplit {
       for (int i = 0; i < total; i++) {
         WALEdit kvs = new WALEdit();
         kvs.add(new KeyValue(Bytes.toBytes(i), tableName, tableName));
-        log.append(regioninfo, tableName, kvs, System.currentTimeMillis());
+        log.append(regioninfo, tableName, kvs, System.currentTimeMillis()).get();
       }
       // Send the data to HDFS datanodes and close the HDFS writer
       log.sync(true);

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
index d1f6890..2297b40 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogActionsListener.java
@@ -97,7 +97,7 @@ public class TestLogActionsListener {
       KeyValue kv = new KeyValue(b,b,b);
       WALEdit edit = new WALEdit();
       edit.add(kv);
-      hlog.append(hri, b, edit, 0);
+      hlog.append(hri, b, edit, 0).get();
       if (i == 10) {
         hlog.addLogActionsListerner(laterList);
       }

http://git-wip-us.apache.org/repos/asf/hbase/blob/c22acc7f/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
index 62e954f..b8571c4 100644
--- a/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
+++ b/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
@@ -432,14 +433,14 @@ public class TestWALReplay {
     long now = ee.currentTimeMillis();
     edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
       now, rowName));
-    wal.append(hri, tableName, edit, now);
+    wal.append(hri, tableName, edit, now).get();
 
     // Delete the c family to verify deletes make it over.
     edit = new WALEdit();
     now = ee.currentTimeMillis();
     edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
       KeyValue.Type.DeleteFamily));
-    wal.append(hri, tableName, edit, now);
+    wal.append(hri, tableName, edit, now).get();
 
     // Sync.
     wal.sync();
@@ -579,7 +580,7 @@ public class TestWALReplay {
   private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
       final byte [] rowName, final byte [] family,
       final int count, EnvironmentEdge ee, final HLog wal)
-  throws IOException {
+  throws IOException, InterruptedException, ExecutionException {
     String familyStr = Bytes.toString(family);
     for (int j = 0; j < count; j++) {
       byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
@@ -587,7 +588,7 @@ public class TestWALReplay {
       WALEdit edit = new WALEdit();
       edit.add(new KeyValue(rowName, family, qualifierBytes,
         ee.currentTimeMillis(), columnBytes));
-      wal.append(hri, tableName, edit, ee.currentTimeMillis());
+      wal.append(hri, tableName, edit, ee.currentTimeMillis()).checkedGet();
     }
   }