You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nk...@apache.org on 2013/06/24 20:42:31 UTC

svn commit: r1496157 - in /hbase/trunk/hbase-client/src: main/java/org/apache/hadoop/hbase/client/AsyncProcess.java test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java

Author: nkeywal
Date: Mon Jun 24 18:42:30 2013
New Revision: 1496157

URL: http://svn.apache.org/r1496157
Log:
HBASE-6295  Possible performance improvement in client batch operations: presplit and send in background - round 2

Added:
    hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
    hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java

Added: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java?rev=1496157&view=auto
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java (added)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java Mon Jun 24 18:42:30 2013
@@ -0,0 +1,728 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.Pair;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * This class  allows a continuous flow of requests. It's written to be compatible with a
+ * synchronous caller such as HTable.
+ * <p>
+ * The caller sends a buffer of operation, by calling submit. This class extract from this list
+ * the operations it can send, i.e. the operations that are on region that are not considered
+ * as busy. The process is asynchronous, i.e. it returns immediately when if has finished to
+ * iterate on the list. If, and only if, the maximum number of current task is reached, the call
+ * to submit will block.
+ * </p>
+ * <p>
+ * The class manages internally the retries.
+ * </p>
+ * <p>
+ * The class includes an error marker: it allows to know if an operation has failed or not, and
+ * to get the exception details, i.e. the full list of throwables for each attempt. This marker
+ * is here to help the backward compatibility in HTable. In most (new) cases, it should be
+ * managed by the callbacks.
+ * </p>
+ * <p>
+ * A callback is available, in order to: <list>
+ * <li>Get the result of the operation (failure or success)</li>
+ * <li>When an operation fails but could be retried, allows or not to retry</li>
+ * <li>When an operation fails for good (can't be retried or already retried the maximum number
+ * time), register the error or not.
+ * </list>
+ * <p>
+ * This class is not thread safe externally; only one thread should submit operations at a time.
+ * Internally, the class is thread safe enough to manage simultaneously new submission and results
+ * arising from older operations.
+ * </p>
+ * <p>
+ * Internally, this class works with {@link Row}, this mean it could be theoretically used for
+ * gets as well.
+ * </p>
+ */
+class AsyncProcess<CResult> {
+  private static final Log LOG = LogFactory.getLog(AsyncProcess.class);
+
+  protected final HConnection hConnection;
+  protected final byte[] tableName;
+  protected final ExecutorService pool;
+  protected final AsyncProcessCallback<CResult> callback;
+  protected final BatchErrors errors = new BatchErrors();
+  protected final BatchErrors retriedErrors = new BatchErrors();
+  protected final AtomicBoolean hasError = new AtomicBoolean(false);
+  protected final AtomicLong tasksSent = new AtomicLong(0);
+  protected final AtomicLong tasksDone = new AtomicLong(0);
+  protected final ConcurrentMap<String, AtomicInteger> taskCounterPerRegion =
+      new ConcurrentHashMap<String, AtomicInteger>();
+  protected final int maxTotalConcurrentTasks;
+  protected final int maxConcurrentTasksPerRegion;
+  protected final long pause;
+  protected int numTries;
+  protected final boolean useServerTrackerForRetries;
+  protected int serverTrackerTimeout;
+
+
+  /**
+   * This interface allows to keep the interface of the previous synchronous interface, that uses
+   * an array of object to return the result.
+   * <p/>
+   * This interface allows the caller to specify the behavior on errors: <list>
+   * <li>If we have not yet reach the maximum number of retries, the user can nevertheless
+   * specify if this specific operation should be retried or not.
+   * </li>
+   * <li>If an operation fails (i.e. is not retried or fails after all retries), the user can
+   * specify is we should mark this AsyncProcess as in error or not.
+   * </li>
+   * </list>
+   */
+  static interface AsyncProcessCallback<CResult> {
+
+    /**
+     * Called on success. originalIndex holds the index in the action list.
+     */
+    void success(int originalIndex, byte[] region, Row row, CResult result);
+
+    /**
+     * called on failure, if we don't retry (i.e. called once per failed operation).
+     *
+     * @return true if we should store the error and tag this async process as being in error.
+     *         false if the failure of this operation can be safely ignored, and does not require
+     *         the current process to be stopped without proceeding with the other operations in
+     *         the queue.
+     */
+    boolean failure(int originalIndex, byte[] region, Row row, Throwable t);
+
+    /**
+     * Called on a failure we plan to retry. This allows the user to stop retrying. Will be
+     * called multiple times for a single action if it fails multiple times.
+     *
+     * @return false if we should retry, true otherwise.
+     */
+    boolean retriableFailure(int originalIndex, Row row, byte[] region, Throwable exception);
+  }
+
+  private static class BatchErrors {
+    private List<Throwable> throwables = new ArrayList<Throwable>();
+    private List<Row> actions = new ArrayList<Row>();
+    private List<String> addresses = new ArrayList<String>();
+
+    public void add(Throwable ex, Row row, HRegionLocation location) {
+      throwables.add(ex);
+      actions.add(row);
+      addresses.add(location != null ? location.getHostnamePort() : "null location");
+    }
+
+    private RetriesExhaustedWithDetailsException makeException() {
+      return new RetriesExhaustedWithDetailsException(
+          new ArrayList<Throwable>(throwables),
+          new ArrayList<Row>(actions), new ArrayList<String>(addresses));
+    }
+
+    public void clear() {
+      throwables.clear();
+      actions.clear();
+      addresses.clear();
+    }
+  }
+
+  public AsyncProcess(HConnection hc, byte[] tableName, ExecutorService pool,
+                      AsyncProcessCallback<CResult> callback, Configuration conf) {
+    this.hConnection = hc;
+    this.tableName = tableName;
+    this.pool = pool;
+    this.callback = callback;
+
+    this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
+        HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
+    this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
+        HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
+
+    this.maxTotalConcurrentTasks = conf.getInt("hbase.client.max.total.tasks", 200);
+
+    // With one, we ensure that the ordering of the queries is respected: we don't start
+    //  a set of operations on a region before the previous one is done. As well, this limits
+    //  the pressure we put on the region server.
+    this.maxConcurrentTasksPerRegion = conf.getInt("hbase.client.max.perregion.tasks", 1);
+
+    this.useServerTrackerForRetries =
+        conf.getBoolean(HConnectionManager.RETRIES_BY_SERVER_KEY, true);
+
+    if (this.useServerTrackerForRetries) {
+      // Server tracker allows us to do faster, and yet useful (hopefully), retries.
+      // However, if we are too useful, we might fail very quickly due to retry count limit.
+      // To avoid this, we are going to cheat for now (see HBASE-7659), and calculate maximum
+      // retry time if normal retries were used. Then we will retry until this time runs out.
+      // If we keep hitting one server, the net effect will be the incremental backoff, and
+      // essentially the same number of retries as planned. If we have to do faster retries,
+      // we will do more retries in aggregate, but the user will be none the wiser.
+      this.serverTrackerTimeout = 0;
+      for (int i = 0; i < this.numTries; ++i) {
+        serverTrackerTimeout += ConnectionUtils.getPauseTime(this.pause, i);
+      }
+    }
+  }
+
+  /**
+   * Extract from the rows list what we can submit. The rows we can not submit are kept in the
+   * list.
+   *
+   * @param rows - the submitted row. Modified by the method: we remove the rows we took.
+   * @param atLeastOne true if we should submit at least a subset.
+   */
+  public void submit(List<? extends Row> rows, boolean atLeastOne) throws InterruptedIOException {
+    if (rows.isEmpty()){
+      return;
+    }
+
+    Map<HRegionLocation, MultiAction<Row>> actionsByServer =
+        new HashMap<HRegionLocation, MultiAction<Row>>();
+    List<Action<Row>> retainedActions = new ArrayList<Action<Row>>(rows.size());
+
+    do {
+      Map<String, Boolean> regionIncluded = new HashMap<String, Boolean>();
+      long currentTaskNumber = waitForMaximumCurrentTasks(maxTotalConcurrentTasks);
+      int posInList = -1;
+      Iterator<? extends Row> it = rows.iterator();
+      while (it.hasNext()) {
+        Row r = it.next();
+        HRegionLocation loc = findDestLocation(r, 1, posInList, false, regionIncluded);
+
+        if (loc != null) {   // loc is null if the dest is too busy or there is an error
+          Action<Row> action = new Action<Row>(r, ++posInList);
+          retainedActions.add(action);
+          addAction(loc, action, actionsByServer);
+          it.remove();
+        }
+      }
+
+      if (retainedActions.isEmpty() && atLeastOne && !hasError()) {
+        waitForNextTaskDone(currentTaskNumber);
+      }
+
+    } while (retainedActions.isEmpty() && atLeastOne && !hasError());
+
+    HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
+    sendMultiAction(retainedActions, actionsByServer, 1, errorsByServer);
+  }
+
+  /**
+   * Group the actions per region server.
+   *
+   * @param loc - the destination. Must not be null.
+   * @param action - the action to add to the multiaction
+   * @param actionsByServer the multiaction per server
+   */
+  private void addAction(HRegionLocation loc, Action<Row> action, Map<HRegionLocation,
+      MultiAction<Row>> actionsByServer) {
+    final byte[] regionName = loc.getRegionInfo().getRegionName();
+    MultiAction<Row> multiAction = actionsByServer.get(loc);
+    if (multiAction == null) {
+      multiAction = new MultiAction<Row>();
+      actionsByServer.put(loc, multiAction);
+    }
+
+    multiAction.add(regionName, action);
+  }
+
+  /**
+   * Find the destination, if this destination is not considered as busy.
+   *
+   * @param row          the row
+   * @param numAttempt   the num attempt
+   * @param posInList    the position in the list
+   * @param force        if we must submit whatever the server load
+   * @param regionStatus the
+   * @return null if we should not submit, the destination otherwise.
+   */
+  private HRegionLocation findDestLocation(Row row, int numAttempt,
+                                           int posInList, boolean force,
+                                           Map<String, Boolean> regionStatus) {
+    HRegionLocation loc = null;
+    IOException locationException = null;
+    try {
+      loc = hConnection.locateRegion(this.tableName, row.getRow());
+      if (loc == null) {
+        locationException = new IOException("No location found, aborting submit for" +
+            " tableName=" + Bytes.toString(tableName) +
+            " rowkey=" + Arrays.toString(row.getRow()));
+      }
+    } catch (IOException e) {
+      locationException = e;
+    }
+    if (locationException != null) {
+      // There are multiple retries in locateRegion already. No need to add new.
+      // We can't continue with this row, hence it's the last retry.
+      manageError(numAttempt, posInList, row, false, locationException, null);
+      return null;
+    }
+
+    if (force) {
+      return loc;
+    }
+
+    String regionName = loc.getRegionInfo().getEncodedName();
+    Boolean addIt = regionStatus.get(regionName);
+    if (addIt == null) {
+      addIt = canTakeNewOperations(regionName);
+      regionStatus.put(regionName, addIt);
+    }
+
+    return addIt ? loc : null;
+  }
+
+
+  /**
+   * Check if we should send new operations to this region.
+   *
+   * @param encodedRegionName region name
+   * @return true if this region is considered as busy.
+   */
+  protected boolean canTakeNewOperations(String encodedRegionName) {
+    AtomicInteger ct = taskCounterPerRegion.get(encodedRegionName);
+    return ct == null || ct.get() < maxConcurrentTasksPerRegion;
+  }
+
+  /**
+   * Submit immediately the list of rows, whatever the server status. Kept for backward
+   * compatibility: it allows to be used with the batch interface that return an array of objects.
+   *
+   * @param rows the list of rows.
+   */
+  public void submitAll(List<? extends Row> rows) {
+    List<Action<Row>> actions = new ArrayList<Action<Row>>(rows.size());
+
+    // The position will be used by the processBatch to match the object array returned.
+    int posInList = -1;
+    for (Row r : rows) {
+      posInList++;
+      Action<Row> action = new Action<Row>(r, posInList);
+      actions.add(action);
+    }
+    HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
+    submit(actions, actions, 1, true, errorsByServer);
+  }
+
+
+  /**
+   * Group a list of actions per region servers, and send them. The created MultiActions are
+   * added to the inProgress list.
+   *
+   * @param initialActions - the full list of the actions in progress
+   * @param currentActions - the list of row to submit
+   * @param numAttempt - the current numAttempt (first attempt is 1)
+   * @param force - true if we submit the rowList without taking into account the server load
+   */
+  private void submit(List<Action<Row>> initialActions,
+                      List<Action<Row>> currentActions, int numAttempt, boolean force,
+                      final HConnectionManager.ServerErrorTracker errorsByServer) {
+    // group per location => regions server
+    final Map<HRegionLocation, MultiAction<Row>> actionsByServer =
+        new HashMap<HRegionLocation, MultiAction<Row>>();
+
+    // We have the same policy for a single region per call to submit: we don't want
+    //  to send half of the actions because the status changed in the middle. So we keep the
+    //  status
+    Map<String, Boolean> regionIncluded = new HashMap<String, Boolean>();
+
+    for (Action<Row> action : currentActions) {
+      HRegionLocation loc = findDestLocation(
+          action.getAction(), 1, action.getOriginalIndex(), force, regionIncluded);
+
+      if (loc != null) {
+        addAction(loc, action, actionsByServer);
+      }
+    }
+
+    if (!actionsByServer.isEmpty()) {
+      sendMultiAction(initialActions, actionsByServer, numAttempt, errorsByServer);
+    }
+  }
+
+  /**
+   * Send a multi action structure to the servers, after a delay depending on the attempt
+   * number. Asynchronous.
+   *
+   * @param initialActions  the list of the actions, flat.
+   * @param actionsByServer the actions structured by regions
+   * @param numAttempt      the attempt number.
+   */
+  public void sendMultiAction(final List<Action<Row>> initialActions,
+                              Map<HRegionLocation, MultiAction<Row>> actionsByServer,
+                              final int numAttempt,
+                              final HConnectionManager.ServerErrorTracker errorsByServer) {
+
+    // Send the queries and add them to the inProgress list
+    for (Map.Entry<HRegionLocation, MultiAction<Row>> e : actionsByServer.entrySet()) {
+      final HRegionLocation loc = e.getKey();
+      final MultiAction<Row> multi = e.getValue();
+      final String regionName = loc.getRegionInfo().getEncodedName();
+
+      incTaskCounters(regionName);
+
+      Runnable runnable = new Runnable() {
+        @Override
+        public void run() {
+          MultiResponse res;
+          try {
+            ServerCallable<MultiResponse> callable = createCallable(loc, multi);
+            try {
+              res = callable.withoutRetries();
+            } catch (IOException e) {
+              LOG.warn("The call to the RS failed, we don't know where we stand. regionName="
+                  + regionName, e);
+              resubmitAll(initialActions, multi, loc, numAttempt + 1, e, errorsByServer);
+              return;
+            }
+
+            receiveMultiAction(initialActions, multi, loc, res, numAttempt, errorsByServer);
+          } finally {
+            decTaskCounters(regionName);
+          }
+        }
+      };
+
+      try {
+        this.pool.submit(runnable);
+      } catch (RejectedExecutionException ree) {
+        // This should never happen. But as the pool is provided by the end user, let's secure
+        //  this a little.
+        decTaskCounters(regionName);
+        LOG.warn("The task was rejected by the pool. This is unexpected. " +
+            "regionName=" + regionName, ree);
+        // We're likely to fail again, but this will increment the attempt counter, so it will
+        //  finish.
+        resubmitAll(initialActions, multi, loc, numAttempt + 1, ree, errorsByServer);
+      }
+    }
+  }
+
+  /**
+   * Create a callable. Isolated to be easily overridden in the tests.
+   */
+  protected ServerCallable<MultiResponse> createCallable(
+      final HRegionLocation loc, final MultiAction<Row> multi) {
+
+    return new MultiServerCallable<Row>(hConnection, tableName, loc, multi);
+  }
+
+  /**
+   * Check that we can retry acts accordingly: logs, set the error status, call the callbacks.
+   *
+   * @param numAttempt    the number of this attempt
+   * @param originalIndex the position in the list sent
+   * @param row           the row
+   * @param canRetry      if false, we won't retry whatever the settings.
+   * @param throwable     the throwable, if any (can be null)
+   * @param location      the location, if any (can be null)
+   * @return true if the action can be retried, false otherwise.
+   */
+  private boolean manageError(int numAttempt, int originalIndex, Row row, boolean canRetry,
+                              Throwable throwable, HRegionLocation location) {
+    if (canRetry) {
+      if (numAttempt >= numTries ||
+          (throwable != null && throwable instanceof DoNotRetryIOException)) {
+        canRetry = false;
+      }
+    }
+    byte[] region = location == null ? null : location.getRegionInfo().getEncodedNameAsBytes();
+
+    if (canRetry && callback != null) {
+      canRetry = callback.retriableFailure(originalIndex, row, region, throwable);
+    }
+
+    if (canRetry) {
+      if (LOG.isTraceEnabled()) {
+        retriedErrors.add(throwable, row, location);
+      }
+    } else {
+      if (callback != null) {
+        callback.failure(originalIndex, region, row, throwable);
+      }
+      this.hasError.set(true);
+      errors.add(throwable, row, location);
+    }
+
+    return canRetry;
+  }
+
+  /**
+   * Resubmit all the actions from this multiaction after a failure.
+   *
+   * @param initialActions the full initial action list
+   * @param rsActions  the actions still to do from the initial list
+   * @param location   the destination
+   * @param numAttempt the number of attempts so far
+   * @param t the throwable (if any) that caused the resubmit
+   */
+  private void resubmitAll(List<Action<Row>> initialActions, MultiAction<Row> rsActions,
+                           HRegionLocation location, int numAttempt, Throwable t,
+                           HConnectionManager.ServerErrorTracker errorsByServer) {
+    // Do not use the exception for updating cache because it might be coming from
+    // any of the regions in the MultiAction.
+    hConnection.updateCachedLocations(tableName,
+        rsActions.actions.values().iterator().next().get(0).getAction().getRow(), null, location);
+    errorsByServer.reportServerError(location);
+
+    List<Action<Row>> toReplay = new ArrayList<Action<Row>>();
+    for (List<Action<Row>> actions : rsActions.actions.values()) {
+      for (Action<Row> action : actions) {
+        if (manageError(numAttempt, action.getOriginalIndex(), action.getAction(),
+            true, t, location)) {
+          toReplay.add(action);
+        }
+      }
+    }
+
+    submit(initialActions, toReplay, numAttempt, true, errorsByServer);
+  }
+
+  /**
+   * Called when we receive the result of a server query.
+   *
+   * @param initialActions - the whole action list
+   * @param rsActions      - the actions for this location
+   * @param location       - the location
+   * @param responses      - the response, if any
+   * @param numAttempt     - the attempt
+   */
+  private void receiveMultiAction(List<Action<Row>> initialActions,
+                                  MultiAction<Row> rsActions, HRegionLocation location,
+                                  MultiResponse responses, int numAttempt,
+                                  HConnectionManager.ServerErrorTracker errorsByServer) {
+
+    if (responses == null) {
+      LOG.info("Attempt #" + numAttempt + " failed for all operations on server " +
+          location.getServerName() + " , trying to resubmit.");
+      resubmitAll(initialActions, rsActions, location, numAttempt + 1, null, errorsByServer);
+      return;
+    }
+
+    // Success or partial success
+    // Analyze detailed results. We can still have individual failures to be redo.
+    // two specific throwables are managed:
+    //  - DoNotRetryIOException: we continue to retry for other actions
+    //  - RegionMovedException: we update the cache with the new region location
+
+    List<Action<Row>> toReplay = new ArrayList<Action<Row>>();
+    Throwable throwable = null;
+
+    int failureCount = 0;
+    boolean canRetry = true;
+    for (Map.Entry<byte[], List<Pair<Integer, Object>>> resultsForRS :
+        responses.getResults().entrySet()) {
+
+      for (Pair<Integer, Object> regionResult : resultsForRS.getValue()) {
+        Object result = regionResult.getSecond();
+
+        // Failure: retry if it's make sense else update the errors lists
+        if (result == null || result instanceof Throwable) {
+          throwable = (Throwable) result;
+          Action<Row> correspondingAction = initialActions.get(regionResult.getFirst());
+          Row row = correspondingAction.getAction();
+
+          if (failureCount++ == 0) { // We're doing this once per location.
+            hConnection.updateCachedLocations(this.tableName, row.getRow(), result, location);
+            if (errorsByServer != null) {
+              errorsByServer.reportServerError(location);
+              canRetry = errorsByServer.canRetryMore();
+            }
+          }
+
+          if (manageError(numAttempt, correspondingAction.getOriginalIndex(), row, canRetry,
+              throwable, location)) {
+            toReplay.add(correspondingAction);
+          }
+        } else { // success
+          if (callback != null) {
+            Action<Row> correspondingAction = initialActions.get(regionResult.getFirst());
+            Row row = correspondingAction.getAction();
+            //noinspection unchecked
+            this.callback.success(correspondingAction.getOriginalIndex(),
+                resultsForRS.getKey(), row, (CResult) result);
+          }
+        }
+      }
+    }
+
+    if (!toReplay.isEmpty()) {
+      if (numAttempt > 2) {
+        // We use this value to have some logs when we have multiple failures, but not too many
+        //  logs as errors are to be expected wehn region moves, split and so on
+        LOG.debug("Attempt #" + numAttempt + " failed for " + failureCount +
+            " operations on server " + location.getServerName() + ", resubmitting " +
+            toReplay.size() + ", tableName=" + Bytes.toString(tableName) +
+            ", last exception was: " + throwable);
+      }
+      long backOffTime = (errorsByServer != null ?
+          errorsByServer.calculateBackoffTime(location, pause) :
+          ConnectionUtils.getPauseTime(pause, numAttempt));
+      try {
+        Thread.sleep(backOffTime);
+      } catch (InterruptedException e) {
+        LOG.warn("Not sent: " + toReplay.size() +
+            " operations,  tableName=" + Bytes.toString(tableName), e);
+        Thread.interrupted();
+        return;
+      }
+
+      submit(initialActions, toReplay, numAttempt + 1, true, errorsByServer);
+    } else if (failureCount != 0) {
+      LOG.warn("Attempt #" + numAttempt + " failed for " + failureCount +
+          " operations on server " + location.getServerName() + " NOT resubmitting." +
+          ", tableName=" + Bytes.toString(tableName));
+    }
+  }
+
+  /**
+   * Waits for another task to finish.
+   * @param currentNumberOfTask - the number of task finished when calling the method.
+   */
+  protected void waitForNextTaskDone(long currentNumberOfTask) throws InterruptedIOException {
+    while (currentNumberOfTask == tasksDone.get()) {
+      try {
+        synchronized (this.tasksDone) {
+          this.tasksDone.wait(100);
+        }
+      } catch (InterruptedException e) {
+        throw new InterruptedIOException("Interrupted." +
+            " currentNumberOfTask=" + currentNumberOfTask +
+            ",  tableName=" + Bytes.toString(tableName) + ", tasksDone=" + tasksDone.get());
+      }
+    }
+  }
+
+  /**
+   * Wait until the async does not have more than max tasks in progress.
+   */
+  private long waitForMaximumCurrentTasks(int max) throws InterruptedIOException {
+    long lastLog = 0;
+    long currentTasksDone = this.tasksDone.get();
+
+    while ((tasksSent.get() - currentTasksDone) > max) {
+      long now = EnvironmentEdgeManager.currentTimeMillis();
+      if (now > lastLog + 5000) {
+        lastLog = now;
+        LOG.info(Bytes.toString(tableName) +
+            ": Waiting for the global number of tasks to be equals or less than " + max +
+            ", currently it's " + this.tasksDone.get());
+      }
+      waitForNextTaskDone(currentTasksDone);
+      currentTasksDone = this.tasksDone.get();
+    }
+
+    return currentTasksDone;
+  }
+
+  /**
+   * Wait until all tasks are executed, successfully or not.
+   */
+  public void waitUntilDone() throws InterruptedIOException {
+    waitForMaximumCurrentTasks(0);
+  }
+
+
+  public boolean hasError() {
+    return hasError.get();
+  }
+
+  public List<? extends Row> getFailedOperations() {
+    return errors.actions;
+  }
+
+  /**
+   * Clean the errors stacks. Should be called only when there are no actions in progress.
+   */
+  public void clearErrors() {
+    errors.clear();
+    retriedErrors.clear();
+    hasError.set(false);
+  }
+
+  public RetriesExhaustedWithDetailsException getErrors() {
+    return errors.makeException();
+  }
+
+  /**
+   * incrementer the tasks counters for a given region. MT safe.
+   */
+  protected void incTaskCounters(String encodedRegionName) {
+    tasksSent.incrementAndGet();
+
+    AtomicInteger counterPerServer = taskCounterPerRegion.get(encodedRegionName);
+    if (counterPerServer == null) {
+      taskCounterPerRegion.putIfAbsent(encodedRegionName, new AtomicInteger());
+      counterPerServer = taskCounterPerRegion.get(encodedRegionName);
+    }
+    counterPerServer.incrementAndGet();
+  }
+
+  /**
+   * Decrements the counters for a given region
+   */
+  protected void decTaskCounters(String encodedRegionName) {
+    AtomicInteger counterPerServer = taskCounterPerRegion.get(encodedRegionName);
+    counterPerServer.decrementAndGet();
+
+    tasksDone.incrementAndGet();
+    synchronized (tasksDone) {
+      tasksDone.notifyAll();
+    }
+  }
+
+  /**
+   * Creates the server error tracker to use inside process.
+   * Currently, to preserve the main assumption about current retries, and to work well with
+   * the retry-limit-based calculation, the calculation is local per Process object.
+   * We may benefit from connection-wide tracking of server errors.
+   * @return ServerErrorTracker to use, null if there is no ServerErrorTracker on this connection
+   */
+  protected HConnectionManager.ServerErrorTracker createServerErrorTracker() {
+    if (useServerTrackerForRetries){
+      return new HConnectionManager.ServerErrorTracker(this.serverTrackerTimeout);
+    }else {
+      return null;
+    }
+  }
+}

Added: hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java?rev=1496157&view=auto
==============================================================================
--- hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java (added)
+++ hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java Mon Jun 24 18:42:30 2013
@@ -0,0 +1,635 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.util.Threads;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+@Category(MediumTests.class)
+public class TestAsyncProcess {
+  private static final byte[] DUMMY_TABLE = "DUMMY_TABLE".getBytes();
+  private static final byte[] DUMMY_BYTES_1 = "DUMMY_BYTES_1".getBytes();
+  private static final byte[] DUMMY_BYTES_2 = "DUMMY_BYTES_2".getBytes();
+  private static final byte[] FAILS = "FAILS".getBytes();
+  private Configuration conf = new Configuration();
+
+
+  private static ServerName sn = new ServerName("localhost:10,1254");
+  private static HRegionInfo hri1 = new HRegionInfo(DUMMY_BYTES_1);
+  private static HRegionInfo hri2 = new HRegionInfo(DUMMY_BYTES_1);
+  private static HRegionLocation loc1 = new HRegionLocation(hri1, sn);
+  private static HRegionLocation loc2 = new HRegionLocation(hri2, sn);
+
+  private static final String success = "success";
+  private static Exception failure = new Exception("failure");
+
+  static class MyAsyncProcess<Res> extends AsyncProcess<Res> {
+
+    public MyAsyncProcess(HConnection hc,
+                          AsyncProcessCallback<Res> callback, Configuration conf) {
+      super(hc, DUMMY_TABLE, new ThreadPoolExecutor(1, 10, 60, TimeUnit.SECONDS,
+          new SynchronousQueue<Runnable>(),
+          Threads.newDaemonThreadFactory("test-TestAsyncProcess"))
+          , callback, conf);
+    }
+
+    /**
+     * Do not call a server, fails if the rowkey of the operation is{@link #FAILS}
+     */
+    @Override
+    protected ServerCallable<MultiResponse> createCallable(
+        final HRegionLocation loc, final MultiAction<Row> multi) {
+
+      final MultiResponse mr = new MultiResponse();
+      for (Map.Entry<byte[], List<Action<Row>>> entry : multi.actions.entrySet()) {
+        for (Action a : entry.getValue()) {
+          if (Arrays.equals(FAILS, a.getAction().getRow())) {
+            mr.add(loc.getRegionInfo().getRegionName(), a.getOriginalIndex(), failure);
+          } else {
+            mr.add(loc.getRegionInfo().getRegionName(), a.getOriginalIndex(), success);
+          }
+        }
+      }
+
+      return new MultiServerCallable<Row>(hConnection, tableName, null, null) {
+        @Override
+        public MultiResponse withoutRetries() {
+          return mr;
+        }
+      };
+    }
+  }
+
+  /**
+   * Returns our async process.
+   */
+  static class MyConnectionImpl extends HConnectionManager.HConnectionImplementation {
+    MyAsyncProcess<?> ap;
+    final static Configuration c = new Configuration();
+
+    static {
+      c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
+    }
+
+    protected MyConnectionImpl() {
+      super(c);
+    }
+
+    protected MyConnectionImpl(Configuration conf) {
+      super(conf);
+    }
+
+    @Override
+    protected <R> AsyncProcess createAsyncProcess(byte[] tableName, ExecutorService pool,
+                                                  AsyncProcess.AsyncProcessCallback<R> callback,
+                                                  Configuration conf) {
+      ap = new MyAsyncProcess<R>(this, callback, conf);
+      return ap;
+    }
+
+    @Override
+    public HRegionLocation locateRegion(final byte[] tableName,
+                                        final byte[] row) {
+      return loc1;
+    }
+
+  }
+
+  @Test
+  public void testSubmit() throws Exception {
+    HConnection hc = createHConnection();
+    AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
+
+    List<Put> puts = new ArrayList<Put>();
+    puts.add(createPut(true, true));
+
+    ap.submit(puts, false);
+    Assert.assertTrue(puts.isEmpty());
+  }
+
+  @Test
+  public void testSubmitWithCB() throws Exception {
+    HConnection hc = createHConnection();
+    MyCB mcb = new MyCB();
+    AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+
+    List<Put> puts = new ArrayList<Put>();
+    puts.add(createPut(true, true));
+
+    ap.submit(puts, false);
+    Assert.assertTrue(puts.isEmpty());
+
+    while (!(mcb.successCalled.get() == 1) && !ap.hasError()) {
+      Thread.sleep(1);
+    }
+    Assert.assertEquals(mcb.successCalled.get(), 1);
+  }
+
+  @Test
+  public void testSubmitBusyRegion() throws Exception {
+    HConnection hc = createHConnection();
+    AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
+
+    List<Put> puts = new ArrayList<Put>();
+    puts.add(createPut(true, true));
+
+    ap.incTaskCounters(hri1.getEncodedName());
+    ap.submit(puts, false);
+    Assert.assertEquals(puts.size(), 1);
+
+    ap.decTaskCounters(hri1.getEncodedName());
+    ap.submit(puts, false);
+    Assert.assertTrue(puts.isEmpty());
+  }
+
+  @Test
+  public void testFail() throws Exception {
+    HConnection hc = createHConnection();
+    MyCB mcb = new MyCB();
+    AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+
+    List<Put> puts = new ArrayList<Put>();
+    Put p = createPut(true, false);
+    puts.add(p);
+
+    ap.submit(puts, false);
+    Assert.assertTrue(puts.isEmpty());
+
+    while (!ap.hasError()) {
+      Thread.sleep(1);
+    }
+
+    Assert.assertEquals(0, mcb.successCalled.get());
+    Assert.assertEquals(2, mcb.retriableFailure.get());
+    Assert.assertEquals(1, mcb.failureCalled.get());
+
+    Assert.assertEquals(1, ap.getErrors().exceptions.size());
+    Assert.assertTrue("was: " + ap.getErrors().exceptions.get(0),
+        failure.equals(ap.getErrors().exceptions.get(0)));
+    Assert.assertTrue("was: " + ap.getErrors().exceptions.get(0),
+        failure.equals(ap.getErrors().exceptions.get(0)));
+
+    Assert.assertEquals(1, ap.getFailedOperations().size());
+    Assert.assertTrue("was: " + ap.getFailedOperations().get(0),
+        p.equals(ap.getFailedOperations().get(0)));
+  }
+
+  @Test
+  public void testWaitForNextTaskDone() throws IOException {
+    HConnection hc = createHConnection();
+    MyCB mcb = new MyCB();
+    final AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+    ap.tasksSent.incrementAndGet();
+
+    final AtomicBoolean checkPoint = new AtomicBoolean(false);
+    final AtomicBoolean checkPoint2 = new AtomicBoolean(false);
+
+    Thread t = new Thread(){
+      @Override
+      public void run(){
+        Threads.sleep(1000);
+        Assert.assertFalse(checkPoint.get());
+        ap.tasksDone.incrementAndGet();
+        checkPoint2.set(true);
+      }
+    };
+
+    t.start();
+    ap.waitForNextTaskDone(0);
+    checkPoint.set(true);
+    while (!checkPoint2.get()){
+      Threads.sleep(1);
+    }
+  }
+
+  @Test
+  public void testSubmitTrue() throws IOException {
+    HConnection hc = createHConnection();
+    MyCB mcb = new MyCB();
+    final AsyncProcess<Object> ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+    ap.tasksSent.incrementAndGet();
+    final AtomicInteger ai = new AtomicInteger(1);
+    ap.taskCounterPerRegion.put(hri1.getEncodedName(), ai);
+
+    final AtomicBoolean checkPoint = new AtomicBoolean(false);
+    final AtomicBoolean checkPoint2 = new AtomicBoolean(false);
+
+    Thread t = new Thread(){
+      @Override
+      public void run(){
+        Threads.sleep(1000);
+        Assert.assertFalse(checkPoint.get());
+        ai.decrementAndGet();
+        ap.tasksDone.incrementAndGet();
+        checkPoint2.set(true);
+      }
+    };
+
+    List<Put> puts = new ArrayList<Put>();
+    Put p = createPut(true, true);
+    puts.add(p);
+
+    ap.submit(puts, false);
+    Assert.assertFalse(puts.isEmpty());
+
+    t.start();
+
+    ap.submit(puts, true);
+    Assert.assertTrue(puts.isEmpty());
+
+    checkPoint.set(true);
+    while (!checkPoint2.get()){
+      Threads.sleep(1);
+    }
+  }
+
+  @Test
+  public void testFailAndSuccess() throws Exception {
+    HConnection hc = createHConnection();
+    MyCB mcb = new MyCB();
+    AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+
+    List<Put> puts = new ArrayList<Put>();
+    puts.add(createPut(true, false));
+    puts.add(createPut(true, true));
+    puts.add(createPut(true, true));
+
+    ap.submit(puts, false);
+    Assert.assertTrue(puts.isEmpty());
+
+    while (!ap.hasError()) {
+      Thread.sleep(1);
+    }
+    Assert.assertEquals(mcb.successCalled.get(), 2);
+    Assert.assertEquals(mcb.retriableFailure.get(), 2);
+    Assert.assertEquals(mcb.failureCalled.get(), 1);
+
+    Assert.assertEquals(1, ap.getErrors().actions.size());
+
+
+    puts.add(createPut(true, true));
+    ap.submit(puts, false);
+    Assert.assertTrue(puts.isEmpty());
+
+    while (mcb.successCalled.get() != 3) {
+      Thread.sleep(1);
+    }
+    Assert.assertEquals(mcb.retriableFailure.get(), 2);
+    Assert.assertEquals(mcb.failureCalled.get(), 1);
+
+    ap.clearErrors();
+    Assert.assertTrue(ap.getErrors().actions.isEmpty());
+  }
+
+  @Test
+  public void testFlush() throws Exception {
+    HConnection hc = createHConnection();
+    MyCB mcb = new MyCB();
+    AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+
+    List<Put> puts = new ArrayList<Put>();
+    puts.add(createPut(true, false));
+    puts.add(createPut(true, true));
+    puts.add(createPut(true, true));
+
+    ap.submit(puts, false);
+    ap.waitUntilDone();
+
+    Assert.assertEquals(mcb.successCalled.get(), 2);
+    Assert.assertEquals(mcb.retriableFailure.get(), 2);
+    Assert.assertEquals(mcb.failureCalled.get(), 1);
+
+    Assert.assertEquals(1, ap.getFailedOperations().size());
+  }
+
+  @Test
+  public void testMaxTask() throws Exception {
+    HConnection hc = createHConnection();
+    final AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
+
+    for (int i = 0; i < 1000; i++) {
+      ap.incTaskCounters("dummy");
+    }
+
+    final Thread myThread = Thread.currentThread();
+
+    Thread t = new Thread() {
+      public void run() {
+        Threads.sleep(2000);
+        myThread.interrupt();
+      }
+    };
+
+    List<Put> puts = new ArrayList<Put>();
+    puts.add(createPut(true, true));
+
+    t.start();
+
+    try {
+      ap.submit(puts, false);
+      Assert.fail("We should have been interrupted.");
+    } catch (InterruptedIOException expected) {
+    }
+
+    final long sleepTime = 2000;
+
+    Thread t2 = new Thread() {
+      public void run() {
+        Threads.sleep(sleepTime);
+        while (ap.tasksDone.get() > 0) {
+          ap.decTaskCounters("dummy");
+        }
+      }
+    };
+    t2.start();
+
+    long start = System.currentTimeMillis();
+    ap.submit(new ArrayList<Row>(), false);
+    long end = System.currentTimeMillis();
+
+    //Adds 100 to secure us against approximate timing.
+    Assert.assertTrue(start + 100L + sleepTime > end);
+  }
+
+
+  private class MyCB implements AsyncProcess.AsyncProcessCallback<Object> {
+    private AtomicInteger successCalled = new AtomicInteger(0);
+    private AtomicInteger failureCalled = new AtomicInteger(0);
+    private AtomicInteger retriableFailure = new AtomicInteger(0);
+
+
+    @Override
+    public void success(int originalIndex, byte[] region, Row row, Object o) {
+      successCalled.incrementAndGet();
+    }
+
+    @Override
+    public boolean failure(int originalIndex, byte[] region, Row row, Throwable t) {
+      failureCalled.incrementAndGet();
+      return true;
+    }
+
+    @Override
+    public boolean retriableFailure(int originalIndex, Row row, byte[] region,
+                                    Throwable exception) {
+      // We retry once only.
+      return (retriableFailure.incrementAndGet() < 2);
+    }
+  }
+
+
+  private static HConnection createHConnection() throws IOException {
+    HConnection hc = Mockito.mock(HConnection.class);
+
+    Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
+        Mockito.eq(DUMMY_BYTES_1), Mockito.anyBoolean())).thenReturn(loc1);
+    Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
+        Mockito.eq(DUMMY_BYTES_1))).thenReturn(loc1);
+
+    Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
+        Mockito.eq(DUMMY_BYTES_2), Mockito.anyBoolean())).thenReturn(loc2);
+    Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
+        Mockito.eq(DUMMY_BYTES_2))).thenReturn(loc2);
+
+    Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
+        Mockito.eq(FAILS), Mockito.anyBoolean())).thenReturn(loc2);
+    Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
+        Mockito.eq(FAILS))).thenReturn(loc2);
+
+    return hc;
+  }
+
+  @Test
+  public void testHTablePutSuccess() throws Exception {
+    HTable ht = Mockito.mock(HTable.class);
+    HConnection hc = createHConnection();
+    ht.ap = new MyAsyncProcess<Object>(hc, null, conf);
+
+    Put put = createPut(true, true);
+
+    Assert.assertEquals(0, ht.getWriteBufferSize());
+    ht.put(put);
+    Assert.assertEquals(0, ht.getWriteBufferSize());
+  }
+
+  private void doHTableFailedPut(boolean bufferOn) throws Exception {
+    HTable ht = new HTable();
+    HConnection hc = createHConnection();
+    MyCB mcb = new MyCB(); // This allows to have some hints on what's going on.
+    ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+    ht.setAutoFlush(true, true);
+    if (bufferOn) {
+      ht.setWriteBufferSize(1024L * 1024L);
+    } else {
+      ht.setWriteBufferSize(0L);
+    }
+
+    Put put = createPut(true, false);
+
+    Assert.assertEquals(0L, ht.currentWriteBufferSize);
+    try {
+      ht.put(put);
+      if (bufferOn) {
+        ht.flushCommits();
+      }
+      Assert.fail();
+    } catch (RetriesExhaustedException expected) {
+    }
+    Assert.assertEquals(0L, ht.currentWriteBufferSize);
+    Assert.assertEquals(0, mcb.successCalled.get());
+    Assert.assertEquals(2, mcb.retriableFailure.get());
+    Assert.assertEquals(1, mcb.failureCalled.get());
+
+    // This should not raise any exception, puts have been 'received' before by the catch.
+    ht.close();
+  }
+
+  @Test
+  public void testHTableFailedPutWithBuffer() throws Exception {
+    doHTableFailedPut(true);
+  }
+
+  @Test
+  public void doHTableFailedPutWithoutBuffer() throws Exception {
+    doHTableFailedPut(false);
+  }
+
+  @Test
+  public void testHTableFailedPutAndNewPut() throws Exception {
+    HTable ht = new HTable();
+    HConnection hc = createHConnection();
+    MyCB mcb = new MyCB(); // This allows to have some hints on what's going on.
+    ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+    ht.setAutoFlush(false, true);
+    ht.setWriteBufferSize(0);
+
+    Put p = createPut(true, false);
+    ht.put(p);
+
+    ht.ap.waitUntilDone(); // Let's do all the retries.
+
+    // We're testing that we're behaving as we were behaving in 0.94: sending exceptions in the
+    //  doPut if it fails.
+    // This said, it's not a very easy going behavior. For example, when we insert a list of
+    //  puts, we may raise an exception in the middle of the list. It's then up to the caller to
+    //  manage what was inserted, what was tried but failed, and what was not even tried.
+    p = createPut(true, true);
+    Assert.assertEquals(0, ht.writeAsyncBuffer.size());
+    try {
+      ht.put(p);
+      Assert.fail();
+    } catch (RetriesExhaustedException expected) {
+    }
+    Assert.assertEquals("the put should not been inserted.", 0, ht.writeAsyncBuffer.size());
+  }
+
+
+  @Test
+  public void testWithNoClearOnFail() throws IOException {
+    HTable ht = new HTable();
+    HConnection hc = createHConnection();
+    MyCB mcb = new MyCB();
+    ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+    ht.setAutoFlush(false, false);
+
+    Put p = createPut(true, false);
+    ht.put(p);
+    Assert.assertEquals(0, ht.writeAsyncBuffer.size());
+    try {
+      ht.flushCommits();
+    } catch (RetriesExhaustedWithDetailsException expected) {
+    }
+    Assert.assertEquals(1, ht.writeAsyncBuffer.size());
+
+    try {
+      ht.close();
+    } catch (RetriesExhaustedWithDetailsException expected) {
+    }
+    Assert.assertEquals(1, ht.writeAsyncBuffer.size());
+  }
+
+  @Test
+  public void testBatch() throws IOException, InterruptedException {
+    HTable ht = new HTable();
+    ht.connection = new MyConnectionImpl();
+
+    List<Put> puts = new ArrayList<Put>();
+    puts.add(createPut(true, true));
+    puts.add(createPut(true, true));
+    puts.add(createPut(true, true));
+    puts.add(createPut(true, true));
+    puts.add(createPut(true, false)); // <=== the bad apple, position 4
+    puts.add(createPut(true, true));
+    puts.add(createPut(true, false)); // <=== another bad apple, position 6
+
+    Object[] res = new Object[puts.size()];
+    try {
+      ht.processBatch(puts, res);
+      Assert.fail();
+    } catch (RetriesExhaustedException expected) {
+    }
+
+    Assert.assertEquals(res[0], success);
+    Assert.assertEquals(res[1], success);
+    Assert.assertEquals(res[2], success);
+    Assert.assertEquals(res[3], success);
+    Assert.assertEquals(res[4], failure);
+    Assert.assertEquals(res[5], success);
+    Assert.assertEquals(res[6], failure);
+  }
+
+  @Test
+  public void testErrorsServers() throws InterruptedIOException,
+      RetriesExhaustedWithDetailsException {
+    HTable ht = new HTable();
+    Configuration configuration = new Configuration(conf);
+    configuration.setBoolean(HConnectionManager.RETRIES_BY_SERVER_KEY, true);
+    configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 20);
+
+    MyConnectionImpl mci = new MyConnectionImpl(configuration);
+    ht.connection = mci;
+    ht.ap = new MyAsyncProcess<Object>(mci, null, configuration);
+
+
+    Assert.assertTrue(ht.ap.useServerTrackerForRetries);
+    Assert.assertNotNull(ht.ap.createServerErrorTracker());
+    Assert.assertTrue(ht.ap.serverTrackerTimeout > 10000);
+    ht.ap.serverTrackerTimeout = 1;
+
+
+    Put p = createPut(true, false);
+    ht.setAutoFlush(false);
+    ht.put(p);
+
+    long start = System.currentTimeMillis();
+    try {
+      ht.flushCommits();
+      Assert.fail();
+    } catch (RetriesExhaustedWithDetailsException expected) {
+    }
+    // Checking that the ErrorsServers came into play and made us stop immediately
+    Assert.assertTrue((System.currentTimeMillis() - start) < 10000);
+  }
+
+
+  /**
+   * @param reg1    if true, creates a put on region 1, region 2 otherwise
+   * @param success if true, the put will succeed.
+   * @return a put
+   */
+  private Put createPut(boolean reg1, boolean success) {
+    Put p;
+    if (!success) {
+      p = new Put(FAILS);
+    } else if (reg1) {
+      p = new Put(DUMMY_BYTES_1);
+    } else {
+      p = new Put(DUMMY_BYTES_2);
+    }
+
+    p.add(DUMMY_BYTES_1, DUMMY_BYTES_1, DUMMY_BYTES_1);
+
+    return p;
+  }
+}