You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nk...@apache.org on 2013/06/24 20:42:31 UTC
svn commit: r1496157 - in /hbase/trunk/hbase-client/src:
main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java
Author: nkeywal
Date: Mon Jun 24 18:42:30 2013
New Revision: 1496157
URL: http://svn.apache.org/r1496157
Log:
HBASE-6295 Possible performance improvement in client batch operations: presplit and send in background - round 2
Added:
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java
Added: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java?rev=1496157&view=auto
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java (added)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java Mon Jun 24 18:42:30 2013
@@ -0,0 +1,728 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.Pair;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * This class allows a continuous flow of requests. It's written to be compatible with a
+ * synchronous caller such as HTable.
+ * <p>
+ * The caller sends a buffer of operation, by calling submit. This class extract from this list
+ * the operations it can send, i.e. the operations that are on region that are not considered
+ * as busy. The process is asynchronous, i.e. it returns immediately when if has finished to
+ * iterate on the list. If, and only if, the maximum number of current task is reached, the call
+ * to submit will block.
+ * </p>
+ * <p>
+ * The class manages internally the retries.
+ * </p>
+ * <p>
+ * The class includes an error marker: it allows to know if an operation has failed or not, and
+ * to get the exception details, i.e. the full list of throwables for each attempt. This marker
+ * is here to help the backward compatibility in HTable. In most (new) cases, it should be
+ * managed by the callbacks.
+ * </p>
+ * <p>
+ * A callback is available, in order to: <list>
+ * <li>Get the result of the operation (failure or success)</li>
+ * <li>When an operation fails but could be retried, allows or not to retry</li>
+ * <li>When an operation fails for good (can't be retried or already retried the maximum number
+ * time), register the error or not.
+ * </list>
+ * <p>
+ * This class is not thread safe externally; only one thread should submit operations at a time.
+ * Internally, the class is thread safe enough to manage simultaneously new submission and results
+ * arising from older operations.
+ * </p>
+ * <p>
+ * Internally, this class works with {@link Row}, this mean it could be theoretically used for
+ * gets as well.
+ * </p>
+ */
+class AsyncProcess<CResult> {
+ private static final Log LOG = LogFactory.getLog(AsyncProcess.class);
+
+ protected final HConnection hConnection;
+ protected final byte[] tableName;
+ protected final ExecutorService pool;
+ protected final AsyncProcessCallback<CResult> callback;
+ protected final BatchErrors errors = new BatchErrors();
+ protected final BatchErrors retriedErrors = new BatchErrors();
+ protected final AtomicBoolean hasError = new AtomicBoolean(false);
+ protected final AtomicLong tasksSent = new AtomicLong(0);
+ protected final AtomicLong tasksDone = new AtomicLong(0);
+ protected final ConcurrentMap<String, AtomicInteger> taskCounterPerRegion =
+ new ConcurrentHashMap<String, AtomicInteger>();
+ protected final int maxTotalConcurrentTasks;
+ protected final int maxConcurrentTasksPerRegion;
+ protected final long pause;
+ protected int numTries;
+ protected final boolean useServerTrackerForRetries;
+ protected int serverTrackerTimeout;
+
+
+ /**
+ * This interface allows to keep the interface of the previous synchronous interface, that uses
+ * an array of object to return the result.
+ * <p/>
+ * This interface allows the caller to specify the behavior on errors: <list>
+ * <li>If we have not yet reach the maximum number of retries, the user can nevertheless
+ * specify if this specific operation should be retried or not.
+ * </li>
+ * <li>If an operation fails (i.e. is not retried or fails after all retries), the user can
+ * specify is we should mark this AsyncProcess as in error or not.
+ * </li>
+ * </list>
+ */
+ static interface AsyncProcessCallback<CResult> {
+
+ /**
+ * Called on success. originalIndex holds the index in the action list.
+ */
+ void success(int originalIndex, byte[] region, Row row, CResult result);
+
+ /**
+ * called on failure, if we don't retry (i.e. called once per failed operation).
+ *
+ * @return true if we should store the error and tag this async process as being in error.
+ * false if the failure of this operation can be safely ignored, and does not require
+ * the current process to be stopped without proceeding with the other operations in
+ * the queue.
+ */
+ boolean failure(int originalIndex, byte[] region, Row row, Throwable t);
+
+ /**
+ * Called on a failure we plan to retry. This allows the user to stop retrying. Will be
+ * called multiple times for a single action if it fails multiple times.
+ *
+ * @return false if we should retry, true otherwise.
+ */
+ boolean retriableFailure(int originalIndex, Row row, byte[] region, Throwable exception);
+ }
+
+ private static class BatchErrors {
+ private List<Throwable> throwables = new ArrayList<Throwable>();
+ private List<Row> actions = new ArrayList<Row>();
+ private List<String> addresses = new ArrayList<String>();
+
+ public void add(Throwable ex, Row row, HRegionLocation location) {
+ throwables.add(ex);
+ actions.add(row);
+ addresses.add(location != null ? location.getHostnamePort() : "null location");
+ }
+
+ private RetriesExhaustedWithDetailsException makeException() {
+ return new RetriesExhaustedWithDetailsException(
+ new ArrayList<Throwable>(throwables),
+ new ArrayList<Row>(actions), new ArrayList<String>(addresses));
+ }
+
+ public void clear() {
+ throwables.clear();
+ actions.clear();
+ addresses.clear();
+ }
+ }
+
+ public AsyncProcess(HConnection hc, byte[] tableName, ExecutorService pool,
+ AsyncProcessCallback<CResult> callback, Configuration conf) {
+ this.hConnection = hc;
+ this.tableName = tableName;
+ this.pool = pool;
+ this.callback = callback;
+
+ this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
+ HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
+ this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
+ HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
+
+ this.maxTotalConcurrentTasks = conf.getInt("hbase.client.max.total.tasks", 200);
+
+ // With one, we ensure that the ordering of the queries is respected: we don't start
+ // a set of operations on a region before the previous one is done. As well, this limits
+ // the pressure we put on the region server.
+ this.maxConcurrentTasksPerRegion = conf.getInt("hbase.client.max.perregion.tasks", 1);
+
+ this.useServerTrackerForRetries =
+ conf.getBoolean(HConnectionManager.RETRIES_BY_SERVER_KEY, true);
+
+ if (this.useServerTrackerForRetries) {
+ // Server tracker allows us to do faster, and yet useful (hopefully), retries.
+ // However, if we are too useful, we might fail very quickly due to retry count limit.
+ // To avoid this, we are going to cheat for now (see HBASE-7659), and calculate maximum
+ // retry time if normal retries were used. Then we will retry until this time runs out.
+ // If we keep hitting one server, the net effect will be the incremental backoff, and
+ // essentially the same number of retries as planned. If we have to do faster retries,
+ // we will do more retries in aggregate, but the user will be none the wiser.
+ this.serverTrackerTimeout = 0;
+ for (int i = 0; i < this.numTries; ++i) {
+ serverTrackerTimeout += ConnectionUtils.getPauseTime(this.pause, i);
+ }
+ }
+ }
+
+ /**
+ * Extract from the rows list what we can submit. The rows we can not submit are kept in the
+ * list.
+ *
+ * @param rows - the submitted row. Modified by the method: we remove the rows we took.
+ * @param atLeastOne true if we should submit at least a subset.
+ */
+ public void submit(List<? extends Row> rows, boolean atLeastOne) throws InterruptedIOException {
+ if (rows.isEmpty()){
+ return;
+ }
+
+ Map<HRegionLocation, MultiAction<Row>> actionsByServer =
+ new HashMap<HRegionLocation, MultiAction<Row>>();
+ List<Action<Row>> retainedActions = new ArrayList<Action<Row>>(rows.size());
+
+ do {
+ Map<String, Boolean> regionIncluded = new HashMap<String, Boolean>();
+ long currentTaskNumber = waitForMaximumCurrentTasks(maxTotalConcurrentTasks);
+ int posInList = -1;
+ Iterator<? extends Row> it = rows.iterator();
+ while (it.hasNext()) {
+ Row r = it.next();
+ HRegionLocation loc = findDestLocation(r, 1, posInList, false, regionIncluded);
+
+ if (loc != null) { // loc is null if the dest is too busy or there is an error
+ Action<Row> action = new Action<Row>(r, ++posInList);
+ retainedActions.add(action);
+ addAction(loc, action, actionsByServer);
+ it.remove();
+ }
+ }
+
+ if (retainedActions.isEmpty() && atLeastOne && !hasError()) {
+ waitForNextTaskDone(currentTaskNumber);
+ }
+
+ } while (retainedActions.isEmpty() && atLeastOne && !hasError());
+
+ HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
+ sendMultiAction(retainedActions, actionsByServer, 1, errorsByServer);
+ }
+
+ /**
+ * Group the actions per region server.
+ *
+ * @param loc - the destination. Must not be null.
+ * @param action - the action to add to the multiaction
+ * @param actionsByServer the multiaction per server
+ */
+ private void addAction(HRegionLocation loc, Action<Row> action, Map<HRegionLocation,
+ MultiAction<Row>> actionsByServer) {
+ final byte[] regionName = loc.getRegionInfo().getRegionName();
+ MultiAction<Row> multiAction = actionsByServer.get(loc);
+ if (multiAction == null) {
+ multiAction = new MultiAction<Row>();
+ actionsByServer.put(loc, multiAction);
+ }
+
+ multiAction.add(regionName, action);
+ }
+
+ /**
+ * Find the destination, if this destination is not considered as busy.
+ *
+ * @param row the row
+ * @param numAttempt the num attempt
+ * @param posInList the position in the list
+ * @param force if we must submit whatever the server load
+ * @param regionStatus the
+ * @return null if we should not submit, the destination otherwise.
+ */
+ private HRegionLocation findDestLocation(Row row, int numAttempt,
+ int posInList, boolean force,
+ Map<String, Boolean> regionStatus) {
+ HRegionLocation loc = null;
+ IOException locationException = null;
+ try {
+ loc = hConnection.locateRegion(this.tableName, row.getRow());
+ if (loc == null) {
+ locationException = new IOException("No location found, aborting submit for" +
+ " tableName=" + Bytes.toString(tableName) +
+ " rowkey=" + Arrays.toString(row.getRow()));
+ }
+ } catch (IOException e) {
+ locationException = e;
+ }
+ if (locationException != null) {
+ // There are multiple retries in locateRegion already. No need to add new.
+ // We can't continue with this row, hence it's the last retry.
+ manageError(numAttempt, posInList, row, false, locationException, null);
+ return null;
+ }
+
+ if (force) {
+ return loc;
+ }
+
+ String regionName = loc.getRegionInfo().getEncodedName();
+ Boolean addIt = regionStatus.get(regionName);
+ if (addIt == null) {
+ addIt = canTakeNewOperations(regionName);
+ regionStatus.put(regionName, addIt);
+ }
+
+ return addIt ? loc : null;
+ }
+
+
+ /**
+ * Check if we should send new operations to this region.
+ *
+ * @param encodedRegionName region name
+ * @return true if this region is considered as busy.
+ */
+ protected boolean canTakeNewOperations(String encodedRegionName) {
+ AtomicInteger ct = taskCounterPerRegion.get(encodedRegionName);
+ return ct == null || ct.get() < maxConcurrentTasksPerRegion;
+ }
+
+ /**
+ * Submit immediately the list of rows, whatever the server status. Kept for backward
+ * compatibility: it allows to be used with the batch interface that return an array of objects.
+ *
+ * @param rows the list of rows.
+ */
+ public void submitAll(List<? extends Row> rows) {
+ List<Action<Row>> actions = new ArrayList<Action<Row>>(rows.size());
+
+ // The position will be used by the processBatch to match the object array returned.
+ int posInList = -1;
+ for (Row r : rows) {
+ posInList++;
+ Action<Row> action = new Action<Row>(r, posInList);
+ actions.add(action);
+ }
+ HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
+ submit(actions, actions, 1, true, errorsByServer);
+ }
+
+
+ /**
+ * Group a list of actions per region servers, and send them. The created MultiActions are
+ * added to the inProgress list.
+ *
+ * @param initialActions - the full list of the actions in progress
+ * @param currentActions - the list of row to submit
+ * @param numAttempt - the current numAttempt (first attempt is 1)
+ * @param force - true if we submit the rowList without taking into account the server load
+ */
+ private void submit(List<Action<Row>> initialActions,
+ List<Action<Row>> currentActions, int numAttempt, boolean force,
+ final HConnectionManager.ServerErrorTracker errorsByServer) {
+ // group per location => regions server
+ final Map<HRegionLocation, MultiAction<Row>> actionsByServer =
+ new HashMap<HRegionLocation, MultiAction<Row>>();
+
+ // We have the same policy for a single region per call to submit: we don't want
+ // to send half of the actions because the status changed in the middle. So we keep the
+ // status
+ Map<String, Boolean> regionIncluded = new HashMap<String, Boolean>();
+
+ for (Action<Row> action : currentActions) {
+ HRegionLocation loc = findDestLocation(
+ action.getAction(), 1, action.getOriginalIndex(), force, regionIncluded);
+
+ if (loc != null) {
+ addAction(loc, action, actionsByServer);
+ }
+ }
+
+ if (!actionsByServer.isEmpty()) {
+ sendMultiAction(initialActions, actionsByServer, numAttempt, errorsByServer);
+ }
+ }
+
+ /**
+ * Send a multi action structure to the servers, after a delay depending on the attempt
+ * number. Asynchronous.
+ *
+ * @param initialActions the list of the actions, flat.
+ * @param actionsByServer the actions structured by regions
+ * @param numAttempt the attempt number.
+ */
+ public void sendMultiAction(final List<Action<Row>> initialActions,
+ Map<HRegionLocation, MultiAction<Row>> actionsByServer,
+ final int numAttempt,
+ final HConnectionManager.ServerErrorTracker errorsByServer) {
+
+ // Send the queries and add them to the inProgress list
+ for (Map.Entry<HRegionLocation, MultiAction<Row>> e : actionsByServer.entrySet()) {
+ final HRegionLocation loc = e.getKey();
+ final MultiAction<Row> multi = e.getValue();
+ final String regionName = loc.getRegionInfo().getEncodedName();
+
+ incTaskCounters(regionName);
+
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ MultiResponse res;
+ try {
+ ServerCallable<MultiResponse> callable = createCallable(loc, multi);
+ try {
+ res = callable.withoutRetries();
+ } catch (IOException e) {
+ LOG.warn("The call to the RS failed, we don't know where we stand. regionName="
+ + regionName, e);
+ resubmitAll(initialActions, multi, loc, numAttempt + 1, e, errorsByServer);
+ return;
+ }
+
+ receiveMultiAction(initialActions, multi, loc, res, numAttempt, errorsByServer);
+ } finally {
+ decTaskCounters(regionName);
+ }
+ }
+ };
+
+ try {
+ this.pool.submit(runnable);
+ } catch (RejectedExecutionException ree) {
+ // This should never happen. But as the pool is provided by the end user, let's secure
+ // this a little.
+ decTaskCounters(regionName);
+ LOG.warn("The task was rejected by the pool. This is unexpected. " +
+ "regionName=" + regionName, ree);
+ // We're likely to fail again, but this will increment the attempt counter, so it will
+ // finish.
+ resubmitAll(initialActions, multi, loc, numAttempt + 1, ree, errorsByServer);
+ }
+ }
+ }
+
+ /**
+ * Create a callable. Isolated to be easily overridden in the tests.
+ */
+ protected ServerCallable<MultiResponse> createCallable(
+ final HRegionLocation loc, final MultiAction<Row> multi) {
+
+ return new MultiServerCallable<Row>(hConnection, tableName, loc, multi);
+ }
+
+ /**
+ * Check that we can retry acts accordingly: logs, set the error status, call the callbacks.
+ *
+ * @param numAttempt the number of this attempt
+ * @param originalIndex the position in the list sent
+ * @param row the row
+ * @param canRetry if false, we won't retry whatever the settings.
+ * @param throwable the throwable, if any (can be null)
+ * @param location the location, if any (can be null)
+ * @return true if the action can be retried, false otherwise.
+ */
+ private boolean manageError(int numAttempt, int originalIndex, Row row, boolean canRetry,
+ Throwable throwable, HRegionLocation location) {
+ if (canRetry) {
+ if (numAttempt >= numTries ||
+ (throwable != null && throwable instanceof DoNotRetryIOException)) {
+ canRetry = false;
+ }
+ }
+ byte[] region = location == null ? null : location.getRegionInfo().getEncodedNameAsBytes();
+
+ if (canRetry && callback != null) {
+ canRetry = callback.retriableFailure(originalIndex, row, region, throwable);
+ }
+
+ if (canRetry) {
+ if (LOG.isTraceEnabled()) {
+ retriedErrors.add(throwable, row, location);
+ }
+ } else {
+ if (callback != null) {
+ callback.failure(originalIndex, region, row, throwable);
+ }
+ this.hasError.set(true);
+ errors.add(throwable, row, location);
+ }
+
+ return canRetry;
+ }
+
+ /**
+ * Resubmit all the actions from this multiaction after a failure.
+ *
+ * @param initialActions the full initial action list
+ * @param rsActions the actions still to do from the initial list
+ * @param location the destination
+ * @param numAttempt the number of attempts so far
+ * @param t the throwable (if any) that caused the resubmit
+ */
+ private void resubmitAll(List<Action<Row>> initialActions, MultiAction<Row> rsActions,
+ HRegionLocation location, int numAttempt, Throwable t,
+ HConnectionManager.ServerErrorTracker errorsByServer) {
+ // Do not use the exception for updating cache because it might be coming from
+ // any of the regions in the MultiAction.
+ hConnection.updateCachedLocations(tableName,
+ rsActions.actions.values().iterator().next().get(0).getAction().getRow(), null, location);
+ errorsByServer.reportServerError(location);
+
+ List<Action<Row>> toReplay = new ArrayList<Action<Row>>();
+ for (List<Action<Row>> actions : rsActions.actions.values()) {
+ for (Action<Row> action : actions) {
+ if (manageError(numAttempt, action.getOriginalIndex(), action.getAction(),
+ true, t, location)) {
+ toReplay.add(action);
+ }
+ }
+ }
+
+ submit(initialActions, toReplay, numAttempt, true, errorsByServer);
+ }
+
+ /**
+ * Called when we receive the result of a server query.
+ *
+ * @param initialActions - the whole action list
+ * @param rsActions - the actions for this location
+ * @param location - the location
+ * @param responses - the response, if any
+ * @param numAttempt - the attempt
+ */
+ private void receiveMultiAction(List<Action<Row>> initialActions,
+ MultiAction<Row> rsActions, HRegionLocation location,
+ MultiResponse responses, int numAttempt,
+ HConnectionManager.ServerErrorTracker errorsByServer) {
+
+ if (responses == null) {
+ LOG.info("Attempt #" + numAttempt + " failed for all operations on server " +
+ location.getServerName() + " , trying to resubmit.");
+ resubmitAll(initialActions, rsActions, location, numAttempt + 1, null, errorsByServer);
+ return;
+ }
+
+ // Success or partial success
+ // Analyze detailed results. We can still have individual failures to be redo.
+ // two specific throwables are managed:
+ // - DoNotRetryIOException: we continue to retry for other actions
+ // - RegionMovedException: we update the cache with the new region location
+
+ List<Action<Row>> toReplay = new ArrayList<Action<Row>>();
+ Throwable throwable = null;
+
+ int failureCount = 0;
+ boolean canRetry = true;
+ for (Map.Entry<byte[], List<Pair<Integer, Object>>> resultsForRS :
+ responses.getResults().entrySet()) {
+
+ for (Pair<Integer, Object> regionResult : resultsForRS.getValue()) {
+ Object result = regionResult.getSecond();
+
+ // Failure: retry if it's make sense else update the errors lists
+ if (result == null || result instanceof Throwable) {
+ throwable = (Throwable) result;
+ Action<Row> correspondingAction = initialActions.get(regionResult.getFirst());
+ Row row = correspondingAction.getAction();
+
+ if (failureCount++ == 0) { // We're doing this once per location.
+ hConnection.updateCachedLocations(this.tableName, row.getRow(), result, location);
+ if (errorsByServer != null) {
+ errorsByServer.reportServerError(location);
+ canRetry = errorsByServer.canRetryMore();
+ }
+ }
+
+ if (manageError(numAttempt, correspondingAction.getOriginalIndex(), row, canRetry,
+ throwable, location)) {
+ toReplay.add(correspondingAction);
+ }
+ } else { // success
+ if (callback != null) {
+ Action<Row> correspondingAction = initialActions.get(regionResult.getFirst());
+ Row row = correspondingAction.getAction();
+ //noinspection unchecked
+ this.callback.success(correspondingAction.getOriginalIndex(),
+ resultsForRS.getKey(), row, (CResult) result);
+ }
+ }
+ }
+ }
+
+ if (!toReplay.isEmpty()) {
+ if (numAttempt > 2) {
+ // We use this value to have some logs when we have multiple failures, but not too many
+ // logs as errors are to be expected wehn region moves, split and so on
+ LOG.debug("Attempt #" + numAttempt + " failed for " + failureCount +
+ " operations on server " + location.getServerName() + ", resubmitting " +
+ toReplay.size() + ", tableName=" + Bytes.toString(tableName) +
+ ", last exception was: " + throwable);
+ }
+ long backOffTime = (errorsByServer != null ?
+ errorsByServer.calculateBackoffTime(location, pause) :
+ ConnectionUtils.getPauseTime(pause, numAttempt));
+ try {
+ Thread.sleep(backOffTime);
+ } catch (InterruptedException e) {
+ LOG.warn("Not sent: " + toReplay.size() +
+ " operations, tableName=" + Bytes.toString(tableName), e);
+ Thread.interrupted();
+ return;
+ }
+
+ submit(initialActions, toReplay, numAttempt + 1, true, errorsByServer);
+ } else if (failureCount != 0) {
+ LOG.warn("Attempt #" + numAttempt + " failed for " + failureCount +
+ " operations on server " + location.getServerName() + " NOT resubmitting." +
+ ", tableName=" + Bytes.toString(tableName));
+ }
+ }
+
+ /**
+ * Waits for another task to finish.
+ * @param currentNumberOfTask - the number of task finished when calling the method.
+ */
+ protected void waitForNextTaskDone(long currentNumberOfTask) throws InterruptedIOException {
+ while (currentNumberOfTask == tasksDone.get()) {
+ try {
+ synchronized (this.tasksDone) {
+ this.tasksDone.wait(100);
+ }
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException("Interrupted." +
+ " currentNumberOfTask=" + currentNumberOfTask +
+ ", tableName=" + Bytes.toString(tableName) + ", tasksDone=" + tasksDone.get());
+ }
+ }
+ }
+
+ /**
+ * Wait until the async does not have more than max tasks in progress.
+ */
+ private long waitForMaximumCurrentTasks(int max) throws InterruptedIOException {
+ long lastLog = 0;
+ long currentTasksDone = this.tasksDone.get();
+
+ while ((tasksSent.get() - currentTasksDone) > max) {
+ long now = EnvironmentEdgeManager.currentTimeMillis();
+ if (now > lastLog + 5000) {
+ lastLog = now;
+ LOG.info(Bytes.toString(tableName) +
+ ": Waiting for the global number of tasks to be equals or less than " + max +
+ ", currently it's " + this.tasksDone.get());
+ }
+ waitForNextTaskDone(currentTasksDone);
+ currentTasksDone = this.tasksDone.get();
+ }
+
+ return currentTasksDone;
+ }
+
+ /**
+ * Wait until all tasks are executed, successfully or not.
+ */
+ public void waitUntilDone() throws InterruptedIOException {
+ waitForMaximumCurrentTasks(0);
+ }
+
+
+ public boolean hasError() {
+ return hasError.get();
+ }
+
+ public List<? extends Row> getFailedOperations() {
+ return errors.actions;
+ }
+
+ /**
+ * Clean the errors stacks. Should be called only when there are no actions in progress.
+ */
+ public void clearErrors() {
+ errors.clear();
+ retriedErrors.clear();
+ hasError.set(false);
+ }
+
+ public RetriesExhaustedWithDetailsException getErrors() {
+ return errors.makeException();
+ }
+
+ /**
+ * incrementer the tasks counters for a given region. MT safe.
+ */
+ protected void incTaskCounters(String encodedRegionName) {
+ tasksSent.incrementAndGet();
+
+ AtomicInteger counterPerServer = taskCounterPerRegion.get(encodedRegionName);
+ if (counterPerServer == null) {
+ taskCounterPerRegion.putIfAbsent(encodedRegionName, new AtomicInteger());
+ counterPerServer = taskCounterPerRegion.get(encodedRegionName);
+ }
+ counterPerServer.incrementAndGet();
+ }
+
+ /**
+ * Decrements the counters for a given region
+ */
+ protected void decTaskCounters(String encodedRegionName) {
+ AtomicInteger counterPerServer = taskCounterPerRegion.get(encodedRegionName);
+ counterPerServer.decrementAndGet();
+
+ tasksDone.incrementAndGet();
+ synchronized (tasksDone) {
+ tasksDone.notifyAll();
+ }
+ }
+
+ /**
+ * Creates the server error tracker to use inside process.
+ * Currently, to preserve the main assumption about current retries, and to work well with
+ * the retry-limit-based calculation, the calculation is local per Process object.
+ * We may benefit from connection-wide tracking of server errors.
+ * @return ServerErrorTracker to use, null if there is no ServerErrorTracker on this connection
+ */
+ protected HConnectionManager.ServerErrorTracker createServerErrorTracker() {
+ if (useServerTrackerForRetries){
+ return new HConnectionManager.ServerErrorTracker(this.serverTrackerTimeout);
+ }else {
+ return null;
+ }
+ }
+}
Added: hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java?rev=1496157&view=auto
==============================================================================
--- hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java (added)
+++ hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestAsyncProcess.java Mon Jun 24 18:42:30 2013
@@ -0,0 +1,635 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.util.Threads;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+@Category(MediumTests.class)
+public class TestAsyncProcess {
+ private static final byte[] DUMMY_TABLE = "DUMMY_TABLE".getBytes();
+ private static final byte[] DUMMY_BYTES_1 = "DUMMY_BYTES_1".getBytes();
+ private static final byte[] DUMMY_BYTES_2 = "DUMMY_BYTES_2".getBytes();
+ private static final byte[] FAILS = "FAILS".getBytes();
+ private Configuration conf = new Configuration();
+
+
+ private static ServerName sn = new ServerName("localhost:10,1254");
+ private static HRegionInfo hri1 = new HRegionInfo(DUMMY_BYTES_1);
+ private static HRegionInfo hri2 = new HRegionInfo(DUMMY_BYTES_1);
+ private static HRegionLocation loc1 = new HRegionLocation(hri1, sn);
+ private static HRegionLocation loc2 = new HRegionLocation(hri2, sn);
+
+ private static final String success = "success";
+ private static Exception failure = new Exception("failure");
+
+ static class MyAsyncProcess<Res> extends AsyncProcess<Res> {
+
+ public MyAsyncProcess(HConnection hc,
+ AsyncProcessCallback<Res> callback, Configuration conf) {
+ super(hc, DUMMY_TABLE, new ThreadPoolExecutor(1, 10, 60, TimeUnit.SECONDS,
+ new SynchronousQueue<Runnable>(),
+ Threads.newDaemonThreadFactory("test-TestAsyncProcess"))
+ , callback, conf);
+ }
+
+ /**
+ * Do not call a server, fails if the rowkey of the operation is{@link #FAILS}
+ */
+ @Override
+ protected ServerCallable<MultiResponse> createCallable(
+ final HRegionLocation loc, final MultiAction<Row> multi) {
+
+ final MultiResponse mr = new MultiResponse();
+ for (Map.Entry<byte[], List<Action<Row>>> entry : multi.actions.entrySet()) {
+ for (Action a : entry.getValue()) {
+ if (Arrays.equals(FAILS, a.getAction().getRow())) {
+ mr.add(loc.getRegionInfo().getRegionName(), a.getOriginalIndex(), failure);
+ } else {
+ mr.add(loc.getRegionInfo().getRegionName(), a.getOriginalIndex(), success);
+ }
+ }
+ }
+
+ return new MultiServerCallable<Row>(hConnection, tableName, null, null) {
+ @Override
+ public MultiResponse withoutRetries() {
+ return mr;
+ }
+ };
+ }
+ }
+
+ /**
+ * Returns our async process.
+ */
+ static class MyConnectionImpl extends HConnectionManager.HConnectionImplementation {
+ MyAsyncProcess<?> ap;
+ final static Configuration c = new Configuration();
+
+ static {
+ c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
+ }
+
+ protected MyConnectionImpl() {
+ super(c);
+ }
+
+ protected MyConnectionImpl(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ protected <R> AsyncProcess createAsyncProcess(byte[] tableName, ExecutorService pool,
+ AsyncProcess.AsyncProcessCallback<R> callback,
+ Configuration conf) {
+ ap = new MyAsyncProcess<R>(this, callback, conf);
+ return ap;
+ }
+
+ @Override
+ public HRegionLocation locateRegion(final byte[] tableName,
+ final byte[] row) {
+ return loc1;
+ }
+
+ }
+
+ @Test
+ public void testSubmit() throws Exception {
+ HConnection hc = createHConnection();
+ AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
+
+ List<Put> puts = new ArrayList<Put>();
+ puts.add(createPut(true, true));
+
+ ap.submit(puts, false);
+ Assert.assertTrue(puts.isEmpty());
+ }
+
+ @Test
+ public void testSubmitWithCB() throws Exception {
+ HConnection hc = createHConnection();
+ MyCB mcb = new MyCB();
+ AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+
+ List<Put> puts = new ArrayList<Put>();
+ puts.add(createPut(true, true));
+
+ ap.submit(puts, false);
+ Assert.assertTrue(puts.isEmpty());
+
+ while (!(mcb.successCalled.get() == 1) && !ap.hasError()) {
+ Thread.sleep(1);
+ }
+ Assert.assertEquals(mcb.successCalled.get(), 1);
+ }
+
+ @Test
+ public void testSubmitBusyRegion() throws Exception {
+ HConnection hc = createHConnection();
+ AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
+
+ List<Put> puts = new ArrayList<Put>();
+ puts.add(createPut(true, true));
+
+ ap.incTaskCounters(hri1.getEncodedName());
+ ap.submit(puts, false);
+ Assert.assertEquals(puts.size(), 1);
+
+ ap.decTaskCounters(hri1.getEncodedName());
+ ap.submit(puts, false);
+ Assert.assertTrue(puts.isEmpty());
+ }
+
+ @Test
+ public void testFail() throws Exception {
+ HConnection hc = createHConnection();
+ MyCB mcb = new MyCB();
+ AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+
+ List<Put> puts = new ArrayList<Put>();
+ Put p = createPut(true, false);
+ puts.add(p);
+
+ ap.submit(puts, false);
+ Assert.assertTrue(puts.isEmpty());
+
+ while (!ap.hasError()) {
+ Thread.sleep(1);
+ }
+
+ Assert.assertEquals(0, mcb.successCalled.get());
+ Assert.assertEquals(2, mcb.retriableFailure.get());
+ Assert.assertEquals(1, mcb.failureCalled.get());
+
+ Assert.assertEquals(1, ap.getErrors().exceptions.size());
+ Assert.assertTrue("was: " + ap.getErrors().exceptions.get(0),
+ failure.equals(ap.getErrors().exceptions.get(0)));
+ Assert.assertTrue("was: " + ap.getErrors().exceptions.get(0),
+ failure.equals(ap.getErrors().exceptions.get(0)));
+
+ Assert.assertEquals(1, ap.getFailedOperations().size());
+ Assert.assertTrue("was: " + ap.getFailedOperations().get(0),
+ p.equals(ap.getFailedOperations().get(0)));
+ }
+
+ @Test
+ public void testWaitForNextTaskDone() throws IOException {
+ HConnection hc = createHConnection();
+ MyCB mcb = new MyCB();
+ final AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+ ap.tasksSent.incrementAndGet();
+
+ final AtomicBoolean checkPoint = new AtomicBoolean(false);
+ final AtomicBoolean checkPoint2 = new AtomicBoolean(false);
+
+ Thread t = new Thread(){
+ @Override
+ public void run(){
+ Threads.sleep(1000);
+ Assert.assertFalse(checkPoint.get());
+ ap.tasksDone.incrementAndGet();
+ checkPoint2.set(true);
+ }
+ };
+
+ t.start();
+ ap.waitForNextTaskDone(0);
+ checkPoint.set(true);
+ while (!checkPoint2.get()){
+ Threads.sleep(1);
+ }
+ }
+
+ @Test
+ public void testSubmitTrue() throws IOException {
+ HConnection hc = createHConnection();
+ MyCB mcb = new MyCB();
+ final AsyncProcess<Object> ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+ ap.tasksSent.incrementAndGet();
+ final AtomicInteger ai = new AtomicInteger(1);
+ ap.taskCounterPerRegion.put(hri1.getEncodedName(), ai);
+
+ final AtomicBoolean checkPoint = new AtomicBoolean(false);
+ final AtomicBoolean checkPoint2 = new AtomicBoolean(false);
+
+ Thread t = new Thread(){
+ @Override
+ public void run(){
+ Threads.sleep(1000);
+ Assert.assertFalse(checkPoint.get());
+ ai.decrementAndGet();
+ ap.tasksDone.incrementAndGet();
+ checkPoint2.set(true);
+ }
+ };
+
+ List<Put> puts = new ArrayList<Put>();
+ Put p = createPut(true, true);
+ puts.add(p);
+
+ ap.submit(puts, false);
+ Assert.assertFalse(puts.isEmpty());
+
+ t.start();
+
+ ap.submit(puts, true);
+ Assert.assertTrue(puts.isEmpty());
+
+ checkPoint.set(true);
+ while (!checkPoint2.get()){
+ Threads.sleep(1);
+ }
+ }
+
+ @Test
+ public void testFailAndSuccess() throws Exception {
+ HConnection hc = createHConnection();
+ MyCB mcb = new MyCB();
+ AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+
+ List<Put> puts = new ArrayList<Put>();
+ puts.add(createPut(true, false));
+ puts.add(createPut(true, true));
+ puts.add(createPut(true, true));
+
+ ap.submit(puts, false);
+ Assert.assertTrue(puts.isEmpty());
+
+ while (!ap.hasError()) {
+ Thread.sleep(1);
+ }
+ Assert.assertEquals(mcb.successCalled.get(), 2);
+ Assert.assertEquals(mcb.retriableFailure.get(), 2);
+ Assert.assertEquals(mcb.failureCalled.get(), 1);
+
+ Assert.assertEquals(1, ap.getErrors().actions.size());
+
+
+ puts.add(createPut(true, true));
+ ap.submit(puts, false);
+ Assert.assertTrue(puts.isEmpty());
+
+ while (mcb.successCalled.get() != 3) {
+ Thread.sleep(1);
+ }
+ Assert.assertEquals(mcb.retriableFailure.get(), 2);
+ Assert.assertEquals(mcb.failureCalled.get(), 1);
+
+ ap.clearErrors();
+ Assert.assertTrue(ap.getErrors().actions.isEmpty());
+ }
+
+ @Test
+ public void testFlush() throws Exception {
+ HConnection hc = createHConnection();
+ MyCB mcb = new MyCB();
+ AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+
+ List<Put> puts = new ArrayList<Put>();
+ puts.add(createPut(true, false));
+ puts.add(createPut(true, true));
+ puts.add(createPut(true, true));
+
+ ap.submit(puts, false);
+ ap.waitUntilDone();
+
+ Assert.assertEquals(mcb.successCalled.get(), 2);
+ Assert.assertEquals(mcb.retriableFailure.get(), 2);
+ Assert.assertEquals(mcb.failureCalled.get(), 1);
+
+ Assert.assertEquals(1, ap.getFailedOperations().size());
+ }
+
+ @Test
+ public void testMaxTask() throws Exception {
+ HConnection hc = createHConnection();
+ final AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
+
+ for (int i = 0; i < 1000; i++) {
+ ap.incTaskCounters("dummy");
+ }
+
+ final Thread myThread = Thread.currentThread();
+
+ Thread t = new Thread() {
+ public void run() {
+ Threads.sleep(2000);
+ myThread.interrupt();
+ }
+ };
+
+ List<Put> puts = new ArrayList<Put>();
+ puts.add(createPut(true, true));
+
+ t.start();
+
+ try {
+ ap.submit(puts, false);
+ Assert.fail("We should have been interrupted.");
+ } catch (InterruptedIOException expected) {
+ }
+
+ final long sleepTime = 2000;
+
+ Thread t2 = new Thread() {
+ public void run() {
+ Threads.sleep(sleepTime);
+ while (ap.tasksDone.get() > 0) {
+ ap.decTaskCounters("dummy");
+ }
+ }
+ };
+ t2.start();
+
+ long start = System.currentTimeMillis();
+ ap.submit(new ArrayList<Row>(), false);
+ long end = System.currentTimeMillis();
+
+ //Adds 100 to secure us against approximate timing.
+ Assert.assertTrue(start + 100L + sleepTime > end);
+ }
+
+
+ private class MyCB implements AsyncProcess.AsyncProcessCallback<Object> {
+ private AtomicInteger successCalled = new AtomicInteger(0);
+ private AtomicInteger failureCalled = new AtomicInteger(0);
+ private AtomicInteger retriableFailure = new AtomicInteger(0);
+
+
+ @Override
+ public void success(int originalIndex, byte[] region, Row row, Object o) {
+ successCalled.incrementAndGet();
+ }
+
+ @Override
+ public boolean failure(int originalIndex, byte[] region, Row row, Throwable t) {
+ failureCalled.incrementAndGet();
+ return true;
+ }
+
+ @Override
+ public boolean retriableFailure(int originalIndex, Row row, byte[] region,
+ Throwable exception) {
+ // We retry once only.
+ return (retriableFailure.incrementAndGet() < 2);
+ }
+ }
+
+
+ private static HConnection createHConnection() throws IOException {
+ HConnection hc = Mockito.mock(HConnection.class);
+
+ Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
+ Mockito.eq(DUMMY_BYTES_1), Mockito.anyBoolean())).thenReturn(loc1);
+ Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
+ Mockito.eq(DUMMY_BYTES_1))).thenReturn(loc1);
+
+ Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
+ Mockito.eq(DUMMY_BYTES_2), Mockito.anyBoolean())).thenReturn(loc2);
+ Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
+ Mockito.eq(DUMMY_BYTES_2))).thenReturn(loc2);
+
+ Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
+ Mockito.eq(FAILS), Mockito.anyBoolean())).thenReturn(loc2);
+ Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
+ Mockito.eq(FAILS))).thenReturn(loc2);
+
+ return hc;
+ }
+
+ @Test
+ public void testHTablePutSuccess() throws Exception {
+ HTable ht = Mockito.mock(HTable.class);
+ HConnection hc = createHConnection();
+ ht.ap = new MyAsyncProcess<Object>(hc, null, conf);
+
+ Put put = createPut(true, true);
+
+ Assert.assertEquals(0, ht.getWriteBufferSize());
+ ht.put(put);
+ Assert.assertEquals(0, ht.getWriteBufferSize());
+ }
+
+ private void doHTableFailedPut(boolean bufferOn) throws Exception {
+ HTable ht = new HTable();
+ HConnection hc = createHConnection();
+ MyCB mcb = new MyCB(); // This allows to have some hints on what's going on.
+ ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+ ht.setAutoFlush(true, true);
+ if (bufferOn) {
+ ht.setWriteBufferSize(1024L * 1024L);
+ } else {
+ ht.setWriteBufferSize(0L);
+ }
+
+ Put put = createPut(true, false);
+
+ Assert.assertEquals(0L, ht.currentWriteBufferSize);
+ try {
+ ht.put(put);
+ if (bufferOn) {
+ ht.flushCommits();
+ }
+ Assert.fail();
+ } catch (RetriesExhaustedException expected) {
+ }
+ Assert.assertEquals(0L, ht.currentWriteBufferSize);
+ Assert.assertEquals(0, mcb.successCalled.get());
+ Assert.assertEquals(2, mcb.retriableFailure.get());
+ Assert.assertEquals(1, mcb.failureCalled.get());
+
+ // This should not raise any exception, puts have been 'received' before by the catch.
+ ht.close();
+ }
+
+ @Test
+ public void testHTableFailedPutWithBuffer() throws Exception {
+ doHTableFailedPut(true);
+ }
+
+ @Test
+ public void doHTableFailedPutWithoutBuffer() throws Exception {
+ doHTableFailedPut(false);
+ }
+
+ @Test
+ public void testHTableFailedPutAndNewPut() throws Exception {
+ HTable ht = new HTable();
+ HConnection hc = createHConnection();
+ MyCB mcb = new MyCB(); // This allows to have some hints on what's going on.
+ ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+ ht.setAutoFlush(false, true);
+ ht.setWriteBufferSize(0);
+
+ Put p = createPut(true, false);
+ ht.put(p);
+
+ ht.ap.waitUntilDone(); // Let's do all the retries.
+
+ // We're testing that we're behaving as we were behaving in 0.94: sending exceptions in the
+ // doPut if it fails.
+ // This said, it's not a very easy going behavior. For example, when we insert a list of
+ // puts, we may raise an exception in the middle of the list. It's then up to the caller to
+ // manage what was inserted, what was tried but failed, and what was not even tried.
+ p = createPut(true, true);
+ Assert.assertEquals(0, ht.writeAsyncBuffer.size());
+ try {
+ ht.put(p);
+ Assert.fail();
+ } catch (RetriesExhaustedException expected) {
+ }
+ Assert.assertEquals("the put should not been inserted.", 0, ht.writeAsyncBuffer.size());
+ }
+
+
+ @Test
+ public void testWithNoClearOnFail() throws IOException {
+ HTable ht = new HTable();
+ HConnection hc = createHConnection();
+ MyCB mcb = new MyCB();
+ ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
+ ht.setAutoFlush(false, false);
+
+ Put p = createPut(true, false);
+ ht.put(p);
+ Assert.assertEquals(0, ht.writeAsyncBuffer.size());
+ try {
+ ht.flushCommits();
+ } catch (RetriesExhaustedWithDetailsException expected) {
+ }
+ Assert.assertEquals(1, ht.writeAsyncBuffer.size());
+
+ try {
+ ht.close();
+ } catch (RetriesExhaustedWithDetailsException expected) {
+ }
+ Assert.assertEquals(1, ht.writeAsyncBuffer.size());
+ }
+
+ @Test
+ public void testBatch() throws IOException, InterruptedException {
+ HTable ht = new HTable();
+ ht.connection = new MyConnectionImpl();
+
+ List<Put> puts = new ArrayList<Put>();
+ puts.add(createPut(true, true));
+ puts.add(createPut(true, true));
+ puts.add(createPut(true, true));
+ puts.add(createPut(true, true));
+ puts.add(createPut(true, false)); // <=== the bad apple, position 4
+ puts.add(createPut(true, true));
+ puts.add(createPut(true, false)); // <=== another bad apple, position 6
+
+ Object[] res = new Object[puts.size()];
+ try {
+ ht.processBatch(puts, res);
+ Assert.fail();
+ } catch (RetriesExhaustedException expected) {
+ }
+
+ Assert.assertEquals(res[0], success);
+ Assert.assertEquals(res[1], success);
+ Assert.assertEquals(res[2], success);
+ Assert.assertEquals(res[3], success);
+ Assert.assertEquals(res[4], failure);
+ Assert.assertEquals(res[5], success);
+ Assert.assertEquals(res[6], failure);
+ }
+
+ @Test
+ public void testErrorsServers() throws InterruptedIOException,
+ RetriesExhaustedWithDetailsException {
+ HTable ht = new HTable();
+ Configuration configuration = new Configuration(conf);
+ configuration.setBoolean(HConnectionManager.RETRIES_BY_SERVER_KEY, true);
+ configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 20);
+
+ MyConnectionImpl mci = new MyConnectionImpl(configuration);
+ ht.connection = mci;
+ ht.ap = new MyAsyncProcess<Object>(mci, null, configuration);
+
+
+ Assert.assertTrue(ht.ap.useServerTrackerForRetries);
+ Assert.assertNotNull(ht.ap.createServerErrorTracker());
+ Assert.assertTrue(ht.ap.serverTrackerTimeout > 10000);
+ ht.ap.serverTrackerTimeout = 1;
+
+
+ Put p = createPut(true, false);
+ ht.setAutoFlush(false);
+ ht.put(p);
+
+ long start = System.currentTimeMillis();
+ try {
+ ht.flushCommits();
+ Assert.fail();
+ } catch (RetriesExhaustedWithDetailsException expected) {
+ }
+ // Checking that the ErrorsServers came into play and made us stop immediately
+ Assert.assertTrue((System.currentTimeMillis() - start) < 10000);
+ }
+
+
+ /**
+ * @param reg1 if true, creates a put on region 1, region 2 otherwise
+ * @param success if true, the put will succeed.
+ * @return a put
+ */
+ private Put createPut(boolean reg1, boolean success) {
+ Put p;
+ if (!success) {
+ p = new Put(FAILS);
+ } else if (reg1) {
+ p = new Put(DUMMY_BYTES_1);
+ } else {
+ p = new Put(DUMMY_BYTES_2);
+ }
+
+ p.add(DUMMY_BYTES_1, DUMMY_BYTES_1, DUMMY_BYTES_1);
+
+ return p;
+ }
+}