You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/04/04 22:49:17 UTC
svn commit: r1464739 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/client/
main/java/org/apache/hadoop/hbase/mapreduce/
main/java/org/apache/hadoop/hbase/util/ test/java/org/apache/hadoop/hbase/
test/java/org/apache/hadoop/hbase/c...
Author: liyin
Date: Thu Apr 4 20:49:16 2013
New Revision: 1464739
URL: http://svn.apache.org/r1464739
Log:
[HBASE-8249] Adds client side context information
Author: rshroff
Summary:
The motive is to capture failures at per op level. There might be situations in
which the retries will succeed, but the overall operation failed from AppServer
perspective, as the timeout is only 5 secs. Hence, we need a way to get more
context about these operations and record them in Scuba.
This diff adds more information to client side operations performed via
HTable. It adds a ThreadLocal Object at HConnectionManager#TableServers level to
record the exceptions occuring for the current operation. Provides APIs
to fetch this context information and hooks to enable/disable this
recording of context.
Test Plan: mr, Also modified TestHCM to verify this feature.
Reviewers: aaiyer, liyintang
Reviewed By: liyintang
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D750533
Task ID: 2186415
Added:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/OperationContext.java
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java?rev=1464739&r1=1464738&r2=1464739&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java Thu Apr 4 20:49:16 2013
@@ -377,4 +377,18 @@ public interface HConnection extends Clo
* @param options -- hbase rpc options to use when talking to regionservers
*/
public void endBatchedLoad(byte[] tableName, HBaseRPCOptions options) throws IOException;
+
+ /**
+ * Get the context of the last operation. This call will return a copy
+ * of the context.
+ *
+ * @return List<OperationContext> operation context of the last operation
+ */
+ public List<OperationContext> getAndResetOperationContext();
+
+ /**
+ * Resets the operation context for the next operation.
+ */
+ public void resetOperationContext();
+
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1464739&r1=1464738&r2=1464739&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Thu Apr 4 20:49:16 2013
@@ -406,6 +406,42 @@ public class HConnectionManager {
// A safety valve, in case the client does not exit the
// fast fail mode for any reason.
private long fastFailClearingTimeMilliSec;
+ private final boolean recordClientContext;
+
+ private ThreadLocal<List<OperationContext>> operationContextPerThread =
+ new ThreadLocal<List<OperationContext>>();
+
+ public void resetOperationContext() {
+ if (!recordClientContext || this.operationContextPerThread == null) {
+ return;
+ }
+
+ List<OperationContext> currContext = this.operationContextPerThread.get();
+
+ if (currContext != null) {
+ currContext.clear();
+ }
+ }
+
+ public List<OperationContext> getAndResetOperationContext() {
+ if (!recordClientContext || this.operationContextPerThread == null) {
+ return null;
+ }
+
+ List<OperationContext> currContext = this.operationContextPerThread.get();
+
+ if (currContext == null) {
+ return null;
+ }
+
+ ArrayList<OperationContext> context =
+ new ArrayList<OperationContext>(currContext);
+
+ // Made a copy, clear the context
+ currContext.clear();
+
+ return context;
+ }
/**
* Keeps track of repeated failures to any region server.
@@ -525,6 +561,9 @@ public class HConnectionManager {
conf.getLong("hbase.client.batched-upload.softflush.timeout.ms", 60000L); // 1 min
batchedUploadUpdatesMap = new ConcurrentHashMap<String,
ConcurrentMap<HRegionInfo, HRegionLocation>>();
+
+ this.recordClientContext = conf.getBoolean("hbase.client.record.context", false);
+
}
private long getPauseTime(int tries) {
@@ -1626,6 +1665,7 @@ public class HConnectionManager {
handleFailureToServer(server, t2);
}
+ updateClientContext(callable, t2);
if (t2 instanceof IOException) {
throw (IOException)t2;
} else {
@@ -1637,6 +1677,21 @@ public class HConnectionManager {
}
}
+ private <T> void updateClientContext(final ServerCallable<T> callable, final Throwable t) {
+ if (!recordClientContext) {
+ return;
+ }
+
+ List<OperationContext> currContext = this.operationContextPerThread.get();
+ if (currContext == null) {
+ currContext = new ArrayList<OperationContext>();
+ this.operationContextPerThread.set(currContext);
+ }
+
+ currContext.add(
+ new OperationContext(callable.location, t));
+ }
+
/**
* Handles failures encountered when communicating with a server.
*
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1464739&r1=1464738&r2=1464739&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTable.java Thu Apr 4 20:49:16 2013
@@ -32,7 +32,6 @@ import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -81,6 +80,7 @@ public class HTable implements HTableInt
protected int scannerCaching;
private int maxKeyValueSize;
private HBaseRPCOptions options;
+ private boolean recordClientContext = false;
private long maxScannerResultSize;
@@ -155,7 +155,7 @@ public class HTable implements HTableInt
HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
this.configuration = conf;
- this.connection.locateRegion(tableName, HConstants.EMPTY_START_ROW);
+ this.getConnectionAndResetOperationContext().locateRegion(tableName, HConstants.EMPTY_START_ROW);
this.writeBufferSize = conf.getLong("hbase.client.write.buffer", 2097152);
this.clearBufferOnFail = true;
this.autoFlush = true;
@@ -166,6 +166,8 @@ public class HTable implements HTableInt
HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
this.maxKeyValueSize = conf.getInt("hbase.client.keyvalue.maxsize", -1);
+ this.recordClientContext = conf.getBoolean("hbase.client.record.context", false);
+
this.options = new HBaseRPCOptions ();
String compressionAlgo = conf.get(HConstants.HBASE_RPC_COMPRESSION_KEY);
if (compressionAlgo != null) {
@@ -180,6 +182,10 @@ public class HTable implements HTableInt
return configuration;
}
+ public List<OperationContext> getAndResetOperationContext() {
+ return this.connection.getAndResetOperationContext();
+ }
+
/**
* TODO Might want to change this to public, would be nice if the number
* of threads would automatically change when servers were added and removed
@@ -281,12 +287,19 @@ public class HTable implements HTableInt
}
/**
- * <em>INTERNAL</em> Used by unit tests and tools to do low-level
- * manipulations.
+ * Returns the connection to the HConnectionManager and also resets the
+ * operationContext. The context maintained is thread local and since we
+ * don't own the thread we cannot control the lifetime of this context object.
+ * Resetting it on the next getConnectionAndResetOperationContext()
+ * call is the best option. Hence, this call resets the context maintained
+ * in the connection object.
* @return An HConnection instance.
*/
- // TODO(tsuna): Remove this. Unit tests shouldn't require public helpers.
- public HConnection getConnection() {
+
+ public HConnection getConnectionAndResetOperationContext() {
+ if (recordClientContext) {
+ this.connection.resetOperationContext();
+ }
return this.connection;
}
@@ -389,33 +402,33 @@ public class HTable implements HTableInt
* @throws IOException
*/
public Pair<byte[][], byte[][]> getStartKeysAndFavoredNodes()
- throws IOException {
- final List<byte[]> startKeyList = new ArrayList<byte[]>();
- final List<byte[]> favoredNodes =
- new ArrayList<byte[]>();
- MetaScannerVisitor visitor = new MetaScannerVisitor() {
- public boolean processRow(Result rowResult) throws IOException {
- HRegionInfo info = Writables.getHRegionInfo(
- rowResult.getValue(HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER));
- byte[] favoredNodesBytes = rowResult.getValue(
- HConstants.CATALOG_FAMILY,
- HConstants.FAVOREDNODES_QUALIFIER);
- if (Bytes.equals(info.getTableDesc().getName(), getTableName())) {
- if (!(info.isOffline() || info.isSplit())) {
- startKeyList.add(info.getStartKey());
- favoredNodes.add(favoredNodesBytes);
- }
- }
- return true;
- }
- };
- MetaScanner.metaScan(configuration, visitor, this.tableName);
- return new Pair<byte[][], byte[][]>(
- startKeyList.toArray(new byte[startKeyList.size()][]),
- favoredNodes.toArray(new byte[favoredNodes.size()][]));
+ throws IOException {
+ final List<byte[]> startKeyList = new ArrayList<byte[]>();
+ final List<byte[]> favoredNodes =
+ new ArrayList<byte[]>();
+ MetaScannerVisitor visitor = new MetaScannerVisitor() {
+ public boolean processRow(Result rowResult) throws IOException {
+ HRegionInfo info = Writables.getHRegionInfo(
+ rowResult.getValue(HConstants.CATALOG_FAMILY,
+ HConstants.REGIONINFO_QUALIFIER));
+ byte[] favoredNodesBytes = rowResult.getValue(
+ HConstants.CATALOG_FAMILY,
+ HConstants.FAVOREDNODES_QUALIFIER);
+ if (Bytes.equals(info.getTableDesc().getName(), getTableName())) {
+ if (!(info.isOffline() || info.isSplit())) {
+ startKeyList.add(info.getStartKey());
+ favoredNodes.add(favoredNodesBytes);
+ }
+ }
+ return true;
+ }
+ };
+ MetaScanner.metaScan(configuration, visitor, this.tableName);
+ return new Pair<byte[][], byte[][]>(
+ startKeyList.toArray(new byte[startKeyList.size()][]),
+ favoredNodes.toArray(new byte[favoredNodes.size()][]));
}
-
+
/**
* Gets all the regions and their address for this table.
* <p>
@@ -548,7 +561,7 @@ public class HTable implements HTableInt
public Result getRowOrBefore(final byte[] row, final byte[] family)
throws IOException {
- return connection.getRegionServerWithRetries(
+ return this.getConnectionAndResetOperationContext().getRegionServerWithRetries(
new ServerCallable<Result>(connection, tableName, row, this.options) {
public Result call() throws IOException {
return server.getClosestRowBefore(location.getRegionInfo().getRegionName(),
@@ -577,7 +590,7 @@ public class HTable implements HTableInt
}
public Result get(final Get get) throws IOException {
- return connection.getRegionServerWithRetries(
+ return this.getConnectionAndResetOperationContext().getRegionServerWithRetries(
new ServerCallable<Result>(connection, tableName, get.getRow(), this.options) {
public Result call() throws IOException {
return server.get(location.getRegionInfo().getRegionName(), get);
@@ -587,7 +600,7 @@ public class HTable implements HTableInt
}
public Result[] get(List<Get> gets) throws IOException {
- return connection.processBatchOfGets(gets, tableName, this.options);
+ return this.getConnectionAndResetOperationContext().processBatchOfGets(gets, tableName, this.options);
}
/**
@@ -608,7 +621,7 @@ public class HTable implements HTableInt
throws IOException {
Result[] results = new Result[actions.size()];
try {
- connection.processBatchedGets(actions, tableName, multiActionThreadPool,
+ this.getConnectionAndResetOperationContext().processBatchedGets(actions, tableName, multiActionThreadPool,
results, this.options);
} catch (Exception e) {
throw new IOException(e);
@@ -623,7 +636,7 @@ public class HTable implements HTableInt
public void batchMutate(final List<Mutation> actions)
throws IOException {
try {
- connection.processBatchedMutations(actions,
+ this.getConnectionAndResetOperationContext().processBatchedMutations(actions,
tableName, multiActionThreadPool, null, this.options);
} catch (Exception e) {
throw new IOException(e);
@@ -636,7 +649,7 @@ public class HTable implements HTableInt
@Override
public void delete(final Delete delete)
throws IOException {
- connection.getRegionServerWithRetries(
+ this.getConnectionAndResetOperationContext().getRegionServerWithRetries(
new ServerCallable<Boolean>(connection,
tableName, delete.getRow(), this.options) {
public Boolean call() throws IOException {
@@ -651,7 +664,7 @@ public class HTable implements HTableInt
throws IOException {
int last = 0;
try {
- last = connection.processBatchOfDeletes(deletes, this.tableName, this.options);
+ last = this.getConnectionAndResetOperationContext().processBatchOfDeletes(deletes, this.tableName, this.options);
} finally {
deletes.subList(0, last).clear();
}
@@ -696,7 +709,7 @@ public class HTable implements HTableInt
throw new IOException(
"Invalid arguments to incrementColumnValue", npe);
}
- return connection.getRegionServerWithRetries(
+ return this.getConnectionAndResetOperationContext().getRegionServerWithRetries(
new ServerCallable<Long>(connection, tableName, row, this.options) {
public Long call() throws IOException {
return server.incrementColumnValue(
@@ -724,7 +737,7 @@ public class HTable implements HTableInt
final byte [] family, final byte [] qualifier, final byte [] value,
final Put put)
throws IOException {
- return connection.getRegionServerWithRetries(
+ return this.getConnectionAndResetOperationContext().getRegionServerWithRetries(
new ServerCallable<Boolean>(connection, tableName, row, this.options) {
public Boolean call() throws IOException {
return server.checkAndPut(location.getRegionInfo().getRegionName(),
@@ -751,7 +764,7 @@ public class HTable implements HTableInt
final byte [] family, final byte [] qualifier, final byte [] value,
final Delete delete)
throws IOException {
- return connection.getRegionServerWithRetries(
+ return this.getConnectionAndResetOperationContext().getRegionServerWithRetries(
new ServerCallable<Boolean>(connection, tableName, row, this.options) {
public Boolean call() throws IOException {
return server.checkAndDelete(
@@ -768,8 +781,8 @@ public class HTable implements HTableInt
*/
@Override
public void mutateRow(final RowMutations arm) throws IOException {
- connection.getRegionServerWithRetries(
- new ServerCallable<Void>(connection, tableName, arm.getRow(), this.options) {
+ this.getConnectionAndResetOperationContext().getRegionServerWithRetries(
+ new ServerCallable<Void>(this.connection, tableName, arm.getRow(), this.options) {
public Void call() throws IOException {
server.mutateRow(location.getRegionInfo().getRegionName(), arm);
return null;
@@ -782,7 +795,7 @@ public class HTable implements HTableInt
*/
@Override
public void mutateRow(final List<RowMutations> armList) throws IOException {
- connection.processBatchOfRowMutations(armList, this.tableName, this.options);
+ this.getConnectionAndResetOperationContext().processBatchOfRowMutations(armList, this.tableName, this.options);
}
/**
@@ -797,7 +810,7 @@ public class HTable implements HTableInt
* @throws IOException
*/
public boolean exists(final Get get) throws IOException {
- return connection.getRegionServerWithRetries(
+ return this.getConnectionAndResetOperationContext().getRegionServerWithRetries(
new ServerCallable<Boolean>(connection, tableName, get.getRow(), this.options) {
public Boolean call() throws IOException {
return server.
@@ -809,7 +822,7 @@ public class HTable implements HTableInt
public void flushCommits() throws IOException {
try {
- connection.processBatchOfPuts(writeBuffer, tableName, this.options);
+ this.getConnectionAndResetOperationContext().processBatchOfPuts(writeBuffer, tableName, this.options);
} finally {
if (clearBufferOnFail) {
writeBuffer.clear();
@@ -850,7 +863,7 @@ public class HTable implements HTableInt
public RowLock lockRow(final byte [] row)
throws IOException {
- return connection.getRegionServerWithRetries(
+ return this.getConnectionAndResetOperationContext().getRegionServerWithRetries(
new ServerCallable<RowLock>(connection, tableName, row, this.options) {
public RowLock call() throws IOException {
long lockId =
@@ -863,7 +876,7 @@ public class HTable implements HTableInt
public void unlockRow(final RowLock rl)
throws IOException {
- connection.getRegionServerWithRetries(
+ this.getConnectionAndResetOperationContext().getRegionServerWithRetries(
new ServerCallable<Boolean>(connection, tableName, rl.getRow(), this.options) {
public Boolean call() throws IOException {
server.unlockRow(location.getRegionInfo().getRegionName(),
@@ -1040,7 +1053,7 @@ public class HTable implements HTableInt
// Close the previous scanner if it's open
if (this.callable != null) {
this.callable.setClose();
- getConnection().getRegionServerWithRetries(callable);
+ getConnectionAndResetOperationContext().getRegionServerWithRetries(callable);
this.callable = null;
}
@@ -1076,7 +1089,7 @@ public class HTable implements HTableInt
callable = getScannerCallable(localStartKey, nbRows, options);
// Open a scanner on the region server starting at the
// beginning of the region
- getConnection().getRegionServerWithRetries(callable);
+ getConnectionAndResetOperationContext().getRegionServerWithRetries(callable);
this.currentRegion = callable.getHRegionInfo();
} catch (IOException e) {
close();
@@ -1088,7 +1101,7 @@ public class HTable implements HTableInt
protected ScannerCallable getScannerCallable(byte [] localStartKey,
int nbRows, HBaseRPCOptions options) {
scan.setStartRow(localStartKey);
- ScannerCallable s = new ScannerCallable(getConnection(),
+ ScannerCallable s = new ScannerCallable(getConnectionAndResetOperationContext(),
getTableName(), scan, options);
s.setCaching(nbRows);
return s;
@@ -1114,11 +1127,11 @@ public class HTable implements HTableInt
// Server returns a null values if scanning is to stop. Else,
// returns an empty array if scanning is to go on and we've just
// exhausted current region.
- values = getConnection().getRegionServerWithRetries(callable);
+ values = getConnectionAndResetOperationContext().getRegionServerWithRetries(callable);
if (skipFirst) {
skipFirst = false;
// Reget.
- values = getConnection().getRegionServerWithRetries(callable);
+ values = getConnectionAndResetOperationContext().getRegionServerWithRetries(callable);
}
} catch (DoNotRetryIOException e) {
if (e instanceof UnknownScannerException) {
@@ -1194,7 +1207,7 @@ public class HTable implements HTableInt
if (callable != null) {
callable.setClose();
try {
- getConnection().getRegionServerWithRetries(callable);
+ getConnectionAndResetOperationContext().getRegionServerWithRetries(callable);
} catch (IOException e) {
// We used to catch this error, interpret, and rethrow. However, we
// have since decided that it's not nice for a scanner's close to
Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/OperationContext.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/OperationContext.java?rev=1464739&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/OperationContext.java (added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/OperationContext.java Thu Apr 4 20:49:16 2013
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2013 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.hbase.HRegionLocation;
+
+public class OperationContext {
+ private final HRegionLocation location;
+ private final Throwable error;
+
+ OperationContext(final HRegionLocation location, final Throwable e) {
+ this.location = location;
+ this.error = e;
+ }
+
+ OperationContext(final OperationContext c) {
+ this.location = c.location;
+ this.error = c.error;
+ }
+
+ public HRegionLocation getLocation() {
+ return location;
+ }
+
+ public Throwable getError() {
+ return error;
+ }
+}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java?rev=1464739&r1=1464738&r2=1464739&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java Thu Apr 4 20:49:16 2013
@@ -157,7 +157,7 @@ public class LoadIncrementalHFiles exten
public void doBulkLoad(Path hfofDir, HTable table)
throws TableNotFoundException, IOException
{
- HConnection conn = table.getConnection();
+ HConnection conn = table.getConnectionAndResetOperationContext();
if (!conn.isTableAvailable(table.getTableName())) {
throw new TableNotFoundException("Table " +
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java?rev=1464739&r1=1464738&r2=1464739&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java Thu Apr 4 20:49:16 2013
@@ -394,7 +394,7 @@ public class RegionSplitter {
HTable table = new HTable(conf, tableName);
// max outstanding splits. default == 50% of servers
- final int MAX_OUTSTANDING = Math.max(table.getConnection().getMaster()
+ final int MAX_OUTSTANDING = Math.max(table.getConnectionAndResetOperationContext().getMaster()
.getClusterStatus().getServers() / 2, minOS);
Path hbDir = new Path(conf.get(HConstants.HBASE_DIR));
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1464739&r1=1464738&r2=1464739&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Thu Apr 4 20:49:16 2013
@@ -812,7 +812,7 @@ public class HBaseTestingUtility {
meta.delete(new Delete(row));
}
// flush cache of regions
- HConnection conn = table.getConnection();
+ HConnection conn = table.getConnectionAndResetOperationContext();
conn.clearRegionCache();
return count;
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java?rev=1464739&r1=1464738&r2=1464739&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java Thu Apr 4 20:49:16 2013
@@ -4010,9 +4010,9 @@ public class TestFromClientSide {
table.getRegionsInfo();
// set the deserialized regions to the global cache.
- table.getConnection().clearRegionCache();
+ table.getConnectionAndResetOperationContext().clearRegionCache();
- table.getConnection().prewarmRegionCache(table.getTableName(),
+ table.getConnectionAndResetOperationContext().prewarmRegionCache(table.getTableName(),
deserRegions);
// verify whether the 2 maps are identical or not.
@@ -4058,7 +4058,7 @@ public class TestFromClientSide {
// This count effectively waits until the regions have been
// fully assigned
TEST_UTIL.countRows(table);
- table.getConnection().clearRegionCache();
+ table.getConnectionAndResetOperationContext().clearRegionCache();
assertEquals("Clearing cache should have 0 cached ", 0,
HConnectionManager.getCachedRegionCount(conf, TABLENAME));
@@ -4083,7 +4083,7 @@ public class TestFromClientSide {
assertTrue("The table is enabled for region cache prefetch",
HTable.getRegionCachePrefetch(conf, TABLENAME));
- table.getConnection().clearRegionCache();
+ table.getConnectionAndResetOperationContext().clearRegionCache();
assertEquals("Number of cached region is incorrect ", 0,
HConnectionManager.getCachedRegionCount(conf, TABLENAME));
@@ -4101,7 +4101,7 @@ public class TestFromClientSide {
LOG.info("Testing how many regions cached");
assertTrue(prefetchRegionNumber < HConnectionManager.getCachedRegionCount(conf, TABLENAME));
- table.getConnection().clearRegionCache();
+ table.getConnectionAndResetOperationContext().clearRegionCache();
Get g3 = new Get(Bytes.toBytes("abc"));
table.get(g3);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java?rev=1464739&r1=1464738&r2=1464739&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java Thu Apr 4 20:49:16 2013
@@ -54,6 +54,7 @@ public class TestHCM {
TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 3);
TEST_UTIL.getConfiguration().set("hbase.loadbalancer.impl",
"org.apache.hadoop.hbase.master.RegionManager$AssignmentLoadBalancer");
+ TEST_UTIL.getConfiguration().setBoolean("hbase.client.record.context", true);
TEST_UTIL.startMiniCluster(REGION_SERVERS);
}
@@ -71,7 +72,7 @@ public class TestHCM {
put.add(FAM_NAM, ROW, ROW);
table.put(put);
HConnectionManager.TableServers conn =
- (HConnectionManager.TableServers) table.getConnection();
+ (HConnectionManager.TableServers) table.getConnectionAndResetOperationContext();
assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
conn.deleteCachedLocation(TABLE_NAME, ROW, null);
HRegionLocation rl = conn.getCachedLocation(TABLE_NAME, ROW);
@@ -91,12 +92,14 @@ public class TestHCM {
Put put = new Put(ROW);
put.add(FAM_NAM, ROW, ROW);
- setupFailure(table);
+ setupFailure(table,
+ TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size());
try {
table.put(put);
assertNull("Put should not succeed");
} catch (IOException e) {
- verifyException(e);
+ verifyFailure(table, e);
+
} finally {
ServerManager.clearRSBlacklistInTest();
waitForRegionsToGetReAssigned();
@@ -124,13 +127,14 @@ public class TestHCM {
put.add(FAM_NAM, ROW, ROW);
table.put(put);
- setupFailure(table);
+ setupFailure(table,
+ TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size());
try {
Get get = new Get(ROW);
table.get(get);
assertNull("Get should not succeed");
} catch (IOException e) {
- verifyException(e);
+ verifyFailure(table, e);
} finally {
ServerManager.clearRSBlacklistInTest();
waitForRegionsToGetReAssigned();
@@ -151,13 +155,14 @@ public class TestHCM {
put.add(FAM_NAM, ROW, ROW);
table.put(put);
- setupFailure(table);
+ setupFailure(table,
+ TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size());
try {
Delete delete = new Delete(ROW);
table.delete(delete);
assertNull("Delete should not succeed");
} catch (IOException e) {
- verifyException(e);
+ verifyFailure(table, e);
} finally {
ServerManager.clearRSBlacklistInTest();
waitForRegionsToGetReAssigned();
@@ -165,15 +170,53 @@ public class TestHCM {
}
- private void setupFailure(HTable table) throws IOException {
+ /**
+ * Simulates a case where the get operation succeeds after a retry
+ * @throws Exception
+ */
+ @Test
+ public void testClientGetSuccess() throws Exception {
+
+ HTable table = TEST_UTIL.createTable(Bytes.toBytes("testClientSuccess"), FAM_NAM);
+ TEST_UTIL.createMultiRegions(table, FAM_NAM);
+
+ Put put = new Put(ROW);
+ put.add(FAM_NAM, ROW, ROW);
+ table.put(put);
+
+ setupFailure(table, 0);
+ try {
+ Get g = new Get(ROW);
+ table.get(g);
+ List<OperationContext> op = table.getAndResetOperationContext();
+ assertTrue(op.size() > 0);
+ for (OperationContext p : op) {
+ assertTrue(p.getError() != null);
+ assertTrue(p.getLocation() != null);
+ }
+ } catch (IOException e) {
+
+ } finally {
+ waitForRegionsToGetReAssigned();
+ }
+ }
+
+ /**
+ * Assign noOfRS to the dead maps list, so that no regions are assigned to
+ * them and then kill the current region server.
+ * @param table
+ * @param noOfRS
+ * @throws IOException
+ */
+ private void setupFailure(HTable table, int noOfRS) throws IOException {
HRegionLocation regLoc = table.getRegionLocation(ROW);
List<RegionServerThread> regionServers =
TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads();
// Add all region servers to the black list to prevent fail overs.
- for(RegionServerThread r : regionServers) {
+ for(int i = 0; i < noOfRS; i++) {
ServerManager.blacklistRSHostPortInTest(
- r.getRegionServer().getHServerInfo().getHostnamePort());
+ regionServers.get(i).getRegionServer().getHServerInfo().getHostnamePort());
}
// abort the region server
@@ -182,7 +225,14 @@ public class TestHCM {
TEST_UTIL.getHBaseCluster().stopRegionServer(srcRSIdx);
}
- private void verifyException(Exception e) {
+ private void verifyFailure(HTable table, Exception e) {
+
+ List<OperationContext> context = table.getAndResetOperationContext();
+ assertTrue(context.size() != 0);
+ for (OperationContext c : context) {
+ assertTrue(c.getError() != null);
+ assertTrue(c.getLocation() != null);
+ }
assertTrue(e instanceof RetriesExhaustedException);
RetriesExhaustedException exp = (RetriesExhaustedException)(e);
assertTrue(exp.getRegionNames() != null);