You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/10/14 01:12:31 UTC
svn commit: r1183158 [2/3] - in /hbase/trunk: ./
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/catalog/
src/main/java/org/apache/hadoop/hbase/client/
src/main/java/org/apache/hadoop/hbase/ipc/ src/main/java/org/apache/had...
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Thu Oct 13 23:12:30 2011
@@ -54,7 +54,6 @@ import org.apache.hadoop.hbase.ZooKeeper
import org.apache.hadoop.hbase.catalog.CatalogTracker;
import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
@@ -138,7 +137,6 @@ public class HBaseAdmin implements Abort
CatalogTracker ct = null;
try {
ct = new CatalogTracker(this.conf);
-
ct.start();
} catch (InterruptedException e) {
// Let it out as an IOE for now until we redo all so tolerate IEs
@@ -1597,19 +1595,19 @@ public class HBaseAdmin implements Abort
* get the regions of a given table.
*
* @param tableName the name of the table
- * @return Ordered list of {@link HRegionInfo}. *
+ * @return Ordered list of {@link HRegionInfo}.
* @throws IOException
*/
public List<HRegionInfo> getTableRegions(final byte[] tableName)
throws IOException {
CatalogTracker ct = getCatalogTracker();
- List<HRegionInfo> Regions;
+ List<HRegionInfo> Regions = null;
try {
Regions = MetaReader.getTableRegions(ct, tableName, true);
} finally {
cleanupCatalogTracker(ct);
}
- return Regions;
+ return Regions;
}
public void close() throws IOException {
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Thu Oct 13 23:12:30 2011
@@ -130,8 +130,10 @@ import org.apache.zookeeper.KeeperExcept
*/
@SuppressWarnings("serial")
public class HConnectionManager {
- // A LRU Map of HConnectionKey -> HConnection (TableServer).
- private static final Map<HConnectionKey, HConnectionImplementation> HBASE_INSTANCES;
+ // An LRU Map of HConnectionKey -> HConnection (TableServer). All
+ // access must be synchronized. This map is not private because tests
+ // need to be able to tinker with it.
+ static final Map<HConnectionKey, HConnectionImplementation> HBASE_INSTANCES;
public static final int MAX_CACHED_HBASE_INSTANCES;
@@ -895,7 +897,7 @@ public class HConnectionManager {
deleteCachedLocation(tableName, row);
}
- // Query the root or meta region for the location of the meta region
+ // Query the root or meta region for the location of the meta region
regionInfoRow = server.getClosestRowBefore(
metaLocation.getRegionInfo().getRegionName(), metaKey,
HConstants.CATALOG_FAMILY);
@@ -962,8 +964,8 @@ public class HConnectionManager {
if (LOG.isDebugEnabled()) {
LOG.debug("locateRegionInMeta parentTable=" +
Bytes.toString(parentTable) + ", metaLocation=" +
- ((metaLocation == null)? "null": metaLocation) + ", attempt=" +
- tries + " of " +
+ ((metaLocation == null)? "null": "{" + metaLocation + "}") +
+ ", attempt=" + tries + " of " +
this.numRetries + " failed; retrying after sleep of " +
getPauseTime(tries) + " because: " + e.getMessage());
}
@@ -1201,7 +1203,7 @@ public class HConnectionManager {
} catch (RemoteException e) {
LOG.warn("RemoteException connecting to RS", e);
// Throw what the RemoteException was carrying.
- throw RemoteExceptionHandler.decodeRemoteException(e);
+ throw e.unwrapRemoteException();
}
}
}
@@ -1233,19 +1235,22 @@ public class HConnectionManager {
public <T> T getRegionServerWithRetries(ServerCallable<T> callable)
throws IOException, RuntimeException {
- List<Throwable> exceptions = new ArrayList<Throwable>();
+ List<RetriesExhaustedException.ThrowableWithExtraContext> exceptions =
+ new ArrayList<RetriesExhaustedException.ThrowableWithExtraContext>();
for(int tries = 0; tries < numRetries; tries++) {
try {
- callable.instantiateServer(tries != 0);
callable.beforeCall();
+ callable.connect(tries != 0);
return callable.call();
} catch (Throwable t) {
callable.shouldRetry(t);
t = translateException(t);
- exceptions.add(t);
+ RetriesExhaustedException.ThrowableWithExtraContext qt =
+ new RetriesExhaustedException.ThrowableWithExtraContext(t,
+ System.currentTimeMillis(), callable.toString());
+ exceptions.add(qt);
if (tries == numRetries - 1) {
- throw new RetriesExhaustedException(callable.getServerName(),
- callable.getRegionName(), callable.getRow(), tries, exceptions);
+ throw new RetriesExhaustedException(tries, exceptions);
}
} finally {
callable.afterCall();
@@ -1254,7 +1259,7 @@ public class HConnectionManager {
Thread.sleep(getPauseTime(tries));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
- throw new IOException("Giving up trying to get region server: thread is interrupted.");
+ throw new IOException("Giving up after tries=" + tries, e);
}
}
return null;
@@ -1263,8 +1268,8 @@ public class HConnectionManager {
public <T> T getRegionServerWithoutRetries(ServerCallable<T> callable)
throws IOException, RuntimeException {
try {
- callable.instantiateServer(false);
callable.beforeCall();
+ callable.connect(false);
return callable.call();
} catch (Throwable t) {
Throwable t2 = translateException(t);
@@ -1289,7 +1294,7 @@ public class HConnectionManager {
return server.multi(multi);
}
@Override
- public void instantiateServer(boolean reload) throws IOException {
+ public void connect(boolean reload) throws IOException {
server =
connection.getHRegionConnection(loc.getHostname(), loc.getPort());
}
@@ -1804,6 +1809,25 @@ public class HConnectionManager {
}
return hTableDescriptor;
}
+ }
+ /**
+ * Set the number of retries to use serverside when trying to communicate
+ * with another server over {@link HConnection}. Used updating catalog
+ * tables, etc. Call this method before we create any Connections.
+ * @param c The Configuration instance to set the retries into.
+ * @param log Used to log what we set in here.
+ */
+ public static void setServerSideHConnectionRetries(final Configuration c,
+ final Log log) {
+ int hcRetries = c.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
+ HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
+ // Go big. Multiply by 10. If we can't get to meta after this many retries
+ // then something seriously wrong.
+ int serversideMultiplier =
+ c.getInt("hbase.client.serverside.retries.multiplier", 10);
+ int retries = hcRetries * serversideMultiplier;
+ c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, retries);
+ log.debug("Set serverside HConnection retries=" + retries);
}
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HTable.java Thu Oct 13 23:12:30 2011
@@ -53,7 +53,6 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.UnknownScannerException;
-import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
@@ -194,7 +193,8 @@ public class HTable implements HTableInt
}
this.connection = HConnectionManager.getConnection(conf);
this.scannerTimeout =
- (int) conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
+ (int) conf.getLong(HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
+ HConstants.DEFAULT_HBASE_REGIONSERVER_LEASE_PERIOD);
this.operationTimeout = HTableDescriptor.isMetaTable(tableName) ? HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT
: conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
@@ -418,8 +418,9 @@ public class HTable implements HTableInt
}
};
MetaScanner.metaScan(configuration, visitor, this.tableName);
- return new Pair(startKeyList.toArray(new byte[startKeyList.size()][]),
- endKeyList.toArray(new byte[endKeyList.size()][]));
+ return new Pair<byte [][], byte [][]>(
+ startKeyList.toArray(new byte[startKeyList.size()][]),
+ endKeyList.toArray(new byte[endKeyList.size()][]));
}
/**
@@ -872,7 +873,7 @@ public class HTable implements HTableInt
@Override
public void flushCommits() throws IOException {
try {
- connection.processBatchOfPuts(writeBuffer, tableName, pool);
+ this.connection.processBatchOfPuts(writeBuffer, tableName, pool);
} finally {
if (clearBufferOnFail) {
writeBuffer.clear();
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java Thu Oct 13 23:12:30 2011
@@ -187,7 +187,7 @@ public class MetaScanner {
if (LOG.isDebugEnabled()) {
LOG.debug("Scanning " + Bytes.toString(metaTableName) +
" starting at row=" + Bytes.toStringBinary(startRow) + " for max=" +
- rowUpperLimit + " rows");
+ rowUpperLimit + " rows using " + connection.toString());
}
callable = new ScannerCallable(connection, metaTableName, scan, null);
// Open scanner
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Result.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Result.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Result.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Result.java Thu Oct 13 23:12:30 2011
@@ -20,12 +20,16 @@
package org.apache.hadoop.hbase.client;
-import com.google.common.collect.Ordering;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.SplitKeyValue;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.io.WritableWithSize;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.Writable;
import java.io.DataInput;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java Thu Oct 13 23:12:30 2011
@@ -15,9 +15,8 @@
*/
package org.apache.hadoop.hbase.client;
-import org.apache.hadoop.hbase.util.Bytes;
-
import java.io.IOException;
+import java.util.Date;
import java.util.List;
/**
@@ -36,28 +35,53 @@ public class RetriesExhaustedException e
}
/**
+ * Datastructure that allows adding more info around Throwable incident.
+ */
+ public static class ThrowableWithExtraContext {
+ private final Throwable t;
+ private final long when;
+ private final String extras;
+
+ public ThrowableWithExtraContext(final Throwable t, final long when,
+ final String extras) {
+ this.t = t;
+ this.when = when;
+ this.extras = extras;
+ }
+
+ @Override
+ public String toString() {
+ return new Date(this.when).toString() + ", " + extras + ", " + t.toString();
+ }
+ }
+
+ /**
* Create a new RetriesExhaustedException from the list of prior failures.
- * @param serverName name of HRegionServer
- * @param regionName name of region
- * @param row The row we were pursuing when we ran out of retries
+ * @param callableVitals Details from the {@link ServerCallable} we were using
+ * when we got this exception.
* @param numTries The number of tries we made
* @param exceptions List of exceptions that failed before giving up
*/
- public RetriesExhaustedException(String serverName, final byte [] regionName,
- final byte [] row, int numTries, List<Throwable> exceptions) {
- super(getMessage(serverName, regionName, row, numTries, exceptions));
+ public RetriesExhaustedException(final String callableVitals, int numTries,
+ List<Throwable> exceptions) {
+ super(getMessage(callableVitals, numTries, exceptions));
+ }
+
+ /**
+ * Create a new RetriesExhaustedException from the list of prior failures.
+ * @param numTries
+ * @param exceptions List of exceptions that failed before giving up
+ */
+ public RetriesExhaustedException(final int numTries,
+ final List<ThrowableWithExtraContext> exceptions) {
+ super(getMessage(numTries, exceptions));
}
- private static String getMessage(String serverName, final byte [] regionName,
- final byte [] row,
- int numTries, List<Throwable> exceptions) {
- StringBuilder buffer = new StringBuilder("Trying to contact region server ");
- buffer.append(serverName);
- buffer.append(" for region ");
- buffer.append(regionName == null? "": Bytes.toStringBinary(regionName));
- buffer.append(", row '");
- buffer.append(row == null? "": Bytes.toStringBinary(row));
- buffer.append("', but failed after ");
+ private static String getMessage(String callableVitals, int numTries,
+ List<Throwable> exceptions) {
+ StringBuilder buffer = new StringBuilder("Failed contacting ");
+ buffer.append(callableVitals);
+ buffer.append(" after ");
buffer.append(numTries + 1);
buffer.append(" attempts.\nExceptions:\n");
for (Throwable t : exceptions) {
@@ -66,4 +90,16 @@ public class RetriesExhaustedException e
}
return buffer.toString();
}
+
+ private static String getMessage(final int numTries,
+ final List<ThrowableWithExtraContext> exceptions) {
+ StringBuilder buffer = new StringBuilder("Failed after attempts=");
+ buffer.append(numTries + 1);
+ buffer.append(", exceptions:\n");
+ for (ThrowableWithExtraContext t : exceptions) {
+ buffer.append(t.toString());
+ buffer.append("\n");
+ }
+ return buffer.toString();
+ }
}
\ No newline at end of file
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java Thu Oct 13 23:12:30 2011
@@ -67,9 +67,9 @@ public class ScannerCallable extends Ser
* @throws IOException
*/
@Override
- public void instantiateServer(boolean reload) throws IOException {
+ public void connect(boolean reload) throws IOException {
if (!instantiated || reload) {
- super.instantiateServer(reload);
+ super.connect(reload);
checkIfRegionServerIsRemote();
instantiated = true;
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java Thu Oct 13 23:12:30 2011
@@ -31,7 +31,14 @@ import org.apache.hadoop.hbase.ipc.HRegi
import org.apache.hadoop.hbase.util.Bytes;
/**
- * Abstract class that implements Callable, used by retryable actions.
+ * Abstract class that implements {@link Callable}. Implementation stipulates
+ * return type and method we actually invoke on remote Server. Usually
+ * used inside a try/catch that fields usual connection failures all wrapped
+ * up in a retry loop.
+ * <p>Call {@link #connect(boolean)} to connect to server hosting region
+ * that contains the passed row in the passed table before invoking
+ * {@link #call()}.
+ * @see HConnection#getRegionServerWithoutRetries(ServerCallable)
* @param <T> the class that the ServerCallable handles
*/
public abstract class ServerCallable<T> implements Callable<T> {
@@ -44,9 +51,9 @@ public abstract class ServerCallable<T>
protected long startTime, endTime;
/**
- * @param connection connection callable is on
- * @param tableName table name callable is on
- * @param row row we are querying
+ * @param connection Connection to use.
+ * @param tableName Table name to which <code>row</code> belongs.
+ * @param row The row we want in <code>tableName</code>.
*/
public ServerCallable(HConnection connection, byte [] tableName, byte [] row) {
this(connection, tableName, row, HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT);
@@ -58,34 +65,37 @@ public abstract class ServerCallable<T>
this.row = row;
this.callTimeout = callTimeout;
}
+
/**
- *
- * @param reload set this to true if connection should re-find the region
+ * Connect to the server hosting region with row from tablename.
+ * @param reload Set this to true if connection should re-find the region
* @throws IOException e
*/
- public void instantiateServer(boolean reload) throws IOException {
+ public void connect(final boolean reload) throws IOException {
this.location = connection.getRegionLocation(tableName, row, reload);
this.server = connection.getHRegionConnection(location.getHostname(),
location.getPort());
}
- /** @return the server name */
+ /** @return the server name
+ * @deprecated Just use {@link #toString()} instead.
+ */
public String getServerName() {
- if (location == null) {
- return null;
- }
+ if (location == null) return null;
return location.getHostnamePort();
}
- /** @return the region name */
+ /** @return the region name
+ * @deprecated Just use {@link #toString()} instead.
+ */
public byte[] getRegionName() {
- if (location == null) {
- return null;
- }
+ if (location == null) return null;
return location.getRegionInfo().getRegionName();
}
- /** @return the row */
+ /** @return the row
+ * @deprecated Just use {@link #toString()} instead.
+ */
public byte [] getRow() {
return row;
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java Thu Oct 13 23:12:30 2011
@@ -87,7 +87,7 @@ public interface HMasterInterface extend
* @throws IOException
*/
public Pair<Integer, Integer> getAlterStatus(byte[] tableName)
- throws IOException;
+ throws IOException;
/**
* Adds a column to the specified table
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Thu Oct 13 23:12:30 2011
@@ -267,11 +267,12 @@ public class AssignmentManager extends Z
* @param tableName
* @return Pair indicating the status of the alter command
* @throws IOException
+ * @throws InterruptedException
*/
public Pair<Integer, Integer> getReopenStatus(byte[] tableName)
- throws IOException {
- List <HRegionInfo> hris = MetaReader.getTableRegions(
- this.master.getCatalogTracker(), tableName);
+ throws IOException, InterruptedException {
+ List <HRegionInfo> hris =
+ MetaReader.getTableRegions(this.master.getCatalogTracker(), tableName);
Integer pending = 0;
for(HRegionInfo hri : hris) {
if(regionsToReopen.get(hri.getEncodedName()) != null) {
@@ -730,7 +731,7 @@ public class AssignmentManager extends Z
case RS_ZK_REGION_OPENING:
// Should see OPENING after we have asked it to OPEN or additional
// times after already being in state of OPENING
- if(regionState == null ||
+ if (regionState == null ||
(!regionState.isPendingOpen() && !regionState.isOpening())) {
LOG.warn("Received OPENING for region " +
prettyPrintedRegionName +
@@ -1764,7 +1765,6 @@ public class AssignmentManager extends Z
// Presume that master has stale data. Presume remote side just split.
// Presume that the split message when it comes in will fix up the master's
// in memory cluster state.
- return;
} catch (Throwable t) {
if (t instanceof RemoteException) {
t = ((RemoteException)t).unwrapRemoteException();
@@ -2082,13 +2082,13 @@ public class AssignmentManager extends Z
Map<ServerName, List<Pair<HRegionInfo, Result>>> rebuildUserRegions()
throws IOException, KeeperException {
// Region assignment from META
- List<Result> results = MetaReader.fullScanOfResults(this.catalogTracker);
+ List<Result> results = MetaReader.fullScan(this.catalogTracker);
// Map of offline servers and their regions to be returned
Map<ServerName, List<Pair<HRegionInfo,Result>>> offlineServers =
new TreeMap<ServerName, List<Pair<HRegionInfo, Result>>>();
// Iterate regions in META
for (Result result : results) {
- Pair<HRegionInfo, ServerName> region = MetaReader.metaRowToRegionPair(result);
+ Pair<HRegionInfo, ServerName> region = MetaReader.parseCatalogResult(result);
if (region == null) continue;
HRegionInfo regionInfo = region.getFirst();
ServerName regionLocation = region.getSecond();
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Thu Oct 13 23:12:30 2011
@@ -25,7 +25,6 @@ import java.lang.reflect.InvocationTarge
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
@@ -41,21 +40,18 @@ import org.apache.hadoop.hbase.HRegionIn
import org.apache.hadoop.hbase.HServerLoad;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
-import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
-import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
-import org.apache.hadoop.hbase.catalog.MetaEditor;
import org.apache.hadoop.hbase.catalog.MetaReader;
-import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.MetaScanner;
-import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.executor.ExecutorService;
import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
@@ -64,7 +60,7 @@ import org.apache.hadoop.hbase.ipc.HBase
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.HMasterRegionInterface;
import org.apache.hadoop.hbase.ipc.RpcServer;
-import org.apache.hadoop.hbase.master.RegionPlan;
+import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
import org.apache.hadoop.hbase.master.handler.DeleteTableHandler;
import org.apache.hadoop.hbase.master.handler.DisableTableHandler;
import org.apache.hadoop.hbase.master.handler.EnableTableHandler;
@@ -72,18 +68,14 @@ import org.apache.hadoop.hbase.master.ha
import org.apache.hadoop.hbase.master.handler.TableAddFamilyHandler;
import org.apache.hadoop.hbase.master.handler.TableDeleteFamilyHandler;
import org.apache.hadoop.hbase.master.handler.TableModifyFamilyHandler;
-import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
import org.apache.hadoop.hbase.monitoring.MemoryBoundedLogMessageBuffer;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.replication.regionserver.Replication;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
-import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.InfoServer;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Sleeper;
@@ -205,7 +197,8 @@ implements HMasterInterface, HMasterRegi
public HMaster(final Configuration conf)
throws IOException, KeeperException, InterruptedException {
this.conf = conf;
-
+ // Set how many times to retry talking to another server over HConnection.
+ HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);
// Server to handle client requests.
String hostname = DNS.getDefaultHost(
conf.get("hbase.master.dns.interface", "default"),
@@ -470,7 +463,9 @@ implements HMasterInterface, HMasterRegi
// Update meta with new HRI if required. i.e migrate all HRI with HTD to
// HRI with out HTD in meta and update the status in ROOT. This must happen
// before we assign all user regions or else the assignment will fail.
- updateMetaWithNewHRI();
+ // TODO: Remove this when we do 0.94.
+ org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD.
+ updateMetaWithNewHRI(this);
// Fixup assignment manager status
status.setStatus("Starting assignment manager");
@@ -500,45 +495,6 @@ implements HMasterInterface, HMasterRegi
}
}
- public boolean isMetaHRIUpdated()
- throws IOException {
- boolean metaUpdated = false;
- Get get = new Get(HRegionInfo.FIRST_META_REGIONINFO.getRegionName());
- get.addColumn(HConstants.CATALOG_FAMILY,
- HConstants.META_MIGRATION_QUALIFIER);
- Result r =
- catalogTracker.waitForRootServerConnectionDefault().get(
- HRegionInfo.ROOT_REGIONINFO.getRegionName(), get);
- if (r != null && r.getBytes() != null)
- {
- byte[] metaMigrated = r.getValue(HConstants.CATALOG_FAMILY,
- HConstants.META_MIGRATION_QUALIFIER);
- String migrated = Bytes.toString(metaMigrated);
- metaUpdated = new Boolean(migrated).booleanValue();
- } else {
- LOG.info("metaUpdated = NULL.");
- }
- LOG.info("Meta updated status = " + metaUpdated);
- return metaUpdated;
- }
-
-
- boolean updateMetaWithNewHRI() throws IOException {
- if (!isMetaHRIUpdated()) {
- LOG.info("Meta has HRI with HTDs. Updating meta now.");
- try {
- MetaEditor.migrateRootAndMeta(this);
- LOG.info("ROOT and Meta updated with new HRI.");
- return true;
- } catch (IOException e) {
- throw new RuntimeException("Update ROOT/Meta with new HRI failed." +
- "Master startup aborted.");
- }
- }
- LOG.info("ROOT/Meta already up-to date with new HRI.");
- return true;
- }
-
/**
* Check <code>-ROOT-</code> and <code>.META.</code> are assigned. If not,
* assign them.
@@ -1018,6 +974,7 @@ implements HMasterInterface, HMasterRegi
Bytes.equals(tableName, HConstants.META_TABLE_NAME);
}
+ @Override
public void deleteTable(final byte [] tableName) throws IOException {
if (cpHost != null) {
cpHost.preDeleteTable(tableName);
@@ -1029,16 +986,13 @@ implements HMasterInterface, HMasterRegi
}
}
- /**
- * Get the number of regions of the table that have been updated by the alter.
- *
- * @return Pair indicating the number of regions updated Pair.getFirst is the
- * regions that are yet to be updated Pair.getSecond is the total number
- * of regions of the table
- */
public Pair<Integer, Integer> getAlterStatus(byte[] tableName)
throws IOException {
- return this.assignmentManager.getReopenStatus(tableName);
+ try {
+ return this.assignmentManager.getReopenStatus(tableName);
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted", e);
+ }
}
public void addColumn(byte [] tableName, HColumnDescriptor column)
@@ -1123,7 +1077,7 @@ implements HMasterInterface, HMasterRegi
if (data == null || data.size() <= 0) {
return true;
}
- Pair<HRegionInfo, ServerName> pair = MetaReader.metaRowToRegionPair(data);
+ Pair<HRegionInfo, ServerName> pair = MetaReader.parseCatalogResult(data);
if (pair == null) {
return false;
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java Thu Oct 13 23:12:30 2011
@@ -111,8 +111,8 @@ public class EnableTableHandler extends
boolean done = false;
// Get the regions of this table. We're done when all listed
// tables are onlined.
- List<HRegionInfo> regionsInMeta =
- MetaReader.getTableRegions(this.ct, tableName, true);
+ List<HRegionInfo> regionsInMeta;
+ regionsInMeta = MetaReader.getTableRegions(this.ct, tableName, true);
int countOfRegionsInTable = regionsInMeta.size();
List<HRegionInfo> regions = regionsToAssign(regionsInMeta);
int regionsCount = regions.size();
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java Thu Oct 13 23:12:30 2011
@@ -41,7 +41,6 @@ import org.apache.hadoop.hbase.master.De
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Writables;
import org.apache.zookeeper.KeeperException;
/**
@@ -265,14 +264,16 @@ public class ServerShutdownHandler exten
LOG.debug("Removed " + rit.getRegion().getRegionNameAsString() +
" from list of regions to assign because in RIT" + " region state: "
+ rit.getState());
- hris.remove(rit.getRegion());
+ if (hris != null) hris.remove(rit.getRegion());
}
}
- LOG.info("Reassigning " + (hris == null? 0: hris.size()) +
- " region(s) that " + serverName +
- " was carrying (skipping " + regionsInTransition.size() +
- " regions(s) that are already in transition)");
+ assert regionsInTransition != null;
+ LOG.info("Reassigning " + ((hris == null)? 0: hris.size()) +
+ " region(s) that " + (serverName == null? "null": serverName) +
+ " was carrying (skipping " +
+ regionsInTransition.size() +
+ " regions(s) that are already in transition)");
// Iterate regions that were on this server and assign them
if (hris != null) {
@@ -342,7 +343,8 @@ public class ServerShutdownHandler exten
final AssignmentManager assignmentManager,
final CatalogTracker catalogTracker)
throws IOException {
- HRegionInfo daughter = getHRegionInfo(result, qualifier);
+ HRegionInfo daughter =
+ MetaReader.parseHRegionInfoFromCatalogResult(result, qualifier);
if (daughter == null) return;
if (isDaughterMissing(catalogTracker, daughter)) {
LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString());
@@ -360,21 +362,6 @@ public class ServerShutdownHandler exten
}
/**
- * Interpret the content of the cell at {@link HConstants#CATALOG_FAMILY} and
- * <code>qualifier</code> as an HRegionInfo and return it, or null.
- * @param r Result instance to pull from.
- * @param qualifier Column family qualifier
- * @return An HRegionInfo instance or null.
- * @throws IOException
- */
- private static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier)
- throws IOException {
- byte [] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
- if (bytes == null || bytes.length <= 0) return null;
- return Writables.getHRegionInfoOrNull(bytes);
- }
-
- /**
* Look for presence of the daughter OR of a split of the daughter in .META.
* Daughter could have been split over on regionserver before a run of the
* catalogJanitor had chance to clear reference from parent.
@@ -416,7 +403,8 @@ public class ServerShutdownHandler exten
@Override
public boolean visit(Result r) throws IOException {
- HRegionInfo hri = getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
+ HRegionInfo hri =
+ MetaReader.parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
if (hri == null) {
LOG.warn("No serialized HRegionInfo in " + r);
return true;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Thu Oct 13 23:12:30 2011
@@ -3061,9 +3061,9 @@ public class HRegion implements HeapSize
EnvironmentEdgeManager.currentTimeMillis(),
Writables.getBytes(r.getRegionInfo())));
edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
- HConstants.META_MIGRATION_QUALIFIER,
+ org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER,
EnvironmentEdgeManager.currentTimeMillis(),
- Bytes.toBytes("true")));
+ Bytes.toBytes(true)));
meta.put(HConstants.CATALOG_FAMILY, edits);
} finally {
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Thu Oct 13 23:12:30 2011
@@ -83,6 +83,7 @@ import org.apache.hadoop.hbase.client.Ac
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.MultiAction;
import org.apache.hadoop.hbase.client.MultiPut;
@@ -334,6 +335,8 @@ public class HRegionServer implements HR
throws IOException, InterruptedException {
this.fsOk = true;
this.conf = conf;
+ // Set how many times to retry talking to another server over HConnection.
+ HConnectionManager.setServerSideHConnectionRetries(this.conf, LOG);
this.isOnline = false;
checkCodecs(this.conf);
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java Thu Oct 13 23:12:30 2011
@@ -78,4 +78,4 @@ public interface RegionServerServices ex
*/
public Map<byte[], Boolean> getRegionsInTransitionInRS();
-}
+}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java Thu Oct 13 23:12:30 2011
@@ -274,17 +274,19 @@ public class ThriftServer {
public List<TRegionInfo> getTableRegions(ByteBuffer tableName)
throws IOError {
try{
- List<HRegionInfo> HRegions = this.admin.getTableRegions(tableName.array());
+ List<HRegionInfo> hris = this.admin.getTableRegions(tableName.array());
List<TRegionInfo> regions = new ArrayList<TRegionInfo>();
- for (HRegionInfo regionInfo : HRegions){
- TRegionInfo region = new TRegionInfo();
- region.startKey = ByteBuffer.wrap(regionInfo.getStartKey());
- region.endKey = ByteBuffer.wrap(regionInfo.getEndKey());
- region.id = regionInfo.getRegionId();
- region.name = ByteBuffer.wrap(regionInfo.getRegionName());
- region.version = regionInfo.getVersion();
- regions.add(region);
+ if (hris != null) {
+ for (HRegionInfo regionInfo : hris){
+ TRegionInfo region = new TRegionInfo();
+ region.startKey = ByteBuffer.wrap(regionInfo.getStartKey());
+ region.endKey = ByteBuffer.wrap(regionInfo.getEndKey());
+ region.id = regionInfo.getRegionId();
+ region.name = ByteBuffer.wrap(regionInfo.getRegionName());
+ region.version = regionInfo.getVersion();
+ regions.add(region);
+ }
}
return regions;
} catch (IOException e){
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Thu Oct 13 23:12:30 2011
@@ -891,8 +891,7 @@ public class HBaseFsck {
// record the latest modification of this META record
long ts = Collections.max(result.list(), comp).getTimestamp();
- Pair<HRegionInfo, ServerName> pair =
- MetaReader.metaRowToRegionPair(result);
+ Pair<HRegionInfo, ServerName> pair = MetaReader.parseCatalogResult(result);
if (pair == null || pair.getFirst() == null) {
emptyRegionInfoQualifiers.add(result);
return true;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java Thu Oct 13 23:12:30 2011
@@ -19,52 +19,29 @@
*/
package org.apache.hadoop.hbase.zookeeper;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.catalog.CatalogTracker;
/**
* Tracks the unassigned zookeeper node used by the META table.
- *
- * A callback is made into the passed {@link CatalogTracker} when
- * <code>.META.</code> completes a new assignment.
* <p>
* If META is already assigned when instantiating this class, you will not
* receive any notification for that assignment. You will receive a
* notification after META has been successfully assigned to a new location.
*/
public class MetaNodeTracker extends ZooKeeperNodeTracker {
- private static final Log LOG = LogFactory.getLog(MetaNodeTracker.class);
-
- /** Catalog tracker to notify when META has a new assignment completed. */
- private final CatalogTracker catalogTracker;
-
/**
* Creates a meta node tracker.
* @param watcher
* @param abortable
*/
- public MetaNodeTracker(final ZooKeeperWatcher watcher,
- final CatalogTracker catalogTracker, final Abortable abortable) {
+ public MetaNodeTracker(final ZooKeeperWatcher watcher, final Abortable abortable) {
super(watcher, ZKUtil.joinZNode(watcher.assignmentZNode,
HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()), abortable);
- this.catalogTracker = catalogTracker;
}
@Override
public void nodeDeleted(String path) {
super.nodeDeleted(path);
- if (!path.equals(node)) return;
- LOG.info("Detected completed assignment of META, notifying catalog tracker");
- try {
- this.catalogTracker.waitForMetaServerConnectionDefault();
- } catch (IOException e) {
- LOG.warn("Tried to reset META server location after seeing the " +
- "completion of a new META assignment but got an IOE", e);
- }
}
}
\ No newline at end of file
Modified: hbase/trunk/src/main/ruby/hbase/admin.rb
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/ruby/hbase/admin.rb?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/ruby/hbase/admin.rb (original)
+++ hbase/trunk/src/main/ruby/hbase/admin.rb Thu Oct 13 23:12:30 2011
@@ -263,8 +263,13 @@ module Hbase
#----------------------------------------------------------------------------------------------
# Truncates table (deletes all records by recreating the table)
- def truncate(table_name)
- h_table = org.apache.hadoop.hbase.client.HTable.new(table_name)
+ def truncate(table_name, conf = nil)
+ h_table = nil
+ unless conf
+ h_table = org.apache.hadoop.hbase.client.HTable.new(table_name)
+ else
+ h_table = org.apache.hadoop.hbase.client.HTable.new(conf, table_name)
+ end
table_description = h_table.getTableDescriptor()
yield 'Disabling table...' if block_given?
disable(table_name)
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java Thu Oct 13 23:12:30 2011
@@ -25,7 +25,6 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -81,15 +80,11 @@ public class TestRegionRebalancing {
this.table = new HTable(UTIL.getConfiguration(), this.desc.getName());
CatalogTracker ct = new CatalogTracker(UTIL.getConfiguration());
ct.start();
- Map<HRegionInfo, ServerName> regions = null;
try {
- regions = MetaReader.fullScan(ct);
+ MetaReader.fullScanMetaAndPrint(ct);
} finally {
ct.stop();
}
- for (Map.Entry<HRegionInfo, ServerName> e: regions.entrySet()) {
- LOG.info(e);
- }
assertEquals("Test table should have right number of regions",
HBaseTestingUtility.KEYS.length + 1/*One extra to account for start/end keys*/,
this.table.getStartKeys().length);
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java Thu Oct 13 23:12:30 2011
@@ -35,16 +35,20 @@ import org.apache.hadoop.hbase.Abortable
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ServerCallable;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.hadoop.util.Progressable;
@@ -160,27 +164,44 @@ public class TestCatalogTracker {
t.join();
}
- @Test public void testGetMetaServerConnectionFails()
+ @Test
+ public void testGetMetaServerConnectionFails()
throws IOException, InterruptedException, KeeperException {
- HConnection connection = Mockito.mock(HConnection.class);
- ConnectException connectException =
- new ConnectException("Connection refused");
- final HRegionInterface implementation =
- Mockito.mock(HRegionInterface.class);
- Mockito.when(implementation.get((byte [])Mockito.any(), (Get)Mockito.any())).
- thenThrow(connectException);
- Mockito.when(connection.getHRegionConnection((HServerAddress)Matchers.anyObject(), Matchers.anyBoolean())).
- thenReturn(implementation);
- Assert.assertNotNull(connection.getHRegionConnection(new HServerAddress(), false));
- final CatalogTracker ct = constructAndStartCatalogTracker(connection);
+ HConnection connection =
+ HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
try {
- RootLocationEditor.setRootLocation(this.watcher,
- new ServerName("example.com", 1234, System.currentTimeMillis()));
- Assert.assertFalse(ct.verifyMetaRegionLocation(100));
+ // Mock an HRegionInterface.
+ final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
+ // If a 'get' is called on mocked interface, throw connection refused.
+ Mockito.when(implementation.get((byte[]) Mockito.any(), (Get) Mockito.any())).
+ thenThrow(new ConnectException("Connection refused"));
+ // Make it so our implementation is returned when we do a connection.
+ // Need to fake out the location lookup stuff first.
+ ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis());
+ final HRegionLocation anyLocation =
+ new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(),
+ sn.getPort());
+ Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(),
+ (byte[]) Mockito.any(), Mockito.anyBoolean())).
+ thenReturn(anyLocation);
+ Mockito.when(connection.getHRegionConnection(Mockito.anyString(),
+ Mockito.anyInt(), Matchers.anyBoolean())).
+ thenReturn(implementation);
+ // Now start up the catalogtracker with our doctored Connection.
+ final CatalogTracker ct = constructAndStartCatalogTracker(connection);
+ try {
+ RootLocationEditor.setRootLocation(this.watcher, sn);
+ long timeout = UTIL.getConfiguration().
+ getLong("hbase.catalog.verification.timeout", 1000);
+ Assert.assertFalse(ct.verifyMetaRegionLocation(timeout));
+ } finally {
+ // Clean out root location or later tests will be confused... they
+ // presume start fresh in zk.
+ RootLocationEditor.deleteRootLocation(this.watcher);
+ }
} finally {
- // Clean out root location or later tests will be confused... they presume
- // start fresh in zk.
- RootLocationEditor.deleteRootLocation(this.watcher);
+ // Clear out our doctored connection or could mess up subsequent tests.
+ HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
}
}
@@ -200,9 +221,9 @@ public class TestCatalogTracker {
Mockito.mock(HRegionInterface.class);
Mockito.when(implementation.getRegionInfo((byte [])Mockito.any())).
thenThrow(connectException);
- Mockito.when(connection.getHRegionConnection((HServerAddress)Matchers.anyObject(), Matchers.anyBoolean())).
+ Mockito.when(connection.getHRegionConnection(Mockito.anyString(),
+ Mockito.anyInt(), Mockito.anyBoolean())).
thenReturn(implementation);
- Assert.assertNotNull(connection.getHRegionConnection(new HServerAddress(), false));
final CatalogTracker ct = constructAndStartCatalogTracker(connection);
try {
RootLocationEditor.setRootLocation(this.watcher,
@@ -225,8 +246,14 @@ public class TestCatalogTracker {
@Test (expected = NotAllMetaRegionsOnlineException.class)
public void testTimeoutWaitForMeta()
throws IOException, InterruptedException {
- final CatalogTracker ct = constructAndStartCatalogTracker();
- ct.waitForMeta(100);
+ HConnection connection =
+ HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
+ try {
+ final CatalogTracker ct = constructAndStartCatalogTracker(connection);
+ ct.waitForMeta(100);
+ } finally {
+ HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
+ }
}
/**
@@ -259,62 +286,84 @@ public class TestCatalogTracker {
/**
* Test waiting on meta w/ no timeout specified.
- * @throws IOException
- * @throws InterruptedException
- * @throws KeeperException
+ * @throws Exception
*/
@Test public void testNoTimeoutWaitForMeta()
- throws IOException, InterruptedException, KeeperException {
+ throws Exception {
// Mock an HConnection and a HRegionInterface implementation. Have the
// HConnection return the HRI. Have the HRI return a few mocked up responses
// to make our test work.
- HConnection connection = Mockito.mock(HConnection.class);
- HRegionInterface mockHRI = Mockito.mock(HRegionInterface.class);
- // Make the HRI return an answer no matter how Get is called. Same for
- // getHRegionInfo. Thats enough for this test.
- Mockito.when(connection.getHRegionConnection((String)Mockito.any(),
- Matchers.anyInt())).thenReturn(mockHRI);
-
- final CatalogTracker ct = constructAndStartCatalogTracker(connection);
- ServerName hsa = ct.getMetaLocation();
- Assert.assertNull(hsa);
-
- // Now test waiting on meta location getting set.
- Thread t = new WaitOnMetaThread(ct) {
- @Override
- void doWaiting() throws InterruptedException {
- this.ct.waitForMeta();
- }
- };
- startWaitAliveThenWaitItLives(t, 1000);
+ HConnection connection =
+ HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
+ try {
+ // Mock an HRegionInterface.
+
+ final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
+ // Make it so our implementation is returned when we do a connection.
+ // Need to fake out the location lookup stuff first.
+ ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis());
+ final HRegionLocation anyLocation =
+ new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(),
+ sn.getPort());
+ Mockito.when(connection.getRegionLocation((byte[]) Mockito.any(),
+ (byte[]) Mockito.any(), Mockito.anyBoolean())).
+ thenReturn(anyLocation);
+ // Have implementation returned which ever way getHRegionConnection is called.
+ Mockito.when(connection.getHRegionConnection(Mockito.anyString(),
+ Mockito.anyInt(), Matchers.anyBoolean())).
+ thenReturn(implementation);
+ Mockito.when(connection.getHRegionConnection(Mockito.anyString(),
+ Mockito.anyInt())).
+ thenReturn(implementation);
+
+ final CatalogTracker ct = constructAndStartCatalogTracker(connection);
+ ServerName hsa = ct.getMetaLocation();
+ Assert.assertNull(hsa);
+
+ // Now test waiting on meta location getting set.
+ Thread t = new WaitOnMetaThread(ct) {
+ @Override
+ void doWaiting() throws InterruptedException {
+ this.ct.waitForMeta();
+ }
+ };
+ startWaitAliveThenWaitItLives(t, 1000);
- // Now the ct is up... set into the mocks some answers that make it look
- // like things have been getting assigned. Make it so we'll return a
- // location (no matter what the Get is). Same for getHRegionInfo -- always
- // just return the meta region.
- List<KeyValue> kvs = new ArrayList<KeyValue>();
- kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
- HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
- Bytes.toBytes(SN.getHostAndPort())));
- kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
- HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
- Bytes.toBytes(SN.getStartcode())));
- final Result result = new Result(kvs);
- Mockito.when(mockHRI.get((byte [])Mockito.any(), (Get)Mockito.any())).
- thenReturn(result);
- Mockito.when(mockHRI.getRegionInfo((byte [])Mockito.any())).
- thenReturn(HRegionInfo.FIRST_META_REGIONINFO);
- // This should trigger wake up of meta wait (Its the removal of the meta
- // region unassigned node that triggers catalogtrackers that a meta has
- // been assigned.
- String node = ct.getMetaNodeTracker().getNode();
- ZKUtil.createAndFailSilent(this.watcher, node);
- MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
- ZKUtil.deleteNode(this.watcher, node);
- // Join the thread... should exit shortly.
- t.join();
- // Now meta is available.
- Assert.assertTrue(ct.getMetaLocation().equals(SN));
+ // Now the ct is up... set into the mocks some answers that make it look
+ // like things have been getting assigned. Make it so we'll return a
+ // location (no matter what the Get is). Same for getHRegionInfo -- always
+ // just return the meta region.
+ List<KeyValue> kvs = new ArrayList<KeyValue>();
+ kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
+ HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
+ kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
+ HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
+ Bytes.toBytes(SN.getHostAndPort())));
+ kvs.add(new KeyValue(HConstants.EMPTY_BYTE_ARRAY,
+ HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
+ Bytes.toBytes(SN.getStartcode())));
+ final Result result = new Result(kvs);
+ Mockito.when(connection.getRegionServerWithRetries((ServerCallable<Result>)Mockito.any())).
+ thenReturn(result);
+ Mockito.when(implementation.getRegionInfo((byte[]) Mockito.any())).
+ thenReturn(HRegionInfo.FIRST_META_REGIONINFO);
+ // This should trigger wake up of meta wait (Its the removal of the meta
+ // region unassigned node that triggers catalogtrackers that a meta has
+ // been assigned).
+ String node = ct.getMetaNodeTracker().getNode();
+ ZKUtil.createAndFailSilent(this.watcher, node);
+ MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN);
+ ZKUtil.deleteNode(this.watcher, node);
+ // Go get the new meta location. waitForMeta gets and verifies meta.
+ Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
+ // Join the thread... should exit shortly.
+ t.join();
+ // Now meta is available.
+ Assert.assertTrue(ct.waitForMeta(10000).equals(SN));
+ } finally {
+ HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
+ }
}
private void startWaitAliveThenWaitItLives(final Thread t, final int ms) {
@@ -361,4 +410,4 @@ public class TestCatalogTracker {
this.ct.waitForRoot();
}
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java Thu Oct 13 23:12:30 2011
@@ -19,7 +19,10 @@
*/
package org.apache.hadoop.hbase.catalog;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.List;
@@ -34,8 +37,6 @@ import org.apache.hadoop.hbase.HConstant
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -70,11 +71,15 @@ public class TestMetaReaderEditor {
};
@BeforeClass public static void beforeClass() throws Exception {
- UTIL.startMiniCluster();
+ UTIL.startMiniCluster(3);
}
@Before public void setup() throws IOException, InterruptedException {
Configuration c = new Configuration(UTIL.getConfiguration());
+ // Tests to 4 retries every 5 seconds. Make it try every 1 second so more
+ // responsive. 1 second is default as is ten retries.
+ c.setLong("hbase.client.pause", 1000);
+ c.setInt("hbase.client.retries.number", 10);
zkw = new ZooKeeperWatcher(c, "TestMetaReaderEditor", ABORTABLE);
ct = new CatalogTracker(zkw, c, ABORTABLE);
ct.start();
@@ -84,6 +89,111 @@ public class TestMetaReaderEditor {
UTIL.shutdownMiniCluster();
}
+ /**
+ * Does {@link MetaReader#getRegion(CatalogTracker, byte[])} and a write
+ * against .META. while its hosted server is restarted to prove our retrying
+ * works.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test (timeout = 180000) public void testRetrying()
+ throws IOException, InterruptedException {
+ final String name = "testRetrying";
+ LOG.info("Started " + name);
+ final byte [] nameBytes = Bytes.toBytes(name);
+ HTable t = UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY);
+ int regionCount = UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY);
+ // Test it works getting a region from just made user table.
+ final List<HRegionInfo> regions =
+ testGettingTableRegions(this.ct, nameBytes, regionCount);
+ MetaTask reader = new MetaTask(this.ct, "reader") {
+ @Override
+ void metaTask() throws Throwable {
+ testGetRegion(this.ct, regions.get(0));
+ LOG.info("Read " + regions.get(0).getEncodedName());
+ }
+ };
+ MetaTask writer = new MetaTask(this.ct, "writer") {
+ @Override
+ void metaTask() throws Throwable {
+ MetaEditor.addRegionToMeta(this.ct, regions.get(0));
+ LOG.info("Wrote " + regions.get(0).getEncodedName());
+ }
+ };
+ reader.start();
+ writer.start();
+ // Make sure reader and writer are working.
+ assertTrue(reader.isProgressing());
+ assertTrue(writer.isProgressing());
+ // Kill server hosting meta -- twice . See if our reader/writer ride over the
+ // meta moves. They'll need to retry.
+ for (int i = 0; i < 2; i++) {
+ LOG.info("Restart=" + i);
+ UTIL.ensureSomeRegionServersAvailable(2);
+ int index = -1;
+ do {
+ index = UTIL.getMiniHBaseCluster().getServerWithMeta();
+ } while (index == -1);
+ UTIL.getMiniHBaseCluster().abortRegionServer(index);
+ UTIL.getMiniHBaseCluster().waitOnRegionServer(index);
+ }
+ assertTrue(reader.toString(), reader.isProgressing());
+ assertTrue(writer.toString(), writer.isProgressing());
+ reader.stop = true;
+ writer.stop = true;
+ reader.join();
+ writer.join();
+ }
+
+ /**
+ * Thread that runs a MetaReader/MetaEditor task until asked stop.
+ */
+ abstract static class MetaTask extends Thread {
+ boolean stop = false;
+ int count = 0;
+ Throwable t = null;
+ final CatalogTracker ct;
+
+ MetaTask(final CatalogTracker ct, final String name) {
+ super(name);
+ this.ct = ct;
+ }
+
+ @Override
+ public void run() {
+ try {
+ while(!this.stop) {
+ LOG.info("Before " + this.getName()+ ", count=" + this.count);
+ metaTask();
+ this.count += 1;
+ LOG.info("After " + this.getName() + ", count=" + this.count);
+ Thread.sleep(100);
+ }
+ } catch (Throwable t) {
+ LOG.info(this.getName() + " failed", t);
+ this.t = t;
+ }
+ }
+
+ boolean isProgressing() throws InterruptedException {
+ int currentCount = this.count;
+ while(currentCount == this.count) {
+ if (!isAlive()) return false;
+ if (this.t != null) return false;
+ Thread.sleep(10);
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "count=" + this.count + ", t=" +
+ (this.t == null? "null": this.t.toString());
+ }
+
+ abstract void metaTask() throws Throwable;
+ }
+
@Test public void testGetRegionsCatalogTables()
throws IOException, InterruptedException {
List<HRegionInfo> regions =
@@ -114,19 +224,9 @@ public class TestMetaReaderEditor {
@Test public void testGetRegion() throws IOException, InterruptedException {
final String name = "testGetRegion";
LOG.info("Started " + name);
- final byte [] nameBytes = Bytes.toBytes(name);
- HTable t = UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY);
- int regionCount = UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY);
-
- // Test it works getting a region from user table.
- List<HRegionInfo> regions = MetaReader.getTableRegions(ct, nameBytes);
- assertEquals(regionCount, regions.size());
- Pair<HRegionInfo, ServerName> pair =
- MetaReader.getRegion(ct, regions.get(0).getRegionName());
- assertEquals(regions.get(0).getEncodedName(),
- pair.getFirst().getEncodedName());
// Test get on non-existent region.
- pair = MetaReader.getRegion(ct, Bytes.toBytes("nonexistent-region"));
+ Pair<HRegionInfo, ServerName> pair =
+ MetaReader.getRegion(ct, Bytes.toBytes("nonexistent-region"));
assertNull(pair);
// Test it works getting a region from meta/root.
pair =
@@ -137,7 +237,8 @@ public class TestMetaReaderEditor {
}
// Test for the optimization made in HBASE-3650
- @Test public void testScanMetaForTable() throws IOException {
+ @Test public void testScanMetaForTable()
+ throws IOException, InterruptedException {
final String name = "testScanMetaForTable";
LOG.info("Started " + name);
@@ -165,4 +266,25 @@ public class TestMetaReaderEditor {
}
assertEquals(1, MetaReader.getTableRegions(ct, greaterName).size());
}
-}
\ No newline at end of file
+
+ private static List<HRegionInfo> testGettingTableRegions(final CatalogTracker ct,
+ final byte [] nameBytes, final int regionCount)
+ throws IOException, InterruptedException {
+ List<HRegionInfo> regions = MetaReader.getTableRegions(ct, nameBytes);
+ assertEquals(regionCount, regions.size());
+ Pair<HRegionInfo, ServerName> pair =
+ MetaReader.getRegion(ct, regions.get(0).getRegionName());
+ assertEquals(regions.get(0).getEncodedName(),
+ pair.getFirst().getEncodedName());
+ return regions;
+ }
+
+ private static void testGetRegion(final CatalogTracker ct,
+ final HRegionInfo region)
+ throws IOException, InterruptedException {
+ Pair<HRegionInfo, ServerName> pair =
+ MetaReader.getRegion(ct, region.getRegionName());
+ assertEquals(region.getEncodedName(),
+ pair.getFirst().getEncodedName());
+ }
+}
Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java?rev=1183158&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java Thu Oct 13 23:12:30 2011
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.catalog;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test MetaReader/Editor but without spinning up a cluster.
+ * We mock regionserver back and forth (we do spin up a zk cluster).
+ */
+public class TestMetaReaderEditorNoCluster {
+ private static final Log LOG = LogFactory.getLog(TestMetaReaderEditorNoCluster.class);
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ private static final Abortable ABORTABLE = new Abortable() {
+ boolean aborted = false;
+ @Override
+ public void abort(String why, Throwable e) {
+ LOG.info(why, e);
+ this.aborted = true;
+ throw new RuntimeException(e);
+ }
+ @Override
+ public boolean isAborted() {
+ return this.aborted;
+ }
+ };
+
+ @Before
+ public void before() throws Exception {
+ UTIL.startMiniZKCluster();
+ }
+
+ @After
+ public void after() throws IOException {
+ UTIL.shutdownMiniZKCluster();
+ }
+
+ /**
+ * Test that MetaReader will ride over server throwing
+ * "Server not running" IOEs.
+ * @see https://issues.apache.org/jira/browse/HBASE-3446
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ public void testRideOverServerNotRunning() throws IOException, InterruptedException {
+ // Need a zk watcher.
+ ZooKeeperWatcher zkw = new ZooKeeperWatcher(UTIL.getConfiguration(),
+ this.getClass().getSimpleName(), ABORTABLE, true);
+ // This is a servername we use in a few places below.
+ ServerName sn = new ServerName("example.com", 1234, System.currentTimeMillis());
+
+ HConnection connection = null;
+ CatalogTracker ct = null;
+ try {
+ // Mock an HRegionInterface. Our mock implementation will fail a few
+ // times when we go to open a scanner.
+ final HRegionInterface implementation = Mockito.mock(HRegionInterface.class);
+ // When openScanner called throw IOE 'Server not running' a few times
+ // before we return a scanner id. Whats WEIRD is that these
+ // exceptions do not show in the log because they are caught and only
+ // printed if we FAIL. We eventually succeed after retry so these don't
+ // show. We will know if they happened or not because we will ask
+ // mockito at the end of this test to verify that openscanner was indeed
+ // called the wanted number of times.
+ final long scannerid = 123L;
+ Mockito.when(implementation.openScanner((byte [])Mockito.any(),
+ (Scan)Mockito.any())).
+ thenThrow(new IOException("Server not running (1 of 3)")).
+ thenThrow(new IOException("Server not running (2 of 3)")).
+ thenThrow(new IOException("Server not running (3 of 3)")).
+ thenReturn(scannerid);
+ // Make it so a verifiable answer comes back when next is called. Return
+ // the verifiable answer and then a null so we stop scanning. Our
+ // verifiable answer is something that looks like a row in META with
+ // a server and startcode that is that of the above defined servername.
+ List<KeyValue> kvs = new ArrayList<KeyValue>();
+ final byte [] rowToVerify = Bytes.toBytes("rowToVerify");
+ kvs.add(new KeyValue(rowToVerify,
+ HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(HRegionInfo.FIRST_META_REGIONINFO)));
+ kvs.add(new KeyValue(rowToVerify,
+ HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
+ Bytes.toBytes(sn.getHostAndPort())));
+ kvs.add(new KeyValue(rowToVerify,
+ HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
+ Bytes.toBytes(sn.getStartcode())));
+ final Result [] result = new Result [] {new Result(kvs)};
+ Mockito.when(implementation.next(Mockito.anyLong(), Mockito.anyInt())).
+ thenReturn(result).
+ thenReturn(null);
+
+ // Associate a spied-upon HConnection with UTIL.getConfiguration. Need
+ // to shove this in here first so it gets picked up all over; e.g. by
+ // HTable.
+ connection = HConnectionTestingUtility.getSpiedConnection(UTIL.getConfiguration());
+ // Fix the location lookup so it 'works' though no network. First
+ // make an 'any location' object.
+ final HRegionLocation anyLocation =
+ new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, sn.getHostname(),
+ sn.getPort());
+ // Return the any location object when locateRegion is called in HTable
+ // constructor and when its called by ServerCallable (it uses getRegionLocation).
+ // The ugly format below comes of 'Important gotcha on spying real objects!' from
+ // http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html
+ Mockito.doReturn(anyLocation).
+ when(connection).locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any());
+ Mockito.doReturn(anyLocation).
+ when(connection).getRegionLocation((byte[]) Mockito.any(),
+ (byte[]) Mockito.any(), Mockito.anyBoolean());
+
+ // Now shove our HRI implementation into the spied-upon connection.
+ Mockito.doReturn(implementation).
+ when(connection).getHRegionConnection(Mockito.anyString(), Mockito.anyInt());
+
+ // Now start up the catalogtracker with our doctored Connection.
+ ct = new CatalogTracker(zkw, null, connection, ABORTABLE, 0);
+ ct.start();
+ // Scan meta for user tables and verify we got back expected answer.
+ NavigableMap<HRegionInfo, Result> hris = MetaReader.getServerUserRegions(ct, sn);
+ assertTrue(hris.size() == 1);
+ assertTrue(hris.firstEntry().getKey().equals(HRegionInfo.FIRST_META_REGIONINFO));
+ assertTrue(Bytes.equals(rowToVerify, hris.firstEntry().getValue().getRow()));
+ // Finally verify that openscanner was called four times -- three times
+ // with exception and then on 4th attempt we succeed.
+ Mockito.verify(implementation, Mockito.times(4)).
+ openScanner((byte [])Mockito.any(), (Scan)Mockito.any());
+ } finally {
+ if (ct != null) ct.stop();
+ HConnectionManager.deleteConnection(UTIL.getConfiguration(), true);
+ zkw.close();
+ }
+ }
+}
Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java?rev=1183158&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java Thu Oct 13 23:12:30 2011
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionImplementation;
+import org.apache.hadoop.hbase.client.HConnectionManager.HConnectionKey;
+import org.mockito.Mockito;
+
+/**
+ * {@link HConnection} testing utility.
+ */
+public class HConnectionTestingUtility {
+ /*
+ * Not part of {@link HBaseTestingUtility} because this class is not
+ * in same package as {@link HConnection}. Would have to reveal ugly
+ * {@link HConnectionManager} innards to HBaseTestingUtility to give it access.
+ */
+ /**
+ * Get a Mocked {@link HConnection} that goes with the passed <code>conf</code>
+ * configuration instance. Minimally the mock will return
+ * <code>conf</conf> when {@link HConnection#getConfiguration()} is invoked.
+ * Be sure to shutdown the connection when done by calling
+ * {@link HConnectionManager#deleteConnection(Configuration, boolean)} else it
+ * will stick around; this is probably not what you want.
+ * @param conf configuration
+ * @return HConnection object for <code>conf</code>
+ * @throws ZooKeeperConnectionException
+ */
+ public static HConnection getMockedConnection(final Configuration conf)
+ throws ZooKeeperConnectionException {
+ HConnectionKey connectionKey = new HConnectionKey(conf);
+ synchronized (HConnectionManager.HBASE_INSTANCES) {
+ HConnectionImplementation connection =
+ HConnectionManager.HBASE_INSTANCES.get(connectionKey);
+ if (connection == null) {
+ connection = Mockito.mock(HConnectionImplementation.class);
+ Mockito.when(connection.getConfiguration()).thenReturn(conf);
+ HConnectionManager.HBASE_INSTANCES.put(connectionKey, connection);
+ }
+ return connection;
+ }
+ }
+
+ /**
+ * Get a Mockito spied-upon {@link HConnection} that goes with the passed
+ * <code>conf</code> configuration instance.
+ * Be sure to shutdown the connection when done by calling
+ * {@link HConnectionManager#deleteConnection(Configuration, boolean)} else it
+ * will stick around; this is probably not what you want.
+ * @param conf configuration
+ * @return HConnection object for <code>conf</code>
+ * @throws ZooKeeperConnectionException
+ * @see http://mockito.googlecode.com/svn/branches/1.6/javadoc/org/mockito/Mockito.html#spy(T)
+ */
+ public static HConnection getSpiedConnection(final Configuration conf)
+ throws ZooKeeperConnectionException {
+ HConnectionKey connectionKey = new HConnectionKey(conf);
+ synchronized (HConnectionManager.HBASE_INSTANCES) {
+ HConnectionImplementation connection =
+ HConnectionManager.HBASE_INSTANCES.get(connectionKey);
+ if (connection == null) {
+ connection = Mockito.spy(new HConnectionImplementation(conf));
+ HConnectionManager.HBASE_INSTANCES.put(connectionKey, connection);
+ }
+ return connection;
+ }
+ }
+}
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java Thu Oct 13 23:12:30 2011
@@ -19,6 +19,10 @@
*/
package org.apache.hadoop.hbase.client;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashSet;
@@ -27,6 +31,8 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -38,12 +44,6 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
/**
* This class is for testing HCM features
@@ -225,4 +225,4 @@ public class TestHCM {
Thread.sleep(50);
}
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java Thu Oct 13 23:12:30 2011
@@ -28,7 +28,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.catalog.MetaEditor;
+import org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD;
import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -77,7 +77,8 @@ public class TestMetaMigration {
@Test
public void testMetaUpdatedFlagInROOT() throws Exception {
LOG.info("Starting testMetaUpdatedFlagInROOT");
- boolean metaUpdated = miniHBaseCluster.getMaster().isMetaHRIUpdated();
+ boolean metaUpdated =
+ MetaMigrationRemovingHTD.isMetaHRIUpdated(miniHBaseCluster.getMaster());
assertEquals(true, metaUpdated);
LOG.info("END testMetaUpdatedFlagInROOT");
}
@@ -97,16 +98,17 @@ public class TestMetaMigration {
Bytes.toBytes("region_b")});
CatalogTracker ct = miniHBaseCluster.getMaster().getCatalogTracker();
// just for this test set it to false.
- MetaEditor.updateRootWithMetaMigrationStatus(ct, false);
+ MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(ct, false);
MetaReader.fullScanMetaAndPrint(ct);
- LOG.info("MEta Print completed.testUpdatesOnMetaWithLegacyHRI");
+ LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI");
- List<HTableDescriptor> htds = MetaEditor.updateMetaWithNewRegionInfo(
+ List<HTableDescriptor> htds = MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo(
TEST_UTIL.getHBaseCluster().getMaster());
MetaReader.fullScanMetaAndPrint(ct);
assertEquals(3, htds.size());
// Assert that the flag in ROOT is updated to reflect the correct status
- boolean metaUpdated = miniHBaseCluster.getMaster().isMetaHRIUpdated();
+ boolean metaUpdated =
+ MetaMigrationRemovingHTD.isMetaHRIUpdated(miniHBaseCluster.getMaster());
assertEquals(true, metaUpdated);
LOG.info("END testMetaWithLegacyHRI");
@@ -138,15 +140,16 @@ public class TestMetaMigration {
TEST_UTIL.createMultiRegionsWithLegacyHRI(conf, htd, FAMILY, 10);
CatalogTracker ct = miniHBaseCluster.getMaster().getCatalogTracker();
// just for this test set it to false.
- MetaEditor.updateRootWithMetaMigrationStatus(ct, false);
+ MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(ct, false);
//MetaReader.fullScanMetaAndPrint(ct);
LOG.info("MEta Print completed.testUpdatesOnMetaWithLegacyHRI");
- List<HTableDescriptor> htds = MetaEditor.updateMetaWithNewRegionInfo(
+ List<HTableDescriptor> htds = MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo(
TEST_UTIL.getHBaseCluster().getMaster());
assertEquals(10, htds.size());
// Assert that the flag in ROOT is updated to reflect the correct status
- boolean metaUpdated = miniHBaseCluster.getMaster().isMetaHRIUpdated();
+ boolean metaUpdated =
+ MetaMigrationRemovingHTD.isMetaHRIUpdated(miniHBaseCluster.getMaster());
assertEquals(true, metaUpdated);
LOG.info("END testMetaWithLegacyHRI");