You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/10/14 01:12:31 UTC
svn commit: r1183158 [1/3] - in /hbase/trunk: ./
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/catalog/
src/main/java/org/apache/hadoop/hbase/client/
src/main/java/org/apache/hadoop/hbase/ipc/ src/main/java/org/apache/had...
Author: stack
Date: Thu Oct 13 23:12:30 2011
New Revision: 1183158
URL: http://svn.apache.org/viewvc?rev=1183158&view=rev
Log:
HBASE-3446 ProcessServerShutdown fails if META moves, orphaning lots of regions
Added:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HTable.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Result.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java
hbase/trunk/src/main/ruby/hbase/admin.rb
hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
hbase/trunk/src/test/ruby/hbase/admin_test.rb
hbase/trunk/src/test/ruby/shell/shell_test.rb
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Thu Oct 13 23:12:30 2011
@@ -361,6 +361,8 @@ Release 0.92.0 - Unreleased
names, need to use a more consistent block naming scheme (jgray)
HBASE-4551 Fix pom and some test cases to compile and run against
Hadoop 0.23 (todd)
+ HBASE-3446 ProcessServerShutdown fails if META moves, orphaning lots of
+ regions
TESTS
HBASE-4450 test for number of blocks read: to serve as baseline for expected
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java Thu Oct 13 23:12:30 2011
@@ -185,10 +185,6 @@ public final class HConstants {
/** The file name used to store HTD in HDFS */
public static final String TABLEINFO_NAME = ".tableinfo";
- /** The metaupdated column qualifier */
- public static final byte [] META_MIGRATION_QUALIFIER = Bytes.toBytes("metamigrated");
-
-
/** Default maximum file size */
public static final long DEFAULT_MAX_FILE_SIZE = 256 * 1024 * 1024;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java Thu Oct 13 23:12:30 2011
@@ -1043,6 +1043,7 @@ public class KeyValue implements Writabl
* @return True if this KV is a {@link KeyValue.Type#Delete} type.
*/
public boolean isDeleteType() {
+ // TODO: Fix this method name vis-a-vis isDelete!
return getType() == Type.Delete.getCode();
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java Thu Oct 13 23:12:30 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.hadoop.hbase.master.HMaster;
@@ -238,6 +239,7 @@ public class LocalHBaseCluster {
List<RegionServerThread> list = getRegionServers();
for (JVMClusterUtil.RegionServerThread rst: list) {
if (rst.isAlive()) liveServers.add(rst);
+ else LOG.info("Not alive " + rst.getName());
}
return liveServers;
}
@@ -386,12 +388,12 @@ public class LocalHBaseCluster {
*/
public void join() {
if (this.regionThreads != null) {
- for(Thread t: this.regionThreads) {
- if (t.isAlive()) {
- try {
- t.join();
+ for(Thread t: this.regionThreads) {
+ if (t.isAlive()) {
+ try {
+ Threads.threadDumpingIsAlive(t);
} catch (InterruptedException e) {
- // continue
+ LOG.debug("Interrupted", e);
}
}
}
@@ -400,9 +402,9 @@ public class LocalHBaseCluster {
for (Thread t : this.masterThreads) {
if (t.isAlive()) {
try {
- t.join();
+ Threads.threadDumpingIsAlive(t);
} catch (InterruptedException e) {
- // continue
+ LOG.debug("Interrupted", e);
}
}
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java Thu Oct 13 23:12:30 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2010 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -33,7 +31,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
-import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
@@ -60,30 +57,76 @@ import org.apache.hadoop.ipc.RemoteExcep
* interrupt waits and close up shop.
*/
public class CatalogTracker {
+ // TODO: This class needs a rethink. The original intent was that it would be
+ // the one-stop-shop for root and meta locations and that it would get this
+ // info from reading and watching zk state. The class was to be used by
+ // servers when they needed to know of root and meta movement but also by
+ // client-side (inside in HTable) so rather than figure root and meta
+ // locations on fault, the client would instead get notifications out of zk.
+ //
+ // But this original intent is frustrated by the fact that this class has to
+ // read an hbase table, the -ROOT- table, to figure out the .META. region
+ // location which means we depend on an HConnection. HConnection will do
+ // retrying but also, it has its own mechanism for finding root and meta
+ // locations (and for 'verifying'; it tries the location and if it fails, does
+ // new lookup, etc.). So, at least for now, HConnection (or HTable) can't
+ // have a CT since CT needs a HConnection (Even then, do want HT to have a CT?
+ // For HT keep up a session with ZK? Rather, shouldn't we do like asynchbase
+ // where we'd open a connection to zk, read what we need then let the
+ // connection go?). The 'fix' is make it so both root and meta addresses
+ // are wholey up in zk -- not in zk (root) -- and in an hbase table (meta).
+ //
+ // But even then, this class does 'verification' of the location and it does
+ // this by making a call over an HConnection (which will do its own root
+ // and meta lookups). Isn't this verification 'useless' since when we
+ // return, whatever is dependent on the result of this call then needs to
+ // use HConnection; what we have verified may change in meantime (HConnection
+ // uses the CT primitives, the root and meta trackers finding root locations).
+ //
+ // When meta is moved to zk, this class may make more sense. In the
+ // meantime, it does not cohere. It should just watch meta and root and not
+ // NOT do verification -- let that be out in HConnection since its going to
+ // be done there ultimately anyways.
+ //
+ // This class has spread throughout the codebase. It needs to be reigned in.
+ // This class should be used server-side only, even if we move meta location
+ // up into zk. Currently its used over in the client package. Its used in
+ // MetaReader and MetaEditor classes usually just to get the Configuration
+ // its using (It does this indirectly by asking its HConnection for its
+ // Configuration and even then this is just used to get an HConnection out on
+ // the other end). I made https://issues.apache.org/jira/browse/HBASE-4495 for
+ // doing CT fixup. St.Ack 09/30/2011.
+ //
private static final Log LOG = LogFactory.getLog(CatalogTracker.class);
- private final Configuration conf;
private final HConnection connection;
private final ZooKeeperWatcher zookeeper;
private final RootRegionTracker rootRegionTracker;
private final MetaNodeTracker metaNodeTracker;
private final AtomicBoolean metaAvailable = new AtomicBoolean(false);
- /**
+ private boolean instantiatedzkw = false;
+
+ /*
* Do not clear this address once set. Its needed when we do
* server shutdown processing -- we need to know who had .META. last. If you
* want to know if the address is good, rely on {@link #metaAvailable} value.
*/
private ServerName metaLocation;
+
+ /*
+ * Timeout waiting on root or meta to be set.
+ */
private final int defaultTimeout;
+
private boolean stopped = false;
- public static final byte [] ROOT_REGION =
+ static final byte [] ROOT_REGION_NAME =
HRegionInfo.ROOT_REGIONINFO.getRegionName();
- public static final byte [] META_REGION =
+ static final byte [] META_REGION_NAME =
HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
/**
- * Constructs a catalog tracker. Find current state of catalog tables and
- * begin active tracking by executing {@link #start()} post construction. Does
+ * Constructs a catalog tracker. Find current state of catalog tables.
+ * Begin active tracking by executing {@link #start()} post construction. Does
* not timeout.
*
* @param conf
@@ -97,29 +140,36 @@ public class CatalogTracker {
}
/**
- * Constructs the catalog tracker. Find current state of catalog tables and
- * begin active tracking by executing {@link #start()} post construction.
+ * Constructs the catalog tracker. Find current state of catalog tables.
+ * Begin active tracking by executing {@link #start()} post construction.
* Does not timeout.
- * @param zk
+ * @param zk If zk is null, we'll create an instance (and shut it down
+ * when {@link #stop()} is called) else we'll use what is passed.
* @param connection server connection
- * @param abortable if fatal exception
+ * @param abortable If fatal exception we'll call abort on this. May be null.
+ * If it is we'll use the Connection associated with the passed
+ * {@link Configuration} as our Abortable.
* @throws IOException
*/
public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
final Abortable abortable)
throws IOException {
- this(zk, conf, abortable, 0);
+ this(zk, conf, abortable,
+ conf.getInt("hbase.catalogtracker.default.timeout", 1000));
}
/**
- * Constructs the catalog tracker. Find current state of catalog tables and
- * begin active tracking by executing {@link #start()} post construction.
- * @param zk
- * @param connection server connection
- * @param abortable if fatal exception
+ * Constructs the catalog tracker. Find current state of catalog tables.
+ * Begin active tracking by executing {@link #start()} post construction.
+ * @param zk If zk is null, we'll create an instance (and shut it down
+ * when {@link #stop()} is called) else we'll use what is passed.
+ * @param conf
+ * @param abortable If fatal exception we'll call abort on this. May be null.
+ * If it is we'll use the Connection associated with the passed
+ * {@link Configuration} as our Abortable.
* @param defaultTimeout Timeout to use. Pass zero for no timeout
* ({@link Object#wait(long)} when passed a <code>0</code> waits for ever).
- * @throws IOException
+ * @throws IOException
*/
public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
Abortable abortable, final int defaultTimeout)
@@ -130,14 +180,29 @@ public class CatalogTracker {
CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
HConnection connection, Abortable abortable, final int defaultTimeout)
throws IOException {
- this.conf = conf;
this.connection = connection;
- this.zookeeper = (zk == null) ? this.connection.getZooKeeperWatcher() : zk;
if (abortable == null) {
+ // A connection is abortable.
abortable = this.connection;
}
+ if (zk == null) {
+ // Create our own. Set flag so we tear it down on stop.
+ this.zookeeper =
+ new ZooKeeperWatcher(conf, "catalogtracker-on-" + connection.toString(),
+ abortable);
+ instantiatedzkw = true;
+ } else {
+ this.zookeeper = zk;
+ }
this.rootRegionTracker = new RootRegionTracker(zookeeper, abortable);
- this.metaNodeTracker = new MetaNodeTracker(zookeeper, this, abortable);
+ final CatalogTracker ct = this;
+ // Override nodeDeleted so we get notified when meta node deleted
+ this.metaNodeTracker = new MetaNodeTracker(zookeeper, abortable) {
+ public void nodeDeleted(String path) {
+ if (!path.equals(node)) return;
+ ct.resetMetaLocation();
+ }
+ };
this.defaultTimeout = defaultTimeout;
}
@@ -149,9 +214,9 @@ public class CatalogTracker {
* @throws InterruptedException
*/
public void start() throws IOException, InterruptedException {
+ LOG.debug("Starting catalog tracker " + this);
this.rootRegionTracker.start();
this.metaNodeTracker.start();
- LOG.debug("Starting catalog tracker " + this);
}
/**
@@ -173,6 +238,9 @@ public class CatalogTracker {
// IOException}, in reality, the implementation would never do that.
LOG.error("Attempt to close catalog tracker's connection failed.", e);
}
+ if (this.instantiatedzkw) {
+ this.zookeeper.close();
+ }
// Call this and it will interrupt any ongoing waits on meta.
synchronized (this.metaAvailable) {
this.metaAvailable.notifyAll();
@@ -183,7 +251,8 @@ public class CatalogTracker {
/**
* Gets the current location for <code>-ROOT-</code> or null if location is
* not currently available.
- * @return server name
+ * @return {@link ServerName} for server hosting <code>-ROOT-</code> or null
+ * if none available
* @throws InterruptedException
*/
public ServerName getRootLocation() throws InterruptedException {
@@ -191,8 +260,8 @@ public class CatalogTracker {
}
/**
- * @return Location of server hosting meta region formatted as per
- * {@link ServerName}, or null if none available
+ * @return {@link ServerName} for server hosting <code>.META.</code> or null
+ * if none available
*/
public ServerName getMetaLocation() {
return this.metaLocation;
@@ -213,7 +282,8 @@ public class CatalogTracker {
* for up to the specified timeout if not immediately available. Returns null
* if the timeout elapses before root is available.
* @param timeout maximum time to wait for root availability, in milliseconds
- * @return Location of server hosting root region or null if none available
+ * @return {@link ServerName} for server hosting <code>-ROOT-</code> or null
+ * if none available
* @throws InterruptedException if interrupted while waiting
* @throws NotAllMetaRegionsOnlineException if root not available before
* timeout
@@ -230,14 +300,32 @@ public class CatalogTracker {
/**
* Gets a connection to the server hosting root, as reported by ZooKeeper,
* waiting up to the specified timeout for availability.
+ * @param timeout How long to wait on root location
* @see #waitForRoot(long) for additional information
* @return connection to server hosting root
* @throws InterruptedException
* @throws NotAllMetaRegionsOnlineException if timed out waiting
* @throws IOException
+ * @deprecated Use {@link #getRootServerConnection(long)}
*/
public HRegionInterface waitForRootServerConnection(long timeout)
throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
+ return getRootServerConnection(timeout);
+ }
+
+ /**
+ * Gets a connection to the server hosting root, as reported by ZooKeeper,
+ * waiting up to the specified timeout for availability.
+ * <p>WARNING: Does not retry. Use an {@link HTable} instead.
+ * @param timeout How long to wait on root location
+ * @see #waitForRoot(long) for additional information
+ * @return connection to server hosting root
+ * @throws InterruptedException
+ * @throws NotAllMetaRegionsOnlineException if timed out waiting
+ * @throws IOException
+ */
+ HRegionInterface getRootServerConnection(long timeout)
+ throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
return getCachedConnection(waitForRoot(timeout));
}
@@ -248,30 +336,18 @@ public class CatalogTracker {
* @return connection to server hosting root
* @throws NotAllMetaRegionsOnlineException if timed out waiting
* @throws IOException
+ * @deprecated Use {@link #getRootServerConnection(long)}
*/
public HRegionInterface waitForRootServerConnectionDefault()
throws NotAllMetaRegionsOnlineException, IOException {
try {
- return getCachedConnection(waitForRoot(defaultTimeout));
+ return getRootServerConnection(this.defaultTimeout);
} catch (InterruptedException e) {
throw new NotAllMetaRegionsOnlineException("Interrupted");
}
}
/**
- * Gets a connection to the server hosting root, as reported by ZooKeeper,
- * if available. Returns null if no location is immediately available.
- * @return connection to server hosting root, null if not available
- * @throws IOException
- * @throws InterruptedException
- */
- private HRegionInterface getRootServerConnection()
- throws IOException, InterruptedException {
- ServerName sn = this.rootRegionTracker.getRootRegionLocation();
- return getCachedConnection(sn);
- }
-
- /**
* Gets a connection to the server currently hosting <code>.META.</code> or
* null if location is not currently available.
* <p>
@@ -292,24 +368,28 @@ public class CatalogTracker {
synchronized (metaAvailable) {
if (metaAvailable.get()) {
HRegionInterface current = getCachedConnection(this.metaLocation);
- if (verifyRegionLocation(current, this.metaLocation, META_REGION)) {
+ // If we are to refresh, verify we have a good connection by making
+ // an invocation on it.
+ if (verifyRegionLocation(current, this.metaLocation, META_REGION_NAME)) {
return current;
}
resetMetaLocation();
}
- HRegionInterface rootConnection = getRootServerConnection();
- if (rootConnection == null) {
- LOG.debug("-ROOT- server unavailable.");
- return null;
- }
- ServerName newLocation = MetaReader.readMetaLocation(rootConnection);
+ // We got here because there is no meta available or because whats
+ // available is bad.
+
+ // Now read the current .META. content from -ROOT-. Note: This goes via
+ // an HConnection. It has its own way of figuring root and meta locations
+ // which we have to wait on.
+ ServerName newLocation =
+ MetaReader.readRegionLocation(this, META_REGION_NAME);
if (newLocation == null) {
LOG.debug(".META. server unavailable.");
return null;
}
HRegionInterface newConnection = getCachedConnection(newLocation);
- if (verifyRegionLocation(newConnection, newLocation, META_REGION)) {
+ if (verifyRegionLocation(newConnection, newLocation, META_REGION_NAME)) {
setMetaLocation(newLocation);
return newConnection;
} else {
@@ -322,13 +402,19 @@ public class CatalogTracker {
/**
* Waits indefinitely for availability of <code>.META.</code>. Used during
- * cluster startup.
+ * cluster startup. Does not verify meta, just that something has been
+ * set up in zk.
+ * @see #waitForMeta(long)
* @throws InterruptedException if interrupted while waiting
*/
public void waitForMeta() throws InterruptedException {
- synchronized (metaAvailable) {
- while (!stopped && !metaAvailable.get()) {
- metaAvailable.wait();
+ while (!this.stopped) {
+ try {
+ if (waitForMeta(100) != null) break;
+ } catch (NotAllMetaRegionsOnlineException e) {
+ LOG.info("Retrying", e);
+ } catch (IOException e) {
+ LOG.info("Retrying", e);
}
}
}
@@ -340,7 +426,8 @@ public class CatalogTracker {
* in that it will go ahead and verify the location gotten from ZooKeeper and
* -ROOT- region by trying to use returned connection.
* @param timeout maximum time to wait for meta availability, in milliseconds
- * @return location of meta
+ * @return {@link ServerName} for server hosting <code>.META.</code> or null
+ * if none available
* @throws InterruptedException if interrupted while waiting
* @throws IOException unexpected exception connecting to meta server
* @throws NotAllMetaRegionsOnlineException if meta not available before
@@ -359,8 +446,7 @@ public class CatalogTracker {
metaAvailable.wait(waitTime);
}
if (getMetaServerConnection() == null) {
- throw new NotAllMetaRegionsOnlineException(
- "Timed out (" + timeout + "ms)");
+ throw new NotAllMetaRegionsOnlineException("Timed out (" + timeout + "ms)");
}
return metaLocation;
}
@@ -374,6 +460,7 @@ public class CatalogTracker {
* @throws InterruptedException
* @throws NotAllMetaRegionsOnlineException if timed out waiting
* @throws IOException
+ * @deprecated Does not retry; use an HTable instance instead.
*/
public HRegionInterface waitForMetaServerConnection(long timeout)
throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
@@ -383,10 +470,12 @@ public class CatalogTracker {
/**
* Gets a connection to the server hosting meta, as reported by ZooKeeper,
* waiting up to the specified timeout for availability.
+ * Used in tests.
* @see #waitForMeta(long) for additional information
* @return connection to server hosting meta
* @throws NotAllMetaRegionsOnlineException if timed out or interrupted
* @throws IOException
+ * @deprecated Does not retry; use an HTable instance instead.
*/
public HRegionInterface waitForMetaServerConnectionDefault()
throws NotAllMetaRegionsOnlineException, IOException {
@@ -397,12 +486,22 @@ public class CatalogTracker {
}
}
- private void resetMetaLocation() {
- LOG.debug("Current cached META location: " + metaLocation +
- " is not valid, resetting");
- this.metaAvailable.set(false);
+ /**
+ * Called when we figure current meta is off (called from zk callback).
+ */
+ public void resetMetaLocation() {
+ LOG.debug("Current cached META location, " + metaLocation +
+ ", is not valid, resetting");
+ synchronized(this.metaAvailable) {
+ this.metaAvailable.set(false);
+ this.metaAvailable.notifyAll();
+ }
}
+ /**
+ * Caller must be synchronized on this.metaAvailable
+ * @param metaLocation
+ */
private void setMetaLocation(final ServerName metaLocation) {
LOG.debug("set new cached META location: " + metaLocation);
metaAvailable.set(true);
@@ -411,6 +510,13 @@ public class CatalogTracker {
this.metaAvailable.notifyAll();
}
+ /**
+ * @param sn ServerName to get a connection against.
+ * @return The HRegionInterface we got when we connected to <code>sn</code>
+ * May have come from cache, may not be good, may have been setup by this
+ * invocation, or may be null.
+ * @throws IOException
+ */
private HRegionInterface getCachedConnection(ServerName sn)
throws IOException {
if (sn == null) {
@@ -446,17 +552,32 @@ public class CatalogTracker {
return protocol;
}
- private boolean verifyRegionLocation(HRegionInterface metaServer,
- final ServerName address,
- byte [] regionName)
+ /**
+ * Verify we can connect to <code>hostingServer</code> and that its carrying
+ * <code>regionName</code>.
+ * @param hostingServer Interface to the server hosting <code>regionName</code>
+ * @param serverName The servername that goes with the <code>metaServer</code>
+ * Interface. Used logging.
+ * @param regionName The regionname we are interested in.
+ * @return True if we were able to verify the region located at other side of
+ * the Interface.
+ * @throws IOException
+ */
+ // TODO: We should be able to get the ServerName from the HRegionInterface
+ // rather than have to pass it in. Its made awkward by the fact that the
+ // HRI is likely a proxy against remote server so the getServerName needs
+ // to be fixed to go to a local method or to a cache before we can do this.
+ private boolean verifyRegionLocation(HRegionInterface hostingServer,
+ final ServerName address, final byte [] regionName)
throws IOException {
- if (metaServer == null) {
- LOG.info("Passed metaserver is null");
+ if (hostingServer == null) {
+ LOG.info("Passed hostingServer is null");
return false;
}
Throwable t = null;
try {
- return metaServer.getRegionInfo(regionName) != null;
+ // Try and get regioninfo from the hosting server.
+ return hostingServer.getRegionInfo(regionName) != null;
} catch (ConnectException e) {
t = e;
} catch (RemoteException e) {
@@ -498,8 +619,7 @@ public class CatalogTracker {
}
return (connection == null)? false:
verifyRegionLocation(connection,
- this.rootRegionTracker.getRootRegionLocation(),
- HRegionInfo.ROOT_REGIONINFO.getRegionName());
+ this.rootRegionTracker.getRootRegionLocation(), ROOT_REGION_NAME);
}
/**
@@ -523,6 +643,7 @@ public class CatalogTracker {
return connection != null;
}
+ // Used by tests.
MetaNodeTracker getMetaNodeTracker() {
return this.metaNodeTracker;
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java Thu Oct 13 23:12:30 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2010 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -19,40 +17,121 @@
*/
package org.apache.hadoop.hbase.catalog;
-import java.util.ArrayList;
-import java.util.List;
-
import java.io.IOException;
import java.net.ConnectException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.migration.HRegionInfo090x;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.catalog.MetaReader.Visitor;
/**
* Writes region and assignment information to <code>.META.</code>.
- * <p>
- * Uses the {@link CatalogTracker} to obtain locations and connections to
- * catalogs.
*/
public class MetaEditor {
+ // TODO: Strip CatalogTracker from this class. Its all over and in the end
+ // its only used to get its Configuration so we can get associated
+ // Connection.
private static final Log LOG = LogFactory.getLog(MetaEditor.class);
- private static Put makePutFromRegionInfo(HRegionInfo regionInfo) throws IOException {
+ private static Put makePutFromRegionInfo(HRegionInfo regionInfo)
+ throws IOException {
Put put = new Put(regionInfo.getRegionName());
put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
Writables.getBytes(regionInfo));
return put;
}
+
+ /**
+ * Put the passed <code>p</code> to the <code>.META.</code> table.
+ * @param ct CatalogTracker on whose back we will ride the edit.
+ * @param p Put to add to .META.
+ * @throws IOException
+ */
+ static void putToMetaTable(final CatalogTracker ct, final Put p)
+ throws IOException {
+ put(MetaReader.getMetaHTable(ct), p);
+ }
+
+ /**
+ * Put the passed <code>p</code> to the <code>.META.</code> table.
+ * @param ct CatalogTracker on whose back we will ride the edit.
+ * @param p Put to add to .META.
+ * @throws IOException
+ */
+ static void putToRootTable(final CatalogTracker ct, final Put p)
+ throws IOException {
+ put(MetaReader.getRootHTable(ct), p);
+ }
+
+ /**
+ * Put the passed <code>p</code> to a catalog table.
+ * @param ct CatalogTracker on whose back we will ride the edit.
+ * @param regionName Name of the catalog table to put too.
+ * @param p Put to add
+ * @throws IOException
+ */
+ static void putToCatalogTable(final CatalogTracker ct,
+ final byte [] regionName, final Put p)
+ throws IOException {
+ HTable t = MetaReader.getCatalogHTable(ct, regionName);
+ put(t, p);
+ }
+
+ /**
+ * @param t Table to use (will be closed when done).
+ * @param p
+ * @throws IOException
+ */
+ private static void put(final HTable t, final Put p) throws IOException {
+ try {
+ t.put(p);
+ } finally {
+ t.close();
+ }
+ }
+
+ /**
+ * Put the passed <code>ps</code> to the <code>.META.</code> table.
+ * @param ct CatalogTracker on whose back we will ride the edit.
+ * @param ps Put to add to .META.
+ * @throws IOException
+ */
+ static void putsToMetaTable(final CatalogTracker ct, final List<Put> ps)
+ throws IOException {
+ HTable t = MetaReader.getMetaHTable(ct);
+ try {
+ t.put(ps);
+ } finally {
+ t.close();
+ }
+ }
+
+ /**
+ * Delete the passed <code>d</code> from the <code>.META.</code> table.
+ * @param ct CatalogTracker on whose back we will ride the edit.
+ * @param d Delete to add to .META.
+ * @throws IOException
+ */
+ static void deleteMetaTable(final CatalogTracker ct, final Delete d)
+ throws IOException {
+ HTable t = MetaReader.getMetaHTable(ct);
+ try {
+ t.delete(d);
+ } finally {
+ t.close();
+ }
+ }
/**
* Adds a META row for the specified new region.
@@ -62,8 +141,7 @@ public class MetaEditor {
public static void addRegionToMeta(CatalogTracker catalogTracker,
HRegionInfo regionInfo)
throws IOException {
- catalogTracker.waitForMetaServerConnectionDefault().put(
- CatalogTracker.META_REGION, makePutFromRegionInfo(regionInfo));
+ putToMetaTable(catalogTracker, makePutFromRegionInfo(regionInfo));
LOG.info("Added region " + regionInfo.getRegionNameAsString() + " to META");
}
@@ -79,11 +157,9 @@ public class MetaEditor {
List<Put> puts = new ArrayList<Put>();
for (HRegionInfo regionInfo : regionInfos) {
puts.add(makePutFromRegionInfo(regionInfo));
- LOG.debug("Added region " + regionInfo.getRegionNameAsString() + " to META");
}
- catalogTracker.waitForMetaServerConnectionDefault().put(
- CatalogTracker.META_REGION, puts);
- LOG.info("Added " + puts.size() + " regions to META");
+ putsToMetaTable(catalogTracker, puts);
+ LOG.info("Added " + puts.size() + " regions in META");
}
/**
@@ -108,7 +184,7 @@ public class MetaEditor {
Writables.getBytes(a));
put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,
Writables.getBytes(b));
- catalogTracker.waitForMetaServerConnectionDefault().put(CatalogTracker.META_REGION, put);
+ putToMetaTable(catalogTracker, put);
LOG.info("Offlined parent region " + parent.getRegionNameAsString() +
" in META");
}
@@ -116,14 +192,11 @@ public class MetaEditor {
public static void addDaughter(final CatalogTracker catalogTracker,
final HRegionInfo regionInfo, final ServerName sn)
throws NotAllMetaRegionsOnlineException, IOException {
- HRegionInterface server = catalogTracker.waitForMetaServerConnectionDefault();
- byte [] catalogRegionName = CatalogTracker.META_REGION;
Put put = new Put(regionInfo.getRegionName());
addRegionInfo(put, regionInfo);
if (sn != null) addLocation(put, sn);
- server.put(catalogRegionName, put);
+ putToMetaTable(catalogTracker, put);
LOG.info("Added daughter " + regionInfo.getRegionNameAsString() +
- " in region " + Bytes.toString(catalogRegionName) +
(sn == null? ", serverName=null": ", serverName=" + sn.toString()));
}
@@ -145,9 +218,7 @@ public class MetaEditor {
public static void updateMetaLocation(CatalogTracker catalogTracker,
HRegionInfo regionInfo, ServerName sn)
throws IOException, ConnectException {
- HRegionInterface server = catalogTracker.waitForRootServerConnectionDefault();
- if (server == null) throw new IOException("No server for -ROOT-");
- updateLocation(server, CatalogTracker.ROOT_REGION, regionInfo, sn);
+ updateLocation(catalogTracker, regionInfo, sn);
}
/**
@@ -165,8 +236,7 @@ public class MetaEditor {
public static void updateRegionLocation(CatalogTracker catalogTracker,
HRegionInfo regionInfo, ServerName sn)
throws IOException {
- updateLocation(catalogTracker.waitForMetaServerConnectionDefault(),
- CatalogTracker.META_REGION, regionInfo, sn);
+ updateLocation(catalogTracker, regionInfo, sn);
}
/**
@@ -175,22 +245,21 @@ public class MetaEditor {
* Connects to the specified server which should be hosting the specified
* catalog region name to perform the edit.
*
- * @param server connection to server hosting catalog region
- * @param catalogRegionName name of catalog region being updated
+ * @param catalogTracker
* @param regionInfo region to update location of
* @param sn Server name
* @throws IOException In particular could throw {@link java.net.ConnectException}
* if the server is down on other end.
*/
- private static void updateLocation(HRegionInterface server,
- byte [] catalogRegionName, HRegionInfo regionInfo, ServerName sn)
+ private static void updateLocation(final CatalogTracker catalogTracker,
+ HRegionInfo regionInfo, ServerName sn)
throws IOException {
+ final byte [] regionName = regionInfo.getRegionName();
Put put = new Put(regionInfo.getRegionName());
addLocation(put, sn);
- server.put(catalogRegionName, put);
+ putToCatalogTable(catalogTracker, regionName, put);
LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
- " in region " + Bytes.toStringBinary(catalogRegionName) + " with " +
- "serverName=" + sn.toString());
+ " with server=" + sn);
}
/**
@@ -203,8 +272,7 @@ public class MetaEditor {
HRegionInfo regionInfo)
throws IOException {
Delete delete = new Delete(regionInfo.getRegionName());
- catalogTracker.waitForMetaServerConnectionDefault().
- delete(CatalogTracker.META_REGION, delete);
+ deleteMetaTable(catalogTracker, delete);
LOG.info("Deleted region " + regionInfo.getRegionNameAsString() + " from META");
}
@@ -223,142 +291,12 @@ public class MetaEditor {
throws NotAllMetaRegionsOnlineException, IOException {
Delete delete = new Delete(parent.getRegionName());
delete.deleteColumns(HConstants.CATALOG_FAMILY, qualifier);
- catalogTracker.waitForMetaServerConnectionDefault().
- delete(CatalogTracker.META_REGION, delete);
+ deleteMetaTable(catalogTracker, delete);
LOG.info("Deleted daughter reference " + daughter.getRegionNameAsString() +
", qualifier=" + Bytes.toStringBinary(qualifier) + ", from parent " +
parent.getRegionNameAsString());
}
- /**
- * Update the metamigrated flag in -ROOT-.
- * @param catalogTracker
- * @throws IOException
- */
- public static void updateRootWithMetaMigrationStatus(
- CatalogTracker catalogTracker) throws IOException {
- updateRootWithMetaMigrationStatus(catalogTracker, true);
- }
-
- /**
- * Update the metamigrated flag in -ROOT-.
- * @param catalogTracker
- * @param metaUpdated
- * @throws IOException
- */
- public static void updateRootWithMetaMigrationStatus(
- CatalogTracker catalogTracker, boolean metaUpdated)
- throws IOException {
- Put put = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName());
- addMetaUpdateStatus(put, metaUpdated);
- catalogTracker.waitForRootServerConnectionDefault().put(
- CatalogTracker.ROOT_REGION, put);
- LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated);
- }
-
- /**
- * Update legacy META rows, removing HTD from HRI.
- * @param masterServices
- * @return
- * @throws IOException
- */
- public static List<HTableDescriptor> updateMetaWithNewRegionInfo(
- final MasterServices masterServices)
- throws IOException {
- final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
- Visitor v = new Visitor() {
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
- if (hrfm == null) return true;
- htds.add(hrfm.getTableDesc());
- masterServices.getMasterFileSystem()
- .createTableDescriptor(hrfm.getTableDesc());
- updateHRI(masterServices.getCatalogTracker()
- .waitForMetaServerConnectionDefault(),
- hrfm, CatalogTracker.META_REGION);
- return true;
- }
- };
- MetaReader.fullScan(masterServices.getCatalogTracker(), v);
- updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker());
- return htds;
- }
-
- /**
- * Migrate root and meta to newer version. This updates the META and ROOT
- * and removes the HTD from HRI.
- * @param masterServices
- * @throws IOException
- */
- public static void migrateRootAndMeta(final MasterServices masterServices)
- throws IOException {
- updateRootWithNewRegionInfo(masterServices);
- updateMetaWithNewRegionInfo(masterServices);
- }
-
- /**
- * Update the ROOT with new HRI. (HRI with no HTD)
- * @param masterServices
- * @return
- * @throws IOException
- */
- public static List<HTableDescriptor> updateRootWithNewRegionInfo(
- final MasterServices masterServices)
- throws IOException {
- final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
- Visitor v = new Visitor() {
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
- if (hrfm == null) return true;
- htds.add(hrfm.getTableDesc());
- masterServices.getMasterFileSystem().createTableDescriptor(
- hrfm.getTableDesc());
- updateHRI(masterServices.getCatalogTracker()
- .waitForRootServerConnectionDefault(),
- hrfm, CatalogTracker.ROOT_REGION);
- return true;
- }
- };
- MetaReader.fullScan(
- masterServices.getCatalogTracker().waitForRootServerConnectionDefault(),
- v, HRegionInfo.ROOT_REGIONINFO.getRegionName(), null);
- return htds;
- }
-
- private static void updateHRI(HRegionInterface hRegionInterface,
- HRegionInfo090x hRegionInfo090x, byte[] regionName)
- throws IOException {
- HRegionInfo regionInfo = new HRegionInfo(hRegionInfo090x);
- Put put = new Put(regionInfo.getRegionName());
- put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
- Writables.getBytes(regionInfo));
- hRegionInterface.put(regionName, put);
- LOG.info("Updated region " + regionInfo + " to " + Bytes.toString(regionName));
- }
-
- public static HRegionInfo090x getHRegionInfoForMigration(
- Result data) throws IOException {
- HRegionInfo090x info = null;
- byte [] bytes =
- data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
- if (bytes == null) return null;
- try {
- info = Writables.getHRegionInfoForMigration(bytes);
- } catch(IOException ioe) {
- if (ioe.getMessage().equalsIgnoreCase("HTD not found in input buffer")) {
- return null;
- } else {
- throw ioe;
- }
- }
- LOG.info("Current INFO from scan results = " + info);
- return info;
- }
-
public static HRegionInfo getHRegionInfo(
Result data) throws IOException {
byte [] bytes =
@@ -369,20 +307,6 @@ public class MetaEditor {
return info;
}
- private static Put addMetaUpdateStatus(final Put p) {
- p.add(HConstants.CATALOG_FAMILY, HConstants.META_MIGRATION_QUALIFIER,
- Bytes.toBytes("true"));
- return p;
- }
-
-
- private static Put addMetaUpdateStatus(final Put p, final boolean metaUpdated) {
- p.add(HConstants.CATALOG_FAMILY, HConstants.META_MIGRATION_QUALIFIER,
- Bytes.toBytes(metaUpdated));
- return p;
- }
-
-
private static Put addRegionInfo(final Put p, final HRegionInfo hri)
throws IOException {
p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
@@ -397,4 +321,4 @@ public class MetaEditor {
Bytes.toBytes(sn.getStartcode()));
return p;
}
-}
+}
\ No newline at end of file
Added: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java?rev=1183158&view=auto
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java (added)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java Thu Oct 13 23:12:30 2011
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.catalog;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.catalog.MetaReader.Visitor;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.migration.HRegionInfo090x;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+
+/**
+ * Tools to help with migration of meta tables so they no longer host
+ * instances of HTableDescriptor.
+ * @deprecated Used migration from 0.90 to 0.92 so will be going away in next
+ * release
+ */
+public class MetaMigrationRemovingHTD {
+ private static final Log LOG = LogFactory.getLog(MetaMigrationRemovingHTD.class);
+
+ /** The metaupdated column qualifier */
+ public static final byte [] META_MIGRATION_QUALIFIER =
+ Bytes.toBytes("metamigrated");
+
+ /**
+ * Update legacy META rows, removing HTD from HRI.
+ * @param masterServices
+ * @return
+ * @throws IOException
+ */
+ public static List<HTableDescriptor> updateMetaWithNewRegionInfo(
+ final MasterServices masterServices)
+ throws IOException {
+ final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
+ Visitor v = new Visitor() {
+ @Override
+ public boolean visit(Result r) throws IOException {
+ if (r == null || r.isEmpty()) return true;
+ HRegionInfo090x hrfm = MetaMigrationRemovingHTD.getHRegionInfoForMigration(r);
+ if (hrfm == null) return true;
+ htds.add(hrfm.getTableDesc());
+ masterServices.getMasterFileSystem()
+ .createTableDescriptor(hrfm.getTableDesc());
+ updateHRI(masterServices.getCatalogTracker(), false, hrfm);
+ return true;
+ }
+ };
+ MetaReader.fullScan(masterServices.getCatalogTracker(), v);
+ MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker(), true);
+ return htds;
+ }
+
+ /**
+ * Update the ROOT with new HRI. (HRI with no HTD)
+ * @param masterServices
+ * @return
+ * @throws IOException
+ */
+ public static List<HTableDescriptor> updateRootWithNewRegionInfo(
+ final MasterServices masterServices)
+ throws IOException {
+ final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
+ Visitor v = new Visitor() {
+ @Override
+ public boolean visit(Result r) throws IOException {
+ if (r == null || r.isEmpty()) return true;
+ HRegionInfo090x hrfm = MetaMigrationRemovingHTD.getHRegionInfoForMigration(r);
+ if (hrfm == null) return true;
+ htds.add(hrfm.getTableDesc());
+ masterServices.getMasterFileSystem().createTableDescriptor(
+ hrfm.getTableDesc());
+ updateHRI(masterServices.getCatalogTracker(), true, hrfm);
+ return true;
+ }
+ };
+ MetaReader.fullScan(masterServices.getCatalogTracker(), v, null, true);
+ return htds;
+ }
+
+ /**
+ * Migrate root and meta to newer version. This updates the META and ROOT
+ * and removes the HTD from HRI.
+ * @param masterServices
+ * @throws IOException
+ */
+ public static void migrateRootAndMeta(final MasterServices masterServices)
+ throws IOException {
+ updateRootWithNewRegionInfo(masterServices);
+ updateMetaWithNewRegionInfo(masterServices);
+ }
+
+ /**
+ * Update the metamigrated flag in -ROOT-.
+ * @param catalogTracker
+ * @param metaUpdated
+ * @throws IOException
+ */
+ public static void updateRootWithMetaMigrationStatus(
+ CatalogTracker catalogTracker, boolean metaUpdated)
+ throws IOException {
+ Put p = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName());
+ MetaMigrationRemovingHTD.addMetaUpdateStatus(p, metaUpdated);
+ MetaEditor.putToRootTable(catalogTracker, p);
+ LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated);
+ }
+
+ static void updateHRI(final CatalogTracker ct, final boolean rootTable,
+ final HRegionInfo090x hRegionInfo090x)
+ throws IOException {
+ HRegionInfo regionInfo = new HRegionInfo(hRegionInfo090x);
+ Put p = new Put(regionInfo.getRegionName());
+ p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+ Writables.getBytes(regionInfo));
+ if (rootTable) {
+ MetaEditor.putToRootTable(ct, p);
+ } else {
+ MetaEditor.putToMetaTable(ct, p);
+ }
+ LOG.info("Updated region " + regionInfo + " to " +
+ (rootTable? "-ROOT-": ".META."));
+ }
+
+ /**
+ * @deprecated Going away in 0.94; used for migrating to 0.92 only.
+ */
+ public static HRegionInfo090x getHRegionInfoForMigration(
+ Result data) throws IOException {
+ HRegionInfo090x info = null;
+ byte [] bytes =
+ data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+ if (bytes == null) return null;
+ try {
+ info = Writables.getHRegionInfoForMigration(bytes);
+ } catch(IOException ioe) {
+ if (ioe.getMessage().equalsIgnoreCase("HTD not found in input buffer")) {
+ return null;
+ } else {
+ throw ioe;
+ }
+ }
+ LOG.info("Current INFO from scan results = " + info);
+ return info;
+ }
+
+ public static List<HRegionInfo090x> fullScanMetaAndPrintHRIM(
+ CatalogTracker catalogTracker)
+ throws IOException {
+ final List<HRegionInfo090x> regions =
+ new ArrayList<HRegionInfo090x>();
+ Visitor v = new Visitor() {
+ @Override
+ public boolean visit(Result r) throws IOException {
+ if (r == null || r.isEmpty()) return true;
+ LOG.info("fullScanMetaAndPrint1.Current Meta Result: " + r);
+ HRegionInfo090x hrim = getHRegionInfoForMigration(r);
+ LOG.info("fullScanMetaAndPrint.HRIM Print= " + hrim);
+ regions.add(hrim);
+ return true;
+ }
+ };
+ MetaReader.fullScan(catalogTracker, v);
+ return regions;
+ }
+
+ static Put addMetaUpdateStatus(final Put p, final boolean metaUpdated) {
+ p.add(HConstants.CATALOG_FAMILY,
+ MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER,
+ Bytes.toBytes(metaUpdated));
+ return p;
+ }
+
+ /**
+ * @return True if the meta table has been migrated.
+ * @throws IOException
+ */
+ // Public because used in tests
+ public static boolean isMetaHRIUpdated(final MasterServices services)
+ throws IOException {
+ boolean metaUpdated = false;
+ List<Result> results =
+ MetaReader.fullScanOfRoot(services.getCatalogTracker());
+ if (results == null || results.isEmpty()) {
+ LOG.info("metaUpdated = NULL.");
+ return metaUpdated;
+ }
+ // Presume only the one result.
+ Result r = results.get(0);
+ byte [] metaMigrated = r.getValue(HConstants.CATALOG_FAMILY,
+ MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER);
+ if (metaMigrated != null && metaMigrated.length > 0) {
+ metaUpdated = Bytes.toBoolean(metaMigrated);
+ }
+ LOG.info("Meta updated status = " + metaUpdated);
+ return metaUpdated;
+ }
+
+ /**
+ * @return True if migrated.
+ * @throws IOException
+ */
+ public static boolean updateMetaWithNewHRI(final MasterServices services)
+ throws IOException {
+ if (isMetaHRIUpdated(services)) {
+ LOG.info("ROOT/Meta already up-to date with new HRI.");
+ return true;
+ }
+ LOG.info("Meta has HRI with HTDs. Updating meta now.");
+ try {
+ migrateRootAndMeta(services);
+ LOG.info("ROOT and Meta updated with new HRI.");
+ return true;
+ } catch (IOException e) {
+ throw new RuntimeException("Update ROOT/Meta with new HRI failed." +
+ "Master startup aborted.");
+ }
+ }
+}
\ No newline at end of file
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java Thu Oct 13 23:12:30 2011
@@ -1,6 +1,4 @@
/**
- * Copyright 2010 The Apache Software Foundation
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -26,16 +24,20 @@ import java.util.Map;
import java.util.NavigableMap;
import java.util.Set;
import java.util.TreeMap;
-import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.migration.HRegionInfo090x;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Writables;
@@ -43,14 +45,14 @@ import org.apache.hadoop.ipc.RemoteExcep
/**
* Reads region and assignment information from <code>.META.</code>.
- * <p>
- * Uses the {@link CatalogTracker} to obtain locations and connections to
- * catalogs.
*/
public class MetaReader {
+ // TODO: Strip CatalogTracker from this class. Its all over and in the end
+ // its only used to get its Configuration so we can get associated
+ // Connection.
private static final Log LOG = LogFactory.getLog(MetaReader.class);
- public static final byte [] META_REGION_PREFIX;
+ static final byte [] META_REGION_PREFIX;
static {
// Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX.
// FIRST_META_REGIONINFO == '.META.,,1'. META_REGION_PREFIX == '.META.,'
@@ -61,48 +63,6 @@ public class MetaReader {
}
/**
- * @param ct
- * @param tableName A user tablename or a .META. table name.
- * @return Interface on to server hosting the <code>-ROOT-</code> or
- * <code>.META.</code> regions.
- * @throws NotAllMetaRegionsOnlineException
- * @throws IOException
- */
- private static HRegionInterface getCatalogRegionInterface(final CatalogTracker ct,
- final byte [] tableName)
- throws NotAllMetaRegionsOnlineException, IOException {
- return Bytes.equals(HConstants.META_TABLE_NAME, tableName)?
- ct.waitForRootServerConnectionDefault():
- ct.waitForMetaServerConnectionDefault();
- }
-
- /**
- * @param tableName
- * @return Returns region name to look in for regions for <code>tableName</code>;
- * e.g. if we are looking for <code>.META.</code> regions, we need to look
- * in the <code>-ROOT-</code> region, else if a user table, we need to look
- * in the <code>.META.</code> region.
- */
- private static byte [] getCatalogRegionNameForTable(final byte [] tableName) {
- return Bytes.equals(HConstants.META_TABLE_NAME, tableName)?
- HRegionInfo.ROOT_REGIONINFO.getRegionName():
- HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
- }
-
- /**
- * @param regionName
- * @return Returns region name to look in for <code>regionName</code>;
- * e.g. if we are looking for <code>.META.,,1</code> region, we need to look
- * in <code>-ROOT-</code> region, else if a user region, we need to look
- * in the <code>.META.,,1</code> region.
- */
- private static byte [] getCatalogRegionNameForRegion(final byte [] regionName) {
- return isMetaRegion(regionName)?
- HRegionInfo.ROOT_REGIONINFO.getRegionName():
- HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
- }
-
- /**
* @param regionName
* @return True if <code>regionName</code> is from <code>.META.</code> table.
*/
@@ -118,33 +78,13 @@ public class MetaReader {
}
/**
- * Performs a full scan of <code>.META.</code>.
- * <p>
- * Returns a map of every region to it's currently assigned server, according
- * to META. If the region does not have an assignment it will have a null
- * value in the map.
- *
- * @return map of regions to their currently assigned server where server is
- * a String of <host> ':' <port>
- * @throws IOException
- */
- public static Map<HRegionInfo, ServerName> fullScan(
- CatalogTracker catalogTracker)
- throws IOException {
- return fullScan(catalogTracker, new TreeSet<String>());
- }
-
- /**
* Performs a full scan of <code>.META.</code>, skipping regions from any
* tables in the specified set of disabled tables.
- * <p>
- * Returns a map of every region to it's currently assigned server, according
- * to META. If the region does not have an assignment it will have a null
- * value in the map.
- *
* @param catalogTracker
* @param disabledTables set of disabled tables that will not be returned
- * @return map of regions to their currently assigned server
+ * @return Returns a map of every region to it's currently assigned server,
+ * according to META. If the region does not have an assignment it will have
+ * a null value in the map.
* @throws IOException
*/
public static Map<HRegionInfo, ServerName> fullScan(
@@ -156,16 +96,13 @@ public class MetaReader {
/**
* Performs a full scan of <code>.META.</code>, skipping regions from any
* tables in the specified set of disabled tables.
- * <p>
- * Returns a map of every region to it's currently assigned server, according
- * to META. If the region does not have an assignment it will have a null
- * value in the map.
- *
* @param catalogTracker
* @param disabledTables set of disabled tables that will not be returned
* @param excludeOfflinedSplitParents If true, do not include offlined split
* parents in the return.
- * @return map of regions to their currently assigned server
+ * @return Returns a map of every region to it's currently assigned server,
+ * according to META. If the region does not have an assignment it will have
+ * a null value in the map.
* @throws IOException
*/
public static Map<HRegionInfo, ServerName> fullScan(
@@ -178,7 +115,7 @@ public class MetaReader {
@Override
public boolean visit(Result r) throws IOException {
if (r == null || r.isEmpty()) return true;
- Pair<HRegionInfo, ServerName> region = metaRowToRegionPair(r);
+ Pair<HRegionInfo, ServerName> region = parseCatalogResult(r);
if (region == null) return true;
HRegionInfo hri = region.getFirst();
if (disabledTables.contains(
@@ -195,38 +132,32 @@ public class MetaReader {
/**
* Performs a full scan of <code>.META.</code>.
- * <p>
- * Returns a map of every region to it's currently assigned server, according
- * to META. If the region does not have an assignment it will have a null
- * value in the map.
- *
- * @return map of regions to their currently assigned server
+ * @return List of {@link Result}
* @throws IOException
*/
- public static List<Result> fullScanOfResults(
- CatalogTracker catalogTracker)
+ public static List<Result> fullScan(CatalogTracker catalogTracker)
throws IOException {
- final List<Result> regions = new ArrayList<Result>();
- Visitor v = new Visitor() {
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- regions.add(r);
- return true;
- }
- };
- fullScan(catalogTracker, v);
- return regions;
+ CollectAllVisitor v = new CollectAllVisitor();
+ fullScan(catalogTracker, v, null);
+ return v.getResults();
+ }
+
+ /**
+ * Performs a full scan of a <code>-ROOT-</code> table.
+ * @return List of {@link Result}
+ * @throws IOException
+ */
+ public static List<Result> fullScanOfRoot(CatalogTracker catalogTracker)
+ throws IOException {
+ CollectAllVisitor v = new CollectAllVisitor();
+ fullScan(catalogTracker, v, null, true);
+ return v.getResults();
}
/**
* Performs a full scan of <code>.META.</code>.
- * <p>
- * Returns a map of every region to it's currently assigned server, according
- * to META. If the region does not have an assignment it will have a null
- * value in the map.
* @param catalogTracker
- * @param visitor
+ * @param visitor Visitor invoked against each row.
* @throws IOException
*/
public static void fullScan(CatalogTracker catalogTracker,
@@ -237,24 +168,82 @@ public class MetaReader {
/**
* Performs a full scan of <code>.META.</code>.
- * <p>
- * Returns a map of every region to it's currently assigned server, according
- * to META. If the region does not have an assignment it will have a null
- * value in the map.
* @param catalogTracker
- * @param visitor
+ * @param visitor Visitor invoked against each row.
* @param startrow Where to start the scan. Pass null if want to begin scan
- * at first row.
+ * at first row (The visitor will stop the Scan when its done so no need to
+ * pass a stoprow).
* @throws IOException
*/
public static void fullScan(CatalogTracker catalogTracker,
final Visitor visitor, final byte [] startrow)
throws IOException {
- HRegionInterface metaServer =
- catalogTracker.waitForMetaServerConnectionDefault();
- fullScan(metaServer, visitor,
- HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), startrow);
- return;
+ fullScan(catalogTracker, visitor, startrow, false);
+ }
+
+ /**
+ * Callers should call close on the returned {@link HTable} instance.
+ * @param catalogTracker We'll use this catalogtracker's connection
+ * @param tableName Table to get an {@link HTable} against.
+ * @return An {@link HTable} for <code>tableName</code>
+ * @throws IOException
+ */
+ private static HTable getHTable(final CatalogTracker catalogTracker,
+ final byte [] tableName)
+ throws IOException {
+ // Passing the CatalogTracker's connection configuration ensures this
+ // HTable instance uses the CatalogTracker's connection.
+ return new HTable(catalogTracker.getConnection().getConfiguration(), tableName);
+ }
+
+ /**
+ * Callers should call close on the returned {@link HTable} instance.
+ * @param catalogTracker
+ * @param regionName
+ * @return
+ * @throws IOException
+ */
+ static HTable getCatalogHTable(final CatalogTracker catalogTracker,
+ final byte [] regionName)
+ throws IOException {
+ return isMetaRegion(regionName)?
+ getRootHTable(catalogTracker):
+ getMetaHTable(catalogTracker);
+ }
+
+ /**
+ * Callers should call close on the returned {@link HTable} instance.
+ * @param ct
+ * @return An {@link HTable} for <code>.META.</code>
+ * @throws IOException
+ */
+ static HTable getMetaHTable(final CatalogTracker ct)
+ throws IOException {
+ return getHTable(ct, HConstants.META_TABLE_NAME);
+ }
+
+ /**
+ * Callers should call close on the returned {@link HTable} instance.
+ * @param ct
+ * @return An {@link HTable} for <code>-ROOT-</code>
+ * @throws IOException
+ */
+ static HTable getRootHTable(final CatalogTracker ct)
+ throws IOException {
+ return getHTable(ct, HConstants.ROOT_TABLE_NAME);
+ }
+
+ /**
+ * @param t Table to use (will be closed when done).
+ * @param g Get to run
+ * @throws IOException
+ */
+ private static Result get(final HTable t, final Get g) throws IOException {
+ try {
+ return t.get(g);
+ } finally {
+ t.close();
+ }
}
/**
@@ -262,28 +251,29 @@ public class MetaReader {
* @param metaServer connection to server hosting ROOT
* @return location of META in ROOT where location, or null if not available
* @throws IOException
+ * @deprecated Does not retry; use {@link #readRegionLocation(CatalogTracker, byte[])
*/
public static ServerName readMetaLocation(HRegionInterface metaServer)
throws IOException {
- return readLocation(metaServer, CatalogTracker.ROOT_REGION,
- CatalogTracker.META_REGION);
+ return readLocation(metaServer, CatalogTracker.ROOT_REGION_NAME,
+ CatalogTracker.META_REGION_NAME);
}
/**
- * Reads the location of the specified region from META.
+ * Reads the location of the specified region
* @param catalogTracker
- * @param regionName region to read location of
- * @return location of META in ROOT where location is, or null if not available
+ * @param regionName region whose location we are after
+ * @return location of region as a {@link ServerName} or null if not found
* @throws IOException
*/
public static ServerName readRegionLocation(CatalogTracker catalogTracker,
byte [] regionName)
throws IOException {
- if (isMetaRegion(regionName)) throw new IllegalArgumentException("See readMetaLocation");
- return readLocation(catalogTracker.waitForMetaServerConnectionDefault(),
- CatalogTracker.META_REGION, regionName);
+ Pair<HRegionInfo, ServerName> pair = getRegion(catalogTracker, regionName);
+ return (pair == null || pair.getSecond() == null)? null: pair.getSecond();
}
+ // TODO: Remove when deprecated dependencies are removed.
private static ServerName readLocation(HRegionInterface metaServer,
byte [] catalogRegionName, byte [] regionName)
throws IOException {
@@ -324,15 +314,14 @@ public class MetaReader {
if (r == null || r.isEmpty()) {
return null;
}
- return getServerNameFromResult(r);
+ return getServerNameFromCatalogResult(r);
}
/**
- * Gets the region info and assignment for the specified region from META.
+ * Gets the region info and assignment for the specified region.
* @param catalogTracker
- * @param regionName
- * @return location of META in ROOT where location is
- * a String of <host> ':' <port>, or null if not available
+ * @param regionName Region to lookup.
+ * @return Location and HRegionInfo for <code>regionName</code>
* @throws IOException
*/
public static Pair<HRegionInfo, ServerName> getRegion(
@@ -340,44 +329,63 @@ public class MetaReader {
throws IOException {
Get get = new Get(regionName);
get.addFamily(HConstants.CATALOG_FAMILY);
- byte [] meta = getCatalogRegionNameForRegion(regionName);
- Result r = catalogTracker.waitForMetaServerConnectionDefault().get(meta, get);
- return (r == null || r.isEmpty())? null: metaRowToRegionPair(r);
+ Result r = get(getCatalogHTable(catalogTracker, regionName), get);
+ return (r == null || r.isEmpty())? null: parseCatalogResult(r);
}
/**
- * @param data A .META. table row.
- * @return A pair of the regioninfo and the ServerName
- * (or null for server address if no address set in .META.).
- * @throws IOException
- */
- public static Pair<HRegionInfo, ServerName> metaRowToRegionPair(Result data)
- throws IOException {
- byte [] bytes = data.getValue(HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER);
- if (bytes == null) return null;
- HRegionInfo info = Writables.getHRegionInfo(bytes);
- ServerName sn = getServerNameFromResult(data);
- // sn can be null in case where no server inof.
- return new Pair<HRegionInfo, ServerName>(info, sn);
- }
-
- /**
- * @param data Result to interrogate.
+ * Extract a {@link ServerName}
+ * For use on catalog table {@link Result}.
+ * @param r Result to pull from
* @return A ServerName instance or null if necessary fields not found or empty.
*/
- private static ServerName getServerNameFromResult(final Result data) {
- byte[] value = data.getValue(HConstants.CATALOG_FAMILY,
+ public static ServerName getServerNameFromCatalogResult(final Result r) {
+ byte[] value = r.getValue(HConstants.CATALOG_FAMILY,
HConstants.SERVER_QUALIFIER);
if (value == null || value.length == 0) return null;
String hostAndPort = Bytes.toString(value);
- value = data.getValue(HConstants.CATALOG_FAMILY,
+ value = r.getValue(HConstants.CATALOG_FAMILY,
HConstants.STARTCODE_QUALIFIER);
if (value == null || value.length == 0) return null;
return new ServerName(hostAndPort, Bytes.toLong(value));
}
/**
+ * Extract a HRegionInfo and ServerName.
+ * For use on catalog table {@link Result}.
+ * @param r Result to pull from
+ * @return A pair of the {@link HRegionInfo} and the {@link ServerName}
+ * (or null for server address if no address set in .META.).
+ * @throws IOException
+ */
+ public static Pair<HRegionInfo, ServerName> parseCatalogResult(final Result r)
+ throws IOException {
+ HRegionInfo info =
+ parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
+ ServerName sn = getServerNameFromCatalogResult(r);
+ return new Pair<HRegionInfo, ServerName>(info, sn);
+ }
+
+ /**
+ * Parse the content of the cell at {@link HConstants#CATALOG_FAMILY} and
+ * <code>qualifier</code> as an HRegionInfo and return it, or null.
+ * For use on catalog table {@link Result}.
+ * @param r Result instance to pull from.
+ * @param qualifier Column family qualifier -- either
+ * {@link HConstants#SPLITA_QUALIFIER}, {@link HConstants#SPLITB_QUALIFIER} or
+ * {@link HConstants#REGIONINFO_QUALIFIER}.
+ * @return An HRegionInfo instance or null.
+ * @throws IOException
+ */
+ public static HRegionInfo parseHRegionInfoFromCatalogResult(final Result r,
+ byte [] qualifier)
+ throws IOException {
+ byte [] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
+ if (bytes == null || bytes.length <= 0) return null;
+ return Writables.getHRegionInfoOrNull(bytes);
+ }
+
+ /**
* Checks if the specified table exists. Looks at the META table hosted on
* the specified server.
* @param catalogTracker
@@ -393,21 +401,36 @@ public class MetaReader {
// Catalog tables always exist.
return true;
}
- HRegionInterface metaServer =
- catalogTracker.waitForMetaServerConnectionDefault();
- Scan scan = getScanForTableName(Bytes.toBytes(tableName));
- scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
- long scannerid = metaServer.openScanner(
- HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
- try {
- Result data = metaServer.next(scannerid);
- if (data != null && data.size() > 0) {
+ final byte [] tableNameBytes = Bytes.toBytes(tableName);
+ // Make a version of ResultCollectingVisitor that only collects the first
+ CollectingVisitor<HRegionInfo> visitor = new CollectingVisitor<HRegionInfo>() {
+ private HRegionInfo current = null;
+
+ @Override
+ public boolean visit(Result r) throws IOException {
+ this.current =
+ parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
+ if (this.current == null) {
+ LOG.warn("No serialized HRegionInfo in " + r);
return true;
+ }
+ if (!isInsideTable(this.current, tableNameBytes)) return false;
+ if (this.current.isSplitParent()) return true;
+ // Else call super and add this Result to the collection.
+ super.visit(r);
+ // Stop collecting regions from table after we get one.
+ return false;
}
- return false;
- } finally {
- metaServer.close(scannerid);
- }
+
+ @Override
+ void add(Result r) {
+ // Add the current HRI.
+ this.results.add(this.current);
+ }
+ };
+ fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableNameBytes));
+ // If visitor has results >= 1 then table exists.
+ return visitor.getResults().size() >= 1;
}
/**
@@ -435,42 +458,46 @@ public class MetaReader {
public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
byte [] tableName, final boolean excludeOfflinedSplitParents)
throws IOException {
- if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
- // If root, do a bit of special handling.
- List<HRegionInfo> list = new ArrayList<HRegionInfo>();
- list.add(HRegionInfo.ROOT_REGIONINFO);
- return list;
- } else if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
- // Same for .META. table
- List<HRegionInfo> list = new ArrayList<HRegionInfo>();
- list.add(HRegionInfo.FIRST_META_REGIONINFO);
- return list;
+ List<Pair<HRegionInfo, ServerName>> result = null;
+ try {
+ result = getTableRegionsAndLocations(catalogTracker, tableName,
+ excludeOfflinedSplitParents);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
}
+ return getListOfHRegionInfos(result);
+ }
- // Its a user table.
- HRegionInterface metaServer =
- getCatalogRegionInterface(catalogTracker, tableName);
- List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
-
- Scan scan = getScanForTableName(tableName);
- scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
- long scannerid =
- metaServer.openScanner(getCatalogRegionNameForTable(tableName), scan);
- try {
- Result data;
- while((data = metaServer.next(scannerid)) != null) {
- if (data != null && data.size() > 0) {
- HRegionInfo info = Writables.getHRegionInfo(
- data.getValue(HConstants.CATALOG_FAMILY,
- HConstants.REGIONINFO_QUALIFIER));
- if (excludeOfflinedSplitParents && info.isSplitParent()) continue;
- regions.add(info);
- }
- }
- return regions;
- } finally {
- metaServer.close(scannerid);
+ static List<HRegionInfo> getListOfHRegionInfos(final List<Pair<HRegionInfo, ServerName>> pairs) {
+ if (pairs == null || pairs.isEmpty()) return null;
+ List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
+ for (Pair<HRegionInfo, ServerName> pair: pairs) {
+ result.add(pair.getFirst());
}
+ return result;
+ }
+
+ /**
+ * @param current
+ * @param tableName
+ * @return True if <code>current</code> tablename is equal to
+ * <code>tableName</code>
+ */
+ static boolean isInsideTable(final HRegionInfo current, final byte [] tableName) {
+ return Bytes.equals(tableName, current.getTableName());
+ }
+
+ /**
+ * @param tableName
+ * @return Place to start Scan in <code>.META.</code> when passed a
+ * <code>tableName</code>; returns <tableName&rt; <,&rt; <,&rt;
+ */
+ static byte [] getTableStartRowForMeta(final byte [] tableName) {
+ byte [] startRow = new byte[tableName.length + 2];
+ System.arraycopy(tableName, 0, startRow, 0, tableName.length);
+ startRow[startRow.length - 2] = HRegionInfo.DELIMITER;
+ startRow[startRow.length - 1] = HRegionInfo.DELIMITER;
+ return startRow;
}
/**
@@ -504,8 +531,22 @@ public class MetaReader {
public static List<Pair<HRegionInfo, ServerName>>
getTableRegionsAndLocations(CatalogTracker catalogTracker, String tableName)
throws IOException, InterruptedException {
- byte [] tableNameBytes = Bytes.toBytes(tableName);
- if (Bytes.equals(tableNameBytes, HConstants.ROOT_TABLE_NAME)) {
+ return getTableRegionsAndLocations(catalogTracker, Bytes.toBytes(tableName),
+ true);
+ }
+
+ /**
+ * @param catalogTracker
+ * @param tableName
+ * @return Return list of regioninfos and server addresses.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public static List<Pair<HRegionInfo, ServerName>>
+ getTableRegionsAndLocations(final CatalogTracker catalogTracker,
+ final byte [] tableName, final boolean excludeOfflinedSplitParents)
+ throws IOException, InterruptedException {
+ if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
// If root, do a bit of special handling.
ServerName serverName = catalogTracker.getRootLocation();
List<Pair<HRegionInfo, ServerName>> list =
@@ -514,27 +555,36 @@ public class MetaReader {
serverName));
return list;
}
- HRegionInterface metaServer =
- getCatalogRegionInterface(catalogTracker, tableNameBytes);
- List<Pair<HRegionInfo, ServerName>> regions =
- new ArrayList<Pair<HRegionInfo, ServerName>>();
- Scan scan = getScanForTableName(tableNameBytes);
- scan.addFamily(HConstants.CATALOG_FAMILY);
- long scannerid =
- metaServer.openScanner(getCatalogRegionNameForTable(tableNameBytes), scan);
- try {
- Result data;
- while((data = metaServer.next(scannerid)) != null) {
- if (data != null && data.size() > 0) {
- Pair<HRegionInfo, ServerName> region = metaRowToRegionPair(data);
- if (region == null) continue;
- regions.add(region);
+ // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress
+ CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
+ new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
+ private Pair<HRegionInfo, ServerName> current = null;
+
+ @Override
+ public boolean visit(Result r) throws IOException {
+ HRegionInfo hri =
+ parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
+ if (hri == null) {
+ LOG.warn("No serialized HRegionInfo in " + r);
+ return true;
}
+ if (!isInsideTable(hri, tableName)) return false;
+ if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
+ ServerName sn = getServerNameFromCatalogResult(r);
+ // Populate this.current so available when we call #add
+ this.current = new Pair<HRegionInfo, ServerName>(hri, sn);
+ // Else call super and add this Result to the collection.
+ return super.visit(r);
}
- return regions;
- } finally {
- metaServer.close(scannerid);
- }
+
+ @Override
+ void add(Result r) {
+ this.results.add(this.current);
+ }
+ };
+ fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName),
+ Bytes.equals(tableName, HConstants.META_TABLE_NAME));
+ return visitor.getResults();
}
/**
@@ -547,36 +597,31 @@ public class MetaReader {
public static NavigableMap<HRegionInfo, Result>
getServerUserRegions(CatalogTracker catalogTracker, final ServerName serverName)
throws IOException {
- HRegionInterface metaServer =
- catalogTracker.waitForMetaServerConnectionDefault();
- NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
- Scan scan = new Scan();
- scan.addFamily(HConstants.CATALOG_FAMILY);
- long scannerid = metaServer.openScanner(
- HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
- try {
- Result result;
- while((result = metaServer.next(scannerid)) != null) {
- if (result != null && result.size() > 0) {
- Pair<HRegionInfo, ServerName> pair = metaRowToRegionPair(result);
- if (pair == null) continue;
- if (pair.getSecond() == null || !serverName.equals(pair.getSecond())) {
- continue;
- }
- hris.put(pair.getFirst(), result);
- }
+ final NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
+ // Fill the above hris map with entries from .META. that have the passed
+ // servername.
+ CollectingVisitor<Result> v = new CollectingVisitor<Result>() {
+ @Override
+ void add(Result r) {
+ if (r == null || r.isEmpty()) return;
+ ServerName sn = getServerNameFromCatalogResult(r);
+ if (sn != null && sn.equals(serverName)) this.results.add(r);
+ }
+ };
+ fullScan(catalogTracker, v);
+ List<Result> results = v.getResults();
+ if (results != null && !results.isEmpty()) {
+ // Convert results to Map keyed by HRI
+ for (Result r: results) {
+ Pair<HRegionInfo, ServerName> p = parseCatalogResult(r);
+ if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r);
}
- return hris;
- } finally {
- metaServer.close(scannerid);
}
+ return hris;
}
- public static void fullScanMetaAndPrint(
- CatalogTracker catalogTracker)
+ public static void fullScanMetaAndPrint(final CatalogTracker catalogTracker)
throws IOException {
- final List<HRegionInfo090x> regions =
- new ArrayList<HRegionInfo090x>();
Visitor v = new Visitor() {
@Override
public boolean visit(Result r) throws IOException {
@@ -590,27 +635,6 @@ public class MetaReader {
fullScan(catalogTracker, v);
}
-
- public static List<HRegionInfo090x> fullScanMetaAndPrintHRIM(
- CatalogTracker catalogTracker)
- throws IOException {
- final List<HRegionInfo090x> regions =
- new ArrayList<HRegionInfo090x>();
- Visitor v = new Visitor() {
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- LOG.info("fullScanMetaAndPrint1.Current Meta Result: " + r);
- HRegionInfo090x hrim = MetaEditor.getHRegionInfoForMigration(r);
- LOG.info("fullScanMetaAndPrint.HRIM Print= " + hrim);
- regions.add(hrim);
- return true;
- }
- };
- fullScan(catalogTracker, v);
- return regions;
- }
-
/**
* Fully scan a given region, on a given server starting with given row.
* @param hRegionInterface region server
@@ -618,6 +642,8 @@ public class MetaReader {
* @param regionName name of region
* @param startrow start row
* @throws IOException
+ * @deprecated Does not retry; use fullScan xxx instead.
+ x
*/
public static void fullScan(HRegionInterface hRegionInterface,
Visitor visitor, final byte[] regionName,
@@ -638,6 +664,38 @@ public class MetaReader {
return;
}
+ /**
+ * Performs a full scan of a catalog table.
+ * @param catalogTracker
+ * @param visitor Visitor invoked against each row.
+ * @param startrow Where to start the scan. Pass null if want to begin scan
+ * at first row.
+ * @param scanRoot True if we are to scan <code>-ROOT-</code> rather than
+ * <code>.META.</code>, the default (pass false to scan .META.)
+ * @throws IOException
+ */
+ static void fullScan(CatalogTracker catalogTracker,
+ final Visitor visitor, final byte [] startrow, final boolean scanRoot)
+ throws IOException {
+ Scan scan = new Scan();
+ if (startrow != null) scan.setStartRow(startrow);
+ scan.addFamily(HConstants.CATALOG_FAMILY);
+ HTable metaTable = scanRoot?
+ getRootHTable(catalogTracker): getMetaHTable(catalogTracker);
+ ResultScanner scanner = metaTable.getScanner(scan);
+ try {
+ Result data;
+ while((data = scanner.next()) != null) {
+ if (data.isEmpty()) continue;
+ // Break if visit returns false.
+ if (!visitor.visit(data)) break;
+ }
+ } finally {
+ scanner.close();
+ metaTable.close();
+ }
+ return;
+ }
/**
* Implementations 'visit' a catalog table row.
@@ -651,4 +709,37 @@ public class MetaReader {
*/
public boolean visit(final Result r) throws IOException;
}
-}
+
+ /**
+ * A {@link Visitor} that collects content out of passed {@link Result}.
+ */
+ static abstract class CollectingVisitor<T> implements Visitor {
+ final List<T> results = new ArrayList<T>();
+ @Override
+ public boolean visit(Result r) throws IOException {
+ if (r == null || r.isEmpty()) return true;
+ add(r);
+ return true;
+ }
+
+ abstract void add(Result r);
+
+ /**
+ * @return Collected results; wait till visits complete to collect all
+ * possible results
+ */
+ List<T> getResults() {
+ return this.results;
+ }
+ }
+
+ /**
+ * Collects all returned.
+ */
+ static class CollectAllVisitor extends CollectingVisitor<Result> {
+ @Override
+ void add(Result r) {
+ this.results.add(r);
+ }
+ }
+}
\ No newline at end of file