You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/07/03 04:01:33 UTC
[8/8] git commit: HBASE-4495 CatalogTracker has an identity crisis;
needs to be cut-back in scope (Mikhail Antonov)
HBASE-4495 CatalogTracker has an identity crisis; needs to be cut-back in scope (Mikhail Antonov)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ea085c63
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ea085c63
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ea085c63
Branch: refs/heads/master
Commit: ea085c6373ae238e896be755becdec7727bb89de
Parents: 1d89586
Author: stack <st...@apache.org>
Authored: Wed Jul 2 19:01:04 2014 -0700
Committer: stack <st...@apache.org>
Committed: Wed Jul 2 19:01:04 2014 -0700
----------------------------------------------------------------------
bin/region_mover.rb | 8 +-
dev-support/hbasetests.sh | 2 +-
.../org/apache/hadoop/hbase/HRegionInfo.java | 14 +-
.../java/org/apache/hadoop/hbase/Server.java | 22 +-
.../hadoop/hbase/catalog/CatalogTracker.java | 457 ----------
.../apache/hadoop/hbase/catalog/MetaReader.java | 886 -------------------
.../hadoop/hbase/client/ConnectionManager.java | 4 +-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 216 ++---
.../apache/hadoop/hbase/client/HConnection.java | 4 +-
.../apache/hadoop/hbase/client/MetaScanner.java | 12 +-
.../hadoop/hbase/client/ZooKeeperRegistry.java | 4 +-
.../hbase/zookeeper/MetaRegionTracker.java | 195 ----
.../apache/hadoop/hbase/zookeeper/ZKUtil.java | 4 +-
.../apache/hadoop/hbase/catalog/MetaEditor.java | 585 ------------
.../catalog/MetaMigrationConvertingToPB.java | 177 ----
.../hadoop/hbase/mapred/TableMapReduceUtil.java | 12 +-
.../hbase/mapreduce/TableMapReduceUtil.java | 8 +-
.../hadoop/hbase/master/AssignmentManager.java | 49 +-
.../hadoop/hbase/master/CatalogJanitor.java | 14 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 30 +-
.../hadoop/hbase/master/MasterRpcServices.java | 6 +-
.../hbase/master/MasterStatusServlet.java | 8 +-
.../apache/hadoop/hbase/master/RackManager.java | 2 +-
.../hbase/master/RegionPlacementMaintainer.java | 4 +-
.../hadoop/hbase/master/RegionStateStore.java | 20 +-
.../hadoop/hbase/master/RegionStates.java | 4 +-
.../SnapshotOfRegionAssignmentFromMeta.java | 22 +-
.../hbase/master/TableNamespaceManager.java | 6 +-
.../hbase/master/balancer/BaseLoadBalancer.java | 2 +-
.../balancer/FavoredNodeAssignmentHelper.java | 14 +-
.../balancer/FavoredNodeLoadBalancer.java | 2 +-
.../master/handler/CreateTableHandler.java | 17 +-
.../master/handler/DeleteTableHandler.java | 4 +-
.../master/handler/DisableTableHandler.java | 11 +-
.../master/handler/EnableTableHandler.java | 20 +-
.../handler/MetaServerShutdownHandler.java | 10 +-
.../master/handler/ModifyTableHandler.java | 9 +-
.../master/handler/ServerShutdownHandler.java | 14 +-
.../hbase/master/handler/TableEventHandler.java | 6 +-
.../master/handler/TruncateTableHandler.java | 5 +-
.../master/snapshot/CloneSnapshotHandler.java | 7 +-
.../master/snapshot/MasterSnapshotVerifier.java | 6 +-
.../master/snapshot/RestoreSnapshotHandler.java | 14 +-
.../hbase/master/snapshot/SnapshotManager.java | 4 +-
.../master/snapshot/TakeSnapshotHandler.java | 6 +-
.../hbase/migration/NamespaceUpgrade.java | 6 +-
.../flush/MasterFlushTableProcedureManager.java | 7 +-
.../hbase/regionserver/HRegionServer.java | 75 +-
.../hbase/regionserver/RSRpcServices.java | 6 +-
.../regionserver/RegionMergeTransaction.java | 35 +-
.../regionserver/RegionServerServices.java | 9 +-
.../hbase/regionserver/SplitTransaction.java | 28 +-
.../regionserver/handler/OpenRegionHandler.java | 7 +-
.../regionserver/ReplicationSyncUp.java | 10 +-
.../hbase/security/access/AccessController.java | 6 +-
.../visibility/VisibilityController.java | 4 +-
.../hbase/snapshot/RestoreSnapshotHelper.java | 18 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 10 +-
.../hadoop/hbase/util/HBaseFsckRepair.java | 4 +-
.../org/apache/hadoop/hbase/util/HMerge.java | 4 +-
.../hadoop/hbase/util/RegionSplitter.java | 4 +-
.../resources/hbase-webapps/master/table.jsp | 6 +-
.../hadoop/hbase/HBaseTestingUtility.java | 9 +-
.../hadoop/hbase/MockRegionServerServices.java | 14 +-
.../apache/hadoop/hbase/TestDrainingServer.java | 7 +-
.../hadoop/hbase/TestRegionRebalancing.java | 12 +-
.../hadoop/hbase/catalog/MetaMockingUtil.java | 126 ---
.../hbase/catalog/TestCatalogTracker.java | 395 ---------
.../TestMetaMigrationConvertingToPB.java | 440 ---------
.../hbase/catalog/TestMetaReaderEditor.java | 376 --------
.../catalog/TestMetaReaderEditorNoCluster.java | 228 -----
.../apache/hadoop/hbase/client/TestAdmin.java | 22 +-
.../hadoop/hbase/client/TestMetaScanner.java | 10 +-
.../hadoop/hbase/client/TestScannerTimeout.java | 4 +-
.../coprocessor/TestRegionServerObserver.java | 4 +-
.../TestLoadIncrementalHFilesSplitRecovery.java | 12 +-
.../hadoop/hbase/master/MockRegionServer.java | 13 +-
.../hbase/master/TestActiveMasterManager.java | 10 +-
.../hbase/master/TestAssignmentManager.java | 108 ++-
.../master/TestAssignmentManagerOnCluster.java | 22 +-
.../hadoop/hbase/master/TestCatalogJanitor.java | 27 +-
.../hbase/master/TestClockSkewDetection.java | 10 +-
.../apache/hadoop/hbase/master/TestMaster.java | 13 +-
.../hadoop/hbase/master/TestMasterFailover.java | 14 +-
.../hbase/master/TestMasterNoCluster.java | 39 +-
.../TestMasterOperationsForRegionReplicas.java | 78 +-
.../hadoop/hbase/master/TestRestartCluster.java | 8 +-
.../hbase/master/cleaner/TestHFileCleaner.java | 10 +-
.../master/cleaner/TestHFileLinkCleaner.java | 10 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 10 +-
.../TestEndToEndSplitTransaction.java | 6 +-
.../regionserver/TestGetClosestAtOrBefore.java | 4 +-
.../regionserver/TestHeapMemoryManager.java | 10 +-
.../TestRegionMergeTransactionOnCluster.java | 38 +-
.../hbase/regionserver/TestRegionReplicas.java | 6 +-
.../regionserver/TestRegionServerNoMaster.java | 9 +-
.../TestSplitTransactionOnCluster.java | 21 +-
.../replication/TestReplicationStateZKImpl.java | 10 +-
.../TestReplicationTrackerZKImpl.java | 10 +-
.../TestReplicationSourceManager.java | 11 +-
.../security/token/TestTokenAuthentication.java | 10 +-
.../snapshot/TestRestoreSnapshotHelper.java | 3 -
.../apache/hadoop/hbase/util/MockServer.java | 10 +-
.../apache/hadoop/hbase/util/TestHBaseFsck.java | 10 +-
.../hadoop/hbase/util/TestMergeTable.java | 15 +-
.../util/hbck/OfflineMetaRebuildTestCore.java | 4 +-
106 files changed, 796 insertions(+), 4577 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea085c63/bin/region_mover.rb
----------------------------------------------------------------------
diff --git a/bin/region_mover.rb b/bin/region_mover.rb
index 482617c..d3ee796 100644
--- a/bin/region_mover.rb
+++ b/bin/region_mover.rb
@@ -101,15 +101,13 @@ def getServerNameForRegion(admin, r)
if r.isMetaRegion()
# Hack
zkw = org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher.new(admin.getConfiguration(), "region_mover", nil)
+ mtl = org.apache.hadoop.hbase.zookeeper.MetaTableLocator.new()
begin
- tracker = org.apache.hadoop.hbase.zookeeper.MetaRegionTracker.new(zkw, RubyAbortable.new())
- tracker.start()
- while not tracker.isLocationAvailable()
+ while not mtl.isLocationAvailable(zkw)
sleep 0.1
end
# Make a fake servername by appending ','
- metaServer = tracker.getMetaRegionLocation().toString() + ","
- tracker.stop()
+ metaServer = mtl.getMetaRegionLocation(zkw).toString() + ","
return metaServer
ensure
zkw.close()
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea085c63/dev-support/hbasetests.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbasetests.sh b/dev-support/hbasetests.sh
index e129bd4..50c3a51 100755
--- a/dev-support/hbasetests.sh
+++ b/dev-support/hbasetests.sh
@@ -64,7 +64,7 @@ parallelMaven=1
# the others
#The ',' at the end is mandatory
flakyTests=
-#org.apache.hadoop.hbase.mapreduce.TestTableInputFormatScan,org.apache.hadoop.hbase.catalog.TestMetaReaderEditorNoCluster,org.apache.hadoop.hbase.catalog.TestMetaReaderEditor,org.apache.hadoop.hbase.mapreduce.TestHFileOutputFormat,org.apache.hadoop.hbase.mapred.TestTableMapReduce,org.apache.hadoop.hbase.coprocessor.TestMasterCoprocessorExceptionWithAbort,org.apache.hadoop.hbase.coprocessor.TestMasterCoprocessorExceptionWithRemove,org.apache.hadoop.hbase.client.TestAdmin,org.apache.hadoop.hbase.master.TestMasterFailover,org.apache.hadoop.hbase.regionserver.wal.TestLogRolling,org.apache.hadoop.hbase.master.TestDistributedLogSplitting,org.apache.hadoop.hbase.master.TestMasterRestartAfterDisablingTable,org.apache.hadoop.hbase.TestGlobalMemStoreSize,
+#org.apache.hadoop.hbase.mapreduce.TestTableInputFormatScan,org.apache.hadoop.hbase.catalog.TestMetaTableAccessorNoCluster,org.apache.hadoop.hbase.catalog.TestMetaTableAccessor,org.apache.hadoop.hbase.mapreduce.TestHFileOutputFormat,org.apache.hadoop.hbase.mapred.TestTableMapReduce,org.apache.hadoop.hbase.coprocessor.TestMasterCoprocessorExceptionWithAbort,org.apache.hadoop.hbase.coprocessor.TestMasterCoprocessorExceptionWithRemove,org.apache.hadoop.hbase.client.TestAdmin,org.apache.hadoop.hbase.master.TestMasterFailover,org.apache.hadoop.hbase.regionserver.wal.TestLogRolling,org.apache.hadoop.hbase.master.TestDistributedLogSplitting,org.apache.hadoop.hbase.master.TestMasterRestartAfterDisablingTable,org.apache.hadoop.hbase.TestGlobalMemStoreSize,
######################################### Internal parameters
#directory used for surefire & the source code.
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea085c63/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
index 0799fc8..8db7530 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
@@ -1231,7 +1231,7 @@ public class HRegionInfo implements Comparable<HRegionInfo> {
* @return A pair of the {@link HRegionInfo} and the {@link ServerName}
* (or null for server address if no address set in hbase:meta).
* @throws IOException
- * @deprecated use MetaReader methods for interacting with meta layouts
+ * @deprecated use MetaTableAccessor methods for interacting with meta layouts
*/
@Deprecated
public static Pair<HRegionInfo, ServerName> getHRegionInfoAndServerName(final Result r) {
@@ -1247,7 +1247,7 @@ public class HRegionInfo implements Comparable<HRegionInfo> {
* table Result.
* @param data a Result object from the catalog table scan
* @return HRegionInfo or null
- * @deprecated use MetaReader methods for interacting with meta layouts
+ * @deprecated use MetaTableAccessor methods for interacting with meta layouts
*/
@Deprecated
public static HRegionInfo getHRegionInfo(Result data) {
@@ -1260,7 +1260,7 @@ public class HRegionInfo implements Comparable<HRegionInfo> {
* @param data a Result object from the catalog table scan
* @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
* parent
- * @deprecated use MetaReader methods for interacting with meta layouts
+ * @deprecated use MetaTableAccessor methods for interacting with meta layouts
*/
@Deprecated
public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
@@ -1276,7 +1276,7 @@ public class HRegionInfo implements Comparable<HRegionInfo> {
* @param data a Result object from the catalog table scan
* @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
* parent
- * @deprecated use MetaReader methods for interacting with meta layouts
+ * @deprecated use MetaTableAccessor methods for interacting with meta layouts
*/
@Deprecated
public static PairOfSameType<HRegionInfo> getMergeRegions(Result data) throws IOException {
@@ -1294,7 +1294,7 @@ public class HRegionInfo implements Comparable<HRegionInfo> {
* {@link HConstants#SPLITA_QUALIFIER}, {@link HConstants#SPLITB_QUALIFIER} or
* {@link HConstants#REGIONINFO_QUALIFIER}.
* @return An HRegionInfo instance or null.
- * @deprecated use MetaReader methods for interacting with meta layouts
+ * @deprecated use MetaTableAccessor methods for interacting with meta layouts
*/
@Deprecated
public static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier) {
@@ -1305,7 +1305,7 @@ public class HRegionInfo implements Comparable<HRegionInfo> {
}
/**
- * @deprecated use MetaReader methods for interacting with meta layouts
+ * @deprecated use MetaTableAccessor methods for interacting with meta layouts
*/
@Deprecated
public static ServerName getServerName(final Result r) {
@@ -1330,7 +1330,7 @@ public class HRegionInfo implements Comparable<HRegionInfo> {
* E.g. the seqNum when the result of {@link #getServerName(Result)} was written.
* @param r Result to pull the seqNum from
* @return SeqNum, or HConstants.NO_SEQNUM if there's no value written.
- * @deprecated use MetaReader methods for interacting with meta layouts
+ * @deprecated use MetaTableAccessor methods for interacting with meta layouts
*/
@Deprecated
public static long getSeqNumDuringOpen(final Result r) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea085c63/hbase-client/src/main/java/org/apache/hadoop/hbase/Server.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/Server.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/Server.java
index c39d6ad..8f1a004 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/Server.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/Server.java
@@ -20,7 +20,8 @@ package org.apache.hadoop.hbase;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
/**
@@ -40,9 +41,24 @@ public interface Server extends Abortable, Stoppable {
ZooKeeperWatcher getZooKeeper();
/**
- * @return Master's instance of {@link CatalogTracker}
+ * Returns reference to wrapped short-circuit (i.e. local, bypassing RPC layer entirely)
+ * HConnection to this server, which may be used for miscellaneous needs.
+ *
+ * Important note: this method returns reference to connection which is managed
+ * by Server itself, so callers must NOT attempt to close connection obtained.
+ *
+ * See {@link org.apache.hadoop.hbase.client.ConnectionUtils#createShortCircuitHConnection}
+ * for details on short-circuit connections.
*/
- CatalogTracker getCatalogTracker();
+ HConnection getShortCircuitConnection();
+
+ /**
+ * Returns instance of {@link org.apache.hadoop.hbase.zookeeper.MetaTableLocator}
+ * running inside this server. This MetaServerLocator is started and stopped by server, clients
+ * shouldn't manage it's lifecycle.
+ * @return instance of {@link MetaTableLocator} associated with this server.
+ */
+ MetaTableLocator getMetaTableLocator();
/**
* @return The unique server name for this server.
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea085c63/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
deleted file mode 100644
index 3889317..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
+++ /dev/null
@@ -1,457 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.catalog;
-
-import com.google.common.base.Stopwatch;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.RetriesExhaustedException;
-import org.apache.hadoop.hbase.ipc.RpcClient.FailedServerException;
-import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService;
-import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.hadoop.ipc.RemoteException;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.net.ConnectException;
-import java.net.NoRouteToHostException;
-import java.net.SocketException;
-import java.net.SocketTimeoutException;
-import java.net.UnknownHostException;
-
-/**
- * Tracks the availability of the catalog tables
- * <code>hbase:meta</code>.
- *
- * This class is "read-only" in that the locations of the catalog tables cannot
- * be explicitly set. Instead, ZooKeeper is used to learn of the availability
- * and location of <code>hbase:meta</code>.
- *
- * <p>Call {@link #start()} to start up operation. Call {@link #stop()}} to
- * interrupt waits and close up shop.
- */
-@InterfaceAudience.Private
-public class CatalogTracker {
- // TODO JDC 11/30 We don't even have ROOT anymore, revisit
- // TODO: This class needs a rethink. The original intent was that it would be
- // the one-stop-shop for meta locations and that it would get this
- // info from reading and watching zk state. The class was to be used by
- // servers when they needed to know of meta movement but also by
- // client-side (inside in HTable) so rather than figure meta
- // locations on fault, the client would instead get notifications out of zk.
- //
- // But this original intent is frustrated by the fact that this class has to
- // read an hbase table, the -ROOT- table, to figure out the hbase:meta region
- // location which means we depend on an HConnection. HConnection will do
- // retrying but also, it has its own mechanism for finding root and meta
- // locations (and for 'verifying'; it tries the location and if it fails, does
- // new lookup, etc.). So, at least for now, HConnection (or HTable) can't
- // have a CT since CT needs a HConnection (Even then, do want HT to have a CT?
- // For HT keep up a session with ZK? Rather, shouldn't we do like asynchbase
- // where we'd open a connection to zk, read what we need then let the
- // connection go?). The 'fix' is make it so both root and meta addresses
- // are wholey up in zk -- not in zk (root) -- and in an hbase table (meta).
- //
- // But even then, this class does 'verification' of the location and it does
- // this by making a call over an HConnection (which will do its own root
- // and meta lookups). Isn't this verification 'useless' since when we
- // return, whatever is dependent on the result of this call then needs to
- // use HConnection; what we have verified may change in meantime (HConnection
- // uses the CT primitives, the root and meta trackers finding root locations).
- //
- // When meta is moved to zk, this class may make more sense. In the
- // meantime, it does not cohere. It should just watch meta and root and not
- // NOT do verification -- let that be out in HConnection since its going to
- // be done there ultimately anyways.
- //
- // This class has spread throughout the codebase. It needs to be reigned in.
- // This class should be used server-side only, even if we move meta location
- // up into zk. Currently its used over in the client package. Its used in
- // MetaReader and MetaEditor classes usually just to get the Configuration
- // its using (It does this indirectly by asking its HConnection for its
- // Configuration and even then this is just used to get an HConnection out on
- // the other end). I made https://issues.apache.org/jira/browse/HBASE-4495 for
- // doing CT fixup. St.Ack 09/30/2011.
- //
-
- // TODO: Timeouts have never been as advertised in here and its worse now
- // with retries; i.e. the HConnection retries and pause goes ahead whatever
- // the passed timeout is. Fix.
- private static final Log LOG = LogFactory.getLog(CatalogTracker.class);
- private final HConnection connection;
- private final ZooKeeperWatcher zookeeper;
- private final MetaRegionTracker metaRegionTracker;
- private boolean instantiatedzkw = false;
- private Abortable abortable;
-
- private boolean stopped = false;
-
- static final byte [] META_REGION_NAME =
- HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
-
- /**
- * Constructs a catalog tracker. Find current state of catalog tables.
- * Begin active tracking by executing {@link #start()} post construction. Does
- * not timeout.
- *
- * @param conf
- * the {@link Configuration} from which a {@link HConnection} will be
- * obtained; if problem, this connections
- * {@link HConnection#abort(String, Throwable)} will be called.
- * @throws IOException
- */
- public CatalogTracker(final Configuration conf) throws IOException {
- this(null, conf, HConnectionManager.getConnection(conf), null);
- }
-
- /**
- * Constructs the catalog tracker. Find current state of catalog tables.
- * Begin active tracking by executing {@link #start()} post construction.
- * Does not timeout.
- * @param zk If zk is null, we'll create an instance (and shut it down
- * when {@link #stop()} is called) else we'll use what is passed.
- * @param conf
- * @param abortable If fatal exception we'll call abort on this. May be null.
- * If it is we'll use the Connection associated with the passed
- * {@link Configuration} as our Abortable.
- * @throws IOException
- */
- public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
- HConnection connection, Abortable abortable)
- throws IOException {
- this.connection = connection;
- if (abortable == null) {
- // A connection is abortable.
- this.abortable = this.connection;
- } else {
- this.abortable = abortable;
- }
- Abortable throwableAborter = new Abortable() {
-
- @Override
- public void abort(String why, Throwable e) {
- throw new RuntimeException(why, e);
- }
-
- @Override
- public boolean isAborted() {
- return true;
- }
-
- };
- if (zk == null) {
- // Create our own. Set flag so we tear it down on stop.
- this.zookeeper =
- new ZooKeeperWatcher(conf, "catalogtracker-on-" + connection.toString(),
- abortable);
- instantiatedzkw = true;
- } else {
- this.zookeeper = zk;
- }
- this.metaRegionTracker = new MetaRegionTracker(zookeeper, throwableAborter);
- }
-
- /**
- * Starts the catalog tracker.
- * Determines current availability of catalog tables and ensures all further
- * transitions of either region are tracked.
- * @throws IOException
- * @throws InterruptedException
- */
- public void start() throws IOException, InterruptedException {
- LOG.debug("Starting catalog tracker " + this);
- try {
- this.metaRegionTracker.start();
- } catch (RuntimeException e) {
- Throwable t = e.getCause();
- this.abortable.abort(e.getMessage(), t);
- throw new IOException("Attempt to start meta tracker failed.", t);
- }
- }
-
- /**
- * Stop working.
- * Interrupts any ongoing waits.
- */
- public void stop() {
- if (!this.stopped) {
- LOG.debug("Stopping catalog tracker " + this);
- this.stopped = true;
- this.metaRegionTracker.stop();
- try {
- if (this.connection != null) {
- this.connection.close();
- }
- } catch (IOException e) {
- // Although the {@link Closeable} interface throws an {@link
- // IOException}, in reality, the implementation would never do that.
- LOG.error("Attempt to close catalog tracker's connection failed.", e);
- }
- if (this.instantiatedzkw) {
- this.zookeeper.close();
- }
- }
- }
-
- /**
- * Gets the current location for <code>hbase:meta</code> or null if location is
- * not currently available.
- * @return {@link ServerName} for server hosting <code>hbase:meta</code> or null
- * if none available
- * @throws InterruptedException
- */
- public ServerName getMetaLocation() throws InterruptedException {
- return this.metaRegionTracker.getMetaRegionLocation();
- }
-
- /**
- * Checks whether meta regionserver znode has some non null data.
- * @return true if data is not null, false otherwise.
- */
- public boolean isMetaLocationAvailable() {
- return this.metaRegionTracker.isLocationAvailable();
- }
- /**
- * Gets the current location for <code>hbase:meta</code> if available and waits
- * for up to the specified timeout if not immediately available. Returns null
- * if the timeout elapses before root is available.
- * @param timeout maximum time to wait for root availability, in milliseconds
- * @return {@link ServerName} for server hosting <code>hbase:meta</code> or null
- * if none available
- * @throws InterruptedException if interrupted while waiting
- * @throws NotAllMetaRegionsOnlineException if meta not available before
- * timeout
- */
- public ServerName waitForMeta(final long timeout)
- throws InterruptedException, NotAllMetaRegionsOnlineException {
- ServerName sn = metaRegionTracker.waitMetaRegionLocation(timeout);
- if (sn == null) {
- throw new NotAllMetaRegionsOnlineException("Timed out; " + timeout + "ms");
- }
- return sn;
- }
-
- /**
- * Gets a connection to the server hosting meta, as reported by ZooKeeper,
- * waiting up to the specified timeout for availability.
- * @param timeout How long to wait on meta location
- * @see #waitForMeta for additional information
- * @return connection to server hosting meta
- * @throws InterruptedException
- * @throws NotAllMetaRegionsOnlineException if timed out waiting
- * @throws IOException
- * @deprecated Use #getMetaServerConnection(long)
- */
- public AdminService.BlockingInterface waitForMetaServerConnection(long timeout)
- throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
- return getMetaServerConnection(timeout);
- }
-
- /**
- * Gets a connection to the server hosting meta, as reported by ZooKeeper,
- * waiting up to the specified timeout for availability.
- * <p>WARNING: Does not retry. Use an {@link HTable} instead.
- * @param timeout How long to wait on meta location
- * @see #waitForMeta for additional information
- * @return connection to server hosting meta
- * @throws InterruptedException
- * @throws NotAllMetaRegionsOnlineException if timed out waiting
- * @throws IOException
- */
- AdminService.BlockingInterface getMetaServerConnection(long timeout)
- throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
- return getCachedConnection(waitForMeta(timeout));
- }
-
- /**
- * Waits indefinitely for availability of <code>hbase:meta</code>. Used during
- * cluster startup. Does not verify meta, just that something has been
- * set up in zk.
- * @see #waitForMeta(long)
- * @throws InterruptedException if interrupted while waiting
- */
- public void waitForMeta() throws InterruptedException {
- Stopwatch stopwatch = new Stopwatch().start();
- while (!this.stopped) {
- try {
- if (waitForMeta(100) != null) break;
- long sleepTime = stopwatch.elapsedMillis();
- // +1 in case sleepTime=0
- if ((sleepTime + 1) % 10000 == 0) {
- LOG.warn("Have been waiting for meta to be assigned for " + sleepTime + "ms");
- }
- } catch (NotAllMetaRegionsOnlineException e) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("hbase:meta still not available, sleeping and retrying." +
- " Reason: " + e.getMessage());
- }
- }
- }
- }
-
- /**
- * @param sn ServerName to get a connection against.
- * @return The AdminProtocol we got when we connected to <code>sn</code>
- * May have come from cache, may not be good, may have been setup by this
- * invocation, or may be null.
- * @throws IOException
- */
- @SuppressWarnings("deprecation")
- private AdminService.BlockingInterface getCachedConnection(ServerName sn)
- throws IOException {
- if (sn == null) {
- return null;
- }
- AdminService.BlockingInterface service = null;
- try {
- service = connection.getAdmin(sn);
- } catch (RetriesExhaustedException e) {
- if (e.getCause() != null && e.getCause() instanceof ConnectException) {
- // Catch this; presume it means the cached connection has gone bad.
- } else {
- throw e;
- }
- } catch (SocketTimeoutException e) {
- LOG.debug("Timed out connecting to " + sn);
- } catch (NoRouteToHostException e) {
- LOG.debug("Connecting to " + sn, e);
- } catch (SocketException e) {
- LOG.debug("Exception connecting to " + sn);
- } catch (UnknownHostException e) {
- LOG.debug("Unknown host exception connecting to " + sn);
- } catch (FailedServerException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Server " + sn + " is in failed server list.");
- }
- } catch (IOException ioe) {
- Throwable cause = ioe.getCause();
- if (ioe instanceof ConnectException) {
- // Catch. Connect refused.
- } else if (cause != null && cause instanceof EOFException) {
- // Catch. Other end disconnected us.
- } else if (cause != null && cause.getMessage() != null &&
- cause.getMessage().toLowerCase().contains("connection reset")) {
- // Catch. Connection reset.
- } else {
- throw ioe;
- }
-
- }
- return service;
- }
-
- /**
- * Verify we can connect to <code>hostingServer</code> and that its carrying
- * <code>regionName</code>.
- * @param hostingServer Interface to the server hosting <code>regionName</code>
- * @param address The servername that goes with the <code>metaServer</code>
- * Interface. Used logging.
- * @param regionName The regionname we are interested in.
- * @return True if we were able to verify the region located at other side of
- * the Interface.
- * @throws IOException
- */
- // TODO: We should be able to get the ServerName from the AdminProtocol
- // rather than have to pass it in. Its made awkward by the fact that the
- // HRI is likely a proxy against remote server so the getServerName needs
- // to be fixed to go to a local method or to a cache before we can do this.
- private boolean verifyRegionLocation(AdminService.BlockingInterface hostingServer,
- final ServerName address, final byte [] regionName)
- throws IOException {
- if (hostingServer == null) {
- LOG.info("Passed hostingServer is null");
- return false;
- }
- Throwable t = null;
- try {
- // Try and get regioninfo from the hosting server.
- return ProtobufUtil.getRegionInfo(hostingServer, regionName) != null;
- } catch (ConnectException e) {
- t = e;
- } catch (RetriesExhaustedException e) {
- t = e;
- } catch (RemoteException e) {
- IOException ioe = e.unwrapRemoteException();
- t = ioe;
- } catch (IOException e) {
- Throwable cause = e.getCause();
- if (cause != null && cause instanceof EOFException) {
- t = cause;
- } else if (cause != null && cause.getMessage() != null
- && cause.getMessage().contains("Connection reset")) {
- t = cause;
- } else {
- t = e;
- }
- }
- LOG.info("Failed verification of " + Bytes.toStringBinary(regionName) +
- " at address=" + address + ", exception=" + t);
- return false;
- }
-
- /**
- * Verify <code>hbase:meta</code> is deployed and accessible.
- * @param timeout How long to wait on zk for meta address (passed through to
- * the internal call to {@link #waitForMetaServerConnection(long)}.
- * @return True if the <code>hbase:meta</code> location is healthy.
- * @throws IOException
- * @throws InterruptedException
- */
- public boolean verifyMetaRegionLocation(final long timeout)
- throws InterruptedException, IOException {
- AdminService.BlockingInterface service = null;
- try {
- service = waitForMetaServerConnection(timeout);
- } catch (NotAllMetaRegionsOnlineException e) {
- // Pass
- } catch (ServerNotRunningYetException e) {
- // Pass -- remote server is not up so can't be carrying root
- } catch (UnknownHostException e) {
- // Pass -- server name doesn't resolve so it can't be assigned anything.
- } catch (RegionServerStoppedException e) {
- // Pass -- server name sends us to a server that is dying or already dead.
- }
- return (service == null)? false:
- verifyRegionLocation(service,
- this.metaRegionTracker.getMetaRegionLocation(), META_REGION_NAME);
- }
-
- public HConnection getConnection() {
- return this.connection;
- }
-
- @Override
- public String toString() {
- return "CatalogTracker{" + "connection=" + connection + ", zookeeper=" + zookeeper +
- ", metaRegionTracker=" + metaRegionTracker + ", stopped=" + stopped + '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea085c63/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
deleted file mode 100644
index 9517113..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
+++ /dev/null
@@ -1,886 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.catalog;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.NavigableMap;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HRegionLocation;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.RegionLocations;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.RegionReplicaUtil;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.util.PairOfSameType;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Reads region and assignment information from <code>hbase:meta</code>.
- */
-@InterfaceAudience.Private
-public class MetaReader {
-
- /*
- * HBASE-10070 adds a replicaId to HRI, meaning more than one HRI can be defined for the
- * same table range (table, startKey, endKey). For every range, there will be at least one
- * HRI defined which is called default replica.
- *
- * Meta layout (as of 0.98 + HBASE-10070) is like:
- * For each table range, there is a single row, formatted like:
- * <tableName>,<startKey>,<regionId>,<encodedRegionName>. This row corresponds to the regionName
- * of the default region replica.
- * Columns are:
- * info:regioninfo => contains serialized HRI for the default region replica
- * info:server => contains hostname:port (in string form) for the server hosting
- * the default regionInfo replica
- * info:server_<replicaId> => contains hostname:port (in string form) for the server hosting the
- * regionInfo replica with replicaId
- * info:serverstartcode => contains server start code (in binary long form) for the server
- * hosting the default regionInfo replica
- * info:serverstartcode_<replicaId> => contains server start code (in binary long form) for the
- * server hosting the regionInfo replica with replicaId
- * info:seqnumDuringOpen => contains seqNum (in binary long form) for the region at the time
- * the server opened the region with default replicaId
- * info:seqnumDuringOpen_<replicaId> => contains seqNum (in binary long form) for the region at
- * the time the server opened the region with replicaId
- * info:splitA => contains a serialized HRI for the first daughter region if the
- * region is split
- * info:splitB => contains a serialized HRI for the second daughter region if the
- * region is split
- * info:mergeA => contains a serialized HRI for the first parent region if the
- * region is the result of a merge
- * info:mergeB => contains a serialized HRI for the second parent region if the
- * region is the result of a merge
- *
- * The actual layout of meta should be encapsulated inside MetaReader and MetaEditor methods,
- * and should not leak out of those (through Result objects, etc)
- */
-
- // TODO: Strip CatalogTracker from this class. Its all over and in the end
- // its only used to get its Configuration so we can get associated
- // Connection.
- private static final Log LOG = LogFactory.getLog(MetaReader.class);
-
- static final byte [] META_REGION_PREFIX;
- static {
- // Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX.
- // FIRST_META_REGIONINFO == 'hbase:meta,,1'. META_REGION_PREFIX == 'hbase:meta,'
- int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2;
- META_REGION_PREFIX = new byte [len];
- System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0,
- META_REGION_PREFIX, 0, len);
- }
-
- /** The delimiter for meta columns for replicaIds > 0 */
- protected static final char META_REPLICA_ID_DELIMITER = '_';
-
- /** A regex for parsing server columns from meta. See above javadoc for meta layout */
- private static final Pattern SERVER_COLUMN_PATTERN
- = Pattern.compile("^server(_[0-9a-fA-F]{4})?$");
-
- /**
- * Performs a full scan of <code>hbase:meta</code>.
- * @return List of {@link Result}
- * @throws IOException
- */
- public static List<Result> fullScan(CatalogTracker catalogTracker)
- throws IOException {
- CollectAllVisitor v = new CollectAllVisitor();
- fullScan(catalogTracker, v, null);
- return v.getResults();
- }
-
- /**
- * Performs a full scan of a <code>hbase:meta</code> table.
- * @return List of {@link Result}
- * @throws IOException
- */
- public static List<Result> fullScanOfMeta(CatalogTracker catalogTracker)
- throws IOException {
- CollectAllVisitor v = new CollectAllVisitor();
- fullScan(catalogTracker, v, null);
- return v.getResults();
- }
-
- /**
- * Performs a full scan of <code>hbase:meta</code>.
- * @param catalogTracker
- * @param visitor Visitor invoked against each row.
- * @throws IOException
- */
- public static void fullScan(CatalogTracker catalogTracker,
- final Visitor visitor)
- throws IOException {
- fullScan(catalogTracker, visitor, null);
- }
-
- /**
- * Callers should call close on the returned {@link HTable} instance.
- * @param catalogTracker We'll use this catalogtracker's connection
- * @param tableName Table to get an {@link HTable} against.
- * @return An {@link HTable} for <code>tableName</code>
- * @throws IOException
- */
- @SuppressWarnings("deprecation")
- private static HTable getHTable(final CatalogTracker catalogTracker,
- final TableName tableName)
- throws IOException {
- // Passing the CatalogTracker's connection ensures this
- // HTable instance uses the CatalogTracker's connection.
- org.apache.hadoop.hbase.client.HConnection c = catalogTracker.getConnection();
- if (c == null) throw new NullPointerException("No connection");
- return new HTable(tableName, c);
- }
-
- /**
- * Callers should call close on the returned {@link HTable} instance.
- * @param catalogTracker
- * @return An {@link HTable} for <code>hbase:meta</code>
- * @throws IOException
- */
- static HTable getCatalogHTable(final CatalogTracker catalogTracker)
- throws IOException {
- return getMetaHTable(catalogTracker);
- }
-
- /**
- * Callers should call close on the returned {@link HTable} instance.
- * @param ct
- * @return An {@link HTable} for <code>hbase:meta</code>
- * @throws IOException
- */
- static HTable getMetaHTable(final CatalogTracker ct)
- throws IOException {
- return getHTable(ct, TableName.META_TABLE_NAME);
- }
-
- /**
- * @param t Table to use (will be closed when done).
- * @param g Get to run
- * @throws IOException
- */
- private static Result get(final HTable t, final Get g) throws IOException {
- try {
- return t.get(g);
- } finally {
- t.close();
- }
- }
-
- /**
- * Gets the region info and assignment for the specified region.
- * @param catalogTracker
- * @param regionName Region to lookup.
- * @return Location and HRegionInfo for <code>regionName</code>
- * @throws IOException
- * @deprecated use {@link #getRegionLocation(CatalogTracker, byte[])} instead
- */
- @Deprecated
- public static Pair<HRegionInfo, ServerName> getRegion(
- CatalogTracker catalogTracker, byte [] regionName)
- throws IOException {
- HRegionLocation location = getRegionLocation(catalogTracker, regionName);
- return location == null
- ? null
- : new Pair<HRegionInfo, ServerName>(location.getRegionInfo(), location.getServerName());
- }
-
- /**
- * Returns the HRegionLocation from meta for the given region
- * @param catalogTracker
- * @param regionName
- * @return HRegionLocation for the given region
- * @throws IOException
- */
- public static HRegionLocation getRegionLocation(CatalogTracker catalogTracker,
- byte[] regionName) throws IOException {
- byte[] row = regionName;
- HRegionInfo parsedInfo = null;
- try {
- parsedInfo = parseRegionInfoFromRegionName(regionName);
- row = getMetaKeyForRegion(parsedInfo);
- } catch (Exception parseEx) {
- // Ignore. This is used with tableName passed as regionName.
- }
- Get get = new Get(row);
- get.addFamily(HConstants.CATALOG_FAMILY);
- Result r = get(getCatalogHTable(catalogTracker), get);
- RegionLocations locations = getRegionLocations(r);
- return locations == null
- ? null
- : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
- }
-
- /**
- * Returns the HRegionLocation from meta for the given region
- * @param catalogTracker
- * @param regionInfo
- * @return HRegionLocation for the given region
- * @throws IOException
- */
- public static HRegionLocation getRegionLocation(CatalogTracker catalogTracker,
- HRegionInfo regionInfo) throws IOException {
- byte[] row = getMetaKeyForRegion(regionInfo);
- Get get = new Get(row);
- get.addFamily(HConstants.CATALOG_FAMILY);
- Result r = get(getCatalogHTable(catalogTracker), get);
- return getRegionLocation(r, regionInfo, regionInfo.getReplicaId());
- }
-
- /** Returns the row key to use for this regionInfo */
- public static byte[] getMetaKeyForRegion(HRegionInfo regionInfo) {
- return RegionReplicaUtil.getRegionInfoForDefaultReplica(regionInfo).getRegionName();
- }
-
- /** Returns an HRI parsed from this regionName. Not all the fields of the HRI
- * is stored in the name, so the returned object should only be used for the fields
- * in the regionName.
- */
- protected static HRegionInfo parseRegionInfoFromRegionName(byte[] regionName)
- throws IOException {
- byte[][] fields = HRegionInfo.parseRegionName(regionName);
- long regionId = Long.parseLong(Bytes.toString(fields[2]));
- int replicaId = fields.length > 3 ? Integer.parseInt(Bytes.toString(fields[3]), 16) : 0;
- return new HRegionInfo(
- TableName.valueOf(fields[0]), fields[1], fields[1], false, regionId, replicaId);
- }
-
- /**
- * Gets the result in hbase:meta for the specified region.
- * @param catalogTracker
- * @param regionName
- * @return result of the specified region
- * @throws IOException
- */
- public static Result getRegionResult(CatalogTracker catalogTracker,
- byte[] regionName) throws IOException {
- Get get = new Get(regionName);
- get.addFamily(HConstants.CATALOG_FAMILY);
- return get(getCatalogHTable(catalogTracker), get);
- }
-
- /**
- * Get regions from the merge qualifier of the specified merged region
- * @return null if it doesn't contain merge qualifier, else two merge regions
- * @throws IOException
- */
- public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
- CatalogTracker catalogTracker, byte[] regionName) throws IOException {
- Result result = getRegionResult(catalogTracker, regionName);
- HRegionInfo mergeA = getHRegionInfo(result, HConstants.MERGEA_QUALIFIER);
- HRegionInfo mergeB = getHRegionInfo(result, HConstants.MERGEB_QUALIFIER);
- if (mergeA == null && mergeB == null) {
- return null;
- }
- return new Pair<HRegionInfo, HRegionInfo>(mergeA, mergeB);
- }
-
- /**
- * Checks if the specified table exists. Looks at the hbase:meta table hosted on
- * the specified server.
- * @param catalogTracker
- * @param tableName table to check
- * @return true if the table exists in meta, false if not
- * @throws IOException
- */
- public static boolean tableExists(CatalogTracker catalogTracker,
- final TableName tableName)
- throws IOException {
- if (tableName.equals(HTableDescriptor.META_TABLEDESC.getTableName())) {
- // Catalog tables always exist.
- return true;
- }
- // Make a version of ResultCollectingVisitor that only collects the first
- CollectingVisitor<HRegionInfo> visitor = new CollectingVisitor<HRegionInfo>() {
- private HRegionInfo current = null;
-
- @Override
- public boolean visit(Result r) throws IOException {
- RegionLocations locations = getRegionLocations(r);
- if (locations == null || locations.getRegionLocation().getRegionInfo() == null) {
- LOG.warn("No serialized HRegionInfo in " + r);
- return true;
- }
- this.current = locations.getRegionLocation().getRegionInfo();
- if (this.current == null) {
- LOG.warn("No serialized HRegionInfo in " + r);
- return true;
- }
- if (!isInsideTable(this.current, tableName)) return false;
- // Else call super and add this Result to the collection.
- super.visit(r);
- // Stop collecting regions from table after we get one.
- return false;
- }
-
- @Override
- void add(Result r) {
- // Add the current HRI.
- this.results.add(this.current);
- }
- };
- fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
- // If visitor has results >= 1 then table exists.
- return visitor.getResults().size() >= 1;
- }
-
- /**
- * Gets all of the regions of the specified table.
- * @param catalogTracker
- * @param tableName
- * @return Ordered list of {@link HRegionInfo}.
- * @throws IOException
- */
- public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
- TableName tableName)
- throws IOException {
- return getTableRegions(catalogTracker, tableName, false);
- }
-
- /**
- * Gets all of the regions of the specified table.
- * @param catalogTracker
- * @param tableName
- * @param excludeOfflinedSplitParents If true, do not include offlined split
- * parents in the return.
- * @return Ordered list of {@link HRegionInfo}.
- * @throws IOException
- */
- public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
- TableName tableName, final boolean excludeOfflinedSplitParents)
- throws IOException {
- List<Pair<HRegionInfo, ServerName>> result = null;
- try {
- result = getTableRegionsAndLocations(catalogTracker, tableName,
- excludeOfflinedSplitParents);
- } catch (InterruptedException e) {
- throw (InterruptedIOException)new InterruptedIOException().initCause(e);
- }
- return getListOfHRegionInfos(result);
- }
-
- static List<HRegionInfo> getListOfHRegionInfos(final List<Pair<HRegionInfo, ServerName>> pairs) {
- if (pairs == null || pairs.isEmpty()) return null;
- List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
- for (Pair<HRegionInfo, ServerName> pair: pairs) {
- result.add(pair.getFirst());
- }
- return result;
- }
-
- /**
- * @param current
- * @param tableName
- * @return True if <code>current</code> tablename is equal to
- * <code>tableName</code>
- */
- static boolean isInsideTable(final HRegionInfo current, final TableName tableName) {
- return tableName.equals(current.getTable());
- }
-
- /**
- * @param tableName
- * @return Place to start Scan in <code>hbase:meta</code> when passed a
- * <code>tableName</code>; returns <tableName&rt; <,&rt; <,&rt;
- */
- static byte [] getTableStartRowForMeta(TableName tableName) {
- byte [] startRow = new byte[tableName.getName().length + 2];
- System.arraycopy(tableName.getName(), 0, startRow, 0, tableName.getName().length);
- startRow[startRow.length - 2] = HConstants.DELIMITER;
- startRow[startRow.length - 1] = HConstants.DELIMITER;
- return startRow;
- }
-
- /**
- * This method creates a Scan object that will only scan catalog rows that
- * belong to the specified table. It doesn't specify any columns.
- * This is a better alternative to just using a start row and scan until
- * it hits a new table since that requires parsing the HRI to get the table
- * name.
- * @param tableName bytes of table's name
- * @return configured Scan object
- */
- public static Scan getScanForTableName(TableName tableName) {
- String strName = tableName.getNameAsString();
- // Start key is just the table name with delimiters
- byte[] startKey = Bytes.toBytes(strName + ",,");
- // Stop key appends the smallest possible char to the table name
- byte[] stopKey = Bytes.toBytes(strName + " ,,");
-
- Scan scan = new Scan(startKey);
- scan.setStopRow(stopKey);
- return scan;
- }
-
- /**
- * @param catalogTracker
- * @param tableName
- * @return Return list of regioninfos and server.
- * @throws IOException
- * @throws InterruptedException
- */
- public static List<Pair<HRegionInfo, ServerName>>
- getTableRegionsAndLocations(CatalogTracker catalogTracker, TableName tableName)
- throws IOException, InterruptedException {
- return getTableRegionsAndLocations(catalogTracker, tableName,
- true);
- }
-
- /**
- * @param catalogTracker
- * @param tableName
- * @return Return list of regioninfos and server addresses.
- * @throws IOException
- * @throws InterruptedException
- */
- public static List<Pair<HRegionInfo, ServerName>>
- getTableRegionsAndLocations(final CatalogTracker catalogTracker,
- final TableName tableName, final boolean excludeOfflinedSplitParents)
- throws IOException, InterruptedException {
- if (tableName.equals(TableName.META_TABLE_NAME)) {
- // If meta, do a bit of special handling.
- ServerName serverName = catalogTracker.getMetaLocation();
- List<Pair<HRegionInfo, ServerName>> list =
- new ArrayList<Pair<HRegionInfo, ServerName>>();
- list.add(new Pair<HRegionInfo, ServerName>(HRegionInfo.FIRST_META_REGIONINFO,
- serverName));
- return list;
- }
- // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress
- CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
- new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
- private RegionLocations current = null;
-
- @Override
- public boolean visit(Result r) throws IOException {
- current = getRegionLocations(r);
- if (current == null || current.getRegionLocation().getRegionInfo() == null) {
- LOG.warn("No serialized HRegionInfo in " + r);
- return true;
- }
- HRegionInfo hri = current.getRegionLocation().getRegionInfo();
- if (!isInsideTable(hri, tableName)) return false;
- if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
- // Else call super and add this Result to the collection.
- return super.visit(r);
- }
-
- @Override
- void add(Result r) {
- if (current == null) {
- return;
- }
- for (HRegionLocation loc : current.getRegionLocations()) {
- if (loc != null) {
- this.results.add(new Pair<HRegionInfo, ServerName>(
- loc.getRegionInfo(), loc.getServerName()));
- }
- }
- }
- };
- fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
- return visitor.getResults();
- }
-
- /**
- * @param catalogTracker
- * @param serverName
- * @return List of user regions installed on this server (does not include
- * catalog regions).
- * @throws IOException
- */
- public static NavigableMap<HRegionInfo, Result>
- getServerUserRegions(CatalogTracker catalogTracker, final ServerName serverName)
- throws IOException {
- final NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
- // Fill the above hris map with entries from hbase:meta that have the passed
- // servername.
- CollectingVisitor<Result> v = new CollectingVisitor<Result>() {
- @Override
- void add(Result r) {
- if (r == null || r.isEmpty()) return;
- RegionLocations locations = getRegionLocations(r);
- if (locations == null) return;
- for (HRegionLocation loc : locations.getRegionLocations()) {
- if (loc != null) {
- if (loc.getServerName() != null && loc.getServerName().equals(serverName)) {
- hris.put(loc.getRegionInfo(), r);
- }
- }
- }
- }
- };
- fullScan(catalogTracker, v);
- return hris;
- }
-
- public static void fullScanMetaAndPrint(final CatalogTracker catalogTracker)
- throws IOException {
- Visitor v = new Visitor() {
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r);
- RegionLocations locations = getRegionLocations(r);
- if (locations == null) return true;
- for (HRegionLocation loc : locations.getRegionLocations()) {
- if (loc != null) {
- LOG.info("fullScanMetaAndPrint.HRI Print= " + loc.getRegionInfo());
- }
- }
- return true;
- }
- };
- fullScan(catalogTracker, v);
- }
-
- /**
- * Performs a full scan of a catalog table.
- * @param catalogTracker
- * @param visitor Visitor invoked against each row.
- * @param startrow Where to start the scan. Pass null if want to begin scan
- * at first row.
- * <code>hbase:meta</code>, the default (pass false to scan hbase:meta)
- * @throws IOException
- */
- public static void fullScan(CatalogTracker catalogTracker,
- final Visitor visitor, final byte [] startrow)
- throws IOException {
- Scan scan = new Scan();
- if (startrow != null) scan.setStartRow(startrow);
- if (startrow == null) {
- int caching = catalogTracker.getConnection().getConfiguration()
- .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
- scan.setCaching(caching);
- }
- scan.addFamily(HConstants.CATALOG_FAMILY);
- HTable metaTable = getMetaHTable(catalogTracker);
- ResultScanner scanner = null;
- try {
- scanner = metaTable.getScanner(scan);
- Result data;
- while((data = scanner.next()) != null) {
- if (data.isEmpty()) continue;
- // Break if visit returns false.
- if (!visitor.visit(data)) break;
- }
- } finally {
- if (scanner != null) scanner.close();
- metaTable.close();
- }
- return;
- }
-
- /**
- * Returns the column family used for meta columns.
- * @return HConstants.CATALOG_FAMILY.
- */
- protected static byte[] getFamily() {
- return HConstants.CATALOG_FAMILY;
- }
-
- /**
- * Returns the column qualifier for serialized region info
- * @return HConstants.REGIONINFO_QUALIFIER
- */
- protected static byte[] getRegionInfoColumn() {
- return HConstants.REGIONINFO_QUALIFIER;
- }
-
- /**
- * Returns the column qualifier for server column for replicaId
- * @param replicaId the replicaId of the region
- * @return a byte[] for server column qualifier
- */
- @VisibleForTesting
- public static byte[] getServerColumn(int replicaId) {
- return replicaId == 0
- ? HConstants.SERVER_QUALIFIER
- : Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
- + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
- }
-
- /**
- * Returns the column qualifier for server start code column for replicaId
- * @param replicaId the replicaId of the region
- * @return a byte[] for server start code column qualifier
- */
- @VisibleForTesting
- public static byte[] getStartCodeColumn(int replicaId) {
- return replicaId == 0
- ? HConstants.STARTCODE_QUALIFIER
- : Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
- + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
- }
-
- /**
- * Returns the column qualifier for seqNum column for replicaId
- * @param replicaId the replicaId of the region
- * @return a byte[] for seqNum column qualifier
- */
- @VisibleForTesting
- public static byte[] getSeqNumColumn(int replicaId) {
- return replicaId == 0
- ? HConstants.SEQNUM_QUALIFIER
- : Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
- + String.format(HRegionInfo.REPLICA_ID_FORMAT, replicaId));
- }
-
- /**
- * Parses the replicaId from the server column qualifier. See top of the class javadoc
- * for the actual meta layout
- * @param serverColumn the column qualifier
- * @return an int for the replicaId
- */
- @VisibleForTesting
- static int parseReplicaIdFromServerColumn(byte[] serverColumn) {
- String serverStr = Bytes.toString(serverColumn);
-
- Matcher matcher = SERVER_COLUMN_PATTERN.matcher(serverStr);
- if (matcher.matches() && matcher.groupCount() > 0) {
- String group = matcher.group(1);
- if (group != null && group.length() > 0) {
- return Integer.parseInt(group.substring(1), 16);
- } else {
- return 0;
- }
- }
- return -1;
- }
-
- /**
- * Returns a {@link ServerName} from catalog table {@link Result}.
- * @param r Result to pull from
- * @return A ServerName instance or null if necessary fields not found or empty.
- */
- private static ServerName getServerName(final Result r, final int replicaId) {
- byte[] serverColumn = getServerColumn(replicaId);
- Cell cell = r.getColumnLatestCell(getFamily(), serverColumn);
- if (cell == null || cell.getValueLength() == 0) return null;
- String hostAndPort = Bytes.toString(
- cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- byte[] startcodeColumn = getStartCodeColumn(replicaId);
- cell = r.getColumnLatestCell(getFamily(), startcodeColumn);
- if (cell == null || cell.getValueLength() == 0) return null;
- return ServerName.valueOf(hostAndPort,
- Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
- }
-
- /**
- * The latest seqnum that the server writing to meta observed when opening the region.
- * E.g. the seqNum when the result of {@link #getServerName(Result)} was written.
- * @param r Result to pull the seqNum from
- * @return SeqNum, or HConstants.NO_SEQNUM if there's no value written.
- */
- private static long getSeqNumDuringOpen(final Result r, final int replicaId) {
- Cell cell = r.getColumnLatestCell(getFamily(), getSeqNumColumn(replicaId));
- if (cell == null || cell.getValueLength() == 0) return HConstants.NO_SEQNUM;
- return Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
- }
-
- /**
- * Returns an HRegionLocationList extracted from the result.
- * @return an HRegionLocationList containing all locations for the region range or null if
- * we can't deserialize the result.
- */
- public static RegionLocations getRegionLocations(final Result r) {
- if (r == null) return null;
- HRegionInfo regionInfo = getHRegionInfo(r, getRegionInfoColumn());
- if (regionInfo == null) return null;
-
- List<HRegionLocation> locations = new ArrayList<HRegionLocation>(1);
- NavigableMap<byte[],NavigableMap<byte[],byte[]>> familyMap = r.getNoVersionMap();
-
- locations.add(getRegionLocation(r, regionInfo, 0));
-
- NavigableMap<byte[], byte[]> infoMap = familyMap.get(getFamily());
- if (infoMap == null) return new RegionLocations(locations);
-
- // iterate until all serverName columns are seen
- int replicaId = 0;
- byte[] serverColumn = getServerColumn(replicaId);
- SortedMap<byte[], byte[]> serverMap = infoMap.tailMap(serverColumn, false);
- if (serverMap.isEmpty()) return new RegionLocations(locations);
-
- for (Entry<byte[], byte[]> entry : serverMap.entrySet()) {
- replicaId = parseReplicaIdFromServerColumn(entry.getKey());
- if (replicaId < 0) {
- break;
- }
-
- locations.add(getRegionLocation(r, regionInfo, replicaId));
- }
-
- return new RegionLocations(locations);
- }
-
- /**
- * Returns the HRegionLocation parsed from the given meta row Result
- * for the given regionInfo and replicaId. The regionInfo can be the default region info
- * for the replica.
- * @param r the meta row result
- * @param regionInfo RegionInfo for default replica
- * @param replicaId the replicaId for the HRegionLocation
- * @return HRegionLocation parsed from the given meta row Result for the given replicaId
- */
- private static HRegionLocation getRegionLocation(final Result r, final HRegionInfo regionInfo,
- final int replicaId) {
- ServerName serverName = getServerName(r, replicaId);
- long seqNum = getSeqNumDuringOpen(r, replicaId);
- HRegionInfo replicaInfo = RegionReplicaUtil.getRegionInfoForReplica(regionInfo, replicaId);
- return new HRegionLocation(replicaInfo, serverName, seqNum);
- }
-
- /**
- * Returns HRegionInfo object from the column
- * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog
- * table Result.
- * @param data a Result object from the catalog table scan
- * @return HRegionInfo or null
- */
- public static HRegionInfo getHRegionInfo(Result data) {
- return getHRegionInfo(data, HConstants.REGIONINFO_QUALIFIER);
- }
-
- /**
- * Returns the HRegionInfo object from the column {@link HConstants#CATALOG_FAMILY} and
- * <code>qualifier</code> of the catalog table result.
- * @param r a Result object from the catalog table scan
- * @param qualifier Column family qualifier
- * @return An HRegionInfo instance or null.
- */
- private static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier) {
- Cell cell = r.getColumnLatestCell(getFamily(), qualifier);
- if (cell == null) return null;
- return HRegionInfo.parseFromOrNull(cell.getValueArray(),
- cell.getValueOffset(), cell.getValueLength());
- }
-
- /**
- * Returns the daughter regions by reading the corresponding columns of the catalog table
- * Result.
- * @param data a Result object from the catalog table scan
- * @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
- * parent
- */
- public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
- HRegionInfo splitA = getHRegionInfo(data, HConstants.SPLITA_QUALIFIER);
- HRegionInfo splitB = getHRegionInfo(data, HConstants.SPLITB_QUALIFIER);
-
- return new PairOfSameType<HRegionInfo>(splitA, splitB);
- }
-
- /**
- * Returns the merge regions by reading the corresponding columns of the catalog table
- * Result.
- * @param data a Result object from the catalog table scan
- * @return a pair of HRegionInfo or PairOfSameType(null, null) if the region is not a split
- * parent
- */
- public static PairOfSameType<HRegionInfo> getMergeRegions(Result data) throws IOException {
- HRegionInfo mergeA = getHRegionInfo(data, HConstants.MERGEA_QUALIFIER);
- HRegionInfo mergeB = getHRegionInfo(data, HConstants.MERGEB_QUALIFIER);
-
- return new PairOfSameType<HRegionInfo>(mergeA, mergeB);
- }
-
- /**
- * Implementations 'visit' a catalog table row.
- */
- public interface Visitor {
- /**
- * Visit the catalog table row.
- * @param r A row from catalog table
- * @return True if we are to proceed scanning the table, else false if
- * we are to stop now.
- */
- boolean visit(final Result r) throws IOException;
- }
-
- /**
- * A {@link Visitor} that collects content out of passed {@link Result}.
- */
- static abstract class CollectingVisitor<T> implements Visitor {
- final List<T> results = new ArrayList<T>();
- @Override
- public boolean visit(Result r) throws IOException {
- if (r == null || r.isEmpty()) return true;
- add(r);
- return true;
- }
-
- abstract void add(Result r);
-
- /**
- * @return Collected results; wait till visits complete to collect all
- * possible results
- */
- List<T> getResults() {
- return this.results;
- }
- }
-
- /**
- * Collects all returned.
- */
- static class CollectAllVisitor extends CollectingVisitor<Result> {
- @Override
- void add(Result r) {
- this.results.add(r);
- }
- }
-
- /**
- * Count regions in <code>hbase:meta</code> for passed table.
- * @param c
- * @param tableName
- * @return Count or regions in table <code>tableName</code>
- * @throws IOException
- */
- public static int getRegionCount(final Configuration c, final String tableName) throws IOException {
- HTable t = new HTable(c, tableName);
- try {
- return t.getRegionLocations().size();
- } finally {
- t.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea085c63/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
index 2059f91..98c6a4d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.RegionTooBusyException;
import org.apache.hadoop.hbase.ServerName;
@@ -60,7 +61,6 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
-import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.AsyncProcess.AsyncRequestFuture;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitorBase;
@@ -1199,7 +1199,7 @@ class ConnectionManager {
}
// convert the row result into the HRegionLocation we need!
- RegionLocations locations = MetaReader.getRegionLocations(regionInfoRow);
+ RegionLocations locations = MetaTableAccessor.getRegionLocations(regionInfoRow);
if (locations == null || locations.getRegionLocation(replicaId) == null) {
throw new IOException("HRegionInfo was null in " +
tableName + ", row=" + regionInfoRow);