You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/12/11 03:34:24 UTC
[01/16] hbase git commit: HBASE-21541 Move
MetaTableLocator.verifyRegionLocation to hbase-rsgroup module [Forced
Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-21512 22f406c79 -> e7a122780 (forced update)
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java
----------------------------------------------------------------------
diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java
index 7f0fb11..0cebc76 100644
--- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java
+++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,75 +17,55 @@
*/
package org.apache.hadoop.hbase.zookeeper;
-import java.io.EOFException;
-import java.io.IOException;
-import java.net.ConnectException;
-import java.net.NoRouteToHostException;
-import java.net.SocketException;
-import java.net.SocketTimeoutException;
-import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Locale;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
-import org.apache.hadoop.hbase.client.RetriesExhaustedException;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.ipc.FailedServerException;
-import org.apache.hadoop.hbase.ipc.HBaseRpcController;
-import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.master.RegionState;
-import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.ipc.RemoteException;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.MetaRegionServer;
/**
- * Utility class to perform operation (get/wait for/verify/set/delete) on znode in ZooKeeper
- * which keeps hbase:meta region server location.
- *
- * Stateless class with a bunch of static methods. Doesn't manage resources passed in
- * (e.g. Connection, ZKWatcher etc).
- *
- * Meta region location is set by <code>RegionServerServices</code>.
- * This class doesn't use ZK watchers, rather accesses ZK directly.
- *
- * This class it stateless. The only reason it's not made a non-instantiable util class
- * with a collection of static methods is that it'd be rather hard to mock properly in tests.
- *
+ * Utility class to perform operation (get/wait for/verify/set/delete) on znode in ZooKeeper which
+ * keeps hbase:meta region server location.
+ * <p/>
+ * Stateless class with a bunch of static methods. Doesn't manage resources passed in (e.g.
+ * Connection, ZKWatcher etc).
+ * <p/>
+ * Meta region location is set by <code>RegionServerServices</code>. This class doesn't use ZK
+ * watchers, rather accesses ZK directly.
+ * <p/>
* TODO: rewrite using RPC calls to master to find out about hbase:meta.
*/
@InterfaceAudience.Private
-public class MetaTableLocator {
+public final class MetaTableLocator {
private static final Logger LOG = LoggerFactory.getLogger(MetaTableLocator.class);
- // only needed to allow non-timeout infinite waits to stop when cluster shuts down
- private volatile boolean stopped = false;
+ private MetaTableLocator() {
+ }
/**
* Checks if the meta region location is available.
* @return true if meta region location is available, false if not
*/
- public boolean isLocationAvailable(ZKWatcher zkw) {
+ public static boolean isLocationAvailable(ZKWatcher zkw) {
return getMetaRegionLocation(zkw) != null;
}
@@ -93,7 +73,7 @@ public class MetaTableLocator {
* @param zkw ZooKeeper watcher to be used
* @return meta table regions and their locations.
*/
- public List<Pair<RegionInfo, ServerName>> getMetaRegionsAndLocations(ZKWatcher zkw) {
+ public static List<Pair<RegionInfo, ServerName>> getMetaRegionsAndLocations(ZKWatcher zkw) {
return getMetaRegionsAndLocations(zkw, RegionInfo.DEFAULT_REPLICA_ID);
}
@@ -104,7 +84,7 @@ public class MetaTableLocator {
* @param replicaId the ID of the replica
* @return meta table regions and their locations.
*/
- public List<Pair<RegionInfo, ServerName>> getMetaRegionsAndLocations(ZKWatcher zkw,
+ public static List<Pair<RegionInfo, ServerName>> getMetaRegionsAndLocations(ZKWatcher zkw,
int replicaId) {
ServerName serverName = getMetaRegionLocation(zkw, replicaId);
List<Pair<RegionInfo, ServerName>> list = new ArrayList<>(1);
@@ -119,30 +99,30 @@ public class MetaTableLocator {
* @param zkw ZooKeeper watcher to be used
* @return List of meta regions
*/
- public List<RegionInfo> getMetaRegions(ZKWatcher zkw) {
+ public static List<RegionInfo> getMetaRegions(ZKWatcher zkw) {
return getMetaRegions(zkw, RegionInfo.DEFAULT_REPLICA_ID);
}
/**
* Gets the meta regions for the given path and replica ID.
- *
* @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
* @param replicaId the ID of the replica
* @return List of meta regions
*/
- public List<RegionInfo> getMetaRegions(ZKWatcher zkw, int replicaId) {
+ public static List<RegionInfo> getMetaRegions(ZKWatcher zkw, int replicaId) {
List<Pair<RegionInfo, ServerName>> result;
result = getMetaRegionsAndLocations(zkw, replicaId);
return getListOfRegionInfos(result);
}
- private List<RegionInfo> getListOfRegionInfos(final List<Pair<RegionInfo, ServerName>> pairs) {
+ private static List<RegionInfo> getListOfRegionInfos(
+ final List<Pair<RegionInfo, ServerName>> pairs) {
if (pairs == null || pairs.isEmpty()) {
- return Collections.EMPTY_LIST;
+ return Collections.emptyList();
}
List<RegionInfo> result = new ArrayList<>(pairs.size());
- for (Pair<RegionInfo, ServerName> pair: pairs) {
+ for (Pair<RegionInfo, ServerName> pair : pairs) {
result.add(pair.getFirst());
}
return result;
@@ -153,7 +133,7 @@ public class MetaTableLocator {
* @param zkw zookeeper connection to use
* @return server name or null if we failed to get the data.
*/
- public ServerName getMetaRegionLocation(final ZKWatcher zkw) {
+ public static ServerName getMetaRegionLocation(final ZKWatcher zkw) {
try {
RegionState state = getMetaRegionState(zkw);
return state.isOpened() ? state.getServerName() : null;
@@ -168,7 +148,7 @@ public class MetaTableLocator {
* @param replicaId the ID of the replica
* @return server name
*/
- public ServerName getMetaRegionLocation(final ZKWatcher zkw, int replicaId) {
+ public static ServerName getMetaRegionLocation(final ZKWatcher zkw, int replicaId) {
try {
RegionState state = getMetaRegionState(zkw, replicaId);
return state.isOpened() ? state.getServerName() : null;
@@ -178,42 +158,39 @@ public class MetaTableLocator {
}
/**
- * Gets the meta region location, if available, and waits for up to the
- * specified timeout if not immediately available.
- * Given the zookeeper notification could be delayed, we will try to
- * get the latest data.
- *
+ * Gets the meta region location, if available, and waits for up to the specified timeout if not
+ * immediately available. Given the zookeeper notification could be delayed, we will try to get
+ * the latest data.
* @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
* @param timeout maximum time to wait, in millis
- * @return server name for server hosting meta region formatted as per
- * {@link ServerName}, or null if none available
+ * @return server name for server hosting meta region formatted as per {@link ServerName}, or null
+ * if none available
* @throws InterruptedException if interrupted while waiting
* @throws NotAllMetaRegionsOnlineException if a meta or root region is not online
*/
- public ServerName waitMetaRegionLocation(ZKWatcher zkw, long timeout)
- throws InterruptedException, NotAllMetaRegionsOnlineException {
+ public static ServerName waitMetaRegionLocation(ZKWatcher zkw, long timeout)
+ throws InterruptedException, NotAllMetaRegionsOnlineException {
return waitMetaRegionLocation(zkw, RegionInfo.DEFAULT_REPLICA_ID, timeout);
}
/**
* Gets the meta region location, if available, and waits for up to the specified timeout if not
- * immediately available. Given the zookeeper notification could be delayed, we will try to
- * get the latest data.
- *
+ * immediately available. Given the zookeeper notification could be delayed, we will try to get
+ * the latest data.
* @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
* @param replicaId the ID of the replica
* @param timeout maximum time to wait, in millis
- * @return server name for server hosting meta region formatted as per
- * {@link ServerName}, or null if none available
+ * @return server name for server hosting meta region formatted as per {@link ServerName}, or null
+ * if none available
* @throws InterruptedException if waiting for the socket operation fails
* @throws NotAllMetaRegionsOnlineException if a meta or root region is not online
*/
- public ServerName waitMetaRegionLocation(ZKWatcher zkw, int replicaId, long timeout)
- throws InterruptedException, NotAllMetaRegionsOnlineException {
+ public static ServerName waitMetaRegionLocation(ZKWatcher zkw, int replicaId, long timeout)
+ throws InterruptedException, NotAllMetaRegionsOnlineException {
try {
if (ZKUtil.checkExists(zkw, zkw.getZNodePaths().baseZNode) == -1) {
- String errorMsg = "Check the value configured in 'zookeeper.znode.parent'. "
- + "There could be a mismatch with the one configured in the master.";
+ String errorMsg = "Check the value configured in 'zookeeper.znode.parent'. " +
+ "There could be a mismatch with the one configured in the master.";
LOG.error(errorMsg);
throw new IllegalArgumentException(errorMsg);
}
@@ -230,198 +207,6 @@ public class MetaTableLocator {
}
/**
- * Waits indefinitely for availability of <code>hbase:meta</code>. Used during
- * cluster startup. Does not verify meta, just that something has been
- * set up in zk.
- * @see #waitMetaRegionLocation(ZKWatcher, long)
- * @throws InterruptedException if interrupted while waiting
- */
- public void waitMetaRegionLocation(ZKWatcher zkw) throws InterruptedException {
- long startTime = System.currentTimeMillis();
- while (!stopped) {
- try {
- if (waitMetaRegionLocation(zkw, 100) != null) {
- break;
- }
-
- long sleepTime = System.currentTimeMillis() - startTime;
- // +1 in case sleepTime=0
- if ((sleepTime + 1) % 10000 == 0) {
- LOG.warn("Have been waiting for meta to be assigned for " + sleepTime + "ms");
- }
- } catch (NotAllMetaRegionsOnlineException e) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("hbase:meta still not available, sleeping and retrying." +
- " Reason: " + e.getMessage());
- }
- }
- }
- }
-
- /**
- * Verify <code>hbase:meta</code> is deployed and accessible.
- *
- * @param hConnection the connection to use
- * @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
- * @param timeout How long to wait on zk for meta address (passed through to
- * the internal call to {@link #getMetaServerConnection}.
- * @return True if the <code>hbase:meta</code> location is healthy.
- * @throws IOException if the number of retries for getting the connection is exceeded
- * @throws InterruptedException if waiting for the socket operation fails
- */
- public boolean verifyMetaRegionLocation(ClusterConnection hConnection, ZKWatcher zkw,
- final long timeout) throws InterruptedException, IOException {
- return verifyMetaRegionLocation(hConnection, zkw, timeout, RegionInfo.DEFAULT_REPLICA_ID);
- }
-
- /**
- * Verify <code>hbase:meta</code> is deployed and accessible.
- *
- * @param connection the connection to use
- * @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
- * @param timeout How long to wait on zk for meta address (passed through to
- * @param replicaId the ID of the replica
- * @return True if the <code>hbase:meta</code> location is healthy.
- * @throws InterruptedException if waiting for the socket operation fails
- * @throws IOException if the number of retries for getting the connection is exceeded
- */
- public boolean verifyMetaRegionLocation(ClusterConnection connection, ZKWatcher zkw,
- final long timeout, int replicaId) throws InterruptedException, IOException {
- AdminProtos.AdminService.BlockingInterface service = null;
- try {
- service = getMetaServerConnection(connection, zkw, timeout, replicaId);
- } catch (NotAllMetaRegionsOnlineException e) {
- // Pass
- } catch (ServerNotRunningYetException e) {
- // Pass -- remote server is not up so can't be carrying root
- } catch (UnknownHostException e) {
- // Pass -- server name doesn't resolve so it can't be assigned anything.
- } catch (RegionServerStoppedException e) {
- // Pass -- server name sends us to a server that is dying or already dead.
- }
- return (service != null) && verifyRegionLocation(connection, service,
- getMetaRegionLocation(zkw, replicaId), RegionReplicaUtil.getRegionInfoForReplica(
- RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId).getRegionName());
- }
-
- /**
- * Verify we can connect to <code>hostingServer</code> and that its carrying
- * <code>regionName</code>.
- * @param hostingServer Interface to the server hosting <code>regionName</code>
- * @param address The servername that goes with the <code>metaServer</code> interface.
- * Used logging.
- * @param regionName The regionname we are interested in.
- * @return True if we were able to verify the region located at other side of the interface.
- */
- // TODO: We should be able to get the ServerName from the AdminProtocol
- // rather than have to pass it in. Its made awkward by the fact that the
- // HRI is likely a proxy against remote server so the getServerName needs
- // to be fixed to go to a local method or to a cache before we can do this.
- private boolean verifyRegionLocation(final ClusterConnection connection,
- AdminService.BlockingInterface hostingServer, final ServerName address,
- final byte [] regionName) {
- if (hostingServer == null) {
- LOG.info("Passed hostingServer is null");
- return false;
- }
- Throwable t;
- HBaseRpcController controller = connection.getRpcControllerFactory().newController();
- try {
- // Try and get regioninfo from the hosting server.
- return ProtobufUtil.getRegionInfo(controller, hostingServer, regionName) != null;
- } catch (ConnectException e) {
- t = e;
- } catch (RetriesExhaustedException e) {
- t = e;
- } catch (RemoteException e) {
- IOException ioe = e.unwrapRemoteException();
- t = ioe;
- } catch (IOException e) {
- Throwable cause = e.getCause();
- if (cause != null && cause instanceof EOFException) {
- t = cause;
- } else if (cause != null && cause.getMessage() != null
- && cause.getMessage().contains("Connection reset")) {
- t = cause;
- } else {
- t = e;
- }
- }
- LOG.info("Failed verification of " + Bytes.toStringBinary(regionName) +
- " at address=" + address + ", exception=" + t.getMessage());
- return false;
- }
-
- /**
- * Gets a connection to the server hosting meta, as reported by ZooKeeper, waiting up to the
- * specified timeout for availability.
- *
- * <p>WARNING: Does not retry. Use an {@link org.apache.hadoop.hbase.client.HTable} instead.
- *
- * @param connection the connection to use
- * @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
- * @param timeout How long to wait on meta location
- * @param replicaId the ID of the replica
- * @return connection to server hosting meta
- * @throws InterruptedException if waiting for the socket operation fails
- * @throws IOException if the number of retries for getting the connection is exceeded
- */
- private AdminService.BlockingInterface getMetaServerConnection(ClusterConnection connection,
- ZKWatcher zkw, long timeout, int replicaId) throws InterruptedException, IOException {
- return getCachedConnection(connection, waitMetaRegionLocation(zkw, replicaId, timeout));
- }
-
- /**
- * @param sn ServerName to get a connection against.
- * @return The AdminProtocol we got when we connected to <code>sn</code>
- * May have come from cache, may not be good, may have been setup by this invocation, or
- * may be null.
- * @throws IOException if the number of retries for getting the connection is exceeded
- */
- private static AdminService.BlockingInterface getCachedConnection(ClusterConnection connection,
- ServerName sn) throws IOException {
- if (sn == null) {
- return null;
- }
- AdminService.BlockingInterface service = null;
- try {
- service = connection.getAdmin(sn);
- } catch (RetriesExhaustedException e) {
- if (e.getCause() != null && e.getCause() instanceof ConnectException) {
- LOG.debug("Catch this; presume it means the cached connection has gone bad.");
- } else {
- throw e;
- }
- } catch (SocketTimeoutException e) {
- LOG.debug("Timed out connecting to " + sn);
- } catch (NoRouteToHostException e) {
- LOG.debug("Connecting to " + sn, e);
- } catch (SocketException e) {
- LOG.debug("Exception connecting to " + sn);
- } catch (UnknownHostException e) {
- LOG.debug("Unknown host exception connecting to " + sn);
- } catch (FailedServerException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Server " + sn + " is in failed server list.");
- }
- } catch (IOException ioe) {
- Throwable cause = ioe.getCause();
- if (ioe instanceof ConnectException) {
- LOG.debug("Catch. Connect refused.");
- } else if (cause != null && cause instanceof EOFException) {
- LOG.debug("Catch. Other end disconnected us.");
- } else if (cause != null && cause.getMessage() != null &&
- cause.getMessage().toLowerCase(Locale.ROOT).contains("connection reset")) {
- LOG.debug("Catch. Connection reset.");
- } else {
- throw ioe;
- }
-
- }
- return service;
- }
-
- /**
* Sets the location of <code>hbase:meta</code> in ZooKeeper to the
* specified server address.
* @param zookeeper zookeeper reference
@@ -498,7 +283,7 @@ public class MetaTableLocator {
try {
int prefixLen = ProtobufUtil.lengthOfPBMagic();
ZooKeeperProtos.MetaRegionServer rl =
- ZooKeeperProtos.MetaRegionServer.PARSER.parseFrom(data, prefixLen,
+ ZooKeeperProtos.MetaRegionServer.parser().parseFrom(data, prefixLen,
data.length - prefixLen);
if (rl.hasState()) {
state = RegionState.State.convert(rl.getState());
@@ -532,12 +317,12 @@ public class MetaTableLocator {
* @param zookeeper zookeeper reference
* @throws KeeperException unexpected zookeeper exception
*/
- public void deleteMetaLocation(ZKWatcher zookeeper)
+ public static void deleteMetaLocation(ZKWatcher zookeeper)
throws KeeperException {
deleteMetaLocation(zookeeper, RegionInfo.DEFAULT_REPLICA_ID);
}
- public void deleteMetaLocation(ZKWatcher zookeeper, int replicaId)
+ public static void deleteMetaLocation(ZKWatcher zookeeper, int replicaId)
throws KeeperException {
if (replicaId == RegionInfo.DEFAULT_REPLICA_ID) {
LOG.info("Deleting hbase:meta region location in ZooKeeper");
@@ -561,7 +346,7 @@ public class MetaTableLocator {
* @return ServerName or null if we timed out.
* @throws InterruptedException if waiting for the socket operation fails
*/
- public List<ServerName> blockUntilAvailable(final ZKWatcher zkw, final long timeout,
+ public static List<ServerName> blockUntilAvailable(final ZKWatcher zkw, final long timeout,
Configuration conf) throws InterruptedException {
int numReplicasConfigured = 1;
@@ -596,22 +381,21 @@ public class MetaTableLocator {
* @return ServerName or null if we timed out.
* @throws InterruptedException if waiting for the socket operation fails
*/
- public ServerName blockUntilAvailable(final ZKWatcher zkw, final long timeout)
- throws InterruptedException {
+ public static ServerName blockUntilAvailable(final ZKWatcher zkw, final long timeout)
+ throws InterruptedException {
return blockUntilAvailable(zkw, RegionInfo.DEFAULT_REPLICA_ID, timeout);
}
/**
* Wait until the meta region is available and is not in transition.
- *
* @param zkw reference to the {@link ZKWatcher} which also contains configuration and constants
* @param replicaId the ID of the replica
* @param timeout maximum time to wait in millis
* @return ServerName or null if we timed out.
* @throws InterruptedException if waiting for the socket operation fails
*/
- public ServerName blockUntilAvailable(final ZKWatcher zkw, int replicaId, final long timeout)
- throws InterruptedException {
+ public static ServerName blockUntilAvailable(final ZKWatcher zkw, int replicaId,
+ final long timeout) throws InterruptedException {
if (timeout < 0) {
throw new IllegalArgumentException();
}
@@ -624,23 +408,12 @@ public class MetaTableLocator {
ServerName sn = null;
while (true) {
sn = getMetaRegionLocation(zkw, replicaId);
- if (sn != null || (System.currentTimeMillis() - startTime)
- > timeout - HConstants.SOCKET_RETRY_WAIT_MS) {
+ if (sn != null ||
+ (System.currentTimeMillis() - startTime) > timeout - HConstants.SOCKET_RETRY_WAIT_MS) {
break;
}
Thread.sleep(HConstants.SOCKET_RETRY_WAIT_MS);
}
return sn;
}
-
- /**
- * Stop working.
- * Interrupts any ongoing waits.
- */
- public void stop() {
- if (!stopped) {
- LOG.debug("Stopping MetaTableLocator");
- stopped = true;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
----------------------------------------------------------------------
diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index 9153191..57c847c 100644
--- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -34,10 +34,8 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.AuthUtil;
@@ -75,6 +73,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
@@ -1748,13 +1747,13 @@ public final class ZKUtil {
sb.append("\n ").append(child);
}
sb.append("\nRegion server holding hbase:meta: "
- + new MetaTableLocator().getMetaRegionLocation(zkw));
+ + MetaTableLocator.getMetaRegionLocation(zkw));
Configuration conf = HBaseConfiguration.create();
int numMetaReplicas = conf.getInt(HConstants.META_REPLICAS_NUM,
HConstants.DEFAULT_META_REPLICA_NUM);
for (int i = 1; i < numMetaReplicas; i++) {
sb.append("\nRegion server holding hbase:meta, replicaId " + i + " "
- + new MetaTableLocator().getMetaRegionLocation(zkw, i));
+ + MetaTableLocator.getMetaRegionLocation(zkw, i));
}
sb.append("\nRegion servers:");
for (String child : listChildrenNoWatch(zkw, zkw.getZNodePaths().rsZNode)) {
[13/16] hbase git commit: HBASE-21567 Allow overriding configs
starting up the shell
Posted by zh...@apache.org.
HBASE-21567 Allow overriding configs starting up the shell
Adds support for -D as option to 'hbase shell'
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/da9508d4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/da9508d4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/da9508d4
Branch: refs/heads/HBASE-21512
Commit: da9508d4271ea12410e289692f10791b0e05266b
Parents: 79d90c8
Author: stack <st...@apache.org>
Authored: Thu Dec 6 23:05:21 2018 -0800
Committer: stack <st...@apache.org>
Committed: Sat Dec 8 15:08:19 2018 -0800
----------------------------------------------------------------------
bin/hirb.rb | 40 ++++++++++++++++++++++++-----
src/main/asciidoc/_chapters/shell.adoc | 16 ++++++++++++
2 files changed, 49 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/da9508d4/bin/hirb.rb
----------------------------------------------------------------------
diff --git a/bin/hirb.rb b/bin/hirb.rb
index 790ecdc..e857db7 100644
--- a/bin/hirb.rb
+++ b/bin/hirb.rb
@@ -54,21 +54,47 @@ $LOAD_PATH.unshift Pathname.new(sources)
cmdline_help = <<HERE # HERE document output as shell usage
Usage: shell [OPTIONS] [SCRIPTFILE [ARGUMENTS]]
- -d | --debug Set DEBUG log levels.
- -h | --help This help.
- -n | --noninteractive Do not run within an IRB session
- and exit with non-zero status on
- first error.
+ -d | --debug Set DEBUG log levels.
+ -h | --help This help.
+ -n | --noninteractive Do not run within an IRB session and exit with non-zero
+ status on first error.
+ -Dkey=value Pass hbase-*.xml Configuration overrides. For example, to
+ use an alternate zookeeper ensemble, pass:
+ -Dhbase.zookeeper.quorum=zookeeper.example.org
+ For faster fail, pass the below and vary the values:
+ -Dhbase.client.retries.number=7
+ -Dhbase.ipc.client.connect.max.retries=3
HERE
+
+# Takes configuration and an arg that is expected to be key=value format.
+# If c is empty, creates one and returns it
+def add_to_configuration(c, arg)
+ kv = arg.split('=')
+ kv.length == 2 || (raise "Expected parameter #{kv} in key=value format")
+ c = org.apache.hadoop.hbase.HBaseConfiguration.create if c.nil?
+ c.set(kv[0], kv[1])
+ c
+end
+
found = []
script2run = nil
log_level = org.apache.log4j.Level::ERROR
@shell_debug = false
interactive = true
-for arg in ARGV
+_configuration = nil
+D_ARG = '-D'
+while (arg = ARGV.shift)
if arg == '-h' || arg == '--help'
puts cmdline_help
exit
+ elsif arg == D_ARG
+ argValue = ARGV.shift || (raise "#{D_ARG} takes a 'key=value' parameter")
+ _configuration = add_to_configuration(_configuration, argValue)
+ found.push(arg)
+ found.push(argValue)
+ elsif arg.start_with? D_ARG
+ _configuration = add_to_configuration(_configuration, arg[2..-1])
+ found.push(arg)
elsif arg == '-d' || arg == '--debug'
log_level = org.apache.log4j.Level::DEBUG
$fullBackTrace = true
@@ -111,7 +137,7 @@ require 'shell'
require 'shell/formatter'
# Setup the HBase module. Create a configuration.
-@hbase = Hbase::Hbase.new
+@hbase = _configuration.nil? ? Hbase::Hbase.new : Hbase::Hbase.new(_configuration)
# Setup console
@shell = Shell::Shell.new(@hbase, interactive)
http://git-wip-us.apache.org/repos/asf/hbase/blob/da9508d4/src/main/asciidoc/_chapters/shell.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/shell.adoc b/src/main/asciidoc/_chapters/shell.adoc
index 5612e1d..cdfa828 100644
--- a/src/main/asciidoc/_chapters/shell.adoc
+++ b/src/main/asciidoc/_chapters/shell.adoc
@@ -58,6 +58,7 @@ To run one of these files, do as follows:
$ ./bin/hbase org.jruby.Main PATH_TO_SCRIPT
----
+
== Running the Shell in Non-Interactive Mode
A new non-interactive mode has been added to the HBase Shell (link:https://issues.apache.org/jira/browse/HBASE-11658[HBASE-11658)].
@@ -213,6 +214,21 @@ $ HBASE_SHELL_OPTS="-verbose:gc -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCD
-XX:+PrintGCDetails -Xloggc:$HBASE_HOME/logs/gc-hbase.log" ./bin/hbase shell
----
+== Overriding configuration starting the HBase Shell
+
+As of hbase-2.0.5/hbase-2.1.3/hbase-2.2.0/hbase-1.4.10/hbase-1.5.0, you can
+pass or override hbase configuration as specified in `hbase-*.xml` by passing
+your key/values prefixed with `-D` on the command-line as follows:
+[source,bash]
+----
+$ ./bin/hbase shell -Dhbase.zookeeper.quorum=ZK0.remote.cluster.example.org,ZK1.remote.cluster.example.org,ZK2.remote.cluster.example.org -Draining=false
+...
+hbase(main):001:0> @shell.hbase.configuration.get("hbase.zookeeper.quorum")
+=> "ZK0.remote.cluster.example.org,ZK1.remote.cluster.example.org,ZK2.remote.cluster.example.org"
+hbase(main):002:0> @shell.hbase.configuration.get("raining")
+=> "false"
+----
+
== Shell Tricks
=== Table variables
[08/16] hbase git commit: HBASE-21559 The
RestoreSnapshotFromClientTestBase related UT are flaky
Posted by zh...@apache.org.
HBASE-21559 The RestoreSnapshotFromClientTestBase related UT are flaky
Signed-off-by: zhangduo <zh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dfb9ae8e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dfb9ae8e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dfb9ae8e
Branch: refs/heads/HBASE-21512
Commit: dfb9ae8e0e69ad84962b8768190f6891827767fa
Parents: 170df27
Author: huzheng <op...@gmail.com>
Authored: Thu Dec 6 20:35:30 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Fri Dec 7 08:22:22 2018 +0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/master/snapshot/SnapshotManager.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dfb9ae8e/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
index ae9b6fb..2b963b2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
@@ -27,6 +27,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -150,7 +151,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
// The map is always accessed and modified under the object lock using synchronized.
// snapshotTable() will insert an Handler in the table.
// isSnapshotDone() will remove the handler requested if the operation is finished.
- private Map<TableName, SnapshotSentinel> snapshotHandlers = new HashMap<>();
+ private Map<TableName, SnapshotSentinel> snapshotHandlers = new ConcurrentHashMap<>();
// Restore map, with table name as key, procedure ID as value.
// The map is always accessed and modified under the object lock using synchronized.
@@ -419,7 +420,7 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
* @param tableName name of the table being snapshotted.
* @return <tt>true</tt> if there is a snapshot in progress on the specified table.
*/
- public synchronized boolean isTakingSnapshot(final TableName tableName) {
+ public boolean isTakingSnapshot(final TableName tableName) {
SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
return handler != null && !handler.isFinished();
}
[14/16] hbase git commit: HBASE-21515 Also initialize an
AsyncClusterConnection in HRegionServer
Posted by zh...@apache.org.
HBASE-21515 Also initialize an AsyncClusterConnection in HRegionServer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/054b4bab
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/054b4bab
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/054b4bab
Branch: refs/heads/HBASE-21512
Commit: 054b4bab5d6545429dad22d6b6d50951c3448e00
Parents: da9508d
Author: zhangduo <zh...@apache.org>
Authored: Fri Nov 30 08:23:47 2018 +0800
Committer: Duo Zhang <zh...@apache.org>
Committed: Tue Dec 11 11:17:11 2018 +0800
----------------------------------------------------------------------
.../hbase/client/AsyncClusterConnection.java | 38 ++++++++++++
.../hbase/client/AsyncConnectionImpl.java | 39 ++++++------
.../hbase/client/ClusterConnectionFactory.java | 63 ++++++++++++++++++++
.../hadoop/hbase/client/ConnectionFactory.java | 5 +-
.../hadoop/hbase/util/ReflectionUtils.java | 22 ++++---
.../java/org/apache/hadoop/hbase/Server.java | 20 +++++++
.../org/apache/hadoop/hbase/master/HMaster.java | 3 +
.../hbase/regionserver/HRegionServer.java | 56 ++++++++++++-----
.../regionserver/ReplicationSyncUp.java | 6 ++
.../hadoop/hbase/MockRegionServerServices.java | 6 ++
.../client/TestAsyncNonMetaRegionLocator.java | 2 +-
...syncNonMetaRegionLocatorConcurrenyLimit.java | 2 +-
.../client/TestAsyncRegionLocatorTimeout.java | 2 +-
...TestAsyncSingleRequestRpcRetryingCaller.java | 4 +-
.../hbase/client/TestAsyncTableNoncedRetry.java | 2 +-
.../hbase/master/MockNoopMasterServices.java | 6 ++
.../hadoop/hbase/master/MockRegionServer.java | 6 ++
.../hbase/master/TestActiveMasterManager.java | 6 ++
.../hbase/master/cleaner/TestHFileCleaner.java | 6 ++
.../master/cleaner/TestHFileLinkCleaner.java | 6 ++
.../hbase/master/cleaner/TestLogsCleaner.java | 6 ++
.../cleaner/TestReplicationHFileCleaner.java | 6 ++
.../regionserver/TestHeapMemoryManager.java | 6 ++
.../hbase/regionserver/TestSplitLogWorker.java | 6 ++
.../hbase/regionserver/TestWALLockup.java | 6 ++
.../TestReplicationTrackerZKImpl.java | 6 ++
.../TestReplicationSourceManager.java | 6 ++
.../security/token/TestTokenAuthentication.java | 6 ++
.../apache/hadoop/hbase/util/MockServer.java | 6 ++
29 files changed, 304 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
new file mode 100644
index 0000000..c7dea25
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import org.apache.hadoop.hbase.ipc.RpcClient;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * The asynchronous connection for internal usage.
+ */
+@InterfaceAudience.Private
+public interface AsyncClusterConnection extends AsyncConnection {
+
+ /**
+ * Get the nonce generator for this connection.
+ */
+ NonceGenerator getNonceGenerator();
+
+ /**
+ * Get the rpc client we used to communicate with other servers.
+ */
+ RpcClient getRpcClient();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
index a05764e..00c8f55 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
@@ -21,48 +21,48 @@ import static org.apache.hadoop.hbase.client.ConnectionUtils.NO_NONCE_GENERATOR;
import static org.apache.hadoop.hbase.client.ConnectionUtils.getStubKey;
import static org.apache.hadoop.hbase.client.NonceGenerator.CLIENT_NONCES_ENABLED_KEY;
-import org.apache.hadoop.hbase.AuthUtil;
-import org.apache.hadoop.hbase.ChoreService;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-
-import org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer;
-
import java.io.IOException;
+import java.net.SocketAddress;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.AuthUtil;
+import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.ipc.HBaseRpcController;
import org.apache.hadoop.hbase.ipc.RpcClient;
import org.apache.hadoop.hbase.ipc.RpcClientFactory;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.security.User;
+import org.apache.hadoop.hbase.util.CollectionUtils;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
+import org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer;
+
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ClientService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsMasterRunningResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MasterService;
-import org.apache.hadoop.hbase.util.CollectionUtils;
-import org.apache.hadoop.hbase.util.Threads;
/**
* The implementation of AsyncConnection.
*/
@InterfaceAudience.Private
-class AsyncConnectionImpl implements AsyncConnection {
+class AsyncConnectionImpl implements AsyncClusterConnection {
private static final Logger LOG = LoggerFactory.getLogger(AsyncConnectionImpl.class);
@@ -105,7 +105,7 @@ class AsyncConnectionImpl implements AsyncConnection {
private ChoreService authService;
public AsyncConnectionImpl(Configuration conf, AsyncRegistry registry, String clusterId,
- User user) {
+ SocketAddress localAddress, User user) {
this.conf = conf;
this.user = user;
if (user.isLoginFromKeytab()) {
@@ -113,7 +113,7 @@ class AsyncConnectionImpl implements AsyncConnection {
}
this.connConf = new AsyncConnectionConfiguration(conf);
this.registry = registry;
- this.rpcClient = RpcClientFactory.createClient(conf, clusterId);
+ this.rpcClient = RpcClientFactory.createClient(conf, clusterId, localAddress, null);
this.rpcControllerFactory = RpcControllerFactory.instantiate(conf);
this.hostnameCanChange = conf.getBoolean(RESOLVE_HOSTNAME_ON_FAIL_KEY, true);
this.rpcTimeout =
@@ -158,11 +158,16 @@ class AsyncConnectionImpl implements AsyncConnection {
}
// ditto
- @VisibleForTesting
+ @Override
public NonceGenerator getNonceGenerator() {
return nonceGenerator;
}
+ @Override
+ public RpcClient getRpcClient() {
+ return rpcClient;
+ }
+
private ClientService.Interface createRegionServerStub(ServerName serverName) throws IOException {
return ClientService.newStub(rpcClient.createRpcChannel(serverName, user, rpcTimeout));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java
new file mode 100644
index 0000000..68c0630
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClusterConnectionFactory.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.SocketAddress;
+import java.util.concurrent.ExecutionException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
+
+/**
+ * The factory for creating {@link AsyncClusterConnection}.
+ */
+@InterfaceAudience.Private
+public final class ClusterConnectionFactory {
+
+ private ClusterConnectionFactory() {
+ }
+
+ /**
+ * Create a new {@link AsyncClusterConnection} instance.
+ * <p/>
+ * Unlike what we have done in {@link ConnectionFactory}, here we just return an
+ * {@link AsyncClusterConnection} instead of a {@link java.util.concurrent.CompletableFuture},
+ * which means this method could block on fetching the cluster id. This is just used to simplify
+ * the implementation, as when starting new region servers, we do not need to be event-driven. Can
+ * change later if we want a {@link java.util.concurrent.CompletableFuture} here.
+ */
+ public static AsyncClusterConnection createAsyncClusterConnection(Configuration conf,
+ SocketAddress localAddress, User user) throws IOException {
+ AsyncRegistry registry = AsyncRegistryFactory.getRegistry(conf);
+ String clusterId;
+ try {
+ clusterId = registry.getClusterId().get();
+ } catch (InterruptedException e) {
+ throw (IOException) new InterruptedIOException().initCause(e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ Throwables.propagateIfPossible(cause, IOException.class);
+ throw new IOException(cause);
+ }
+ return new AsyncConnectionImpl(conf, registry, clusterId, localAddress, user);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java
index e24af74..2ba732a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionFactory.java
@@ -295,9 +295,8 @@ public class ConnectionFactory {
AsyncConnectionImpl.class, AsyncConnection.class);
try {
future.complete(
- user.runAs((PrivilegedExceptionAction<? extends AsyncConnection>)() ->
- ReflectionUtils.newInstance(clazz, conf, registry, clusterId, user))
- );
+ user.runAs((PrivilegedExceptionAction<? extends AsyncConnection>) () -> ReflectionUtils
+ .newInstance(clazz, conf, registry, clusterId, null, user)));
} catch (Exception e) {
future.completeExceptionally(e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectionUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectionUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectionUtils.java
index a136846..268249d 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectionUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectionUtils.java
@@ -83,15 +83,19 @@ public class ReflectionUtils {
boolean match = true;
for (int i = 0; i < ctorParamTypes.length && match; ++i) {
- Class<?> paramType = paramTypes[i].getClass();
- match = (!ctorParamTypes[i].isPrimitive()) ? ctorParamTypes[i].isAssignableFrom(paramType) :
- ((int.class.equals(ctorParamTypes[i]) && Integer.class.equals(paramType)) ||
- (long.class.equals(ctorParamTypes[i]) && Long.class.equals(paramType)) ||
- (double.class.equals(ctorParamTypes[i]) && Double.class.equals(paramType)) ||
- (char.class.equals(ctorParamTypes[i]) && Character.class.equals(paramType)) ||
- (short.class.equals(ctorParamTypes[i]) && Short.class.equals(paramType)) ||
- (boolean.class.equals(ctorParamTypes[i]) && Boolean.class.equals(paramType)) ||
- (byte.class.equals(ctorParamTypes[i]) && Byte.class.equals(paramType)));
+ if (paramTypes[i] == null) {
+ match = !ctorParamTypes[i].isPrimitive();
+ } else {
+ Class<?> paramType = paramTypes[i].getClass();
+ match = (!ctorParamTypes[i].isPrimitive()) ? ctorParamTypes[i].isAssignableFrom(paramType)
+ : ((int.class.equals(ctorParamTypes[i]) && Integer.class.equals(paramType)) ||
+ (long.class.equals(ctorParamTypes[i]) && Long.class.equals(paramType)) ||
+ (double.class.equals(ctorParamTypes[i]) && Double.class.equals(paramType)) ||
+ (char.class.equals(ctorParamTypes[i]) && Character.class.equals(paramType)) ||
+ (short.class.equals(ctorParamTypes[i]) && Short.class.equals(paramType)) ||
+ (boolean.class.equals(ctorParamTypes[i]) && Boolean.class.equals(paramType)) ||
+ (byte.class.equals(ctorParamTypes[i]) && Byte.class.equals(paramType)));
+ }
}
if (match) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
index fb898ea..c33d5af 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hbase;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
+import org.apache.hadoop.hbase.client.AsyncConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -61,6 +63,24 @@ public interface Server extends Abortable, Stoppable {
ClusterConnection getClusterConnection();
/**
+ * Returns a reference to the servers' async connection.
+ * <p/>
+ * Important note: this method returns a reference to Connection which is managed by Server
+ * itself, so callers must NOT attempt to close connection obtained.
+ */
+ default AsyncConnection getAsyncConnection() {
+ return getAsyncClusterConnection();
+ }
+
+ /**
+ * Returns a reference to the servers' async cluster connection.
+ * <p/>
+ * Important note: this method returns a reference to Connection which is managed by Server
+ * itself, so callers must NOT attempt to close connection obtained.
+ */
+ AsyncClusterConnection getAsyncClusterConnection();
+
+ /**
* @return The unique server name for this server.
*/
ServerName getServerName();
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index a16e09d..968cc27 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -3007,6 +3007,9 @@ public class HMaster extends HRegionServer implements MasterServices {
if (this.clusterConnection != null) {
this.clusterConnection.close();
}
+ if (this.asyncClusterConnection != null) {
+ this.asyncClusterConnection.close();
+ }
}
public void stopMaster() throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 6242d36..3085469 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -76,7 +76,9 @@ import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.YouAreDeadException;
import org.apache.hadoop.hbase.ZNodeClearer;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.client.ClusterConnectionFactory;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionUtils;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -104,7 +106,6 @@ import org.apache.hadoop.hbase.io.util.MemorySizeUtil;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;
import org.apache.hadoop.hbase.ipc.NettyRpcClientConfigHelper;
import org.apache.hadoop.hbase.ipc.RpcClient;
-import org.apache.hadoop.hbase.ipc.RpcClientFactory;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.ipc.RpcServerInterface;
@@ -262,6 +263,11 @@ public class HRegionServer extends HasThread implements
protected ClusterConnection clusterConnection;
/**
+ * The asynchronous cluster connection to be shared by services.
+ */
+ protected AsyncClusterConnection asyncClusterConnection;
+
+ /**
* Go here to get table descriptors.
*/
protected TableDescriptors tableDescriptors;
@@ -775,11 +781,7 @@ public class HRegionServer extends HasThread implements
return true;
}
- /**
- * Create a 'smarter' Connection, one that is capable of by-passing RPC if the request is to the
- * local server; i.e. a short-circuit Connection. Safe to use going to local or remote server.
- */
- private ClusterConnection createClusterConnection() throws IOException {
+ private Configuration unsetClientZookeeperQuorum() {
Configuration conf = this.conf;
if (conf.get(HConstants.CLIENT_ZOOKEEPER_QUORUM) != null) {
// Use server ZK cluster for server-issued connections, so we clone
@@ -787,11 +789,20 @@ public class HRegionServer extends HasThread implements
conf = new Configuration(this.conf);
conf.unset(HConstants.CLIENT_ZOOKEEPER_QUORUM);
}
+ return conf;
+ }
+
+ /**
+ * Create a 'smarter' Connection, one that is capable of by-passing RPC if the request is to the
+ * local server; i.e. a short-circuit Connection. Safe to use going to local or remote server.
+ */
+ private ClusterConnection createClusterConnection() throws IOException {
// Create a cluster connection that when appropriate, can short-circuit and go directly to the
// local server if the request is to the local server bypassing RPC. Can be used for both local
// and remote invocations.
- ClusterConnection conn = ConnectionUtils.createShortCircuitConnection(conf, null,
- userProvider.getCurrent(), serverName, rpcServices, rpcServices);
+ ClusterConnection conn =
+ ConnectionUtils.createShortCircuitConnection(unsetClientZookeeperQuorum(), null,
+ userProvider.getCurrent(), serverName, rpcServices, rpcServices);
// This is used to initialize the batch thread pool inside the connection implementation.
// When deploy a fresh cluster, we may first use the cluster connection in InitMetaProcedure,
// which will be executed inside the PEWorker, and then the batch thread pool will inherit the
@@ -825,9 +836,12 @@ public class HRegionServer extends HasThread implements
/**
* Setup our cluster connection if not already initialized.
*/
- protected synchronized void setupClusterConnection() throws IOException {
+ protected final synchronized void setupClusterConnection() throws IOException {
if (clusterConnection == null) {
clusterConnection = createClusterConnection();
+ asyncClusterConnection =
+ ClusterConnectionFactory.createAsyncClusterConnection(unsetClientZookeeperQuorum(),
+ new InetSocketAddress(this.rpcServices.isa.getAddress(), 0), userProvider.getCurrent());
}
}
@@ -841,8 +855,7 @@ public class HRegionServer extends HasThread implements
initializeZooKeeper();
setupClusterConnection();
// Setup RPC client for master communication
- this.rpcClient = RpcClientFactory.createClient(conf, clusterId, new InetSocketAddress(
- this.rpcServices.isa.getAddress(), 0), clusterConnection.getConnectionMetrics());
+ this.rpcClient = asyncClusterConnection.getRpcClient();
} catch (Throwable t) {
// Call stop if error or process will stick around for ever since server
// puts up non-daemon threads.
@@ -1104,7 +1117,15 @@ public class HRegionServer extends HasThread implements
LOG.warn("Attempt to close server's short circuit ClusterConnection failed.", e);
}
}
-
+ if (this.asyncClusterConnection != null) {
+ try {
+ this.asyncClusterConnection.close();
+ } catch (IOException e) {
+ // Although the {@link Closeable} interface throws an {@link
+ // IOException}, in reality, the implementation would never do that.
+ LOG.warn("Attempt to close server's AsyncClusterConnection failed.", e);
+ }
+ }
// Closing the compactSplit thread before closing meta regions
if (!this.killed && containsMetaTableRegions()) {
if (!abortRequested || this.fsOk) {
@@ -3722,9 +3743,9 @@ public class HRegionServer extends HasThread implements
}
@Override
- public EntityLock regionLock(List<RegionInfo> regionInfos, String description,
- Abortable abort) throws IOException {
- return new LockServiceClient(conf, lockStub, clusterConnection.getNonceGenerator())
+ public EntityLock regionLock(List<RegionInfo> regionInfos, String description, Abortable abort)
+ throws IOException {
+ return new LockServiceClient(conf, lockStub, asyncClusterConnection.getNonceGenerator())
.regionLock(regionInfos, description, abort);
}
@@ -3828,4 +3849,9 @@ public class HRegionServer extends HasThread implements
System.exit(1);
}
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return asyncClusterConnection;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index c7bccb3..7d1245c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -180,5 +181,10 @@ public class ReplicationSyncUp extends Configured implements Tool {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
index ff0a88c..4239498 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
@@ -30,6 +30,7 @@ import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -355,4 +356,9 @@ public class MockRegionServerServices implements RegionServerServices {
public TableDescriptors getTableDescriptors() {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java
index 38dc78d..4fb9716 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java
@@ -78,7 +78,7 @@ public class TestAsyncNonMetaRegionLocator {
TEST_UTIL.getAdmin().balancerSwitch(false, true);
AsyncRegistry registry = AsyncRegistryFactory.getRegistry(TEST_UTIL.getConfiguration());
CONN = new AsyncConnectionImpl(TEST_UTIL.getConfiguration(), registry,
- registry.getClusterId().get(), User.getCurrent());
+ registry.getClusterId().get(), null, User.getCurrent());
LOCATOR = new AsyncNonMetaRegionLocator(CONN);
SPLIT_KEYS = new byte[8][];
for (int i = 111; i < 999; i += 111) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocatorConcurrenyLimit.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocatorConcurrenyLimit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocatorConcurrenyLimit.java
index c6624e7..1f7d1b6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocatorConcurrenyLimit.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocatorConcurrenyLimit.java
@@ -124,7 +124,7 @@ public class TestAsyncNonMetaRegionLocatorConcurrenyLimit {
TEST_UTIL.getAdmin().balancerSwitch(false, true);
AsyncRegistry registry = AsyncRegistryFactory.getRegistry(TEST_UTIL.getConfiguration());
CONN = new AsyncConnectionImpl(TEST_UTIL.getConfiguration(), registry,
- registry.getClusterId().get(), User.getCurrent());
+ registry.getClusterId().get(), null, User.getCurrent());
LOCATOR = new AsyncNonMetaRegionLocator(CONN);
SPLIT_KEYS = IntStream.range(1, 256).mapToObj(i -> Bytes.toBytes(String.format("%02x", i)))
.toArray(byte[][]::new);
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTimeout.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTimeout.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTimeout.java
index 758aa30..0e28f96 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTimeout.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionLocatorTimeout.java
@@ -96,7 +96,7 @@ public class TestAsyncRegionLocatorTimeout {
TEST_UTIL.waitTableAvailable(TABLE_NAME);
AsyncRegistry registry = AsyncRegistryFactory.getRegistry(TEST_UTIL.getConfiguration());
CONN = new AsyncConnectionImpl(TEST_UTIL.getConfiguration(), registry,
- registry.getClusterId().get(), User.getCurrent());
+ registry.getClusterId().get(), null, User.getCurrent());
LOCATOR = CONN.getLocator();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSingleRequestRpcRetryingCaller.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSingleRequestRpcRetryingCaller.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSingleRequestRpcRetryingCaller.java
index a6c2efb..4ce1de8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSingleRequestRpcRetryingCaller.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncSingleRequestRpcRetryingCaller.java
@@ -73,7 +73,7 @@ public class TestAsyncSingleRequestRpcRetryingCaller {
TEST_UTIL.waitTableAvailable(TABLE_NAME);
AsyncRegistry registry = AsyncRegistryFactory.getRegistry(TEST_UTIL.getConfiguration());
CONN = new AsyncConnectionImpl(TEST_UTIL.getConfiguration(), registry,
- registry.getClusterId().get(), User.getCurrent());
+ registry.getClusterId().get(), null, User.getCurrent());
}
@AfterClass
@@ -164,7 +164,7 @@ public class TestAsyncSingleRequestRpcRetryingCaller {
}
};
try (AsyncConnectionImpl mockedConn = new AsyncConnectionImpl(CONN.getConfiguration(),
- CONN.registry, CONN.registry.getClusterId().get(), User.getCurrent()) {
+ CONN.registry, CONN.registry.getClusterId().get(), null, User.getCurrent()) {
@Override
AsyncRegionLocator getLocator() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableNoncedRetry.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableNoncedRetry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableNoncedRetry.java
index 3008561..e1e55f5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableNoncedRetry.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncTableNoncedRetry.java
@@ -85,7 +85,7 @@ public class TestAsyncTableNoncedRetry {
TEST_UTIL.waitTableAvailable(TABLE_NAME);
AsyncRegistry registry = AsyncRegistryFactory.getRegistry(TEST_UTIL.getConfiguration());
ASYNC_CONN = new AsyncConnectionImpl(TEST_UTIL.getConfiguration(), registry,
- registry.getClusterId().get(), User.getCurrent()) {
+ registry.getClusterId().get(), null, User.getCurrent()) {
@Override
public NonceGenerator getNonceGenerator() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index 9c55f57..3ebad66 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.Connection;
@@ -473,4 +474,9 @@ public class MockNoopMasterServices implements MasterServices {
public SyncReplicationReplayWALManager getSyncReplicationReplayWALManager() {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
index 844b705..848d456 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -708,4 +709,9 @@ class MockRegionServer implements AdminProtos.AdminService.BlockingInterface,
public TableDescriptors getTableDescriptors() {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
index 2300f54..77667a7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
@@ -349,5 +350,10 @@ public class TestActiveMasterManager {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
index 5c8db3e..c5fad32 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -279,6 +280,11 @@ public class TestHFileCleaner {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
index 119194b..fd11ff8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.HFileLink;
@@ -213,5 +214,10 @@ public class TestHFileLinkCleaner {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
index 247ed01..3286032 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.master.HMaster;
@@ -409,6 +410,11 @@ public class TestLogsCleaner {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
static class FaultyZooKeeperWatcher extends ZKWatcher {
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
index d162bf3..9791643 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.master.HMaster;
@@ -303,6 +304,11 @@ public class TestReplicationHFileCleaner {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
static class FaultyZooKeeperWatcher extends ZKWatcher {
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
index 8c9ce75..4a359e4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
@@ -862,6 +863,11 @@ public class TestHeapMemoryManager {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
static class CustomHeapMemoryTuner implements HeapMemoryTuner {
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
index cbf932c..5481ff8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.SplitLogCounters;
import org.apache.hadoop.hbase.SplitLogTask;
import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
@@ -159,6 +160,11 @@ public class TestSplitLogWorker {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
private void waitForCounter(LongAdder ctr, long oldval, long newval, long timems)
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
index 0e20252..9e9d1d6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Durability;
@@ -523,6 +524,11 @@ public class TestWALLockup {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
static class DummyWALActionsListener implements WALActionsListener {
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
index 863d558..62ab265 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -263,5 +264,10 @@ public class TestReplicationTrackerZKImpl {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index 86bbb09..427f319 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
@@ -906,5 +907,10 @@ public abstract class TestReplicationSourceManager {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
index e4780f1..92c8e54 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
@@ -363,6 +364,11 @@ public class TestTokenAuthentication {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
@Parameters(name = "{index}: rpcServerImpl={0}")
http://git-wip-us.apache.org/repos/asf/hbase/blob/054b4bab/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java
index c25db01..13212d2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.log.HBaseMarkers;
@@ -143,4 +144,9 @@ public class MockServer implements Server {
public Connection createConnection(Configuration conf) throws IOException {
return null;
}
+
+ @Override
+ public AsyncClusterConnection getAsyncClusterConnection() {
+ return null;
+ }
}
[07/16] hbase git commit: HBASE-21414 : StoreFileSize growth rate
metric
Posted by zh...@apache.org.
HBASE-21414 : StoreFileSize growth rate metric
Signed-off-by: Sergey Shelukhin <se...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/170df27b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/170df27b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/170df27b
Branch: refs/heads/HBASE-21512
Commit: 170df27b8858c93ea22a38f28f653c347cb8ce7f
Parents: 12e75a8
Author: Tommy Li <To...@microsoft.com>
Authored: Thu Dec 6 13:43:13 2018 -0800
Committer: Sergey Shelukhin <se...@microsoft.com>
Committed: Thu Dec 6 14:43:17 2018 -0800
----------------------------------------------------------------------
.../regionserver/MetricsRegionServerSource.java | 3 +++
.../regionserver/MetricsRegionServerWrapper.java | 5 +++++
.../MetricsRegionServerSourceImpl.java | 2 ++
.../MetricsRegionServerWrapperImpl.java | 19 ++++++++++++++-----
.../MetricsRegionServerWrapperStub.java | 7 ++++++-
.../regionserver/TestMetricsRegionServer.java | 1 +
6 files changed, 31 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/170df27b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index b94ee2d..6d9ce54 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -234,6 +234,7 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
String MEMSTORE_SIZE = "memStoreSize";
String MEMSTORE_SIZE_DESC = "Size of the memstore";
String STOREFILE_SIZE = "storeFileSize";
+ String STOREFILE_SIZE_GROWTH_RATE = "storeFileSizeGrowthRate";
String MAX_STORE_FILE_AGE = "maxStoreFileAge";
String MIN_STORE_FILE_AGE = "minStoreFileAge";
String AVG_STORE_FILE_AGE = "avgStoreFileAge";
@@ -243,6 +244,8 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
String AVG_STORE_FILE_AGE_DESC = "Average age of store files hosted on this RegionServer";
String NUM_REFERENCE_FILES_DESC = "Number of reference file on this RegionServer";
String STOREFILE_SIZE_DESC = "Size of storefiles being served.";
+ String STOREFILE_SIZE_GROWTH_RATE_DESC =
+ "Bytes per second by which the size of storefiles being served grows.";
String TOTAL_REQUEST_COUNT = "totalRequestCount";
String TOTAL_REQUEST_COUNT_DESC =
"Total number of requests this RegionServer has answered; increments the count once for " +
http://git-wip-us.apache.org/repos/asf/hbase/blob/170df27b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index b3a8dac..03ebc4c 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -101,6 +101,11 @@ public interface MetricsRegionServerWrapper {
long getStoreFileSize();
/**
+ * Get the growth rate of the store files this region server is serving from.
+ */
+ double getStoreFileSizeGrowthRate();
+
+ /**
* @return Max age of store files hosted on this region server
*/
long getMaxStoreFileAge();
http://git-wip-us.apache.org/repos/asf/hbase/blob/170df27b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 44dbc79..58c42a5 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -482,6 +482,8 @@ public class MetricsRegionServerSourceImpl
rsWrap.getNumStoreFiles())
.addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemStoreSize())
.addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize())
+ .addGauge(Interns.info(STOREFILE_SIZE_GROWTH_RATE, STOREFILE_SIZE_GROWTH_RATE_DESC),
+ rsWrap.getStoreFileSizeGrowthRate())
.addGauge(Interns.info(MAX_STORE_FILE_AGE, MAX_STORE_FILE_AGE_DESC),
rsWrap.getMaxStoreFileAge())
.addGauge(Interns.info(MIN_STORE_FILE_AGE, MIN_STORE_FILE_AGE_DESC),
http://git-wip-us.apache.org/repos/asf/hbase/blob/170df27b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 04a32c8..b38c3e0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -68,6 +68,7 @@ class MetricsRegionServerWrapperImpl
private volatile long numStoreFiles = 0;
private volatile long memstoreSize = 0;
private volatile long storeFileSize = 0;
+ private volatile double storeFileSizeGrowthRate = 0;
private volatile long maxStoreFileAge = 0;
private volatile long minStoreFileAge = 0;
private volatile long avgStoreFileAge = 0;
@@ -512,6 +513,11 @@ class MetricsRegionServerWrapperImpl
return storeFileSize;
}
+ @Override
+ public double getStoreFileSizeGrowthRate() {
+ return storeFileSizeGrowthRate;
+ }
+
@Override public double getRequestsPerSecond() {
return requestsPerSecond;
}
@@ -730,6 +736,7 @@ class MetricsRegionServerWrapperImpl
private long lastRequestCount = 0;
private long lastReadRequestsCount = 0;
private long lastWriteRequestsCount = 0;
+ private long lastStoreFileSize = 0;
@Override
synchronized public void run() {
@@ -870,18 +877,20 @@ class MetricsRegionServerWrapperImpl
long intervalReadRequestsCount = tempReadRequestsCount - lastReadRequestsCount;
long intervalWriteRequestsCount = tempWriteRequestsCount - lastWriteRequestsCount;
- double readRequestsRatePerMilliSecond = ((double)intervalReadRequestsCount/
- (double)period);
- double writeRequestsRatePerMilliSecond = ((double)intervalWriteRequestsCount/
- (double)period);
+ double readRequestsRatePerMilliSecond = (double)intervalReadRequestsCount / period;
+ double writeRequestsRatePerMilliSecond = (double)intervalWriteRequestsCount / period;
readRequestsRatePerSecond = readRequestsRatePerMilliSecond * 1000.0;
writeRequestsRatePerSecond = writeRequestsRatePerMilliSecond * 1000.0;
+ long intervalStoreFileSize = tempStoreFileSize - lastStoreFileSize;
+ storeFileSizeGrowthRate = (double)intervalStoreFileSize * 1000.0 / period;
+
lastReadRequestsCount = tempReadRequestsCount;
lastWriteRequestsCount = tempWriteRequestsCount;
-
+ lastStoreFileSize = tempStoreFileSize;
}
+
lastRan = currentTime;
WALProvider provider = regionServer.walFactory.getWALProvider();
http://git-wip-us.apache.org/repos/asf/hbase/blob/170df27b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 314d69c..b003b44 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -71,6 +71,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
}
@Override
+ public double getStoreFileSizeGrowthRate() {
+ return 50.0;
+ }
+
+ @Override
public long getMaxStoreFileAge() {
return 2;
}
@@ -222,7 +227,7 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
@Override
public long getMemStoreLimit() {
- return 419;
+ return 419;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/170df27b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
index fa8ea46..8215ea1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
@@ -78,6 +78,7 @@ public class TestMetricsRegionServer {
HELPER.assertGauge("storeFileCount", 300, serverSource);
HELPER.assertGauge("memstoreSize", 1025, serverSource);
HELPER.assertGauge("storeFileSize", 1900, serverSource);
+ HELPER.assertGauge("storeFileSizeGrowthRate", 50.0, serverSource);
HELPER.assertCounter("totalRequestCount", 899, serverSource);
HELPER.assertCounter("totalRowActionRequestCount",
HELPER.getCounter("readRequestCount", serverSource)
[15/16] hbase git commit: HBASE-21516 Use AsyncConnection instead of
Connection in SecureBulkLoadManager
Posted by zh...@apache.org.
HBASE-21516 Use AsyncConnection instead of Connection in SecureBulkLoadManager
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/47810209
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/47810209
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/47810209
Branch: refs/heads/HBASE-21512
Commit: 47810209598db31c4ded5986d892eda774f2951f
Parents: 054b4ba
Author: zhangduo <zh...@apache.org>
Authored: Sat Dec 1 21:15:48 2018 +0800
Committer: Duo Zhang <zh...@apache.org>
Committed: Tue Dec 11 11:17:11 2018 +0800
----------------------------------------------------------------------
.../hadoop/hbase/protobuf/ProtobufUtil.java | 5 +-
.../hbase/shaded/protobuf/ProtobufUtil.java | 7 ++-
.../hbase/regionserver/HRegionServer.java | 2 +-
.../regionserver/SecureBulkLoadManager.java | 24 +++++----
.../hadoop/hbase/security/token/TokenUtil.java | 57 +++++++++++++++-----
.../hbase/security/token/TestTokenUtil.java | 42 +++++++++++----
6 files changed, 96 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/47810209/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index 4d54528..a2c7687 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -261,13 +261,12 @@ public final class ProtobufUtil {
* just {@link ServiceException}. Prefer this method to
* {@link #getRemoteException(ServiceException)} because trying to
* contain direct protobuf references.
- * @param e
*/
- public static IOException handleRemoteException(Exception e) {
+ public static IOException handleRemoteException(Throwable e) {
return makeIOExceptionOfException(e);
}
- private static IOException makeIOExceptionOfException(Exception e) {
+ private static IOException makeIOExceptionOfException(Throwable e) {
Throwable t = e;
if (e instanceof ServiceException ||
e instanceof org.apache.hbase.thirdparty.com.google.protobuf.ServiceException) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/47810209/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
index cf4c831..d604013 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/ProtobufUtil.java
@@ -40,7 +40,6 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ByteBufferExtendedCell;
@@ -123,6 +122,7 @@ import org.apache.hbase.thirdparty.com.google.protobuf.Service;
import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
@@ -343,13 +343,12 @@ public final class ProtobufUtil {
* just {@link ServiceException}. Prefer this method to
* {@link #getRemoteException(ServiceException)} because trying to
* contain direct protobuf references.
- * @param e
*/
- public static IOException handleRemoteException(Exception e) {
+ public static IOException handleRemoteException(Throwable e) {
return makeIOExceptionOfException(e);
}
- private static IOException makeIOExceptionOfException(Exception e) {
+ private static IOException makeIOExceptionOfException(Throwable e) {
Throwable t = e;
if (e instanceof ServiceException) {
t = e.getCause();
http://git-wip-us.apache.org/repos/asf/hbase/blob/47810209/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 3085469..0d0a34e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1927,7 +1927,7 @@ public class HRegionServer extends HasThread implements
if (!isStopped() && !isAborted()) {
initializeThreads();
}
- this.secureBulkLoadManager = new SecureBulkLoadManager(this.conf, clusterConnection);
+ this.secureBulkLoadManager = new SecureBulkLoadManager(this.conf, asyncClusterConnection);
this.secureBulkLoadManager.start();
// Health checker thread.
http://git-wip-us.apache.org/repos/asf/hbase/blob/47810209/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java
index 566a6b6..add6519 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SecureBulkLoadManager.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,7 +28,6 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.BiFunction;
import java.util.function.Consumer;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -38,11 +37,12 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.AsyncConnection;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.regionserver.HRegion.BulkLoadListener;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider;
+import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier;
import org.apache.hadoop.hbase.security.token.FsDelegationToken;
import org.apache.hadoop.hbase.security.token.TokenUtil;
import org.apache.hadoop.hbase.util.Bytes;
@@ -56,7 +56,9 @@ import org.apache.hadoop.security.token.Token;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CleanupBulkLoadRequest;
@@ -111,9 +113,9 @@ public class SecureBulkLoadManager {
private UserProvider userProvider;
private ConcurrentHashMap<UserGroupInformation, Integer> ugiReferenceCounter;
- private Connection conn;
+ private AsyncConnection conn;
- SecureBulkLoadManager(Configuration conf, Connection conn) {
+ SecureBulkLoadManager(Configuration conf, AsyncConnection conn) {
this.conf = conf;
this.conn = conn;
}
@@ -212,23 +214,23 @@ public class SecureBulkLoadManager {
familyPaths.add(new Pair<>(el.getFamily().toByteArray(), el.getPath()));
}
- Token userToken = null;
+ Token<AuthenticationTokenIdentifier> userToken = null;
if (userProvider.isHadoopSecurityEnabled()) {
- userToken = new Token(request.getFsToken().getIdentifier().toByteArray(), request.getFsToken()
- .getPassword().toByteArray(), new Text(request.getFsToken().getKind()), new Text(
- request.getFsToken().getService()));
+ userToken = new Token<>(request.getFsToken().getIdentifier().toByteArray(),
+ request.getFsToken().getPassword().toByteArray(), new Text(request.getFsToken().getKind()),
+ new Text(request.getFsToken().getService()));
}
final String bulkToken = request.getBulkToken();
User user = getActiveUser();
final UserGroupInformation ugi = user.getUGI();
if (userProvider.isHadoopSecurityEnabled()) {
try {
- Token tok = TokenUtil.obtainToken(conn);
+ Token<AuthenticationTokenIdentifier> tok = TokenUtil.obtainToken(conn).get();
if (tok != null) {
boolean b = ugi.addToken(tok);
LOG.debug("token added " + tok + " for user " + ugi + " return=" + b);
}
- } catch (IOException ioe) {
+ } catch (Exception ioe) {
LOG.warn("unable to add token", ioe);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/47810209/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java
index c54d905..28efb84 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,27 +15,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.hbase.security.token;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.ServiceException;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.security.PrivilegedExceptionAction;
-
-import com.google.protobuf.ByteString;
-import com.google.protobuf.ServiceException;
-
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import java.util.concurrent.CompletableFuture;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.AsyncConnection;
+import org.apache.hadoop.hbase.client.AsyncTable;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;
+import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.AuthenticationService;
+import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.GetAuthenticationTokenRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.GetAuthenticationTokenResponse;
import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
@@ -45,6 +47,8 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+
/**
* Utility methods for obtaining authentication tokens.
*/
@@ -64,12 +68,39 @@ public class TokenUtil {
/**
* Obtain and return an authentication token for the current user.
+ * @param conn The async HBase cluster connection
+ * @return the authentication token instance, wrapped by a {@link CompletableFuture}.
+ */
+ public static CompletableFuture<Token<AuthenticationTokenIdentifier>> obtainToken(
+ AsyncConnection conn) {
+ CompletableFuture<Token<AuthenticationTokenIdentifier>> future = new CompletableFuture<>();
+ if (injectedException != null) {
+ future.completeExceptionally(injectedException);
+ return future;
+ }
+ AsyncTable<?> table = conn.getTable(TableName.META_TABLE_NAME);
+ table.<AuthenticationService.Interface, GetAuthenticationTokenResponse> coprocessorService(
+ AuthenticationProtos.AuthenticationService::newStub,
+ (s, c, r) -> s.getAuthenticationToken(c,
+ AuthenticationProtos.GetAuthenticationTokenRequest.getDefaultInstance(), r),
+ HConstants.EMPTY_START_ROW).whenComplete((resp, error) -> {
+ if (error != null) {
+ future.completeExceptionally(ProtobufUtil.handleRemoteException(error));
+ } else {
+ future.complete(toToken(resp.getToken()));
+ }
+ });
+ return future;
+ }
+
+ /**
+ * Obtain and return an authentication token for the current user.
* @param conn The HBase cluster connection
* @throws IOException if a remote error or serialization problem occurs.
* @return the authentication token instance
*/
- public static Token<AuthenticationTokenIdentifier> obtainToken(
- Connection conn) throws IOException {
+ public static Token<AuthenticationTokenIdentifier> obtainToken(Connection conn)
+ throws IOException {
Table meta = null;
try {
injectFault();
@@ -77,9 +108,9 @@ public class TokenUtil {
meta = conn.getTable(TableName.META_TABLE_NAME);
CoprocessorRpcChannel rpcChannel = meta.coprocessorService(HConstants.EMPTY_START_ROW);
AuthenticationProtos.AuthenticationService.BlockingInterface service =
- AuthenticationProtos.AuthenticationService.newBlockingStub(rpcChannel);
- AuthenticationProtos.GetAuthenticationTokenResponse response = service.getAuthenticationToken(null,
- AuthenticationProtos.GetAuthenticationTokenRequest.getDefaultInstance());
+ AuthenticationService.newBlockingStub(rpcChannel);
+ GetAuthenticationTokenResponse response =
+ service.getAuthenticationToken(null, GetAuthenticationTokenRequest.getDefaultInstance());
return toToken(response.getToken());
} catch (ServiceException se) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/47810209/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenUtil.java
index 32fcddb..585a3ec 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenUtil.java
@@ -18,35 +18,53 @@
package org.apache.hadoop.hbase.security.token;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.net.URL;
import java.net.URLClassLoader;
-
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.client.AsyncConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.After;
+import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
@Category(SmallTests.class)
public class TestTokenUtil {
+
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestTokenUtil.class);
+ HBaseClassTestRule.forClass(TestTokenUtil.class);
- @Test
- public void testObtainToken() throws Exception {
+ private URLClassLoader cl;
+
+ @Before
+ public void setUp() {
URL urlPU = ProtobufUtil.class.getProtectionDomain().getCodeSource().getLocation();
URL urlTU = TokenUtil.class.getProtectionDomain().getCodeSource().getLocation();
+ cl = new URLClassLoader(new URL[] { urlPU, urlTU }, getClass().getClassLoader());
+ }
- ClassLoader cl = new URLClassLoader(new URL[] { urlPU, urlTU }, getClass().getClassLoader());
+ @After
+ public void tearDown() throws IOException {
+ Closeables.close(cl, true);
+ }
+ @Test
+ public void testObtainToken() throws Exception {
Throwable injected = new com.google.protobuf.ServiceException("injected");
Class<?> tokenUtil = cl.loadClass(TokenUtil.class.getCanonicalName());
@@ -55,8 +73,7 @@ public class TestTokenUtil {
shouldInjectFault.set(null, injected);
try {
- tokenUtil.getMethod("obtainToken", Connection.class)
- .invoke(null, new Object[] { null });
+ tokenUtil.getMethod("obtainToken", Connection.class).invoke(null, new Object[] { null });
fail("Should have injected exception.");
} catch (InvocationTargetException e) {
Throwable t = e;
@@ -72,9 +89,16 @@ public class TestTokenUtil {
}
}
+ CompletableFuture<?> future = (CompletableFuture<?>) tokenUtil
+ .getMethod("obtainToken", AsyncConnection.class).invoke(null, new Object[] { null });
+ try {
+ future.get();
+ fail("Should have injected exception.");
+ } catch (ExecutionException e) {
+ assertSame(injected, e.getCause());
+ }
Boolean loaded = (Boolean) cl.loadClass(ProtobufUtil.class.getCanonicalName())
- .getDeclaredMethod("isClassLoaderLoaded")
- .invoke(null);
+ .getDeclaredMethod("isClassLoaderLoaded").invoke(null);
assertFalse("Should not have loaded DynamicClassLoader", loaded);
}
}
[05/16] hbase git commit: HBASE-21551 Memory leak when use scan with
STREAM at server side - (addendum)
Posted by zh...@apache.org.
HBASE-21551 Memory leak when use scan with STREAM at server side - (addendum)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/67ab8b88
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/67ab8b88
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/67ab8b88
Branch: refs/heads/HBASE-21512
Commit: 67ab8b888f8b393979624a2bd7d527fefd9dd6d7
Parents: 3b85485
Author: huzheng <op...@gmail.com>
Authored: Thu Dec 6 11:26:52 2018 +0800
Committer: huzheng <op...@gmail.com>
Committed: Thu Dec 6 11:26:52 2018 +0800
----------------------------------------------------------------------
.../hbase/regionserver/TestSwitchToStreamRead.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/67ab8b88/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 037b13e..c1cecf8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -47,9 +47,9 @@ import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.junit.AfterClass;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Test;
@@ -74,8 +74,8 @@ public class TestSwitchToStreamRead {
private static HRegion REGION;
- @BeforeClass
- public static void setUp() throws IOException {
+ @Before
+ public void setUp() throws IOException {
UTIL.getConfiguration().setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, 2048);
StringBuilder sb = new StringBuilder(256);
for (int i = 0; i < 255; i++) {
@@ -99,8 +99,8 @@ public class TestSwitchToStreamRead {
}
}
- @AfterClass
- public static void tearDown() throws IOException {
+ @After
+ public void tearDown() throws IOException {
REGION.close(true);
UTIL.cleanupTestDir();
}
[02/16] hbase git commit: HBASE-21541 Move
MetaTableLocator.verifyRegionLocation to hbase-rsgroup module
Posted by zh...@apache.org.
HBASE-21541 Move MetaTableLocator.verifyRegionLocation to hbase-rsgroup module
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8bf966c8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8bf966c8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8bf966c8
Branch: refs/heads/HBASE-21512
Commit: 8bf966c8e936dec4d83bcbe85c5aab543f14a0df
Parents: 59cfe2e
Author: zhangduo <zh...@apache.org>
Authored: Tue Dec 4 22:33:36 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Tue Dec 4 22:33:36 2018 +0800
----------------------------------------------------------------------
.../hbase/rsgroup/RSGroupInfoManagerImpl.java | 3 +-
.../apache/hadoop/hbase/rsgroup/Utility.java | 196 ++++++++++-
.../hadoop/hbase/rsgroup/TestUtility.java | 229 +++++++++++++
.../java/org/apache/hadoop/hbase/Server.java | 15 +-
.../hbase/master/MasterMetaBootstrap.java | 3 +-
.../hadoop/hbase/master/MasterRpcServices.java | 5 +-
.../hbase/master/MasterStatusServlet.java | 8 +-
.../master/procedure/ProcedureSyncWait.java | 8 +-
.../master/snapshot/MasterSnapshotVerifier.java | 4 +-
.../master/snapshot/TakeSnapshotHandler.java | 4 +-
.../flush/MasterFlushTableProcedureManager.java | 6 +-
.../hbase/regionserver/HRegionServer.java | 16 -
.../regionserver/ReplicationSyncUp.java | 6 -
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 10 +-
.../resources/hbase-webapps/master/table.jsp | 4 +-
.../hadoop/hbase/MockRegionServerServices.java | 6 -
.../hadoop/hbase/TestMetaTableAccessor.java | 10 +-
.../hadoop/hbase/TestMetaTableLocator.java | 261 +++------------
.../hbase/client/TestMetaWithReplicas.java | 4 +-
.../hbase/master/MockNoopMasterServices.java | 6 -
.../hadoop/hbase/master/MockRegionServer.java | 6 -
.../hbase/master/TestActiveMasterManager.java | 6 -
.../hbase/master/cleaner/TestHFileCleaner.java | 6 -
.../master/cleaner/TestHFileLinkCleaner.java | 6 -
.../hbase/master/cleaner/TestLogsCleaner.java | 7 -
.../cleaner/TestReplicationHFileCleaner.java | 6 -
.../regionserver/TestHeapMemoryManager.java | 6 -
.../regionserver/TestRegionServerNoMaster.java | 5 +-
.../hbase/regionserver/TestSplitLogWorker.java | 6 -
.../hbase/regionserver/TestWALLockup.java | 6 -
.../TestReplicationTrackerZKImpl.java | 6 -
.../TestReplicationSourceManager.java | 6 -
.../security/token/TestTokenAuthentication.java | 6 -
.../apache/hadoop/hbase/util/MockServer.java | 7 -
.../hbase/zookeeper/MetaTableLocator.java | 329 +++----------------
.../apache/hadoop/hbase/zookeeper/ZKUtil.java | 7 +-
36 files changed, 551 insertions(+), 674 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
----------------------------------------------------------------------
diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
index cdd68d1..6f54c31 100644
--- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
+++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java
@@ -757,8 +757,7 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
found.set(true);
try {
boolean rootMetaFound =
- masterServices.getMetaTableLocator().verifyMetaRegionLocation(
- conn, masterServices.getZooKeeper(), 1);
+ Utility.verifyMetaRegionLocation(conn, masterServices.getZooKeeper(), 1);
if (rootMetaFound) {
MetaTableAccessor.Visitor visitor = new DefaultVisitorBase() {
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/Utility.java
----------------------------------------------------------------------
diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/Utility.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/Utility.java
index 4d7bf43..d5450c4 100644
--- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/Utility.java
+++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/Utility.java
@@ -19,19 +19,49 @@
*/
package org.apache.hadoop.hbase.rsgroup;
+import java.io.EOFException;
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.NoRouteToHostException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
import java.util.HashSet;
+import java.util.Locale;
import java.util.Set;
-
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.RegionReplicaUtil;
+import org.apache.hadoop.hbase.client.RetriesExhaustedException;
+import org.apache.hadoop.hbase.ipc.FailedServerException;
+import org.apache.hadoop.hbase.ipc.HBaseRpcController;
+import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.net.Address;
+import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
/**
* Utility for this RSGroup package in hbase-rsgroup.
*/
@InterfaceAudience.Private
final class Utility {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Utility.class);
+
private Utility() {
}
@@ -45,9 +75,171 @@ final class Utility {
return onlineServers;
}
- for(ServerName server: master.getServerManager().getOnlineServers().keySet()) {
+ for (ServerName server : master.getServerManager().getOnlineServers().keySet()) {
onlineServers.add(server.getAddress());
}
return onlineServers;
}
+
+ /**
+ * Verify <code>hbase:meta</code> is deployed and accessible.
+ * @param hConnection the connection to use
+ * @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
+ * @param timeout How long to wait on zk for meta address (passed through to the internal call to
+ * {@link #getMetaServerConnection}.
+ * @return True if the <code>hbase:meta</code> location is healthy.
+ * @throws IOException if the number of retries for getting the connection is exceeded
+ * @throws InterruptedException if waiting for the socket operation fails
+ */
+ public static boolean verifyMetaRegionLocation(ClusterConnection hConnection, ZKWatcher zkw,
+ final long timeout) throws InterruptedException, IOException {
+ return verifyMetaRegionLocation(hConnection, zkw, timeout, RegionInfo.DEFAULT_REPLICA_ID);
+ }
+
+ /**
+ * Verify <code>hbase:meta</code> is deployed and accessible.
+ * @param connection the connection to use
+ * @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
+ * @param timeout How long to wait on zk for meta address (passed through to
+ * @param replicaId the ID of the replica
+ * @return True if the <code>hbase:meta</code> location is healthy.
+ * @throws InterruptedException if waiting for the socket operation fails
+ * @throws IOException if the number of retries for getting the connection is exceeded
+ */
+ public static boolean verifyMetaRegionLocation(ClusterConnection connection, ZKWatcher zkw,
+ final long timeout, int replicaId) throws InterruptedException, IOException {
+ AdminProtos.AdminService.BlockingInterface service = null;
+ try {
+ service = getMetaServerConnection(connection, zkw, timeout, replicaId);
+ } catch (NotAllMetaRegionsOnlineException e) {
+ // Pass
+ } catch (ServerNotRunningYetException e) {
+ // Pass -- remote server is not up so can't be carrying root
+ } catch (UnknownHostException e) {
+ // Pass -- server name doesn't resolve so it can't be assigned anything.
+ } catch (RegionServerStoppedException e) {
+ // Pass -- server name sends us to a server that is dying or already dead.
+ }
+ return (service != null) && verifyRegionLocation(connection, service,
+ MetaTableLocator.getMetaRegionLocation(zkw, replicaId),
+ RegionReplicaUtil.getRegionInfoForReplica(RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId)
+ .getRegionName());
+ }
+
+ /**
+ * Verify we can connect to <code>hostingServer</code> and that its carrying
+ * <code>regionName</code>.
+ * @param hostingServer Interface to the server hosting <code>regionName</code>
+ * @param address The servername that goes with the <code>metaServer</code> interface. Used
+ * logging.
+ * @param regionName The regionname we are interested in.
+ * @return True if we were able to verify the region located at other side of the interface.
+ */
+ // TODO: We should be able to get the ServerName from the AdminProtocol
+ // rather than have to pass it in. Its made awkward by the fact that the
+ // HRI is likely a proxy against remote server so the getServerName needs
+ // to be fixed to go to a local method or to a cache before we can do this.
+ private static boolean verifyRegionLocation(final ClusterConnection connection,
+ AdminService.BlockingInterface hostingServer, final ServerName address,
+ final byte[] regionName) {
+ if (hostingServer == null) {
+ LOG.info("Passed hostingServer is null");
+ return false;
+ }
+ Throwable t;
+ HBaseRpcController controller = connection.getRpcControllerFactory().newController();
+ try {
+ // Try and get regioninfo from the hosting server.
+ return ProtobufUtil.getRegionInfo(controller, hostingServer, regionName) != null;
+ } catch (ConnectException e) {
+ t = e;
+ } catch (RetriesExhaustedException e) {
+ t = e;
+ } catch (RemoteException e) {
+ IOException ioe = e.unwrapRemoteException();
+ t = ioe;
+ } catch (IOException e) {
+ Throwable cause = e.getCause();
+ if (cause != null && cause instanceof EOFException) {
+ t = cause;
+ } else if (cause != null && cause.getMessage() != null &&
+ cause.getMessage().contains("Connection reset")) {
+ t = cause;
+ } else {
+ t = e;
+ }
+ }
+ LOG.info("Failed verification of " + Bytes.toStringBinary(regionName) + " at address=" +
+ address + ", exception=" + t.getMessage());
+ return false;
+ }
+
+ /**
+ * Gets a connection to the server hosting meta, as reported by ZooKeeper, waiting up to the
+ * specified timeout for availability.
+ * <p>
+ * WARNING: Does not retry. Use an {@link org.apache.hadoop.hbase.client.HTable} instead.
+ * @param connection the connection to use
+ * @param zkw reference to the {@link ZKWatcher} which also contains configuration and operation
+ * @param timeout How long to wait on meta location
+ * @param replicaId the ID of the replica
+ * @return connection to server hosting meta
+ * @throws InterruptedException if waiting for the socket operation fails
+ * @throws IOException if the number of retries for getting the connection is exceeded
+ */
+ private static AdminService.BlockingInterface getMetaServerConnection(
+ ClusterConnection connection, ZKWatcher zkw, long timeout, int replicaId)
+ throws InterruptedException, IOException {
+ return getCachedConnection(connection,
+ MetaTableLocator.waitMetaRegionLocation(zkw, replicaId, timeout));
+ }
+
+ /**
+ * @param sn ServerName to get a connection against.
+ * @return The AdminProtocol we got when we connected to <code>sn</code> May have come from cache,
+ * may not be good, may have been setup by this invocation, or may be null.
+ * @throws IOException if the number of retries for getting the connection is exceeded
+ */
+ private static AdminService.BlockingInterface getCachedConnection(ClusterConnection connection,
+ ServerName sn) throws IOException {
+ if (sn == null) {
+ return null;
+ }
+ AdminService.BlockingInterface service = null;
+ try {
+ service = connection.getAdmin(sn);
+ } catch (RetriesExhaustedException e) {
+ if (e.getCause() != null && e.getCause() instanceof ConnectException) {
+ LOG.debug("Catch this; presume it means the cached connection has gone bad.");
+ } else {
+ throw e;
+ }
+ } catch (SocketTimeoutException e) {
+ LOG.debug("Timed out connecting to " + sn);
+ } catch (NoRouteToHostException e) {
+ LOG.debug("Connecting to " + sn, e);
+ } catch (SocketException e) {
+ LOG.debug("Exception connecting to " + sn);
+ } catch (UnknownHostException e) {
+ LOG.debug("Unknown host exception connecting to " + sn);
+ } catch (FailedServerException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Server " + sn + " is in failed server list.");
+ }
+ } catch (IOException ioe) {
+ Throwable cause = ioe.getCause();
+ if (ioe instanceof ConnectException) {
+ LOG.debug("Catch. Connect refused.");
+ } else if (cause != null && cause instanceof EOFException) {
+ LOG.debug("Catch. Other end disconnected us.");
+ } else if (cause != null && cause.getMessage() != null &&
+ cause.getMessage().toLowerCase(Locale.ROOT).contains("connection reset")) {
+ LOG.debug("Catch. Connection reset.");
+ } else {
+ throw ioe;
+ }
+
+ }
+ return service;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestUtility.java
----------------------------------------------------------------------
diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestUtility.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestUtility.java
new file mode 100644
index 0000000..0065610
--- /dev/null
+++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestUtility.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.rsgroup;
+
+import static org.junit.Assert.assertFalse;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.ipc.HBaseRpcController;
+import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
+import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
+import org.apache.hadoop.hbase.master.RegionState;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.zookeeper.KeeperException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
+import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.GetRequest;
+
+@Category({ MiscTests.class, MediumTests.class })
+public class TestUtility {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestUtility.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestUtility.class);
+
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private static final ServerName SN =
+ ServerName.valueOf("example.org", 1234, System.currentTimeMillis());
+
+ private ZKWatcher watcher;
+
+ private Abortable abortable;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ // Set this down so tests run quicker
+ UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
+ UTIL.startMiniZKCluster();
+ }
+
+ @AfterClass
+ public static void afterClass() throws IOException {
+ UTIL.getZkCluster().shutdown();
+ }
+
+ @Before
+ public void before() throws IOException {
+ this.abortable = new Abortable() {
+ @Override
+ public void abort(String why, Throwable e) {
+ LOG.info(why, e);
+ }
+
+ @Override
+ public boolean isAborted() {
+ return false;
+ }
+ };
+ this.watcher =
+ new ZKWatcher(UTIL.getConfiguration(), this.getClass().getSimpleName(), this.abortable, true);
+ }
+
+ @After
+ public void after() {
+ try {
+ // Clean out meta location or later tests will be confused... they presume
+ // start fresh in zk.
+ MetaTableLocator.deleteMetaLocation(this.watcher);
+ } catch (KeeperException e) {
+ LOG.warn("Unable to delete hbase:meta location", e);
+ }
+
+ this.watcher.close();
+ }
+
+ /**
+ * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely want
+ * to pass a mocked HRS; can be null.
+ * @param client A mocked ClientProtocol instance, can be null
+ * @return Mock up a connection that returns a {@link Configuration} when
+ * {@link org.apache.hadoop.hbase.client.ClusterConnection#getConfiguration()} is called,
+ * a 'location' when
+ * {@link org.apache.hadoop.hbase.client.RegionLocator#getRegionLocation(byte[], boolean)}
+ * is called, and that returns the passed
+ * {@link AdminProtos.AdminService.BlockingInterface} instance when
+ * {@link org.apache.hadoop.hbase.client.ClusterConnection#getAdmin(ServerName)} is
+ * called, returns the passed {@link ClientProtos.ClientService.BlockingInterface}
+ * instance when
+ * {@link org.apache.hadoop.hbase.client.ClusterConnection#getClient(ServerName)} is
+ * called.
+ */
+ private ClusterConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
+ final ClientProtos.ClientService.BlockingInterface client) throws IOException {
+ ClusterConnection connection =
+ HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
+ Mockito.doNothing().when(connection).close();
+ // Make it so we return any old location when asked.
+ final HRegionLocation anyLocation =
+ new HRegionLocation(RegionInfoBuilder.FIRST_META_REGIONINFO, SN);
+ Mockito.when(connection.getRegionLocation((TableName) Mockito.any(), (byte[]) Mockito.any(),
+ Mockito.anyBoolean())).thenReturn(anyLocation);
+ Mockito.when(connection.locateRegion((TableName) Mockito.any(), (byte[]) Mockito.any()))
+ .thenReturn(anyLocation);
+ if (admin != null) {
+ // If a call to getHRegionConnection, return this implementation.
+ Mockito.when(connection.getAdmin(Mockito.any())).thenReturn(admin);
+ }
+ if (client != null) {
+ // If a call to getClient, return this implementation.
+ Mockito.when(connection.getClient(Mockito.any())).thenReturn(client);
+ }
+ return connection;
+ }
+
+ private void testVerifyMetaRegionLocationWithException(Exception ex)
+ throws IOException, InterruptedException, KeeperException, ServiceException {
+ // Mock an ClientProtocol.
+ final ClientProtos.ClientService.BlockingInterface implementation =
+ Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
+
+ ClusterConnection connection = mockConnection(null, implementation);
+
+ // If a 'get' is called on mocked interface, throw connection refused.
+ Mockito.when(implementation.get((RpcController) Mockito.any(), (GetRequest) Mockito.any()))
+ .thenThrow(new ServiceException(ex));
+
+ long timeout = UTIL.getConfiguration().getLong("hbase.catalog.verification.timeout", 1000);
+ MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPENING);
+ assertFalse(Utility.verifyMetaRegionLocation(connection, watcher, timeout));
+
+ MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN);
+ assertFalse(Utility.verifyMetaRegionLocation(connection, watcher, timeout));
+ }
+
+ /**
+ * Test get of meta region fails properly if nothing to connect to.
+ */
+ @Test
+ public void testVerifyMetaRegionLocationFails()
+ throws IOException, InterruptedException, KeeperException, ServiceException {
+ ClusterConnection connection = Mockito.mock(ClusterConnection.class);
+ ServiceException connectException =
+ new ServiceException(new ConnectException("Connection refused"));
+ final AdminProtos.AdminService.BlockingInterface implementation =
+ Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
+ Mockito.when(implementation.getRegionInfo((RpcController) Mockito.any(),
+ (GetRegionInfoRequest) Mockito.any())).thenThrow(connectException);
+ Mockito.when(connection.getAdmin(Mockito.any())).thenReturn(implementation);
+ RpcControllerFactory controllerFactory = Mockito.mock(RpcControllerFactory.class);
+ Mockito.when(controllerFactory.newController())
+ .thenReturn(Mockito.mock(HBaseRpcController.class));
+ Mockito.when(connection.getRpcControllerFactory()).thenReturn(controllerFactory);
+
+ ServerName sn = ServerName.valueOf("example.com", 1234, System.currentTimeMillis());
+ MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPENING);
+ assertFalse(Utility.verifyMetaRegionLocation(connection, watcher, 100));
+ MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPEN);
+ assertFalse(Utility.verifyMetaRegionLocation(connection, watcher, 100));
+ }
+
+ /**
+ * Test we survive a connection refused {@link ConnectException}
+ */
+ @Test
+ public void testGetMetaServerConnectionFails()
+ throws IOException, InterruptedException, KeeperException, ServiceException {
+ testVerifyMetaRegionLocationWithException(new ConnectException("Connection refused"));
+ }
+
+ /**
+ * Test that verifyMetaRegionLocation properly handles getting a ServerNotRunningException. See
+ * HBASE-4470. Note this doesn't check the exact exception thrown in the HBASE-4470 as there it is
+ * thrown from getHConnection() and here it is thrown from get() -- but those are both called from
+ * the same function anyway, and this way is less invasive than throwing from getHConnection would
+ * be.
+ */
+ @Test
+ public void testVerifyMetaRegionServerNotRunning()
+ throws IOException, InterruptedException, KeeperException, ServiceException {
+ testVerifyMetaRegionLocationWithException(new ServerNotRunningYetException("mock"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
index 53a757a..fb898ea 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/Server.java
@@ -1,5 +1,4 @@
-/*
- *
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,16 +17,14 @@
*/
package org.apache.hadoop.hbase;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
-import java.io.IOException;
-
/**
* Defines a curated set of shared functions implemented by HBase servers (Masters
* and RegionServers). For use internally only. Be judicious adding API. Changes cause ripples
@@ -64,14 +61,6 @@ public interface Server extends Abortable, Stoppable {
ClusterConnection getClusterConnection();
/**
- * Returns instance of {@link org.apache.hadoop.hbase.zookeeper.MetaTableLocator}
- * running inside this server. This MetaServerLocator is started and stopped by server, clients
- * shouldn't manage it's lifecycle.
- * @return instance of {@link MetaTableLocator} associated with this server.
- */
- MetaTableLocator getMetaTableLocator();
-
- /**
* @return The unique server name for this server.
*/
ServerName getServerName();
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java
index ce21465..e57817e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterMetaBootstrap.java
@@ -65,8 +65,7 @@ class MasterMetaBootstrap {
throw new IllegalStateException("hbase:meta must be initialized first before we can " +
"assign out its replicas");
}
- ServerName metaServername =
- this.master.getMetaTableLocator().getMetaRegionLocation(this.master.getZooKeeper());
+ ServerName metaServername = MetaTableLocator.getMetaRegionLocation(this.master.getZooKeeper());
for (int i = 1; i < numReplicas; i++) {
// Get current meta state for replica from zk.
RegionState metaState = MetaTableLocator.getMetaRegionState(master.getZooKeeper(), i);
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index fb76bec..cd838d5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -102,6 +102,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -1550,9 +1551,9 @@ public class MasterRpcServices extends RSRpcServices
}
Pair<RegionInfo, ServerName> pair =
MetaTableAccessor.getRegion(master.getConnection(), regionName);
- if (Bytes.equals(RegionInfoBuilder.FIRST_META_REGIONINFO.getRegionName(),regionName)) {
+ if (Bytes.equals(RegionInfoBuilder.FIRST_META_REGIONINFO.getRegionName(), regionName)) {
pair = new Pair<>(RegionInfoBuilder.FIRST_META_REGIONINFO,
- master.getMetaTableLocator().getMetaRegionLocation(master.getZooKeeper()));
+ MetaTableLocator.getMetaRegionLocation(master.getZooKeeper()));
}
if (pair == null) {
throw new UnknownRegionException(Bytes.toString(regionName));
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
index 560a04a..58e57c4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java
@@ -22,17 +22,15 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.tmpl.master.MasterStatusTmpl;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
+import org.apache.yetus.audience.InterfaceAudience;
/**
* The servlet responsible for rendering the index page of the
@@ -82,9 +80,7 @@ public class MasterStatusServlet extends HttpServlet {
}
private ServerName getMetaLocationOrNull(HMaster master) {
- MetaTableLocator metaTableLocator = master.getMetaTableLocator();
- return metaTableLocator == null ? null :
- metaTableLocator.getMetaRegionLocation(master.getZooKeeper());
+ return MetaTableLocator.getMetaRegionLocation(master.getZooKeeper());
}
private Map<String, Integer> getFragmentationInfo(
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
index 9353124..46621da 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ProcedureSyncWait.java
@@ -25,7 +25,6 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -35,6 +34,7 @@ import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
@@ -225,12 +225,12 @@ public final class ProcedureSyncWait {
protected static void waitMetaRegions(final MasterProcedureEnv env) throws IOException {
int timeout = env.getMasterConfiguration().getInt("hbase.client.catalog.timeout", 10000);
try {
- if (env.getMasterServices().getMetaTableLocator().waitMetaRegionLocation(
- env.getMasterServices().getZooKeeper(), timeout) == null) {
+ if (MetaTableLocator.waitMetaRegionLocation(env.getMasterServices().getZooKeeper(),
+ timeout) == null) {
throw new NotAllMetaRegionsOnlineException();
}
} catch (InterruptedException e) {
- throw (InterruptedIOException)new InterruptedIOException().initCause(e);
+ throw (InterruptedIOException) new InterruptedIOException().initCause(e);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
index e78f0b5..c9dc0c2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/MasterSnapshotVerifier.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.MetaTableAccessor;
@@ -42,6 +41,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
@@ -159,7 +159,7 @@ public final class MasterSnapshotVerifier {
private void verifyRegions(final SnapshotManifest manifest) throws IOException {
List<RegionInfo> regions;
if (TableName.META_TABLE_NAME.equals(tableName)) {
- regions = new MetaTableLocator().getMetaRegions(services.getZooKeeper());
+ regions = MetaTableLocator.getMetaRegions(services.getZooKeeper());
} else {
regions = MetaTableAccessor.getTableRegions(services.getConnection(), tableName);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
index 1dce79f..3b7d65a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/TakeSnapshotHandler.java
@@ -24,7 +24,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CancellationException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
@@ -60,6 +59,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
/**
@@ -193,7 +193,7 @@ public abstract class TakeSnapshotHandler extends EventHandler implements Snapsh
List<Pair<RegionInfo, ServerName>> regionsAndLocations;
if (TableName.META_TABLE_NAME.equals(snapshotTable)) {
- regionsAndLocations = new MetaTableLocator().getMetaRegionsAndLocations(
+ regionsAndLocations = MetaTableLocator.getMetaRegionsAndLocations(
server.getZooKeeper());
} else {
regionsAndLocations = MetaTableAccessor.getTableRegionsAndLocations(
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/MasterFlushTableProcedureManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/MasterFlushTableProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/MasterFlushTableProcedureManager.java
index fee3dde..1f7a5e2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/MasterFlushTableProcedureManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/MasterFlushTableProcedureManager.java
@@ -24,7 +24,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ThreadPoolExecutor;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.MetaTableAccessor;
@@ -43,14 +42,15 @@ import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinator;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.access.AccessChecker;
-import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ProcedureDescription;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
@@ -131,7 +131,7 @@ public class MasterFlushTableProcedureManager extends MasterProcedureManager {
List<Pair<RegionInfo, ServerName>> regionsAndLocations;
if (TableName.META_TABLE_NAME.equals(tableName)) {
- regionsAndLocations = new MetaTableLocator().getMetaRegionsAndLocations(
+ regionsAndLocations = MetaTableLocator.getMetaRegionsAndLocations(
master.getZooKeeper());
} else {
regionsAndLocations = MetaTableAccessor.getTableRegionsAndLocations(
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b9d606d..6242d36 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -261,14 +261,6 @@ public class HRegionServer extends HasThread implements
*/
protected ClusterConnection clusterConnection;
- /*
- * Long-living meta table locator, which is created when the server is started and stopped
- * when server shuts down. References to this locator shall be used to perform according
- * operations in EventHandlers. Primary reason for this decision is to make it mockable
- * for tests.
- */
- protected MetaTableLocator metaTableLocator;
-
/**
* Go here to get table descriptors.
*/
@@ -836,7 +828,6 @@ public class HRegionServer extends HasThread implements
protected synchronized void setupClusterConnection() throws IOException {
if (clusterConnection == null) {
clusterConnection = createClusterConnection();
- metaTableLocator = new MetaTableLocator();
}
}
@@ -1104,8 +1095,6 @@ public class HRegionServer extends HasThread implements
LOG.info("stopping server " + this.serverName);
}
- // so callers waiting for meta without timeout can stop
- if (this.metaTableLocator != null) this.metaTableLocator.stop();
if (this.clusterConnection != null && !clusterConnection.isClosed()) {
try {
this.clusterConnection.close();
@@ -2174,11 +2163,6 @@ public class HRegionServer extends HasThread implements
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return this.metaTableLocator;
- }
-
- @Override
public void stop(final String msg) {
stop(msg, false, RpcServer.getRequestUser().orElse(null));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index 62068fd..c7bccb3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -130,11 +129,6 @@ public class ReplicationSyncUp extends Configured implements Tool {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ServerName getServerName() {
return ServerName.valueOf(hostname, 1234, 1L);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index b43262d..14706c5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -134,7 +134,6 @@ import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.hbase.wal.WALSplitter;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
@@ -144,20 +143,20 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.base.Joiner;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
import org.apache.hbase.thirdparty.com.google.common.collect.Ordering;
+import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
import org.apache.hbase.thirdparty.com.google.common.collect.TreeMultimap;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
@@ -2002,11 +2001,6 @@ public class HBaseFsck extends Configured implements Closeable {
});
}
- private ServerName getMetaRegionServerName(int replicaId)
- throws IOException, KeeperException {
- return new MetaTableLocator().getMetaRegionLocation(zkw, replicaId);
- }
-
/**
* Contacts each regionserver and fetches metadata about regions.
* @param regionServerList - the list of region servers to connect to
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
index 372e5ca..5b996eb396 100644
--- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
+++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp
@@ -77,8 +77,6 @@
<%
HMaster master = (HMaster)getServletContext().getAttribute(HMaster.MASTER);
Configuration conf = master.getConfiguration();
-
- MetaTableLocator metaTableLocator = new MetaTableLocator();
String fqtn = request.getParameter("name");
final String escaped_fqtn = StringEscapeUtils.escapeHtml4(fqtn);
Table table;
@@ -202,7 +200,7 @@ if ( fqtn != null ) {
for (int j = 0; j < numMetaReplicas; j++) {
RegionInfo meta = RegionReplicaUtil.getRegionInfoForReplica(
RegionInfoBuilder.FIRST_META_REGIONINFO, j);
- ServerName metaLocation = metaTableLocator.waitMetaRegionLocation(master.getZooKeeper(), j, 1);
+ ServerName metaLocation = MetaTableLocator.waitMetaRegionLocation(master.getZooKeeper(), j, 1);
for (int i = 0; i < 1; i++) {
String hostAndPort = "";
String readReq = "N/A";
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
index c0a2a8c..ff0a88c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
@@ -55,7 +55,6 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequester;
import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -162,11 +161,6 @@ public class MockRegionServerServices implements RegionServerServices {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ZKWatcher getZooKeeper() {
return zkw;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java
index 0d21fb8..5582178 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java
@@ -247,13 +247,11 @@ public class TestMetaTableAccessor {
abstract void metaTask() throws Throwable;
}
- @Test public void testGetRegionsFromMetaTable()
- throws IOException, InterruptedException {
- List<RegionInfo> regions =
- new MetaTableLocator().getMetaRegions(UTIL.getZooKeeperWatcher());
+ @Test
+ public void testGetRegionsFromMetaTable() throws IOException, InterruptedException {
+ List<RegionInfo> regions = MetaTableLocator.getMetaRegions(UTIL.getZooKeeperWatcher());
assertTrue(regions.size() >= 1);
- assertTrue(new MetaTableLocator().getMetaRegionsAndLocations(
- UTIL.getZooKeeperWatcher()).size() >= 1);
+ assertTrue(MetaTableLocator.getMetaRegionsAndLocations(UTIL.getZooKeeperWatcher()).size() >= 1);
}
@Test public void testTableExists() throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java
index db93c0c..9274fa0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java
@@ -18,18 +18,10 @@
package org.apache.hadoop.hbase;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
-import java.net.ConnectException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.ClusterConnection;
-import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
-import org.apache.hadoop.hbase.ipc.HBaseRpcController;
-import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
-import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
@@ -51,8 +43,6 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.GetRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.GetResponse;
@@ -60,31 +50,34 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.GetRespons
/**
* Test {@link org.apache.hadoop.hbase.zookeeper.MetaTableLocator}
*/
-@Category({MiscTests.class, MediumTests.class})
+@Category({ MiscTests.class, MediumTests.class })
public class TestMetaTableLocator {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestMetaTableLocator.class);
+ HBaseClassTestRule.forClass(TestMetaTableLocator.class);
private static final Logger LOG = LoggerFactory.getLogger(TestMetaTableLocator.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static final ServerName SN =
- ServerName.valueOf("example.org", 1234, System.currentTimeMillis());
+ ServerName.valueOf("example.org", 1234, System.currentTimeMillis());
private ZKWatcher watcher;
private Abortable abortable;
- @BeforeClass public static void beforeClass() throws Exception {
+ @BeforeClass
+ public static void beforeClass() throws Exception {
// Set this down so tests run quicker
UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
UTIL.startMiniZKCluster();
}
- @AfterClass public static void afterClass() throws IOException {
+ @AfterClass
+ public static void afterClass() throws IOException {
UTIL.getZkCluster().shutdown();
}
- @Before public void before() throws IOException {
+ @Before
+ public void before() throws IOException {
this.abortable = new Abortable() {
@Override
public void abort(String why, Throwable e) {
@@ -92,19 +85,20 @@ public class TestMetaTableLocator {
}
@Override
- public boolean isAborted() {
+ public boolean isAborted() {
return false;
}
};
- this.watcher = new ZKWatcher(UTIL.getConfiguration(),
- this.getClass().getSimpleName(), this.abortable, true);
+ this.watcher =
+ new ZKWatcher(UTIL.getConfiguration(), this.getClass().getSimpleName(), this.abortable, true);
}
- @After public void after() {
+ @After
+ public void after() {
try {
// Clean out meta location or later tests will be confused... they presume
// start fresh in zk.
- new MetaTableLocator().deleteMetaLocation(this.watcher);
+ MetaTableLocator.deleteMetaLocation(this.watcher);
} catch (KeeperException e) {
LOG.warn("Unable to delete hbase:meta location", e);
}
@@ -115,178 +109,47 @@ public class TestMetaTableLocator {
/**
* Test normal operations
*/
- @Test public void testMetaLookup()
- throws IOException, InterruptedException, ServiceException, KeeperException {
+ @Test
+ public void testMetaLookup()
+ throws IOException, InterruptedException, ServiceException, KeeperException {
final ClientProtos.ClientService.BlockingInterface client =
- Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
+ Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
- Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())).
- thenReturn(GetResponse.newBuilder().build());
+ Mockito.when(client.get((RpcController) Mockito.any(), (GetRequest) Mockito.any()))
+ .thenReturn(GetResponse.newBuilder().build());
- final MetaTableLocator mtl = new MetaTableLocator();
- assertNull(mtl.getMetaRegionLocation(this.watcher));
+ assertNull(MetaTableLocator.getMetaRegionLocation(this.watcher));
for (RegionState.State state : RegionState.State.values()) {
- if (state.equals(RegionState.State.OPEN))
+ if (state.equals(RegionState.State.OPEN)) {
continue;
+ }
MetaTableLocator.setMetaLocation(this.watcher, SN, state);
- assertNull(mtl.getMetaRegionLocation(this.watcher));
+ assertNull(MetaTableLocator.getMetaRegionLocation(this.watcher));
assertEquals(state, MetaTableLocator.getMetaRegionState(this.watcher).getState());
}
MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN);
- assertEquals(SN, mtl.getMetaRegionLocation(this.watcher));
+ assertEquals(SN, MetaTableLocator.getMetaRegionLocation(this.watcher));
assertEquals(RegionState.State.OPEN,
MetaTableLocator.getMetaRegionState(this.watcher).getState());
- mtl.deleteMetaLocation(this.watcher);
+ MetaTableLocator.deleteMetaLocation(this.watcher);
assertNull(MetaTableLocator.getMetaRegionState(this.watcher).getServerName());
assertEquals(RegionState.State.OFFLINE,
- MetaTableLocator.getMetaRegionState(this.watcher).getState());
- assertNull(mtl.getMetaRegionLocation(this.watcher));
- }
-
-
- /**
- * Test interruptable while blocking wait on meta.
- * @throws IOException
- * @throws ServiceException
- * @throws InterruptedException
- */
- @Test public void testInterruptWaitOnMeta()
- throws IOException, InterruptedException, ServiceException {
- final ClientProtos.ClientService.BlockingInterface client =
- Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
-
- Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())).
- thenReturn(GetResponse.newBuilder().build());
-
- final MetaTableLocator mtl = new MetaTableLocator();
- ServerName meta = new MetaTableLocator().getMetaRegionLocation(this.watcher);
- assertNull(meta);
- Thread t = new Thread() {
- @Override
- public void run() {
- try {
- mtl.waitMetaRegionLocation(watcher);
- } catch (InterruptedException e) {
- throw new RuntimeException("Interrupted", e);
- }
- }
- };
- t.start();
- while (!t.isAlive())
- Threads.sleep(1);
- Threads.sleep(1);
- assertTrue(t.isAlive());
- mtl.stop();
- // Join the thread... should exit shortly.
- t.join();
- }
-
- private void testVerifyMetaRegionLocationWithException(Exception ex)
- throws IOException, InterruptedException, KeeperException, ServiceException {
- // Mock an ClientProtocol.
- final ClientProtos.ClientService.BlockingInterface implementation =
- Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
-
- ClusterConnection connection = mockConnection(null, implementation);
-
- // If a 'get' is called on mocked interface, throw connection refused.
- Mockito.when(implementation.get((RpcController) Mockito.any(), (GetRequest) Mockito.any())).
- thenThrow(new ServiceException(ex));
-
- long timeout = UTIL.getConfiguration().
- getLong("hbase.catalog.verification.timeout", 1000);
- MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPENING);
- assertFalse(new MetaTableLocator().verifyMetaRegionLocation(
- connection, watcher, timeout));
-
- MetaTableLocator.setMetaLocation(this.watcher, SN, RegionState.State.OPEN);
- assertFalse(new MetaTableLocator().verifyMetaRegionLocation(
- connection, watcher, timeout));
- }
-
- /**
- * Test we survive a connection refused {@link ConnectException}
- * @throws IOException
- * @throws InterruptedException
- * @throws KeeperException
- * @throws ServiceException
- */
- @Test
- public void testGetMetaServerConnectionFails()
- throws IOException, InterruptedException, KeeperException, ServiceException {
- testVerifyMetaRegionLocationWithException(new ConnectException("Connection refused"));
- }
-
- /**
- * Test that verifyMetaRegionLocation properly handles getting a
- * ServerNotRunningException. See HBASE-4470.
- * Note this doesn't check the exact exception thrown in the
- * HBASE-4470 as there it is thrown from getHConnection() and
- * here it is thrown from get() -- but those are both called
- * from the same function anyway, and this way is less invasive than
- * throwing from getHConnection would be.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws KeeperException
- * @throws ServiceException
- */
- @Test
- public void testVerifyMetaRegionServerNotRunning()
- throws IOException, InterruptedException, KeeperException, ServiceException {
- testVerifyMetaRegionLocationWithException(new ServerNotRunningYetException("mock"));
- }
-
- /**
- * Test get of meta region fails properly if nothing to connect to.
- * @throws IOException
- * @throws InterruptedException
- * @throws KeeperException
- * @throws ServiceException
- */
- @Test
- public void testVerifyMetaRegionLocationFails()
- throws IOException, InterruptedException, KeeperException, ServiceException {
- ClusterConnection connection = Mockito.mock(ClusterConnection.class);
- ServiceException connectException =
- new ServiceException(new ConnectException("Connection refused"));
- final AdminProtos.AdminService.BlockingInterface implementation =
- Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
- Mockito.when(implementation.getRegionInfo((RpcController)Mockito.any(),
- (GetRegionInfoRequest)Mockito.any())).thenThrow(connectException);
- Mockito.when(connection.getAdmin(Mockito.any())).
- thenReturn(implementation);
- RpcControllerFactory controllerFactory = Mockito.mock(RpcControllerFactory.class);
- Mockito.when(controllerFactory.newController()).thenReturn(
- Mockito.mock(HBaseRpcController.class));
- Mockito.when(connection.getRpcControllerFactory()).thenReturn(controllerFactory);
-
- ServerName sn = ServerName.valueOf("example.com", 1234, System.currentTimeMillis());
- MetaTableLocator.setMetaLocation(this.watcher,
- sn,
- RegionState.State.OPENING);
- assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100));
- MetaTableLocator.setMetaLocation(this.watcher, sn, RegionState.State.OPEN);
- assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100));
+ MetaTableLocator.getMetaRegionState(this.watcher).getState());
+ assertNull(MetaTableLocator.getMetaRegionLocation(this.watcher));
}
- @Test (expected = NotAllMetaRegionsOnlineException.class)
- public void testTimeoutWaitForMeta()
- throws IOException, InterruptedException {
- new MetaTableLocator().waitMetaRegionLocation(watcher, 100);
+ @Test(expected = NotAllMetaRegionsOnlineException.class)
+ public void testTimeoutWaitForMeta() throws IOException, InterruptedException {
+ MetaTableLocator.waitMetaRegionLocation(watcher, 100);
}
/**
* Test waiting on meat w/ no timeout specified.
- * @throws IOException
- * @throws InterruptedException
- * @throws KeeperException
*/
- @Test public void testNoTimeoutWaitForMeta()
- throws IOException, InterruptedException, KeeperException {
- final MetaTableLocator mtl = new MetaTableLocator();
- ServerName hsa = mtl.getMetaRegionLocation(watcher);
+ @Test
+ public void testNoTimeoutWaitForMeta() throws IOException, InterruptedException, KeeperException {
+ ServerName hsa = MetaTableLocator.getMetaRegionLocation(watcher);
assertNull(hsa);
// Now test waiting on meta location getting set.
@@ -298,54 +161,12 @@ public class TestMetaTableLocator {
// Join the thread... should exit shortly.
t.join();
// Now meta is available.
- assertTrue(mtl.getMetaRegionLocation(watcher).equals(hsa));
- }
-
- /**
- * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely
- * want to pass a mocked HRS; can be null.
- * @param client A mocked ClientProtocol instance, can be null
- * @return Mock up a connection that returns a {@link Configuration} when
- * {@link org.apache.hadoop.hbase.client.ClusterConnection#getConfiguration()} is called, a 'location' when
- * {@link org.apache.hadoop.hbase.client.RegionLocator#getRegionLocation(byte[], boolean)} is called,
- * and that returns the passed {@link AdminProtos.AdminService.BlockingInterface} instance when
- * {@link org.apache.hadoop.hbase.client.ClusterConnection#getAdmin(ServerName)} is called, returns the passed
- * {@link ClientProtos.ClientService.BlockingInterface} instance when
- * {@link org.apache.hadoop.hbase.client.ClusterConnection#getClient(ServerName)} is called.
- * @throws IOException
- */
- private ClusterConnection mockConnection(final AdminProtos.AdminService.BlockingInterface admin,
- final ClientProtos.ClientService.BlockingInterface client)
- throws IOException {
- ClusterConnection connection =
- HConnectionTestingUtility.getMockedConnection(UTIL.getConfiguration());
- Mockito.doNothing().when(connection).close();
- // Make it so we return any old location when asked.
- final HRegionLocation anyLocation = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, SN);
- Mockito.when(connection.getRegionLocation((TableName) Mockito.any(),
- (byte[]) Mockito.any(), Mockito.anyBoolean())).
- thenReturn(anyLocation);
- Mockito.when(connection.locateRegion((TableName) Mockito.any(),
- (byte[]) Mockito.any())).
- thenReturn(anyLocation);
- if (admin != null) {
- // If a call to getHRegionConnection, return this implementation.
- Mockito.when(connection.getAdmin(Mockito.any())).
- thenReturn(admin);
- }
- if (client != null) {
- // If a call to getClient, return this implementation.
- Mockito.when(connection.getClient(Mockito.any())).
- thenReturn(client);
- }
- return connection;
+ assertTrue(MetaTableLocator.getMetaRegionLocation(watcher).equals(hsa));
}
private void startWaitAliveThenWaitItLives(final Thread t, final int ms) {
t.start();
- while(!t.isAlive()) {
- // Wait
- }
+ UTIL.waitFor(2000, t::isAlive);
// Wait one second.
Threads.sleep(ms);
assertTrue("Assert " + t.getName() + " still waiting", t.isAlive());
@@ -372,9 +193,13 @@ public class TestMetaTableLocator {
void doWaiting() throws InterruptedException {
try {
- while (new MetaTableLocator().waitMetaRegionLocation(watcher, 10000) == null);
+ for (;;) {
+ if (MetaTableLocator.waitMetaRegionLocation(watcher, 10000) != null) {
+ break;
+ }
+ }
} catch (NotAllMetaRegionsOnlineException e) {
- //Ignore
+ // Ignore
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
index bac588a..3b14b7f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
import org.apache.hadoop.hbase.util.HBaseFsckRepair;
import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker;
+import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
@@ -100,8 +101,7 @@ public class TestMetaWithReplicas {
AssignmentManager am = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
Set<ServerName> sns = new HashSet<ServerName>();
ServerName hbaseMetaServerName =
- TEST_UTIL.getMiniHBaseCluster().getMaster().getMetaTableLocator().
- getMetaRegionLocation(TEST_UTIL.getZooKeeperWatcher());
+ MetaTableLocator.getMetaRegionLocation(TEST_UTIL.getZooKeeperWatcher());
LOG.info("HBASE:META DEPLOY: on " + hbaseMetaServerName);
sns.add(hbaseMetaServerName);
for (int replicaId = 1; replicaId < 3; replicaId++) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index ac20dbd..9c55f57 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
public class MockNoopMasterServices implements MasterServices {
@@ -161,11 +160,6 @@ public class MockNoopMasterServices implements MasterServices {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ClusterConnection getConnection() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
index f4c2a33..844b705 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
@@ -68,7 +68,6 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequester;
import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.WAL;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
@@ -305,11 +304,6 @@ class MockRegionServer implements AdminProtos.AdminService.BlockingInterface,
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ServerName getServerName() {
return this.sn;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
index 5851dea..2300f54 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKListener;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -317,11 +316,6 @@ public class TestActiveMasterManager {
return null;
}
- @Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
public ClusterStatusTracker getClusterStatusTracker() {
return clusterStatusTracker;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
index 9da4df4..5c8db3e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileCleaner.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.EnvironmentEdge;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -233,11 +232,6 @@ public class TestHFileCleaner {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ServerName getServerName() {
return ServerName.valueOf("regionserver,60020,000000");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
index c011ea8..119194b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.junit.ClassRule;
import org.junit.Rule;
@@ -169,11 +168,6 @@ public class TestHFileLinkCleaner {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ServerName getServerName() {
return ServerName.valueOf("regionserver,60020,000000");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
index 882ea9d..247ed01 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
@@ -30,7 +30,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.concurrent.atomic.AtomicBoolean;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -55,7 +54,6 @@ import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.zookeeper.KeeperException;
@@ -367,11 +365,6 @@ public class TestLogsCleaner {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ServerName getServerName() {
return ServerName.valueOf("regionserver,60020,000000");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
index 24b930c..d162bf3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
@@ -55,7 +55,6 @@ import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.zookeeper.KeeperException;
@@ -257,11 +256,6 @@ public class TestReplicationHFileCleaner {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ServerName getServerName() {
return ServerName.valueOf("regionserver,60020,000000");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
index 9f05a73..8c9ce75 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.junit.ClassRule;
import org.junit.Test;
@@ -834,11 +833,6 @@ public class TestHeapMemoryManager {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ServerName getServerName() {
return ServerName.valueOf("server1",4000,12345);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
index af2861f..b9f89b7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java
@@ -111,8 +111,7 @@ public class TestRegionServerNoMaster {
HRegionServer hrs = HTU.getHBaseCluster()
.getLiveRegionServerThreads().get(0).getRegionServer();
ZKWatcher zkw = hrs.getZooKeeper();
- MetaTableLocator mtl = new MetaTableLocator();
- ServerName sn = mtl.getMetaRegionLocation(zkw);
+ ServerName sn = MetaTableLocator.getMetaRegionLocation(zkw);
if (sn != null && !masterAddr.equals(sn)) {
return;
}
@@ -120,7 +119,7 @@ public class TestRegionServerNoMaster {
ProtobufUtil.openRegion(null, hrs.getRSRpcServices(),
hrs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO);
while (true) {
- sn = mtl.getMetaRegionLocation(zkw);
+ sn = MetaTableLocator.getMetaRegionLocation(zkw);
if (sn != null && sn.equals(hrs.getServerName())
&& hrs.onlineRegions.containsKey(
HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
index 0c38ee3..cbf932c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.hbase.executor.ExecutorType;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.CancelableProgressable;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -136,11 +135,6 @@ public class TestSplitLogWorker {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ChoreService getChoreService() {
return null;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
index 84b8d6c..0e20252 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWALLockup.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.wal.WALKey;
import org.apache.hadoop.hbase.wal.WALKeyImpl;
import org.apache.hadoop.hbase.wal.WALProvider.Writer;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.junit.After;
import org.junit.Assert;
@@ -475,11 +474,6 @@ public class TestWALLockup {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ServerName getServerName() {
return ServerName.valueOf(this.serverName);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
index 2419095..863d558 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationTrackerZKImpl.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -214,11 +213,6 @@ public class TestReplicationTrackerZKImpl {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ServerName getServerName() {
return ServerName.valueOf(this.serverName);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index 0872ea7..86bbb09 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -88,7 +88,6 @@ import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.hbase.wal.WALKeyImpl;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -858,11 +857,6 @@ public abstract class TestReplicationSourceManager {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ServerName getServerName() {
return ServerName.valueOf(hostname, 1234, 1L);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
index afde037..e4780f1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestTokenAuthentication.java
@@ -70,7 +70,6 @@ import org.apache.hadoop.hbase.util.Sleeper;
import org.apache.hadoop.hbase.util.Strings;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.net.DNS;
@@ -216,11 +215,6 @@ public class TestTokenAuthentication {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ZKWatcher getZooKeeper() {
return zookeeper;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8bf966c8/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java
index 551b940..c25db01 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/MockServer.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.util;
import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.ChoreService;
@@ -30,7 +29,6 @@ import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.log.HBaseMarkers;
-import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -110,11 +108,6 @@ public class MockServer implements Server {
}
@Override
- public MetaTableLocator getMetaTableLocator() {
- return null;
- }
-
- @Override
public ServerName getServerName() {
return NAME;
}
[04/16] hbase git commit: HBASE-21551 Memory leak when use scan with
STREAM at server side
Posted by zh...@apache.org.
HBASE-21551 Memory leak when use scan with STREAM at server side
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b854859
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b854859
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b854859
Branch: refs/heads/HBASE-21512
Commit: 3b854859f6fad44cbf31164374569a6ab23f3623
Parents: f49baf2
Author: huzheng <op...@gmail.com>
Authored: Wed Dec 5 22:57:49 2018 +0800
Committer: huzheng <op...@gmail.com>
Committed: Thu Dec 6 10:55:42 2018 +0800
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/HStoreFile.java | 3 +-
.../hbase/regionserver/StoreFileReader.java | 3 ++
.../regionserver/TestSwitchToStreamRead.java | 50 ++++++++++++++++++++
3 files changed, 55 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
index 4aff949..9c94990 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java
@@ -126,7 +126,8 @@ public class HStoreFile implements StoreFile, StoreFileReader.Listener {
private final AtomicInteger refCount = new AtomicInteger(0);
// Set implementation must be of concurrent type
- private final Set<StoreFileReader> streamReaders;
+ @VisibleForTesting
+ final Set<StoreFileReader> streamReaders;
private final boolean noReadahead;
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
index 3fbddf2..d9008b2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
@@ -186,6 +186,9 @@ public class StoreFileReader {
if (!shared) {
try {
reader.close(false);
+ if (this.listener != null) {
+ this.listener.storeFileReaderClosed(this);
+ }
} catch (IOException e) {
LOG.warn("failed to close stream reader", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b854859/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
index 815643d..037b13e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSwitchToStreamRead.java
@@ -23,8 +23,13 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -33,6 +38,7 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Scan.ReadType;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
@@ -42,6 +48,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Ignore;
@@ -98,6 +105,49 @@ public class TestSwitchToStreamRead {
UTIL.cleanupTestDir();
}
+ private Set<StoreFileReader> getStreamReaders() {
+ List<HStore> stores = REGION.getStores();
+ Assert.assertEquals(1, stores.size());
+ HStore firstStore = stores.get(0);
+ Assert.assertNotNull(firstStore);
+ Collection<HStoreFile> storeFiles = firstStore.getStorefiles();
+ Assert.assertEquals(1, storeFiles.size());
+ HStoreFile firstSToreFile = storeFiles.iterator().next();
+ Assert.assertNotNull(firstSToreFile);
+ return Collections.unmodifiableSet(firstSToreFile.streamReaders);
+ }
+
+ /**
+ * Test Case for HBASE-21551
+ */
+ @Test
+ public void testStreamReadersCleanup() throws IOException {
+ Set<StoreFileReader> streamReaders = getStreamReaders();
+ Assert.assertEquals(0, getStreamReaders().size());
+ try (RegionScannerImpl scanner = REGION.getScanner(new Scan().setReadType(ReadType.STREAM))) {
+ StoreScanner storeScanner =
+ (StoreScanner) (scanner).getStoreHeapForTesting().getCurrentForTesting();
+ List<StoreFileScanner> sfScanners = storeScanner.getAllScannersForTesting().stream()
+ .filter(kvs -> kvs instanceof StoreFileScanner).map(kvs -> (StoreFileScanner) kvs)
+ .collect(Collectors.toList());
+ Assert.assertEquals(1, sfScanners.size());
+ StoreFileScanner sfScanner = sfScanners.get(0);
+ Assert.assertFalse(sfScanner.getReader().shared);
+
+ // There should be a stream reader
+ Assert.assertEquals(1, getStreamReaders().size());
+ }
+ Assert.assertEquals(0, getStreamReaders().size());
+
+ // The streamsReader should be clear after region close even if there're some opened stream
+ // scanner.
+ RegionScannerImpl scanner = REGION.getScanner(new Scan().setReadType(ReadType.STREAM));
+ Assert.assertNotNull(scanner);
+ Assert.assertEquals(1, getStreamReaders().size());
+ REGION.close();
+ Assert.assertEquals(0, streamReaders.size());
+ }
+
@Test
public void test() throws IOException {
try (RegionScannerImpl scanner = REGION.getScanner(new Scan())) {
[10/16] hbase git commit: HBASE-21413 Empty meta log doesn't get
split when restart whole cluster Signed-off-by: stack
Posted by zh...@apache.org.
HBASE-21413 Empty meta log doesn't get split when restart whole cluster
Signed-off-by: stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e0e0694f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e0e0694f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e0e0694f
Branch: refs/heads/HBASE-21512
Commit: e0e0694fa18372e6551420687bcf28ffe0bbc1f8
Parents: 1e65bd5
Author: Allan Yang <al...@apache.org>
Authored: Thu Dec 6 21:13:03 2018 -0800
Committer: stack <st...@stack.corp.apple.com>
Committed: Thu Dec 6 21:21:41 2018 -0800
----------------------------------------------------------------------
.../hadoop/hbase/master/MasterWalManager.java | 40 ++++++++++
.../master/procedure/ServerCrashProcedure.java | 3 +
.../hbase/regionserver/TestCleanupMetaWAL.java | 83 ++++++++++++++++++++
3 files changed, 126 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e0e0694f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
index 2b1a81f..5ab1c28 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
@@ -356,4 +357,43 @@ public class MasterWalManager {
}
}
}
+
+ /**
+ * For meta region open and closed normally on a server, it may leave some meta
+ * WAL in the server's wal dir. Since meta region is no long on this server,
+ * The SCP won't split those meta wals, just leaving them there. So deleting
+ * the wal dir will fail since the dir is not empty. Actually We can safely achive those
+ * meta log and Archiving the meta log and delete the dir.
+ * @param serverName the server to archive meta log
+ */
+ public void archiveMetaLog(final ServerName serverName) {
+ try {
+ Path logDir = new Path(this.rootDir,
+ AbstractFSWALProvider.getWALDirectoryName(serverName.toString()));
+ Path splitDir = logDir.suffix(AbstractFSWALProvider.SPLITTING_EXT);
+ if (fs.exists(splitDir)) {
+ FileStatus[] logfiles = FSUtils.listStatus(fs, splitDir, META_FILTER);
+ if (logfiles != null) {
+ for (FileStatus status : logfiles) {
+ if (!status.isDir()) {
+ Path newPath = AbstractFSWAL.getWALArchivePath(this.oldLogDir,
+ status.getPath());
+ if (!FSUtils.renameAndSetModifyTime(fs, status.getPath(), newPath)) {
+ LOG.warn("Unable to move " + status.getPath() + " to " + newPath);
+ } else {
+ LOG.debug("Archived meta log " + status.getPath() + " to " + newPath);
+ }
+ }
+ }
+ }
+ if (!fs.delete(splitDir, false)) {
+ LOG.warn("Unable to delete log dir. Ignoring. " + splitDir);
+ }
+ }
+ } catch (IOException ie) {
+ LOG.warn("Failed archiving meta log for server " + serverName, ie);
+ }
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e0e0694f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
index 048bca8..b93f8fa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
@@ -221,6 +221,9 @@ public class ServerCrashProcedure
// PROBLEM!!! WE BLOCK HERE.
am.getRegionStates().logSplitting(this.serverName);
mwm.splitLog(this.serverName);
+ if (!carryingMeta) {
+ mwm.archiveMetaLog(this.serverName);
+ }
am.getRegionStates().logSplit(this.serverName);
LOG.debug("Done splitting WALs {}", this);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e0e0694f/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCleanupMetaWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCleanupMetaWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCleanupMetaWAL.java
new file mode 100644
index 0000000..4a723c0
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCleanupMetaWAL.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.hbase.wal.AbstractFSWALProvider.SPLITTING_EXT;
+import static org.junit.Assert.fail;
+
+@Category(MediumTests.class)
+public class TestCleanupMetaWAL {
+ private static final Logger LOG = LoggerFactory.getLogger(TestCleanupMetaWAL.class);
+
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestCleanupMetaWAL.class);
+
+ @BeforeClass
+ public static void before() throws Exception {
+ TEST_UTIL.startMiniCluster(2);
+ }
+
+ @AfterClass
+ public static void after() throws Exception {
+ TEST_UTIL.shutdownMiniZKCluster();
+ }
+
+ @Test
+ public void testCleanupMetaWAL() throws Exception {
+ TEST_UTIL.createTable(TableName.valueOf("test"), "cf");
+ HRegionServer serverWithMeta = TEST_UTIL.getMiniHBaseCluster()
+ .getRegionServer(TEST_UTIL.getMiniHBaseCluster().getServerWithMeta());
+ TEST_UTIL.getAdmin()
+ .move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), null);
+ TEST_UTIL.getMiniHBaseCluster().killRegionServer(serverWithMeta.getServerName());
+ TEST_UTIL.waitFor(10000, () ->
+ TEST_UTIL.getMiniHBaseCluster().getMaster().getProcedures().stream()
+ .filter(p -> p instanceof ServerCrashProcedure && p.isFinished()).count() > 0);
+ MasterFileSystem fs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
+ Path walPath = new Path(fs.getWALRootDir(), HConstants.HREGION_LOGDIR_NAME);
+ for (FileStatus status : FSUtils.listStatus(fs.getFileSystem(), walPath)) {
+ if (status.getPath().toString().contains(SPLITTING_EXT)) {
+ fail("Should not have splitting wal dir here:" + status);
+ }
+ }
+
+
+ }
+}
[06/16] hbase git commit: Add 'strong' notice that 2.1.1 and 2.0.3
have a memory leak
Posted by zh...@apache.org.
Add 'strong' notice that 2.1.1 and 2.0.3 have a memory leak
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/12e75a8a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/12e75a8a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/12e75a8a
Branch: refs/heads/HBASE-21512
Commit: 12e75a8a635785b279900b6905c86a1617526c72
Parents: 67ab8b8
Author: stack <st...@apache.org>
Authored: Wed Dec 5 21:50:39 2018 -0800
Committer: stack <st...@apache.org>
Committed: Wed Dec 5 21:50:39 2018 -0800
----------------------------------------------------------------------
src/site/xdoc/downloads.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/12e75a8a/src/site/xdoc/downloads.xml
----------------------------------------------------------------------
diff --git a/src/site/xdoc/downloads.xml b/src/site/xdoc/downloads.xml
index 7e81afd..5d3f2a6 100644
--- a/src/site/xdoc/downloads.xml
+++ b/src/site/xdoc/downloads.xml
@@ -32,6 +32,7 @@ under the License.
<a href="https://www.apache.org/dyn/closer.cgi#verify">Verify The Integrity Of The Files</a> for
how to verify your mirrored downloads.
</p>
+ <p style="color:red;"><strong>NOTE: 2.1.1 and 2.0.3 have a serious memory leak. See HBASE-21551. We are working on replacement releases.</strong></p>
<section name="Releases">
<table>
<tr>
[12/16] hbase git commit: HBASE-21560 Return a new TableDescriptor
for MasterObserver#preModifyTable to allow coprocessor modify the
TableDescriptor
Posted by zh...@apache.org.
HBASE-21560 Return a new TableDescriptor for MasterObserver#preModifyTable to allow coprocessor modify the TableDescriptor
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/79d90c87
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/79d90c87
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/79d90c87
Branch: refs/heads/HBASE-21512
Commit: 79d90c87b5bc6d4aa50e6edc52a3f20da708ee29
Parents: 8d7061a
Author: Guanghao Zhang <zg...@apache.org>
Authored: Fri Dec 7 16:51:19 2018 +0800
Committer: Guanghao Zhang <zg...@apache.org>
Committed: Sat Dec 8 09:28:14 2018 +0800
----------------------------------------------------------------------
.../hbase/coprocessor/MasterObserver.java | 6 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 11 +-
.../hbase/master/MasterCoprocessorHost.java | 22 ++--
.../hbase/security/access/AccessController.java | 9 +-
.../CoprocessorWhitelistMasterObserver.java | 5 +-
.../visibility/VisibilityController.java | 15 ++-
.../hbase/coprocessor/TestMasterObserver.java | 3 +-
.../TestMasterObserverToModifyTableSchema.java | 128 +++++++++++++++++++
8 files changed, 169 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/79d90c87/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
index a0863e4..1a8db79 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
@@ -240,11 +240,13 @@ public interface MasterObserver {
* @param currentDescriptor current TableDescriptor of the table
* @param newDescriptor after modify operation, table will have this descriptor
*/
- default void preModifyTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
+ default TableDescriptor preModifyTable(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final TableName tableName, TableDescriptor currentDescriptor, TableDescriptor newDescriptor)
- throws IOException {
+ throws IOException {
preModifyTable(ctx, tableName, newDescriptor);
+ return newDescriptor;
}
+
/**
* Called after the modifyTable operation has been requested. Called as part
* of modify table RPC call.
http://git-wip-us.apache.org/repos/asf/hbase/blob/79d90c87/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index e96dc36..a16e09d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2631,13 +2631,12 @@ public class HMaster extends HRegionServer implements MasterServices {
.submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
@Override
protected void run() throws IOException {
- TableDescriptor newDescriptor = newDescriptorGetter.get();
- sanityCheckTableDescriptor(newDescriptor);
TableDescriptor oldDescriptor = getMaster().getTableDescriptors().get(tableName);
- getMaster().getMasterCoprocessorHost().preModifyTable(tableName, oldDescriptor,
- newDescriptor);
-
- LOG.info(getClientIdAuditPrefix() + " modify " + tableName);
+ TableDescriptor newDescriptor = getMaster().getMasterCoprocessorHost()
+ .preModifyTable(tableName, oldDescriptor, newDescriptorGetter.get());
+ sanityCheckTableDescriptor(newDescriptor);
+ LOG.info("{} modify table {} from {} to {}", getClientIdAuditPrefix(), tableName,
+ oldDescriptor, newDescriptor);
// Execute the operation synchronously - wait for the operation completes before
// continuing.
http://git-wip-us.apache.org/repos/asf/hbase/blob/79d90c87/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index 51e30c4..e7b166c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -446,14 +446,20 @@ public class MasterCoprocessorHost
});
}
- public void preModifyTable(final TableName tableName, final TableDescriptor currentDescriptor,
- final TableDescriptor newDescriptor) throws IOException {
- execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
- @Override
- public void call(MasterObserver observer) throws IOException {
- observer.preModifyTable(this, tableName, currentDescriptor, newDescriptor);
- }
- });
+ public TableDescriptor preModifyTable(final TableName tableName,
+ final TableDescriptor currentDescriptor, final TableDescriptor newDescriptor)
+ throws IOException {
+ if (coprocEnvironments.isEmpty()) {
+ return newDescriptor;
+ }
+ return execOperationWithResult(
+ new ObserverOperationWithResult<MasterObserver, TableDescriptor>(masterObserverGetter,
+ newDescriptor) {
+ @Override
+ protected TableDescriptor call(MasterObserver observer) throws IOException {
+ return observer.preModifyTable(this, tableName, currentDescriptor, getResult());
+ }
+ });
}
public void postModifyTable(final TableName tableName, final TableDescriptor oldDescriptor,
http://git-wip-us.apache.org/repos/asf/hbase/blob/79d90c87/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
index 835fc0d..82ec12d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
@@ -970,11 +970,12 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
}
@Override
- public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName,
- TableDescriptor currentDesc, TableDescriptor newDesc) throws IOException {
+ public TableDescriptor preModifyTable(ObserverContext<MasterCoprocessorEnvironment> c,
+ TableName tableName, TableDescriptor currentDesc, TableDescriptor newDesc)
+ throws IOException {
// TODO: potentially check if this is a add/modify/delete column operation
- requirePermission(c, "modifyTable",
- tableName, null, null, Action.ADMIN, Action.CREATE);
+ requirePermission(c, "modifyTable", tableName, null, null, Action.ADMIN, Action.CREATE);
+ return newDesc;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/79d90c87/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/CoprocessorWhitelistMasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/CoprocessorWhitelistMasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/CoprocessorWhitelistMasterObserver.java
index 719fe33..1e83e96 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/CoprocessorWhitelistMasterObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/CoprocessorWhitelistMasterObserver.java
@@ -54,10 +54,11 @@ public class CoprocessorWhitelistMasterObserver implements MasterCoprocessor, Ma
}
@Override
- public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ public TableDescriptor preModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
TableName tableName, TableDescriptor currentDesc, TableDescriptor newDesc)
- throws IOException {
+ throws IOException {
verifyCoprocessors(ctx, newDesc);
+ return newDesc;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/79d90c87/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
index 1f54afc..c4f3b95 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityController.java
@@ -226,14 +226,15 @@ public class VisibilityController implements MasterCoprocessor, RegionCoprocesso
}
@Override
- public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
- TableDescriptor currentDescriptor, TableDescriptor newDescriptor) throws IOException {
- if (!authorizationEnabled) {
- return;
- }
- if (LABELS_TABLE_NAME.equals(tableName)) {
- throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
+ public TableDescriptor preModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
+ TableName tableName, TableDescriptor currentDescriptor, TableDescriptor newDescriptor)
+ throws IOException {
+ if (authorizationEnabled) {
+ if (LABELS_TABLE_NAME.equals(tableName)) {
+ throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
+ }
}
+ return newDescriptor;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/79d90c87/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index d8a5b4c..58f4b9b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -373,10 +373,11 @@ public class TestMasterObserver {
}
@Override
- public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> env,
+ public TableDescriptor preModifyTable(ObserverContext<MasterCoprocessorEnvironment> env,
TableName tableName, final TableDescriptor currentDescriptor,
final TableDescriptor newDescriptor) throws IOException {
preModifyTableCalled = true;
+ return newDescriptor;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/79d90c87/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserverToModifyTableSchema.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserverToModifyTableSchema.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserverToModifyTableSchema.java
new file mode 100644
index 0000000..d23a4a8
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserverToModifyTableSchema.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Optional;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+@Category({ CoprocessorTests.class, MediumTests.class })
+public class TestMasterObserverToModifyTableSchema {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMasterObserverToModifyTableSchema.class);
+
+ private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ private static TableName TABLENAME = TableName.valueOf("TestTable");
+
+ @Rule
+ public TestName name = new TestName();
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ Configuration conf = UTIL.getConfiguration();
+ conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
+ OnlyOneVersionAllowedMasterObserver.class.getName());
+ UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testMasterObserverToModifyTableSchema() throws IOException {
+ TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLENAME);
+ for (int i = 1; i <= 3; i++) {
+ builder.setColumnFamily(
+ ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf" + i)).setMaxVersions(i)
+ .build());
+ }
+ try (Admin admin = UTIL.getAdmin()) {
+ admin.createTable(builder.build());
+ assertOneVersion(admin.getDescriptor(TABLENAME));
+
+ builder.modifyColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf1"))
+ .setMaxVersions(Integer.MAX_VALUE).build());
+ admin.modifyTable(builder.build());
+ assertOneVersion(admin.getDescriptor(TABLENAME));
+ }
+ }
+
+ private void assertOneVersion(TableDescriptor td) {
+ for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) {
+ assertEquals(1, cfd.getMaxVersions());
+ }
+ }
+
+ public static class OnlyOneVersionAllowedMasterObserver
+ implements MasterCoprocessor, MasterObserver {
+
+ @Override
+ public Optional<MasterObserver> getMasterObserver() {
+ return Optional.of(this);
+ }
+
+ @Override
+ public TableDescriptor preCreateTableRegionsInfos(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, TableDescriptor desc)
+ throws IOException {
+ TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(desc);
+ for (ColumnFamilyDescriptor cfd : desc.getColumnFamilies()) {
+ builder.modifyColumnFamily(
+ ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(1).build());
+ }
+ return builder.build();
+ }
+
+ @Override
+ public TableDescriptor preModifyTable(ObserverContext<MasterCoprocessorEnvironment> env,
+ TableName tableName, final TableDescriptor currentDescriptor,
+ final TableDescriptor newDescriptor) throws IOException {
+ TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(newDescriptor);
+ for (ColumnFamilyDescriptor cfd : newDescriptor.getColumnFamilies()) {
+ builder.modifyColumnFamily(
+ ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(1).build());
+ }
+ return builder.build();
+ }
+ }
+}
[09/16] hbase git commit: HBASE-21549 Add shell command for serial
replication peer
Posted by zh...@apache.org.
HBASE-21549 Add shell command for serial replication peer
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1e65bd5c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1e65bd5c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1e65bd5c
Branch: refs/heads/HBASE-21512
Commit: 1e65bd5cf96cf5affd446596ef10b1034e2e0a88
Parents: dfb9ae8
Author: Guanghao Zhang <zg...@apache.org>
Authored: Wed Dec 5 18:05:03 2018 +0800
Committer: Guanghao Zhang <zg...@apache.org>
Committed: Fri Dec 7 10:10:13 2018 +0800
----------------------------------------------------------------------
.../src/main/ruby/hbase/replication_admin.rb | 5 +++++
hbase-shell/src/main/ruby/hbase_constants.rb | 1 +
.../src/main/ruby/shell/commands/add_peer.rb | 4 ++++
.../main/ruby/shell/commands/set_peer_serial.rb | 4 ++--
.../test/ruby/hbase/replication_admin_test.rb | 22 +++++++++++++++++++
src/main/asciidoc/_chapters/ops_mgt.adoc | 23 +++++++++++++++++++-
6 files changed, 56 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e65bd5c/hbase-shell/src/main/ruby/hbase/replication_admin.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase/replication_admin.rb b/hbase-shell/src/main/ruby/hbase/replication_admin.rb
index 5f86365..c01b6ea 100644
--- a/hbase-shell/src/main/ruby/hbase/replication_admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/replication_admin.rb
@@ -66,6 +66,7 @@ module Hbase
namespaces = args.fetch(NAMESPACES, nil)
peer_state = args.fetch(STATE, nil)
remote_wal_dir = args.fetch(REMOTE_WAL_DIR, nil)
+ serial = args.fetch(SERIAL, nil)
# Create and populate a ReplicationPeerConfig
builder = ReplicationPeerConfig.newBuilder()
@@ -79,6 +80,10 @@ module Hbase
builder.setRemoteWALDir(remote_wal_dir)
end
+ unless serial.nil?
+ builder.setSerial(serial)
+ end
+
unless config.nil?
builder.putAllConfiguration(config)
end
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e65bd5c/hbase-shell/src/main/ruby/hbase_constants.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/hbase_constants.rb b/hbase-shell/src/main/ruby/hbase_constants.rb
index 2870dfb..4c1ad22 100644
--- a/hbase-shell/src/main/ruby/hbase_constants.rb
+++ b/hbase-shell/src/main/ruby/hbase_constants.rb
@@ -78,6 +78,7 @@ module HBaseConstants
ENDPOINT_CLASSNAME = 'ENDPOINT_CLASSNAME'.freeze
CLUSTER_KEY = 'CLUSTER_KEY'.freeze
REMOTE_WAL_DIR = 'REMOTE_WAL_DIR'.freeze
+ SERIAL = 'SERIAL'.freeze
TABLE_CFS = 'TABLE_CFS'.freeze
NAMESPACES = 'NAMESPACES'.freeze
STATE = 'STATE'.freeze
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e65bd5c/hbase-shell/src/main/ruby/shell/commands/add_peer.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/add_peer.rb b/hbase-shell/src/main/ruby/shell/commands/add_peer.rb
index 4b6f294..9be42ac 100644
--- a/hbase-shell/src/main/ruby/shell/commands/add_peer.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/add_peer.rb
@@ -34,6 +34,8 @@ An optional parameter for namespaces identifies which namespace's tables will be
to the peer cluster.
An optional parameter for table column families identifies which tables and/or column families
will be replicated to the peer cluster.
+An optional parameter for serial flag identifies whether or not the replication peer is a serial
+replication peer. The default serial flag is false.
Note: Set a namespace in the peer config means that all tables in this namespace
will be replicated to the peer cluster. So if you already have set a namespace in peer config,
@@ -50,6 +52,8 @@ Examples:
NAMESPACES => ["ns1", "ns2", "ns3"]
hbase> add_peer '2', CLUSTER_KEY => "zk1,zk2,zk3:2182:/hbase-prod",
NAMESPACES => ["ns1", "ns2"], TABLE_CFS => { "ns3:table1" => [], "ns3:table2" => ["cf1"] }
+ hbase> add_peer '3', CLUSTER_KEY => "zk1,zk2,zk3:2182:/hbase-prod",
+ NAMESPACES => ["ns1", "ns2", "ns3"], SERIAL => true
For a custom replication endpoint, the ENDPOINT_CLASSNAME can be provided. Two optional arguments
are DATA and CONFIG which can be specified to set different either the peer_data or configuration
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e65bd5c/hbase-shell/src/main/ruby/shell/commands/set_peer_serial.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/set_peer_serial.rb b/hbase-shell/src/main/ruby/shell/commands/set_peer_serial.rb
index d556077..a6484cd 100644
--- a/hbase-shell/src/main/ruby/shell/commands/set_peer_serial.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/set_peer_serial.rb
@@ -41,8 +41,8 @@ module Shell
EOF
end
- def command(id, peer_serial)
- replication_admin.set_peer_serial(id, peer_serial)
+ def command(id, serial)
+ replication_admin.set_peer_serial(id, serial)
end
end
end
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e65bd5c/hbase-shell/src/test/ruby/hbase/replication_admin_test.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/test/ruby/hbase/replication_admin_test.rb b/hbase-shell/src/test/ruby/hbase/replication_admin_test.rb
index f44fd8c..f4c771e 100644
--- a/hbase-shell/src/test/ruby/hbase/replication_admin_test.rb
+++ b/hbase-shell/src/test/ruby/hbase/replication_admin_test.rb
@@ -100,6 +100,27 @@ module Hbase
command(:remove_peer, @peer_id)
end
+ define_test "add_peer: serial" do
+ cluster_key = "server1.cie.com:2181:/hbase"
+ remote_wal_dir = "hdfs://srv1:9999/hbase"
+ table_cfs = { "ns3:table1" => [], "ns3:table2" => [],
+ "ns3:table3" => [] }
+ # add a new replication peer which serial flag is true
+ args = { CLUSTER_KEY => cluster_key, SERIAL => true,
+ TABLE_CFS => table_cfs}
+ command(:add_peer, @peer_id, args)
+
+ assert_equal(1, command(:list_peers).length)
+ peer = command(:list_peers).get(0)
+ assert_equal(@peer_id, peer.getPeerId)
+ assert_equal(cluster_key, peer.getPeerConfig.getClusterKey)
+ assert_equal(true, peer.getPeerConfig.isSerial)
+ assert_tablecfs_equal(table_cfs, peer.getPeerConfig.getTableCFsMap())
+
+ # cleanup for future tests
+ command(:remove_peer, @peer_id)
+ end
+
define_test "add_peer: remote wal dir" do
cluster_key = "server1.cie.com:2181:/hbase"
remote_wal_dir = "hdfs://srv1:9999/hbase"
@@ -490,6 +511,7 @@ module Hbase
assert_equal(1, command(:list_peers).length)
peer_config = command(:list_peers).get(0).getPeerConfig
+ # the default serial flag is false
assert_equal(false, peer_config.isSerial)
command(:set_peer_serial, @peer_id, true)
http://git-wip-us.apache.org/repos/asf/hbase/blob/1e65bd5c/src/main/asciidoc/_chapters/ops_mgt.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc b/src/main/asciidoc/_chapters/ops_mgt.adoc
index f2ee1cc..db85b45 100644
--- a/src/main/asciidoc/_chapters/ops_mgt.adoc
+++ b/src/main/asciidoc/_chapters/ops_mgt.adoc
@@ -1898,7 +1898,28 @@ This treatment can possibly lead to data inconsistency between source and destin
.Serial replication configuration
-. Set the serial flag to true for a repliation peer. You can either set it to true when creating a replication peer, or change it to true later.
+Set the serial flag to true for a repliation peer. And the default serial flag is false.
+
+* Add a new replication peer which serial flag is true
+
+[source,ruby]
+----
+hbase> add_peer '1', CLUSTER_KEY => "server1.cie.com:2181:/hbase", SERIAL => true
+----
+
+* Set a replication peer's serial flag to false
+
+[source,ruby]
+----
+hbase> set_peer_serial '1', false
+----
+
+* Set a replication peer's serial flag to true
+
+[source,ruby]
+----
+hbase> set_peer_serial '1', true
+----
The serial replication feature had been done firstly in link:https://issues.apache.org/jira/browse/HBASE-9465[HBASE-9465] and then reverted and redone in link:https://issues.apache.org/jira/browse/HBASE-20046[HBASE-20046]. You can find more details in these issues.
[11/16] hbase git commit: HBASE-21554 Show replication endpoint
classname for replication peer on master web UI
Posted by zh...@apache.org.
HBASE-21554 Show replication endpoint classname for replication peer on master web UI
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8d7061a4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8d7061a4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8d7061a4
Branch: refs/heads/HBASE-21512
Commit: 8d7061a487357344f10ee260979cc2c47cd833dd
Parents: e0e0694
Author: Guanghao Zhang <zg...@apache.org>
Authored: Thu Dec 6 10:54:14 2018 +0800
Committer: Guanghao Zhang <zg...@apache.org>
Committed: Fri Dec 7 13:31:59 2018 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8d7061a4/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
index 7bb6c40..da44052 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
@@ -621,6 +621,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
<tr>
<th>Peer Id</th>
<th>Cluster Key</th>
+ <th>Endpoint</th>
<th>State</th>
<th>IsSerial</th>
<th>Remote WAL</th>
@@ -641,6 +642,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
<tr>
<td><% peerId %></td>
<td><% peerConfig.getClusterKey() %></td>
+ <td><% peerConfig.getReplicationEndpointImpl() %></td>
<td><% peer.isEnabled() ? "ENABLED" : "DISABLED" %></td>
<td><% peerConfig.isSerial() %></td>
<td><% peerConfig.getRemoteWALDir() == null ? "" : peerConfig.getRemoteWALDir() %>
[16/16] hbase git commit: HBASE-21526 Use AsyncClusterConnection in
ServerManager for getRsAdmin
Posted by zh...@apache.org.
HBASE-21526 Use AsyncClusterConnection in ServerManager for getRsAdmin
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e7a12278
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e7a12278
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e7a12278
Branch: refs/heads/HBASE-21512
Commit: e7a12278080e11c0f1ddb9553b0531f923fd209d
Parents: 4781020
Author: zhangduo <zh...@apache.org>
Authored: Thu Dec 6 21:25:34 2018 +0800
Committer: Duo Zhang <zh...@apache.org>
Committed: Tue Dec 11 11:28:45 2018 +0800
----------------------------------------------------------------------
.../hbase/client/AsyncClusterConnection.java | 6 +
.../hbase/client/AsyncConnectionImpl.java | 5 +
.../hbase/client/AsyncRegionServerAdmin.java | 210 +++++++++++++++++++
.../apache/hadoop/hbase/util/FutureUtils.java | 60 ++++++
.../org/apache/hadoop/hbase/master/HMaster.java | 13 +-
.../hadoop/hbase/master/ServerManager.java | 67 ------
.../master/procedure/RSProcedureDispatcher.java | 44 ++--
7 files changed, 320 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7a12278/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
index c7dea25..1327fd7 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncClusterConnection.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.client;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.ipc.RpcClient;
import org.apache.yetus.audience.InterfaceAudience;
@@ -27,6 +28,11 @@ import org.apache.yetus.audience.InterfaceAudience;
public interface AsyncClusterConnection extends AsyncConnection {
/**
+ * Get the admin service for the given region server.
+ */
+ AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName);
+
+ /**
* Get the nonce generator for this connection.
*/
NonceGenerator getNonceGenerator();
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7a12278/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
index 00c8f55..5625ee8 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java
@@ -330,4 +330,9 @@ class AsyncConnectionImpl implements AsyncClusterConnection {
ExecutorService pool) {
return new AsyncBufferedMutatorBuilderImpl(connConf, getTableBuilder(tableName, pool));
}
+
+ @Override
+ public AsyncRegionServerAdmin getRegionServerAdmin(ServerName serverName) {
+ return new AsyncRegionServerAdmin(serverName, this);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7a12278/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
new file mode 100644
index 0000000..9accd89
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionServerAdmin.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.ipc.HBaseRpcController;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
+import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearCompactionQueuesResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ClearRegionBlockCacheResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactionSwitchRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CompactionSwitchResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.FlushRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetStoreFileResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ReplicateWALEntryRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ReplicateWALEntryResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.RollWALWriterRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.RollWALWriterResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.StopServerRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.StopServerResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateConfigurationRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateConfigurationResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WarmupRegionRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WarmupRegionResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse;
+
+/**
+ * A simple wrapper of the {@link AdminService} for a region server, which returns a
+ * {@link CompletableFuture}. This is easier to use, as if you use the raw protobuf interface, you
+ * need to get the result from the {@link RpcCallback}, and if there is an exception, you need to
+ * get it from the {@link RpcController} passed in.
+ * <p/>
+ * Notice that there is no retry, and this is intentional. We have different retry for different
+ * usage for now, if later we want to unify them, we can move the retry logic into this class.
+ */
+@InterfaceAudience.Private
+public class AsyncRegionServerAdmin {
+
+ private final ServerName server;
+
+ private final AsyncConnectionImpl conn;
+
+ AsyncRegionServerAdmin(ServerName server, AsyncConnectionImpl conn) {
+ this.server = server;
+ this.conn = conn;
+ }
+
+ @FunctionalInterface
+ private interface RpcCall<RESP> {
+ void call(AdminService.Interface stub, HBaseRpcController controller, RpcCallback<RESP> done);
+ }
+
+ private <RESP> CompletableFuture<RESP> call(RpcCall<RESP> rpcCall) {
+ CompletableFuture<RESP> future = new CompletableFuture<>();
+ HBaseRpcController controller = conn.rpcControllerFactory.newController();
+ try {
+ rpcCall.call(conn.getAdminStub(server), controller, new RpcCallback<RESP>() {
+
+ @Override
+ public void run(RESP resp) {
+ if (controller.failed()) {
+ future.completeExceptionally(controller.getFailed());
+ } else {
+ future.complete(resp);
+ }
+ }
+ });
+ } catch (IOException e) {
+ future.completeExceptionally(e);
+ }
+ return future;
+ }
+
+ public CompletableFuture<GetRegionInfoResponse> getRegionInfo(GetRegionInfoRequest request) {
+ return call((stub, controller, done) -> stub.getRegionInfo(controller, request, done));
+ }
+
+ public CompletableFuture<GetStoreFileResponse> getStoreFile(GetStoreFileRequest request) {
+ return call((stub, controller, done) -> stub.getStoreFile(controller, request, done));
+ }
+
+ public CompletableFuture<GetOnlineRegionResponse> getOnlineRegion(
+ GetOnlineRegionRequest request) {
+ return call((stub, controller, done) -> stub.getOnlineRegion(controller, request, done));
+ }
+
+ public CompletableFuture<OpenRegionResponse> openRegion(OpenRegionRequest request) {
+ return call((stub, controller, done) -> stub.openRegion(controller, request, done));
+ }
+
+ public CompletableFuture<WarmupRegionResponse> warmupRegion(WarmupRegionRequest request) {
+ return call((stub, controller, done) -> stub.warmupRegion(controller, request, done));
+ }
+
+ public CompletableFuture<CloseRegionResponse> closeRegion(CloseRegionRequest request) {
+ return call((stub, controller, done) -> stub.closeRegion(controller, request, done));
+ }
+
+ public CompletableFuture<FlushRegionResponse> flushRegion(FlushRegionRequest request) {
+ return call((stub, controller, done) -> stub.flushRegion(controller, request, done));
+ }
+
+ public CompletableFuture<CompactionSwitchResponse> compactionSwitch(
+ CompactionSwitchRequest request) {
+ return call((stub, controller, done) -> stub.compactionSwitch(controller, request, done));
+ }
+
+ public CompletableFuture<CompactRegionResponse> compactRegion(CompactRegionRequest request) {
+ return call((stub, controller, done) -> stub.compactRegion(controller, request, done));
+ }
+
+ public CompletableFuture<ReplicateWALEntryResponse> replicateWALEntry(
+ ReplicateWALEntryRequest request) {
+ return call((stub, controller, done) -> stub.replicateWALEntry(controller, request, done));
+ }
+
+ public CompletableFuture<ReplicateWALEntryResponse> replay(ReplicateWALEntryRequest request) {
+ return call((stub, controller, done) -> stub.replay(controller, request, done));
+ }
+
+ public CompletableFuture<RollWALWriterResponse> rollWALWriter(RollWALWriterRequest request) {
+ return call((stub, controller, done) -> stub.rollWALWriter(controller, request, done));
+ }
+
+ public CompletableFuture<GetServerInfoResponse> getServerInfo(GetServerInfoRequest request) {
+ return call((stub, controller, done) -> stub.getServerInfo(controller, request, done));
+ }
+
+ public CompletableFuture<StopServerResponse> stopServer(StopServerRequest request) {
+ return call((stub, controller, done) -> stub.stopServer(controller, request, done));
+ }
+
+ public CompletableFuture<UpdateFavoredNodesResponse> updateFavoredNodes(
+ UpdateFavoredNodesRequest request) {
+ return call((stub, controller, done) -> stub.updateFavoredNodes(controller, request, done));
+ }
+
+ public CompletableFuture<UpdateConfigurationResponse> updateConfiguration(
+ UpdateConfigurationRequest request) {
+ return call((stub, controller, done) -> stub.updateConfiguration(controller, request, done));
+ }
+
+ public CompletableFuture<GetRegionLoadResponse> getRegionLoad(GetRegionLoadRequest request) {
+ return call((stub, controller, done) -> stub.getRegionLoad(controller, request, done));
+ }
+
+ public CompletableFuture<ClearCompactionQueuesResponse> clearCompactionQueues(
+ ClearCompactionQueuesRequest request) {
+ return call((stub, controller, done) -> stub.clearCompactionQueues(controller, request, done));
+ }
+
+ public CompletableFuture<ClearRegionBlockCacheResponse> clearRegionBlockCache(
+ ClearRegionBlockCacheRequest request) {
+ return call((stub, controller, done) -> stub.clearRegionBlockCache(controller, request, done));
+ }
+
+ public CompletableFuture<GetSpaceQuotaSnapshotsResponse> getSpaceQuotaSnapshots(
+ GetSpaceQuotaSnapshotsRequest request) {
+ return call((stub, controller, done) -> stub.getSpaceQuotaSnapshots(controller, request, done));
+ }
+
+ public CompletableFuture<ExecuteProceduresResponse> executeProcedures(
+ ExecuteProceduresRequest request) {
+ return call((stub, controller, done) -> stub.executeProcedures(controller, request, done));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7a12278/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FutureUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FutureUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FutureUtils.java
new file mode 100644
index 0000000..0a72581
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/FutureUtils.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Consumer;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Helper class for processing futures.
+ */
+@InterfaceAudience.Private
+public final class FutureUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(FutureUtils.class);
+
+ private FutureUtils() {
+ }
+
+ /**
+ * This is method is used when you do not care the result of an asynchronous operation. Ignoring
+ * the return value of a Future is considered as a bad practice as it may suppress exceptions
+ * thrown from the code that completes the future, so you can use method to log the exceptions
+ * when the future is failed.
+ * <p/>
+ * And the error phone check will always report FutureReturnValueIgnored because every method in
+ * the {@link CompletableFuture} class will return a new {@link CompletableFuture}, so you always
+ * have one future that has not been checked. So we introduce this method and add a suppress
+ * warnings annotation here.
+ */
+ @SuppressWarnings("FutureReturnValueIgnored")
+ public static void ifFail(CompletableFuture<?> future, Consumer<Throwable> action) {
+ future.whenComplete((resp, error) -> {
+ if (error != null) {
+ try {
+ action.accept(error);
+ } catch (Throwable e) {
+ LOG.warn("Failed to process error", error);
+ }
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7a12278/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 968cc27..5055652 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -193,6 +193,7 @@ import org.apache.hadoop.hbase.util.BloomFilterUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CompressionTest;
import org.apache.hadoop.hbase.util.EncryptionTest;
+import org.apache.hadoop.hbase.util.FutureUtils;
import org.apache.hadoop.hbase.util.HBaseFsck;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.hbase.util.HasThread;
@@ -225,6 +226,7 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceViolationPolicy;
@@ -1936,6 +1938,13 @@ public class HMaster extends HRegionServer implements MasterServices {
});
}
+ private void warmUpRegion(ServerName server, RegionInfo region) {
+ FutureUtils.ifFail(
+ asyncClusterConnection.getRegionServerAdmin(server)
+ .warmupRegion(RequestConverter.buildWarmupRegionRequest(region)),
+ error -> LOG.warn("Failed to warm up region {} on server {}", region, server, error));
+ }
+
// Public so can be accessed by tests. Blocks until move is done.
// Replace with an async implementation from which you can get
// a success/failure result.
@@ -2007,7 +2016,9 @@ public class HMaster extends HRegionServer implements MasterServices {
// Warmup the region on the destination before initiating the move. this call
// is synchronous and takes some time. doing it before the source region gets
// closed
- serverManager.sendRegionWarmup(rp.getDestination(), hri);
+ // A region server could reject the close request because it either does not
+ // have the specified region or the region is being split.
+ warmUpRegion(rp.getDestination(), hri);
LOG.info(getClientIdAuditPrefix() + " move " + rp + ", running balancer");
Future<byte []> future = this.assignmentManager.moveAsync(rp);
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7a12278/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
index dc76d72..80dbaa1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -51,12 +50,9 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.YouAreDeadException;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.client.RetriesExhaustedException;
import org.apache.hadoop.hbase.ipc.HBaseRpcController;
-import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.master.assignment.RegionStates;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
@@ -159,25 +155,16 @@ public class ServerManager {
private final ConcurrentNavigableMap<ServerName, ServerMetrics> onlineServers =
new ConcurrentSkipListMap<>();
- /**
- * Map of admin interfaces per registered regionserver; these interfaces we use to control
- * regionservers out on the cluster
- */
- private final Map<ServerName, AdminService.BlockingInterface> rsAdmins = new HashMap<>();
-
/** List of region servers that should not get any more new regions. */
private final ArrayList<ServerName> drainingServers = new ArrayList<>();
private final MasterServices master;
- private final ClusterConnection connection;
private final DeadServer deadservers = new DeadServer();
private final long maxSkew;
private final long warningSkew;
- private final RpcControllerFactory rpcControllerFactory;
-
/** Listeners that are called on server events. */
private List<ServerListener> listeners = new CopyOnWriteArrayList<>();
@@ -189,8 +176,6 @@ public class ServerManager {
Configuration c = master.getConfiguration();
maxSkew = c.getLong("hbase.master.maxclockskew", 30000);
warningSkew = c.getLong("hbase.master.warningclockskew", 10000);
- this.connection = master.getClusterConnection();
- this.rpcControllerFactory = this.connection == null? null: connection.getRpcControllerFactory();
persistFlushedSequenceId = c.getBoolean(PERSIST_FLUSHEDSEQUENCEID,
PERSIST_FLUSHEDSEQUENCEID_DEFAULT);
}
@@ -438,7 +423,6 @@ public class ServerManager {
void recordNewServerWithLock(final ServerName serverName, final ServerMetrics sl) {
LOG.info("Registering regionserver=" + serverName);
this.onlineServers.put(serverName, sl);
- this.rsAdmins.remove(serverName);
}
@VisibleForTesting
@@ -630,7 +614,6 @@ public class ServerManager {
this.onlineServers.remove(sn);
onlineServers.notifyAll();
}
- this.rsAdmins.remove(sn);
}
/*
@@ -673,34 +656,6 @@ public class ServerManager {
return this.drainingServers.add(sn);
}
- // RPC methods to region servers
-
- private HBaseRpcController newRpcController() {
- return rpcControllerFactory == null ? null : rpcControllerFactory.newController();
- }
-
- /**
- * Sends a WARMUP RPC to the specified server to warmup the specified region.
- * <p>
- * A region server could reject the close request because it either does not
- * have the specified region or the region is being split.
- * @param server server to warmup a region
- * @param region region to warmup
- */
- public void sendRegionWarmup(ServerName server,
- RegionInfo region) {
- if (server == null) return;
- try {
- AdminService.BlockingInterface admin = getRsAdmin(server);
- HBaseRpcController controller = newRpcController();
- ProtobufUtil.warmupRegion(controller, admin, region);
- } catch (IOException e) {
- LOG.error("Received exception in RPC for warmup server:" +
- server + "region: " + region +
- "exception: " + e);
- }
- }
-
/**
* Contacts a region server and waits up to timeout ms
* to close the region. This bypasses the active hmaster.
@@ -734,28 +689,6 @@ public class ServerManager {
}
/**
- * @param sn
- * @return Admin interface for the remote regionserver named <code>sn</code>
- * @throws IOException
- * @throws RetriesExhaustedException wrapping a ConnectException if failed
- */
- public AdminService.BlockingInterface getRsAdmin(final ServerName sn)
- throws IOException {
- AdminService.BlockingInterface admin = this.rsAdmins.get(sn);
- if (admin == null) {
- LOG.debug("New admin connection to " + sn.toString());
- if (sn.equals(master.getServerName()) && master instanceof HRegionServer) {
- // A master is also a region server now, see HBASE-10569 for details
- admin = ((HRegionServer)master).getRSRpcServices();
- } else {
- admin = this.connection.getAdmin(sn);
- }
- this.rsAdmins.put(sn, admin);
- }
- return admin;
- }
-
- /**
* Calculate min necessary to start. This is not an absolute. It is just
* a friction that will cause us hang around a bit longer waiting on
* RegionServers to check-in.
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7a12278/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
index 638f9d3..f3ab4b3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RSProcedureDispatcher.java
@@ -18,12 +18,15 @@
package org.apache.hadoop.hbase.master.procedure;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.AsyncRegionServerAdmin;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil;
import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
@@ -37,11 +40,11 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
import org.apache.hbase.thirdparty.com.google.common.collect.ArrayListMultimap;
import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
-import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
@@ -159,13 +162,8 @@ public class RSProcedureDispatcher
this.serverName = serverName;
}
- protected AdminService.BlockingInterface getRsAdmin() throws IOException {
- final AdminService.BlockingInterface admin = master.getServerManager().getRsAdmin(serverName);
- if (admin == null) {
- throw new IOException("Attempting to send OPEN RPC to server " + getServerName() +
- " failed because no RPC connection found to this server");
- }
- return admin;
+ protected AsyncRegionServerAdmin getRsAdmin() throws IOException {
+ return master.getAsyncClusterConnection().getRegionServerAdmin(serverName);
}
protected ServerName getServerName() {
@@ -344,9 +342,13 @@ public class RSProcedureDispatcher
protected ExecuteProceduresResponse sendRequest(final ServerName serverName,
final ExecuteProceduresRequest request) throws IOException {
try {
- return getRsAdmin().executeProcedures(null, request);
- } catch (ServiceException se) {
- throw ProtobufUtil.getRemoteException(se);
+ return getRsAdmin().executeProcedures(request).get();
+ } catch (InterruptedException e) {
+ throw (IOException) new InterruptedIOException().initCause(e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ Throwables.propagateIfPossible(cause, IOException.class);
+ throw new IOException(cause);
}
}
@@ -407,9 +409,13 @@ public class RSProcedureDispatcher
private OpenRegionResponse sendRequest(final ServerName serverName,
final OpenRegionRequest request) throws IOException {
try {
- return getRsAdmin().openRegion(null, request);
- } catch (ServiceException se) {
- throw ProtobufUtil.getRemoteException(se);
+ return getRsAdmin().openRegion(request).get();
+ } catch (InterruptedException e) {
+ throw (IOException) new InterruptedIOException().initCause(e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ Throwables.propagateIfPossible(cause, IOException.class);
+ throw new IOException(cause);
}
}
@@ -453,9 +459,13 @@ public class RSProcedureDispatcher
private CloseRegionResponse sendRequest(final ServerName serverName,
final CloseRegionRequest request) throws IOException {
try {
- return getRsAdmin().closeRegion(null, request);
- } catch (ServiceException se) {
- throw ProtobufUtil.getRemoteException(se);
+ return getRsAdmin().closeRegion(request).get();
+ } catch (InterruptedException e) {
+ throw (IOException) new InterruptedIOException().initCause(e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ Throwables.propagateIfPossible(cause, IOException.class);
+ throw new IOException(cause);
}
}
[03/16] hbase git commit: HBASE-21550 Add a new method
preCreateTableRegionInfos for MasterObserver which allows CPs to modify the
TableDescriptor
Posted by zh...@apache.org.
HBASE-21550 Add a new method preCreateTableRegionInfos for MasterObserver which allows CPs to modify the TableDescriptor
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f49baf25
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f49baf25
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f49baf25
Branch: refs/heads/HBASE-21512
Commit: f49baf259ec6bc2c8634debd2dbfc592753245d3
Parents: 8bf966c
Author: Duo Zhang <zh...@apache.org>
Authored: Wed Dec 5 18:19:15 2018 +0800
Committer: zhangduo <zh...@apache.org>
Committed: Thu Dec 6 08:30:32 2018 +0800
----------------------------------------------------------------------
.../hbase/coprocessor/MasterObserver.java | 15 +++++
.../org/apache/hadoop/hbase/master/HMaster.java | 68 ++++++++++----------
.../hbase/master/MasterCoprocessorHost.java | 14 ++++
.../hbase/coprocessor/TestMasterObserver.java | 14 +++-
4 files changed, 75 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f49baf25/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
index 573ac7a..a0863e4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
@@ -70,6 +70,21 @@ import org.apache.yetus.audience.InterfaceStability;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC)
@InterfaceStability.Evolving
public interface MasterObserver {
+
+ /**
+ * Called before we create the region infos for this table. Called as part of create table RPC
+ * call.
+ * @param ctx the environment to interact with the framework and master
+ * @param desc the TableDescriptor for the table
+ * @return the TableDescriptor used to create the table. Default is the one passed in. Return
+ * {@code null} means cancel the creation.
+ */
+ default TableDescriptor preCreateTableRegionsInfos(
+ final ObserverContext<MasterCoprocessorEnvironment> ctx, TableDescriptor desc)
+ throws IOException {
+ return desc;
+ }
+
/**
* Called before a new table is created by
* {@link org.apache.hadoop.hbase.master.HMaster}. Called as part of create
http://git-wip-us.apache.org/repos/asf/hbase/blob/f49baf25/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 132e271..e96dc36 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2030,45 +2030,45 @@ public class HMaster extends HRegionServer implements MasterServices {
}
@Override
- public long createTable(
- final TableDescriptor tableDescriptor,
- final byte [][] splitKeys,
- final long nonceGroup,
- final long nonce) throws IOException {
+ public long createTable(final TableDescriptor tableDescriptor, final byte[][] splitKeys,
+ final long nonceGroup, final long nonce) throws IOException {
checkInitialized();
-
- String namespace = tableDescriptor.getTableName().getNamespaceAsString();
+ TableDescriptor desc = getMasterCoprocessorHost().preCreateTableRegionsInfos(tableDescriptor);
+ if (desc == null) {
+ throw new IOException("Creation for " + tableDescriptor + " is canceled by CP");
+ }
+ String namespace = desc.getTableName().getNamespaceAsString();
this.clusterSchemaService.getNamespace(namespace);
- RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(tableDescriptor, splitKeys);
- sanityCheckTableDescriptor(tableDescriptor);
-
- return MasterProcedureUtil.submitProcedure(
- new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
- @Override
- protected void run() throws IOException {
- getMaster().getMasterCoprocessorHost().preCreateTable(tableDescriptor, newRegions);
-
- LOG.info(getClientIdAuditPrefix() + " create " + tableDescriptor);
-
- // TODO: We can handle/merge duplicate requests, and differentiate the case of
- // TableExistsException by saying if the schema is the same or not.
- //
- // We need to wait for the procedure to potentially fail due to "prepare" sanity
- // checks. This will block only the beginning of the procedure. See HBASE-19953.
- ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
- submitProcedure(new CreateTableProcedure(
- procedureExecutor.getEnvironment(), tableDescriptor, newRegions, latch));
- latch.await();
+ RegionInfo[] newRegions = ModifyRegionUtils.createRegionInfos(desc, splitKeys);
+ sanityCheckTableDescriptor(desc);
- getMaster().getMasterCoprocessorHost().postCreateTable(tableDescriptor, newRegions);
- }
+ return MasterProcedureUtil
+ .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
+ @Override
+ protected void run() throws IOException {
+ getMaster().getMasterCoprocessorHost().preCreateTable(desc, newRegions);
+
+ LOG.info(getClientIdAuditPrefix() + " create " + desc);
+
+ // TODO: We can handle/merge duplicate requests, and differentiate the case of
+ // TableExistsException by saying if the schema is the same or not.
+ //
+ // We need to wait for the procedure to potentially fail due to "prepare" sanity
+ // checks. This will block only the beginning of the procedure. See HBASE-19953.
+ ProcedurePrepareLatch latch = ProcedurePrepareLatch.createBlockingLatch();
+ submitProcedure(
+ new CreateTableProcedure(procedureExecutor.getEnvironment(), desc, newRegions, latch));
+ latch.await();
+
+ getMaster().getMasterCoprocessorHost().postCreateTable(desc, newRegions);
+ }
- @Override
- protected String getDescription() {
- return "CreateTableProcedure";
- }
- });
+ @Override
+ protected String getDescription() {
+ return "CreateTableProcedure";
+ }
+ });
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/f49baf25/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index 019c64f..51e30c4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -317,6 +317,20 @@ public class MasterCoprocessorHost
/* Implementation of hooks for invoking MasterObservers */
+ public TableDescriptor preCreateTableRegionsInfos(TableDescriptor desc) throws IOException {
+ if (coprocEnvironments.isEmpty()) {
+ return desc;
+ }
+ return execOperationWithResult(
+ new ObserverOperationWithResult<MasterObserver, TableDescriptor>(masterObserverGetter, desc) {
+
+ @Override
+ protected TableDescriptor call(MasterObserver observer) throws IOException {
+ return observer.preCreateTableRegionsInfos(this, getResult());
+ }
+ });
+ }
+
public void preCreateTable(final TableDescriptor htd, final RegionInfo[] regions)
throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/f49baf25/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
index a606e27..d8a5b4c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
@@ -94,6 +94,7 @@ public class TestMasterObserver {
public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
+ private boolean preCreateTableRegionInfosCalled;
private boolean preCreateTableCalled;
private boolean postCreateTableCalled;
private boolean preDeleteTableCalled;
@@ -186,6 +187,7 @@ public class TestMasterObserver {
private boolean postLockHeartbeatCalled;
public void resetStates() {
+ preCreateTableRegionInfosCalled = false;
preCreateTableCalled = false;
postCreateTableCalled = false;
preDeleteTableCalled = false;
@@ -298,6 +300,14 @@ public class TestMasterObserver {
}
@Override
+ public TableDescriptor preCreateTableRegionsInfos(
+ ObserverContext<MasterCoprocessorEnvironment> ctx, TableDescriptor desc)
+ throws IOException {
+ preCreateTableRegionInfosCalled = true;
+ return desc;
+ }
+
+ @Override
public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> env,
TableDescriptor desc, RegionInfo[] regions) throws IOException {
preCreateTableCalled = true;
@@ -310,11 +320,11 @@ public class TestMasterObserver {
}
public boolean wasCreateTableCalled() {
- return preCreateTableCalled && postCreateTableCalled;
+ return preCreateTableRegionInfosCalled && preCreateTableCalled && postCreateTableCalled;
}
public boolean preCreateTableCalledOnly() {
- return preCreateTableCalled && !postCreateTableCalled;
+ return preCreateTableRegionInfosCalled && preCreateTableCalled && !postCreateTableCalled;
}
@Override