You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2019/11/22 20:16:28 UTC
[hbase] branch HBASE-18095/client-locate-meta-no-zookeeper updated
(cc611c9 -> efab11a)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a change to branch HBASE-18095/client-locate-meta-no-zookeeper
in repository https://gitbox.apache.org/repos/asf/hbase.git.
discard cc611c9 HBASE-23275: Track active master's address in ActiveMasterManager (#812)
add 77b4e8c HBASE-23322 [hbck2] Simplification on HBCKSCP scheduling
add 4ea7922 HBASE-23223 Support the offsetLock of bucketCache to use strong ref (#764)
add ea52866 HBASE-23234 Provide .editorconfig based on checkstyle configuration (#846)
add 0b49e3a HBASE-23237 Prevent Negative values in metrics requestsPerSecond
add ca6059f HBASE-23325 [UI]rsgoup average load keep two decimals (#860)
add 54ad797 HBASE-23328 info:regioninfo goes wrong when region replicas enabled (#863)
add 3b0c276 HBASE-23329 Remove unused methods from RequestConverter (#870)
add 8e52339 HBASE-23307 Add running of ReplicationBarrierCleaner to hbck2 fixMeta invocation (#859)
new efab11a HBASE-23275: Track active master's address in ActiveMasterManager (#812)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (cc611c9)
\
N -- N -- N refs/heads/HBASE-18095/client-locate-meta-no-zookeeper (efab11a)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.editorconfig | 693 +++++++++++++++++++++
.../org/apache/hadoop/hbase/MetaTableAccessor.java | 5 +-
.../hbase/shaded/protobuf/RequestConverter.java | 186 ------
.../hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon | 3 +-
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 19 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 8 +
.../org/apache/hadoop/hbase/master/HbckChore.java | 24 +-
.../hadoop/hbase/master/MasterRpcServices.java | 37 +-
.../apache/hadoop/hbase/master/MasterServices.java | 4 +
.../org/apache/hadoop/hbase/master/MetaFixer.java | 3 +
.../apache/hadoop/hbase/master/ServerManager.java | 38 +-
.../hadoop/hbase/master/TableStateManager.java | 3 +-
.../hbase/master/assignment/AssignmentManager.java | 68 +-
.../hbase/master/assignment/RegionStates.java | 3 +
.../hbase/master/assignment/ServerStateNode.java | 6 +-
.../master/cleaner/ReplicationBarrierCleaner.java | 13 +-
.../master/procedure/HBCKServerCrashProcedure.java | 3 +
.../hadoop/hbase/regionserver/HRegionServer.java | 6 +-
.../MetricsRegionServerWrapperImpl.java | 64 +-
.../apache/hadoop/hbase/util/IdReadWriteLock.java | 80 +--
...Property.java => IdReadWriteLockStrongRef.java} | 36 +-
...ock.java => IdReadWriteLockWithObjectPool.java} | 40 +-
.../hbase/master/MockNoopMasterServices.java | 5 +-
.../hbase/master/TestClusterRestartFailover.java | 8 +-
.../assignment/TestAssignmentManagerBase.java | 2 +-
.../regionserver/TestRequestsPerSecondMetric.java | 100 +++
.../hbase/util/TestIdReadWriteLockStrongRef.java | 35 +-
...java => TestIdReadWriteLockWithObjectPool.java} | 16 +-
28 files changed, 1047 insertions(+), 461 deletions(-)
create mode 100644 .editorconfig
copy hbase-server/src/main/java/org/apache/hadoop/hbase/util/{GetJavaProperty.java => IdReadWriteLockStrongRef.java} (56%)
copy hbase-server/src/main/java/org/apache/hadoop/hbase/util/{IdReadWriteLock.java => IdReadWriteLockWithObjectPool.java} (77%)
create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRequestsPerSecondMetric.java
copy hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestShowProperties.java => hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestIdReadWriteLockStrongRef.java (62%)
rename hbase-server/src/test/java/org/apache/hadoop/hbase/util/{TestIdReadWriteLock.java => TestIdReadWriteLockWithObjectPool.java} (89%)
[hbase] 01/01: HBASE-23275: Track active master's address in
ActiveMasterManager (#812)
Posted by nd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch HBASE-18095/client-locate-meta-no-zookeeper
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit efab11afc52df4ae6fc3d883251728a221afb38f
Author: Bharath Vissapragada <bh...@apache.org>
AuthorDate: Wed Nov 20 11:41:36 2019 -0800
HBASE-23275: Track active master's address in ActiveMasterManager (#812)
* HBASE-23275: Track active master's address in ActiveMasterManager
Currently we just track whether an active master exists.
It helps to also track the address of the active master in
all the masters to help serve the client RPC requests to
know which master is active.
Signed-off-by: Nick Dimiduk <nd...@apache.org>
Signed-off-by: Andrew Purtell <ap...@apache.org>
---
.../hadoop/hbase/master/ActiveMasterManager.java | 63 +++++++++++++++++-----
.../org/apache/hadoop/hbase/master/HMaster.java | 4 ++
.../hbase/master/TestActiveMasterManager.java | 10 ++++
3 files changed, 64 insertions(+), 13 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
index 50798ed..99cab62 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ActiveMasterManager.java
@@ -1,4 +1,4 @@
-/**
+/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -17,25 +17,24 @@
* limitations under the License.
*/
package org.apache.hadoop.hbase.master;
-
import java.io.IOException;
+import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.ZNodeClearer;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
import org.apache.hadoop.hbase.zookeeper.ZKListener;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
/**
* Handles everything on master-side related to master election.
@@ -57,12 +56,18 @@ public class ActiveMasterManager extends ZKListener {
final AtomicBoolean clusterHasActiveMaster = new AtomicBoolean(false);
final AtomicBoolean clusterShutDown = new AtomicBoolean(false);
+ // This server's information.
private final ServerName sn;
private int infoPort;
private final Server master;
+ // Active master's server name. Invalidated anytime active master changes (based on ZK
+ // notifications) and lazily fetched on-demand.
+ // ServerName is immutable, so we don't need heavy synchronization around it.
+ private volatile ServerName activeMasterServerName;
+
/**
- * @param watcher
+ * @param watcher ZK watcher
* @param sn ServerName
* @param master In an instance of a Master.
*/
@@ -107,6 +112,30 @@ public class ActiveMasterManager extends ZKListener {
}
/**
+ * Fetches the active master's ServerName from zookeeper.
+ */
+ private void fetchAndSetActiveMasterServerName() {
+ LOG.debug("Attempting to fetch active master sn from zk");
+ try {
+ activeMasterServerName = MasterAddressTracker.getMasterAddress(watcher);
+ } catch (IOException | KeeperException e) {
+ // Log and ignore for now and re-fetch later if needed.
+ LOG.error("Error fetching active master information", e);
+ }
+ }
+
+ public Optional<ServerName> getActiveMasterServerName() {
+ if (!clusterHasActiveMaster.get()) {
+ return Optional.empty();
+ }
+ if (activeMasterServerName == null) {
+ fetchAndSetActiveMasterServerName();
+ }
+ // It could still be null, but return whatever we have.
+ return Optional.ofNullable(activeMasterServerName);
+ }
+
+ /**
* Handle a change in the master node. Doesn't matter whether this was called
* from a nodeCreated or nodeDeleted event because there are no guarantees
* that the current state of the master node matches the event at the time of
@@ -134,6 +163,9 @@ public class ActiveMasterManager extends ZKListener {
// Notify any thread waiting to become the active master
clusterHasActiveMaster.notifyAll();
}
+ // Reset the active master sn. Will be re-fetched later if needed.
+ // We don't want to make a synchronous RPC under a monitor.
+ activeMasterServerName = null;
}
} catch (KeeperException ke) {
master.abort("Received an unexpected KeeperException, aborting", ke);
@@ -151,8 +183,8 @@ public class ActiveMasterManager extends ZKListener {
* @param checkInterval the interval to check if the master is stopped
* @param startupStatus the monitor status to track the progress
* @return True if no issue becoming active master else false if another
- * master was running or if some other problem (zookeeper, stop flag has been
- * set on this Master)
+ * master was running or if some other problem (zookeeper, stop flag has been
+ * set on this Master)
*/
boolean blockUntilBecomingActiveMaster(
int checkInterval, MonitoredTask startupStatus) {
@@ -178,10 +210,14 @@ public class ActiveMasterManager extends ZKListener {
// We are the master, return
startupStatus.setStatus("Successfully registered as active master.");
this.clusterHasActiveMaster.set(true);
+ activeMasterServerName = sn;
LOG.info("Registered as active master=" + this.sn);
return true;
}
+ // Invalidate the active master name so that subsequent requests do not get any stale
+ // master information. Will be re-fetched if needed.
+ activeMasterServerName = null;
// There is another active master running elsewhere or this is a restart
// and the master ephemeral node has not expired yet.
this.clusterHasActiveMaster.set(true);
@@ -208,7 +244,8 @@ public class ActiveMasterManager extends ZKListener {
ZKUtil.deleteNode(this.watcher, this.watcher.getZNodePaths().masterAddressZNode);
// We may have failed to delete the znode at the previous step, but
- // we delete the file anyway: a second attempt to delete the znode is likely to fail again.
+ // we delete the file anyway: a second attempt to delete the znode is likely to fail
+ // again.
ZNodeClearer.deleteMyEphemeralNodeOnDisk();
} else {
msg = "Another master is the active master, " + currentMaster +
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index cc226bd..b444d3b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -3869,6 +3869,10 @@ public class HMaster extends HRegionServer implements MasterServices {
return cachedClusterId.getFromCacheOrFetch();
}
+ public Optional<ServerName> getActiveMaster() {
+ return activeMasterManager.getActiveMasterServerName();
+ }
+
@Override
public void runReplicationBarrierCleaner() {
ReplicationBarrierCleaner rbc = this.replicationBarrierCleaner;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
index 4649eea..84837f9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestActiveMasterManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.master;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -91,6 +92,7 @@ public class TestActiveMasterManager {
ActiveMasterManager activeMasterManager =
dummyMaster.getActiveMasterManager();
assertFalse(activeMasterManager.clusterHasActiveMaster.get());
+ assertFalse(activeMasterManager.getActiveMasterServerName().isPresent());
// First test becoming the active master uninterrupted
MonitoredTask status = Mockito.mock(MonitoredTask.class);
@@ -99,6 +101,7 @@ public class TestActiveMasterManager {
activeMasterManager.blockUntilBecomingActiveMaster(100, status);
assertTrue(activeMasterManager.clusterHasActiveMaster.get());
assertMaster(zk, master);
+ assertMaster(zk, activeMasterManager.getActiveMasterServerName().get());
// Now pretend master restart
DummyMaster secondDummyMaster = new DummyMaster(zk,master);
@@ -108,6 +111,8 @@ public class TestActiveMasterManager {
activeMasterManager.blockUntilBecomingActiveMaster(100, status);
assertTrue(activeMasterManager.clusterHasActiveMaster.get());
assertMaster(zk, master);
+ assertMaster(zk, activeMasterManager.getActiveMasterServerName().get());
+ assertMaster(zk, secondActiveMasterManager.getActiveMasterServerName().get());
}
/**
@@ -135,6 +140,7 @@ public class TestActiveMasterManager {
ActiveMasterManager activeMasterManager =
ms1.getActiveMasterManager();
assertFalse(activeMasterManager.clusterHasActiveMaster.get());
+ assertFalse(activeMasterManager.getActiveMasterServerName().isPresent());
// First test becoming the active master uninterrupted
ClusterStatusTracker clusterStatusTracker =
@@ -144,6 +150,7 @@ public class TestActiveMasterManager {
Mockito.mock(MonitoredTask.class));
assertTrue(activeMasterManager.clusterHasActiveMaster.get());
assertMaster(zk, firstMasterAddress);
+ assertMaster(zk, activeMasterManager.getActiveMasterServerName().get());
// New manager will now try to become the active master in another thread
WaitToBeMasterThread t = new WaitToBeMasterThread(zk, secondMasterAddress);
@@ -161,6 +168,8 @@ public class TestActiveMasterManager {
assertTrue(t.manager.clusterHasActiveMaster.get());
// But secondary one should not be the active master
assertFalse(t.isActiveMaster);
+ // Verify the active master ServerName is populated in standby master.
+ assertEquals(firstMasterAddress, t.manager.getActiveMasterServerName().get());
// Close the first server and delete it's master node
ms1.stop("stopping first server");
@@ -189,6 +198,7 @@ public class TestActiveMasterManager {
assertTrue(t.manager.clusterHasActiveMaster.get());
assertTrue(t.isActiveMaster);
+ assertEquals(secondMasterAddress, t.manager.getActiveMasterServerName().get());
LOG.info("Deleting master node");