You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2023/03/18 13:57:23 UTC
[hbase] branch HBASE-27109/table_based_rqs updated (1f14f4ae436 -> 49589ef2853)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
discard 1f14f4ae436 HBASE-27216 Revisit the ReplicationSyncUp tool (#4966)
omit c1d126dd078 HBASE-27430 Should disable replication log cleaner when migrating replication queue data (#4901)
omit 2fefe53e047 HBASE-27429 Add exponential retry backoff support for MigrateReplicationQueueFromZkToTableProcedure
omit 10e73d39ad4 HBASE-27217 Revisit the DumpReplicationQueues tool (#4810)
omit 46108163991 HBASE-27218 Support rolling upgrading (#4808)
omit 154bafe610f HBASE-27405 Fix the replication hfile/log cleaner report that the replication table does not exist (#4811)
omit 7489817e7cb HBASE-27392 Add a new procedure type for implementing some global operations such as migration (#4803)
omit fc2a456e45d HBASE-27215 Add support for sync replication (#4762)
omit 3be514d150a HBASE-27214 Implement the new replication hfile/log cleaner (#4722)
omit 3c960ee4409 HBASE-27213 Add support for claim queue operation (#4708)
omit 957929fdc45 HBASE-27212 Implement a new table based replication queue storage and make the minimum replication system work (#4672)
add b0cfd74edd3 Add tianhang to the developer list
add 22b0c3e2bd6 HBASE-27708 CPU hot-spot resolving User subject
add 58cb1f4799e HBASE-27686: Recovery of BucketCache and Prefetched data after RS Crash (#5080)
add 16737627414 HBASE-27710 ByteBuff ref counting is too expensive for on-heap buffers (#5104)
add d5c86243e10 HBASE-27671 Client should not be able to restore/clone a snapshot aft… (#5109)
new a878920ad10 HBASE-27212 Implement a new table based replication queue storage and make the minimum replication system work (#4672)
new a38d5744845 HBASE-27213 Add support for claim queue operation (#4708)
new a7345b0e301 HBASE-27214 Implement the new replication hfile/log cleaner (#4722)
new d24e7cf39f9 HBASE-27215 Add support for sync replication (#4762)
new 31c8f8c40b0 HBASE-27392 Add a new procedure type for implementing some global operations such as migration (#4803)
new 26d6ba2d0ad HBASE-27405 Fix the replication hfile/log cleaner report that the replication table does not exist (#4811)
new 1158247dbbf HBASE-27218 Support rolling upgrading (#4808)
new 96f15b3d2d3 HBASE-27217 Revisit the DumpReplicationQueues tool (#4810)
new f1d491205ce HBASE-27429 Add exponential retry backoff support for MigrateReplicationQueueFromZkToTableProcedure
new b59ec9baf7c HBASE-27430 Should disable replication log cleaner when migrating replication queue data (#4901)
new 49589ef2853 HBASE-27216 Revisit the ReplicationSyncUp tool (#4966)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1f14f4ae436)
\
N -- N -- N refs/heads/HBASE-27109/table_based_rqs (49589ef2853)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 11 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
...ption.java => SnapshotTTLExpiredException.java} | 17 +-
.../java/org/apache/hadoop/hbase/nio/ByteBuff.java | 11 +-
.../java/org/apache/hadoop/hbase/nio/RefCnt.java | 7 +
.../org/apache/hadoop/hbase/security/User.java | 13 ++
.../hadoop/hbase/io/TestByteBuffAllocator.java | 105 ++++++++-
.../apache/hadoop/hbase/io/hfile/CacheConfig.java | 6 +
.../hadoop/hbase/io/hfile/PrefetchExecutor.java | 2 +-
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 29 ++-
.../hfile/bucket/BucketCachePersister.java} | 44 ++--
.../hbase/master/cleaner/SnapshotCleanerChore.java | 22 +-
.../master/procedure/CloneSnapshotProcedure.java | 10 +
.../master/procedure/RestoreSnapshotProcedure.java | 10 +
.../hbase/snapshot/SnapshotDescriptionUtils.java | 14 ++
.../client/TestSnapshotWithTTLFromClient.java | 240 +++++++++++++++++++++
.../hadoop/hbase/io/hfile/TestPrefetchRSClose.java | 26 +--
...sistence.java => TestBucketCachePersister.java} | 139 ++++++------
.../hbase/snapshot/SnapshotTestingUtils.java | 12 +-
pom.xml | 6 +
18 files changed, 567 insertions(+), 146 deletions(-)
copy hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/{SnapshotExistsException.java => SnapshotTTLExpiredException.java} (68%)
copy hbase-server/src/main/java/org/apache/hadoop/hbase/{master/balancer/BalancerChore.java => io/hfile/bucket/BucketCachePersister.java} (51%)
create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotWithTTLFromClient.java
copy hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/{TestPrefetchPersistence.java => TestBucketCachePersister.java} (64%)
[hbase] 01/11: HBASE-27212 Implement a new table based replication queue storage and make the minimum replication system work (#4672)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a878920ad105c1b11da8e0ba33b8e58b0debec2b
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Mon Aug 15 10:46:27 2022 +0800
HBASE-27212 Implement a new table based replication queue storage and make the minimum replication system work (#4672)
Signed-off-by: Xin Sun <dd...@gmail.com>
---
.../protobuf/server/master/MasterProcedure.proto | 17 +-
...ageFactory.java => ReplicationGroupOffset.java} | 44 +-
...orageFactory.java => ReplicationQueueData.java} | 34 +-
.../hbase/replication/ReplicationQueueId.java | 129 +++
.../hbase/replication/ReplicationQueueStorage.java | 154 ++--
.../replication/ReplicationStorageFactory.java | 45 +-
.../hadoop/hbase/replication/ReplicationUtils.java | 15 -
.../replication/TableReplicationQueueStorage.java | 535 ++++++++++++
.../replication/ZKReplicationQueueStorage.java | 689 ---------------
.../replication/TestReplicationStateBasic.java | 243 +-----
.../replication/TestReplicationStateZKImpl.java | 1 -
.../replication/TestZKReplicationQueueStorage.java | 341 --------
.../org/apache/hadoop/hbase/master/HMaster.java | 2 +-
.../master/procedure/ServerCrashProcedure.java | 26 +-
.../AssignReplicationQueuesProcedure.java | 204 +++++
.../ClaimReplicationQueueRemoteProcedure.java | 44 +-
.../ClaimReplicationQueuesProcedure.java | 44 +-
.../master/replication/ReplicationPeerManager.java | 128 ++-
.../hbase/regionserver/wal/AbstractFSWAL.java | 6 +-
.../master/ReplicationHFileCleaner.java | 71 +-
.../replication/master/ReplicationLogCleaner.java | 25 +-
.../ClaimReplicationQueueCallable.java | 17 +-
.../regionserver/DumpReplicationQueues.java | 62 +-
.../regionserver/PeerProcedureHandler.java | 5 +-
.../regionserver/PeerProcedureHandlerImpl.java | 6 +-
.../regionserver/RecoveredReplicationSource.java | 138 +--
.../RecoveredReplicationSourceShipper.java | 64 +-
.../replication/regionserver/Replication.java | 2 +-
.../replication/regionserver/ReplicationLoad.java | 4 +-
.../regionserver/ReplicationSource.java | 70 +-
.../regionserver/ReplicationSourceFactory.java | 7 +-
.../regionserver/ReplicationSourceInterface.java | 26 +-
.../regionserver/ReplicationSourceLogQueue.java | 4 +-
.../regionserver/ReplicationSourceManager.java | 302 ++++---
.../regionserver/ReplicationSourceShipper.java | 6 +-
.../ReplicationSourceWALActionListener.java | 5 -
.../regionserver/ReplicationSyncUp.java | 23 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 4 +-
.../hadoop/hbase/util/hbck/ReplicationChecker.java | 46 +-
.../hadoop/hbase/wal/AbstractFSWALProvider.java | 40 +
.../hbase/client/TestAsyncReplicationAdminApi.java | 10 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 42 +-
.../cleaner/TestReplicationHFileCleaner.java | 90 +-
.../TestBulkLoadReplicationHFileRefs.java | 2 +-
...StartupWithLegacyRegionReplicationEndpoint.java | 35 +-
.../hbase/replication/ReplicationSourceDummy.java | 13 +-
.../TestAddToSerialReplicationPeer.java | 5 +-
.../replication/TestClaimReplicationQueue.java | 8 +-
...TestNamespaceReplicationWithBulkLoadedData.java | 4 +-
.../replication/TestReplicationDroppedTables.java | 20 +-
.../TestReplicationEmptyWALRecovery.java | 1 -
.../replication/TestReplicationSyncUpTool.java | 3 +
...estReplicationSyncUpToolWithBulkLoadedData.java | 3 +
.../replication/TestSerialReplicationFailover.java | 3 +
.../TestTableReplicationQueueStorage.java | 423 ++++++++++
...tReplicationSyncUpToolWithMultipleAsyncWAL.java | 3 +
.../TestReplicationSyncUpToolWithMultipleWAL.java | 3 +
.../TestDrainReplicationQueuesForStandBy.java | 3 +
.../regionserver/TestDumpReplicationQueues.java | 3 +
.../regionserver/TestReplicationSource.java | 82 +-
.../regionserver/TestReplicationSourceManager.java | 928 +++++----------------
.../TestReplicationSourceManagerZkImpl.java | 107 ---
.../regionserver/TestReplicationWALEdits.java | 147 ++++
.../regionserver/TestSerialReplicationChecker.java | 17 +-
.../regionserver/TestWALEntrySinkFilter.java | 23 -
.../TestHBaseFsckCleanReplicationBarriers.java | 20 +-
.../hadoop/hbase/util/TestHBaseFsckEncryption.java | 3 +
.../apache/hadoop/hbase/util/TestHBaseFsckMOB.java | 3 +
.../hbase/util/TestHBaseFsckReplication.java | 103 ++-
69 files changed, 2646 insertions(+), 3089 deletions(-)
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
index 59bb031589a..2e0da0deb84 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
@@ -679,16 +679,13 @@ message ClaimReplicationQueueRemoteStateData {
required ServerName crashed_server = 1;
required string queue = 2;
required ServerName target_server = 3;
+ optional ServerName source_server = 4;
}
message ClaimReplicationQueueRemoteParameter {
required ServerName crashed_server = 1;
required string queue = 2;
-}
-
-enum ClaimReplicationQueuesState {
- CLAIM_REPLICATION_QUEUES_DISPATCH = 1;
- CLAIM_REPLICATION_QUEUES_FINISH = 2;
+ optional ServerName source_server = 3;
}
enum ModifyTableDescriptorState {
@@ -715,3 +712,13 @@ message ModifyStoreFileTrackerStateData {
message ModifyColumnFamilyStoreFileTrackerStateData {
required bytes family = 1;
}
+
+enum AssignReplicationQueuesState {
+ ASSIGN_REPLICATION_QUEUES_PRE_CHECK = 1;
+ ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES = 2;
+ ASSIGN_REPLICATION_QUEUES_CLAIM = 3;
+}
+
+message AssignReplicationQueuesStateData {
+ required ServerName crashed_server = 1;
+}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationGroupOffset.java
similarity index 56%
copy from hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
copy to hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationGroupOffset.java
index 1080b2125c7..bd13594b99a 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationGroupOffset.java
@@ -17,33 +17,41 @@
*/
package org.apache.hadoop.hbase.replication;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
-/**
- * Used to create replication storage(peer, queue) classes.
- * <p>
- * For now we only have zk based implementation.
- */
@InterfaceAudience.Private
-public final class ReplicationStorageFactory {
+public class ReplicationGroupOffset {
+
+ public static final ReplicationGroupOffset BEGIN = new ReplicationGroupOffset("", 0L);
+
+ private final String wal;
- private ReplicationStorageFactory() {
+ private final long offset;
+
+ public ReplicationGroupOffset(String wal, long offset) {
+ this.wal = wal;
+ this.offset = offset;
}
- /**
- * Create a new {@link ReplicationPeerStorage}.
- */
- public static ReplicationPeerStorage getReplicationPeerStorage(ZKWatcher zk, Configuration conf) {
- return new ZKReplicationPeerStorage(zk, conf);
+ public String getWal() {
+ return wal;
}
/**
- * Create a new {@link ReplicationQueueStorage}.
+ * A negative value means this file has already been fully replicated out
*/
- public static ReplicationQueueStorage getReplicationQueueStorage(ZKWatcher zk,
- Configuration conf) {
- return new ZKReplicationQueueStorage(zk, conf);
+ public long getOffset() {
+ return offset;
+ }
+
+ @Override
+ public String toString() {
+ return wal + ":" + offset;
+ }
+
+ public static ReplicationGroupOffset parse(String str) {
+ int index = str.lastIndexOf(':');
+ return new ReplicationGroupOffset(str.substring(0, index),
+ Long.parseLong(str.substring(index + 1)));
}
}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueData.java
similarity index 56%
copy from hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
copy to hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueData.java
index 1080b2125c7..794ae9d3a55 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueData.java
@@ -17,33 +17,31 @@
*/
package org.apache.hadoop.hbase.replication;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
/**
- * Used to create replication storage(peer, queue) classes.
- * <p>
- * For now we only have zk based implementation.
+ * Representing all the information for a replication queue.
*/
@InterfaceAudience.Private
-public final class ReplicationStorageFactory {
+public class ReplicationQueueData {
+
+ private final ReplicationQueueId id;
+
+ private final ImmutableMap<String, ReplicationGroupOffset> offsets;
- private ReplicationStorageFactory() {
+ public ReplicationQueueData(ReplicationQueueId id,
+ ImmutableMap<String, ReplicationGroupOffset> offsets) {
+ this.id = id;
+ this.offsets = offsets;
}
- /**
- * Create a new {@link ReplicationPeerStorage}.
- */
- public static ReplicationPeerStorage getReplicationPeerStorage(ZKWatcher zk, Configuration conf) {
- return new ZKReplicationPeerStorage(zk, conf);
+ public ReplicationQueueId getId() {
+ return id;
}
- /**
- * Create a new {@link ReplicationQueueStorage}.
- */
- public static ReplicationQueueStorage getReplicationQueueStorage(ZKWatcher zk,
- Configuration conf) {
- return new ZKReplicationQueueStorage(zk, conf);
+ public ImmutableMap<String, ReplicationGroupOffset> getOffsets() {
+ return offsets;
}
}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueId.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueId.java
new file mode 100644
index 00000000000..5ee97fbaaf6
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueId.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.Objects;
+import java.util.Optional;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class ReplicationQueueId {
+
+ private final ServerName serverName;
+
+ private final String peerId;
+
+ private final Optional<ServerName> sourceServerName;
+
+ public ReplicationQueueId(ServerName serverName, String peerId) {
+ this.serverName = Objects.requireNonNull(serverName);
+ this.peerId = Objects.requireNonNull(peerId);
+ this.sourceServerName = Optional.empty();
+ }
+
+ public ReplicationQueueId(ServerName serverName, String peerId, ServerName sourceServerName) {
+ this.serverName = Objects.requireNonNull(serverName);
+ this.peerId = Objects.requireNonNull(peerId);
+ this.sourceServerName = Optional.of(sourceServerName);
+ }
+
+ public ServerName getServerName() {
+ return serverName;
+ }
+
+ public String getPeerId() {
+ return peerId;
+ }
+
+ public Optional<ServerName> getSourceServerName() {
+ return sourceServerName;
+ }
+
+ public ServerName getServerWALsBelongTo() {
+ return sourceServerName.orElse(serverName);
+ }
+
+ public boolean isRecovered() {
+ return sourceServerName.isPresent();
+ }
+
+ public ReplicationQueueId claim(ServerName targetServerName) {
+ ServerName newSourceServerName = sourceServerName.orElse(serverName);
+ return new ReplicationQueueId(targetServerName, peerId, newSourceServerName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(peerId, serverName, sourceServerName);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof ReplicationQueueId)) {
+ return false;
+ }
+ ReplicationQueueId other = (ReplicationQueueId) obj;
+ return Objects.equals(peerId, other.peerId) && Objects.equals(serverName, other.serverName)
+ && Objects.equals(sourceServerName, other.sourceServerName);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder().append(peerId).append('-').append(serverName);
+ sourceServerName.ifPresent(s -> sb.append('\t').append(s.toString()));
+ return sb.toString();
+ }
+
+ public static ReplicationQueueId parse(String str) {
+ int dashIndex = str.indexOf('-');
+ String peerId = str.substring(0, dashIndex);
+ int tabIndex = str.indexOf('\t', dashIndex + 1);
+ if (tabIndex < 0) {
+ String serverName = str.substring(dashIndex + 1);
+ return new ReplicationQueueId(ServerName.valueOf(serverName), peerId);
+ } else {
+ String serverName = str.substring(dashIndex + 1, tabIndex);
+ String sourceServerName = str.substring(tabIndex + 1);
+ return new ReplicationQueueId(ServerName.valueOf(serverName), peerId,
+ ServerName.valueOf(sourceServerName));
+ }
+ }
+
+ public static String getPeerId(String str) {
+ int dashIndex = str.indexOf('-');
+ return str.substring(0, dashIndex);
+ }
+
+ public static byte[] getScanPrefix(ServerName serverName, String peerId) {
+ return Bytes.toBytes(peerId + "-" + serverName.toString());
+ }
+
+ public static byte[] getScanPrefix(String peerId) {
+ return Bytes.toBytes(peerId + "-");
+ }
+
+ public static byte[] getScanStartRowForNextPeerId(String peerId) {
+ // '.' is the next char after '-'
+ return Bytes.toBytes(peerId + ".");
+ }
+}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
index 0f95c04b254..c4204f0e8c4 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.replication;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.SortedSet;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.util.Pair;
@@ -33,40 +32,79 @@ import org.apache.yetus.audience.InterfaceAudience;
public interface ReplicationQueueStorage {
/**
- * Remove a replication queue for a given regionserver.
- * @param serverName the name of the regionserver
- * @param queueId a String that identifies the queue.
+ * Set the current offset for a specific WAL group in a given queue.
+ * @param queueId the id of the queue
+ * @param walGroup the group of the WAL, can be empty if multi wal is not enabled
+ * @param offset the current offset of replication progress
+ * @param lastSeqIds map with {encodedRegionName, sequenceId} pairs for serial replication.
+ */
+ void setOffset(ReplicationQueueId queueId, String walGroup, ReplicationGroupOffset offset,
+ Map<String, Long> lastSeqIds) throws ReplicationException;
+
+ /**
+ * Get the current offset of all the WAL groups for a queue
+ * @param queueId the id of the queue
+ * @return a map of all offsets of the WAL groups. The key the is WAL group and the value is the
+ * position.
+ */
+ Map<String, ReplicationGroupOffset> getOffsets(ReplicationQueueId queueId)
+ throws ReplicationException;
+
+ /**
+ * Get a list of all queues for the specific peer.
+ * @param peerId the id of the peer
+ * @return a list of queueIds
*/
- void removeQueue(ServerName serverName, String queueId) throws ReplicationException;
+ List<ReplicationQueueId> listAllQueueIds(String peerId) throws ReplicationException;
/**
- * Add a new WAL file to the given queue for a given regionserver. If the queue does not exist it
- * is created.
- * @param serverName the name of the regionserver
- * @param queueId a String that identifies the queue.
- * @param fileName name of the WAL
+ * Get a list of all queues for the specific region server.
+ * @param serverName the server name of the region server that owns the set of queues
+ * @return a list of queueIds
*/
- void addWAL(ServerName serverName, String queueId, String fileName) throws ReplicationException;
+ List<ReplicationQueueId> listAllQueueIds(ServerName serverName) throws ReplicationException;
/**
- * Remove an WAL file from the given queue for a given regionserver.
- * @param serverName the name of the regionserver
- * @param queueId a String that identifies the queue.
- * @param fileName name of the WAL
+ * Get a list of all queues for the specific region server and the specific peer
+ * @param peerId the id of the peer
+ * @param serverName the server name of the region server that owns the set of queues
+ * @return a list of queueIds
*/
- void removeWAL(ServerName serverName, String queueId, String fileName)
+ List<ReplicationQueueId> listAllQueueIds(String peerId, ServerName serverName)
throws ReplicationException;
/**
- * Set the current position for a specific WAL in a given queue for a given regionserver.
- * @param serverName the name of the regionserver
- * @param queueId a String that identifies the queue
- * @param fileName name of the WAL
- * @param position the current position in the file. Will ignore if less than or equal to 0.
- * @param lastSeqIds map with {encodedRegionName, sequenceId} pairs for serial replication.
+ * Get a list of all queues and the offsets.
*/
- void setWALPosition(ServerName serverName, String queueId, String fileName, long position,
- Map<String, Long> lastSeqIds) throws ReplicationException;
+ List<ReplicationQueueData> listAllQueues() throws ReplicationException;
+
+ /**
+ * Get a list of all region servers that have outstanding replication queues. These servers could
+ * be alive, dead or from a previous run of the cluster.
+ * @return a list of server names
+ */
+ List<ServerName> listAllReplicators() throws ReplicationException;
+
+ /**
+ * Change ownership for the queue identified by queueId and belongs to a dead region server.
+ * @param queueId the id of the queue
+ * @param targetServerName the name of the target region server
+ * @return the new PeerId and A SortedSet of WALs in its queue
+ */
+ Map<String, ReplicationGroupOffset> claimQueue(ReplicationQueueId queueId,
+ ServerName targetServerName) throws ReplicationException;
+
+ /**
+ * Remove a replication queue
+ * @param queueId the id of the queue to remove
+ */
+ void removeQueue(ReplicationQueueId queueId) throws ReplicationException;
+
+ /**
+ * Remove all the replication queues for the given peer. Usually used when removing a peer.
+ * @param peerId the id of the peer
+ */
+ void removeAllQueues(String peerId) throws ReplicationException;
/**
* Read the max sequence id of the specific region for a given peer. For serial replication, we
@@ -99,67 +137,6 @@ public interface ReplicationQueueStorage {
void removeLastSequenceIds(String peerId, List<String> encodedRegionNames)
throws ReplicationException;
- /**
- * Get the current position for a specific WAL in a given queue for a given regionserver.
- * @param serverName the name of the regionserver
- * @param queueId a String that identifies the queue
- * @param fileName name of the WAL
- * @return the current position in the file
- */
- long getWALPosition(ServerName serverName, String queueId, String fileName)
- throws ReplicationException;
-
- /**
- * Get a list of all WALs in the given queue on the given region server.
- * @param serverName the server name of the region server that owns the queue
- * @param queueId a String that identifies the queue
- * @return a list of WALs
- */
- List<String> getWALsInQueue(ServerName serverName, String queueId) throws ReplicationException;
-
- /**
- * Get a list of all queues for the specified region server.
- * @param serverName the server name of the region server that owns the set of queues
- * @return a list of queueIds
- */
- List<String> getAllQueues(ServerName serverName) throws ReplicationException;
-
- /**
- * Change ownership for the queue identified by queueId and belongs to a dead region server.
- * @param sourceServerName the name of the dead region server
- * @param destServerName the name of the target region server
- * @param queueId the id of the queue
- * @return the new PeerId and A SortedSet of WALs in its queue
- */
- Pair<String, SortedSet<String>> claimQueue(ServerName sourceServerName, String queueId,
- ServerName destServerName) throws ReplicationException;
-
- /**
- * Remove the record of region server if the queue is empty.
- */
- void removeReplicatorIfQueueIsEmpty(ServerName serverName) throws ReplicationException;
-
- /**
- * Get a list of all region servers that have outstanding replication queues. These servers could
- * be alive, dead or from a previous run of the cluster.
- * @return a list of server names
- */
- List<ServerName> getListOfReplicators() throws ReplicationException;
-
- /**
- * Load all wals in all replication queues. This method guarantees to return a snapshot which
- * contains all WALs at the start of this call even there is concurrent queue failover. However,
- * some newly created WALs during the call may not be included.
- */
- Set<String> getAllWALs() throws ReplicationException;
-
- /**
- * Add a peer to hfile reference queue if peer does not exist.
- * @param peerId peer cluster id to be added
- * @throws ReplicationException if fails to add a peer id to hfile reference queue
- */
- void addPeerToHFileRefs(String peerId) throws ReplicationException;
-
/**
* Remove a peer from hfile reference queue.
* @param peerId peer cluster id to be removed
@@ -201,11 +178,4 @@ public interface ReplicationQueueStorage {
* created hfile references during the call may not be included.
*/
Set<String> getAllHFileRefs() throws ReplicationException;
-
- /**
- * Get full znode name for given region server
- * @param serverName the name of the region server
- * @return full znode name
- */
- String getRsNode(ServerName serverName);
}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
index 1080b2125c7..dc4317feaa4 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
@@ -17,18 +17,46 @@
*/
package org.apache.hadoop.hbase.replication;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.CoprocessorDescriptorBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
/**
* Used to create replication storage(peer, queue) classes.
- * <p>
- * For now we only have zk based implementation.
*/
@InterfaceAudience.Private
public final class ReplicationStorageFactory {
+ public static final String REPLICATION_QUEUE_TABLE_NAME = "hbase.replication.queue.table.name";
+
+ public static final TableName REPLICATION_QUEUE_TABLE_NAME_DEFAULT =
+ TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "replication");
+
+ public static TableDescriptor createReplicationQueueTableDescriptor(TableName tableName)
+ throws IOException {
+ return TableDescriptorBuilder.newBuilder(tableName)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TableReplicationQueueStorage.QUEUE_FAMILY))
+ .setColumnFamily(
+ ColumnFamilyDescriptorBuilder.of(TableReplicationQueueStorage.LAST_SEQUENCE_ID_FAMILY))
+ .setColumnFamily(
+ ColumnFamilyDescriptorBuilder.of(TableReplicationQueueStorage.HFILE_REF_FAMILY))
+ .setValue("hbase.regionserver.region.split_restriction.type", "DelimitedKeyPrefix")
+ .setValue("hbase.regionserver.region.split_restriction.delimiter", "-")
+ .setCoprocessor(CoprocessorDescriptorBuilder
+ .newBuilder("org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint")
+ .setPriority(Coprocessor.PRIORITY_SYSTEM).build())
+ .build();
+ }
+
private ReplicationStorageFactory() {
}
@@ -42,8 +70,17 @@ public final class ReplicationStorageFactory {
/**
* Create a new {@link ReplicationQueueStorage}.
*/
- public static ReplicationQueueStorage getReplicationQueueStorage(ZKWatcher zk,
+ public static ReplicationQueueStorage getReplicationQueueStorage(Connection conn,
Configuration conf) {
- return new ZKReplicationQueueStorage(zk, conf);
+ return getReplicationQueueStorage(conn, TableName.valueOf(conf.get(REPLICATION_QUEUE_TABLE_NAME,
+ REPLICATION_QUEUE_TABLE_NAME_DEFAULT.getNameAsString())));
+ }
+
+ /**
+ * Create a new {@link ReplicationQueueStorage}.
+ */
+ public static ReplicationQueueStorage getReplicationQueueStorage(Connection conn,
+ TableName tableName) {
+ return new TableReplicationQueueStorage(conn, tableName);
}
}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index d1bca8b4b04..ae78781a313 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CompoundConfiguration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -81,20 +80,6 @@ public final class ReplicationUtils {
return otherConf;
}
- public static void removeAllQueues(ReplicationQueueStorage queueStorage, String peerId)
- throws ReplicationException {
- for (ServerName replicator : queueStorage.getListOfReplicators()) {
- List<String> queueIds = queueStorage.getAllQueues(replicator);
- for (String queueId : queueIds) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (queueInfo.getPeerId().equals(peerId)) {
- queueStorage.removeQueue(replicator, queueId);
- }
- }
- queueStorage.removeReplicatorIfQueueIsEmpty(replicator);
- }
- }
-
private static boolean isCollectionEqual(Collection<String> c1, Collection<String> c2) {
if (c1 == null) {
return c2 == null;
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
new file mode 100644
index 00000000000..0c9553f4fd8
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
@@ -0,0 +1,535 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.function.Supplier;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellScanner;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.CompareOperator;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.AsyncTable;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Scan.ReadType;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FutureUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos;
+
+/**
+ * HBase table based replication queue storage.
+ */
+@InterfaceAudience.Private
+public class TableReplicationQueueStorage implements ReplicationQueueStorage {
+
+ public static final byte[] QUEUE_FAMILY = Bytes.toBytes("queue");
+
+ public static final byte[] LAST_SEQUENCE_ID_FAMILY = Bytes.toBytes("sid");
+
+ public static final byte[] HFILE_REF_FAMILY = Bytes.toBytes("hfileref");
+
+ private final Connection conn;
+
+ private final TableName tableName;
+
+ @FunctionalInterface
+ private interface TableCreator {
+
+ void create() throws IOException;
+ }
+
+ public TableReplicationQueueStorage(Connection conn, TableName tableName) {
+ this.conn = conn;
+ this.tableName = tableName;
+ }
+
+ private void addLastSeqIdsPut(MultiRowMutationProtos.MutateRowsRequest.Builder builder,
+ String peerId, Map<String, Long> lastSeqIds, AsyncTable<?> table) throws IOException {
+ // get the previous sequence ids first
+ byte[] row = Bytes.toBytes(peerId);
+ Get get = new Get(row);
+ lastSeqIds.keySet().forEach(encodedRegionName -> get.addColumn(LAST_SEQUENCE_ID_FAMILY,
+ Bytes.toBytes(encodedRegionName)));
+ Result result = FutureUtils.get(table.get(get));
+ Put put = new Put(row);
+ for (Map.Entry<String, Long> entry : lastSeqIds.entrySet()) {
+ String encodedRegionName = entry.getKey();
+ long lastSeqId = entry.getValue();
+ byte[] encodedRegionNameAsBytes = Bytes.toBytes(encodedRegionName);
+ byte[] previousLastSeqIdAsBytes =
+ result.getValue(LAST_SEQUENCE_ID_FAMILY, encodedRegionNameAsBytes);
+ if (previousLastSeqIdAsBytes != null) {
+ long previousLastSeqId = Bytes.toLong(previousLastSeqIdAsBytes);
+ if (lastSeqId > previousLastSeqId) {
+ // update last seq id when it is greater, and use CAS to make sure we do not overwrite
+ // other's value.
+ put.addColumn(LAST_SEQUENCE_ID_FAMILY, encodedRegionNameAsBytes,
+ Bytes.toBytes(lastSeqId));
+ builder.addCondition(ProtobufUtil.toCondition(row, LAST_SEQUENCE_ID_FAMILY,
+ encodedRegionNameAsBytes, CompareOperator.EQUAL, previousLastSeqIdAsBytes, null));
+ }
+ } else {
+ // also update last seq id when there is no value yet, and use CAS to make sure we do not
+ // overwrite
+ // other's value.
+ put.addColumn(LAST_SEQUENCE_ID_FAMILY, encodedRegionNameAsBytes, Bytes.toBytes(lastSeqId));
+ builder.addCondition(ProtobufUtil.toCondition(row, LAST_SEQUENCE_ID_FAMILY,
+ encodedRegionNameAsBytes, CompareOperator.EQUAL, null, null));
+ }
+ }
+ if (!put.isEmpty()) {
+ builder.addMutationRequest(ProtobufUtil.toMutation(MutationType.PUT, put));
+ }
+ }
+
+ @Override
+ public void setOffset(ReplicationQueueId queueId, String walGroup, ReplicationGroupOffset offset,
+ Map<String, Long> lastSeqIds) throws ReplicationException {
+ Put put = new Put(Bytes.toBytes(queueId.toString())).addColumn(QUEUE_FAMILY,
+ Bytes.toBytes(walGroup), Bytes.toBytes(offset.toString()));
+ AsyncTable<?> asyncTable = conn.toAsyncConnection().getTable(tableName);
+ try {
+ if (lastSeqIds.isEmpty()) {
+ FutureUtils.get(asyncTable.put(put));
+ } else {
+ for (;;) {
+ MultiRowMutationProtos.MutateRowsRequest.Builder builder =
+ MultiRowMutationProtos.MutateRowsRequest.newBuilder();
+ addLastSeqIdsPut(builder, queueId.getPeerId(), lastSeqIds, asyncTable);
+ if (builder.getMutationRequestCount() > 0) {
+ // use MultiRowMutationService to atomically update offset and last sequence ids
+ MultiRowMutationProtos.MutateRowsRequest request =
+ builder.addMutationRequest(ProtobufUtil.toMutation(MutationType.PUT, put)).build();
+ MultiRowMutationProtos.MutateRowsResponse responose =
+ FutureUtils.get(asyncTable.<MultiRowMutationProtos.MultiRowMutationService.Interface,
+ MultiRowMutationProtos.MutateRowsResponse> coprocessorService(
+ MultiRowMutationProtos.MultiRowMutationService::newStub,
+ (stub, controller, done) -> stub.mutateRows(controller, request, done),
+ put.getRow()));
+ if (responose.getProcessed()) {
+ break;
+ }
+ } else {
+ // we do not need to update last seq id, fallback to single put
+ FutureUtils.get(asyncTable.put(put));
+ break;
+ }
+ }
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to setOffset, queueId=" + queueId + ", walGroup="
+ + walGroup + ", offset=" + offset + ", lastSeqIds=" + lastSeqIds, e);
+ }
+ }
+
+ private ImmutableMap<String, ReplicationGroupOffset> parseOffsets(Result result) {
+ ImmutableMap.Builder<String, ReplicationGroupOffset> builder =
+ ImmutableMap.builderWithExpectedSize(result.size());
+ NavigableMap<byte[], byte[]> map = result.getFamilyMap(QUEUE_FAMILY);
+ if (map != null) {
+ map.forEach((k, v) -> {
+ String walGroup = Bytes.toString(k);
+ ReplicationGroupOffset offset = ReplicationGroupOffset.parse(Bytes.toString(v));
+ builder.put(walGroup, offset);
+ });
+ }
+ return builder.build();
+ }
+
+ private Map<String, ReplicationGroupOffset> getOffsets0(Table table, ReplicationQueueId queueId)
+ throws IOException {
+ Result result = table.get(new Get(Bytes.toBytes(queueId.toString())).addFamily(QUEUE_FAMILY));
+ return parseOffsets(result);
+ }
+
+ @Override
+ public Map<String, ReplicationGroupOffset> getOffsets(ReplicationQueueId queueId)
+ throws ReplicationException {
+ try (Table table = conn.getTable(tableName)) {
+ return getOffsets0(table, queueId);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to getOffsets, queueId=" + queueId, e);
+ }
+ }
+
+ private void listAllQueueIds(Table table, Scan scan, List<ReplicationQueueId> queueIds)
+ throws IOException {
+ try (ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ ReplicationQueueId queueId = ReplicationQueueId.parse(Bytes.toString(result.getRow()));
+ queueIds.add(queueId);
+ }
+ }
+ }
+
+ private void listAllQueueIds(Table table, String peerId, ServerName serverName,
+ List<ReplicationQueueId> queueIds) throws IOException {
+ listAllQueueIds(table,
+ new Scan().setStartStopRowForPrefixScan(ReplicationQueueId.getScanPrefix(serverName, peerId))
+ .addFamily(QUEUE_FAMILY).setFilter(new KeyOnlyFilter()),
+ queueIds);
+ }
+
+ @Override
+ public List<ReplicationQueueId> listAllQueueIds(String peerId) throws ReplicationException {
+ Scan scan = new Scan().setStartStopRowForPrefixScan(ReplicationQueueId.getScanPrefix(peerId))
+ .addFamily(QUEUE_FAMILY).setFilter(new KeyOnlyFilter());
+ List<ReplicationQueueId> queueIds = new ArrayList<>();
+ try (Table table = conn.getTable(tableName)) {
+ listAllQueueIds(table, scan, queueIds);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to listAllQueueIds, peerId=" + peerId, e);
+ }
+ return queueIds;
+ }
+
+ @Override
+ public List<ReplicationQueueId> listAllQueueIds(ServerName serverName)
+ throws ReplicationException {
+ List<ReplicationQueueId> queueIds = new ArrayList<>();
+ try (Table table = conn.getTable(tableName)) {
+ KeyOnlyFilter keyOnlyFilter = new KeyOnlyFilter();
+ String previousPeerId = null;
+ for (;;) {
+ // first, get the next peerId
+ Scan peerScan =
+ new Scan().addFamily(QUEUE_FAMILY).setOneRowLimit().setFilter(keyOnlyFilter);
+ if (previousPeerId != null) {
+ peerScan.withStartRow(ReplicationQueueId.getScanStartRowForNextPeerId(previousPeerId));
+ }
+ String peerId;
+ try (ResultScanner scanner = table.getScanner(peerScan)) {
+ Result result = scanner.next();
+ if (result == null) {
+ // no more peers, break
+ break;
+ }
+ peerId = ReplicationQueueId.getPeerId(Bytes.toString(result.getRow()));
+ }
+ listAllQueueIds(table, peerId, serverName, queueIds);
+ previousPeerId = peerId;
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to listAllQueueIds, serverName=" + serverName, e);
+ }
+ return queueIds;
+ }
+
+ @Override
+ public List<ReplicationQueueId> listAllQueueIds(String peerId, ServerName serverName)
+ throws ReplicationException {
+ List<ReplicationQueueId> queueIds = new ArrayList<>();
+ try (Table table = conn.getTable(tableName)) {
+ listAllQueueIds(table, peerId, serverName, queueIds);
+ } catch (IOException e) {
+ throw new ReplicationException(
+ "failed to listAllQueueIds, peerId=" + peerId + ", serverName=" + serverName, e);
+ }
+ return queueIds;
+ }
+
+ @Override
+ public List<ReplicationQueueData> listAllQueues() throws ReplicationException {
+ List<ReplicationQueueData> queues = new ArrayList<>();
+ Scan scan = new Scan().addFamily(QUEUE_FAMILY).setReadType(ReadType.STREAM);
+ try (Table table = conn.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ ReplicationQueueId queueId = ReplicationQueueId.parse(Bytes.toString(result.getRow()));
+ ReplicationQueueData queueData = new ReplicationQueueData(queueId, parseOffsets(result));
+ queues.add(queueData);
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to listAllQueues", e);
+ }
+ return queues;
+ }
+
+ @Override
+ public List<ServerName> listAllReplicators() throws ReplicationException {
+ Set<ServerName> replicators = new HashSet<>();
+ Scan scan = new Scan().addFamily(QUEUE_FAMILY).setFilter(new KeyOnlyFilter())
+ .setReadType(ReadType.STREAM);
+ try (Table table = conn.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ ReplicationQueueId queueId = ReplicationQueueId.parse(Bytes.toString(result.getRow()));
+ replicators.add(queueId.getServerName());
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to listAllReplicators", e);
+ }
+ return new ArrayList<>(replicators);
+ }
+
+ @Override
+ public Map<String, ReplicationGroupOffset> claimQueue(ReplicationQueueId queueId,
+ ServerName targetServerName) throws ReplicationException {
+ ReplicationQueueId newQueueId = queueId.claim(targetServerName);
+ byte[] coprocessorRow = ReplicationQueueId.getScanPrefix(queueId.getPeerId());
+ AsyncTable<?> asyncTable = conn.toAsyncConnection().getTable(tableName);
+ try (Table table = conn.getTable(tableName)) {
+ for (;;) {
+ Map<String, ReplicationGroupOffset> offsets = getOffsets0(table, queueId);
+ if (offsets.isEmpty()) {
+ return Collections.emptyMap();
+ }
+ Map.Entry<String, ReplicationGroupOffset> entry = offsets.entrySet().iterator().next();
+ ClientProtos.Condition condition = ProtobufUtil.toCondition(
+ Bytes.toBytes(queueId.toString()), QUEUE_FAMILY, Bytes.toBytes(entry.getKey()),
+ CompareOperator.EQUAL, Bytes.toBytes(entry.getValue().toString()), null);
+ Delete delete = new Delete(Bytes.toBytes(queueId.toString())).addFamily(QUEUE_FAMILY);
+ Put put = new Put(Bytes.toBytes(newQueueId.toString()));
+ offsets.forEach((walGroup, offset) -> put.addColumn(QUEUE_FAMILY, Bytes.toBytes(walGroup),
+ Bytes.toBytes(offset.toString())));
+ MultiRowMutationProtos.MutateRowsRequest request =
+ MultiRowMutationProtos.MutateRowsRequest.newBuilder().addCondition(condition)
+ .addMutationRequest(ProtobufUtil.toMutation(MutationType.DELETE, delete))
+ .addMutationRequest(ProtobufUtil.toMutation(MutationType.PUT, put)).build();
+ MultiRowMutationProtos.MutateRowsResponse resp =
+ FutureUtils.get(asyncTable.<MultiRowMutationProtos.MultiRowMutationService.Interface,
+ MultiRowMutationProtos.MutateRowsResponse> coprocessorService(
+ MultiRowMutationProtos.MultiRowMutationService::newStub,
+ (stub, controller, done) -> stub.mutateRows(controller, request, done),
+ coprocessorRow));
+ if (resp.getProcessed()) {
+ return offsets;
+ }
+ // if the multi is not processed, which usually the queue has already been claimed by
+ // others, for safety, let's try claiming again, usually the next get operation above will
+ // return an empty map and we will quit the loop.
+ }
+ } catch (IOException e) {
+ throw new ReplicationException(
+ "failed to claimQueue, queueId=" + queueId + ", targetServerName=" + targetServerName, e);
+ }
+ }
+
+ @Override
+ public void removeQueue(ReplicationQueueId queueId) throws ReplicationException {
+ try (Table table = conn.getTable(tableName)) {
+ table.delete(new Delete(Bytes.toBytes(queueId.toString())).addFamily(QUEUE_FAMILY));
+ } catch (IOException e) {
+ throw new ReplicationException("failed to removeQueue, queueId=" + queueId, e);
+ }
+ }
+
+ @Override
+ public void removeAllQueues(String peerId) throws ReplicationException {
+ Scan scan = new Scan().setStartStopRowForPrefixScan(ReplicationQueueId.getScanPrefix(peerId))
+ .addFamily(QUEUE_FAMILY).setFilter(new KeyOnlyFilter());
+ try (Table table = conn.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ table.delete(new Delete(result.getRow()));
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to listAllQueueIds, peerId=" + peerId, e);
+ }
+ }
+
+ @Override
+ public long getLastSequenceId(String encodedRegionName, String peerId)
+ throws ReplicationException {
+ byte[] qual = Bytes.toBytes(encodedRegionName);
+ try (Table table = conn.getTable(tableName)) {
+ Result result =
+ table.get(new Get(Bytes.toBytes(peerId)).addColumn(LAST_SEQUENCE_ID_FAMILY, qual));
+ byte[] lastSeqId = result.getValue(LAST_SEQUENCE_ID_FAMILY, qual);
+ return lastSeqId != null ? Bytes.toLong(lastSeqId) : HConstants.NO_SEQNUM;
+ } catch (IOException e) {
+ throw new ReplicationException("failed to getLastSequenceId, encodedRegionName="
+ + encodedRegionName + ", peerId=" + peerId, e);
+ }
+ }
+
+ @Override
+ public void setLastSequenceIds(String peerId, Map<String, Long> lastSeqIds)
+ throws ReplicationException {
+ // No need CAS and retry here, because it'll call setLastSequenceIds() for disabled peers
+ // only, so no conflict happen.
+ Put put = new Put(Bytes.toBytes(peerId));
+ lastSeqIds.forEach((encodedRegionName, lastSeqId) -> put.addColumn(LAST_SEQUENCE_ID_FAMILY,
+ Bytes.toBytes(encodedRegionName), Bytes.toBytes(lastSeqId)));
+ try (Table table = conn.getTable(tableName)) {
+ table.put(put);
+ } catch (IOException e) {
+ throw new ReplicationException(
+ "failed to setLastSequenceIds, peerId=" + peerId + ", lastSeqIds=" + lastSeqIds, e);
+ }
+ }
+
+ @Override
+ public void removeLastSequenceIds(String peerId) throws ReplicationException {
+ Delete delete = new Delete(Bytes.toBytes(peerId)).addFamily(LAST_SEQUENCE_ID_FAMILY);
+ try (Table table = conn.getTable(tableName)) {
+ table.delete(delete);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to removeLastSequenceIds, peerId=" + peerId, e);
+ }
+ }
+
+ @Override
+ public void removeLastSequenceIds(String peerId, List<String> encodedRegionNames)
+ throws ReplicationException {
+ Delete delete = new Delete(Bytes.toBytes(peerId));
+ encodedRegionNames.forEach(n -> delete.addColumns(LAST_SEQUENCE_ID_FAMILY, Bytes.toBytes(n)));
+ try (Table table = conn.getTable(tableName)) {
+ table.delete(delete);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to removeLastSequenceIds, peerId=" + peerId
+ + ", encodedRegionNames=" + encodedRegionNames, e);
+ }
+ }
+
+ @Override
+ public void removePeerFromHFileRefs(String peerId) throws ReplicationException {
+ try (Table table = conn.getTable(tableName)) {
+ table.delete(new Delete(Bytes.toBytes(peerId)).addFamily(HFILE_REF_FAMILY));
+ } catch (IOException e) {
+ throw new ReplicationException("failed to removePeerFromHFileRefs, peerId=" + peerId, e);
+ }
+ }
+
+ @Override
+ public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs)
+ throws ReplicationException {
+ Put put = new Put(Bytes.toBytes(peerId));
+ pairs.forEach(p -> put.addColumn(HFILE_REF_FAMILY, Bytes.toBytes(p.getSecond().getName()),
+ HConstants.EMPTY_BYTE_ARRAY));
+ try (Table table = conn.getTable(tableName)) {
+ table.put(put);
+ } catch (IOException e) {
+ throw new ReplicationException(
+ "failed to addHFileRefs, peerId=" + peerId + ", pairs=" + pairs, e);
+ }
+ }
+
+ @Override
+ public void removeHFileRefs(String peerId, List<String> files) throws ReplicationException {
+ Delete delete = new Delete(Bytes.toBytes(peerId));
+ files.forEach(f -> delete.addColumns(HFILE_REF_FAMILY, Bytes.toBytes(f)));
+ try (Table table = conn.getTable(tableName)) {
+ table.delete(delete);
+ } catch (IOException e) {
+ throw new ReplicationException(
+ "failed to removeHFileRefs, peerId=" + peerId + ", files=" + files, e);
+ }
+ }
+
+ @Override
+ public List<String> getAllPeersFromHFileRefsQueue() throws ReplicationException {
+ List<String> peerIds = new ArrayList<>();
+ Scan scan = new Scan().addFamily(HFILE_REF_FAMILY).setReadType(ReadType.STREAM)
+ .setFilter(new KeyOnlyFilter());
+ try (Table table = conn.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ peerIds.add(Bytes.toString(result.getRow()));
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to getAllPeersFromHFileRefsQueue", e);
+ }
+ return peerIds;
+ }
+
+ private <T extends Collection<String>> T scanHFiles(Scan scan, Supplier<T> creator)
+ throws IOException {
+ T files = creator.get();
+ try (Table table = conn.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ CellScanner cellScanner = result.cellScanner();
+ while (cellScanner.advance()) {
+ Cell cell = cellScanner.current();
+ files.add(Bytes.toString(CellUtil.cloneQualifier(cell)));
+ }
+ }
+ }
+ return files;
+ }
+
+ @Override
+ public List<String> getReplicableHFiles(String peerId) throws ReplicationException {
+ // use scan to avoid getting a too large row one time, which may cause a very huge memory usage.
+ Scan scan = new Scan().addFamily(HFILE_REF_FAMILY)
+ .setStartStopRowForPrefixScan(Bytes.toBytes(peerId)).setAllowPartialResults(true);
+ try {
+ return scanHFiles(scan, ArrayList::new);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to getReplicableHFiles, peerId=" + peerId, e);
+ }
+ }
+
+ @Override
+ public Set<String> getAllHFileRefs() throws ReplicationException {
+ Scan scan = new Scan().addFamily(HFILE_REF_FAMILY).setReadType(ReadType.STREAM)
+ .setAllowPartialResults(true);
+ try {
+ return scanHFiles(scan, HashSet::new);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to getAllHFileRefs", e);
+ }
+ }
+}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
deleted file mode 100644
index f3506ad3555..00000000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
+++ /dev/null
@@ -1,689 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import static java.util.stream.Collectors.toList;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.KeeperException.BadVersionException;
-import org.apache.zookeeper.KeeperException.NoNodeException;
-import org.apache.zookeeper.KeeperException.NodeExistsException;
-import org.apache.zookeeper.KeeperException.NotEmptyException;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
-
-/**
- * ZK based replication queue storage.
- * <p>
- * The base znode for each regionserver is the regionserver name. For example:
- *
- * <pre>
- * /hbase/replication/rs/hostname.example.org,6020,1234
- * </pre>
- *
- * Within this znode, the region server maintains a set of WAL replication queues. These queues are
- * represented by child znodes named using there give queue id. For example:
- *
- * <pre>
- * /hbase/replication/rs/hostname.example.org,6020,1234/1
- * /hbase/replication/rs/hostname.example.org,6020,1234/2
- * </pre>
- *
- * Each queue has one child znode for every WAL that still needs to be replicated. The value of
- * these WAL child znodes is the latest position that has been replicated. This position is updated
- * every time a WAL entry is replicated. For example:
- *
- * <pre>
- * /hbase/replication/rs/hostname.example.org,6020,1234/1/23522342.23422 [VALUE: 254]
- * </pre>
- */
-@InterfaceAudience.Private
-class ZKReplicationQueueStorage extends ZKReplicationStorageBase
- implements ReplicationQueueStorage {
-
- private static final Logger LOG = LoggerFactory.getLogger(ZKReplicationQueueStorage.class);
-
- public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY =
- "zookeeper.znode.replication.hfile.refs";
- public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT = "hfile-refs";
-
- public static final String ZOOKEEPER_ZNODE_REPLICATION_REGIONS_KEY =
- "zookeeper.znode.replication.regions";
- public static final String ZOOKEEPER_ZNODE_REPLICATION_REGIONS_DEFAULT = "regions";
-
- /**
- * The name of the znode that contains all replication queues
- */
- private final String queuesZNode;
-
- /**
- * The name of the znode that contains queues of hfile references to be replicated
- */
- private final String hfileRefsZNode;
-
- final String regionsZNode;
-
- public ZKReplicationQueueStorage(ZKWatcher zookeeper, Configuration conf) {
- super(zookeeper, conf);
-
- String queuesZNodeName = conf.get("zookeeper.znode.replication.rs", "rs");
- String hfileRefsZNodeName = conf.get(ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
- ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
- this.queuesZNode = ZNodePaths.joinZNode(replicationZNode, queuesZNodeName);
- this.hfileRefsZNode = ZNodePaths.joinZNode(replicationZNode, hfileRefsZNodeName);
- this.regionsZNode = ZNodePaths.joinZNode(replicationZNode, conf
- .get(ZOOKEEPER_ZNODE_REPLICATION_REGIONS_KEY, ZOOKEEPER_ZNODE_REPLICATION_REGIONS_DEFAULT));
- }
-
- @Override
- public String getRsNode(ServerName serverName) {
- return ZNodePaths.joinZNode(queuesZNode, serverName.getServerName());
- }
-
- private String getQueueNode(ServerName serverName, String queueId) {
- return ZNodePaths.joinZNode(getRsNode(serverName), queueId);
- }
-
- private String getFileNode(String queueNode, String fileName) {
- return ZNodePaths.joinZNode(queueNode, fileName);
- }
-
- private String getFileNode(ServerName serverName, String queueId, String fileName) {
- return getFileNode(getQueueNode(serverName, queueId), fileName);
- }
-
- /**
- * <p>
- * Put all regions under /hbase/replication/regions znode will lead to too many children because
- * of the huge number of regions in real production environment. So here we will distribute the
- * znodes to multiple directories.
- * </p>
- * <p>
- * So the final znode path will be format like this:
- *
- * <pre>
- * /hbase/replication/regions/dd/04/e76a6966d4ffa908ed0586764767-100
- * </pre>
- *
- * Here the full encoded region name is dd04e76a6966d4ffa908ed0586764767, and we use the first two
- * characters 'dd' as the first level directory name, and use the next two characters '04' as the
- * second level directory name, and the rest part as the prefix of the znode, and the suffix '100'
- * is the peer id.
- * </p>
- * @param encodedRegionName the encoded region name.
- * @param peerId peer id for replication.
- * @return ZNode path to persist the max sequence id that we've pushed for the given region and
- * peer.
- */
- String getSerialReplicationRegionPeerNode(String encodedRegionName, String peerId) {
- if (encodedRegionName == null || encodedRegionName.length() != RegionInfo.MD5_HEX_LENGTH) {
- throw new IllegalArgumentException(
- "Invalid encoded region name: " + encodedRegionName + ", length should be 32.");
- }
- return new StringBuilder(regionsZNode).append(ZNodePaths.ZNODE_PATH_SEPARATOR)
- .append(encodedRegionName, 0, 2).append(ZNodePaths.ZNODE_PATH_SEPARATOR)
- .append(encodedRegionName, 2, 4).append(ZNodePaths.ZNODE_PATH_SEPARATOR)
- .append(encodedRegionName, 4, encodedRegionName.length()).append("-").append(peerId)
- .toString();
- }
-
- @Override
- public void removeQueue(ServerName serverName, String queueId) throws ReplicationException {
- try {
- ZKUtil.deleteNodeRecursively(zookeeper, getQueueNode(serverName, queueId));
- } catch (KeeperException e) {
- throw new ReplicationException(
- "Failed to delete queue (serverName=" + serverName + ", queueId=" + queueId + ")", e);
- }
- }
-
- @Override
- public void addWAL(ServerName serverName, String queueId, String fileName)
- throws ReplicationException {
- try {
- ZKUtil.createWithParents(zookeeper, getFileNode(serverName, queueId, fileName));
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to add wal to queue (serverName=" + serverName
- + ", queueId=" + queueId + ", fileName=" + fileName + ")", e);
- }
- }
-
- @Override
- public void removeWAL(ServerName serverName, String queueId, String fileName)
- throws ReplicationException {
- String fileNode = getFileNode(serverName, queueId, fileName);
- try {
- ZKUtil.deleteNode(zookeeper, fileNode);
- } catch (NoNodeException e) {
- LOG.warn("{} already deleted when removing log", fileNode);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to remove wal from queue (serverName=" + serverName
- + ", queueId=" + queueId + ", fileName=" + fileName + ")", e);
- }
- }
-
- private void addLastSeqIdsToOps(String queueId, Map<String, Long> lastSeqIds,
- List<ZKUtilOp> listOfOps) throws KeeperException, ReplicationException {
- String peerId = new ReplicationQueueInfo(queueId).getPeerId();
- for (Entry<String, Long> lastSeqEntry : lastSeqIds.entrySet()) {
- String path = getSerialReplicationRegionPeerNode(lastSeqEntry.getKey(), peerId);
- Pair<Long, Integer> p = getLastSequenceIdWithVersion(lastSeqEntry.getKey(), peerId);
- byte[] data = ZKUtil.positionToByteArray(lastSeqEntry.getValue());
- if (p.getSecond() < 0) { // ZNode does not exist.
- ZKUtil.createWithParents(zookeeper,
- path.substring(0, path.lastIndexOf(ZNodePaths.ZNODE_PATH_SEPARATOR)));
- listOfOps.add(ZKUtilOp.createAndFailSilent(path, data));
- continue;
- }
- // Perform CAS in a specific version v0 (HBASE-20138)
- int v0 = p.getSecond();
- long lastPushedSeqId = p.getFirst();
- if (lastSeqEntry.getValue() <= lastPushedSeqId) {
- continue;
- }
- listOfOps.add(ZKUtilOp.setData(path, data, v0));
- }
- }
-
- @Override
- public void setWALPosition(ServerName serverName, String queueId, String fileName, long position,
- Map<String, Long> lastSeqIds) throws ReplicationException {
- try {
- for (int retry = 0;; retry++) {
- List<ZKUtilOp> listOfOps = new ArrayList<>();
- if (position > 0) {
- listOfOps.add(ZKUtilOp.setData(getFileNode(serverName, queueId, fileName),
- ZKUtil.positionToByteArray(position)));
- }
- // Persist the max sequence id(s) of regions for serial replication atomically.
- addLastSeqIdsToOps(queueId, lastSeqIds, listOfOps);
- if (listOfOps.isEmpty()) {
- return;
- }
- try {
- ZKUtil.multiOrSequential(zookeeper, listOfOps, false);
- return;
- } catch (KeeperException.BadVersionException | KeeperException.NodeExistsException e) {
- LOG.warn(
- "Bad version(or node exist) when persist the last pushed sequence id to zookeeper "
- + "storage, Retry = " + retry + ", serverName=" + serverName + ", queueId=" + queueId
- + ", fileName=" + fileName);
- }
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to set log position (serverName=" + serverName
- + ", queueId=" + queueId + ", fileName=" + fileName + ", position=" + position + ")", e);
- }
- }
-
- /**
- * Return the {lastPushedSequenceId, ZNodeDataVersion} pair. if ZNodeDataVersion is -1, it means
- * that the ZNode does not exist.
- */
- protected Pair<Long, Integer> getLastSequenceIdWithVersion(String encodedRegionName,
- String peerId) throws KeeperException {
- Stat stat = new Stat();
- String path = getSerialReplicationRegionPeerNode(encodedRegionName, peerId);
- byte[] data = ZKUtil.getDataNoWatch(zookeeper, path, stat);
- if (data == null) {
- // ZNode does not exist, so just return version -1 to indicate that no node exist.
- return Pair.newPair(HConstants.NO_SEQNUM, -1);
- }
- try {
- return Pair.newPair(ZKUtil.parseWALPositionFrom(data), stat.getVersion());
- } catch (DeserializationException de) {
- LOG.warn("Failed to parse log position (region=" + encodedRegionName + ", peerId=" + peerId
- + "), data=" + Bytes.toStringBinary(data));
- }
- return Pair.newPair(HConstants.NO_SEQNUM, stat.getVersion());
- }
-
- @Override
- public long getLastSequenceId(String encodedRegionName, String peerId)
- throws ReplicationException {
- try {
- return getLastSequenceIdWithVersion(encodedRegionName, peerId).getFirst();
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get last pushed sequence id (encodedRegionName="
- + encodedRegionName + ", peerId=" + peerId + ")", e);
- }
- }
-
- @Override
- public void setLastSequenceIds(String peerId, Map<String, Long> lastSeqIds)
- throws ReplicationException {
- try {
- // No need CAS and retry here, because it'll call setLastSequenceIds() for disabled peers
- // only, so no conflict happen.
- List<ZKUtilOp> listOfOps = new ArrayList<>();
- for (Entry<String, Long> lastSeqEntry : lastSeqIds.entrySet()) {
- String path = getSerialReplicationRegionPeerNode(lastSeqEntry.getKey(), peerId);
- ZKUtil.createWithParents(zookeeper, path);
- listOfOps.add(ZKUtilOp.setData(path, ZKUtil.positionToByteArray(lastSeqEntry.getValue())));
- }
- if (!listOfOps.isEmpty()) {
- ZKUtil.multiOrSequential(zookeeper, listOfOps, true);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to set last sequence ids, peerId=" + peerId
- + ", size of lastSeqIds=" + lastSeqIds.size(), e);
- }
- }
-
- @Override
- public void removeLastSequenceIds(String peerId) throws ReplicationException {
- String suffix = "-" + peerId;
- try {
- StringBuilder sb = new StringBuilder(regionsZNode);
- int regionsZNodeLength = regionsZNode.length();
- int levelOneLength = regionsZNodeLength + 3;
- int levelTwoLength = levelOneLength + 3;
- List<String> levelOneDirs = ZKUtil.listChildrenNoWatch(zookeeper, regionsZNode);
- // it is possible that levelOneDirs is null if we haven't write any last pushed sequence ids
- // yet, so we need an extra check here.
- if (CollectionUtils.isEmpty(levelOneDirs)) {
- return;
- }
- for (String levelOne : levelOneDirs) {
- sb.append(ZNodePaths.ZNODE_PATH_SEPARATOR).append(levelOne);
- for (String levelTwo : ZKUtil.listChildrenNoWatch(zookeeper, sb.toString())) {
- sb.append(ZNodePaths.ZNODE_PATH_SEPARATOR).append(levelTwo);
- for (String znode : ZKUtil.listChildrenNoWatch(zookeeper, sb.toString())) {
- if (znode.endsWith(suffix)) {
- sb.append(ZNodePaths.ZNODE_PATH_SEPARATOR).append(znode);
- ZKUtil.deleteNode(zookeeper, sb.toString());
- sb.setLength(levelTwoLength);
- }
- }
- sb.setLength(levelOneLength);
- }
- sb.setLength(regionsZNodeLength);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to remove all last sequence ids, peerId=" + peerId, e);
- }
- }
-
- @Override
- public void removeLastSequenceIds(String peerId, List<String> encodedRegionNames)
- throws ReplicationException {
- try {
- List<ZKUtilOp> listOfOps =
- encodedRegionNames.stream().map(n -> getSerialReplicationRegionPeerNode(n, peerId))
- .map(ZKUtilOp::deleteNodeFailSilent).collect(Collectors.toList());
- ZKUtil.multiOrSequential(zookeeper, listOfOps, true);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to remove last sequence ids, peerId=" + peerId
- + ", encodedRegionNames.size=" + encodedRegionNames.size(), e);
- }
- }
-
- @Override
- public long getWALPosition(ServerName serverName, String queueId, String fileName)
- throws ReplicationException {
- byte[] bytes;
- try {
- bytes = ZKUtil.getData(zookeeper, getFileNode(serverName, queueId, fileName));
- } catch (KeeperException | InterruptedException e) {
- throw new ReplicationException("Failed to get log position (serverName=" + serverName
- + ", queueId=" + queueId + ", fileName=" + fileName + ")", e);
- }
- try {
- return ZKUtil.parseWALPositionFrom(bytes);
- } catch (DeserializationException de) {
- LOG.warn("Failed parse log position (serverName={}, queueId={}, fileName={})", serverName,
- queueId, fileName);
- }
- // if we can not parse the position, start at the beginning of the wal file again
- return 0;
- }
-
- /**
- * This implement must update the cversion of root {@link #queuesZNode}. The optimistic lock of
- * the {@link #getAllWALs()} method is based on the cversion of root {@link #queuesZNode}.
- * @see #getAllWALs() to show the usage of the cversion of root {@link #queuesZNode} .
- */
- @Override
- public Pair<String, SortedSet<String>> claimQueue(ServerName sourceServerName, String queueId,
- ServerName destServerName) throws ReplicationException {
- LOG.info("Atomically moving {}/{}'s WALs to {}", sourceServerName, queueId, destServerName);
- try {
- ZKUtil.createWithParents(zookeeper, getRsNode(destServerName));
- } catch (KeeperException e) {
- throw new ReplicationException("Claim queue queueId=" + queueId + " from " + sourceServerName
- + " to " + destServerName + " failed when creating the node for " + destServerName, e);
- }
- String newQueueId = queueId + "-" + sourceServerName;
- try {
- String oldQueueNode = getQueueNode(sourceServerName, queueId);
- List<String> wals = ZKUtil.listChildrenNoWatch(zookeeper, oldQueueNode);
- if (CollectionUtils.isEmpty(wals)) {
- ZKUtil.deleteNodeFailSilent(zookeeper, oldQueueNode);
- LOG.info("Removed empty {}/{}", sourceServerName, queueId);
- return new Pair<>(newQueueId, Collections.emptySortedSet());
- }
- String newQueueNode = getQueueNode(destServerName, newQueueId);
- List<ZKUtilOp> listOfOps = new ArrayList<>();
- SortedSet<String> logQueue = new TreeSet<>();
- // create the new cluster znode
- listOfOps.add(ZKUtilOp.createAndFailSilent(newQueueNode, HConstants.EMPTY_BYTE_ARRAY));
- // get the offset of the logs and set it to new znodes
- for (String wal : wals) {
- String oldWalNode = getFileNode(oldQueueNode, wal);
- byte[] logOffset = ZKUtil.getData(this.zookeeper, oldWalNode);
- LOG.debug("Creating {} with data {}", wal, Bytes.toStringBinary(logOffset));
- String newWalNode = getFileNode(newQueueNode, wal);
- listOfOps.add(ZKUtilOp.createAndFailSilent(newWalNode, logOffset));
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldWalNode));
- logQueue.add(wal);
- }
- // add delete op for peer
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldQueueNode));
- // Append new queue id for prevent lock competition in zookeeper server.
- String claimLockZNode = ZNodePaths.joinZNode(queuesZNode, "cversion_" + newQueueId);
- // A trick for update the cversion of root queuesZNode .
- // The optimistic lock of the getAllWALs() method is based on the cversion of root queuesZNode
- listOfOps.add(ZKUtilOp.createAndFailSilent(claimLockZNode, HConstants.EMPTY_BYTE_ARRAY));
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(claimLockZNode));
-
- LOG.trace("The multi list size is {}", listOfOps.size());
- ZKUtil.multiOrSequential(zookeeper, listOfOps, false);
-
- LOG.info("Atomically moved {}/{}'s WALs to {}", sourceServerName, queueId, destServerName);
- return new Pair<>(newQueueId, logQueue);
- } catch (NoNodeException | NodeExistsException | NotEmptyException | BadVersionException e) {
- // Multi call failed; it looks like some other regionserver took away the logs.
- // These exceptions mean that zk tells us the request can not be execute. So return an empty
- // queue to tell the upper layer that claim nothing. For other types of exception should be
- // thrown out to notify the upper layer.
- LOG.info("Claim queue queueId={} from {} to {} failed with {}, someone else took the log?",
- queueId, sourceServerName, destServerName, e.toString());
- return new Pair<>(newQueueId, Collections.emptySortedSet());
- } catch (KeeperException | InterruptedException e) {
- throw new ReplicationException("Claim queue queueId=" + queueId + " from " + sourceServerName
- + " to " + destServerName + " failed", e);
- }
- }
-
- @Override
- public void removeReplicatorIfQueueIsEmpty(ServerName serverName) throws ReplicationException {
- try {
- ZKUtil.deleteNodeFailSilent(zookeeper, getRsNode(serverName));
- } catch (NotEmptyException e) {
- // keep silence to avoid logging too much.
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to remove replicator for " + serverName, e);
- }
- }
-
- private List<ServerName> getListOfReplicators0() throws KeeperException {
- List<String> children = ZKUtil.listChildrenNoWatch(zookeeper, queuesZNode);
- if (children == null) {
- children = Collections.emptyList();
- }
- return children.stream().map(ServerName::parseServerName).collect(toList());
- }
-
- @Override
- public List<ServerName> getListOfReplicators() throws ReplicationException {
- try {
- return getListOfReplicators0();
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get list of replicators", e);
- }
- }
-
- private List<String> getWALsInQueue0(ServerName serverName, String queueId)
- throws KeeperException {
- List<String> children =
- ZKUtil.listChildrenNoWatch(zookeeper, getQueueNode(serverName, queueId));
- return children != null ? children : Collections.emptyList();
- }
-
- @Override
- public List<String> getWALsInQueue(ServerName serverName, String queueId)
- throws ReplicationException {
- try {
- return getWALsInQueue0(serverName, queueId);
- } catch (KeeperException e) {
- throw new ReplicationException(
- "Failed to get wals in queue (serverName=" + serverName + ", queueId=" + queueId + ")", e);
- }
- }
-
- private List<String> getAllQueues0(ServerName serverName) throws KeeperException {
- List<String> children = ZKUtil.listChildrenNoWatch(zookeeper, getRsNode(serverName));
- return children != null ? children : Collections.emptyList();
- }
-
- @Override
- public List<String> getAllQueues(ServerName serverName) throws ReplicationException {
- try {
- return getAllQueues0(serverName);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get all queues (serverName=" + serverName + ")", e);
- }
- }
-
- // will be overridden in UTs
- protected int getQueuesZNodeCversion() throws KeeperException {
- Stat stat = new Stat();
- ZKUtil.getDataNoWatch(this.zookeeper, this.queuesZNode, stat);
- return stat.getCversion();
- }
-
- /**
- * The optimistic lock of this implement is based on the cversion of root {@link #queuesZNode}.
- * Therefore, we must update the cversion of root {@link #queuesZNode} when migrate wal nodes to
- * other queues.
- * @see #claimQueue(ServerName, String, ServerName) as an example of updating root
- * {@link #queuesZNode} cversion.
- */
- @Override
- public Set<String> getAllWALs() throws ReplicationException {
- try {
- for (int retry = 0;; retry++) {
- int v0 = getQueuesZNodeCversion();
- List<ServerName> rss = getListOfReplicators0();
- if (rss.isEmpty()) {
- LOG.debug("Didn't find a RegionServer that replicates, won't prevent deletions.");
- return Collections.emptySet();
- }
- Set<String> wals = new HashSet<>();
- for (ServerName rs : rss) {
- for (String queueId : getAllQueues0(rs)) {
- wals.addAll(getWALsInQueue0(rs, queueId));
- }
- }
- int v1 = getQueuesZNodeCversion();
- if (v0 == v1) {
- return wals;
- }
- LOG.info("Replication queue node cversion changed from %d to %d, retry = %d", v0, v1,
- retry);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get all wals", e);
- }
- }
-
- private String getHFileRefsPeerNode(String peerId) {
- return ZNodePaths.joinZNode(hfileRefsZNode, peerId);
- }
-
- private String getHFileNode(String peerNode, String fileName) {
- return ZNodePaths.joinZNode(peerNode, fileName);
- }
-
- @Override
- public void addPeerToHFileRefs(String peerId) throws ReplicationException {
- String peerNode = getHFileRefsPeerNode(peerId);
- try {
- if (ZKUtil.checkExists(zookeeper, peerNode) == -1) {
- LOG.info("Adding peer {} to hfile reference queue.", peerId);
- ZKUtil.createWithParents(zookeeper, peerNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to add peer " + peerId + " to hfile reference queue.",
- e);
- }
- }
-
- @Override
- public void removePeerFromHFileRefs(String peerId) throws ReplicationException {
- String peerNode = getHFileRefsPeerNode(peerId);
- try {
- if (ZKUtil.checkExists(zookeeper, peerNode) == -1) {
- LOG.debug("Peer {} not found in hfile reference queue.", peerNode);
- } else {
- LOG.info("Removing peer {} from hfile reference queue.", peerNode);
- ZKUtil.deleteNodeRecursively(zookeeper, peerNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException(
- "Failed to remove peer " + peerId + " from hfile reference queue.", e);
- }
- }
-
- @Override
- public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs)
- throws ReplicationException {
- String peerNode = getHFileRefsPeerNode(peerId);
- LOG.debug("Adding hfile references {} in queue {}", pairs, peerNode);
- List<ZKUtilOp> listOfOps =
- pairs.stream().map(p -> p.getSecond().getName()).map(n -> getHFileNode(peerNode, n))
- .map(f -> ZKUtilOp.createAndFailSilent(f, HConstants.EMPTY_BYTE_ARRAY)).collect(toList());
- LOG.debug("The multi list size for adding hfile references in zk for node {} is {}", peerNode,
- listOfOps.size());
- try {
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to add hfile reference to peer " + peerId, e);
- }
- }
-
- @Override
- public void removeHFileRefs(String peerId, List<String> files) throws ReplicationException {
- String peerNode = getHFileRefsPeerNode(peerId);
- LOG.debug("Removing hfile references {} from queue {}", files, peerNode);
-
- List<ZKUtilOp> listOfOps = files.stream().map(n -> getHFileNode(peerNode, n))
- .map(ZKUtilOp::deleteNodeFailSilent).collect(toList());
- LOG.debug("The multi list size for removing hfile references in zk for node {} is {}", peerNode,
- listOfOps.size());
- try {
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to remove hfile reference from peer " + peerId, e);
- }
- }
-
- private List<String> getAllPeersFromHFileRefsQueue0() throws KeeperException {
- List<String> children = ZKUtil.listChildrenNoWatch(zookeeper, hfileRefsZNode);
- return children != null ? children : Collections.emptyList();
- }
-
- @Override
- public List<String> getAllPeersFromHFileRefsQueue() throws ReplicationException {
- try {
- return getAllPeersFromHFileRefsQueue0();
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get list of all peers in hfile references node.",
- e);
- }
- }
-
- private List<String> getReplicableHFiles0(String peerId) throws KeeperException {
- List<String> children =
- ZKUtil.listChildrenNoWatch(this.zookeeper, getHFileRefsPeerNode(peerId));
- return children != null ? children : Collections.emptyList();
- }
-
- @Override
- public List<String> getReplicableHFiles(String peerId) throws ReplicationException {
- try {
- return getReplicableHFiles0(peerId);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get list of hfile references for peer " + peerId,
- e);
- }
- }
-
- // will be overridden in UTs
- protected int getHFileRefsZNodeCversion() throws ReplicationException {
- Stat stat = new Stat();
- try {
- ZKUtil.getDataNoWatch(zookeeper, hfileRefsZNode, stat);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get stat of replication hfile references node.", e);
- }
- return stat.getCversion();
- }
-
- @Override
- public Set<String> getAllHFileRefs() throws ReplicationException {
- try {
- for (int retry = 0;; retry++) {
- int v0 = getHFileRefsZNodeCversion();
- List<String> peers = getAllPeersFromHFileRefsQueue();
- if (peers.isEmpty()) {
- LOG.debug("Didn't find any peers with hfile references, won't prevent deletions.");
- return Collections.emptySet();
- }
- Set<String> hfileRefs = new HashSet<>();
- for (String peer : peers) {
- hfileRefs.addAll(getReplicableHFiles0(peer));
- }
- int v1 = getHFileRefsZNodeCversion();
- if (v0 == v1) {
- return hfileRefs;
- }
- LOG.debug("Replication hfile references node cversion changed from %d to %d, retry = %d",
- v0, v1, retry);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get all hfile refs", e);
- }
- }
-}
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
index 15cf5b1f1f6..dc46e4f1c7c 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
@@ -17,30 +17,18 @@
*/
package org.apache.hadoop.hbase.replication;
-import static org.hamcrest.CoreMatchers.hasItems;
-import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
-import org.apache.zookeeper.KeeperException;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
-
/**
* White box testing for replication state interfaces. Implementations should extend this class, and
* initialize the interfaces properly.
@@ -49,7 +37,6 @@ public abstract class TestReplicationStateBasic {
private static final Logger LOG = LoggerFactory.getLogger(TestReplicationStateBasic.class);
- protected ReplicationQueueStorage rqs;
protected ServerName server1 = ServerName.valueOf("hostname1.example.org", 1234, 12345);
protected ServerName server2 = ServerName.valueOf("hostname2.example.org", 1234, 12345);
protected ServerName server3 = ServerName.valueOf("hostname3.example.org", 1234, 12345);
@@ -66,161 +53,6 @@ public abstract class TestReplicationStateBasic {
protected static final int ZK_MAX_COUNT = 300;
protected static final int ZK_SLEEP_INTERVAL = 100; // millis
- @Test
- public void testReplicationQueueStorage() throws ReplicationException {
- // Test methods with empty state
- assertEquals(0, rqs.getListOfReplicators().size());
- assertTrue(rqs.getWALsInQueue(server1, "qId1").isEmpty());
- assertTrue(rqs.getAllQueues(server1).isEmpty());
-
- /*
- * Set up data Two replicators: -- server1: three queues with 0, 1 and 2 log files each --
- * server2: zero queues
- */
- rqs.addWAL(server1, "qId1", "trash");
- rqs.removeWAL(server1, "qId1", "trash");
- rqs.addWAL(server1, "qId2", "filename1");
- rqs.addWAL(server1, "qId3", "filename2");
- rqs.addWAL(server1, "qId3", "filename3");
- rqs.addWAL(server2, "trash", "trash");
- rqs.removeQueue(server2, "trash");
-
- List<ServerName> reps = rqs.getListOfReplicators();
- assertEquals(2, reps.size());
- assertTrue(server1.getServerName(), reps.contains(server1));
- assertTrue(server2.getServerName(), reps.contains(server2));
-
- assertTrue(rqs.getWALsInQueue(ServerName.valueOf("bogus", 12345, 12345), "bogus").isEmpty());
- assertTrue(rqs.getWALsInQueue(server1, "bogus").isEmpty());
- assertEquals(0, rqs.getWALsInQueue(server1, "qId1").size());
- assertEquals(1, rqs.getWALsInQueue(server1, "qId2").size());
- assertEquals("filename1", rqs.getWALsInQueue(server1, "qId2").get(0));
-
- assertTrue(rqs.getAllQueues(ServerName.valueOf("bogus", 12345, -1L)).isEmpty());
- assertEquals(0, rqs.getAllQueues(server2).size());
- List<String> list = rqs.getAllQueues(server1);
- assertEquals(3, list.size());
- assertTrue(list.contains("qId2"));
- assertTrue(list.contains("qId3"));
- }
-
- private void removeAllQueues(ServerName serverName) throws ReplicationException {
- for (String queue : rqs.getAllQueues(serverName)) {
- rqs.removeQueue(serverName, queue);
- }
- }
-
- @Test
- public void testReplicationQueues() throws ReplicationException {
- // Initialize ReplicationPeer so we can add peers (we don't transfer lone queues)
- rp.init();
-
- rqs.removeQueue(server1, "bogus");
- rqs.removeWAL(server1, "bogus", "bogus");
- removeAllQueues(server1);
- assertEquals(0, rqs.getAllQueues(server1).size());
- assertEquals(0, rqs.getWALPosition(server1, "bogus", "bogus"));
- assertTrue(rqs.getWALsInQueue(server1, "bogus").isEmpty());
- assertTrue(rqs.getAllQueues(ServerName.valueOf("bogus", 1234, 12345)).isEmpty());
-
- populateQueues();
-
- assertEquals(3, rqs.getListOfReplicators().size());
- assertEquals(0, rqs.getWALsInQueue(server2, "qId1").size());
- assertEquals(5, rqs.getWALsInQueue(server3, "qId5").size());
- assertEquals(0, rqs.getWALPosition(server3, "qId1", "filename0"));
- rqs.setWALPosition(server3, "qId5", "filename4", 354L, Collections.emptyMap());
- assertEquals(354L, rqs.getWALPosition(server3, "qId5", "filename4"));
-
- assertEquals(5, rqs.getWALsInQueue(server3, "qId5").size());
- assertEquals(0, rqs.getWALsInQueue(server2, "qId1").size());
- assertEquals(0, rqs.getAllQueues(server1).size());
- assertEquals(1, rqs.getAllQueues(server2).size());
- assertEquals(5, rqs.getAllQueues(server3).size());
-
- assertEquals(0, rqs.getAllQueues(server1).size());
- rqs.removeReplicatorIfQueueIsEmpty(server1);
- assertEquals(2, rqs.getListOfReplicators().size());
-
- List<String> queues = rqs.getAllQueues(server3);
- assertEquals(5, queues.size());
- for (String queue : queues) {
- rqs.claimQueue(server3, queue, server2);
- }
- rqs.removeReplicatorIfQueueIsEmpty(server3);
- assertEquals(1, rqs.getListOfReplicators().size());
-
- assertEquals(6, rqs.getAllQueues(server2).size());
- removeAllQueues(server2);
- rqs.removeReplicatorIfQueueIsEmpty(server2);
- assertEquals(0, rqs.getListOfReplicators().size());
- }
-
- @Test
- public void testHfileRefsReplicationQueues() throws ReplicationException, KeeperException {
- rp.init();
-
- List<Pair<Path, Path>> files1 = new ArrayList<>(3);
- files1.add(new Pair<>(null, new Path("file_1")));
- files1.add(new Pair<>(null, new Path("file_2")));
- files1.add(new Pair<>(null, new Path("file_3")));
- assertTrue(rqs.getReplicableHFiles(ID_ONE).isEmpty());
- assertEquals(0, rqs.getAllPeersFromHFileRefsQueue().size());
- rp.getPeerStorage().addPeer(ID_ONE,
- ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build(), true,
- SyncReplicationState.NONE);
- rqs.addPeerToHFileRefs(ID_ONE);
- rqs.addHFileRefs(ID_ONE, files1);
- assertEquals(1, rqs.getAllPeersFromHFileRefsQueue().size());
- assertEquals(3, rqs.getReplicableHFiles(ID_ONE).size());
- List<String> hfiles2 = new ArrayList<>(files1.size());
- for (Pair<Path, Path> p : files1) {
- hfiles2.add(p.getSecond().getName());
- }
- String removedString = hfiles2.remove(0);
- rqs.removeHFileRefs(ID_ONE, hfiles2);
- assertEquals(1, rqs.getReplicableHFiles(ID_ONE).size());
- hfiles2 = new ArrayList<>(1);
- hfiles2.add(removedString);
- rqs.removeHFileRefs(ID_ONE, hfiles2);
- assertEquals(0, rqs.getReplicableHFiles(ID_ONE).size());
- rp.getPeerStorage().removePeer(ID_ONE);
- }
-
- @Test
- public void testRemovePeerForHFileRefs() throws ReplicationException, KeeperException {
- rp.init();
- rp.getPeerStorage().addPeer(ID_ONE,
- ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build(), true,
- SyncReplicationState.NONE);
- rqs.addPeerToHFileRefs(ID_ONE);
- rp.getPeerStorage().addPeer(ID_TWO,
- ReplicationPeerConfig.newBuilder().setClusterKey(KEY_TWO).build(), true,
- SyncReplicationState.NONE);
- rqs.addPeerToHFileRefs(ID_TWO);
-
- List<Pair<Path, Path>> files1 = new ArrayList<>(3);
- files1.add(new Pair<>(null, new Path("file_1")));
- files1.add(new Pair<>(null, new Path("file_2")));
- files1.add(new Pair<>(null, new Path("file_3")));
- rqs.addHFileRefs(ID_ONE, files1);
- rqs.addHFileRefs(ID_TWO, files1);
- assertEquals(2, rqs.getAllPeersFromHFileRefsQueue().size());
- assertEquals(3, rqs.getReplicableHFiles(ID_ONE).size());
- assertEquals(3, rqs.getReplicableHFiles(ID_TWO).size());
-
- rp.getPeerStorage().removePeer(ID_ONE);
- rqs.removePeerFromHFileRefs(ID_ONE);
- assertEquals(1, rqs.getAllPeersFromHFileRefsQueue().size());
- assertTrue(rqs.getReplicableHFiles(ID_ONE).isEmpty());
- assertEquals(3, rqs.getReplicableHFiles(ID_TWO).size());
-
- rp.getPeerStorage().removePeer(ID_TWO);
- rqs.removePeerFromHFileRefs(ID_TWO);
- assertEquals(0, rqs.getAllPeersFromHFileRefsQueue().size());
- assertTrue(rqs.getReplicableHFiles(ID_TWO).isEmpty());
- }
-
@Test
public void testReplicationPeers() throws Exception {
rp.init();
@@ -286,55 +118,7 @@ public abstract class TestReplicationStateBasic {
assertNumberOfPeers(2);
}
- private String getFileName(String base, int i) {
- return String.format(base + "-%04d", i);
- }
-
- @Test
- public void testPersistLogPositionAndSeqIdAtomically() throws Exception {
- ServerName serverName1 = ServerName.valueOf("127.0.0.1", 8000, 10000);
- assertTrue(rqs.getAllQueues(serverName1).isEmpty());
- String queue1 = "1";
- String region0 = "6b2c8f8555335cc9af74455b94516cbe",
- region1 = "6ecd2e9e010499f8ddef97ee8f70834f";
- for (int i = 0; i < 10; i++) {
- rqs.addWAL(serverName1, queue1, getFileName("file1", i));
- }
- List<String> queueIds = rqs.getAllQueues(serverName1);
- assertEquals(1, queueIds.size());
- assertThat(queueIds, hasItems("1"));
-
- List<String> wals1 = rqs.getWALsInQueue(serverName1, queue1);
- assertEquals(10, wals1.size());
- for (int i = 0; i < 10; i++) {
- assertThat(wals1, hasItems(getFileName("file1", i)));
- }
-
- for (int i = 0; i < 10; i++) {
- assertEquals(0, rqs.getWALPosition(serverName1, queue1, getFileName("file1", i)));
- }
- assertEquals(HConstants.NO_SEQNUM, rqs.getLastSequenceId(region0, queue1));
- assertEquals(HConstants.NO_SEQNUM, rqs.getLastSequenceId(region1, queue1));
-
- for (int i = 0; i < 10; i++) {
- rqs.setWALPosition(serverName1, queue1, getFileName("file1", i), (i + 1) * 100,
- ImmutableMap.of(region0, i * 100L, region1, (i + 1) * 100L));
- }
-
- for (int i = 0; i < 10; i++) {
- assertEquals((i + 1) * 100, rqs.getWALPosition(serverName1, queue1, getFileName("file1", i)));
- }
- assertEquals(900L, rqs.getLastSequenceId(region0, queue1));
- assertEquals(1000L, rqs.getLastSequenceId(region1, queue1));
-
- // Try to decrease the last pushed id by setWALPosition method.
- rqs.setWALPosition(serverName1, queue1, getFileName("file1", 0), 11 * 100,
- ImmutableMap.of(region0, 899L, region1, 1001L));
- assertEquals(900L, rqs.getLastSequenceId(region0, queue1));
- assertEquals(1001L, rqs.getLastSequenceId(region1, queue1));
- }
-
- protected void assertConnectedPeerStatus(boolean status, String peerId) throws Exception {
+ private void assertConnectedPeerStatus(boolean status, String peerId) throws Exception {
// we can first check if the value was changed in the store, if it wasn't then fail right away
if (status != rp.getPeerStorage().isPeerEnabled(peerId)) {
fail("ConnectedPeerStatus was " + !status + " but expected " + status + " in ZK");
@@ -353,30 +137,7 @@ public abstract class TestReplicationStateBasic {
}
}
- protected void assertNumberOfPeers(int total) throws ReplicationException {
+ private void assertNumberOfPeers(int total) throws ReplicationException {
assertEquals(total, rp.getPeerStorage().listPeerIds().size());
}
-
- /*
- * three replicators: rq1 has 0 queues, rq2 has 1 queue with no logs, rq3 has 5 queues with 1, 2,
- * 3, 4, 5 log files respectively
- */
- protected void populateQueues() throws ReplicationException {
- rqs.addWAL(server1, "trash", "trash");
- rqs.removeQueue(server1, "trash");
-
- rqs.addWAL(server2, "qId1", "trash");
- rqs.removeWAL(server2, "qId1", "trash");
-
- for (int i = 1; i < 6; i++) {
- for (int j = 0; j < i; j++) {
- rqs.addWAL(server3, "qId" + i, "filename" + j);
- }
- // Add peers for the corresponding queues so they are not orphans
- rp.getPeerStorage().addPeer("qId" + i,
- ReplicationPeerConfig.newBuilder()
- .setClusterKey(MiniZooKeeperCluster.HOST + ":2818:/bogus" + i).build(),
- true, SyncReplicationState.NONE);
- }
- }
}
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
index 311e7e337f9..1aa5ef8dc0c 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
@@ -79,7 +79,6 @@ public class TestReplicationStateZKImpl extends TestReplicationStateBasic {
@Before
public void setUp() {
zkTimeoutCount = 0;
- rqs = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
rp = ReplicationFactory.getReplicationPeers(zkw, conf);
OUR_KEY = ZKConfig.getZooKeeperClusterKey(conf);
}
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
deleted file mode 100644
index ccd3c17f3bc..00000000000
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
+++ /dev/null
@@ -1,341 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import static org.hamcrest.CoreMatchers.hasItems;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.SortedSet;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseZKTestingUtil;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.ReplicationTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.MD5Hash;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.zookeeper.KeeperException;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
-
-@Category({ ReplicationTests.class, MediumTests.class })
-public class TestZKReplicationQueueStorage {
-
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestZKReplicationQueueStorage.class);
-
- private static final HBaseZKTestingUtil UTIL = new HBaseZKTestingUtil();
-
- private static ZKReplicationQueueStorage STORAGE;
-
- @BeforeClass
- public static void setUp() throws Exception {
- UTIL.startMiniZKCluster();
- STORAGE = new ZKReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
- }
-
- @AfterClass
- public static void tearDown() throws IOException {
- UTIL.shutdownMiniZKCluster();
- }
-
- @After
- public void tearDownAfterTest() throws ReplicationException, KeeperException, IOException {
- for (ServerName serverName : STORAGE.getListOfReplicators()) {
- for (String queue : STORAGE.getAllQueues(serverName)) {
- STORAGE.removeQueue(serverName, queue);
- }
- STORAGE.removeReplicatorIfQueueIsEmpty(serverName);
- }
- for (String peerId : STORAGE.getAllPeersFromHFileRefsQueue()) {
- STORAGE.removePeerFromHFileRefs(peerId);
- }
- }
-
- private ServerName getServerName(int i) {
- return ServerName.valueOf("127.0.0.1", 8000 + i, 10000 + i);
- }
-
- @Test
- public void testReplicator() throws ReplicationException {
- assertTrue(STORAGE.getListOfReplicators().isEmpty());
- String queueId = "1";
- for (int i = 0; i < 10; i++) {
- STORAGE.addWAL(getServerName(i), queueId, "file" + i);
- }
- List<ServerName> replicators = STORAGE.getListOfReplicators();
- assertEquals(10, replicators.size());
- for (int i = 0; i < 10; i++) {
- assertThat(replicators, hasItems(getServerName(i)));
- }
- for (int i = 0; i < 5; i++) {
- STORAGE.removeQueue(getServerName(i), queueId);
- }
- for (int i = 0; i < 10; i++) {
- STORAGE.removeReplicatorIfQueueIsEmpty(getServerName(i));
- }
- replicators = STORAGE.getListOfReplicators();
- assertEquals(5, replicators.size());
- for (int i = 5; i < 10; i++) {
- assertThat(replicators, hasItems(getServerName(i)));
- }
- }
-
- private String getFileName(String base, int i) {
- return String.format(base + "-%04d", i);
- }
-
- @Test
- public void testAddRemoveLog() throws ReplicationException {
- ServerName serverName1 = ServerName.valueOf("127.0.0.1", 8000, 10000);
- assertTrue(STORAGE.getAllQueues(serverName1).isEmpty());
- String queue1 = "1";
- String queue2 = "2";
- for (int i = 0; i < 10; i++) {
- STORAGE.addWAL(serverName1, queue1, getFileName("file1", i));
- STORAGE.addWAL(serverName1, queue2, getFileName("file2", i));
- }
- List<String> queueIds = STORAGE.getAllQueues(serverName1);
- assertEquals(2, queueIds.size());
- assertThat(queueIds, hasItems("1", "2"));
-
- List<String> wals1 = STORAGE.getWALsInQueue(serverName1, queue1);
- List<String> wals2 = STORAGE.getWALsInQueue(serverName1, queue2);
- assertEquals(10, wals1.size());
- assertEquals(10, wals2.size());
- for (int i = 0; i < 10; i++) {
- assertThat(wals1, hasItems(getFileName("file1", i)));
- assertThat(wals2, hasItems(getFileName("file2", i)));
- }
-
- for (int i = 0; i < 10; i++) {
- assertEquals(0, STORAGE.getWALPosition(serverName1, queue1, getFileName("file1", i)));
- assertEquals(0, STORAGE.getWALPosition(serverName1, queue2, getFileName("file2", i)));
- STORAGE.setWALPosition(serverName1, queue1, getFileName("file1", i), (i + 1) * 100,
- Collections.emptyMap());
- STORAGE.setWALPosition(serverName1, queue2, getFileName("file2", i), (i + 1) * 100 + 10,
- Collections.emptyMap());
- }
-
- for (int i = 0; i < 10; i++) {
- assertEquals((i + 1) * 100,
- STORAGE.getWALPosition(serverName1, queue1, getFileName("file1", i)));
- assertEquals((i + 1) * 100 + 10,
- STORAGE.getWALPosition(serverName1, queue2, getFileName("file2", i)));
- }
-
- for (int i = 0; i < 10; i++) {
- if (i % 2 == 0) {
- STORAGE.removeWAL(serverName1, queue1, getFileName("file1", i));
- } else {
- STORAGE.removeWAL(serverName1, queue2, getFileName("file2", i));
- }
- }
-
- queueIds = STORAGE.getAllQueues(serverName1);
- assertEquals(2, queueIds.size());
- assertThat(queueIds, hasItems("1", "2"));
-
- ServerName serverName2 = ServerName.valueOf("127.0.0.1", 8001, 10001);
- Pair<String, SortedSet<String>> peer1 = STORAGE.claimQueue(serverName1, "1", serverName2);
-
- assertEquals("1-" + serverName1.getServerName(), peer1.getFirst());
- assertEquals(5, peer1.getSecond().size());
- int i = 1;
- for (String wal : peer1.getSecond()) {
- assertEquals(getFileName("file1", i), wal);
- assertEquals((i + 1) * 100,
- STORAGE.getWALPosition(serverName2, peer1.getFirst(), getFileName("file1", i)));
- i += 2;
- }
-
- queueIds = STORAGE.getAllQueues(serverName1);
- assertEquals(1, queueIds.size());
- assertThat(queueIds, hasItems("2"));
- wals2 = STORAGE.getWALsInQueue(serverName1, queue2);
- assertEquals(5, wals2.size());
- for (i = 0; i < 10; i += 2) {
- assertThat(wals2, hasItems(getFileName("file2", i)));
- }
-
- queueIds = STORAGE.getAllQueues(serverName2);
- assertEquals(1, queueIds.size());
- assertThat(queueIds, hasItems(peer1.getFirst()));
- wals1 = STORAGE.getWALsInQueue(serverName2, peer1.getFirst());
- assertEquals(5, wals1.size());
- for (i = 1; i < 10; i += 2) {
- assertThat(wals1, hasItems(getFileName("file1", i)));
- }
-
- Set<String> allWals = STORAGE.getAllWALs();
- assertEquals(10, allWals.size());
- for (i = 0; i < 10; i++) {
- assertThat(allWals, hasItems(i % 2 == 0 ? getFileName("file2", i) : getFileName("file1", i)));
- }
- }
-
- // For HBASE-12865, HBASE-26482
- @Test
- public void testClaimQueueChangeCversion() throws ReplicationException, KeeperException {
- ServerName serverName1 = ServerName.valueOf("127.0.0.1", 8000, 10000);
- STORAGE.addWAL(serverName1, "1", "file");
- STORAGE.addWAL(serverName1, "2", "file");
-
- ServerName serverName2 = ServerName.valueOf("127.0.0.1", 8001, 10001);
- // Avoid claimQueue update cversion for prepare server2 rsNode.
- STORAGE.addWAL(serverName2, "1", "file");
- STORAGE.addWAL(serverName2, "2", "file");
-
- int v0 = STORAGE.getQueuesZNodeCversion();
-
- STORAGE.claimQueue(serverName1, "1", serverName2);
- int v1 = STORAGE.getQueuesZNodeCversion();
- // cversion should be increased by claimQueue method.
- assertTrue(v1 > v0);
-
- STORAGE.claimQueue(serverName1, "2", serverName2);
- int v2 = STORAGE.getQueuesZNodeCversion();
- // cversion should be increased by claimQueue method.
- assertTrue(v2 > v1);
- }
-
- private ZKReplicationQueueStorage createWithUnstableVersion() throws IOException {
- return new ZKReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration()) {
-
- private int called = 0;
- private int getLastSeqIdOpIndex = 0;
-
- @Override
- protected int getQueuesZNodeCversion() throws KeeperException {
- if (called < 4) {
- called++;
- }
- return called;
- }
-
- @Override
- protected Pair<Long, Integer> getLastSequenceIdWithVersion(String encodedRegionName,
- String peerId) throws KeeperException {
- Pair<Long, Integer> oldPair = super.getLastSequenceIdWithVersion(encodedRegionName, peerId);
- if (getLastSeqIdOpIndex < 100) {
- // Let the ZNode version increase.
- String path = getSerialReplicationRegionPeerNode(encodedRegionName, peerId);
- ZKUtil.createWithParents(zookeeper, path);
- ZKUtil.setData(zookeeper, path, ZKUtil.positionToByteArray(100L));
- }
- getLastSeqIdOpIndex++;
- return oldPair;
- }
- };
- }
-
- @Test
- public void testGetAllWALsCversionChange() throws IOException, ReplicationException {
- ZKReplicationQueueStorage storage = createWithUnstableVersion();
- storage.addWAL(getServerName(0), "1", "file");
- // This should return eventually when cversion stabilizes
- Set<String> allWals = storage.getAllWALs();
- assertEquals(1, allWals.size());
- assertThat(allWals, hasItems("file"));
- }
-
- // For HBASE-14621
- @Test
- public void testGetAllHFileRefsCversionChange() throws IOException, ReplicationException {
- ZKReplicationQueueStorage storage = createWithUnstableVersion();
- storage.addPeerToHFileRefs("1");
- Path p = new Path("/test");
- storage.addHFileRefs("1", Arrays.asList(Pair.newPair(p, p)));
- // This should return eventually when cversion stabilizes
- Set<String> allHFileRefs = storage.getAllHFileRefs();
- assertEquals(1, allHFileRefs.size());
- assertThat(allHFileRefs, hasItems("test"));
- }
-
- // For HBASE-20138
- @Test
- public void testSetWALPositionBadVersion() throws IOException, ReplicationException {
- ZKReplicationQueueStorage storage = createWithUnstableVersion();
- ServerName serverName1 = ServerName.valueOf("128.0.0.1", 8000, 10000);
- assertTrue(storage.getAllQueues(serverName1).isEmpty());
- String queue1 = "1";
- String fileName = getFileName("file1", 0);
- String encodedRegionName = "31d9792f4435b99d9fb1016f6fbc8dc6";
- storage.addWAL(serverName1, queue1, fileName);
-
- List<String> wals1 = storage.getWALsInQueue(serverName1, queue1);
- assertEquals(1, wals1.size());
-
- assertEquals(0, storage.getWALPosition(serverName1, queue1, fileName));
- // This should return eventually when data version stabilizes
- storage.setWALPosition(serverName1, queue1, fileName, 100,
- ImmutableMap.of(encodedRegionName, 120L));
-
- assertEquals(100, storage.getWALPosition(serverName1, queue1, fileName));
- assertEquals(120L, storage.getLastSequenceId(encodedRegionName, queue1));
- }
-
- @Test
- public void testRegionsZNodeLayout() throws Exception {
- String peerId = "1";
- String encodedRegionName = "31d9792f4435b99d9fb1016f6fbc8dc7";
- String expectedPath = "/hbase/replication/regions/31/d9/792f4435b99d9fb1016f6fbc8dc7-" + peerId;
- String path = STORAGE.getSerialReplicationRegionPeerNode(encodedRegionName, peerId);
- assertEquals(expectedPath, path);
- }
-
- @Test
- public void testRemoveAllLastPushedSeqIdsForPeer() throws Exception {
- String peerId = "1";
- String peerIdToDelete = "2";
- for (int i = 0; i < 100; i++) {
- String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
- STORAGE.setLastSequenceIds(peerId, ImmutableMap.of(encodedRegionName, (long) i));
- STORAGE.setLastSequenceIds(peerIdToDelete, ImmutableMap.of(encodedRegionName, (long) i));
- }
- for (int i = 0; i < 100; i++) {
- String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
- assertEquals(i, STORAGE.getLastSequenceId(encodedRegionName, peerId));
- assertEquals(i, STORAGE.getLastSequenceId(encodedRegionName, peerIdToDelete));
- }
- STORAGE.removeLastSequenceIds(peerIdToDelete);
- for (int i = 0; i < 100; i++) {
- String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
- assertEquals(i, STORAGE.getLastSequenceId(encodedRegionName, peerId));
- assertEquals(HConstants.NO_SEQNUM,
- STORAGE.getLastSequenceId(encodedRegionName, peerIdToDelete));
- }
- }
-}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 58a5d7cae47..ce3e81ad04b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -783,7 +783,7 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
}
this.rsGroupInfoManager = RSGroupInfoManager.create(this);
- this.replicationPeerManager = ReplicationPeerManager.create(zooKeeper, conf, clusterId);
+ this.replicationPeerManager = ReplicationPeerManager.create(this, clusterId);
this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this, this.serverManager);
this.drainingServerTracker.start();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
index 0bd0f3ba0c7..487c45e5c5c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.master.SplitWALManager;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
-import org.apache.hadoop.hbase.master.replication.ClaimReplicationQueuesProcedure;
+import org.apache.hadoop.hbase.master.replication.AssignReplicationQueuesProcedure;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.procedure2.Procedure;
@@ -240,15 +240,33 @@ public class ServerCrashProcedure extends
}
assignRegions(env, regionsOnCrashedServer);
}
- setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
+ // If there is no replication peer, we do not need to enter the claim queues stage.
+ // This is also very important that now we will later initialize ReplicationQueueStorage
+ // so if there is no replication peer added yet, the storage can not be accessed.
+ // And there will be no race because:
+ // 1. For adding replication peer, if the peer storage has not been updated yet, the crash
+ // region server will not have any replication queues for this peer, so it is safe to skip
+ // claiming.
+ // 2. For removing replication peer, it it has already updated the peer storage, then
+ // there is no way to rollback and region servers are already started to close and delete
+ // replication queues, so it is also safe to skip claiming.
+ if (env.getReplicationPeerManager().listPeers(null).isEmpty()) {
+ setNextState(ServerCrashState.SERVER_CRASH_FINISH);
+ } else {
+ setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
+ }
break;
case SERVER_CRASH_HANDLE_RIT2:
// Noop. Left in place because we used to call handleRIT here for a second time
// but no longer necessary since HBASE-20634.
- setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
+ if (env.getReplicationPeerManager().listPeers(null).isEmpty()) {
+ setNextState(ServerCrashState.SERVER_CRASH_FINISH);
+ } else {
+ setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
+ }
break;
case SERVER_CRASH_CLAIM_REPLICATION_QUEUES:
- addChildProcedure(new ClaimReplicationQueuesProcedure(serverName));
+ addChildProcedure(new AssignReplicationQueuesProcedure(serverName));
setNextState(ServerCrashState.SERVER_CRASH_FINISH);
break;
case SERVER_CRASH_FINISH:
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
new file mode 100644
index 00000000000..e7fb5e51715
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.util.RetryCounter;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.AssignReplicationQueuesState;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.AssignReplicationQueuesStateData;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+
+@InterfaceAudience.Private
+public class AssignReplicationQueuesProcedure
+ extends StateMachineProcedure<MasterProcedureEnv, AssignReplicationQueuesState>
+ implements ServerProcedureInterface {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AssignReplicationQueuesProcedure.class);
+
+ private ServerName crashedServer;
+
+ private RetryCounter retryCounter;
+
+ public AssignReplicationQueuesProcedure() {
+ }
+
+ public AssignReplicationQueuesProcedure(ServerName crashedServer) {
+ this.crashedServer = crashedServer;
+ }
+
+ @Override
+ public ServerName getServerName() {
+ return crashedServer;
+ }
+
+ @Override
+ public boolean hasMetaTableRegion() {
+ return false;
+ }
+
+ @Override
+ public ServerOperationType getServerOperationType() {
+ return ServerOperationType.CLAIM_REPLICATION_QUEUES;
+ }
+
+ private void addMissingQueues(MasterProcedureEnv env) throws ReplicationException {
+ ReplicationQueueStorage storage = env.getReplicationPeerManager().getQueueStorage();
+
+ Set<String> existingQueuePeerIds = new HashSet<>();
+ List<ReplicationQueueId> queueIds = storage.listAllQueueIds(crashedServer);
+ for (Iterator<ReplicationQueueId> iter = queueIds.iterator(); iter.hasNext();) {
+ ReplicationQueueId queueId = iter.next();
+ if (!queueId.isRecovered()) {
+ existingQueuePeerIds.add(queueId.getPeerId());
+ }
+ }
+ List<ReplicationPeerDescription> peers = env.getReplicationPeerManager().listPeers(null);
+ for (ReplicationPeerDescription peer : peers) {
+ if (!existingQueuePeerIds.contains(peer.getPeerId())) {
+ ReplicationQueueId queueId = new ReplicationQueueId(crashedServer, peer.getPeerId());
+ LOG.debug("Add replication queue {} for claiming", queueId);
+ env.getReplicationPeerManager().getQueueStorage().setOffset(queueId,
+ crashedServer.toString(), ReplicationGroupOffset.BEGIN, Collections.emptyMap());
+ }
+ }
+ }
+
+ private Flow claimQueues(MasterProcedureEnv env) throws ReplicationException {
+ ReplicationQueueStorage storage = env.getReplicationPeerManager().getQueueStorage();
+ List<ReplicationQueueId> queueIds = storage.listAllQueueIds(crashedServer);
+ if (queueIds.isEmpty()) {
+ LOG.debug("Finish claiming replication queues for {}", crashedServer);
+ // we are done
+ return Flow.NO_MORE_STATE;
+ }
+ LOG.debug("There are {} replication queues need to be claimed for {}", queueIds.size(),
+ crashedServer);
+ List<ServerName> targetServers =
+ env.getMasterServices().getServerManager().getOnlineServersList();
+ if (targetServers.isEmpty()) {
+ throw new ReplicationException("no region server available");
+ }
+ Collections.shuffle(targetServers);
+ for (int i = 0, n = Math.min(queueIds.size(), targetServers.size()); i < n; i++) {
+ addChildProcedure(
+ new ClaimReplicationQueueRemoteProcedure(queueIds.get(i), targetServers.get(i)));
+ }
+ retryCounter = null;
+ return Flow.HAS_MORE_STATE;
+ }
+
+ @Override
+ protected Flow executeFromState(MasterProcedureEnv env, AssignReplicationQueuesState state)
+ throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+ try {
+ switch (state) {
+ case ASSIGN_REPLICATION_QUEUES_PRE_CHECK:
+ // TODO: reserved for implementing the fencing logic with Add/Remove/UpdatePeerProcedure
+ setNextState(AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES);
+ return Flow.HAS_MORE_STATE;
+ case ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES:
+ addMissingQueues(env);
+ retryCounter = null;
+ setNextState(AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_CLAIM);
+ return Flow.HAS_MORE_STATE;
+ case ASSIGN_REPLICATION_QUEUES_CLAIM:
+ return claimQueues(env);
+ default:
+ throw new UnsupportedOperationException("unhandled state=" + state);
+ }
+ } catch (Exception e) {
+ if (retryCounter == null) {
+ retryCounter = ProcedureUtil.createRetryCounter(env.getMasterConfiguration());
+ }
+ long backoff = retryCounter.getBackoffTimeAndIncrementAttempts();
+ LOG.warn("Failed to claim replication queues for {}, suspend {}secs {}; {};", crashedServer,
+ backoff / 1000, e);
+ setTimeout(Math.toIntExact(backoff));
+ setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
+ skipPersistence();
+ throw new ProcedureSuspendedException();
+ }
+ }
+
+ @Override
+ protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
+ setState(ProcedureProtos.ProcedureState.RUNNABLE);
+ env.getProcedureScheduler().addFront(this);
+ return false;
+ }
+
+ @Override
+ protected void rollbackState(MasterProcedureEnv env, AssignReplicationQueuesState state)
+ throws IOException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected AssignReplicationQueuesState getState(int stateId) {
+ return AssignReplicationQueuesState.forNumber(stateId);
+ }
+
+ @Override
+ protected int getStateId(AssignReplicationQueuesState state) {
+ return state.getNumber();
+ }
+
+ @Override
+ protected AssignReplicationQueuesState getInitialState() {
+ return AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_PRE_CHECK;
+ }
+
+ @Override
+ protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.serializeStateData(serializer);
+ serializer.serialize(AssignReplicationQueuesStateData.newBuilder()
+ .setCrashedServer(ProtobufUtil.toServerName(crashedServer)).build());
+ }
+
+ @Override
+ protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.deserializeStateData(serializer);
+ AssignReplicationQueuesStateData proto =
+ serializer.deserialize(AssignReplicationQueuesStateData.class);
+ crashedServer = ProtobufUtil.toServerName(proto.getCrashedServer());
+ }
+
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
index 9ef97d1fff6..7b637384398 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.master.procedure.ServerRemoteProcedure;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteOperation;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.regionserver.ClaimReplicationQueueCallable;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -43,34 +44,33 @@ public class ClaimReplicationQueueRemoteProcedure extends ServerRemoteProcedure
private static final Logger LOG =
LoggerFactory.getLogger(ClaimReplicationQueueRemoteProcedure.class);
- private ServerName crashedServer;
-
- private String queue;
+ private ReplicationQueueId queueId;
public ClaimReplicationQueueRemoteProcedure() {
}
- public ClaimReplicationQueueRemoteProcedure(ServerName crashedServer, String queue,
- ServerName targetServer) {
- this.crashedServer = crashedServer;
- this.queue = queue;
+ public ClaimReplicationQueueRemoteProcedure(ReplicationQueueId queueId, ServerName targetServer) {
+ this.queueId = queueId;
this.targetServer = targetServer;
}
@Override
public Optional<RemoteOperation> remoteCallBuild(MasterProcedureEnv env, ServerName remote) {
assert targetServer.equals(remote);
+ ClaimReplicationQueueRemoteParameter.Builder builder = ClaimReplicationQueueRemoteParameter
+ .newBuilder().setCrashedServer(ProtobufUtil.toServerName(queueId.getServerName()))
+ .setQueue(queueId.getPeerId());
+ queueId.getSourceServerName()
+ .ifPresent(sourceServer -> builder.setSourceServer(ProtobufUtil.toServerName(sourceServer)));
return Optional.of(new ServerOperation(this, getProcId(), ClaimReplicationQueueCallable.class,
- ClaimReplicationQueueRemoteParameter.newBuilder()
- .setCrashedServer(ProtobufUtil.toServerName(crashedServer)).setQueue(queue).build()
- .toByteArray()));
+ builder.build().toByteArray()));
}
@Override
public ServerName getServerName() {
// return crashed server here, as we are going to recover its replication queues so we should
// use its scheduler queue instead of the one for the target server.
- return crashedServer;
+ return queueId.getServerName();
}
@Override
@@ -86,8 +86,7 @@ public class ClaimReplicationQueueRemoteProcedure extends ServerRemoteProcedure
@Override
protected void complete(MasterProcedureEnv env, Throwable error) {
if (error != null) {
- LOG.warn("Failed to claim replication queue {} of crashed server on server {} ", queue,
- crashedServer, targetServer, error);
+ LOG.warn("Failed to claim replication queue {} on server {} ", queueId, targetServer, error);
this.succ = false;
} else {
this.succ = true;
@@ -111,17 +110,26 @@ public class ClaimReplicationQueueRemoteProcedure extends ServerRemoteProcedure
@Override
protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
- serializer.serialize(ClaimReplicationQueueRemoteStateData.newBuilder()
- .setCrashedServer(ProtobufUtil.toServerName(crashedServer)).setQueue(queue)
- .setTargetServer(ProtobufUtil.toServerName(targetServer)).build());
+ ClaimReplicationQueueRemoteStateData.Builder builder = ClaimReplicationQueueRemoteStateData
+ .newBuilder().setCrashedServer(ProtobufUtil.toServerName(queueId.getServerName()))
+ .setQueue(queueId.getPeerId()).setTargetServer(ProtobufUtil.toServerName(targetServer));
+ queueId.getSourceServerName()
+ .ifPresent(sourceServer -> builder.setSourceServer(ProtobufUtil.toServerName(sourceServer)));
+ serializer.serialize(builder.build());
}
@Override
protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
ClaimReplicationQueueRemoteStateData data =
serializer.deserialize(ClaimReplicationQueueRemoteStateData.class);
- crashedServer = ProtobufUtil.toServerName(data.getCrashedServer());
- queue = data.getQueue();
targetServer = ProtobufUtil.toServerName(data.getTargetServer());
+ ServerName crashedServer = ProtobufUtil.toServerName(data.getCrashedServer());
+ String queue = data.getQueue();
+ if (data.hasSourceServer()) {
+ queueId = new ReplicationQueueId(crashedServer, queue,
+ ProtobufUtil.toServerName(data.getSourceServer()));
+ } else {
+ queueId = new ReplicationQueueId(crashedServer, queue);
+ }
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueuesProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueuesProcedure.java
index 5eb6608f4ee..747d352d2aa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueuesProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueuesProcedure.java
@@ -19,8 +19,10 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface;
@@ -30,7 +32,9 @@ import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.util.RetryCounter;
import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
@@ -44,7 +48,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
/**
* Used to assign the replication queues of a dead server to other region servers.
+ * @deprecated Use {@link AssignReplicationQueuesProcedure} instead, kept only for keeping
+ * compatibility.
*/
+@Deprecated
@InterfaceAudience.Private
public class ClaimReplicationQueuesProcedure extends Procedure<MasterProcedureEnv>
implements ServerProcedureInterface {
@@ -82,22 +89,36 @@ public class ClaimReplicationQueuesProcedure extends Procedure<MasterProcedureEn
throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
ReplicationQueueStorage storage = env.getReplicationPeerManager().getQueueStorage();
try {
- List<String> queues = storage.getAllQueues(crashedServer);
+ List<ReplicationQueueId> queues = storage.listAllQueueIds(crashedServer);
+ Set<String> existQueuePeerIds = new HashSet<>();
// this is for upgrading to the new region replication framework, where we will delete the
- // legacy region_replica_replication peer directly, without deleting the replication queues,
- // as it may still be used by region servers which have not been upgraded yet.
- for (Iterator<String> iter = queues.iterator(); iter.hasNext();) {
- ReplicationQueueInfo queue = new ReplicationQueueInfo(iter.next());
- if (queue.getPeerId().equals(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER)) {
+ // legacy region_replica_replication peer directly, without deleting the replication queues
+ for (Iterator<ReplicationQueueId> iter = queues.iterator(); iter.hasNext();) {
+ ReplicationQueueId queueId = iter.next();
+ if (queueId.getPeerId().equals(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER)) {
LOG.info("Found replication queue {} for legacy region replication peer, "
- + "skipping claiming and removing...", queue.getQueueId());
+ + "skipping claiming and removing...", queueId);
iter.remove();
- storage.removeQueue(crashedServer, queue.getQueueId());
+ storage.removeQueue(queueId);
+ } else if (!queueId.isRecovered()) {
+ existQueuePeerIds.add(queueId.getPeerId());
+ }
+ }
+ List<ReplicationPeerDescription> peers = env.getReplicationPeerManager().listPeers(null);
+ // TODO: the implementation is not enough yet, if there are retries, we need to know whether
+ // the replication queue for the given peer has been claimed or not, otherwise this logic will
+ // introduce redundant replication queues for the same peer. Add this logic to make some UTs
+ // pass first.
+ for (ReplicationPeerDescription peer : peers) {
+ if (!existQueuePeerIds.contains(peer.getPeerId())) {
+ ReplicationQueueId queueId = new ReplicationQueueId(crashedServer, peer.getPeerId());
+ env.getReplicationPeerManager().getQueueStorage().setOffset(queueId,
+ crashedServer.toString(), ReplicationGroupOffset.BEGIN, Collections.emptyMap());
+ queues.add(queueId);
}
}
if (queues.isEmpty()) {
LOG.debug("Finish claiming replication queues for {}", crashedServer);
- storage.removeReplicatorIfQueueIsEmpty(crashedServer);
// we are done
return null;
}
@@ -112,8 +133,7 @@ public class ClaimReplicationQueuesProcedure extends Procedure<MasterProcedureEn
ClaimReplicationQueueRemoteProcedure[] procs =
new ClaimReplicationQueueRemoteProcedure[Math.min(queues.size(), targetServers.size())];
for (int i = 0; i < procs.length; i++) {
- procs[i] = new ClaimReplicationQueueRemoteProcedure(crashedServer, queues.get(i),
- targetServers.get(i));
+ procs[i] = new ClaimReplicationQueueRemoteProcedure(queues.get(i), targetServers.get(i));
}
return procs;
} catch (ReplicationException e) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 06cf559d492..59fd5414236 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.master.replication;
+import com.google.errorprone.annotations.RestrictedApi;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -29,6 +30,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
@@ -37,9 +39,12 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.replication.BaseReplicationEndpoint;
import org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
@@ -48,11 +53,12 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfigBuilder;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -95,26 +101,33 @@ public class ReplicationPeerManager {
private final Configuration conf;
+ @FunctionalInterface
+ private interface ReplicationQueueStorageInitializer {
+
+ void initialize() throws IOException;
+ }
+
+ private final ReplicationQueueStorageInitializer queueStorageInitializer;
+
+ // we will mock this class in UT so leave the constructor as package private and not mark the
+ // class as final, since mockito can not mock a final class
ReplicationPeerManager(ReplicationPeerStorage peerStorage, ReplicationQueueStorage queueStorage,
- ConcurrentMap<String, ReplicationPeerDescription> peers, Configuration conf, String clusterId) {
+ ConcurrentMap<String, ReplicationPeerDescription> peers, Configuration conf, String clusterId,
+ ReplicationQueueStorageInitializer queueStorageInitializer) {
this.peerStorage = peerStorage;
this.queueStorage = queueStorage;
this.peers = peers;
this.conf = conf;
this.clusterId = clusterId;
+ this.queueStorageInitializer = queueStorageInitializer;
}
private void checkQueuesDeleted(String peerId)
throws ReplicationException, DoNotRetryIOException {
- for (ServerName replicator : queueStorage.getListOfReplicators()) {
- List<String> queueIds = queueStorage.getAllQueues(replicator);
- for (String queueId : queueIds) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (queueInfo.getPeerId().equals(peerId)) {
- throw new DoNotRetryIOException("undeleted queue for peerId: " + peerId + ", replicator: "
- + replicator + ", queueId: " + queueId);
- }
- }
+ List<ReplicationQueueId> queueIds = queueStorage.listAllQueueIds(peerId);
+ if (!queueIds.isEmpty()) {
+ throw new DoNotRetryIOException("There are still " + queueIds.size()
+ + " undeleted queue(s) for peerId: " + peerId + ", first is " + queueIds.get(0));
}
if (queueStorage.getAllPeersFromHFileRefsQueue().contains(peerId)) {
throw new DoNotRetryIOException("Undeleted queue for peer " + peerId + " in hfile-refs");
@@ -122,7 +135,7 @@ public class ReplicationPeerManager {
}
void preAddPeer(String peerId, ReplicationPeerConfig peerConfig)
- throws DoNotRetryIOException, ReplicationException {
+ throws ReplicationException, IOException {
if (peerId.contains("-")) {
throw new DoNotRetryIOException("Found invalid peer name: " + peerId);
}
@@ -133,6 +146,9 @@ public class ReplicationPeerManager {
if (peers.containsKey(peerId)) {
throw new DoNotRetryIOException("Replication peer " + peerId + " already exists");
}
+
+ // lazy create table
+ queueStorageInitializer.initialize();
// make sure that there is no queues with the same peer id. This may happen when we create a
// peer with the same id with a old deleted peer. If the replication queues for the old peer
// have not been cleaned up yet then we should not create the new peer, otherwise the old wal
@@ -352,8 +368,8 @@ public class ReplicationPeerManager {
// claimed once after the refresh peer procedure done(as the next claim queue will just delete
// it), so we can make sure that a two pass scan will finally find the queue and remove it,
// unless it has already been removed by others.
- ReplicationUtils.removeAllQueues(queueStorage, peerId);
- ReplicationUtils.removeAllQueues(queueStorage, peerId);
+ queueStorage.removeAllQueues(peerId);
+ queueStorage.removeAllQueues(peerId);
}
public void removeAllQueuesAndHFileRefs(String peerId) throws ReplicationException {
@@ -555,14 +571,68 @@ public class ReplicationPeerManager {
.collect(Collectors.toList());
}
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ public ReplicationPeerStorage getPeerStorage() {
+ return peerStorage;
+ }
+
public ReplicationQueueStorage getQueueStorage() {
return queueStorage;
}
- public static ReplicationPeerManager create(ZKWatcher zk, Configuration conf, String clusterId)
- throws ReplicationException {
+ private static Pair<ReplicationQueueStorage, ReplicationQueueStorageInitializer>
+ createReplicationQueueStorage(MasterServices services) throws IOException {
+ Configuration conf = services.getConfiguration();
+ TableName replicationQueueTableName =
+ TableName.valueOf(conf.get(ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME,
+ ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME_DEFAULT.getNameAsString()));
+ ReplicationQueueStorageInitializer initializer;
+ if (services.getTableDescriptors().exists(replicationQueueTableName)) {
+ // no need to create the table
+ initializer = () -> {
+ };
+ } else {
+ // lazy create the replication table.
+ initializer = new ReplicationQueueStorageInitializer() {
+
+ private volatile boolean created = false;
+
+ @Override
+ public void initialize() throws IOException {
+ if (created) {
+ return;
+ }
+ synchronized (this) {
+ if (created) {
+ return;
+ }
+ if (services.getTableDescriptors().exists(replicationQueueTableName)) {
+ created = true;
+ return;
+ }
+ long procId = services.createSystemTable(ReplicationStorageFactory
+ .createReplicationQueueTableDescriptor(replicationQueueTableName));
+ ProcedureExecutor<MasterProcedureEnv> procExec = services.getMasterProcedureExecutor();
+ ProcedureSyncWait.waitFor(procExec.getEnvironment(), TimeUnit.MINUTES.toMillis(1),
+ "Creating table " + replicationQueueTableName, () -> procExec.isFinished(procId));
+ }
+ }
+ };
+ }
+ return Pair.newPair(ReplicationStorageFactory.getReplicationQueueStorage(
+ services.getConnection(), replicationQueueTableName), initializer);
+ }
+
+ public static ReplicationPeerManager create(MasterServices services, String clusterId)
+ throws ReplicationException, IOException {
+ Configuration conf = services.getConfiguration();
+ ZKWatcher zk = services.getZooKeeper();
ReplicationPeerStorage peerStorage =
ReplicationStorageFactory.getReplicationPeerStorage(zk, conf);
+ Pair<ReplicationQueueStorage, ReplicationQueueStorageInitializer> pair =
+ createReplicationQueueStorage(services);
+ ReplicationQueueStorage queueStorage = pair.getFirst();
ConcurrentMap<String, ReplicationPeerDescription> peers = new ConcurrentHashMap<>();
for (String peerId : peerStorage.listPeerIds()) {
ReplicationPeerConfig peerConfig = peerStorage.getPeerConfig(peerId);
@@ -572,7 +642,24 @@ public class ReplicationPeerManager {
) {
// we do not use this endpoint for region replication any more, see HBASE-26233
LOG.info("Legacy region replication peer found, removing: {}", peerConfig);
- peerStorage.removePeer(peerId);
+ // do it asynchronous to not block the start up of HMaster
+ new Thread("Remove legacy replication peer " + peerId) {
+
+ @Override
+ public void run() {
+ try {
+ // need to delete two times to make sure we delete all the queues, see the comments in
+ // above
+ // removeAllQueues method for more details.
+ queueStorage.removeAllQueues(peerId);
+ queueStorage.removeAllQueues(peerId);
+ // delete queue first and then peer, because we use peer as a flag.
+ peerStorage.removePeer(peerId);
+ } catch (Exception e) {
+ LOG.warn("Failed to delete legacy replication peer {}", peerId);
+ }
+ }
+ }.start();
continue;
}
peerConfig = ReplicationPeerConfigUtil.updateReplicationBasePeerConfigs(conf, peerConfig);
@@ -581,8 +668,9 @@ public class ReplicationPeerManager {
SyncReplicationState state = peerStorage.getPeerSyncReplicationState(peerId);
peers.put(peerId, new ReplicationPeerDescription(peerId, enabled, peerConfig, state));
}
- return new ReplicationPeerManager(peerStorage,
- ReplicationStorageFactory.getReplicationQueueStorage(zk, conf), peers, conf, clusterId);
+
+ return new ReplicationPeerManager(peerStorage, queueStorage, peers, conf, clusterId,
+ pair.getSecond());
}
/**
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index 6a7d30b9560..88662385131 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -32,6 +32,7 @@ import java.io.IOException;
import java.io.InterruptedIOException;
import java.lang.management.MemoryType;
import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -431,8 +432,9 @@ public abstract class AbstractFSWAL<W extends WriterBase> implements WAL {
}
// If prefix is null||empty then just name it wal
- this.walFilePrefix =
- prefix == null || prefix.isEmpty() ? "wal" : URLEncoder.encode(prefix, "UTF8");
+ this.walFilePrefix = prefix == null || prefix.isEmpty()
+ ? "wal"
+ : URLEncoder.encode(prefix, StandardCharsets.UTF_8.name());
// we only correctly differentiate suffices when numeric ones start with '.'
if (suffix != null && !(suffix.isEmpty()) && !(suffix.startsWith(WAL_FILE_NAME_DELIMITER))) {
throw new IllegalArgumentException("WAL suffix must start with '" + WAL_FILE_NAME_DELIMITER
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java
index 819e4c5e54a..00e875f8be5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java
@@ -19,23 +19,28 @@ package org.apache.hadoop.hbase.replication.master;
import java.io.IOException;
import java.util.Collections;
+import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.base.Predicate;
import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hbase.thirdparty.org.apache.commons.collections4.MapUtils;
/**
* Implementation of a file cleaner that checks if a hfile is still scheduled for replication before
@@ -44,15 +49,20 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class ReplicationHFileCleaner extends BaseHFileCleanerDelegate {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationHFileCleaner.class);
- private ZKWatcher zkw;
+ private Connection conn;
+ private boolean shareConn;
private ReplicationQueueStorage rqs;
private boolean stopped = false;
@Override
public Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
- // all members of this class are null if replication is disabled,
- // so we cannot filter the files
- if (this.getConf() == null) {
+ if (
+ !(getConf().getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
+ HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT))
+ ) {
+ LOG.warn(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY + " is not enabled. Better to remove "
+ + ReplicationHFileCleaner.class + " from " + HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS
+ + " configuration.");
return files;
}
@@ -88,51 +98,34 @@ public class ReplicationHFileCleaner extends BaseHFileCleanerDelegate {
}
@Override
- public void setConf(Configuration config) {
- // If either replication or replication of bulk load hfiles is disabled, keep all members null
- if (
- !(config.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
- HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT))
- ) {
- LOG.warn(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY + " is not enabled. Better to remove "
- + ReplicationHFileCleaner.class + " from " + HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS
- + " configuration.");
- return;
- }
- // Make my own Configuration. Then I'll have my own connection to zk that
- // I can close myself when time comes.
- Configuration conf = new Configuration(config);
+ public void init(Map<String, Object> params) {
+ super.init(params);
try {
- setConf(conf, new ZKWatcher(conf, "replicationHFileCleaner", null));
+ if (MapUtils.isNotEmpty(params)) {
+ Object master = params.get(HMaster.MASTER);
+ if (master != null && master instanceof Server) {
+ conn = ((Server) master).getConnection();
+ shareConn = true;
+ }
+ }
+ if (conn == null) {
+ conn = ConnectionFactory.createConnection(getConf());
+ }
+ this.rqs = ReplicationStorageFactory.getReplicationQueueStorage(conn, getConf());
} catch (IOException e) {
LOG.error("Error while configuring " + this.getClass().getName(), e);
}
}
- @InterfaceAudience.Private
- public void setConf(Configuration conf, ZKWatcher zk) {
- super.setConf(conf);
- try {
- initReplicationQueueStorage(conf, zk);
- } catch (Exception e) {
- LOG.error("Error while configuring " + this.getClass().getName(), e);
- }
- }
-
- private void initReplicationQueueStorage(Configuration conf, ZKWatcher zk) {
- this.zkw = zk;
- this.rqs = ReplicationStorageFactory.getReplicationQueueStorage(zk, conf);
- }
-
@Override
public void stop(String why) {
if (this.stopped) {
return;
}
this.stopped = true;
- if (this.zkw != null) {
- LOG.info("Stopping " + this.zkw);
- this.zkw.close();
+ if (!shareConn && this.conn != null) {
+ LOG.info("Stopping " + this.conn);
+ IOUtils.closeQuietly(conn);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index 54e600e09ad..7135ca9a9b2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -26,9 +26,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
-import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -56,14 +54,15 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
@Override
public void preClean() {
readZKTimestamp = EnvironmentEdgeManager.currentTime();
- try {
- // The concurrently created new WALs may not be included in the return list,
- // but they won't be deleted because they're not in the checking set.
- wals = queueStorage.getAllWALs();
- } catch (ReplicationException e) {
- LOG.warn("Failed to read zookeeper, skipping checking deletable files");
- wals = null;
- }
+ // TODO: revisit the implementation
+ // try {
+ // // The concurrently created new WALs may not be included in the return list,
+ // // but they won't be deleted because they're not in the checking set.
+ // wals = queueStorage.getAllWALs();
+ // } catch (ReplicationException e) {
+ // LOG.warn("Failed to read zookeeper, skipping checking deletable files");
+ // wals = null;
+ // }
}
@Override
@@ -115,7 +114,8 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
if (zkw == null) {
zkw = new ZKWatcher(getConf(), "replicationLogCleaner", null);
}
- this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
+ // TODO: revisit the implementation
+ // this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
} catch (IOException e) {
LOG.error("Error while configuring " + this.getClass().getName(), e);
}
@@ -126,7 +126,8 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
super.setConf(conf);
try {
this.zkw = zk;
- this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zk, conf);
+ // TODO: revisit the implementation
+ // this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zk, conf);
} catch (Exception e) {
LOG.error("Error while configuring " + this.getClass().getName(), e);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ClaimReplicationQueueCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ClaimReplicationQueueCallable.java
index b9a7be813af..2b7e14f9f7a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ClaimReplicationQueueCallable.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ClaimReplicationQueueCallable.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.replication.regionserver;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.executor.EventType;
import org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
@@ -30,9 +31,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.C
@InterfaceAudience.Private
public class ClaimReplicationQueueCallable extends BaseRSProcedureCallable {
- private ServerName crashedServer;
-
- private String queue;
+ private ReplicationQueueId queueId;
@Override
public EventType getEventType() {
@@ -42,14 +41,20 @@ public class ClaimReplicationQueueCallable extends BaseRSProcedureCallable {
@Override
protected void doCall() throws Exception {
PeerProcedureHandler handler = rs.getReplicationSourceService().getPeerProcedureHandler();
- handler.claimReplicationQueue(crashedServer, queue);
+ handler.claimReplicationQueue(queueId);
}
@Override
protected void initParameter(byte[] parameter) throws InvalidProtocolBufferException {
ClaimReplicationQueueRemoteParameter param =
ClaimReplicationQueueRemoteParameter.parseFrom(parameter);
- crashedServer = ProtobufUtil.toServerName(param.getCrashedServer());
- queue = param.getQueue();
+ ServerName crashedServer = ProtobufUtil.toServerName(param.getCrashedServer());
+ String queue = param.getQueue();
+ if (param.hasSourceServer()) {
+ ServerName sourceServer = ProtobufUtil.toServerName(param.getSourceServer());
+ queueId = new ReplicationQueueId(crashedServer, queue, sourceServer);
+ } else {
+ queueId = new ReplicationQueueId(crashedServer, queue);
+ }
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
index 4636e239904..98d0a55fbc4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -21,7 +21,6 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -45,10 +44,8 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.zookeeper.ZKDump;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -59,6 +56,8 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.AtomicLongMap;
/**
+ * TODO: reimplement this tool
+ * <p/>
* Provides information about the existing states of replication, replication peers and queues.
* Usage: hbase org.apache.hadoop.hbase.replication.regionserver.DumpReplicationQueues [args]
* Arguments: --distributed Polls each RS to dump information about the queue --hdfs Reports HDFS
@@ -299,32 +298,33 @@ public class DumpReplicationQueues extends Configured implements Tool {
ReplicationQueueStorage queueStorage;
StringBuilder sb = new StringBuilder();
- queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
- Set<ServerName> liveRegionServers = ZKUtil.listChildrenNoWatch(zkw, zkw.getZNodePaths().rsZNode)
- .stream().map(ServerName::parseServerName).collect(Collectors.toSet());
-
+ // queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
+ // Set<ServerName> liveRegionServers = ZKUtil.listChildrenNoWatch(zkw,
+ // zkw.getZNodePaths().rsZNode)
+ // .stream().map(ServerName::parseServerName).collect(Collectors.toSet());
+ //
// Loops each peer on each RS and dumps the queues
- List<ServerName> regionservers = queueStorage.getListOfReplicators();
- if (regionservers == null || regionservers.isEmpty()) {
- return sb.toString();
- }
- for (ServerName regionserver : regionservers) {
- List<String> queueIds = queueStorage.getAllQueues(regionserver);
- if (!liveRegionServers.contains(regionserver)) {
- deadRegionServers.add(regionserver.getServerName());
- }
- for (String queueId : queueIds) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- List<String> wals = queueStorage.getWALsInQueue(regionserver, queueId);
- Collections.sort(wals);
- if (!peerIds.contains(queueInfo.getPeerId())) {
- deletedQueues.add(regionserver + "/" + queueId);
- sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, true, hdfs));
- } else {
- sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, false, hdfs));
- }
- }
- }
+ // List<ServerName> regionservers = queueStorage.getListOfReplicators();
+ // if (regionservers == null || regionservers.isEmpty()) {
+ // return sb.toString();
+ // }
+ // for (ServerName regionserver : regionservers) {
+ // List<String> queueIds = queueStorage.getAllQueues(regionserver);
+ // if (!liveRegionServers.contains(regionserver)) {
+ // deadRegionServers.add(regionserver.getServerName());
+ // }
+ // for (String queueId : queueIds) {
+ // ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
+ // List<String> wals = queueStorage.getWALsInQueue(regionserver, queueId);
+ // Collections.sort(wals);
+ // if (!peerIds.contains(queueInfo.getPeerId())) {
+ // deletedQueues.add(regionserver + "/" + queueId);
+ // sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, true, hdfs));
+ // } else {
+ // sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, false, hdfs));
+ // }
+ // }
+ // }
return sb.toString();
}
@@ -350,9 +350,9 @@ public class DumpReplicationQueues extends Configured implements Tool {
peersQueueSize.addAndGet(queueInfo.getPeerId(), wals.size());
for (String wal : wals) {
- long position = queueStorage.getWALPosition(regionserver, queueInfo.getPeerId(), wal);
- sb.append(" Replication position for " + wal + ": "
- + (position > 0 ? position : "0" + " (not started or nothing to replicate)") + "\n");
+ // long position = queueStorage.getWALPosition(regionserver, queueInfo.getPeerId(), wal);
+ // sb.append(" Replication position for " + wal + ": "
+ // + (position > 0 ? position : "0" + " (not started or nothing to replicate)") + "\n");
}
if (hdfs) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
index 3df78c1d831..b2cffd59fd4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.yetus.audience.InterfaceAudience;
/**
@@ -42,6 +42,5 @@ public interface PeerProcedureHandler {
void transitSyncReplicationPeerState(String peerId, int stage, HRegionServer rs)
throws ReplicationException, IOException;
- void claimReplicationQueue(ServerName crashedServer, String queue)
- throws ReplicationException, IOException;
+ void claimReplicationQueue(ReplicationQueueId queueId) throws ReplicationException, IOException;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index 0187de14f80..cd3db44d8fa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.concurrent.locks.Lock;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.LogRoller;
import org.apache.hadoop.hbase.replication.ReplicationException;
@@ -28,6 +27,7 @@ import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
import org.apache.hadoop.hbase.util.KeyLocker;
@@ -226,8 +226,8 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
}
@Override
- public void claimReplicationQueue(ServerName crashedServer, String queue)
+ public void claimReplicationQueue(ReplicationQueueId queueId)
throws ReplicationException, IOException {
- replicationSourceManager.claimQueue(crashedServer, queue);
+ replicationSourceManager.claimQueue(queueId);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index 024248a3f8c..e740a01dc4f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -17,23 +17,7 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
-import java.io.IOException;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.PriorityBlockingQueue;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.replication.ReplicationPeer;
-import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.util.CommonFSUtils;
-import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Class that handles the recovered source of a replication stream, which is transfered from another
@@ -42,124 +26,14 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public class RecoveredReplicationSource extends ReplicationSource {
- private static final Logger LOG = LoggerFactory.getLogger(RecoveredReplicationSource.class);
-
- private String actualPeerId;
-
- @Override
- public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueueStorage queueStorage, ReplicationPeer replicationPeer, Server server,
- String peerClusterZnode, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
- MetricsSource metrics) throws IOException {
- super.init(conf, fs, manager, queueStorage, replicationPeer, server, peerClusterZnode,
- clusterId, walFileLengthProvider, metrics);
- this.actualPeerId = this.replicationQueueInfo.getPeerId();
- }
-
@Override
protected RecoveredReplicationSourceShipper createNewShipper(String walGroupId) {
- return new RecoveredReplicationSourceShipper(conf, walGroupId, logQueue, this, queueStorage);
- }
-
- public void locateRecoveredPaths(String walGroupId) throws IOException {
- boolean hasPathChanged = false;
- PriorityBlockingQueue<Path> queue = logQueue.getQueue(walGroupId);
- PriorityBlockingQueue<Path> newPaths = new PriorityBlockingQueue<Path>(queueSizePerGroup,
- new AbstractFSWALProvider.WALStartTimeComparator());
- pathsLoop: for (Path path : queue) {
- if (fs.exists(path)) { // still in same location, don't need to do anything
- newPaths.add(path);
- continue;
- }
- // Path changed - try to find the right path.
- hasPathChanged = true;
- if (server instanceof ReplicationSyncUp.DummyServer) {
- // In the case of disaster/recovery, HMaster may be shutdown/crashed before flush data
- // from .logs to .oldlogs. Loop into .logs folders and check whether a match exists
- Path newPath = getReplSyncUpPath(path);
- newPaths.add(newPath);
- continue;
- } else {
- // See if Path exists in the dead RS folder (there could be a chain of failures
- // to look at)
- List<ServerName> deadRegionServers = this.replicationQueueInfo.getDeadRegionServers();
- LOG.info("NB dead servers : " + deadRegionServers.size());
- final Path walDir = CommonFSUtils.getWALRootDir(conf);
- for (ServerName curDeadServerName : deadRegionServers) {
- final Path deadRsDirectory = new Path(walDir,
- AbstractFSWALProvider.getWALDirectoryName(curDeadServerName.getServerName()));
- Path[] locs = new Path[] { new Path(deadRsDirectory, path.getName()),
- new Path(deadRsDirectory.suffix(AbstractFSWALProvider.SPLITTING_EXT), path.getName()) };
- for (Path possibleLogLocation : locs) {
- LOG.info("Possible location " + possibleLogLocation.toUri().toString());
- if (manager.getFs().exists(possibleLogLocation)) {
- // We found the right new location
- LOG.info("Log " + path + " still exists at " + possibleLogLocation);
- newPaths.add(possibleLogLocation);
- continue pathsLoop;
- }
- }
- }
- // didn't find a new location
- LOG.error(
- String.format("WAL Path %s doesn't exist and couldn't find its new location", path));
- newPaths.add(path);
- }
- }
-
- if (hasPathChanged) {
- if (newPaths.size() != queue.size()) { // this shouldn't happen
- LOG.error("Recovery queue size is incorrect");
- throw new IOException("Recovery queue size error");
- }
- // put the correct locations in the queue
- // since this is a recovered queue with no new incoming logs,
- // there shouldn't be any concurrency issues
- logQueue.clear(walGroupId);
- for (Path path : newPaths) {
- logQueue.enqueueLog(path, walGroupId);
- }
- }
- }
-
- // N.B. the ReplicationSyncUp tool sets the manager.getWALDir to the root of the wal
- // area rather than to the wal area for a particular region server.
- private Path getReplSyncUpPath(Path path) throws IOException {
- FileStatus[] rss = fs.listStatus(manager.getLogDir());
- for (FileStatus rs : rss) {
- Path p = rs.getPath();
- FileStatus[] logs = fs.listStatus(p);
- for (FileStatus log : logs) {
- p = new Path(p, log.getPath().getName());
- if (p.getName().equals(path.getName())) {
- LOG.info("Log " + p.getName() + " found at " + p);
- return p;
+ return new RecoveredReplicationSourceShipper(conf, walGroupId, logQueue, this, queueStorage,
+ () -> {
+ if (workerThreads.isEmpty()) {
+ this.getSourceMetrics().clear();
+ manager.finishRecoveredSource(this);
}
- }
- }
- LOG.error("Didn't find path for: " + path.getName());
- return path;
- }
-
- void tryFinish() {
- if (workerThreads.isEmpty()) {
- this.getSourceMetrics().clear();
- manager.finishRecoveredSource(this);
- }
- }
-
- @Override
- public String getPeerId() {
- return this.actualPeerId;
- }
-
- @Override
- public ServerName getServerWALsBelongTo() {
- return this.replicationQueueInfo.getDeadRegionServers().get(0);
- }
-
- @Override
- public boolean isRecovered() {
- return true;
+ });
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
index 4f2bafcf156..2bb3a7c3591 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
@@ -17,83 +17,27 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
-import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.util.Threads;
import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Used by a {@link RecoveredReplicationSource}.
*/
@InterfaceAudience.Private
public class RecoveredReplicationSourceShipper extends ReplicationSourceShipper {
- private static final Logger LOG =
- LoggerFactory.getLogger(RecoveredReplicationSourceShipper.class);
- protected final RecoveredReplicationSource source;
- private final ReplicationQueueStorage replicationQueues;
+ private final Runnable tryFinish;
public RecoveredReplicationSourceShipper(Configuration conf, String walGroupId,
ReplicationSourceLogQueue logQueue, RecoveredReplicationSource source,
- ReplicationQueueStorage queueStorage) {
+ ReplicationQueueStorage queueStorage, Runnable tryFinish) {
super(conf, walGroupId, logQueue, source);
- this.source = source;
- this.replicationQueues = queueStorage;
+ this.tryFinish = tryFinish;
}
@Override
protected void postFinish() {
- source.tryFinish();
- }
-
- @Override
- public long getStartPosition() {
- long startPosition = getRecoveredQueueStartPos();
- int numRetries = 0;
- while (numRetries <= maxRetriesMultiplier) {
- try {
- source.locateRecoveredPaths(walGroupId);
- break;
- } catch (IOException e) {
- LOG.error("Error while locating recovered queue paths, attempt #" + numRetries, e);
- numRetries++;
- }
- }
- return startPosition;
- }
-
- // If this is a recovered queue, the queue is already full and the first log
- // normally has a position (unless the RS failed between 2 logs)
- private long getRecoveredQueueStartPos() {
- long startPosition = 0;
- String peerClusterZNode = source.getQueueId();
- try {
- startPosition = this.replicationQueues.getWALPosition(source.getServer().getServerName(),
- peerClusterZNode, this.logQueue.getQueue(walGroupId).peek().getName());
- LOG.trace("Recovered queue started with log {} at position {}",
- this.logQueue.getQueue(walGroupId).peek(), startPosition);
- } catch (ReplicationException e) {
- terminate("Couldn't get the position of this recovered queue " + peerClusterZNode, e);
- }
- return startPosition;
- }
-
- private void terminate(String reason, Exception cause) {
- if (cause == null) {
- LOG.info("Closing worker for wal group {} because: {}", this.walGroupId, reason);
- } else {
- LOG.error(
- "Closing worker for wal group " + this.walGroupId + " because an error occurred: " + reason,
- cause);
- }
- entryReader.interrupt();
- Threads.shutdown(entryReader, sleepForRetries);
- this.interrupt();
- Threads.shutdown(this, sleepForRetries);
- LOG.info("ReplicationSourceWorker {} terminated", this.getName());
+ tryFinish.run();
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index ea28a20c56b..9e26726c065 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -95,7 +95,7 @@ public class Replication implements ReplicationSourceService {
try {
this.queueStorage =
- ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
+ ReplicationStorageFactory.getReplicationQueueStorage(server.getConnection(), conf);
this.replicationPeers =
ReplicationFactory.getReplicationPeers(server.getZooKeeper(), this.conf);
this.replicationPeers.init();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java
index 00306dd1702..7e8ca88b730 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java
@@ -89,8 +89,8 @@ public class ReplicationLoad {
rLoadSourceBuild.setOPsShipped(oPsShipped);
if (source instanceof ReplicationSource) {
ReplicationSource replSource = (ReplicationSource) source;
- rLoadSourceBuild.setRecovered(replSource.getReplicationQueueInfo().isQueueRecovered());
- rLoadSourceBuild.setQueueId(replSource.getReplicationQueueInfo().getQueueId());
+ rLoadSourceBuild.setRecovered(replSource.getQueueId().isRecovered());
+ rLoadSourceBuild.setQueueId(replSource.getQueueId().toString());
rLoadSourceBuild.setRunning(replSource.isWorkerRunning());
rLoadSourceBuild.setEditsSinceRestart(timeStampOfNextToReplicate > 0);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 74a430a7f38..e078722b157 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.replication.regionserver;
import static org.apache.hadoop.hbase.wal.AbstractFSWALProvider.findArchivedLog;
+import com.google.errorprone.annotations.RestrictedApi;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
@@ -52,8 +53,10 @@ import org.apache.hadoop.hbase.replication.ChainWALEntryFilter;
import org.apache.hadoop.hbase.replication.ClusterMarkingEntryFilter;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.SystemTableWALEntryFilter;
import org.apache.hadoop.hbase.replication.WALEntryFilter;
@@ -67,6 +70,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
/**
@@ -90,7 +94,6 @@ public class ReplicationSource implements ReplicationSourceInterface {
protected ReplicationPeer replicationPeer;
protected Configuration conf;
- protected ReplicationQueueInfo replicationQueueInfo;
// The manager of all sources to which we ping back our progress
protected ReplicationSourceManager manager;
@@ -103,8 +106,11 @@ public class ReplicationSource implements ReplicationSourceInterface {
private UUID clusterId;
// total number of edits we replicated
private AtomicLong totalReplicatedEdits = new AtomicLong(0);
- // The znode we currently play with
- protected String queueId;
+ // The id of the replication queue
+ protected ReplicationQueueId queueId;
+ // The start offsets. Usually only recovered replication queue needs this, but probably when we
+ // update the peer config and restart the replication peer, we also need this?
+ protected ImmutableMap<String, ReplicationGroupOffset> startOffsets;
// Maximum number of retries before taking bold actions
private int maxRetriesMultiplier;
// Indicates if this particular source is running
@@ -186,14 +192,14 @@ public class ReplicationSource implements ReplicationSourceInterface {
* @param fs file system to use
* @param manager replication manager to ping to
* @param server the server for this region server
- * @param queueId the id of our replication queue
+ * @param queueData the id and offsets of our replication queue
* @param clusterId unique UUID for the cluster
* @param metrics metrics for replication source
*/
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
ReplicationQueueStorage queueStorage, ReplicationPeer replicationPeer, Server server,
- String queueId, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
+ ReplicationQueueData queueData, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
MetricsSource metrics) throws IOException {
this.server = server;
this.conf = HBaseConfiguration.create(conf);
@@ -213,8 +219,8 @@ public class ReplicationSource implements ReplicationSourceInterface {
this.metrics = metrics;
this.clusterId = clusterId;
- this.queueId = queueId;
- this.replicationQueueInfo = new ReplicationQueueInfo(queueId);
+ this.queueId = queueData.getId();
+ this.startOffsets = queueData.getOffsets();
// A defaultBandwidth of '0' means no bandwidth; i.e. no throttling.
defaultBandwidth = this.conf.getLong("replication.source.per.peer.node.bandwidth", 0);
@@ -243,24 +249,24 @@ public class ReplicationSource implements ReplicationSourceInterface {
return;
}
// Use WAL prefix as the WALGroupId for this peer.
- String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal.getName());
- boolean queueExists = logQueue.enqueueLog(wal, walPrefix);
+ String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(wal.getName());
+ boolean queueExists = logQueue.enqueueLog(wal, walGroupId);
if (!queueExists) {
if (this.isSourceActive() && this.walEntryFilter != null) {
// new wal group observed after source startup, start a new worker thread to track it
// notice: it's possible that wal enqueued when this.running is set but worker thread
// still not launched, so it's necessary to check workerThreads before start the worker
- tryStartNewShipper(walPrefix);
+ tryStartNewShipper(walGroupId);
}
}
if (LOG.isTraceEnabled()) {
- LOG.trace("{} Added wal {} to queue of source {}.", logPeerId(), walPrefix,
- this.replicationQueueInfo.getQueueId());
+ LOG.trace("{} Added wal {} to queue of source {}.", logPeerId(), walGroupId, queueId);
}
}
- @InterfaceAudience.Private
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
public Map<String, PriorityBlockingQueue<Path>> getQueues() {
return logQueue.getQueues();
}
@@ -333,6 +339,30 @@ public class ReplicationSource implements ReplicationSourceInterface {
this.walEntryFilter = new ChainWALEntryFilter(filters);
}
+ private long getStartOffset(String walGroupId) {
+ ReplicationGroupOffset startOffset = startOffsets.get(walGroupId);
+ if (startOffset == null || startOffset == ReplicationGroupOffset.BEGIN) {
+ return 0L;
+ }
+ // this method will only be called when start new shipper, and we will only start new shipper
+ // when there is a new queue, so here the queue for walGroupId will never be null.
+ Path first = logQueue.getQueue(walGroupId).peek();
+ if (!startOffset.getWal().equals(first.getName())) {
+ return 0L;
+ }
+ // Usually, if we arrive here, the start offset should never be -1, as it means this file has
+ // been fully replicated so we should have filtered it out in upper layer, usually in
+ // ReplicationSourceManager. Add a warn message for safety, as usually replicate more data will
+ // not cause big problems.
+ if (startOffset.getOffset() < 0) {
+ LOG.warn("Should have already replicated wal {}, return start offset as 0",
+ startOffset.getWal());
+ return 0L;
+ } else {
+ return startOffset.getOffset();
+ }
+ }
+
private void tryStartNewShipper(String walGroupId) {
workerThreads.compute(walGroupId, (key, value) -> {
if (value != null) {
@@ -342,7 +372,7 @@ public class ReplicationSource implements ReplicationSourceInterface {
LOG.debug("{} starting shipping worker for walGroupId={}", logPeerId(), walGroupId);
ReplicationSourceShipper worker = createNewShipper(walGroupId);
ReplicationSourceWALReader walReader =
- createNewWALReader(walGroupId, worker.getStartPosition());
+ createNewWALReader(walGroupId, getStartOffset(walGroupId));
Threads.setDaemonThreadRunning(
walReader, Thread.currentThread().getName() + ".replicationSource.wal-reader."
+ walGroupId + "," + queueId,
@@ -571,7 +601,7 @@ public class ReplicationSource implements ReplicationSourceInterface {
}
LOG.info("{} queueId={} (queues={}) is replicating from cluster={} to cluster={}", logPeerId(),
- this.replicationQueueInfo.getQueueId(), logQueue.getNumQueues(), clusterId, peerClusterId);
+ queueId, logQueue.getNumQueues(), clusterId, peerClusterId);
initializeWALEntryFilter(peerClusterId);
// Start workers
for (String walGroupId : logQueue.getQueues().keySet()) {
@@ -718,7 +748,7 @@ public class ReplicationSource implements ReplicationSourceInterface {
}
@Override
- public String getQueueId() {
+ public ReplicationQueueId getQueueId() {
return this.queueId;
}
@@ -738,10 +768,6 @@ public class ReplicationSource implements ReplicationSourceInterface {
return !this.server.isStopped() && this.sourceRunning;
}
- public ReplicationQueueInfo getReplicationQueueInfo() {
- return replicationQueueInfo;
- }
-
public boolean isWorkerRunning() {
for (ReplicationSourceShipper worker : this.workerThreads.values()) {
if (worker.isActive()) {
@@ -796,7 +822,7 @@ public class ReplicationSource implements ReplicationSourceInterface {
@Override
public ServerName getServerWALsBelongTo() {
- return server.getServerName();
+ return queueId.getServerWALsBelongTo();
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java
index 331f2269cf9..225c6fd4d74 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hbase.replication.regionserver;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,9 +34,8 @@ public final class ReplicationSourceFactory {
private ReplicationSourceFactory() {
}
- static ReplicationSourceInterface create(Configuration conf, String queueId) {
- ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queueId);
- boolean isQueueRecovered = replicationQueueInfo.isQueueRecovered();
+ static ReplicationSourceInterface create(Configuration conf, ReplicationQueueId queueId) {
+ boolean isQueueRecovered = queueId.isRecovered();
ReplicationSourceInterface src;
try {
String defaultReplicationSourceImpl = isQueueRecovered
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
index 02e7a7ac532..65261e52dd8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.WAL.Entry;
@@ -43,14 +45,22 @@ import org.apache.yetus.audience.InterfaceAudience;
public interface ReplicationSourceInterface {
/**
* Initializer for the source
- * @param conf the configuration to use
- * @param fs the file system to use
- * @param manager the manager to use
- * @param server the server for this region server
+ * @param conf the configuration to use
+ * @param fs the file system to use
+ * @param manager the manager to use
+ * @param queueStorage the replication queue storage
+ * @param replicationPeer the replication peer
+ * @param server the server for this region server
+ * @param queueData the existing replication queue data, contains the queue id and
+ * replication start offsets
+ * @param clusterId the cluster id
+ * @param walFileLengthProvider for getting the length of the WAL file which is currently being
+ * written
+ * @param metrics the replication metrics
*/
void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
ReplicationQueueStorage queueStorage, ReplicationPeer replicationPeer, Server server,
- String queueId, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
+ ReplicationQueueData queueData, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
MetricsSource metrics) throws IOException;
/**
@@ -106,14 +116,14 @@ public interface ReplicationSourceInterface {
* Get the queue id that the source is replicating to
* @return queue id
*/
- String getQueueId();
+ ReplicationQueueId getQueueId();
/**
* Get the id that the source is replicating to.
* @return peer id
*/
default String getPeerId() {
- return getPeer().getId();
+ return getQueueId().getPeerId();
}
/**
@@ -183,7 +193,7 @@ public interface ReplicationSourceInterface {
/** Returns whether this is a replication source for recovery. */
default boolean isRecovered() {
- return false;
+ return getQueueId().isRecovered();
}
/** Returns The instance of queueStorage used by this ReplicationSource. */
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceLogQueue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceLogQueue.java
index a0e6f1b8d1f..93a28b60d27 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceLogQueue.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceLogQueue.java
@@ -97,7 +97,7 @@ public class ReplicationSourceLogQueue {
* @param walGroupId walGroupId
*/
public int getQueueSize(String walGroupId) {
- Queue queue = queues.get(walGroupId);
+ Queue<Path> queue = queues.get(walGroupId);
if (queue == null) {
return 0;
}
@@ -117,7 +117,7 @@ public class ReplicationSourceLogQueue {
/**
* Return queue for the given walGroupId Please don't add or remove elements from the returned
- * queue. Use @enqueueLog and @remove methods respectively.
+ * queue. Use {@link #enqueueLog(Path, String)} and {@link #remove(String)} methods respectively.
* @param walGroupId walGroupId
*/
public PriorityBlockingQueue<Path> getQueue(String walGroupId) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 04e306c219d..309ae522e2c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -17,17 +17,22 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
+import com.google.errorprone.annotations.RestrictedApi;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
import java.util.OptionalLong;
+import java.util.PriorityQueue;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -49,12 +54,13 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
-import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
@@ -67,6 +73,7 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -89,26 +96,25 @@ import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFacto
* operations.</li>
* <li>Need synchronized on {@link #walsById}. There are four methods which modify it,
* {@link #addPeer(String)}, {@link #removePeer(String)},
- * {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)} and {@link #preLogRoll(Path)}.
- * {@link #walsById} is a ConcurrentHashMap and there is a Lock for peer id in
- * {@link PeerProcedureHandlerImpl}. So there is no race between {@link #addPeer(String)} and
+ * {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)} and
+ * {@link #postLogRoll(Path)}. {@link #walsById} is a ConcurrentHashMap and there is a Lock for peer
+ * id in {@link PeerProcedureHandlerImpl}. So there is no race between {@link #addPeer(String)} and
* {@link #removePeer(String)}. {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)}
* is called by {@link ReplicationSourceInterface}. So no race with {@link #addPeer(String)}.
* {@link #removePeer(String)} will terminate the {@link ReplicationSourceInterface} firstly, then
* remove the wals from {@link #walsById}. So no race with {@link #removePeer(String)}. The only
* case need synchronized is {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)} and
- * {@link #preLogRoll(Path)}.</li>
+ * {@link #postLogRoll(Path)}.</li>
* <li>No need synchronized on {@link #walsByIdRecoveredQueues}. There are three methods which
* modify it, {@link #removePeer(String)} ,
* {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)} and
- * {@link ReplicationSourceManager#claimQueue(ServerName, String)}.
+ * {@link #claimQueue(ReplicationQueueId)}.
* {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)} is called by
* {@link ReplicationSourceInterface}. {@link #removePeer(String)} will terminate the
* {@link ReplicationSourceInterface} firstly, then remove the wals from
- * {@link #walsByIdRecoveredQueues}. And
- * {@link ReplicationSourceManager#claimQueue(ServerName, String)} will add the wals to
- * {@link #walsByIdRecoveredQueues} firstly, then start up a {@link ReplicationSourceInterface}. So
- * there is no race here. For {@link ReplicationSourceManager#claimQueue(ServerName, String)} and
+ * {@link #walsByIdRecoveredQueues}. And {@link #claimQueue(ReplicationQueueId)} will add the wals
+ * to {@link #walsByIdRecoveredQueues} firstly, then start up a {@link ReplicationSourceInterface}.
+ * So there is no race here. For {@link #claimQueue(ReplicationQueueId)} and
* {@link #removePeer(String)}, there is already synchronized on {@link #oldsources}. So no need
* synchronized on {@link #walsByIdRecoveredQueues}.</li>
* <li>Need synchronized on {@link #latestPaths} to avoid the new open source miss new log.</li>
@@ -140,11 +146,12 @@ public class ReplicationSourceManager {
// All logs we are currently tracking
// Index structure of the map is: queue_id->logPrefix/logGroup->logs
// For normal replication source, the peer id is same with the queue id
- private final ConcurrentMap<String, Map<String, NavigableSet<String>>> walsById;
+ private final ConcurrentMap<ReplicationQueueId, Map<String, NavigableSet<String>>> walsById;
// Logs for recovered sources we are currently tracking
// the map is: queue_id->logPrefix/logGroup->logs
// For recovered source, the queue id's format is peer_id-servername-*
- private final ConcurrentMap<String, Map<String, NavigableSet<String>>> walsByIdRecoveredQueues;
+ private final ConcurrentMap<ReplicationQueueId,
+ Map<String, NavigableSet<String>>> walsByIdRecoveredQueues;
private final SyncReplicationPeerMappingManager syncReplicationPeerMappingManager;
@@ -162,8 +169,6 @@ public class ReplicationSourceManager {
// Homemade executer service for replication
private final ThreadPoolExecutor executor;
- private final boolean replicationForBulkLoadDataEnabled;
-
private AtomicLong totalBufferUsed = new AtomicLong();
// How long should we sleep for each retry when deleting remote wal files for sync replication
@@ -218,8 +223,6 @@ public class ReplicationSourceManager {
tfb.setDaemon(true);
this.executor.setThreadFactory(tfb.build());
this.latestPaths = new HashMap<>();
- this.replicationForBulkLoadDataEnabled = conf.getBoolean(
- HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
this.sleepForRetries = this.conf.getLong("replication.source.sync.sleepforretries", 1000);
this.maxRetriesMultiplier =
this.conf.getInt("replication.source.sync.maxretriesmultiplier", 60);
@@ -234,11 +237,6 @@ public class ReplicationSourceManager {
void init() throws IOException {
for (String id : this.replicationPeers.getAllPeerIds()) {
addSource(id);
- if (replicationForBulkLoadDataEnabled) {
- // Check if peer exists in hfile-refs queue, if not add it. This can happen in the case
- // when a peer was added before replication for bulk loaded data was enabled.
- throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(id));
- }
}
}
@@ -259,9 +257,6 @@ public class ReplicationSourceManager {
}
if (added) {
addSource(peerId);
- if (replicationForBulkLoadDataEnabled) {
- throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(peerId));
- }
}
}
@@ -292,26 +287,17 @@ public class ReplicationSourceManager {
removeRecoveredSource(src);
}
}
- LOG
- .info("Number of deleted recovered sources for " + peerId + ": " + oldSourcesToDelete.size());
+ LOG.info("Number of deleted recovered sources for {}: {}", peerId, oldSourcesToDelete.size());
// Now close the normal source for this peer
ReplicationSourceInterface srcToRemove = this.sources.get(peerId);
if (srcToRemove != null) {
srcToRemove.terminate(terminateMessage);
removeSource(srcToRemove);
- } else {
- // This only happened in unit test TestReplicationSourceManager#testPeerRemovalCleanup
- // Delete queue from storage and memory and queue id is same with peer id for normal
- // source
- deleteQueue(peerId);
- this.walsById.remove(peerId);
}
ReplicationPeerConfig peerConfig = peer.getPeerConfig();
if (peerConfig.isSyncReplication()) {
syncReplicationPeerMappingManager.remove(peerId, peerConfig);
}
- // Remove HFile Refs
- abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(peerId));
}
/**
@@ -319,17 +305,17 @@ public class ReplicationSourceManager {
* @param queueId the id of the replication queue to associate the ReplicationSource with.
* @see #createCatalogReplicationSource(RegionInfo) for creating a ReplicationSource for meta.
*/
- private ReplicationSourceInterface createSource(String queueId, ReplicationPeer replicationPeer)
- throws IOException {
- ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, queueId);
+ private ReplicationSourceInterface createSource(ReplicationQueueData queueData,
+ ReplicationPeer replicationPeer) throws IOException {
+ ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, queueData.getId());
// Init the just created replication source. Pass the default walProvider's wal file length
// provider. Presumption is we replicate user-space Tables only. For hbase:meta region replica
// replication, see #createCatalogReplicationSource().
WALFileLengthProvider walFileLengthProvider = this.walFactory.getWALProvider() != null
? this.walFactory.getWALProvider().getWALFileLengthProvider()
: p -> OptionalLong.empty();
- src.init(conf, fs, this, queueStorage, replicationPeer, server, queueId, clusterId,
- walFileLengthProvider, new MetricsSource(queueId));
+ src.init(conf, fs, this, queueStorage, replicationPeer, server, queueData, clusterId,
+ walFileLengthProvider, new MetricsSource(queueData.getId().toString()));
return src;
}
@@ -350,12 +336,14 @@ public class ReplicationSourceManager {
LOG.info("Legacy region replication peer found, skip adding: {}", peer.getPeerConfig());
return;
}
- ReplicationSourceInterface src = createSource(peerId, peer);
+ ReplicationQueueId queueId = new ReplicationQueueId(server.getServerName(), peerId);
+ ReplicationSourceInterface src =
+ createSource(new ReplicationQueueData(queueId, ImmutableMap.of()), peer);
// synchronized on latestPaths to avoid missing the new log
synchronized (this.latestPaths) {
this.sources.put(peerId, src);
Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();
- this.walsById.put(peerId, walsByGroup);
+ this.walsById.put(queueId, walsByGroup);
// Add the latest wal to that source's queue
if (!latestPaths.isEmpty()) {
for (Map.Entry<String, Path> walPrefixAndPath : latestPaths.entrySet()) {
@@ -364,8 +352,10 @@ public class ReplicationSourceManager {
wals.add(walPath.getName());
walsByGroup.put(walPrefixAndPath.getKey(), wals);
// Abort RS and throw exception to make add peer failed
+ // TODO: can record the length of the current wal file so we could replicate less data
abortAndThrowIOExceptionWhenFail(
- () -> this.queueStorage.addWAL(server.getServerName(), peerId, walPath.getName()));
+ () -> this.queueStorage.setOffset(queueId, walPrefixAndPath.getKey(),
+ new ReplicationGroupOffset(walPath.getName(), 0), Collections.emptyMap()));
src.enqueueLog(walPath);
LOG.trace("Enqueued {} to source {} during source creation.", walPath, src.getQueueId());
}
@@ -398,7 +388,10 @@ public class ReplicationSourceManager {
+ " is transiting to STANDBY. Will close the previous replication source and open a new one";
ReplicationPeer peer = replicationPeers.getPeer(peerId);
assert peer.getPeerConfig().isSyncReplication();
- ReplicationSourceInterface src = createSource(peerId, peer);
+ ReplicationQueueId queueId = new ReplicationQueueId(server.getServerName(), peerId);
+ // TODO: use empty initial offsets for now, revisit when adding support for sync replication
+ ReplicationSourceInterface src =
+ createSource(new ReplicationQueueData(queueId, ImmutableMap.of()), peer);
// synchronized here to avoid race with preLogRoll where we add new log to source and also
// walsById.
ReplicationSourceInterface toRemove;
@@ -415,17 +408,18 @@ public class ReplicationSourceManager {
// map from walsById since later we may fail to delete them from the replication queue
// storage, and when we retry next time, we can not know the wal files that need to be deleted
// from the replication queue storage.
- walsById.get(peerId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));
+ walsById.get(queueId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));
}
LOG.info("Startup replication source for " + src.getPeerId());
src.startup();
for (NavigableSet<String> walsByGroup : wals.values()) {
- for (String wal : walsByGroup) {
- queueStorage.removeWAL(server.getServerName(), peerId, wal);
- }
+ // TODO: just need to reset the replication offset
+ // for (String wal : walsByGroup) {
+ // queueStorage.removeWAL(server.getServerName(), peerId, wal);
+ // }
}
synchronized (walsById) {
- Map<String, NavigableSet<String>> oldWals = walsById.get(peerId);
+ Map<String, NavigableSet<String>> oldWals = walsById.get(queueId);
wals.forEach((k, v) -> {
NavigableSet<String> walsByGroup = oldWals.get(k);
if (walsByGroup != null) {
@@ -440,17 +434,28 @@ public class ReplicationSourceManager {
for (Iterator<ReplicationSourceInterface> iter = oldsources.iterator(); iter.hasNext();) {
ReplicationSourceInterface oldSource = iter.next();
if (oldSource.getPeerId().equals(peerId)) {
- String queueId = oldSource.getQueueId();
+ ReplicationQueueId oldSourceQueueId = oldSource.getQueueId();
oldSource.terminate(terminateMessage);
oldSource.getSourceMetrics().clear();
- queueStorage.removeQueue(server.getServerName(), queueId);
- walsByIdRecoveredQueues.remove(queueId);
+ queueStorage.removeQueue(oldSourceQueueId);
+ walsByIdRecoveredQueues.remove(oldSourceQueueId);
iter.remove();
}
}
}
}
+ private ReplicationSourceInterface createRefreshedSource(ReplicationQueueId queueId,
+ ReplicationPeer peer) throws IOException {
+ Map<String, ReplicationGroupOffset> offsets;
+ try {
+ offsets = queueStorage.getOffsets(queueId);
+ } catch (ReplicationException e) {
+ throw new IOException(e);
+ }
+ return createSource(new ReplicationQueueData(queueId, ImmutableMap.copyOf(offsets)), peer);
+ }
+
/**
* Close the previous replication sources of this peer id and open new sources to trigger the new
* replication state changes or new replication config changes. Here we don't need to change
@@ -461,6 +466,7 @@ public class ReplicationSourceManager {
String terminateMessage = "Peer " + peerId
+ " state or config changed. Will close the previous replication source and open a new one";
ReplicationPeer peer = replicationPeers.getPeer(peerId);
+ ReplicationQueueId queueId = new ReplicationQueueId(server.getServerName(), peerId);
ReplicationSourceInterface src;
// synchronized on latestPaths to avoid missing the new log
synchronized (this.latestPaths) {
@@ -470,9 +476,9 @@ public class ReplicationSourceManager {
// Do not clear metrics
toRemove.terminate(terminateMessage, null, false);
}
- src = createSource(peerId, peer);
+ src = createRefreshedSource(queueId, peer);
this.sources.put(peerId, src);
- for (NavigableSet<String> walsByGroup : walsById.get(peerId).values()) {
+ for (NavigableSet<String> walsByGroup : walsById.get(queueId).values()) {
walsByGroup.forEach(wal -> src.enqueueLog(new Path(this.logDir, wal)));
}
}
@@ -482,20 +488,22 @@ public class ReplicationSourceManager {
List<ReplicationSourceInterface> toStartup = new ArrayList<>();
// synchronized on oldsources to avoid race with NodeFailoverWorker
synchronized (this.oldsources) {
- List<String> previousQueueIds = new ArrayList<>();
+ List<ReplicationQueueId> oldSourceQueueIds = new ArrayList<>();
for (Iterator<ReplicationSourceInterface> iter = this.oldsources.iterator(); iter
.hasNext();) {
ReplicationSourceInterface oldSource = iter.next();
if (oldSource.getPeerId().equals(peerId)) {
- previousQueueIds.add(oldSource.getQueueId());
+ oldSourceQueueIds.add(oldSource.getQueueId());
oldSource.terminate(terminateMessage);
iter.remove();
}
}
- for (String queueId : previousQueueIds) {
- ReplicationSourceInterface recoveredReplicationSource = createSource(queueId, peer);
+ for (ReplicationQueueId oldSourceQueueId : oldSourceQueueIds) {
+ ReplicationSourceInterface recoveredReplicationSource =
+ createRefreshedSource(oldSourceQueueId, peer);
this.oldsources.add(recoveredReplicationSource);
- for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(queueId).values()) {
+ for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(oldSourceQueueId)
+ .values()) {
walsByGroup.forEach(wal -> recoveredReplicationSource.enqueueLog(new Path(wal)));
}
toStartup.add(recoveredReplicationSource);
@@ -548,8 +556,8 @@ public class ReplicationSourceManager {
* Delete a complete queue of wals associated with a replication source
* @param queueId the id of replication queue to delete
*/
- private void deleteQueue(String queueId) {
- abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), queueId));
+ private void deleteQueue(ReplicationQueueId queueId) {
+ abortWhenFail(() -> this.queueStorage.removeQueue(queueId));
}
@FunctionalInterface
@@ -615,10 +623,15 @@ public class ReplicationSourceManager {
*/
public void logPositionAndCleanOldLogs(ReplicationSourceInterface source,
WALEntryBatch entryBatch) {
- String fileName = entryBatch.getLastWalPath().getName();
- interruptOrAbortWhenFail(() -> this.queueStorage.setWALPosition(server.getServerName(),
- source.getQueueId(), fileName, entryBatch.getLastWalPosition(), entryBatch.getLastSeqIds()));
- cleanOldLogs(fileName, entryBatch.isEndOfFile(), source);
+ String walName = entryBatch.getLastWalPath().getName();
+ String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(walName);
+ // if end of file, we just set the offset to -1 so we know that this file has already been fully
+ // replicated, otherwise we need to compare the file length
+ ReplicationGroupOffset offset = new ReplicationGroupOffset(walName,
+ entryBatch.isEndOfFile() ? -1 : entryBatch.getLastWalPosition());
+ interruptOrAbortWhenFail(() -> this.queueStorage.setOffset(source.getQueueId(), walPrefix,
+ offset, entryBatch.getLastSeqIds()));
+ cleanOldLogs(walName, entryBatch.isEndOfFile(), source);
}
/**
@@ -643,7 +656,7 @@ public class ReplicationSourceManager {
} else {
NavigableSet<String> wals;
NavigableSet<String> walsToRemove;
- // synchronized on walsById to avoid race with preLogRoll
+ // synchronized on walsById to avoid race with postLogRoll
synchronized (this.walsById) {
wals = walsById.get(source.getQueueId()).get(logPrefix);
if (wals == null) {
@@ -725,33 +738,21 @@ public class ReplicationSourceManager {
}
}
}
- String queueId = source.getQueueId();
- for (String wal : wals) {
- interruptOrAbortWhenFail(
- () -> this.queueStorage.removeWAL(server.getServerName(), queueId, wal));
- }
}
// public because of we call it in TestReplicationEmptyWALRecovery
- public void preLogRoll(Path newLog) throws IOException {
+ public void postLogRoll(Path newLog) throws IOException {
String logName = newLog.getName();
String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(logName);
// synchronized on latestPaths to avoid the new open source miss the new log
synchronized (this.latestPaths) {
- // Add log to queue storage
- for (ReplicationSourceInterface source : this.sources.values()) {
- // If record log to queue storage failed, abort RS and throw exception to make log roll
- // failed
- abortAndThrowIOExceptionWhenFail(
- () -> this.queueStorage.addWAL(server.getServerName(), source.getQueueId(), logName));
- }
-
// synchronized on walsById to avoid race with cleanOldLogs
synchronized (this.walsById) {
// Update walsById map
- for (Map.Entry<String, Map<String, NavigableSet<String>>> entry : this.walsById
+ for (Map.Entry<ReplicationQueueId, Map<String, NavigableSet<String>>> entry : this.walsById
.entrySet()) {
- String peerId = entry.getKey();
+ ReplicationQueueId queueId = entry.getKey();
+ String peerId = queueId.getPeerId();
Map<String, NavigableSet<String>> walsByPrefix = entry.getValue();
boolean existingPrefix = false;
for (Map.Entry<String, NavigableSet<String>> walsEntry : walsByPrefix.entrySet()) {
@@ -779,10 +780,6 @@ public class ReplicationSourceManager {
// Add to latestPaths
latestPaths.put(logPrefix, newLog);
}
- }
-
- // public because of we call it in TestReplicationEmptyWALRecovery
- public void postLogRoll(Path newLog) throws IOException {
// This only updates the sources we own, not the recovered ones
for (ReplicationSourceInterface source : this.sources.values()) {
source.enqueueLog(newLog);
@@ -791,7 +788,29 @@ public class ReplicationSourceManager {
}
}
- void claimQueue(ServerName deadRS, String queue) {
+ /**
+ * Check whether we should replicate the given {@code wal}.
+ * @param wal the file name of the wal
+ * @return {@code true} means we should replicate the given {@code wal}, otherwise {@code false}.
+ */
+ private boolean shouldReplicate(ReplicationGroupOffset offset, String wal) {
+ if (offset == null || offset == ReplicationGroupOffset.BEGIN) {
+ return false;
+ }
+ long walTs = AbstractFSWALProvider.getTimestamp(wal);
+ long startWalTs = AbstractFSWALProvider.getTimestamp(offset.getWal());
+ if (walTs < startWalTs) {
+ return false;
+ } else if (walTs > startWalTs) {
+ return true;
+ }
+ // if the timestamp equals, usually it means we should include this wal but there is a special
+ // case, a negative offset means the wal has already been fully replicated, so here we should
+ // check the offset.
+ return offset.getOffset() >= 0;
+ }
+
+ void claimQueue(ReplicationQueueId queueId) {
// Wait a bit before transferring the queues, we may be shutting down.
// This sleep may not be enough in some cases.
try {
@@ -806,66 +825,83 @@ public class ReplicationSourceManager {
LOG.info("Not transferring queue since we are shutting down");
return;
}
- // After claim the queues from dead region server, wewill skip to start the
+ // After claim the queues from dead region server, we will skip to start the
// RecoveredReplicationSource if the peer has been removed. but there's possible that remove a
// peer with peerId = 2 and add a peer with peerId = 2 again during failover. So we need to get
// a copy of the replication peer first to decide whether we should start the
// RecoveredReplicationSource. If the latest peer is not the old peer, we should also skip to
// start the RecoveredReplicationSource, Otherwise the rs will abort (See HBASE-20475).
- String peerId = new ReplicationQueueInfo(queue).getPeerId();
+ String peerId = queueId.getPeerId();
ReplicationPeerImpl oldPeer = replicationPeers.getPeer(peerId);
if (oldPeer == null) {
LOG.info("Not transferring queue since the replication peer {} for queue {} does not exist",
- peerId, queue);
+ peerId, queueId);
return;
}
- Pair<String, SortedSet<String>> claimedQueue;
+ Map<String, ReplicationGroupOffset> offsets;
try {
- claimedQueue = queueStorage.claimQueue(deadRS, queue, server.getServerName());
+ offsets = queueStorage.claimQueue(queueId, server.getServerName());
} catch (ReplicationException e) {
- LOG.error(
- "ReplicationException: cannot claim dead region ({})'s " + "replication queue. Znode : ({})"
- + " Possible solution: check if znode size exceeds jute.maxBuffer value. "
- + " If so, increase it for both client and server side.",
- deadRS, queueStorage.getRsNode(deadRS), e);
+ LOG.error("ReplicationException: cannot claim dead region ({})'s replication queue",
+ queueId.getServerName(), e);
server.abort("Failed to claim queue from dead regionserver.", e);
return;
}
- if (claimedQueue.getSecond().isEmpty()) {
+ if (offsets.isEmpty()) {
+ // someone else claimed the queue
return;
}
- String queueId = claimedQueue.getFirst();
- Set<String> walsSet = claimedQueue.getSecond();
+ ServerName sourceRS = queueId.getServerWALsBelongTo();
+ ReplicationQueueId claimedQueueId = queueId.claim(server.getServerName());
ReplicationPeerImpl peer = replicationPeers.getPeer(peerId);
if (peer == null || peer != oldPeer) {
- LOG.warn("Skipping failover for peer {} of node {}, peer is null", peerId, deadRS);
- abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), queueId));
+ LOG.warn("Skipping failover for peer {} of node {}, peer is null", peerId, sourceRS);
+ deleteQueue(claimedQueueId);
return;
}
- if (
- server instanceof ReplicationSyncUp.DummyServer
- && peer.getPeerState().equals(PeerState.DISABLED)
- ) {
- LOG.warn(
- "Peer {} is disabled. ReplicationSyncUp tool will skip " + "replicating data to this peer.",
- peerId);
- return;
- }
-
ReplicationSourceInterface src;
try {
- src = createSource(queueId, peer);
+ src =
+ createSource(new ReplicationQueueData(claimedQueueId, ImmutableMap.copyOf(offsets)), peer);
} catch (IOException e) {
- LOG.error("Can not create replication source for peer {} and queue {}", peerId, queueId, e);
+ LOG.error("Can not create replication source for peer {} and queue {}", peerId,
+ claimedQueueId, e);
server.abort("Failed to create replication source after claiming queue.", e);
return;
}
+ List<Path> walFiles;
+ try {
+ walFiles = AbstractFSWALProvider.getArchivedWALFiles(conf, sourceRS,
+ URLEncoder.encode(sourceRS.toString(), StandardCharsets.UTF_8.name()));
+ } catch (IOException e) {
+ LOG.error("Can not list all wal files for peer {} and queue {}", peerId, queueId, e);
+ server.abort("Can not list all wal files after claiming queue.", e);
+ return;
+ }
+ PriorityQueue<Path> walFilesPQ = new PriorityQueue<>(
+ Comparator.<Path, Long> comparing(p -> AbstractFSWALProvider.getTimestamp(p.getName()))
+ .thenComparing(Path::getName));
+ // sort the wal files and also filter out replicated files
+ for (Path file : walFiles) {
+ String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(file.getName());
+ ReplicationGroupOffset groupOffset = offsets.get(walGroupId);
+ if (shouldReplicate(groupOffset, file.getName())) {
+ walFilesPQ.add(file);
+ } else {
+ LOG.debug("Skip enqueuing log {} because it is before the start offset {}", file.getName(),
+ groupOffset);
+ }
+ walFilesPQ.add(file);
+ }
+ // the method is a bit long, so assign it to null here to avoid later we reuse it again by
+ // mistake, we should use the sorted walFilesPQ instead
+ walFiles = null;
// synchronized on oldsources to avoid adding recovered source for the to-be-removed peer
synchronized (oldsources) {
peer = replicationPeers.getPeer(src.getPeerId());
if (peer == null || peer != oldPeer) {
src.terminate("Recovered queue doesn't belong to any current peer");
- deleteQueue(queueId);
+ deleteQueue(claimedQueueId);
return;
}
// Do not setup recovered queue if a sync replication peer is in STANDBY state, or is
@@ -881,26 +917,26 @@ public class ReplicationSourceManager {
|| stateAndNewState.getSecond().equals(SyncReplicationState.STANDBY)
) {
src.terminate("Sync replication peer is in STANDBY state");
- deleteQueue(queueId);
+ deleteQueue(claimedQueueId);
return;
}
}
// track sources in walsByIdRecoveredQueues
Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();
- walsByIdRecoveredQueues.put(queueId, walsByGroup);
- for (String wal : walsSet) {
- String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal);
+ walsByIdRecoveredQueues.put(claimedQueueId, walsByGroup);
+ for (Path wal : walFilesPQ) {
+ String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal.getName());
NavigableSet<String> wals = walsByGroup.get(walPrefix);
if (wals == null) {
wals = new TreeSet<>();
walsByGroup.put(walPrefix, wals);
}
- wals.add(wal);
+ wals.add(wal.getName());
}
oldsources.add(src);
- LOG.info("Added source for recovered queue {}", src.getQueueId());
- for (String wal : walsSet) {
- LOG.trace("Enqueueing log from recovered queue for source: " + src.getQueueId());
+ LOG.info("Added source for recovered queue {}", claimedQueueId);
+ for (Path wal : walFilesPQ) {
+ LOG.debug("Enqueueing log {} from recovered queue for source: {}", wal, claimedQueueId);
src.enqueueLog(new Path(oldLogDir, wal));
}
src.startup();
@@ -926,7 +962,9 @@ public class ReplicationSourceManager {
* Get a copy of the wals of the normal sources on this rs
* @return a sorted set of wal names
*/
- public Map<String, Map<String, NavigableSet<String>>> getWALs() {
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ public Map<ReplicationQueueId, Map<String, NavigableSet<String>>> getWALs() {
return Collections.unmodifiableMap(walsById);
}
@@ -934,7 +972,9 @@ public class ReplicationSourceManager {
* Get a copy of the wals of the recovered sources on this rs
* @return a sorted set of wal names
*/
- Map<String, Map<String, NavigableSet<String>>> getWalsByIdRecoveredQueues() {
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ Map<ReplicationQueueId, Map<String, NavigableSet<String>>> getWalsByIdRecoveredQueues() {
return Collections.unmodifiableMap(walsByIdRecoveredQueues);
}
@@ -962,16 +1002,6 @@ public class ReplicationSourceManager {
return this.sources.get(peerId);
}
- List<String> getAllQueues() throws IOException {
- List<String> allQueues = Collections.emptyList();
- try {
- allQueues = queueStorage.getAllQueues(server.getServerName());
- } catch (ReplicationException e) {
- throw new IOException(e);
- }
- return allQueues;
- }
-
int getSizeOfLatestPath() {
synchronized (latestPaths) {
return latestPaths.size();
@@ -1067,6 +1097,8 @@ public class ReplicationSourceManager {
return this.globalMetrics;
}
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
ReplicationQueueStorage getQueueStorage() {
return queueStorage;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java
index 16c54191c2b..9ecb71613b5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java
@@ -58,7 +58,7 @@ public class ReplicationSourceShipper extends Thread {
private final Configuration conf;
protected final String walGroupId;
protected final ReplicationSourceLogQueue logQueue;
- private final ReplicationSource source;
+ protected final ReplicationSource source;
// Last position in the log that we sent to ZooKeeper
// It will be accessed by the stats thread so make it volatile
@@ -313,10 +313,6 @@ public class ReplicationSourceShipper extends Thread {
this.entryReader = entryReader;
}
- long getStartPosition() {
- return 0;
- }
-
protected boolean isActive() {
return source.isSourceActive() && state == WorkerState.RUNNING && !isInterrupted();
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java
index 7337694addb..9380c6b6305 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java
@@ -43,11 +43,6 @@ class ReplicationSourceWALActionListener implements WALActionsListener {
this.manager = manager;
}
- @Override
- public void preLogRoll(Path oldPath, Path newPath) throws IOException {
- manager.preLogRoll(newPath);
- }
-
@Override
public void postLogRoll(Path oldPath, Path newPath) throws IOException {
manager.postLogRoll(newPath);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index daf9081234d..50ffd6df1af 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -80,17 +80,18 @@ public class ReplicationSyncUp extends Configured implements Tool {
// replication queues for the dead region servers first and then replicate the data out.
private void claimReplicationQueues(ZKWatcher zkw, ReplicationSourceManager mgr)
throws ReplicationException, KeeperException {
- List<ServerName> replicators = mgr.getQueueStorage().getListOfReplicators();
- Set<ServerName> liveRegionServers = getLiveRegionServers(zkw);
- for (ServerName sn : replicators) {
- if (!liveRegionServers.contains(sn)) {
- List<String> replicationQueues = mgr.getQueueStorage().getAllQueues(sn);
- System.out.println(sn + " is dead, claim its replication queues: " + replicationQueues);
- for (String queue : replicationQueues) {
- mgr.claimQueue(sn, queue);
- }
- }
- }
+ // TODO: reimplement this tool
+ // List<ServerName> replicators = mgr.getQueueStorage().getListOfReplicators();
+ // Set<ServerName> liveRegionServers = getLiveRegionServers(zkw);
+ // for (ServerName sn : replicators) {
+ // if (!liveRegionServers.contains(sn)) {
+ // List<String> replicationQueues = mgr.getQueueStorage().getAllQueues(sn);
+ // System.out.println(sn + " is dead, claim its replication queues: " + replicationQueues);
+ // for (String queue : replicationQueues) {
+ // mgr.claimQueue(sn, queue);
+ // }
+ // }
+ // }
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index ab8d9fcde6a..439c1adce04 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2558,7 +2558,7 @@ public class HBaseFsck extends Configured implements Closeable {
}
private void checkAndFixReplication() throws ReplicationException {
- ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, errors);
+ ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, connection, errors);
checker.checkUnDeletedQueues();
if (checker.hasUnDeletedQueues() && this.fixReplication) {
@@ -3831,7 +3831,7 @@ public class HBaseFsck extends Configured implements Closeable {
return;
}
ReplicationQueueStorage queueStorage =
- ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
+ ReplicationStorageFactory.getReplicationQueueStorage(connection, getConf());
List<ReplicationPeerDescription> peerDescriptions = admin.listReplicationPeers();
if (peerDescriptions != null && peerDescriptions.size() > 0) {
List<String> peers = peerDescriptions.stream()
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
index 234daef85b5..f4d12ba0326 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
@@ -25,9 +25,11 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.util.HbckErrorReporter;
@@ -46,16 +48,17 @@ public class ReplicationChecker {
private final HbckErrorReporter errorReporter;
// replicator with its queueIds for removed peers
- private Map<ServerName, List<String>> undeletedQueueIds = new HashMap<>();
+ private Map<ServerName, List<ReplicationQueueId>> undeletedQueueIds = new HashMap<>();
// replicator with its undeleted queueIds for removed peers in hfile-refs queue
private Set<String> undeletedHFileRefsPeerIds = new HashSet<>();
private final ReplicationPeerStorage peerStorage;
private final ReplicationQueueStorage queueStorage;
- public ReplicationChecker(Configuration conf, ZKWatcher zkw, HbckErrorReporter errorReporter) {
+ public ReplicationChecker(Configuration conf, ZKWatcher zkw, Connection conn,
+ HbckErrorReporter errorReporter) {
this.peerStorage = ReplicationStorageFactory.getReplicationPeerStorage(zkw, conf);
- this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
+ this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(conn, conf);
this.errorReporter = errorReporter;
}
@@ -64,19 +67,19 @@ public class ReplicationChecker {
.contains(HbckErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE);
}
- private Map<ServerName, List<String>> getUnDeletedQueues() throws ReplicationException {
- Map<ServerName, List<String>> undeletedQueues = new HashMap<>();
+ private Map<ServerName, List<ReplicationQueueId>> getUnDeletedQueues()
+ throws ReplicationException {
+ Map<ServerName, List<ReplicationQueueId>> undeletedQueues = new HashMap<>();
Set<String> peerIds = new HashSet<>(peerStorage.listPeerIds());
- for (ServerName replicator : queueStorage.getListOfReplicators()) {
- for (String queueId : queueStorage.getAllQueues(replicator)) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (!peerIds.contains(queueInfo.getPeerId())) {
- undeletedQueues.computeIfAbsent(replicator, key -> new ArrayList<>()).add(queueId);
- LOG.debug(
- "Undeleted replication queue for removed peer found: "
- + "[removedPeerId={}, replicator={}, queueId={}]",
- queueInfo.getPeerId(), replicator, queueId);
- }
+ for (ReplicationQueueData queueData : queueStorage.listAllQueues()) {
+ ReplicationQueueId queueId = queueData.getId();
+ if (!peerIds.contains(queueId.getPeerId())) {
+ undeletedQueues.computeIfAbsent(queueId.getServerName(), key -> new ArrayList<>())
+ .add(queueId);
+ LOG.debug(
+ "Undeleted replication queue for removed peer found: "
+ + "[removedPeerId={}, replicator={}, queueId={}]",
+ queueId.getPeerId(), queueId.getServerName(), queueId);
}
}
return undeletedQueues;
@@ -99,9 +102,8 @@ public class ReplicationChecker {
undeletedQueueIds = getUnDeletedQueues();
undeletedQueueIds.forEach((replicator, queueIds) -> {
queueIds.forEach(queueId -> {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
String msg = "Undeleted replication queue for removed peer found: "
- + String.format("[removedPeerId=%s, replicator=%s, queueId=%s]", queueInfo.getPeerId(),
+ + String.format("[removedPeerId=%s, replicator=%s, queueId=%s]", queueId.getPeerId(),
replicator, queueId);
errorReporter.reportError(HbckErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE, msg);
});
@@ -114,12 +116,12 @@ public class ReplicationChecker {
}
public void fixUnDeletedQueues() throws ReplicationException {
- for (Map.Entry<ServerName, List<String>> replicatorAndQueueIds : undeletedQueueIds.entrySet()) {
+ for (Map.Entry<ServerName, List<ReplicationQueueId>> replicatorAndQueueIds : undeletedQueueIds
+ .entrySet()) {
ServerName replicator = replicatorAndQueueIds.getKey();
- for (String queueId : replicatorAndQueueIds.getValue()) {
- queueStorage.removeQueue(replicator, queueId);
+ for (ReplicationQueueId queueId : replicatorAndQueueIds.getValue()) {
+ queueStorage.removeQueue(queueId);
}
- queueStorage.removeReplicatorIfQueueIsEmpty(replicator);
}
for (String peerId : undeletedHFileRefsPeerIds) {
queueStorage.removePeerFromHFileRefs(peerId);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
index 1c77d775610..f9331e25402 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.wal;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -28,6 +29,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
@@ -335,6 +337,44 @@ public abstract class AbstractFSWALProvider<T extends AbstractFSWAL<?>> implemen
return dirName.toString();
}
+ /**
+ * List all the old wal files for a dead region server.
+ * <p/>
+ * Initially added for supporting replication, where we need to get the wal files to replicate for
+ * a dead region server.
+ */
+ public static List<Path> getArchivedWALFiles(Configuration conf, ServerName serverName,
+ String logPrefix) throws IOException {
+ Path walRootDir = CommonFSUtils.getWALRootDir(conf);
+ FileSystem fs = walRootDir.getFileSystem(conf);
+ List<Path> archivedWalFiles = new ArrayList<>();
+ // list both the root old wal dir and the separate old wal dir, so we will not miss any files if
+ // the SEPARATE_OLDLOGDIR config is changed
+ Path oldWalDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ try {
+ for (FileStatus status : fs.listStatus(oldWalDir, p -> p.getName().startsWith(logPrefix))) {
+ if (status.isFile()) {
+ archivedWalFiles.add(status.getPath());
+ }
+ }
+ } catch (FileNotFoundException e) {
+ LOG.info("Old WAL dir {} not exists", oldWalDir);
+ return Collections.emptyList();
+ }
+ Path separatedOldWalDir = new Path(oldWalDir, serverName.toString());
+ try {
+ for (FileStatus status : fs.listStatus(separatedOldWalDir,
+ p -> p.getName().startsWith(logPrefix))) {
+ if (status.isFile()) {
+ archivedWalFiles.add(status.getPath());
+ }
+ }
+ } catch (FileNotFoundException e) {
+ LOG.info("Seprated old WAL dir {} not exists", separatedOldWalDir);
+ }
+ return archivedWalFiles;
+ }
+
/**
* Pulls a ServerName out of a Path generated according to our layout rules. In the below layouts,
* this method ignores the format of the logfile component. Current format: [base directory for
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
index e50b14aa3ff..167b000bece 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
@@ -41,12 +41,12 @@ import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfigBuilder;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint;
@@ -100,11 +100,9 @@ public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
} catch (Exception e) {
}
ReplicationQueueStorage queueStorage = ReplicationStorageFactory
- .getReplicationQueueStorage(TEST_UTIL.getZooKeeperWatcher(), TEST_UTIL.getConfiguration());
- for (ServerName serverName : queueStorage.getListOfReplicators()) {
- for (String queue : queueStorage.getAllQueues(serverName)) {
- queueStorage.removeQueue(serverName, queue);
- }
+ .getReplicationQueueStorage(TEST_UTIL.getConnection(), TEST_UTIL.getConfiguration());
+ for (ReplicationQueueData queueData : queueStorage.listAllQueues()) {
+ queueStorage.removeQueue(queueData.getId());
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
index e9e5f973cf8..1a0537bcbaf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.master.cleaner;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
@@ -45,8 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
@@ -62,13 +61,14 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+// revisit later after we implement new replication log cleaner
+@Ignore
@Category({ MasterTests.class, MediumTests.class })
public class TestLogsCleaner {
@@ -131,8 +131,8 @@ public class TestLogsCleaner {
HMaster.decorateMasterConfiguration(conf);
Server server = new DummyServer();
- ReplicationQueueStorage queueStorage =
- ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
+ ReplicationQueueStorage queueStorage = ReplicationStorageFactory
+ .getReplicationQueueStorage(ConnectionFactory.createConnection(conf), conf);
String fakeMachineName =
URLEncoder.encode(server.getServerName().toString(), StandardCharsets.UTF_8.name());
@@ -162,7 +162,7 @@ public class TestLogsCleaner {
// Case 4: put 3 WALs in ZK indicating that they are scheduled for replication so these
// files would pass TimeToLiveLogCleaner but would be rejected by ReplicationLogCleaner
if (i % (30 / 3) == 0) {
- queueStorage.addWAL(server.getServerName(), fakeMachineName, fileName.getName());
+ // queueStorage.addWAL(server.getServerName(), fakeMachineName, fileName.getName());
LOG.info("Replication log file: " + fileName);
}
}
@@ -222,20 +222,20 @@ public class TestLogsCleaner {
try {
faultyZK.init(false);
- ReplicationQueueStorage queueStorage =
- spy(ReplicationStorageFactory.getReplicationQueueStorage(faultyZK, conf));
- doAnswer(new Answer<Object>() {
- @Override
- public Object answer(InvocationOnMock invocation) throws Throwable {
- try {
- return invocation.callRealMethod();
- } catch (ReplicationException e) {
- LOG.debug("Caught Exception", e);
- getListOfReplicatorsFailed.set(true);
- throw e;
- }
- }
- }).when(queueStorage).getAllWALs();
+ ReplicationQueueStorage queueStorage = spy(ReplicationStorageFactory
+ .getReplicationQueueStorage(ConnectionFactory.createConnection(conf), conf));
+ // doAnswer(new Answer<Object>() {
+ // @Override
+ // public Object answer(InvocationOnMock invocation) throws Throwable {
+ // try {
+ // return invocation.callRealMethod();
+ // } catch (ReplicationException e) {
+ // LOG.debug("Caught Exception", e);
+ // getListOfReplicatorsFailed.set(true);
+ // throw e;
+ // }
+ // }
+ // }).when(queueStorage).getAllWALs();
cleaner.setConf(conf, faultyZK, queueStorage);
// should keep all files due to a ConnectionLossException getting the queues znodes
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
index 899d3eb4722..1d527f35bb0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
@@ -17,14 +17,12 @@
*/
package org.apache.hadoop.hbase.master.cleaner;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.spy;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -32,12 +30,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
@@ -49,25 +46,21 @@ import org.apache.hadoop.hbase.replication.SyncReplicationState;
import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.MockServer;
import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.data.Stat;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-
+// TODO: revisit later
+@Ignore
@Category({ MasterTests.class, SmallTests.class })
public class TestReplicationHFileCleaner {
@@ -87,13 +80,13 @@ public class TestReplicationHFileCleaner {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- TEST_UTIL.startMiniZKCluster();
+ TEST_UTIL.startMiniCluster();
server = new DummyServer();
conf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
HMaster.decorateMasterConfiguration(conf);
rp = ReplicationFactory.getReplicationPeers(server.getZooKeeper(), conf);
rp.init();
- rq = ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
+ rq = ReplicationStorageFactory.getReplicationQueueStorage(server.getConnection(), conf);
fs = FileSystem.get(conf);
}
@@ -108,7 +101,6 @@ public class TestReplicationHFileCleaner {
rp.getPeerStorage().addPeer(peerId,
ReplicationPeerConfig.newBuilder().setClusterKey(TEST_UTIL.getClusterKey()).build(), true,
SyncReplicationState.NONE);
- rq.addPeerToHFileRefs(peerId);
}
@After
@@ -183,47 +175,6 @@ public class TestReplicationHFileCleaner {
assertTrue(deletableFilesIterator.next().getPath().equals(deletablefile));
}
- /**
- * ReplicationHFileCleaner should be able to ride over ZooKeeper errors without aborting.
- */
- @Test
- public void testZooKeeperAbort() throws Exception {
- ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner();
-
- List<FileStatus> dummyFiles = Lists.newArrayList(
- new FileStatus(100, false, 3, 100, EnvironmentEdgeManager.currentTime(), new Path("hfile1")),
- new FileStatus(100, false, 3, 100, EnvironmentEdgeManager.currentTime(), new Path("hfile2")));
-
- FaultyZooKeeperWatcher faultyZK =
- new FaultyZooKeeperWatcher(conf, "testZooKeeperAbort-faulty", null);
- try {
- faultyZK.init();
- cleaner.setConf(conf, faultyZK);
- // should keep all files due to a ConnectionLossException getting the queues znodes
- Iterable<FileStatus> toDelete = cleaner.getDeletableFiles(dummyFiles);
- assertFalse(toDelete.iterator().hasNext());
- assertFalse(cleaner.isStopped());
- } finally {
- faultyZK.close();
- }
-
- // when zk is working both files should be returned
- cleaner = new ReplicationHFileCleaner();
- ZKWatcher zkw = new ZKWatcher(conf, "testZooKeeperAbort-normal", null);
- try {
- cleaner.setConf(conf, zkw);
- Iterable<FileStatus> filesToDelete = cleaner.getDeletableFiles(dummyFiles);
- Iterator<FileStatus> iter = filesToDelete.iterator();
- assertTrue(iter.hasNext());
- assertEquals(new Path("hfile1"), iter.next().getPath());
- assertTrue(iter.hasNext());
- assertEquals(new Path("hfile2"), iter.next().getPath());
- assertFalse(iter.hasNext());
- } finally {
- zkw.close();
- }
- }
-
static class DummyServer extends MockServer {
@Override
@@ -232,33 +183,12 @@ public class TestReplicationHFileCleaner {
}
@Override
- public ZKWatcher getZooKeeper() {
+ public Connection getConnection() {
try {
- return new ZKWatcher(getConfiguration(), "dummy server", this);
+ return TEST_UTIL.getConnection();
} catch (IOException e) {
- e.printStackTrace();
+ throw new UncheckedIOException(e);
}
- return null;
- }
- }
-
- static class FaultyZooKeeperWatcher extends ZKWatcher {
- private RecoverableZooKeeper zk;
-
- public FaultyZooKeeperWatcher(Configuration conf, String identifier, Abortable abortable)
- throws ZooKeeperConnectionException, IOException {
- super(conf, identifier, abortable);
- }
-
- public void init() throws Exception {
- this.zk = spy(super.getRecoverableZooKeeper());
- doThrow(new KeeperException.ConnectionLossException()).when(zk)
- .getData("/hbase/replication/hfile-refs", null, new Stat());
- }
-
- @Override
- public RecoverableZooKeeper getRecoverableZooKeeper() {
- return zk;
}
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplicationHFileRefs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplicationHFileRefs.java
index 70a6e88552b..787784c8ec4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplicationHFileRefs.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplicationHFileRefs.java
@@ -114,7 +114,7 @@ public class TestBulkLoadReplicationHFileRefs extends TestReplicationBase {
admin1 = UTIL1.getConnection().getAdmin();
admin2 = UTIL2.getConnection().getAdmin();
- queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(UTIL1.getZooKeeperWatcher(),
+ queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(UTIL1.getConnection(),
UTIL1.getConfiguration());
admin1.createNamespace(NamespaceDescriptor.create(REPLICATE_NAMESPACE).build());
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/regionreplication/TestStartupWithLegacyRegionReplicationEndpoint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/regionreplication/TestStartupWithLegacyRegionReplicationEndpoint.java
index 66eaff0493e..5af9edb8efc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/regionreplication/TestStartupWithLegacyRegionReplicationEndpoint.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/regionreplication/TestStartupWithLegacyRegionReplicationEndpoint.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.Collections;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
@@ -29,9 +30,13 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
@@ -57,6 +62,10 @@ public class TestStartupWithLegacyRegionReplicationEndpoint {
@BeforeClass
public static void setUp() throws Exception {
UTIL.startMiniCluster(1);
+ // add a peer to force initialize the replication storage
+ UTIL.getAdmin().addReplicationPeer("1", ReplicationPeerConfig.newBuilder()
+ .setClusterKey(UTIL.getZkCluster().getAddress().toString() + ":/1").build());
+ UTIL.getAdmin().removeReplicationPeer("1");
}
@AfterClass
@@ -66,40 +75,42 @@ public class TestStartupWithLegacyRegionReplicationEndpoint {
@Test
public void test() throws Exception {
+ String peerId = "legacy";
ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
.setClusterKey("127.0.0.1:2181:/hbase")
.setReplicationEndpointImpl(ReplicationUtils.LEGACY_REGION_REPLICATION_ENDPOINT_NAME).build();
SingleProcessHBaseCluster cluster = UTIL.getMiniHBaseCluster();
HMaster master = cluster.getMaster();
// can not use Admin.addPeer as it will fail with ClassNotFound
- master.getReplicationPeerManager().addPeer("legacy", peerConfig, true);
+ master.getReplicationPeerManager().addPeer(peerId, peerConfig, true);
// add a wal file to the queue
ServerName rsName = cluster.getRegionServer(0).getServerName();
- master.getReplicationPeerManager().getQueueStorage().addWAL(rsName,
- ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER, "test-wal-file");
+ master.getReplicationPeerManager().getQueueStorage().setOffset(
+ new ReplicationQueueId(rsName, ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER), "",
+ new ReplicationGroupOffset("test-wal-file", 0), Collections.emptyMap());
cluster.stopRegionServer(0);
RegionServerThread rst = cluster.startRegionServer();
// we should still have this peer
- assertNotNull(UTIL.getAdmin().getReplicationPeerConfig("legacy"));
+ assertNotNull(UTIL.getAdmin().getReplicationPeerConfig(peerId));
// but at RS side, we should not have this peer loaded as replication source
- assertTrue(rst.getRegionServer().getReplicationSourceService().getReplicationManager()
- .getSources().isEmpty());
+ assertTrue(
+ rst.getRegionServer().getReplicationSourceService().getReplicationManager().getSources()
+ .stream().map(ReplicationSourceInterface::getPeerId).noneMatch(p -> p.equals(peerId)));
UTIL.shutdownMiniHBaseCluster();
UTIL.restartHBaseCluster(1);
// now we should have removed the peer
assertThrows(ReplicationPeerNotFoundException.class,
() -> UTIL.getAdmin().getReplicationPeerConfig("legacy"));
- // at rs side, we should not have the peer this time, not only for not having replication source
- assertTrue(UTIL.getMiniHBaseCluster().getRegionServer(0).getReplicationSourceService()
- .getReplicationManager().getReplicationPeers().getAllPeerIds().isEmpty());
- // make sure that we can finish the SCP and delete the test-wal-file
+ // make sure that we can finish the SCP
UTIL.waitFor(15000,
() -> UTIL.getMiniHBaseCluster().getMaster().getProcedures().stream()
.filter(p -> p instanceof ServerCrashProcedure).map(p -> (ServerCrashProcedure) p)
.allMatch(Procedure::isSuccess));
- assertTrue(UTIL.getMiniHBaseCluster().getMaster().getReplicationPeerManager().getQueueStorage()
- .getAllQueues(rsName).isEmpty());
+ // the deletion is async, so wait until they get deleted
+ ReplicationPeerManager ppm = UTIL.getMiniHBaseCluster().getMaster().getReplicationPeerManager();
+ UTIL.waitFor(15000, () -> !ppm.getPeerStorage().listPeerIds().contains(peerId)
+ && ppm.getQueueStorage().listAllQueueIds(peerId, rsName).isEmpty());
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
index aa31e44b888..80847be3cf3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
@@ -41,7 +41,7 @@ public class ReplicationSourceDummy implements ReplicationSourceInterface {
private ReplicationSourceManager manager;
private ReplicationPeer replicationPeer;
- private String peerClusterId;
+ private ReplicationQueueId queueId;
private Path currentPath;
private MetricsSource metrics;
private WALFileLengthProvider walFileLengthProvider;
@@ -49,11 +49,11 @@ public class ReplicationSourceDummy implements ReplicationSourceInterface {
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueueStorage rq, ReplicationPeer rp, Server server, String peerClusterId,
+ ReplicationQueueStorage rq, ReplicationPeer rp, Server server, ReplicationQueueData queueData,
UUID clusterId, WALFileLengthProvider walFileLengthProvider, MetricsSource metrics)
throws IOException {
this.manager = manager;
- this.peerClusterId = peerClusterId;
+ this.queueId = queueData.getId();
this.metrics = metrics;
this.walFileLengthProvider = walFileLengthProvider;
this.replicationPeer = rp;
@@ -100,14 +100,13 @@ public class ReplicationSourceDummy implements ReplicationSourceInterface {
}
@Override
- public String getQueueId() {
- return peerClusterId;
+ public ReplicationQueueId getQueueId() {
+ return queueId;
}
@Override
public String getPeerId() {
- String[] parts = peerClusterId.split("-", 2);
- return parts.length != 1 ? parts[0] : peerClusterId;
+ return queueId.getPeerId();
}
@Override
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestAddToSerialReplicationPeer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestAddToSerialReplicationPeer.java
index 229da6b0712..5e764ebb0ef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestAddToSerialReplicationPeer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestAddToSerialReplicationPeer.java
@@ -79,8 +79,9 @@ public class TestAddToSerialReplicationPeer extends SerialReplicationTestBase {
ReplicationSourceManager manager =
((Replication) rs.getReplicationSourceService()).getReplicationManager();
// Make sure replication moves to the new file.
- return (manager.getWALs().get(PEER_ID).get(logPrefix).size() == 1)
- && !oldWalName.equals(manager.getWALs().get(PEER_ID).get(logPrefix).first());
+ ReplicationQueueId queueId = new ReplicationQueueId(rs.getServerName(), PEER_ID);
+ return (manager.getWALs().get(queueId).get(logPrefix).size() == 1)
+ && !oldWalName.equals(manager.getWALs().get(queueId).get(logPrefix).first());
}
@Override
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java
index b8718f3526b..a12081a7636 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionServerList;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
-import org.apache.hadoop.hbase.master.replication.ClaimReplicationQueuesProcedure;
+import org.apache.hadoop.hbase.master.replication.AssignReplicationQueuesProcedure;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
@@ -47,7 +47,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.Procedu
/**
* In HBASE-26029, we reimplement the claim queue operation with proc-v2 and make it a step in SCP,
- * this is a UT to make sure the {@link ClaimReplicationQueuesProcedure} works correctly.
+ * this is a UT to make sure the {@link AssignReplicationQueuesProcedure} works correctly.
*/
@Category({ ReplicationTests.class, LargeTests.class })
public class TestClaimReplicationQueue extends TestReplicationBase {
@@ -77,7 +77,7 @@ public class TestClaimReplicationQueue extends TestReplicationBase {
// return no region server to make the procedure hang
if (EMPTY) {
for (StackTraceElement e : Thread.currentThread().getStackTrace()) {
- if (e.getClassName().equals(ClaimReplicationQueuesProcedure.class.getName())) {
+ if (e.getClassName().equals(AssignReplicationQueuesProcedure.class.getName())) {
return Collections.emptyList();
}
}
@@ -149,7 +149,7 @@ public class TestClaimReplicationQueue extends TestReplicationBase {
HMaster master = UTIL1.getMiniHBaseCluster().getMaster();
UTIL1.waitFor(30000,
() -> master.getProcedures().stream()
- .filter(p -> p instanceof ClaimReplicationQueuesProcedure)
+ .filter(p -> p instanceof AssignReplicationQueuesProcedure)
.anyMatch(p -> p.getState() == ProcedureState.WAITING_TIMEOUT));
hbaseAdmin.enableReplicationPeer(PEER_ID2);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplicationWithBulkLoadedData.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplicationWithBulkLoadedData.java
index 180991c4a7f..5fc48b2d729 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplicationWithBulkLoadedData.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplicationWithBulkLoadedData.java
@@ -259,8 +259,8 @@ public final class TestNamespaceReplicationWithBulkLoadedData extends TestBulkLo
MiniZooKeeperCluster zkCluster = UTIL1.getZkCluster();
ZKWatcher watcher = new ZKWatcher(UTIL1.getConfiguration(), "TestZnodeHFiles-refs", null);
RecoverableZooKeeper zk = RecoverableZooKeeper.connect(UTIL1.getConfiguration(), watcher);
- ZKReplicationQueueStorage replicationQueueStorage =
- new ZKReplicationQueueStorage(watcher, UTIL1.getConfiguration());
+ ReplicationQueueStorage replicationQueueStorage = ReplicationStorageFactory
+ .getReplicationQueueStorage(UTIL1.getConnection(), UTIL1.getConfiguration());
Set<String> hfiles = replicationQueueStorage.getAllHFileRefs();
assertTrue(hfiles.isEmpty());
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDroppedTables.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDroppedTables.java
index 7df9a8e6bf7..f38ae7b9cc8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDroppedTables.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationDroppedTables.java
@@ -21,6 +21,7 @@ import static org.apache.hadoop.hbase.replication.regionserver.HBaseInterCluster
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.NamespaceDescriptor;
@@ -59,6 +60,8 @@ public class TestReplicationDroppedTables extends TestReplicationBase {
private static final Logger LOG = LoggerFactory.getLogger(TestReplicationDroppedTables.class);
private static final int ROWS_COUNT = 1000;
+ private byte[] value;
+
@Before
public void setUpBase() throws Exception {
// Starting and stopping replication can make us miss new logs,
@@ -95,12 +98,17 @@ public class TestReplicationDroppedTables extends TestReplicationBase {
break;
}
}
- // Set the max request size to a tiny 10K for dividing the replication WAL entries into multiple
+ // Set the max request size to a tiny 64K for dividing the replication WAL entries into multiple
// batches. the default max request size is 256M, so all replication entries are in a batch, but
// when replicate at sink side, it'll apply to rs group by table name, so the WAL of test table
// may apply first, and then test_dropped table, and we will believe that the replication is not
// got stuck (HBASE-20475).
- CONF1.setInt(RpcServer.MAX_REQUEST_SIZE, 10 * 1024);
+ // we used to use 10K but the regionServerReport is greater than this limit in this test which
+ // makes this test fail, increase to 64K
+ CONF1.setInt(RpcServer.MAX_REQUEST_SIZE, 64 * 1024);
+ // set a large value size to make sure we will split the replication to several batches
+ value = new byte[4096];
+ ThreadLocalRandom.current().nextBytes(value);
}
@Test
@@ -171,13 +179,13 @@ public class TestReplicationDroppedTables extends TestReplicationBase {
try (Table droppedTable = connection1.getTable(tablename)) {
byte[] rowKey = Bytes.toBytes(0 + " put on table to be dropped");
Put put = new Put(rowKey);
- put.addColumn(familyName, row, row);
+ put.addColumn(familyName, row, value);
droppedTable.put(put);
}
try (Table table1 = connection1.getTable(tableName)) {
for (int i = 0; i < ROWS_COUNT; i++) {
- Put put = new Put(generateRowKey(i)).addColumn(famName, row, row);
+ Put put = new Put(generateRowKey(i)).addColumn(famName, row, value);
table1.put(put);
}
}
@@ -243,13 +251,13 @@ public class TestReplicationDroppedTables extends TestReplicationBase {
try (Table droppedTable = connection1.getTable(tablename)) {
byte[] rowKey = Bytes.toBytes(0 + " put on table to be dropped");
Put put = new Put(rowKey);
- put.addColumn(familyName, row, row);
+ put.addColumn(familyName, row, value);
droppedTable.put(put);
}
try (Table table1 = connection1.getTable(tableName)) {
for (int i = 0; i < ROWS_COUNT; i++) {
- Put put = new Put(generateRowKey(i)).addColumn(famName, row, row);
+ Put put = new Put(generateRowKey(i)).addColumn(famName, row, value);
table1.put(put);
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java
index 63cbfe3119c..67546febab7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java
@@ -332,7 +332,6 @@ public class TestReplicationEmptyWALRecovery extends TestReplicationBase {
for (int i = 0; i < numRs; i++) {
HRegionServer hrs = UTIL1.getHBaseCluster().getRegionServer(i);
Replication replicationService = (Replication) hrs.getReplicationSourceService();
- replicationService.getReplicationManager().preLogRoll(emptyWalPaths.get(i));
replicationService.getReplicationManager().postLogRoll(emptyWalPaths.get(i));
RegionInfo regionInfo =
UTIL1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
index bf65d4db82e..7a89af15902 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
@@ -32,11 +32,14 @@ import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+// revisit later when we implement the new ReplicationSyncUpTool
+@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpTool extends TestReplicationSyncUpToolBase {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java
index fbf8ac6b3c9..b5de8e6324f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java
@@ -45,11 +45,14 @@ import org.apache.hadoop.hbase.tool.BulkLoadHFiles;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.HFileTestUtil;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+//revisit later when we implement the new ReplicationSyncUpTool
+@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpToolWithBulkLoadedData extends TestReplicationSyncUpToolBase {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java
index 1295ea14abc..6906db4cd46 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java
@@ -32,9 +32,12 @@ import org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.junit.Before;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+// revisit later when we reviewing the implementation for serial replication
+@Ignore
@Category({ ReplicationTests.class, MediumTests.class })
public class TestSerialReplicationFailover extends SerialReplicationTestBase {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestTableReplicationQueueStorage.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestTableReplicationQueueStorage.java
new file mode 100644
index 00000000000..4148c1c1a2c
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestTableReplicationQueueStorage.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNameTestRule;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.zookeeper.KeeperException;
+import org.hamcrest.Matchers;
+import org.hamcrest.collection.IsEmptyCollection;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+
+@Category({ ReplicationTests.class, MediumTests.class })
+public class TestTableReplicationQueueStorage {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestTableReplicationQueueStorage.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestTableReplicationQueueStorage.class);
+
+ private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
+
+ @Rule
+ public TableNameTestRule tableNameRule = new TableNameTestRule();
+
+ private TableReplicationQueueStorage storage;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UTIL.startMiniCluster();
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void setUpBeforeTest() throws Exception {
+ TableName tableName = tableNameRule.getTableName();
+ TableDescriptor td = ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName);
+ UTIL.getAdmin().createTable(td);
+ UTIL.waitTableAvailable(tableName);
+ storage = new TableReplicationQueueStorage(UTIL.getConnection(), tableName);
+ }
+
+ private ServerName getServerName(int i) {
+ return ServerName.valueOf("127.0.0.1", 8000 + i, 10000 + i);
+ }
+
+ private String getFileName(String base, int i) {
+ return String.format(base + "-%04d", i);
+ }
+
+ @Test
+ public void testReplicator() throws ReplicationException {
+ assertTrue(storage.listAllReplicators().isEmpty());
+ String peerId = "1";
+ for (int i = 0; i < 10; i++) {
+ ReplicationQueueId queueId = new ReplicationQueueId(getServerName(i), peerId);
+ storage.setOffset(queueId, "group-" + i, new ReplicationGroupOffset("file-" + i, i * 100),
+ Collections.emptyMap());
+ }
+ List<ServerName> replicators = storage.listAllReplicators();
+ assertEquals(10, replicators.size());
+ for (int i = 0; i < 10; i++) {
+ assertThat(replicators, hasItem(getServerName(i)));
+ }
+ for (int i = 0; i < 5; i++) {
+ ReplicationQueueId queueId = new ReplicationQueueId(getServerName(i), peerId);
+ storage.removeQueue(queueId);
+ }
+ replicators = storage.listAllReplicators();
+ assertEquals(5, replicators.size());
+ for (int i = 0; i < 5; i++) {
+ assertThat(replicators, not(hasItem(getServerName(i))));
+ }
+ for (int i = 5; i < 10; i++) {
+ assertThat(replicators, hasItem(getServerName(i)));
+ }
+ }
+
+ @Test
+ public void testGetSetOffset() {
+
+ }
+
+ private void assertQueueId(String peerId, ServerName serverName, ReplicationQueueId queueId) {
+ assertEquals(peerId, queueId.getPeerId());
+ assertEquals(serverName, queueId.getServerName());
+ assertFalse(queueId.getSourceServerName().isPresent());
+ }
+
+ @Test
+ public void testPersistLogPositionAndSeqIdAtomically() throws Exception {
+ ServerName serverName1 = ServerName.valueOf("127.0.0.1", 8000, 10000);
+ assertTrue(storage.listAllQueueIds(serverName1).isEmpty());
+ String peerId1 = "1";
+ String region0 = "6b2c8f8555335cc9af74455b94516cbe";
+ String region1 = "6ecd2e9e010499f8ddef97ee8f70834f";
+
+ for (int i = 0; i < 10; i++) {
+ ReplicationQueueId queueId = new ReplicationQueueId(serverName1, peerId1);
+ assertTrue(storage.getOffsets(queueId).isEmpty());
+ }
+ assertEquals(HConstants.NO_SEQNUM, storage.getLastSequenceId(region0, peerId1));
+ assertEquals(HConstants.NO_SEQNUM, storage.getLastSequenceId(region1, peerId1));
+
+ for (int i = 0; i < 10; i++) {
+ ReplicationQueueId queueId = new ReplicationQueueId(serverName1, peerId1);
+ storage.setOffset(queueId, "group1-" + i,
+ new ReplicationGroupOffset(getFileName("file1", i), (i + 1) * 100),
+ ImmutableMap.of(region0, i * 100L, region1, (i + 1) * 100L));
+ }
+
+ List<ReplicationQueueId> queueIds = storage.listAllQueueIds(serverName1);
+ assertEquals(1, queueIds.size());
+ assertQueueId(peerId1, serverName1, queueIds.get(0));
+
+ Map<String, ReplicationGroupOffset> offsets =
+ storage.getOffsets(new ReplicationQueueId(serverName1, peerId1));
+ for (int i = 0; i < 10; i++) {
+ ReplicationGroupOffset offset = offsets.get("group1-" + i);
+ assertEquals(getFileName("file1", i), offset.getWal());
+ assertEquals((i + 1) * 100, offset.getOffset());
+ }
+ assertEquals(900L, storage.getLastSequenceId(region0, peerId1));
+ assertEquals(1000L, storage.getLastSequenceId(region1, peerId1));
+
+ // Try to decrease the last pushed id by setWALPosition method.
+ storage.setOffset(new ReplicationQueueId(serverName1, peerId1), "group1-0",
+ new ReplicationGroupOffset(getFileName("file1", 0), 11 * 100),
+ ImmutableMap.of(region0, 899L, region1, 1001L));
+ assertEquals(900L, storage.getLastSequenceId(region0, peerId1));
+ assertEquals(1001L, storage.getLastSequenceId(region1, peerId1));
+ }
+
+ private void assertGroupOffset(String wal, long offset, ReplicationGroupOffset groupOffset) {
+ assertEquals(wal, groupOffset.getWal());
+ assertEquals(offset, groupOffset.getOffset());
+ }
+
+ @Test
+ public void testClaimQueue() throws Exception {
+ String peerId = "1";
+ ServerName serverName1 = getServerName(1);
+ ReplicationQueueId queueId = new ReplicationQueueId(serverName1, peerId);
+ for (int i = 0; i < 10; i++) {
+ storage.setOffset(queueId, "group-" + i, new ReplicationGroupOffset("wal-" + i, i),
+ Collections.emptyMap());
+ }
+
+ ServerName serverName2 = getServerName(2);
+ Map<String, ReplicationGroupOffset> offsets2 = storage.claimQueue(queueId, serverName2);
+ assertEquals(10, offsets2.size());
+ for (int i = 0; i < 10; i++) {
+ assertGroupOffset("wal-" + i, i, offsets2.get("group-" + i));
+ }
+ ReplicationQueueId claimedQueueId2 = new ReplicationQueueId(serverName2, peerId, serverName1);
+ assertThat(storage.listAllQueueIds(peerId, serverName1), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId, serverName2),
+ Matchers.<List<ReplicationQueueId>> both(hasItem(claimedQueueId2)).and(hasSize(1)));
+ offsets2 = storage.getOffsets(claimedQueueId2);
+ assertEquals(10, offsets2.size());
+ for (int i = 0; i < 10; i++) {
+ assertGroupOffset("wal-" + i, i, offsets2.get("group-" + i));
+ }
+
+ ServerName serverName3 = getServerName(3);
+ Map<String, ReplicationGroupOffset> offsets3 = storage.claimQueue(claimedQueueId2, serverName3);
+ assertEquals(10, offsets3.size());
+ for (int i = 0; i < 10; i++) {
+ assertGroupOffset("wal-" + i, i, offsets3.get("group-" + i));
+ }
+ ReplicationQueueId claimedQueueId3 = new ReplicationQueueId(serverName3, peerId, serverName1);
+ assertThat(storage.listAllQueueIds(peerId, serverName1), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId, serverName2), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId, serverName3),
+ Matchers.<List<ReplicationQueueId>> both(hasItem(claimedQueueId3)).and(hasSize(1)));
+ offsets3 = storage.getOffsets(claimedQueueId3);
+ assertEquals(10, offsets3.size());
+ for (int i = 0; i < 10; i++) {
+ assertGroupOffset("wal-" + i, i, offsets3.get("group-" + i));
+ }
+ storage.removeQueue(claimedQueueId3);
+ assertThat(storage.listAllQueueIds(peerId), IsEmptyCollection.empty());
+ }
+
+ @Test
+ public void testClaimQueueMultiThread() throws Exception {
+ String peerId = "3";
+ String walGroup = "group";
+ ReplicationGroupOffset groupOffset = new ReplicationGroupOffset("wal", 123);
+ ServerName sourceServerName = getServerName(100);
+ ReplicationQueueId queueId = new ReplicationQueueId(sourceServerName, peerId);
+ storage.setOffset(queueId, walGroup, groupOffset, Collections.emptyMap());
+ List<ServerName> serverNames =
+ IntStream.range(0, 10).mapToObj(this::getServerName).collect(Collectors.toList());
+ for (int i = 0; i < 10; i++) {
+ final ReplicationQueueId toClaim = queueId;
+ List<Thread> threads = new ArrayList<>();
+ Map<ServerName, Map<String, ReplicationGroupOffset>> claimed = new ConcurrentHashMap<>();
+ Set<ServerName> failed = ConcurrentHashMap.newKeySet();
+ for (ServerName serverName : serverNames) {
+ if (serverName.equals(queueId.getServerName())) {
+ continue;
+ }
+ threads.add(new Thread("Claim-" + i + "-" + serverName) {
+
+ @Override
+ public void run() {
+ try {
+ Map<String, ReplicationGroupOffset> offsets = storage.claimQueue(toClaim, serverName);
+ if (!offsets.isEmpty()) {
+ claimed.put(serverName, offsets);
+ }
+ } catch (ReplicationException e) {
+ LOG.error("failed to claim queue", e);
+ failed.add(serverName);
+ }
+ }
+ });
+ }
+ LOG.info("Claim round {}, there are {} threads to claim {}", i, threads.size(), toClaim);
+ for (Thread thread : threads) {
+ thread.start();
+ }
+ for (Thread thread : threads) {
+ thread.join(30000);
+ assertFalse(thread.isAlive());
+ }
+ LOG.info("Finish claim round {}, claimed={}, failed={}", i, claimed, failed);
+ assertThat(failed, IsEmptyCollection.empty());
+ assertEquals(1, claimed.size());
+ Map<String, ReplicationGroupOffset> offsets = Iterables.getOnlyElement(claimed.values());
+ assertEquals(1, offsets.size());
+ assertGroupOffset("wal", 123, offsets.get("group"));
+ queueId = new ReplicationQueueId(Iterables.getOnlyElement(claimed.keySet()), peerId,
+ sourceServerName);
+ assertThat(storage.listAllQueueIds(peerId),
+ Matchers.<List<ReplicationQueueId>> both(hasItem(queueId)).and(hasSize(1)));
+ }
+ }
+
+ @Test
+ public void testListRemovePeerAllQueues() throws Exception {
+ String peerId1 = "1";
+ String peerId2 = "2";
+ for (int i = 0; i < 100; i++) {
+ ServerName serverName = getServerName(i);
+ String group = "group";
+ ReplicationGroupOffset offset = new ReplicationGroupOffset("wal", i);
+ ReplicationQueueId queueId1 = new ReplicationQueueId(serverName, peerId1);
+ ReplicationQueueId queueId2 = new ReplicationQueueId(serverName, peerId2);
+ storage.setOffset(queueId1, group, offset, Collections.emptyMap());
+ storage.setOffset(queueId2, group, offset, Collections.emptyMap());
+ }
+ List<ReplicationQueueData> queueDatas = storage.listAllQueues();
+ assertThat(queueDatas, hasSize(200));
+ for (int i = 0; i < 100; i++) {
+ ReplicationQueueData peerId1Data = queueDatas.get(i);
+ ReplicationQueueData peerId2Data = queueDatas.get(i + 100);
+ ServerName serverName = getServerName(i);
+ assertEquals(new ReplicationQueueId(serverName, peerId1), peerId1Data.getId());
+ assertEquals(new ReplicationQueueId(serverName, peerId2), peerId2Data.getId());
+ assertEquals(1, peerId1Data.getOffsets().size());
+ assertEquals(1, peerId2Data.getOffsets().size());
+ assertGroupOffset("wal", i, peerId1Data.getOffsets().get("group"));
+ assertGroupOffset("wal", i, peerId2Data.getOffsets().get("group"));
+ }
+ List<ReplicationQueueId> queueIds1 = storage.listAllQueueIds(peerId1);
+ assertThat(queueIds1, hasSize(100));
+ for (int i = 0; i < 100; i++) {
+ ServerName serverName = getServerName(i);
+ assertEquals(new ReplicationQueueId(serverName, peerId1), queueIds1.get(i));
+ }
+ List<ReplicationQueueId> queueIds2 = storage.listAllQueueIds(peerId2);
+ assertThat(queueIds2, hasSize(100));
+ for (int i = 0; i < 100; i++) {
+ ServerName serverName = getServerName(i);
+ assertEquals(new ReplicationQueueId(serverName, peerId2), queueIds2.get(i));
+ }
+
+ storage.removeAllQueues(peerId1);
+ assertThat(storage.listAllQueues(), hasSize(100));
+ assertThat(storage.listAllQueueIds(peerId1), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId2), hasSize(100));
+
+ storage.removeAllQueues(peerId2);
+ assertThat(storage.listAllQueues(), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId1), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId2), IsEmptyCollection.empty());
+ }
+
+ @Test
+ public void testRemoveAllLastPushedSeqIdsForPeer() throws Exception {
+ String peerId = "1";
+ String peerIdToDelete = "2";
+ for (int i = 0; i < 100; i++) {
+ String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
+ storage.setLastSequenceIds(peerId, ImmutableMap.of(encodedRegionName, (long) i));
+ storage.setLastSequenceIds(peerIdToDelete, ImmutableMap.of(encodedRegionName, (long) i));
+ }
+ for (int i = 0; i < 100; i++) {
+ String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
+ assertEquals(i, storage.getLastSequenceId(encodedRegionName, peerId));
+ assertEquals(i, storage.getLastSequenceId(encodedRegionName, peerIdToDelete));
+ }
+ storage.removeLastSequenceIds(peerIdToDelete);
+ for (int i = 0; i < 100; i++) {
+ String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
+ assertEquals(i, storage.getLastSequenceId(encodedRegionName, peerId));
+ assertEquals(HConstants.NO_SEQNUM,
+ storage.getLastSequenceId(encodedRegionName, peerIdToDelete));
+ }
+ }
+
+ @Test
+ public void testHfileRefsReplicationQueues() throws ReplicationException, KeeperException {
+ String peerId1 = "1";
+
+ List<Pair<Path, Path>> files1 = new ArrayList<>(3);
+ files1.add(new Pair<>(null, new Path("file_1")));
+ files1.add(new Pair<>(null, new Path("file_2")));
+ files1.add(new Pair<>(null, new Path("file_3")));
+ assertTrue(storage.getReplicableHFiles(peerId1).isEmpty());
+ assertEquals(0, storage.getAllPeersFromHFileRefsQueue().size());
+
+ storage.addHFileRefs(peerId1, files1);
+ assertEquals(1, storage.getAllPeersFromHFileRefsQueue().size());
+ assertEquals(3, storage.getReplicableHFiles(peerId1).size());
+ List<String> hfiles2 = new ArrayList<>(files1.size());
+ for (Pair<Path, Path> p : files1) {
+ hfiles2.add(p.getSecond().getName());
+ }
+ String removedString = hfiles2.remove(0);
+ storage.removeHFileRefs(peerId1, hfiles2);
+ assertEquals(1, storage.getReplicableHFiles(peerId1).size());
+ hfiles2 = new ArrayList<>(1);
+ hfiles2.add(removedString);
+ storage.removeHFileRefs(peerId1, hfiles2);
+ assertEquals(0, storage.getReplicableHFiles(peerId1).size());
+ }
+
+ @Test
+ public void testRemovePeerForHFileRefs() throws ReplicationException, KeeperException {
+ String peerId1 = "1";
+ String peerId2 = "2";
+
+ List<Pair<Path, Path>> files1 = new ArrayList<>(3);
+ files1.add(new Pair<>(null, new Path("file_1")));
+ files1.add(new Pair<>(null, new Path("file_2")));
+ files1.add(new Pair<>(null, new Path("file_3")));
+ storage.addHFileRefs(peerId1, files1);
+ storage.addHFileRefs(peerId2, files1);
+ assertEquals(2, storage.getAllPeersFromHFileRefsQueue().size());
+ assertEquals(3, storage.getReplicableHFiles(peerId1).size());
+ assertEquals(3, storage.getReplicableHFiles(peerId2).size());
+
+ storage.removePeerFromHFileRefs(peerId1);
+ assertEquals(1, storage.getAllPeersFromHFileRefsQueue().size());
+ assertTrue(storage.getReplicableHFiles(peerId1).isEmpty());
+ assertEquals(3, storage.getReplicableHFiles(peerId2).size());
+
+ storage.removePeerFromHFileRefs(peerId2);
+ assertEquals(0, storage.getAllPeersFromHFileRefsQueue().size());
+ assertTrue(storage.getReplicableHFiles(peerId2).isEmpty());
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java
index 83cd41773ca..28779be4399 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java
@@ -25,8 +25,11 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.wal.RegionGroupingProvider;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.experimental.categories.Category;
+//revisit later when we implement the new ReplicationSyncUpTool
+@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpToolWithMultipleAsyncWAL extends TestReplicationSyncUpTool {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java
index 673b841430e..f495f433bc9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java
@@ -25,8 +25,11 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.wal.RegionGroupingProvider;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.experimental.categories.Category;
+//revisit later when we implement the new ReplicationSyncUpTool
+@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpToolWithMultipleWAL extends TestReplicationSyncUpTool {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java
index 0189d475575..8918f8422e1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java
@@ -35,9 +35,12 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+// TODO: revisit later
+@Ignore
@Category({ ReplicationTests.class, MediumTests.class })
public class TestDrainReplicationQueuesForStandBy extends SyncReplicationTestBase {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java
index d78a45ca6b9..3475ae5c192 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java
@@ -36,12 +36,15 @@ import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
/**
* Tests for DumpReplicationQueues tool
*/
+// TODO: reimplement
+@Ignore
@Category({ ReplicationTests.class, SmallTests.class })
public class TestDumpReplicationQueues {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java
index 78296b7cab3..edc7dd7079e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -58,7 +57,8 @@ import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.WALEntryFilter;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
@@ -80,6 +80,8 @@ import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
@Category({ ReplicationTests.class, MediumTests.class })
public class TestReplicationSource {
@@ -136,11 +138,13 @@ public class TestReplicationSource {
Mockito.when(manager.getTotalBufferUsed()).thenReturn(new AtomicLong());
Mockito.when(manager.getGlobalMetrics())
.thenReturn(mock(MetricsReplicationGlobalSourceSource.class));
- String queueId = "qid";
+
RegionServerServices rss =
TEST_UTIL.createMockRegionServerService(ServerName.parseServerName("a.b.c,1,1"));
- rs.init(conf, null, manager, null, mockPeer, rss, queueId, null, p -> OptionalLong.empty(),
- new MetricsSource(queueId));
+ ReplicationQueueId queueId = new ReplicationQueueId(rss.getServerName(), "qid");
+ rs.init(conf, null, manager, null, mockPeer, rss,
+ new ReplicationQueueData(queueId, ImmutableMap.of()), null, p -> OptionalLong.empty(),
+ new MetricsSource(queueId.toString()));
try {
rs.startup();
assertTrue(rs.isSourceActive());
@@ -174,11 +178,12 @@ public class TestReplicationSource {
Mockito.when(mockPeer.getPeerConfig()).thenReturn(peerConfig);
ReplicationSourceManager manager = Mockito.mock(ReplicationSourceManager.class);
Mockito.when(manager.getTotalBufferUsed()).thenReturn(new AtomicLong());
- String queueId = "qid";
RegionServerServices rss =
TEST_UTIL.createMockRegionServerService(ServerName.parseServerName("a.b.c,1,1"));
- rs.init(conf, null, manager, null, mockPeer, rss, queueId, uuid, p -> OptionalLong.empty(),
- new MetricsSource(queueId));
+ ReplicationQueueId queueId = new ReplicationQueueId(rss.getServerName(), "qid");
+ rs.init(conf, null, manager, null, mockPeer, rss,
+ new ReplicationQueueData(queueId, ImmutableMap.of()), uuid, p -> OptionalLong.empty(),
+ new MetricsSource(queueId.toString()));
try {
rs.startup();
TEST_UTIL.waitFor(30000, () -> rs.getWalEntryFilter() != null);
@@ -261,8 +266,11 @@ public class TestReplicationSource {
testConf.setInt("replication.source.maxretriesmultiplier", 1);
ReplicationSourceManager manager = Mockito.mock(ReplicationSourceManager.class);
Mockito.when(manager.getTotalBufferUsed()).thenReturn(new AtomicLong());
- source.init(testConf, null, manager, null, mockPeer, null, "testPeer", null,
- p -> OptionalLong.empty(), null);
+ ReplicationQueueId queueId =
+ new ReplicationQueueId(ServerName.valueOf("test,123,123"), "testPeer");
+ source.init(testConf, null, manager, null, mockPeer, null,
+ new ReplicationQueueData(queueId, ImmutableMap.of()), null, p -> OptionalLong.empty(),
+ null);
ExecutorService executor = Executors.newSingleThreadExecutor();
Future<?> future = executor.submit(() -> source.terminate("testing source termination"));
long sleepForRetries = testConf.getLong("replication.source.sleepforretries", 1000);
@@ -284,8 +292,11 @@ public class TestReplicationSource {
ReplicationPeer mockPeer = mock(ReplicationPeer.class);
Mockito.when(mockPeer.getPeerBandwidth()).thenReturn(0L);
Configuration testConf = HBaseConfiguration.create();
- source.init(testConf, null, mockManager, null, mockPeer, Mockito.mock(Server.class), "testPeer",
- null, p -> OptionalLong.empty(), mock(MetricsSource.class));
+ ReplicationQueueId queueId =
+ new ReplicationQueueId(ServerName.valueOf("test,123,123"), "testPeer");
+ source.init(testConf, null, mockManager, null, mockPeer, Mockito.mock(Server.class),
+ new ReplicationQueueData(queueId, ImmutableMap.of()), null, p -> OptionalLong.empty(),
+ mock(MetricsSource.class));
ReplicationSourceWALReader reader =
new ReplicationSourceWALReader(null, conf, null, 0, null, source, null);
ReplicationSourceShipper shipper = new ReplicationSourceShipper(conf, null, null, source);
@@ -488,35 +499,6 @@ public class TestReplicationSource {
}
- /**
- * Test HBASE-20497 Moved here from TestReplicationSource because doesn't need cluster.
- */
- @Test
- public void testRecoveredReplicationSourceShipperGetPosition() throws Exception {
- String walGroupId = "fake-wal-group-id";
- ServerName serverName = ServerName.valueOf("www.example.com", 12006, 1524679704418L);
- ServerName deadServer = ServerName.valueOf("www.deadServer.com", 12006, 1524679704419L);
- RecoveredReplicationSource source = mock(RecoveredReplicationSource.class);
- Server server = mock(Server.class);
- Mockito.when(server.getServerName()).thenReturn(serverName);
- Mockito.when(source.getServer()).thenReturn(server);
- Mockito.when(source.getServerWALsBelongTo()).thenReturn(deadServer);
- ReplicationQueueStorage storage = mock(ReplicationQueueStorage.class);
- Mockito.when(storage.getWALPosition(Mockito.eq(serverName), Mockito.any(), Mockito.any()))
- .thenReturn(1001L);
- Mockito.when(storage.getWALPosition(Mockito.eq(deadServer), Mockito.any(), Mockito.any()))
- .thenReturn(-1L);
- Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
- conf.setInt("replication.source.maxretriesmultiplier", -1);
- MetricsSource metricsSource = mock(MetricsSource.class);
- doNothing().when(metricsSource).incrSizeOfLogQueue();
- ReplicationSourceLogQueue logQueue = new ReplicationSourceLogQueue(conf, metricsSource, source);
- logQueue.enqueueLog(new Path("/www/html/test"), walGroupId);
- RecoveredReplicationSourceShipper shipper =
- new RecoveredReplicationSourceShipper(conf, walGroupId, logQueue, source, storage);
- assertEquals(1001L, shipper.getStartPosition());
- }
-
private RegionServerServices setupForAbortTests(ReplicationSource rs, Configuration conf,
String endpointName) throws IOException {
conf.setInt("replication.source.maxretriesmultiplier", 1);
@@ -531,11 +513,12 @@ public class TestReplicationSource {
Mockito.when(manager.getTotalBufferUsed()).thenReturn(new AtomicLong());
Mockito.when(manager.getGlobalMetrics())
.thenReturn(mock(MetricsReplicationGlobalSourceSource.class));
- String queueId = "qid";
RegionServerServices rss =
TEST_UTIL.createMockRegionServerService(ServerName.parseServerName("a.b.c,1,1"));
- rs.init(conf, null, manager, null, mockPeer, rss, queueId, null, p -> OptionalLong.empty(),
- new MetricsSource(queueId));
+ ReplicationQueueId queueId = new ReplicationQueueId(rss.getServerName(), "qid");
+ rs.init(conf, null, manager, null, mockPeer, rss,
+ new ReplicationQueueData(queueId, ImmutableMap.of()), null, p -> OptionalLong.empty(),
+ new MetricsSource(queueId.toString()));
return rss;
}
@@ -633,8 +616,8 @@ public class TestReplicationSource {
ManualEnvironmentEdge manualEdge = new ManualEnvironmentEdge();
EnvironmentEdgeManager.injectEdge(manualEdge);
- String id = "1";
- MetricsSource metrics = new MetricsSource(id);
+ String peerId = "1";
+ MetricsSource metrics = new MetricsSource(peerId);
Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
conf.setInt("replication.source.maxretriesmultiplier", 1);
ReplicationPeer mockPeer = Mockito.mock(ReplicationPeer.class);
@@ -650,16 +633,17 @@ public class TestReplicationSource {
.thenReturn(mock(MetricsReplicationGlobalSourceSource.class));
RegionServerServices rss =
TEST_UTIL.createMockRegionServerService(ServerName.parseServerName("a.b.c,1,1"));
-
+ ReplicationQueueId queueId = new ReplicationQueueId(rss.getServerName(), peerId);
ReplicationSource source = new ReplicationSource();
- source.init(conf, null, manager, null, mockPeer, rss, id, null, p -> OptionalLong.empty(),
+ source.init(conf, null, manager, null, mockPeer, rss,
+ new ReplicationQueueData(queueId, ImmutableMap.of()), null, p -> OptionalLong.empty(),
metrics);
final Path log1 = new Path(logDir, "log-walgroup-a.8");
manualEdge.setValue(10);
// Diff of current time (10) and log-walgroup-a.8 timestamp will be 2.
source.enqueueLog(log1);
- MetricsReplicationSourceSource metricsSource1 = getSourceMetrics(id);
+ MetricsReplicationSourceSource metricsSource1 = getSourceMetrics(peerId);
assertEquals(2, metricsSource1.getOldestWalAge());
final Path log2 = new Path(logDir, "log-walgroup-b.4");
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index dd989293ff5..6aba327d791 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -17,578 +17,326 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
-import java.lang.reflect.Field;
-import java.net.URLEncoder;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.Collections;
import java.util.NavigableMap;
-import java.util.NavigableSet;
import java.util.Set;
-import java.util.SortedSet;
import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.ChoreService;
-import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
+import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
-import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
-import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
-import org.apache.hadoop.hbase.regionserver.RegionServerServices;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationPeer;
+import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter;
+import org.apache.hadoop.hbase.replication.DummyReplicationEndpoint;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfigBuilder;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
-import org.apache.hadoop.hbase.replication.ZKReplicationPeerStorage;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.JVMClusterUtil;
-import org.apache.hadoop.hbase.util.MockServer;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.hbase.wal.WALKeyImpl;
-import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
-import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
-
-/**
- * An abstract class that tests ReplicationSourceManager. Classes that extend this class should set
- * up the proper config for this class and initialize the proper cluster using HBaseTestingUtility.
- */
@Category({ ReplicationTests.class, MediumTests.class })
-public abstract class TestReplicationSourceManager {
+public class TestReplicationSourceManager {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestReplicationSourceManager.class);
- protected static final Logger LOG = LoggerFactory.getLogger(TestReplicationSourceManager.class);
-
- protected static Configuration conf;
+ public static final class ReplicationEndpointForTest extends DummyReplicationEndpoint {
- protected static HBaseTestingUtil utility;
+ private String clusterKey;
- protected static Replication replication;
-
- protected static ReplicationSourceManager manager;
-
- protected static ReplicationSourceManager managerOfCluster;
+ @Override
+ public boolean replicate(ReplicateContext replicateContext) {
+ // if you want to block the replication, for example, do not want the recovered source to be
+ // removed
+ if (clusterKey.endsWith("error")) {
+ throw new RuntimeException("Inject error");
+ }
+ return true;
+ }
- protected static ZKWatcher zkw;
+ @Override
+ public void init(Context context) throws IOException {
+ super.init(context);
+ this.clusterKey = context.getReplicationPeer().getPeerConfig().getClusterKey();
+ }
- protected static TableDescriptor htd;
+ }
- protected static RegionInfo hri;
+ private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
- protected static final byte[] r1 = Bytes.toBytes("r1");
+ private static Configuration CONF;
- protected static final byte[] r2 = Bytes.toBytes("r2");
+ private static FileSystem FS;
- protected static final byte[] f1 = Bytes.toBytes("f1");
+ private static final byte[] F1 = Bytes.toBytes("f1");
- protected static final byte[] f2 = Bytes.toBytes("f2");
+ private static final byte[] F2 = Bytes.toBytes("f2");
- protected static final TableName test = TableName.valueOf("test");
+ private static final TableName TABLE_NAME = TableName.valueOf("test");
- protected static final String slaveId = "1";
+ private static TableDescriptor TD;
- protected static FileSystem fs;
+ private static RegionInfo RI;
- protected static Path oldLogDir;
+ private static NavigableMap<byte[], Integer> SCOPES;
- protected static Path logDir;
+ @Rule
+ public final TestName name = new TestName();
- protected static Path remoteLogDir;
+ private Path oldLogDir;
- protected static CountDownLatch latch;
+ private Path logDir;
- protected static List<String> files = new ArrayList<>();
- protected static NavigableMap<byte[], Integer> scopes;
+ private Path remoteLogDir;
- protected static void setupZkAndReplication() throws Exception {
- // The implementing class should set up the conf
- assertNotNull(conf);
- zkw = new ZKWatcher(conf, "test", null);
- ZKUtil.createWithParents(zkw, "/hbase/replication");
- ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1");
- ZKUtil.setData(zkw, "/hbase/replication/peers/1",
- Bytes.toBytes(conf.get(HConstants.ZOOKEEPER_QUORUM) + ":"
- + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":/1"));
- ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/peer-state");
- ZKUtil.setData(zkw, "/hbase/replication/peers/1/peer-state",
- ZKReplicationPeerStorage.ENABLED_ZNODE_BYTES);
- ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/sync-rep-state");
- ZKUtil.setData(zkw, "/hbase/replication/peers/1/sync-rep-state",
- ZKReplicationPeerStorage.NONE_STATE_ZNODE_BYTES);
- ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/new-sync-rep-state");
- ZKUtil.setData(zkw, "/hbase/replication/peers/1/new-sync-rep-state",
- ZKReplicationPeerStorage.NONE_STATE_ZNODE_BYTES);
- ZKUtil.createWithParents(zkw, "/hbase/replication/state");
- ZKUtil.setData(zkw, "/hbase/replication/state", ZKReplicationPeerStorage.ENABLED_ZNODE_BYTES);
+ private Server server;
- ZKClusterId.setClusterId(zkw, new ClusterId());
- CommonFSUtils.setRootDir(utility.getConfiguration(), utility.getDataTestDir());
- fs = FileSystem.get(conf);
- oldLogDir = utility.getDataTestDir(HConstants.HREGION_OLDLOGDIR_NAME);
- logDir = utility.getDataTestDir(HConstants.HREGION_LOGDIR_NAME);
- remoteLogDir = utility.getDataTestDir(ReplicationUtils.REMOTE_WAL_DIR_NAME);
- replication = new Replication();
- replication.initialize(new DummyServer(), fs, logDir, oldLogDir,
- new WALFactory(conf, "test", null, false));
- managerOfCluster = getManagerFromCluster();
- if (managerOfCluster != null) {
- // After replication procedure, we need to add peer by hand (other than by receiving
- // notification from zk)
- managerOfCluster.addPeer(slaveId);
- }
+ private Replication replication;
- manager = replication.getReplicationManager();
- manager.addSource(slaveId);
- if (managerOfCluster != null) {
- waitPeer(slaveId, managerOfCluster, true);
- }
- waitPeer(slaveId, manager, true);
+ private ReplicationSourceManager manager;
- htd = TableDescriptorBuilder.newBuilder(test)
- .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(f1)
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ UTIL.startMiniCluster(1);
+ FS = UTIL.getTestFileSystem();
+ CONF = new Configuration(UTIL.getConfiguration());
+ CONF.setLong("replication.sleep.before.failover", 0);
+ TD = TableDescriptorBuilder.newBuilder(TABLE_NAME)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(F1)
.setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
- .setColumnFamily(ColumnFamilyDescriptorBuilder.of(f2)).build();
-
- scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (byte[] fam : htd.getColumnFamilyNames()) {
- scopes.put(fam, 0);
- }
- hri = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(r1).setEndKey(r2).build();
- }
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.of(F2)).build();
- private static ReplicationSourceManager getManagerFromCluster() {
- // TestReplicationSourceManagerZkImpl won't start the mini hbase cluster.
- if (utility.getMiniHBaseCluster() == null) {
- return null;
- }
- return utility.getMiniHBaseCluster().getRegionServerThreads().stream()
- .map(JVMClusterUtil.RegionServerThread::getRegionServer).findAny()
- .map(RegionServerServices::getReplicationSourceService).map(r -> (Replication) r)
- .map(Replication::getReplicationManager).get();
+ RI = RegionInfoBuilder.newBuilder(TABLE_NAME).build();
+ SCOPES = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ SCOPES.put(F1, 1);
+ SCOPES.put(F2, 0);
}
@AfterClass
- public static void tearDownAfterClass() throws Exception {
- if (manager != null) {
- manager.join();
- }
- utility.shutdownMiniCluster();
- }
-
- @Rule
- public TestName testName = new TestName();
-
- private void cleanLogDir() throws IOException {
- fs.delete(logDir, true);
- fs.delete(oldLogDir, true);
- fs.delete(remoteLogDir, true);
+ public static void tearDownAfterClass() throws IOException {
+ UTIL.shutdownMiniCluster();
}
@Before
public void setUp() throws Exception {
- LOG.info("Start " + testName.getMethodName());
- cleanLogDir();
- }
-
- @After
- public void tearDown() throws Exception {
- LOG.info("End " + testName.getMethodName());
- cleanLogDir();
- List<String> ids = manager.getSources().stream().map(ReplicationSourceInterface::getPeerId)
- .collect(Collectors.toList());
- for (String id : ids) {
- if (slaveId.equals(id)) {
- continue;
- }
- removePeerAndWait(id);
- }
- }
-
- @Test
- public void testLogRoll() throws Exception {
- long baseline = 1000;
- long time = baseline;
- MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
- KeyValue kv = new KeyValue(r1, f1, r1);
- WALEdit edit = new WALEdit();
- edit.add(kv);
-
- WALFactory wals =
- new WALFactory(utility.getConfiguration(), URLEncoder.encode("regionserver:60020", "UTF8"));
- ReplicationSourceManager replicationManager = replication.getReplicationManager();
- wals.getWALProvider()
- .addWALActionsListener(new ReplicationSourceWALActionListener(conf, replicationManager));
- final WAL wal = wals.getWAL(hri);
- manager.init();
- TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf("tableame"))
- .setColumnFamily(ColumnFamilyDescriptorBuilder.of(f1)).build();
- NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (byte[] fam : htd.getColumnFamilyNames()) {
- scopes.put(fam, 0);
- }
- // Testing normal log rolling every 20
- for (long i = 1; i < 101; i++) {
- if (i > 1 && i % 20 == 0) {
- wal.rollWriter();
- }
- LOG.info(Long.toString(i));
- final long txid = wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test,
- EnvironmentEdgeManager.currentTime(), mvcc, scopes), edit);
- wal.sync(txid);
- }
-
- // Simulate a rapid insert that's followed
- // by a report that's still not totally complete (missing last one)
- LOG.info(baseline + " and " + time);
- baseline += 101;
- time = baseline;
- LOG.info(baseline + " and " + time);
-
- for (int i = 0; i < 3; i++) {
- wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test,
- EnvironmentEdgeManager.currentTime(), mvcc, scopes), edit);
- }
- wal.sync();
-
- int logNumber = 0;
- for (Map.Entry<String, NavigableSet<String>> entry : manager.getWALs().get(slaveId)
- .entrySet()) {
- logNumber += entry.getValue().size();
- }
- assertEquals(6, logNumber);
-
- wal.rollWriter();
-
- ReplicationSourceInterface source = mock(ReplicationSourceInterface.class);
- when(source.getQueueId()).thenReturn("1");
- when(source.isRecovered()).thenReturn(false);
- when(source.isSyncReplication()).thenReturn(false);
- manager.logPositionAndCleanOldLogs(source,
- new WALEntryBatch(0, manager.getSources().get(0).getCurrentPath()));
-
- wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test,
- EnvironmentEdgeManager.currentTime(), mvcc, scopes), edit);
- wal.sync();
-
- assertEquals(1, manager.getWALs().size());
-
- // TODO Need a case with only 2 WALs and we only want to delete the first one
- }
+ Path rootDir = UTIL.getDataTestDirOnTestFS(name.getMethodName());
+ CommonFSUtils.setRootDir(CONF, rootDir);
+ server = mock(Server.class);
+ when(server.getConfiguration()).thenReturn(CONF);
+ when(server.getZooKeeper()).thenReturn(UTIL.getZooKeeperWatcher());
+ when(server.getConnection()).thenReturn(UTIL.getConnection());
+ when(server.getServerName()).thenReturn(ServerName.valueOf("hostname.example.org", 1234, 1));
+ oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ FS.mkdirs(oldLogDir);
+ logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+ FS.mkdirs(logDir);
+ remoteLogDir = new Path(rootDir, ReplicationUtils.REMOTE_WAL_DIR_NAME);
+ FS.mkdirs(remoteLogDir);
+ TableName tableName = TableName.valueOf("replication_" + name.getMethodName());
+ UTIL.getAdmin()
+ .createTable(ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName));
+ CONF.set(ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME, tableName.getNameAsString());
- @Test
- public void testClaimQueues() throws Exception {
- Server server = new DummyServer("hostname0.example.org");
- ReplicationQueueStorage rq = ReplicationStorageFactory
- .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
- // populate some znodes in the peer znode
- files.add("log1");
- files.add("log2");
- for (String file : files) {
- rq.addWAL(server.getServerName(), "1", file);
- }
- // create 3 DummyServers
- Server s1 = new DummyServer("dummyserver1.example.org");
- Server s2 = new DummyServer("dummyserver2.example.org");
- Server s3 = new DummyServer("dummyserver3.example.org");
-
- // create 3 DummyNodeFailoverWorkers
- DummyNodeFailoverWorker w1 = new DummyNodeFailoverWorker(server.getServerName(), s1);
- DummyNodeFailoverWorker w2 = new DummyNodeFailoverWorker(server.getServerName(), s2);
- DummyNodeFailoverWorker w3 = new DummyNodeFailoverWorker(server.getServerName(), s3);
-
- latch = new CountDownLatch(3);
- // start the threads
- w1.start();
- w2.start();
- w3.start();
- // make sure only one is successful
- int populatedMap = 0;
- // wait for result now... till all the workers are done.
- latch.await();
- populatedMap +=
- w1.isLogZnodesMapPopulated() + w2.isLogZnodesMapPopulated() + w3.isLogZnodesMapPopulated();
- assertEquals(1, populatedMap);
- server.abort("", null);
+ replication = new Replication();
+ replication.initialize(server, FS, logDir, oldLogDir,
+ new WALFactory(CONF, "test", null, false));
+ manager = replication.getReplicationManager();
}
- @Test
- public void testCleanupFailoverQueues() throws Exception {
- Server server = new DummyServer("hostname1.example.org");
- ReplicationQueueStorage rq = ReplicationStorageFactory
- .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
- // populate some znodes in the peer znode
- SortedSet<String> files = new TreeSet<>();
- String group = "testgroup";
- String file1 = group + "." + EnvironmentEdgeManager.currentTime() + ".log1";
- String file2 = group + "." + EnvironmentEdgeManager.currentTime() + ".log2";
- files.add(file1);
- files.add(file2);
- for (String file : files) {
- rq.addWAL(server.getServerName(), "1", file);
- }
- Server s1 = new DummyServer("dummyserver1.example.org");
- ReplicationPeers rp1 =
- ReplicationFactory.getReplicationPeers(s1.getZooKeeper(), s1.getConfiguration());
- rp1.init();
- manager.claimQueue(server.getServerName(), "1");
- assertEquals(1, manager.getWalsByIdRecoveredQueues().size());
- String id = "1-" + server.getServerName().getServerName();
- assertEquals(files, manager.getWalsByIdRecoveredQueues().get(id).get(group));
- ReplicationSourceInterface source = mock(ReplicationSourceInterface.class);
- when(source.getQueueId()).thenReturn(id);
- when(source.isRecovered()).thenReturn(true);
- when(source.isSyncReplication()).thenReturn(false);
- manager.cleanOldLogs(file2, false, source);
- // log1 should be deleted
- assertEquals(Sets.newHashSet(file2), manager.getWalsByIdRecoveredQueues().get(id).get(group));
+ @After
+ public void tearDown() {
+ replication.stopReplicationService();
}
- @Test
- public void testCleanupUnknownPeerZNode() throws Exception {
- Server server = new DummyServer("hostname2.example.org");
- ReplicationQueueStorage rq = ReplicationStorageFactory
- .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
- // populate some znodes in the peer znode
- // add log to an unknown peer
- String group = "testgroup";
- rq.addWAL(server.getServerName(), "2", group + ".log1");
- rq.addWAL(server.getServerName(), "2", group + ".log2");
-
- manager.claimQueue(server.getServerName(), "2");
-
- // The log of the unknown peer should be removed from zk
- for (String peer : manager.getAllQueues()) {
- assertTrue(peer.startsWith("1"));
- }
+ /**
+ * Add a peer and wait for it to initialize
+ */
+ private void addPeerAndWait(String peerId, String clusterKey, boolean syncRep)
+ throws ReplicationException, IOException {
+ ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder()
+ .setClusterKey(UTIL.getZkCluster().getAddress().toString() + ":/" + clusterKey)
+ .setReplicationEndpointImpl(ReplicationEndpointForTest.class.getName());
+ if (syncRep) {
+ builder.setTableCFsMap(ImmutableMap.of(TABLE_NAME, Collections.emptyList()))
+ .setRemoteWALDir(FS.makeQualified(remoteLogDir).toString());
+ }
+
+ manager.getReplicationPeers().getPeerStorage().addPeer(peerId, builder.build(), true,
+ syncRep ? SyncReplicationState.DOWNGRADE_ACTIVE : SyncReplicationState.NONE);
+ manager.addPeer(peerId);
+ UTIL.waitFor(20000, () -> {
+ ReplicationSourceInterface rs = manager.getSource(peerId);
+ return rs != null && rs.isSourceActive();
+ });
}
/**
- * Test for HBASE-9038, Replication.scopeWALEdits would NPE if it wasn't filtering out the
- * compaction WALEdit.
+ * Remove a peer and wait for it to get cleaned up
*/
- @Test
- public void testCompactionWALEdits() throws Exception {
- TableName tableName = TableName.valueOf("testCompactionWALEdits");
- WALProtos.CompactionDescriptor compactionDescriptor =
- WALProtos.CompactionDescriptor.getDefaultInstance();
- RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(HConstants.EMPTY_START_ROW)
- .setEndKey(HConstants.EMPTY_END_ROW).build();
- WALEdit edit = WALEdit.createCompaction(hri, compactionDescriptor);
- ReplicationSourceWALActionListener.scopeWALEdits(new WALKeyImpl(), edit, conf);
+ private void removePeerAndWait(String peerId) throws Exception {
+ ReplicationPeers rp = manager.getReplicationPeers();
+ rp.getPeerStorage().removePeer(peerId);
+ manager.removePeer(peerId);
+ UTIL.waitFor(20000, () -> {
+ if (rp.getPeer(peerId) != null) {
+ return false;
+ }
+ if (manager.getSource(peerId) != null) {
+ return false;
+ }
+ return manager.getOldSources().stream().noneMatch(rs -> rs.getPeerId().equals(peerId));
+ });
}
- @Test
- public void testBulkLoadWALEditsWithoutBulkLoadReplicationEnabled() throws Exception {
- NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- // 1. Get the bulk load wal edit event
- WALEdit logEdit = getBulkLoadWALEdit(scope);
- // 2. Create wal key
- WALKeyImpl logKey = new WALKeyImpl(scope);
-
- // 3. Get the scopes for the key
- ReplicationSourceWALActionListener.scopeWALEdits(logKey, logEdit, conf);
-
- // 4. Assert that no bulk load entry scopes are added if bulk load hfile replication is disabled
- assertNull("No bulk load entries scope should be added if bulk load replication is disabled.",
- logKey.getReplicationScopes());
+ private void createWALFile(Path file) throws Exception {
+ ProtobufLogWriter writer = new ProtobufLogWriter();
+ try {
+ writer.init(FS, file, CONF, false, FS.getDefaultBlockSize(file), null);
+ WALKeyImpl key = new WALKeyImpl(RI.getEncodedNameAsBytes(), TABLE_NAME,
+ EnvironmentEdgeManager.currentTime(), SCOPES);
+ WALEdit edit = new WALEdit();
+ edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(F1).setFamily(F1)
+ .setQualifier(F1).setType(Cell.Type.Put).setValue(F1).build());
+ edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(F2).setFamily(F2)
+ .setQualifier(F2).setType(Cell.Type.Put).setValue(F2).build());
+ writer.append(new WAL.Entry(key, edit));
+ writer.sync(false);
+ } finally {
+ writer.close();
+ }
}
@Test
- public void testBulkLoadWALEdits() throws Exception {
- // 1. Get the bulk load wal edit event
- NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- WALEdit logEdit = getBulkLoadWALEdit(scope);
- // 2. Create wal key
- WALKeyImpl logKey = new WALKeyImpl(scope);
- // 3. Enable bulk load hfile replication
- Configuration bulkLoadConf = HBaseConfiguration.create(conf);
- bulkLoadConf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
-
- // 4. Get the scopes for the key
- ReplicationSourceWALActionListener.scopeWALEdits(logKey, logEdit, bulkLoadConf);
-
- NavigableMap<byte[], Integer> scopes = logKey.getReplicationScopes();
- // Assert family with replication scope global is present in the key scopes
- assertTrue("This family scope is set to global, should be part of replication key scopes.",
- scopes.containsKey(f1));
- // Assert family with replication scope local is not present in the key scopes
- assertFalse("This family scope is set to local, should not be part of replication key scopes",
- scopes.containsKey(f2));
+ public void testClaimQueue() throws Exception {
+ String peerId = "1";
+ addPeerAndWait(peerId, "error", false);
+ ServerName serverName = ServerName.valueOf("hostname0.example.org", 12345, 123);
+ String walName1 = serverName.toString() + ".1";
+ createWALFile(new Path(oldLogDir, walName1));
+ ReplicationQueueId queueId = new ReplicationQueueId(serverName, peerId);
+ ReplicationQueueStorage queueStorage = manager.getQueueStorage();
+ queueStorage.setOffset(queueId, "", new ReplicationGroupOffset(peerId, 0),
+ Collections.emptyMap());
+ manager.claimQueue(queueId);
+ assertThat(manager.getOldSources(), hasSize(1));
}
- /**
- * Test whether calling removePeer() on a ReplicationSourceManager that failed on initializing the
- * corresponding ReplicationSourceInterface correctly cleans up the corresponding replication
- * queue and ReplicationPeer. See HBASE-16096.
- */
@Test
- public void testPeerRemovalCleanup() throws Exception {
- String replicationSourceImplName = conf.get("replication.replicationsource.implementation");
- final String peerId = "FakePeer";
- final ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
- .setClusterKey(utility.getZkCluster().getAddress().toString() + ":/hbase").build();
- try {
- DummyServer server = new DummyServer();
- ReplicationQueueStorage rq = ReplicationStorageFactory
- .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
- // Purposely fail ReplicationSourceManager.addSource() by causing ReplicationSourceInterface
- // initialization to throw an exception.
- conf.set("replication.replicationsource.implementation",
- FailInitializeDummyReplicationSource.class.getName());
- manager.getReplicationPeers();
- // Set up the znode and ReplicationPeer for the fake peer
- // Don't wait for replication source to initialize, we know it won't.
- addPeerAndWait(peerId, peerConfig, false);
-
- // Sanity check
- assertNull(manager.getSource(peerId));
-
- // Create a replication queue for the fake peer
- rq.addWAL(server.getServerName(), peerId, "FakeFile");
- // Unregister peer, this should remove the peer and clear all queues associated with it
- // Need to wait for the ReplicationTracker to pick up the changes and notify listeners.
- removePeerAndWait(peerId);
- assertFalse(rq.getAllQueues(server.getServerName()).contains(peerId));
- } finally {
- conf.set("replication.replicationsource.implementation", replicationSourceImplName);
- removePeerAndWait(peerId);
- }
- }
+ public void testSameWALPrefix() throws IOException {
+ String walName1 = "localhost,8080,12345-45678-Peer.34567";
+ String walName2 = "localhost,8080,12345.56789";
+ manager.postLogRoll(new Path(walName1));
+ manager.postLogRoll(new Path(walName2));
- private static MetricsReplicationSourceSource getGlobalSource() throws Exception {
- ReplicationSourceInterface source = manager.getSource(slaveId);
- // Retrieve the global replication metrics source
- Field f = MetricsSource.class.getDeclaredField("globalSourceSource");
- f.setAccessible(true);
- return (MetricsReplicationSourceSource) f.get(source.getSourceMetrics());
+ Set<String> latestWals =
+ manager.getLastestPath().stream().map(Path::getName).collect(Collectors.toSet());
+ assertThat(latestWals,
+ Matchers.<Set<String>> both(hasSize(2)).and(hasItems(walName1, walName2)));
}
- private static long getSizeOfLatestPath() {
- // If no mini cluster is running, there are extra replication manager influencing the metrics.
- if (utility.getMiniHBaseCluster() == null) {
- return 0;
- }
- return utility.getMiniHBaseCluster().getRegionServerThreads().stream()
- .map(JVMClusterUtil.RegionServerThread::getRegionServer)
- .map(RegionServerServices::getReplicationSourceService).map(r -> (Replication) r)
- .map(Replication::getReplicationManager)
- .mapToLong(ReplicationSourceManager::getSizeOfLatestPath).sum();
+ private MetricsReplicationSourceSource getGlobalSource() {
+ return CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class)
+ .getGlobalSource();
}
@Test
public void testRemovePeerMetricsCleanup() throws Exception {
- final String peerId = "DummyPeer";
- final ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
- .setClusterKey(utility.getZkCluster().getAddress().toString() + ":/hbase").build();
- try {
- MetricsReplicationSourceSource globalSource = getGlobalSource();
- final int globalLogQueueSizeInitial = globalSource.getSizeOfLogQueue();
- final long sizeOfLatestPath = getSizeOfLatestPath();
- addPeerAndWait(peerId, peerConfig, true);
- assertEquals(sizeOfLatestPath + globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
- ReplicationSourceInterface source = manager.getSource(peerId);
- // Sanity check
- assertNotNull(source);
- final int sizeOfSingleLogQueue = source.getSourceMetrics().getSizeOfLogQueue();
- // Enqueue log and check if metrics updated
- source.enqueueLog(new Path("abc"));
- assertEquals(1 + sizeOfSingleLogQueue, source.getSourceMetrics().getSizeOfLogQueue());
- assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
- globalSource.getSizeOfLogQueue());
-
- // Removing the peer should reset the global metrics
- removePeerAndWait(peerId);
- assertEquals(globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
-
- // Adding the same peer back again should reset the single source metrics
- addPeerAndWait(peerId, peerConfig, true);
- source = manager.getSource(peerId);
- assertNotNull(source);
- assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
- globalSource.getSizeOfLogQueue());
- } finally {
- removePeerAndWait(peerId);
- }
+ MetricsReplicationSourceSource globalSource = getGlobalSource();
+ int globalLogQueueSizeInitial = globalSource.getSizeOfLogQueue();
+ String peerId = "DummyPeer";
+ addPeerAndWait(peerId, "hbase", false);
+ // there is no latestPaths so the size of log queue should not change
+ assertEquals(globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
+
+ ReplicationSourceInterface source = manager.getSource(peerId);
+ // Sanity check
+ assertNotNull(source);
+ int sizeOfSingleLogQueue = source.getSourceMetrics().getSizeOfLogQueue();
+ // Enqueue log and check if metrics updated
+ Path serverLogDir = new Path(logDir, server.getServerName().toString());
+ source.enqueueLog(new Path(serverLogDir, server.getServerName() + ".1"));
+ assertEquals(1 + sizeOfSingleLogQueue, source.getSourceMetrics().getSizeOfLogQueue());
+ assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
+ globalSource.getSizeOfLogQueue());
+
+ // Removing the peer should reset the global metrics
+ removePeerAndWait(peerId);
+ assertEquals(globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
+
+ // Adding the same peer back again should reset the single source metrics
+ addPeerAndWait(peerId, "hbase", false);
+ source = manager.getSource(peerId);
+ assertNotNull(source);
+ assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
+ globalSource.getSizeOfLogQueue());
}
@Test
public void testDisablePeerMetricsCleanup() throws Exception {
final String peerId = "DummyPeer";
- final ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
- .setClusterKey(utility.getZkCluster().getAddress().toString() + ":/hbase").build();
try {
MetricsReplicationSourceSource globalSource = getGlobalSource();
final int globalLogQueueSizeInitial = globalSource.getSizeOfLogQueue();
- final long sizeOfLatestPath = getSizeOfLatestPath();
- addPeerAndWait(peerId, peerConfig, true);
- assertEquals(sizeOfLatestPath + globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
+ addPeerAndWait(peerId, "hbase", false);
+ assertEquals(globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
ReplicationSourceInterface source = manager.getSource(peerId);
// Sanity check
assertNotNull(source);
final int sizeOfSingleLogQueue = source.getSourceMetrics().getSizeOfLogQueue();
// Enqueue log and check if metrics updated
- source.enqueueLog(new Path("abc"));
+ Path serverLogDir = new Path(logDir, server.getServerName().toString());
+ source.enqueueLog(new Path(serverLogDir, server.getServerName() + ".1"));
assertEquals(1 + sizeOfSingleLogQueue, source.getSourceMetrics().getSizeOfLogQueue());
assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
globalSource.getSizeOfLogQueue());
@@ -607,269 +355,27 @@ public abstract class TestReplicationSourceManager {
}
}
- private ReplicationSourceInterface mockReplicationSource(String peerId) {
- ReplicationSourceInterface source = mock(ReplicationSourceInterface.class);
- when(source.getPeerId()).thenReturn(peerId);
- when(source.getQueueId()).thenReturn(peerId);
- when(source.isRecovered()).thenReturn(false);
- when(source.isSyncReplication()).thenReturn(true);
- ReplicationPeerConfig config = mock(ReplicationPeerConfig.class);
- when(config.getRemoteWALDir())
- .thenReturn(remoteLogDir.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString());
- ReplicationPeer peer = mock(ReplicationPeer.class);
- when(peer.getPeerConfig()).thenReturn(config);
- when(source.getPeer()).thenReturn(peer);
- return source;
- }
-
@Test
public void testRemoveRemoteWALs() throws Exception {
- String peerId2 = slaveId + "_2";
- addPeerAndWait(peerId2,
- ReplicationPeerConfig.newBuilder()
- .setClusterKey("localhost:" + utility.getZkCluster().getClientPort() + ":/hbase").build(),
- true);
- try {
- // make sure that we can deal with files which does not exist
- String walNameNotExists =
- "remoteWAL-12345-" + slaveId + ".12345" + ReplicationUtils.SYNC_WAL_SUFFIX;
- Path wal = new Path(logDir, walNameNotExists);
- manager.preLogRoll(wal);
- manager.postLogRoll(wal);
-
- Path remoteLogDirForPeer = new Path(remoteLogDir, slaveId);
- fs.mkdirs(remoteLogDirForPeer);
- String walName = "remoteWAL-12345-" + slaveId + ".23456" + ReplicationUtils.SYNC_WAL_SUFFIX;
- Path remoteWAL =
- new Path(remoteLogDirForPeer, walName).makeQualified(fs.getUri(), fs.getWorkingDirectory());
- fs.create(remoteWAL).close();
- wal = new Path(logDir, walName);
- manager.preLogRoll(wal);
- manager.postLogRoll(wal);
-
- ReplicationSourceInterface source = mockReplicationSource(peerId2);
- manager.cleanOldLogs(walName, true, source);
- // still there if peer id does not match
- assertTrue(fs.exists(remoteWAL));
-
- source = mockReplicationSource(slaveId);
- manager.cleanOldLogs(walName, true, source);
- assertFalse(fs.exists(remoteWAL));
- } finally {
- removePeerAndWait(peerId2);
- }
- }
-
- @Test
- public void testSameWALPrefix() throws IOException {
- Set<String> latestWalsBefore =
- manager.getLastestPath().stream().map(Path::getName).collect(Collectors.toSet());
- String walName1 = "localhost,8080,12345-45678-Peer.34567";
- String walName2 = "localhost,8080,12345.56789";
- manager.preLogRoll(new Path(walName1));
- manager.preLogRoll(new Path(walName2));
-
- Set<String> latestWals = manager.getLastestPath().stream().map(Path::getName)
- .filter(n -> !latestWalsBefore.contains(n)).collect(Collectors.toSet());
- assertEquals(2, latestWals.size());
- assertTrue(latestWals.contains(walName1));
- assertTrue(latestWals.contains(walName2));
- }
-
- /**
- * Add a peer and wait for it to initialize
- * @param waitForSource Whether to wait for replication source to initialize
- */
- private void addPeerAndWait(final String peerId, final ReplicationPeerConfig peerConfig,
- final boolean waitForSource) throws Exception {
- final ReplicationPeers rp = manager.getReplicationPeers();
- rp.getPeerStorage().addPeer(peerId, peerConfig, true, SyncReplicationState.NONE);
- try {
- manager.addPeer(peerId);
- } catch (Exception e) {
- // ignore the failed exception, because we'll test both success & failed case.
- }
- waitPeer(peerId, manager, waitForSource);
- if (managerOfCluster != null) {
- managerOfCluster.addPeer(peerId);
- waitPeer(peerId, managerOfCluster, waitForSource);
- }
- }
-
- private static void waitPeer(final String peerId, ReplicationSourceManager manager,
- final boolean waitForSource) {
- ReplicationPeers rp = manager.getReplicationPeers();
- Waiter.waitFor(conf, 20000, () -> {
- if (waitForSource) {
- ReplicationSourceInterface rs = manager.getSource(peerId);
- if (rs == null) {
- return false;
- }
- if (rs instanceof ReplicationSourceDummy) {
- return ((ReplicationSourceDummy) rs).isStartup();
- }
- return true;
- } else {
- return (rp.getPeer(peerId) != null);
- }
- });
- }
-
- /**
- * Remove a peer and wait for it to get cleaned up
- */
- private void removePeerAndWait(final String peerId) throws Exception {
- final ReplicationPeers rp = manager.getReplicationPeers();
- if (rp.getPeerStorage().listPeerIds().contains(peerId)) {
- rp.getPeerStorage().removePeer(peerId);
- try {
- manager.removePeer(peerId);
- } catch (Exception e) {
- // ignore the failed exception and continue.
- }
- }
- Waiter.waitFor(conf, 20000, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- Collection<String> peers = rp.getPeerStorage().listPeerIds();
- return (!manager.getAllQueues().contains(peerId)) && (rp.getPeer(peerId) == null)
- && (!peers.contains(peerId)) && manager.getSource(peerId) == null;
- }
- });
- }
-
- private WALEdit getBulkLoadWALEdit(NavigableMap<byte[], Integer> scope) {
- // 1. Create store files for the families
- Map<byte[], List<Path>> storeFiles = new HashMap<>(1);
- Map<String, Long> storeFilesSize = new HashMap<>(1);
- List<Path> p = new ArrayList<>(1);
- Path hfilePath1 = new Path(Bytes.toString(f1));
- p.add(hfilePath1);
- try {
- storeFilesSize.put(hfilePath1.getName(), fs.getFileStatus(hfilePath1).getLen());
- } catch (IOException e) {
- LOG.debug("Failed to calculate the size of hfile " + hfilePath1);
- storeFilesSize.put(hfilePath1.getName(), 0L);
- }
- storeFiles.put(f1, p);
- scope.put(f1, 1);
- p = new ArrayList<>(1);
- Path hfilePath2 = new Path(Bytes.toString(f2));
- p.add(hfilePath2);
- try {
- storeFilesSize.put(hfilePath2.getName(), fs.getFileStatus(hfilePath2).getLen());
- } catch (IOException e) {
- LOG.debug("Failed to calculate the size of hfile " + hfilePath2);
- storeFilesSize.put(hfilePath2.getName(), 0L);
- }
- storeFiles.put(f2, p);
- // 2. Create bulk load descriptor
- BulkLoadDescriptor desc = ProtobufUtil.toBulkLoadDescriptor(hri.getTable(),
- UnsafeByteOperations.unsafeWrap(hri.getEncodedNameAsBytes()), storeFiles, storeFilesSize, 1);
-
- // 3. create bulk load wal edit event
- WALEdit logEdit = WALEdit.createBulkLoadEvent(hri, desc);
- return logEdit;
- }
-
- static class DummyNodeFailoverWorker extends Thread {
- private Map<String, Set<String>> logZnodesMap;
- Server server;
- private ServerName deadRS;
- ReplicationQueueStorage rq;
-
- public DummyNodeFailoverWorker(ServerName deadRS, Server s) throws Exception {
- this.deadRS = deadRS;
- this.server = s;
- this.rq = ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(),
- server.getConfiguration());
- }
-
- @Override
- public void run() {
- try {
- logZnodesMap = new HashMap<>();
- List<String> queues = rq.getAllQueues(deadRS);
- for (String queue : queues) {
- Pair<String, SortedSet<String>> pair =
- rq.claimQueue(deadRS, queue, server.getServerName());
- if (pair != null) {
- logZnodesMap.put(pair.getFirst(), pair.getSecond());
- }
- }
- server.abort("Done with testing", null);
- } catch (Exception e) {
- LOG.error("Got exception while running NodeFailoverWorker", e);
- } finally {
- latch.countDown();
- }
- }
-
- /** Returns 1 when the map is not empty. */
- private int isLogZnodesMapPopulated() {
- Collection<Set<String>> sets = logZnodesMap.values();
- if (sets.size() > 1) {
- throw new RuntimeException("unexpected size of logZnodesMap: " + sets.size());
- }
- if (sets.size() == 1) {
- Set<String> s = sets.iterator().next();
- for (String file : files) {
- // at least one file was missing
- if (!s.contains(file)) {
- return 0;
- }
- }
- return 1; // we found all the files
- }
- return 0;
- }
- }
-
- static class FailInitializeDummyReplicationSource extends ReplicationSourceDummy {
-
- @Override
- public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueueStorage rq, ReplicationPeer rp, Server server, String peerClusterId,
- UUID clusterId, WALFileLengthProvider walFileLengthProvider, MetricsSource metrics)
- throws IOException {
- throw new IOException("Failing deliberately");
- }
- }
-
- static class DummyServer extends MockServer {
- String hostname;
-
- DummyServer() {
- hostname = "hostname.example.org";
- }
-
- DummyServer(String hostname) {
- this.hostname = hostname;
- }
-
- @Override
- public Configuration getConfiguration() {
- return conf;
- }
-
- @Override
- public ZKWatcher getZooKeeper() {
- return zkw;
- }
-
- @Override
- public Connection getConnection() {
- return null;
- }
-
- @Override
- public ChoreService getChoreService() {
- return null;
- }
-
- @Override
- public ServerName getServerName() {
- return ServerName.valueOf(hostname, 1234, 1L);
- }
+ String peerId = "2";
+ addPeerAndWait(peerId, "hbase", true);
+ // make sure that we can deal with files which does not exist
+ String walNameNotExists =
+ "remoteWAL-12345-" + peerId + ".12345" + ReplicationUtils.SYNC_WAL_SUFFIX;
+ Path wal = new Path(logDir, walNameNotExists);
+ manager.postLogRoll(wal);
+
+ Path remoteLogDirForPeer = new Path(remoteLogDir, peerId);
+ FS.mkdirs(remoteLogDirForPeer);
+ String walName = "remoteWAL-12345-" + peerId + ".23456" + ReplicationUtils.SYNC_WAL_SUFFIX;
+ Path remoteWAL =
+ new Path(remoteLogDirForPeer, walName).makeQualified(FS.getUri(), FS.getWorkingDirectory());
+ FS.create(remoteWAL).close();
+ wal = new Path(logDir, walName);
+ manager.postLogRoll(wal);
+
+ ReplicationSourceInterface source = manager.getSource(peerId);
+ manager.cleanOldLogs(walName, true, source);
+ assertFalse(FS.exists(remoteWAL));
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
deleted file mode 100644
index b26505a6270..00000000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication.regionserver;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.List;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtil;
-import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
-import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
-import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.ReplicationTests;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Tests the ReplicationSourceManager with ReplicationQueueZkImpl's and
- * ReplicationQueuesClientZkImpl. Also includes extra tests outside of those in
- * TestReplicationSourceManager that test ReplicationQueueZkImpl-specific behaviors.
- */
-@Category({ ReplicationTests.class, MediumTests.class })
-public class TestReplicationSourceManagerZkImpl extends TestReplicationSourceManager {
-
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestReplicationSourceManagerZkImpl.class);
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- conf = HBaseConfiguration.create();
- conf.set("replication.replicationsource.implementation",
- ReplicationSourceDummy.class.getCanonicalName());
- conf.setLong("replication.sleep.before.failover", 2000);
- conf.setInt("replication.source.maxretriesmultiplier", 10);
- utility = new HBaseTestingUtil(conf);
- utility.startMiniZKCluster();
- setupZkAndReplication();
- }
-
- // Tests the naming convention of adopted queues for ReplicationQueuesZkImpl
- @Test
- public void testNodeFailoverDeadServerParsing() throws Exception {
- Server server = new DummyServer("ec2-54-234-230-108.compute-1.amazonaws.com");
- ReplicationQueueStorage queueStorage =
- ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
- // populate some znodes in the peer znode
- files.add("log1");
- files.add("log2");
- for (String file : files) {
- queueStorage.addWAL(server.getServerName(), "1", file);
- }
-
- // create 3 DummyServers
- Server s1 = new DummyServer("ip-10-8-101-114.ec2.internal");
- Server s2 = new DummyServer("ec2-107-20-52-47.compute-1.amazonaws.com");
- Server s3 = new DummyServer("ec2-23-20-187-167.compute-1.amazonaws.com");
-
- // simulate three servers fail sequentially
- ServerName serverName = server.getServerName();
- List<String> unclaimed = queueStorage.getAllQueues(serverName);
- queueStorage.claimQueue(serverName, unclaimed.get(0), s1.getServerName());
- queueStorage.removeReplicatorIfQueueIsEmpty(serverName);
-
- serverName = s1.getServerName();
- unclaimed = queueStorage.getAllQueues(serverName);
- queueStorage.claimQueue(serverName, unclaimed.get(0), s2.getServerName());
- queueStorage.removeReplicatorIfQueueIsEmpty(serverName);
-
- serverName = s2.getServerName();
- unclaimed = queueStorage.getAllQueues(serverName);
- String queue3 =
- queueStorage.claimQueue(serverName, unclaimed.get(0), s3.getServerName()).getFirst();
- queueStorage.removeReplicatorIfQueueIsEmpty(serverName);
-
- ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queue3);
- List<ServerName> result = replicationQueueInfo.getDeadRegionServers();
- // verify
- assertTrue(result.contains(server.getServerName()));
- assertTrue(result.contains(s1.getServerName()));
- assertTrue(result.contains(s2.getServerName()));
-
- server.stop("");
- }
-}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALEdits.java
new file mode 100644
index 00000000000..bc885db0df5
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALEdits.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.wal.WALEdit;
+import org.apache.hadoop.hbase.wal.WALKeyImpl;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
+
+@Category({ ReplicationTests.class, SmallTests.class })
+public class TestReplicationWALEdits {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReplicationWALEdits.class);
+
+ private static final Configuration CONF = HBaseConfiguration.create();
+
+ private static final TableName TABLE_NAME = TableName.valueOf("test");
+
+ private static final byte[] F1 = Bytes.toBytes("f1");
+
+ private static final byte[] F2 = Bytes.toBytes("f2");
+
+ private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TABLE_NAME).build();
+
+ /**
+ * Test for HBASE-9038, Replication.scopeWALEdits would NPE if it wasn't filtering out the
+ * compaction WALEdit.
+ */
+ @Test
+ public void testCompactionWALEdits() throws Exception {
+ TableName tableName = TableName.valueOf("testCompactionWALEdits");
+ WALProtos.CompactionDescriptor compactionDescriptor =
+ WALProtos.CompactionDescriptor.getDefaultInstance();
+ RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(HConstants.EMPTY_START_ROW)
+ .setEndKey(HConstants.EMPTY_END_ROW).build();
+ WALEdit edit = WALEdit.createCompaction(hri, compactionDescriptor);
+ ReplicationSourceWALActionListener.scopeWALEdits(new WALKeyImpl(), edit, CONF);
+ }
+
+ private WALEdit getBulkLoadWALEdit(NavigableMap<byte[], Integer> scope) {
+ // 1. Create store files for the families
+ Map<byte[], List<Path>> storeFiles = new HashMap<>(1);
+ Map<String, Long> storeFilesSize = new HashMap<>(1);
+ List<Path> p = new ArrayList<>(1);
+ Path hfilePath1 = new Path(Bytes.toString(F1));
+ p.add(hfilePath1);
+ storeFilesSize.put(hfilePath1.getName(), 0L);
+ storeFiles.put(F1, p);
+ scope.put(F1, 1);
+ p = new ArrayList<>(1);
+ Path hfilePath2 = new Path(Bytes.toString(F2));
+ p.add(hfilePath2);
+ storeFilesSize.put(hfilePath2.getName(), 0L);
+ storeFiles.put(F2, p);
+ // 2. Create bulk load descriptor
+ BulkLoadDescriptor desc = ProtobufUtil.toBulkLoadDescriptor(RI.getTable(),
+ UnsafeByteOperations.unsafeWrap(RI.getEncodedNameAsBytes()), storeFiles, storeFilesSize, 1);
+
+ // 3. create bulk load wal edit event
+ WALEdit logEdit = WALEdit.createBulkLoadEvent(RI, desc);
+ return logEdit;
+ }
+
+ @Test
+ public void testBulkLoadWALEditsWithoutBulkLoadReplicationEnabled() throws Exception {
+ NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ // 1. Get the bulk load wal edit event
+ WALEdit logEdit = getBulkLoadWALEdit(scope);
+ // 2. Create wal key
+ WALKeyImpl logKey = new WALKeyImpl(scope);
+
+ // 3. Get the scopes for the key
+ ReplicationSourceWALActionListener.scopeWALEdits(logKey, logEdit, CONF);
+
+ // 4. Assert that no bulk load entry scopes are added if bulk load hfile replication is disabled
+ assertNull("No bulk load entries scope should be added if bulk load replication is disabled.",
+ logKey.getReplicationScopes());
+ }
+
+ @Test
+ public void testBulkLoadWALEdits() throws Exception {
+ // 1. Get the bulk load wal edit event
+ NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ WALEdit logEdit = getBulkLoadWALEdit(scope);
+ // 2. Create wal key
+ WALKeyImpl logKey = new WALKeyImpl(scope);
+ // 3. Enable bulk load hfile replication
+ Configuration bulkLoadConf = HBaseConfiguration.create(CONF);
+ bulkLoadConf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
+
+ // 4. Get the scopes for the key
+ ReplicationSourceWALActionListener.scopeWALEdits(logKey, logEdit, bulkLoadConf);
+
+ NavigableMap<byte[], Integer> scopes = logKey.getReplicationScopes();
+ // Assert family with replication scope global is present in the key scopes
+ assertTrue("This family scope is set to global, should be part of replication key scopes.",
+ scopes.containsKey(F1));
+ // Assert family with replication scope local is not present in the key scopes
+ assertFalse("This family scope is set to local, should not be part of replication key scopes",
+ scopes.containsKey(F2));
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java
index 9da36769429..1544265435c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java
@@ -46,6 +46,8 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.replication.ReplicationBarrierFamilyFormat;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -94,10 +96,11 @@ public class TestSerialReplicationChecker {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
UTIL.startMiniCluster(1);
- QUEUE_STORAGE = ReplicationStorageFactory.getReplicationQueueStorage(UTIL.getZooKeeperWatcher(),
- UTIL.getConfiguration());
- QUEUE_STORAGE.addWAL(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(), PEER_ID,
- WAL_FILE_NAME);
+ TableName repTable = TableName.valueOf("test_serial_rep");
+ UTIL.getAdmin()
+ .createTable(ReplicationStorageFactory.createReplicationQueueTableDescriptor(repTable));
+ QUEUE_STORAGE =
+ ReplicationStorageFactory.getReplicationQueueStorage(UTIL.getConnection(), repTable);
}
@AfterClass
@@ -174,8 +177,10 @@ public class TestSerialReplicationChecker {
}
private void updatePushedSeqId(RegionInfo region, long seqId) throws ReplicationException {
- QUEUE_STORAGE.setWALPosition(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(),
- PEER_ID, WAL_FILE_NAME, 10, ImmutableMap.of(region.getEncodedName(), seqId));
+ ReplicationQueueId queueId = new ReplicationQueueId(
+ UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(), PEER_ID);
+ QUEUE_STORAGE.setOffset(queueId, "", new ReplicationGroupOffset(WAL_FILE_NAME, 10),
+ ImmutableMap.of(region.getEncodedName(), seqId));
}
private void addParents(RegionInfo region, List<RegionInfo> parents) throws IOException {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java
index d66aef492ff..93fa22c00fd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java
@@ -25,7 +25,6 @@ import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
@@ -35,7 +34,6 @@ import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.AdvancedScanResultConsumer;
import org.apache.hadoop.hbase.client.AsyncClusterConnection;
@@ -55,8 +53,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
@@ -72,31 +68,12 @@ public class TestWALEntrySinkFilter {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALEntrySinkFilter.class);
- private static final Logger LOG = LoggerFactory.getLogger(TestReplicationSink.class);
@Rule
public TestName name = new TestName();
static final int BOUNDARY = 5;
static final AtomicInteger UNFILTERED = new AtomicInteger();
static final AtomicInteger FILTERED = new AtomicInteger();
- /**
- * Implemetentation of Stoppable to pass into ReplicationSink.
- */
- private static Stoppable STOPPABLE = new Stoppable() {
- private final AtomicBoolean stop = new AtomicBoolean(false);
-
- @Override
- public boolean isStopped() {
- return this.stop.get();
- }
-
- @Override
- public void stop(String why) {
- LOG.info("STOPPING BECAUSE: " + why);
- this.stop.set(true);
- }
- };
-
/**
* Test filter. Filter will filter out any write time that is <= 5 (BOUNDARY). We count how many
* items we filter out and we count how many cells make it through for distribution way down below
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java
index d4f0ec664e3..20ed3796dbd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MetaTableAccessor;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put;
@@ -43,7 +44,9 @@ import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.replication.ReplicationBarrierFamilyFormat;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -78,13 +81,9 @@ public class TestHBaseFsckCleanReplicationBarriers {
@BeforeClass
public static void setUp() throws Exception {
UTIL.startMiniCluster(1);
- QUEUE_STORAGE = ReplicationStorageFactory.getReplicationQueueStorage(UTIL.getZooKeeperWatcher(),
+ QUEUE_STORAGE = ReplicationStorageFactory.getReplicationQueueStorage(UTIL.getConnection(),
UTIL.getConfiguration());
createPeer();
- QUEUE_STORAGE.addWAL(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(), PEER_1,
- WAL_FILE_NAME);
- QUEUE_STORAGE.addWAL(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(), PEER_2,
- WAL_FILE_NAME);
}
@AfterClass
@@ -205,9 +204,12 @@ public class TestHBaseFsckCleanReplicationBarriers {
}
private void updatePushedSeqId(RegionInfo region, long seqId) throws ReplicationException {
- QUEUE_STORAGE.setWALPosition(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(),
- PEER_1, WAL_FILE_NAME, 10, ImmutableMap.of(region.getEncodedName(), seqId));
- QUEUE_STORAGE.setWALPosition(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(),
- PEER_2, WAL_FILE_NAME, 10, ImmutableMap.of(region.getEncodedName(), seqId));
+ ServerName sn = UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName();
+ QUEUE_STORAGE.setOffset(new ReplicationQueueId(sn, PEER_1), "",
+ new ReplicationGroupOffset(WAL_FILE_NAME, 10),
+ ImmutableMap.of(region.getEncodedName(), seqId));
+ QUEUE_STORAGE.setOffset(new ReplicationQueueId(sn, PEER_2), "",
+ new ReplicationGroupOffset(WAL_FILE_NAME, 10),
+ ImmutableMap.of(region.getEncodedName(), seqId));
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java
index 66fd10bd156..aeed1a9a483 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java
@@ -55,9 +55,12 @@ import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+//revisit later
+@Ignore
@Category({ MiscTests.class, MediumTests.class })
public class TestHBaseFsckEncryption {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
index 70ea559e330..b24b721762d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
@@ -40,11 +40,14 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+// revisit later
+@Ignore
@Category({ MiscTests.class, MediumTests.class })
public class TestHBaseFsckMOB extends BaseTestHBaseFsck {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
index dc8fb849633..fdf0d2d6a25 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
@@ -17,25 +17,14 @@
*/
package org.apache.hadoop.hbase.util;
-import static org.junit.Assert.assertEquals;
-
-import java.util.List;
-import java.util.stream.Stream;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
-import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
-import org.apache.hadoop.hbase.replication.SyncReplicationState;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
-import org.apache.hadoop.hbase.util.HbckErrorReporter.ERROR_CODE;
-import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -59,51 +48,53 @@ public class TestHBaseFsckReplication {
UTIL.shutdownMiniCluster();
}
+ // TODO: reimplement
+ @Ignore
@Test
public void test() throws Exception {
- ReplicationPeerStorage peerStorage = ReplicationStorageFactory
- .getReplicationPeerStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
- ReplicationQueueStorage queueStorage = ReplicationStorageFactory
- .getReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
-
- String peerId1 = "1";
- String peerId2 = "2";
- peerStorage.addPeer(peerId1, ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
- true, SyncReplicationState.NONE);
- peerStorage.addPeer(peerId2, ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
- true, SyncReplicationState.NONE);
- for (int i = 0; i < 10; i++) {
- queueStorage.addWAL(ServerName.valueOf("localhost", 10000 + i, 100000 + i), peerId1,
- "file-" + i);
- }
- queueStorage.addWAL(ServerName.valueOf("localhost", 10000, 100000), peerId2, "file");
- HBaseFsck fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), true);
- HbckTestingUtil.assertNoErrors(fsck);
-
- // should not remove anything since the replication peer is still alive
- assertEquals(10, queueStorage.getListOfReplicators().size());
- peerStorage.removePeer(peerId1);
- // there should be orphan queues
- assertEquals(10, queueStorage.getListOfReplicators().size());
- fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), false);
- HbckTestingUtil.assertErrors(fsck, Stream.generate(() -> {
- return ERROR_CODE.UNDELETED_REPLICATION_QUEUE;
- }).limit(10).toArray(ERROR_CODE[]::new));
-
- // should not delete anything when fix is false
- assertEquals(10, queueStorage.getListOfReplicators().size());
-
- fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), true);
- HbckTestingUtil.assertErrors(fsck, Stream.generate(() -> {
- return ERROR_CODE.UNDELETED_REPLICATION_QUEUE;
- }).limit(10).toArray(ERROR_CODE[]::new));
-
- List<ServerName> replicators = queueStorage.getListOfReplicators();
- // should not remove the server with queue for peerId2
- assertEquals(1, replicators.size());
- assertEquals(ServerName.valueOf("localhost", 10000, 100000), replicators.get(0));
- for (String queueId : queueStorage.getAllQueues(replicators.get(0))) {
- assertEquals(peerId2, queueId);
- }
+ // ReplicationPeerStorage peerStorage = ReplicationStorageFactory
+ // .getReplicationPeerStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
+ // ReplicationQueueStorage queueStorage = ReplicationStorageFactory
+ // .getReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
+ //
+ // String peerId1 = "1";
+ // String peerId2 = "2";
+ // peerStorage.addPeer(peerId1, ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
+ // true, SyncReplicationState.NONE);
+ // peerStorage.addPeer(peerId2, ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
+ // true, SyncReplicationState.NONE);
+ // for (int i = 0; i < 10; i++) {
+ // queueStorage.addWAL(ServerName.valueOf("localhost", 10000 + i, 100000 + i), peerId1,
+ // "file-" + i);
+ // }
+ // queueStorage.addWAL(ServerName.valueOf("localhost", 10000, 100000), peerId2, "file");
+ // HBaseFsck fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), true);
+ // HbckTestingUtil.assertNoErrors(fsck);
+ //
+ // // should not remove anything since the replication peer is still alive
+ // assertEquals(10, queueStorage.getListOfReplicators().size());
+ // peerStorage.removePeer(peerId1);
+ // // there should be orphan queues
+ // assertEquals(10, queueStorage.getListOfReplicators().size());
+ // fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), false);
+ // HbckTestingUtil.assertErrors(fsck, Stream.generate(() -> {
+ // return ERROR_CODE.UNDELETED_REPLICATION_QUEUE;
+ // }).limit(10).toArray(ERROR_CODE[]::new));
+ //
+ // // should not delete anything when fix is false
+ // assertEquals(10, queueStorage.getListOfReplicators().size());
+ //
+ // fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), true);
+ // HbckTestingUtil.assertErrors(fsck, Stream.generate(() -> {
+ // return ERROR_CODE.UNDELETED_REPLICATION_QUEUE;
+ // }).limit(10).toArray(ERROR_CODE[]::new));
+ //
+ // List<ServerName> replicators = queueStorage.getListOfReplicators();
+ // // should not remove the server with queue for peerId2
+ // assertEquals(1, replicators.size());
+ // assertEquals(ServerName.valueOf("localhost", 10000, 100000), replicators.get(0));
+ // for (String queueId : queueStorage.getAllQueues(replicators.get(0))) {
+ // assertEquals(peerId2, queueId);
+ // }
}
}
[hbase] 03/11: HBASE-27214 Implement the new replication hfile/log cleaner (#4722)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a7345b0e3016a42467fdc1a39bacd58627a79668
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Wed Aug 31 21:24:09 2022 +0800
HBASE-27214 Implement the new replication hfile/log cleaner (#4722)
Signed-off-by: Xin Sun <dd...@gmail.com>
---
.../org/apache/hadoop/hbase/master/HMaster.java | 1 -
.../hbase/master/cleaner/FileCleanerDelegate.java | 2 +-
.../hadoop/hbase/master/region/MasterRegion.java | 2 +-
.../hbase/master/replication/AddPeerProcedure.java | 15 +-
.../master/replication/ReplicationPeerManager.java | 8 +
.../hadoop/hbase/regionserver/HRegionServer.java | 2 +-
.../hbase/replication/ReplicationOffsetUtil.java | 47 +++
.../replication/master/ReplicationLogCleaner.java | 234 +++++++++----
.../master/ReplicationLogCleanerBarrier.java | 85 +++++
.../regionserver/ReplicationSourceManager.java | 18 +-
.../regionserver/ReplicationSyncUp.java | 5 +-
.../hadoop/hbase/wal/AbstractFSWALProvider.java | 29 ++
.../org/apache/hadoop/hbase/wal/WALFactory.java | 29 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 227 +++++-------
.../cleaner/TestReplicationHFileCleaner.java | 43 ++-
.../replication/TestReplicationOffsetUtil.java | 52 +++
.../replication/master/TestLogCleanerBarrier.java | 60 ++++
.../master/TestReplicationLogCleaner.java | 385 +++++++++++++++++++++
.../regionserver/TestReplicationSourceManager.java | 2 +-
.../apache/hadoop/hbase/wal/TestWALFactory.java | 2 +-
.../apache/hadoop/hbase/wal/TestWALMethods.java | 14 +
21 files changed, 1008 insertions(+), 254 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index ce3e81ad04b..118457648de 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -4289,5 +4289,4 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
// initialize master side coprocessors before we start handling requests
this.cpHost = new MasterCoprocessorHost(this, conf);
}
-
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
index d37bb620273..e08f5329433 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/FileCleanerDelegate.java
@@ -50,7 +50,7 @@ public interface FileCleanerDelegate extends Configurable, Stoppable {
}
/**
- * Used to do some cleanup work
+ * Will be called after cleaner run.
*/
default void postClean() {
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java
index 86c23114458..e45b6271f7b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java
@@ -380,7 +380,7 @@ public final class MasterRegion {
params.archivedWalSuffix(), params.rollPeriodMs(), params.flushSize());
walRoller.start();
- WALFactory walFactory = new WALFactory(conf, server.getServerName().toString(), server, false);
+ WALFactory walFactory = new WALFactory(conf, server.getServerName(), server, false);
Path tableDir = CommonFSUtils.getTableDir(rootDir, td.getTableName());
Path initializingFlag = new Path(tableDir, INITIALIZING_FLAG);
Path initializedFlag = new Path(tableDir, INITIALIZED_FLAG);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
index 6d0acee76ca..25a4cd4b08e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AddPeerProcedure.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
-import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.replication.ReplicationException;
@@ -84,15 +83,21 @@ public class AddPeerProcedure extends ModifyPeerProcedure {
@Override
protected void releaseLatch(MasterProcedureEnv env) {
+ env.getReplicationPeerManager().getReplicationLogCleanerBarrier().enable();
if (peerConfig.isSyncReplication()) {
env.getReplicationPeerManager().releaseSyncReplicationPeerLock();
}
- ProcedurePrepareLatch.releaseLatch(latch, this);
+ super.releaseLatch(env);
}
@Override
protected void prePeerModification(MasterProcedureEnv env)
throws IOException, ReplicationException, ProcedureSuspendedException {
+ if (!env.getReplicationPeerManager().getReplicationLogCleanerBarrier().disable()) {
+ throw suspend(env.getMasterConfiguration(),
+ backoff -> LOG.warn("LogCleaner is run at the same time when adding peer {}, sleep {} secs",
+ peerId, backoff / 1000));
+ }
MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
if (cpHost != null) {
cpHost.preAddReplicationPeer(peerId, peerConfig);
@@ -128,9 +133,13 @@ public class AddPeerProcedure extends ModifyPeerProcedure {
@Override
protected void afterReplay(MasterProcedureEnv env) {
if (getCurrentState() == getInitialState()) {
- // will try to acquire the lock when executing the procedure, no need to acquire it here
+ // do not need to disable log cleaner or acquire lock if we are in the initial state, later
+ // when executing the procedure we will try to disable and acquire.
return;
}
+ if (!env.getReplicationPeerManager().getReplicationLogCleanerBarrier().disable()) {
+ throw new IllegalStateException("can not disable log cleaner, this should not happen");
+ }
if (peerConfig.isSyncReplication()) {
if (!env.getReplicationPeerManager().tryAcquireSyncReplicationPeerLock()) {
throw new IllegalStateException(
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 59fd5414236..0a1dbf848bd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
+import org.apache.hadoop.hbase.replication.master.ReplicationLogCleanerBarrier;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
@@ -97,6 +98,9 @@ public class ReplicationPeerManager {
// Only allow to add one sync replication peer concurrently
private final Semaphore syncReplicationPeerLock = new Semaphore(1);
+ private final ReplicationLogCleanerBarrier replicationLogCleanerBarrier =
+ new ReplicationLogCleanerBarrier();
+
private final String clusterId;
private final Configuration conf;
@@ -691,4 +695,8 @@ public class ReplicationPeerManager {
public void releaseSyncReplicationPeerLock() {
syncReplicationPeerLock.release();
}
+
+ public ReplicationLogCleanerBarrier getReplicationLogCleanerBarrier() {
+ return replicationLogCleanerBarrier;
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 5240df5c62a..20598d6752f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1737,7 +1737,7 @@ public class HRegionServer extends HBaseServerBase<RSRpcServices>
* be hooked up to WAL.
*/
private void setupWALAndReplication() throws IOException {
- WALFactory factory = new WALFactory(conf, serverName.toString(), this, true);
+ WALFactory factory = new WALFactory(conf, serverName, this, true);
// TODO Replication make assumptions here based on the default filesystem impl
Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
String logName = AbstractFSWALProvider.getWALDirectoryName(this.serverName.toString());
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationOffsetUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationOffsetUtil.java
new file mode 100644
index 00000000000..052c5542d47
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/ReplicationOffsetUtil.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public final class ReplicationOffsetUtil {
+
+ private ReplicationOffsetUtil() {
+ }
+
+ public static boolean shouldReplicate(ReplicationGroupOffset offset, String wal) {
+ // if no offset or the offset is just a place marker, replicate
+ if (offset == null || offset == ReplicationGroupOffset.BEGIN) {
+ return true;
+ }
+ // otherwise, compare the timestamp
+ long walTs = AbstractFSWALProvider.getTimestamp(wal);
+ long startWalTs = AbstractFSWALProvider.getTimestamp(offset.getWal());
+ if (walTs < startWalTs) {
+ return false;
+ } else if (walTs > startWalTs) {
+ return true;
+ }
+ // if the timestamp equals, usually it means we should include this wal but there is a special
+ // case, a negative offset means the wal has already been fully replicated, so here we should
+ // check the offset.
+ return offset.getOffset() >= 0;
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index 7135ca9a9b2..f1fd8f8d6b3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -17,18 +17,29 @@
*/
package org.apache.hadoop.hbase.replication.master;
-import java.io.IOException;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
-import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationOffsetUtil;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,35 +51,129 @@ import org.apache.hbase.thirdparty.org.apache.commons.collections4.MapUtils;
/**
* Implementation of a log cleaner that checks if a log is still scheduled for replication before
* deleting it when its TTL is over.
+ * <p/>
+ * The logic is a bit complicated after we switch to use table based replication queue storage, see
+ * the design doc in HBASE-27109 and the comments in HBASE-27214 for more details.
*/
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationLogCleaner.class);
- private ZKWatcher zkw = null;
- private boolean shareZK = false;
- private ReplicationQueueStorage queueStorage;
+ private Set<ServerName> notFullyDeadServers;
+ private Set<String> peerIds;
+ // ServerName -> PeerId -> WalGroup -> Offset
+ // Here the server name is the source server name, so we can make sure that there is only one
+ // queue for a given peer, that why we can use a String peerId as key instead of
+ // ReplicationQueueId.
+ private Map<ServerName, Map<String, Map<String, ReplicationGroupOffset>>> replicationOffsets;
+ private ReplicationPeerManager rpm;
+ private Supplier<Set<ServerName>> getNotFullyDeadServers;
+
+ private boolean canFilter;
private boolean stopped = false;
- private Set<String> wals;
- private long readZKTimestamp = 0;
@Override
public void preClean() {
- readZKTimestamp = EnvironmentEdgeManager.currentTime();
- // TODO: revisit the implementation
- // try {
- // // The concurrently created new WALs may not be included in the return list,
- // // but they won't be deleted because they're not in the checking set.
- // wals = queueStorage.getAllWALs();
- // } catch (ReplicationException e) {
- // LOG.warn("Failed to read zookeeper, skipping checking deletable files");
- // wals = null;
- // }
+ if (this.getConf() == null) {
+ return;
+ }
+ canFilter = rpm.getReplicationLogCleanerBarrier().start();
+ if (canFilter) {
+ notFullyDeadServers = getNotFullyDeadServers.get();
+ peerIds = rpm.listPeers(null).stream().map(ReplicationPeerDescription::getPeerId)
+ .collect(Collectors.toSet());
+ // must get the not fully dead servers first and then get the replication queue data, in this
+ // way we can make sure that, we should have added the missing replication queues for the dead
+ // region servers recorded in the above set, otherwise the logic in the
+ // filterForDeadRegionServer method may lead us delete wal still in use.
+ List<ReplicationQueueData> allQueueData;
+ try {
+ allQueueData = rpm.getQueueStorage().listAllQueues();
+ } catch (ReplicationException e) {
+ LOG.error("Can not list all replication queues, give up cleaning", e);
+ rpm.getReplicationLogCleanerBarrier().stop();
+ canFilter = false;
+ notFullyDeadServers = null;
+ peerIds = null;
+ return;
+ }
+ replicationOffsets = new HashMap<>();
+ for (ReplicationQueueData queueData : allQueueData) {
+ ReplicationQueueId queueId = queueData.getId();
+ ServerName serverName = queueId.getServerWALsBelongTo();
+ Map<String, Map<String, ReplicationGroupOffset>> peerId2Offsets =
+ replicationOffsets.computeIfAbsent(serverName, k -> new HashMap<>());
+ Map<String, ReplicationGroupOffset> offsets =
+ peerId2Offsets.computeIfAbsent(queueId.getPeerId(), k -> new HashMap<>());
+ offsets.putAll(queueData.getOffsets());
+ }
+ } else {
+ LOG.info("Skip replication log cleaner because an AddPeerProcedure is running");
+ }
}
@Override
public void postClean() {
- // release memory
- wals = null;
+ if (canFilter) {
+ rpm.getReplicationLogCleanerBarrier().stop();
+ canFilter = false;
+ // release memory
+ notFullyDeadServers = null;
+ peerIds = null;
+ replicationOffsets = null;
+ }
+ }
+
+ private boolean shouldDelete(ReplicationGroupOffset offset, FileStatus file) {
+ return !ReplicationOffsetUtil.shouldReplicate(offset, file.getPath().getName());
+ }
+
+ private boolean filterForLiveRegionServer(ServerName serverName, FileStatus file) {
+ Map<String, Map<String, ReplicationGroupOffset>> peerId2Offsets =
+ replicationOffsets.get(serverName);
+ if (peerId2Offsets == null) {
+ // if there are replication queues missing, we can not delete the wal
+ return false;
+ }
+ for (String peerId : peerIds) {
+ Map<String, ReplicationGroupOffset> offsets = peerId2Offsets.get(peerId);
+ // if no replication queue for a peer, we can not delete the wal
+ if (offsets == null) {
+ return false;
+ }
+ String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(file.getPath().getName());
+ ReplicationGroupOffset offset = offsets.get(walGroupId);
+ // if a replication queue still need to replicate this wal, we can not delete it
+ if (!shouldDelete(offset, file)) {
+ return false;
+ }
+ }
+ // if all replication queues have already finished replicating this wal, we can delete it.
+ return true;
+ }
+
+ private boolean filterForDeadRegionServer(ServerName serverName, FileStatus file) {
+ Map<String, Map<String, ReplicationGroupOffset>> peerId2Offsets =
+ replicationOffsets.get(serverName);
+ if (peerId2Offsets == null) {
+ // no replication queue for this dead rs, we can delete all wal files for it
+ return true;
+ }
+ for (String peerId : peerIds) {
+ Map<String, ReplicationGroupOffset> offsets = peerId2Offsets.get(peerId);
+ if (offsets == null) {
+ // for dead server, we only care about existing replication queues, as we will delete a
+ // queue after we finish replicating it.
+ continue;
+ }
+ String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(file.getPath().getName());
+ ReplicationGroupOffset offset = offsets.get(walGroupId);
+ // if a replication queue still need to replicate this wal, we can not delete it
+ if (!shouldDelete(offset, file)) {
+ return false;
+ }
+ }
+ // if all replication queues have already finished replicating this wal, we can delete it.
+ return true;
}
@Override
@@ -78,10 +183,12 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
if (this.getConf() == null) {
return files;
}
-
- if (wals == null) {
+ if (!canFilter) {
+ // We can not delete anything if there are AddPeerProcedure running at the same time
+ // See HBASE-27214 for more details.
return Collections.emptyList();
}
+
return Iterables.filter(files, new Predicate<FileStatus>() {
@Override
public boolean apply(FileStatus file) {
@@ -90,65 +197,56 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
if (file == null) {
return false;
}
- String wal = file.getPath().getName();
- boolean logInReplicationQueue = wals.contains(wal);
- if (logInReplicationQueue) {
- LOG.debug("Found up in ZooKeeper, NOT deleting={}", wal);
+ if (peerIds.isEmpty()) {
+ // no peer, can always delete
+ return true;
+ }
+ // not a valid wal file name, delete
+ if (!AbstractFSWALProvider.validateWALFilename(file.getPath().getName())) {
+ return true;
+ }
+ // meta wal is always deletable as we will never replicate it
+ if (AbstractFSWALProvider.isMetaFile(file.getPath())) {
+ return true;
+ }
+ ServerName serverName =
+ AbstractFSWALProvider.parseServerNameFromWALName(file.getPath().getName());
+ if (notFullyDeadServers.contains(serverName)) {
+ return filterForLiveRegionServer(serverName, file);
+ } else {
+ return filterForDeadRegionServer(serverName, file);
}
- return !logInReplicationQueue && (file.getModificationTime() < readZKTimestamp);
}
});
}
+ private Set<ServerName> getNotFullyDeadServers(MasterServices services) {
+ List<ServerName> onlineServers = services.getServerManager().getOnlineServersList();
+ return Stream.concat(onlineServers.stream(),
+ services.getMasterProcedureExecutor().getProcedures().stream()
+ .filter(p -> p instanceof ServerCrashProcedure).filter(p -> !p.isFinished())
+ .map(p -> ((ServerCrashProcedure) p).getServerName()))
+ .collect(Collectors.toSet());
+ }
+
@Override
public void init(Map<String, Object> params) {
super.init(params);
- try {
- if (MapUtils.isNotEmpty(params)) {
- Object master = params.get(HMaster.MASTER);
- if (master != null && master instanceof HMaster) {
- zkw = ((HMaster) master).getZooKeeper();
- shareZK = true;
- }
- }
- if (zkw == null) {
- zkw = new ZKWatcher(getConf(), "replicationLogCleaner", null);
+ if (MapUtils.isNotEmpty(params)) {
+ Object master = params.get(HMaster.MASTER);
+ if (master != null && master instanceof MasterServices) {
+ MasterServices m = (MasterServices) master;
+ rpm = m.getReplicationPeerManager();
+ getNotFullyDeadServers = () -> getNotFullyDeadServers(m);
+ return;
}
- // TODO: revisit the implementation
- // this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
- } catch (IOException e) {
- LOG.error("Error while configuring " + this.getClass().getName(), e);
}
- }
-
- @InterfaceAudience.Private
- public void setConf(Configuration conf, ZKWatcher zk) {
- super.setConf(conf);
- try {
- this.zkw = zk;
- // TODO: revisit the implementation
- // this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zk, conf);
- } catch (Exception e) {
- LOG.error("Error while configuring " + this.getClass().getName(), e);
- }
- }
-
- @InterfaceAudience.Private
- public void setConf(Configuration conf, ZKWatcher zk,
- ReplicationQueueStorage replicationQueueStorage) {
- super.setConf(conf);
- this.zkw = zk;
- this.queueStorage = replicationQueueStorage;
+ throw new IllegalArgumentException("Missing " + HMaster.MASTER + " parameter");
}
@Override
public void stop(String why) {
- if (this.stopped) return;
this.stopped = true;
- if (!shareZK && this.zkw != null) {
- LOG.info("Stopping " + this.zkw);
- this.zkw.close();
- }
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleanerBarrier.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleanerBarrier.java
new file mode 100644
index 00000000000..d8756518728
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleanerBarrier.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication.master;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * A barrier to guard the execution of {@link ReplicationLogCleaner}.
+ * <p/>
+ * The reason why we introduce this class is because there could be race between
+ * {@link org.apache.hadoop.hbase.master.replication.AddPeerProcedure} and
+ * {@link ReplicationLogCleaner}. See HBASE-27214 for more details.
+ */
+@InterfaceAudience.Private
+public class ReplicationLogCleanerBarrier {
+
+ private enum State {
+ // the cleaner is not running
+ NOT_RUNNING,
+ // the cleaner is running
+ RUNNING,
+ // the cleaner is disabled
+ DISABLED
+ }
+
+ private State state = State.NOT_RUNNING;
+
+ // we could have multiple AddPeerProcedure running at the same time, so here we need to do
+ // reference counting.
+ private int numberDisabled = 0;
+
+ public synchronized boolean start() {
+ if (state == State.NOT_RUNNING) {
+ state = State.RUNNING;
+ return true;
+ }
+ if (state == State.DISABLED) {
+ return false;
+ }
+ throw new IllegalStateException("Unexpected state " + state);
+ }
+
+ public synchronized void stop() {
+ if (state != State.RUNNING) {
+ throw new IllegalStateException("Unexpected state " + state);
+ }
+ state = State.NOT_RUNNING;
+ }
+
+ public synchronized boolean disable() {
+ if (state == State.RUNNING) {
+ return false;
+ }
+ if (state == State.NOT_RUNNING) {
+ state = State.DISABLED;
+ }
+ numberDisabled++;
+ return true;
+ }
+
+ public synchronized void enable() {
+ if (state != State.DISABLED) {
+ throw new IllegalStateException("Unexpected state " + state);
+ }
+ numberDisabled--;
+ if (numberDisabled == 0) {
+ state = State.NOT_RUNNING;
+ }
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index fff98ae68be..03569be86fc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationOffsetUtil;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerImpl;
@@ -808,22 +809,7 @@ public class ReplicationSourceManager {
if (AbstractFSWALProvider.isMetaFile(wal)) {
return false;
}
- // if no offset or the offset is just a place marker, replicate
- if (offset == null || offset == ReplicationGroupOffset.BEGIN) {
- return true;
- }
- // otherwise, compare the timestamp
- long walTs = AbstractFSWALProvider.getTimestamp(wal);
- long startWalTs = AbstractFSWALProvider.getTimestamp(offset.getWal());
- if (walTs < startWalTs) {
- return false;
- } else if (walTs > startWalTs) {
- return true;
- }
- // if the timestamp equals, usually it means we should include this wal but there is a special
- // case, a negative offset means the wal has already been fully replicated, so here we should
- // check the offset.
- return offset.getOffset() >= 0;
+ return ReplicationOffsetUtil.shouldReplicate(offset, wal);
}
void claimQueue(ReplicationQueueId queueId) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index 50ffd6df1af..b63ad473719 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -117,7 +117,10 @@ public class ReplicationSyncUp extends Configured implements Tool {
System.out.println("Start Replication Server start");
Replication replication = new Replication();
replication.initialize(new DummyServer(zkw), fs, logDir, oldLogDir,
- new WALFactory(conf, "test", null, false));
+ new WALFactory(conf,
+ ServerName
+ .valueOf(getClass().getSimpleName() + ",16010," + EnvironmentEdgeManager.currentTime()),
+ null, false));
ReplicationSourceManager manager = replication.getReplicationManager();
manager.init();
claimReplicationQueues(zkw, manager);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
index f9331e25402..1b387f33ecc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
@@ -19,6 +19,9 @@ package org.apache.hadoop.hbase.wal;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -38,6 +41,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
+import org.apache.hadoop.hbase.util.Addressing;
import org.apache.hadoop.hbase.util.CancelableProgressable;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.RecoverLeaseFSUtils;
@@ -576,4 +580,29 @@ public abstract class AbstractFSWALProvider<T extends AbstractFSWAL<?>> implemen
public static String getWALPrefixFromWALName(String name) {
return getWALNameGroupFromWALName(name, 1);
}
+
+ private static final Pattern SERVER_NAME_PATTERN = Pattern.compile("^[^"
+ + ServerName.SERVERNAME_SEPARATOR + "]+" + ServerName.SERVERNAME_SEPARATOR
+ + Addressing.VALID_PORT_REGEX + ServerName.SERVERNAME_SEPARATOR + Addressing.VALID_PORT_REGEX);
+
+ /**
+ * Parse the server name from wal prefix. A wal's name is always started with a server name in non
+ * test code.
+ * @throws IllegalArgumentException if the name passed in is not started with a server name
+ * @return the server name
+ */
+ public static ServerName parseServerNameFromWALName(String name) {
+ String decoded;
+ try {
+ decoded = URLDecoder.decode(name, StandardCharsets.UTF_8.name());
+ } catch (UnsupportedEncodingException e) {
+ throw new AssertionError("should never happen", e);
+ }
+ Matcher matcher = SERVER_NAME_PATTERN.matcher(decoded);
+ if (matcher.find()) {
+ return ServerName.valueOf(matcher.group());
+ } else {
+ throw new IllegalArgumentException(name + " is not started with a server name");
+ }
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
index 92d96c5e210..bc0a9eec73a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.io.asyncfs.monitor.ExcludeDatanodeManager;
import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
@@ -191,17 +192,35 @@ public class WALFactory {
}
/**
- * @param conf must not be null, will keep a reference to read params in later reader/writer
- * instances.
- * @param factoryId a unique identifier for this factory. used i.e. by filesystem implementations
- * to make a directory
+ * Create a WALFactory.
*/
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*|.*/HBaseTestingUtility.java")
public WALFactory(Configuration conf, String factoryId) throws IOException {
// default enableSyncReplicationWALProvider is true, only disable SyncReplicationWALProvider
// for HMaster or HRegionServer which take system table only. See HBASE-19999
this(conf, factoryId, null, true);
}
+ /**
+ * Create a WALFactory.
+ * <p/>
+ * This is the constructor you should use when creating a WALFactory in normal code, to make sure
+ * that the {@code factoryId} is the server name. We need this assumption in some places for
+ * parsing the server name out from the wal file name.
+ * @param conf must not be null, will keep a reference to read params
+ * in later reader/writer instances.
+ * @param serverName use to generate the factoryId, which will be append at
+ * the first of the final file name
+ * @param abortable the server associated with this WAL file
+ * @param enableSyncReplicationWALProvider whether wrap the wal provider to a
+ * {@link SyncReplicationWALProvider} n
+ */
+ public WALFactory(Configuration conf, ServerName serverName, Abortable abortable,
+ boolean enableSyncReplicationWALProvider) throws IOException {
+ this(conf, serverName.toString(), abortable, enableSyncReplicationWALProvider);
+ }
+
/**
* @param conf must not be null, will keep a reference to read params
* in later reader/writer instances.
@@ -211,7 +230,7 @@ public class WALFactory {
* @param enableSyncReplicationWALProvider whether wrap the wal provider to a
* {@link SyncReplicationWALProvider}
*/
- public WALFactory(Configuration conf, String factoryId, Abortable abortable,
+ private WALFactory(Configuration conf, String factoryId, Abortable abortable,
boolean enableSyncReplicationWALProvider) throws IOException {
// until we've moved reader/writer construction down into providers, this initialization must
// happen prior to provider initialization, in case they need to instantiate a reader/writer.
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
index 1a0537bcbaf..d7ba6c227c6 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
@@ -18,57 +18,60 @@
package org.apache.hadoop.hbase.master.cleaner;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
+import java.util.Collections;
import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNameTestRule;
import org.apache.hadoop.hbase.Waiter;
-import org.apache.hadoop.hbase.ZooKeeperConnectionException;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
-import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
+import org.apache.hadoop.hbase.replication.master.ReplicationLogCleanerBarrier;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.MockServer;
-import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.zookeeper.KeeperException;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-// revisit later after we implement new replication log cleaner
-@Ignore
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
@Category({ MasterTests.class, MediumTests.class })
public class TestLogsCleaner {
@@ -88,22 +91,29 @@ public class TestLogsCleaner {
private static DirScanPool POOL;
+ private static String peerId = "1";
+
+ private MasterServices masterServices;
+
+ private ReplicationQueueStorage queueStorage;
+
+ @Rule
+ public final TableNameTestRule tableNameRule = new TableNameTestRule();
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- TEST_UTIL.startMiniZKCluster();
- TEST_UTIL.startMiniDFSCluster(1);
+ TEST_UTIL.startMiniCluster();
POOL = DirScanPool.getLogCleanerScanPool(TEST_UTIL.getConfiguration());
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniZKCluster();
- TEST_UTIL.shutdownMiniDFSCluster();
+ TEST_UTIL.shutdownMiniCluster();
POOL.shutdownNow();
}
@Before
- public void beforeTest() throws IOException {
+ public void beforeTest() throws Exception {
conf = TEST_UTIL.getConfiguration();
FileSystem fs = TEST_UTIL.getDFSCluster().getFileSystem();
@@ -112,14 +122,51 @@ public class TestLogsCleaner {
// root directory
fs.mkdirs(OLD_WALS_DIR);
+
+ TableName tableName = tableNameRule.getTableName();
+ TableDescriptor td = ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName);
+ TEST_UTIL.getAdmin().createTable(td);
+ TEST_UTIL.waitTableAvailable(tableName);
+ queueStorage =
+ ReplicationStorageFactory.getReplicationQueueStorage(TEST_UTIL.getConnection(), tableName);
+
+ masterServices = mock(MasterServices.class);
+ when(masterServices.getConnection()).thenReturn(TEST_UTIL.getConnection());
+ ReplicationPeerManager rpm = mock(ReplicationPeerManager.class);
+ when(masterServices.getReplicationPeerManager()).thenReturn(rpm);
+ when(rpm.getQueueStorage()).thenReturn(queueStorage);
+ when(rpm.getReplicationLogCleanerBarrier()).thenReturn(new ReplicationLogCleanerBarrier());
+ when(rpm.listPeers(null)).thenReturn(new ArrayList<>());
+ ServerManager sm = mock(ServerManager.class);
+ when(masterServices.getServerManager()).thenReturn(sm);
+ when(sm.getOnlineServersList()).thenReturn(Collections.emptyList());
+ @SuppressWarnings("unchecked")
+ ProcedureExecutor<MasterProcedureEnv> procExec = mock(ProcedureExecutor.class);
+ when(masterServices.getMasterProcedureExecutor()).thenReturn(procExec);
+ when(procExec.getProcedures()).thenReturn(Collections.emptyList());
}
/**
* This tests verifies LogCleaner works correctly with WALs and Procedure WALs located in the same
- * oldWALs directory. Created files: - 2 invalid files - 5 old Procedure WALs - 30 old WALs from
- * which 3 are in replication - 5 recent Procedure WALs - 1 recent WAL - 1 very new WAL (timestamp
- * in future) - masterProcedureWALs subdirectory Files which should stay: - 3 replication WALs - 2
- * new WALs - 5 latest Procedure WALs - masterProcedureWALs subdirectory
+ * oldWALs directory.
+ * <p/>
+ * Created files:
+ * <ul>
+ * <li>2 invalid files</li>
+ * <li>5 old Procedure WALs</li>
+ * <li>30 old WALs from which 3 are in replication</li>
+ * <li>5 recent Procedure WALs</li>
+ * <li>1 recent WAL</li>
+ * <li>1 very new WAL (timestamp in future)</li>
+ * <li>masterProcedureWALs subdirectory</li>
+ * </ul>
+ * Files which should stay:
+ * <ul>
+ * <li>3 replication WALs</li>
+ * <li>2 new WALs</li>
+ * <li>5 latest Procedure WALs</li>
+ * <li>masterProcedureWALs subdirectory</li>
+ * </ul>
*/
@Test
public void testLogCleaning() throws Exception {
@@ -131,9 +178,6 @@ public class TestLogsCleaner {
HMaster.decorateMasterConfiguration(conf);
Server server = new DummyServer();
- ReplicationQueueStorage queueStorage = ReplicationStorageFactory
- .getReplicationQueueStorage(ConnectionFactory.createConnection(conf), conf);
-
String fakeMachineName =
URLEncoder.encode(server.getServerName().toString(), StandardCharsets.UTF_8.name());
@@ -159,14 +203,12 @@ public class TestLogsCleaner {
for (int i = 1; i <= 30; i++) {
Path fileName = new Path(OLD_WALS_DIR, fakeMachineName + "." + (now - i));
fs.createNewFile(fileName);
- // Case 4: put 3 WALs in ZK indicating that they are scheduled for replication so these
- // files would pass TimeToLiveLogCleaner but would be rejected by ReplicationLogCleaner
- if (i % (30 / 3) == 0) {
- // queueStorage.addWAL(server.getServerName(), fakeMachineName, fileName.getName());
- LOG.info("Replication log file: " + fileName);
- }
}
-
+ // Case 4: the newest 3 WALs will be kept because they are beyond the replication offset
+ masterServices.getReplicationPeerManager().listPeers(null)
+ .add(new ReplicationPeerDescription(peerId, true, null, null));
+ queueStorage.setOffset(new ReplicationQueueId(server.getServerName(), peerId), fakeMachineName,
+ new ReplicationGroupOffset(fakeMachineName + "." + (now - 3), 0), Collections.emptyMap());
// Case 5: 5 Procedure WALs that are new, will stay
for (int i = 6; i <= 10; i++) {
Path fileName = new Path(OLD_PROCEDURE_WALS_DIR, String.format("pv2-%020d.log", i));
@@ -189,7 +231,8 @@ public class TestLogsCleaner {
// 10 procedure WALs
assertEquals(10, fs.listStatus(OLD_PROCEDURE_WALS_DIR).length);
- LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, OLD_WALS_DIR, POOL, null);
+ LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, OLD_WALS_DIR, POOL,
+ ImmutableMap.of(HMaster.MASTER, masterServices));
cleaner.chore();
// In oldWALs we end up with the current WAL, a newer WAL, the 3 old WALs which
@@ -208,98 +251,14 @@ public class TestLogsCleaner {
}
}
- @Test
- public void testZooKeeperRecoveryDuringGetListOfReplicators() throws Exception {
- ReplicationLogCleaner cleaner = new ReplicationLogCleaner();
-
- List<FileStatus> dummyFiles = Arrays.asList(
- new FileStatus(100, false, 3, 100, EnvironmentEdgeManager.currentTime(), new Path("log1")),
- new FileStatus(100, false, 3, 100, EnvironmentEdgeManager.currentTime(), new Path("log2")));
-
- FaultyZooKeeperWatcher faultyZK =
- new FaultyZooKeeperWatcher(conf, "testZooKeeperAbort-faulty", null);
- final AtomicBoolean getListOfReplicatorsFailed = new AtomicBoolean(false);
-
- try {
- faultyZK.init(false);
- ReplicationQueueStorage queueStorage = spy(ReplicationStorageFactory
- .getReplicationQueueStorage(ConnectionFactory.createConnection(conf), conf));
- // doAnswer(new Answer<Object>() {
- // @Override
- // public Object answer(InvocationOnMock invocation) throws Throwable {
- // try {
- // return invocation.callRealMethod();
- // } catch (ReplicationException e) {
- // LOG.debug("Caught Exception", e);
- // getListOfReplicatorsFailed.set(true);
- // throw e;
- // }
- // }
- // }).when(queueStorage).getAllWALs();
-
- cleaner.setConf(conf, faultyZK, queueStorage);
- // should keep all files due to a ConnectionLossException getting the queues znodes
- cleaner.preClean();
- Iterable<FileStatus> toDelete = cleaner.getDeletableFiles(dummyFiles);
-
- assertTrue(getListOfReplicatorsFailed.get());
- assertFalse(toDelete.iterator().hasNext());
- assertFalse(cleaner.isStopped());
-
- // zk recovery.
- faultyZK.init(true);
- cleaner.preClean();
- Iterable<FileStatus> filesToDelete = cleaner.getDeletableFiles(dummyFiles);
- Iterator<FileStatus> iter = filesToDelete.iterator();
- assertTrue(iter.hasNext());
- assertEquals(new Path("log1"), iter.next().getPath());
- assertTrue(iter.hasNext());
- assertEquals(new Path("log2"), iter.next().getPath());
- assertFalse(iter.hasNext());
-
- } finally {
- faultyZK.close();
- }
- }
-
- /**
- * When zk is working both files should be returned
- * @throws Exception from ZK watcher
- */
- @Test
- public void testZooKeeperNormal() throws Exception {
- ReplicationLogCleaner cleaner = new ReplicationLogCleaner();
-
- // Subtract 1000 from current time so modtime is for sure older
- // than 'now'.
- long modTime = EnvironmentEdgeManager.currentTime() - 1000;
- List<FileStatus> dummyFiles =
- Arrays.asList(new FileStatus(100, false, 3, 100, modTime, new Path("log1")),
- new FileStatus(100, false, 3, 100, modTime, new Path("log2")));
-
- ZKWatcher zkw = new ZKWatcher(conf, "testZooKeeperAbort-normal", null);
- try {
- cleaner.setConf(conf, zkw);
- cleaner.preClean();
- Iterable<FileStatus> filesToDelete = cleaner.getDeletableFiles(dummyFiles);
- Iterator<FileStatus> iter = filesToDelete.iterator();
- assertTrue(iter.hasNext());
- assertEquals(new Path("log1"), iter.next().getPath());
- assertTrue(iter.hasNext());
- assertEquals(new Path("log2"), iter.next().getPath());
- assertFalse(iter.hasNext());
- } finally {
- zkw.close();
- }
- }
-
@Test
public void testOnConfigurationChange() throws Exception {
// Prepare environments
Server server = new DummyServer();
FileSystem fs = TEST_UTIL.getDFSCluster().getFileSystem();
- LogCleaner cleaner = new LogCleaner(3000, server, conf, fs, OLD_WALS_DIR, POOL, null);
+ LogCleaner cleaner = new LogCleaner(3000, server, conf, fs, OLD_WALS_DIR, POOL,
+ ImmutableMap.of(HMaster.MASTER, masterServices));
int size = cleaner.getSizeOfCleaners();
assertEquals(LogCleaner.DEFAULT_OLD_WALS_CLEANER_THREAD_TIMEOUT_MSEC,
cleaner.getCleanerThreadTimeoutMsec());
@@ -338,7 +297,7 @@ public class TestLogsCleaner {
}
}
- static class DummyServer extends MockServer {
+ private static final class DummyServer extends MockServer {
@Override
public Configuration getConfiguration() {
@@ -355,26 +314,4 @@ public class TestLogsCleaner {
return null;
}
}
-
- static class FaultyZooKeeperWatcher extends ZKWatcher {
- private RecoverableZooKeeper zk;
-
- public FaultyZooKeeperWatcher(Configuration conf, String identifier, Abortable abortable)
- throws ZooKeeperConnectionException, IOException {
- super(conf, identifier, abortable);
- }
-
- public void init(boolean autoRecovery) throws Exception {
- this.zk = spy(super.getRecoverableZooKeeper());
- if (!autoRecovery) {
- doThrow(new KeeperException.ConnectionLossException()).when(zk)
- .getChildren("/hbase/replication/rs", null);
- }
- }
-
- @Override
- public RecoverableZooKeeper getRecoverableZooKeeper() {
- return zk;
- }
- }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
index 1d527f35bb0..452ad981fb1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
@@ -26,6 +26,7 @@ import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -34,7 +35,9 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
@@ -48,19 +51,19 @@ import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.MockServer;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-// TODO: revisit later
-@Ignore
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
@Category({ MasterTests.class, SmallTests.class })
public class TestReplicationHFileCleaner {
@@ -71,19 +74,25 @@ public class TestReplicationHFileCleaner {
private static final Logger LOG = LoggerFactory.getLogger(TestReplicationHFileCleaner.class);
private final static HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
private static Server server;
+ private static final TableName tableName = TableName.valueOf("test_cleaner");
private static ReplicationQueueStorage rq;
private static ReplicationPeers rp;
private static final String peerId = "TestReplicationHFileCleaner";
private static Configuration conf = TEST_UTIL.getConfiguration();
- static FileSystem fs = null;
- Path root;
+ private static FileSystem fs = null;
+ private static Map<String, Object> params;
+ private Path root;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.startMiniCluster();
server = new DummyServer();
+ params = ImmutableMap.of(HMaster.MASTER, server);
conf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
HMaster.decorateMasterConfiguration(conf);
+ TableDescriptor td = ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName);
+ TEST_UTIL.getAdmin().createTable(td);
+ conf.set(ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME, tableName.getNameAsString());
rp = ReplicationFactory.getReplicationPeers(server.getZooKeeper(), conf);
rp.init();
rq = ReplicationStorageFactory.getReplicationQueueStorage(server.getConnection(), conf);
@@ -92,7 +101,7 @@ public class TestReplicationHFileCleaner {
@AfterClass
public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniZKCluster();
+ TEST_UTIL.shutdownMiniCluster();
}
@Before
@@ -115,6 +124,13 @@ public class TestReplicationHFileCleaner {
rp.getPeerStorage().removePeer(peerId);
}
+ private ReplicationHFileCleaner createCleaner() {
+ ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner();
+ cleaner.setConf(conf);
+ cleaner.init(params);
+ return cleaner;
+ }
+
@Test
public void testIsFileDeletable() throws IOException, ReplicationException {
// 1. Create a file
@@ -122,8 +138,7 @@ public class TestReplicationHFileCleaner {
fs.createNewFile(file);
// 2. Assert file is successfully created
assertTrue("Test file not created!", fs.exists(file));
- ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner();
- cleaner.setConf(conf);
+ ReplicationHFileCleaner cleaner = createCleaner();
// 3. Assert that file as is should be deletable
assertTrue("Cleaner should allow to delete this file as there is no hfile reference node "
+ "for it in the queue.", cleaner.isFileDeletable(fs.getFileStatus(file)));
@@ -160,8 +175,7 @@ public class TestReplicationHFileCleaner {
// 2. Add one file to hfile-refs queue
rq.addHFileRefs(peerId, hfiles);
- ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner();
- cleaner.setConf(conf);
+ ReplicationHFileCleaner cleaner = createCleaner();
Iterator<FileStatus> deletableFilesIterator = cleaner.getDeletableFiles(files).iterator();
int i = 0;
while (deletableFilesIterator.hasNext() && i < 2) {
@@ -182,6 +196,15 @@ public class TestReplicationHFileCleaner {
return TEST_UTIL.getConfiguration();
}
+ @Override
+ public ZKWatcher getZooKeeper() {
+ try {
+ return TEST_UTIL.getZooKeeperWatcher();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
@Override
public Connection getConnection() {
try {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationOffsetUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationOffsetUtil.java
new file mode 100644
index 00000000000..f54a4958374
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationOffsetUtil.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.apache.hadoop.hbase.replication.ReplicationOffsetUtil.shouldReplicate;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ ReplicationTests.class, SmallTests.class })
+public class TestReplicationOffsetUtil {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReplicationOffsetUtil.class);
+
+ @Test
+ public void test() {
+ assertTrue(shouldReplicate(null, "whatever"));
+ assertTrue(shouldReplicate(ReplicationGroupOffset.BEGIN, "whatever"));
+ ServerName sn = ServerName.valueOf("host", 16010, EnvironmentEdgeManager.currentTime());
+ ReplicationGroupOffset offset = new ReplicationGroupOffset(sn + ".12345", 100);
+ assertTrue(shouldReplicate(offset, sn + ".12346"));
+ assertFalse(shouldReplicate(offset, sn + ".12344"));
+ assertTrue(shouldReplicate(offset, sn + ".12345"));
+ // -1 means finish replication, so should not replicate
+ assertFalse(shouldReplicate(new ReplicationGroupOffset(sn + ".12345", -1), sn + ".12345"));
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestLogCleanerBarrier.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestLogCleanerBarrier.java
new file mode 100644
index 00000000000..06cb85523d3
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestLogCleanerBarrier.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication.master;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, SmallTests.class })
+public class TestLogCleanerBarrier {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestLogCleanerBarrier.class);
+
+ @Test
+ public void test() {
+ ReplicationLogCleanerBarrier barrier = new ReplicationLogCleanerBarrier();
+ assertThrows(IllegalStateException.class, () -> barrier.stop());
+ assertThrows(IllegalStateException.class, () -> barrier.enable());
+ assertTrue(barrier.start());
+ assertThrows(IllegalStateException.class, () -> barrier.start());
+ assertThrows(IllegalStateException.class, () -> barrier.enable());
+ assertFalse(barrier.disable());
+ assertThrows(IllegalStateException.class, () -> barrier.enable());
+ barrier.stop();
+
+ for (int i = 0; i < 3; i++) {
+ assertTrue(barrier.disable());
+ assertFalse(barrier.start());
+ }
+ for (int i = 0; i < 3; i++) {
+ assertFalse(barrier.start());
+ barrier.enable();
+ }
+ assertTrue(barrier.start());
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java
new file mode 100644
index 00000000000..7a227fb0603
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication.master;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
+@Category({ MasterTests.class, SmallTests.class })
+public class TestReplicationLogCleaner {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReplicationLogCleaner.class);
+
+ private static final Configuration CONF = HBaseConfiguration.create();
+
+ private MasterServices services;
+
+ private ReplicationLogCleaner cleaner;
+
+ @Before
+ public void setUp() throws ReplicationException {
+ services = mock(MasterServices.class);
+ ReplicationPeerManager rpm = mock(ReplicationPeerManager.class);
+ when(rpm.getReplicationLogCleanerBarrier()).thenReturn(new ReplicationLogCleanerBarrier());
+ when(services.getReplicationPeerManager()).thenReturn(rpm);
+ when(rpm.listPeers(null)).thenReturn(new ArrayList<>());
+ ReplicationQueueStorage rqs = mock(ReplicationQueueStorage.class);
+ when(rpm.getQueueStorage()).thenReturn(rqs);
+ when(rqs.listAllQueues()).thenReturn(new ArrayList<>());
+ ServerManager sm = mock(ServerManager.class);
+ when(services.getServerManager()).thenReturn(sm);
+ when(sm.getOnlineServersList()).thenReturn(new ArrayList<>());
+ @SuppressWarnings("unchecked")
+ ProcedureExecutor<MasterProcedureEnv> procExec = mock(ProcedureExecutor.class);
+ when(services.getMasterProcedureExecutor()).thenReturn(procExec);
+ when(procExec.getProcedures()).thenReturn(new ArrayList<>());
+
+ cleaner = new ReplicationLogCleaner();
+ cleaner.setConf(CONF);
+ Map<String, Object> params = ImmutableMap.of(HMaster.MASTER, services);
+ cleaner.init(params);
+ }
+
+ @After
+ public void tearDown() {
+ cleaner.postClean();
+ }
+
+ private static Iterable<FileStatus> runCleaner(ReplicationLogCleaner cleaner,
+ Iterable<FileStatus> files) {
+ cleaner.preClean();
+ return cleaner.getDeletableFiles(files);
+ }
+
+ private static FileStatus createFileStatus(Path path) {
+ return new FileStatus(100, false, 3, 256, EnvironmentEdgeManager.currentTime(), path);
+ }
+
+ private static FileStatus createFileStatus(ServerName sn, int number) {
+ Path path = new Path(sn.toString() + "." + number);
+ return createFileStatus(path);
+ }
+
+ private static ReplicationPeerDescription createPeer(String peerId) {
+ return new ReplicationPeerDescription(peerId, true, null, null);
+ }
+
+ private void addServer(ServerName serverName) {
+ services.getServerManager().getOnlineServersList().add(serverName);
+ }
+
+ private void addSCP(ServerName serverName, boolean finished) {
+ ServerCrashProcedure scp = mock(ServerCrashProcedure.class);
+ when(scp.getServerName()).thenReturn(serverName);
+ when(scp.isFinished()).thenReturn(finished);
+ services.getMasterProcedureExecutor().getProcedures().add(scp);
+ }
+
+ private void addPeer(String... peerIds) {
+ services.getReplicationPeerManager().listPeers(null).addAll(
+ Stream.of(peerIds).map(TestReplicationLogCleaner::createPeer).collect(Collectors.toList()));
+ }
+
+ private void addQueueData(ReplicationQueueData... datas) throws ReplicationException {
+ services.getReplicationPeerManager().getQueueStorage().listAllQueues()
+ .addAll(Arrays.asList(datas));
+ }
+
+ @Test
+ public void testNoConf() {
+ ReplicationLogCleaner cleaner = new ReplicationLogCleaner();
+ List<FileStatus> files = Arrays.asList(new FileStatus());
+ assertSame(files, runCleaner(cleaner, files));
+ cleaner.postClean();
+ }
+
+ @Test
+ public void testCanNotFilter() {
+ assertTrue(services.getReplicationPeerManager().getReplicationLogCleanerBarrier().disable());
+ List<FileStatus> files = Arrays.asList(new FileStatus());
+ assertSame(Collections.emptyList(), runCleaner(cleaner, files));
+ }
+
+ @Test
+ public void testNoPeer() {
+ Path path = new Path("/wal." + EnvironmentEdgeManager.currentTime());
+ assertTrue(AbstractFSWALProvider.validateWALFilename(path.getName()));
+ FileStatus file = createFileStatus(path);
+ Iterator<FileStatus> iter = runCleaner(cleaner, Arrays.asList(file)).iterator();
+ assertSame(file, iter.next());
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testNotValidWalFile() {
+ addPeer("1");
+ Path path = new Path("/whatever");
+ assertFalse(AbstractFSWALProvider.validateWALFilename(path.getName()));
+ FileStatus file = createFileStatus(path);
+ Iterator<FileStatus> iter = runCleaner(cleaner, Arrays.asList(file)).iterator();
+ assertSame(file, iter.next());
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testMetaWalFile() {
+ addPeer("1");
+ Path path = new Path(
+ "/wal." + EnvironmentEdgeManager.currentTime() + AbstractFSWALProvider.META_WAL_PROVIDER_ID);
+ assertTrue(AbstractFSWALProvider.validateWALFilename(path.getName()));
+ assertTrue(AbstractFSWALProvider.isMetaFile(path));
+ FileStatus file = createFileStatus(path);
+ Iterator<FileStatus> iter = runCleaner(cleaner, Arrays.asList(file)).iterator();
+ assertSame(file, iter.next());
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testLiveRegionServerNoQueues() {
+ addPeer("1");
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ addServer(sn);
+ List<FileStatus> files = Arrays.asList(createFileStatus(sn, 1));
+ assertThat(runCleaner(cleaner, files), emptyIterable());
+ }
+
+ @Test
+ public void testLiveRegionServerWithSCPNoQueues() {
+ addPeer("1");
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ addSCP(sn, false);
+ List<FileStatus> files = Arrays.asList(createFileStatus(sn, 1));
+ assertThat(runCleaner(cleaner, files), emptyIterable());
+ }
+
+ @Test
+ public void testDeadRegionServerNoQueues() {
+ addPeer("1");
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ FileStatus file = createFileStatus(sn, 1);
+ Iterator<FileStatus> iter = runCleaner(cleaner, Arrays.asList(file)).iterator();
+ assertSame(file, iter.next());
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testDeadRegionServerWithSCPNoQueues() {
+ addPeer("1");
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ addSCP(sn, true);
+ FileStatus file = createFileStatus(sn, 1);
+ Iterator<FileStatus> iter = runCleaner(cleaner, Arrays.asList(file)).iterator();
+ assertSame(file, iter.next());
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testLiveRegionServerMissingQueue() throws ReplicationException {
+ String peerId1 = "1";
+ String peerId2 = "2";
+ addPeer(peerId1, peerId2);
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ addServer(sn);
+ FileStatus file = createFileStatus(sn, 1);
+ ReplicationQueueData data1 = new ReplicationQueueData(new ReplicationQueueId(sn, peerId1),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), -1)));
+ addQueueData(data1);
+ assertThat(runCleaner(cleaner, Arrays.asList(file)), emptyIterable());
+ }
+
+ @Test
+ public void testLiveRegionServerShouldNotDelete() throws ReplicationException {
+ String peerId = "1";
+ addPeer(peerId);
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ addServer(sn);
+ FileStatus file = createFileStatus(sn, 1);
+ ReplicationQueueData data = new ReplicationQueueData(new ReplicationQueueId(sn, peerId),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), 0)));
+ addQueueData(data);
+ assertThat(runCleaner(cleaner, Arrays.asList(file)), emptyIterable());
+ }
+
+ @Test
+ public void testLiveRegionServerShouldNotDeleteTwoPeers() throws ReplicationException {
+ String peerId1 = "1";
+ String peerId2 = "2";
+ addPeer(peerId1, peerId2);
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ addServer(sn);
+ FileStatus file = createFileStatus(sn, 1);
+ ReplicationQueueData data1 = new ReplicationQueueData(new ReplicationQueueId(sn, peerId1),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), -1)));
+ ReplicationQueueData data2 = new ReplicationQueueData(new ReplicationQueueId(sn, peerId2),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), 0)));
+ addQueueData(data1, data2);
+ assertThat(runCleaner(cleaner, Arrays.asList(file)), emptyIterable());
+ }
+
+ @Test
+ public void testLiveRegionServerShouldDelete() throws ReplicationException {
+ String peerId = "1";
+ addPeer(peerId);
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ addServer(sn);
+ FileStatus file = createFileStatus(sn, 1);
+ ReplicationQueueData data = new ReplicationQueueData(new ReplicationQueueId(sn, peerId),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), -1)));
+ services.getReplicationPeerManager().getQueueStorage().listAllQueues().add(data);
+ Iterator<FileStatus> iter = runCleaner(cleaner, Arrays.asList(file)).iterator();
+ assertSame(file, iter.next());
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testLiveRegionServerShouldDeleteTwoPeers() throws ReplicationException {
+ String peerId1 = "1";
+ String peerId2 = "2";
+ addPeer(peerId1, peerId2);
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ addServer(sn);
+ FileStatus file = createFileStatus(sn, 1);
+ ReplicationQueueData data1 = new ReplicationQueueData(new ReplicationQueueId(sn, peerId1),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), -1)));
+ ReplicationQueueData data2 = new ReplicationQueueData(new ReplicationQueueId(sn, peerId2),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), -1)));
+ addQueueData(data1, data2);
+ Iterator<FileStatus> iter = runCleaner(cleaner, Arrays.asList(file)).iterator();
+ assertSame(file, iter.next());
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testDeadRegionServerMissingQueue() throws ReplicationException {
+ String peerId1 = "1";
+ String peerId2 = "2";
+ addPeer(peerId1, peerId2);
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ FileStatus file = createFileStatus(sn, 1);
+ ReplicationQueueData data1 = new ReplicationQueueData(new ReplicationQueueId(sn, peerId1),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), -1)));
+ addQueueData(data1);
+ Iterator<FileStatus> iter = runCleaner(cleaner, Arrays.asList(file)).iterator();
+ assertSame(file, iter.next());
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testDeadRegionServerShouldNotDelete() throws ReplicationException {
+ String peerId = "1";
+ addPeer(peerId);
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ FileStatus file = createFileStatus(sn, 1);
+ ReplicationQueueData data = new ReplicationQueueData(new ReplicationQueueId(sn, peerId),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), 0)));
+ addQueueData(data);
+ assertThat(runCleaner(cleaner, Arrays.asList(file)), emptyIterable());
+ }
+
+ @Test
+ public void testDeadRegionServerShouldNotDeleteTwoPeers() throws ReplicationException {
+ String peerId1 = "1";
+ String peerId2 = "2";
+ addPeer(peerId1, peerId2);
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ FileStatus file = createFileStatus(sn, 1);
+ ReplicationQueueData data1 = new ReplicationQueueData(new ReplicationQueueId(sn, peerId1),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), -1)));
+ ReplicationQueueData data2 = new ReplicationQueueData(new ReplicationQueueId(sn, peerId2),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), 0)));
+ addQueueData(data1, data2);
+ assertThat(runCleaner(cleaner, Arrays.asList(file)), emptyIterable());
+ }
+
+ @Test
+ public void testDeadRegionServerShouldDelete() throws ReplicationException {
+ String peerId = "1";
+ addPeer(peerId);
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ FileStatus file = createFileStatus(sn, 1);
+ ReplicationQueueData data = new ReplicationQueueData(new ReplicationQueueId(sn, peerId),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), -1)));
+ services.getReplicationPeerManager().getQueueStorage().listAllQueues().add(data);
+ Iterator<FileStatus> iter = runCleaner(cleaner, Arrays.asList(file)).iterator();
+ assertSame(file, iter.next());
+ assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testDeadRegionServerShouldDeleteTwoPeers() throws ReplicationException {
+ String peerId1 = "1";
+ String peerId2 = "2";
+ addPeer(peerId1, peerId2);
+ ServerName sn = ServerName.valueOf("server,123," + EnvironmentEdgeManager.currentTime());
+ FileStatus file = createFileStatus(sn, 1);
+ ReplicationQueueData data1 = new ReplicationQueueData(new ReplicationQueueId(sn, peerId1),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), -1)));
+ ReplicationQueueData data2 = new ReplicationQueueData(new ReplicationQueueId(sn, peerId2),
+ ImmutableMap.of(sn.toString(), new ReplicationGroupOffset(file.getPath().getName(), -1)));
+ addQueueData(data1, data2);
+ Iterator<FileStatus> iter = runCleaner(cleaner, Arrays.asList(file)).iterator();
+ assertSame(file, iter.next());
+ assertFalse(iter.hasNext());
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index 6aba327d791..b7564ed9168 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -190,7 +190,7 @@ public class TestReplicationSourceManager {
replication = new Replication();
replication.initialize(server, FS, logDir, oldLogDir,
- new WALFactory(CONF, "test", null, false));
+ new WALFactory(CONF, server.getServerName(), null, false));
manager = replication.getReplicationManager();
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
index b59ebc0d9a6..26c1152c05a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
@@ -630,7 +630,7 @@ public class TestWALFactory {
assertEquals(wrappedWALProvider.getClass(), walFactory.getMetaProvider().getClass());
// if providers are not set and do not enable SyncReplicationWALProvider
- walFactory = new WALFactory(conf, this.currentServername.toString(), null, false);
+ walFactory = new WALFactory(conf, this.currentServername, null, false);
assertEquals(walFactory.getWALProvider().getClass(), walFactory.getMetaProvider().getClass());
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java
index 8273b3d6041..6a1e98d9fd5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
@@ -183,4 +184,17 @@ public class TestWALMethods {
return entry;
}
+ @Test
+ public void testParseServerNameFromWALName() {
+ assertEquals(ServerName.valueOf("abc,123,123"),
+ AbstractFSWALProvider.parseServerNameFromWALName("abc,123,123.1.12345.meta"));
+ assertEquals(ServerName.valueOf("abc,123,123"),
+ AbstractFSWALProvider.parseServerNameFromWALName("abc,123,123.12345"));
+ assertEquals(ServerName.valueOf("abc,123,123"),
+ AbstractFSWALProvider.parseServerNameFromWALName("abc,123,123"));
+ assertThrows(IllegalArgumentException.class,
+ () -> AbstractFSWALProvider.parseServerNameFromWALName("test,abc,123,123.12345"));
+ assertThrows(IllegalArgumentException.class,
+ () -> AbstractFSWALProvider.parseServerNameFromWALName("abc"));
+ }
}
[hbase] 09/11: HBASE-27429 Add exponential retry backoff support for MigrateReplicationQueueFromZkToTableProcedure
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f1d491205ce0fe8a7a428568d21f5ac65eed32c3
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Tue Oct 18 16:46:03 2022 +0800
HBASE-27429 Add exponential retry backoff support for MigrateReplicationQueueFromZkToTableProcedure
Signed-off-by: Liangjun He <he...@apache.org>
---
.../hbase/procedure2/TimeoutExecutorThread.java | 10 +-
...rateReplicationQueueFromZkToTableProcedure.java | 131 ++++++++++++++-------
.../master/replication/ReplicationPeerManager.java | 45 ++++---
...tReplicationPeerManagerMigrateQueuesFromZk.java | 9 +-
4 files changed, 125 insertions(+), 70 deletions(-)
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java
index 3b99781a558..c0287a99435 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java
@@ -78,9 +78,13 @@ class TimeoutExecutorThread<TEnvironment> extends StoppableThread {
}
public void add(Procedure<TEnvironment> procedure) {
- LOG.info("ADDED {}; timeout={}, timestamp={}", procedure, procedure.getTimeout(),
- procedure.getTimeoutTimestamp());
- queue.add(new DelayedProcedure<>(procedure));
+ if (procedure.getTimeout() > 0) {
+ LOG.info("ADDED {}; timeout={}, timestamp={}", procedure, procedure.getTimeout(),
+ procedure.getTimeoutTimestamp());
+ queue.add(new DelayedProcedure<>(procedure));
+ } else {
+ LOG.info("Got negative timeout {} for {}, skip adding", procedure.getTimeout(), procedure);
+ }
}
public boolean remove(Procedure<TEnvironment> procedure) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
index 536f232338e..93ff27db3f7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
@@ -25,19 +25,25 @@ import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureP
import java.io.IOException;
import java.util.List;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
+import java.util.function.LongConsumer;
import java.util.stream.Collectors;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.master.procedure.GlobalProcedureInterface;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration;
+import org.apache.hadoop.hbase.util.FutureUtils;
+import org.apache.hadoop.hbase.util.IdLock;
+import org.apache.hadoop.hbase.util.RetryCounter;
import org.apache.hadoop.hbase.util.VersionInfo;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
@@ -65,18 +71,34 @@ public class MigrateReplicationQueueFromZkToTableProcedure
private List<String> disabledPeerIds;
- private List<Future<?>> futures;
+ private CompletableFuture<?> future;
private ExecutorService executor;
+ private RetryCounter retryCounter;
+
@Override
public String getGlobalId() {
return getClass().getSimpleName();
}
+ private ProcedureSuspendedException suspend(Configuration conf, LongConsumer backoffConsumer)
+ throws ProcedureSuspendedException {
+ if (retryCounter == null) {
+ retryCounter = ProcedureUtil.createRetryCounter(conf);
+ }
+ long backoff = retryCounter.getBackoffTimeAndIncrementAttempts();
+ backoffConsumer.accept(backoff);
+ throw suspend(Math.toIntExact(backoff), true);
+ }
+
+ private void resetRetry() {
+ retryCounter = null;
+ }
+
private ExecutorService getExecutorService() {
if (executor == null) {
- executor = Executors.newFixedThreadPool(3, new ThreadFactoryBuilder()
+ executor = Executors.newCachedThreadPool(new ThreadFactoryBuilder()
.setNameFormat(getClass().getSimpleName() + "-%d").setDaemon(true).build());
}
return executor;
@@ -95,14 +117,17 @@ public class MigrateReplicationQueueFromZkToTableProcedure
peerProcCount = env.getMasterServices().getProcedures().stream()
.filter(p -> p instanceof PeerProcedureInterface).filter(p -> !p.isFinished()).count();
} catch (IOException e) {
- LOG.warn("failed to check peer procedure status", e);
- throw suspend(5000, true);
+ throw suspend(env.getMasterConfiguration(),
+ backoff -> LOG.warn("failed to check peer procedure status, sleep {} secs and retry later",
+ backoff / 1000, e));
}
if (peerProcCount > 0) {
- LOG.info("There are still {} pending peer procedures, will sleep and check later",
- peerProcCount);
- throw suspend(10_000, true);
+ throw suspend(env.getMasterConfiguration(),
+ backoff -> LOG.info(
+ "There are still {} pending peer procedures, sleep {} secs and retry later",
+ peerProcCount, backoff / 1000));
}
+ resetRetry();
LOG.info("No pending peer procedures found, continue...");
}
@@ -122,8 +147,10 @@ public class MigrateReplicationQueueFromZkToTableProcedure
try {
oldStorage.deleteAllData();
} catch (KeeperException e) {
- LOG.warn("failed to delete old replication queue data, sleep and retry later", e);
- suspend(10_000, true);
+ throw suspend(env.getMasterConfiguration(),
+ backoff -> LOG.warn(
+ "failed to delete old replication queue data, sleep {} secs and retry later",
+ backoff / 1000, e));
}
return Flow.NO_MORE_STATE;
}
@@ -132,6 +159,7 @@ public class MigrateReplicationQueueFromZkToTableProcedure
disabledPeerIds = peers.stream().filter(ReplicationPeerDescription::isEnabled)
.map(ReplicationPeerDescription::getPeerId).collect(Collectors.toList());
setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER);
+ resetRetry();
return Flow.HAS_MORE_STATE;
case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER:
for (String peerId : disabledPeerIds) {
@@ -140,39 +168,52 @@ public class MigrateReplicationQueueFromZkToTableProcedure
setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE);
return Flow.HAS_MORE_STATE;
case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE:
- if (futures != null) {
- // wait until all futures done
- long notDone = futures.stream().filter(f -> !f.isDone()).count();
- if (notDone == 0) {
- boolean succ = true;
- for (Future<?> future : futures) {
- try {
- future.get();
- } catch (Exception e) {
- succ = false;
- LOG.warn("Failed to migrate", e);
- }
- }
- if (succ) {
- shutdownExecutorService();
- setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING);
- return Flow.HAS_MORE_STATE;
- }
- // reschedule to retry migration again
- futures = null;
- } else {
- LOG.info("There still {} pending migration tasks, will sleep and check later", notDone);
- throw suspend(10_000, true);
+ if (future != null) {
+ // should have finished when we arrive here
+ assert future.isDone();
+ try {
+ future.get();
+ } catch (Exception e) {
+ future = null;
+ throw suspend(env.getMasterConfiguration(),
+ backoff -> LOG.warn("failed to migrate queue data, sleep {} secs and retry later",
+ backoff / 1000, e));
}
+ shutdownExecutorService();
+ setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING);
+ resetRetry();
+ return Flow.HAS_MORE_STATE;
}
- try {
- futures = env.getReplicationPeerManager()
- .migrateQueuesFromZk(env.getMasterServices().getZooKeeper(), getExecutorService());
- } catch (IOException e) {
- LOG.warn("failed to submit migration tasks", e);
- throw suspend(10_000, true);
- }
- throw suspend(10_000, true);
+ future = env.getReplicationPeerManager()
+ .migrateQueuesFromZk(env.getMasterServices().getZooKeeper(), getExecutorService());
+ FutureUtils.addListener(future, (r, e) -> {
+ // should acquire procedure execution lock to make sure that the procedure executor has
+ // finished putting this procedure to the WAITING_TIMEOUT state, otherwise there could be
+ // race and cause unexpected result
+ IdLock procLock =
+ env.getMasterServices().getMasterProcedureExecutor().getProcExecutionLock();
+ IdLock.Entry lockEntry;
+ try {
+ lockEntry = procLock.getLockEntry(getProcId());
+ } catch (IOException ioe) {
+ LOG.error("Error while acquiring execution lock for procedure {}"
+ + " when trying to wake it up, aborting...", ioe);
+ env.getMasterServices().abort("Can not acquire procedure execution lock", e);
+ return;
+ }
+ try {
+ setTimeoutFailure(env);
+ } finally {
+ procLock.releaseLockEntry(lockEntry);
+ }
+ });
+ // here we set timeout to -1 so the ProcedureExecutor will not schedule a Timer for us
+ setTimeout(-1);
+ setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
+ // skip persistence is a must now since when restarting, if the procedure is in
+ // WAITING_TIMEOUT state and has -1 as timeout, it will block there forever...
+ skipPersistence();
+ throw new ProcedureSuspendedException();
case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING:
long rsWithLowerVersion =
env.getMasterServices().getServerManager().getOnlineServers().values().stream()
@@ -181,9 +222,11 @@ public class MigrateReplicationQueueFromZkToTableProcedure
setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER);
return Flow.HAS_MORE_STATE;
} else {
- LOG.info("There are still {} region servers which have a major version less than {}, "
- + "will sleep and check later", rsWithLowerVersion, MIN_MAJOR_VERSION);
- throw suspend(10_000, true);
+ throw suspend(env.getMasterConfiguration(),
+ backoff -> LOG.warn(
+ "There are still {} region servers which have a major version"
+ + " less than {}, sleep {} secs and check later",
+ rsWithLowerVersion, MIN_MAJOR_VERSION, backoff / 1000));
}
case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER:
for (String peerId : disabledPeerIds) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 81f569c3f9e..d8c1b5c64c5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -21,7 +21,6 @@ import com.google.errorprone.annotations.RestrictedApi;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
@@ -29,10 +28,10 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
@@ -70,6 +69,7 @@ import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration
import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration.ZkLastPushedSeqId;
import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration.ZkReplicationQueueData;
import org.apache.hadoop.hbase.replication.master.ReplicationLogCleanerBarrier;
+import org.apache.hadoop.hbase.util.FutureUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
@@ -778,25 +778,38 @@ public class ReplicationPeerManager {
}
}
+ private interface ExceptionalRunnable {
+ void run() throws Exception;
+ }
+
+ private CompletableFuture<?> runAsync(ExceptionalRunnable task, ExecutorService executor) {
+ CompletableFuture<?> future = new CompletableFuture<>();
+ executor.execute(() -> {
+ try {
+ task.run();
+ future.complete(null);
+ } catch (Exception e) {
+ future.completeExceptionally(e);
+ }
+ });
+ return future;
+ }
+
/**
- * Submit the migration tasks to the given {@code executor} and return the futures.
+ * Submit the migration tasks to the given {@code executor}.
*/
- List<Future<?>> migrateQueuesFromZk(ZKWatcher zookeeper, ExecutorService executor)
- throws IOException {
+ CompletableFuture<?> migrateQueuesFromZk(ZKWatcher zookeeper, ExecutorService executor) {
// the replication queue table creation is asynchronous and will be triggered by addPeer, so
// here we need to manually initialize it since we will not call addPeer.
- initializeQueueStorage();
+ try {
+ initializeQueueStorage();
+ } catch (IOException e) {
+ return FutureUtils.failedFuture(e);
+ }
ZKReplicationQueueStorageForMigration oldStorage =
new ZKReplicationQueueStorageForMigration(zookeeper, conf);
- return Arrays.asList(executor.submit(() -> {
- migrateQueues(oldStorage);
- return null;
- }), executor.submit(() -> {
- migrateLastPushedSeqIds(oldStorage);
- return null;
- }), executor.submit(() -> {
- migrateHFileRefs(oldStorage);
- return null;
- }));
+ return CompletableFuture.allOf(runAsync(() -> migrateQueues(oldStorage), executor),
+ runAsync(() -> migrateLastPushedSeqIds(oldStorage), executor),
+ runAsync(() -> migrateHFileRefs(oldStorage), executor));
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestReplicationPeerManagerMigrateQueuesFromZk.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestReplicationPeerManagerMigrateQueuesFromZk.java
index 73915e856ea..2d3b950ff82 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestReplicationPeerManagerMigrateQueuesFromZk.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestReplicationPeerManagerMigrateQueuesFromZk.java
@@ -34,7 +34,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
@@ -146,9 +145,7 @@ public class TestReplicationPeerManagerMigrateQueuesFromZk {
@Test
public void testNoPeers() throws Exception {
prepareData();
- for (Future<?> future : manager.migrateQueuesFromZk(UTIL.getZooKeeperWatcher(), EXECUTOR)) {
- future.get(1, TimeUnit.MINUTES);
- }
+ manager.migrateQueuesFromZk(UTIL.getZooKeeperWatcher(), EXECUTOR).get(1, TimeUnit.MINUTES);
// should have called initializer
verify(queueStorageInitializer).initialize();
// should have not migrated any data since there is no peer
@@ -165,9 +162,7 @@ public class TestReplicationPeerManagerMigrateQueuesFromZk {
// value is not used in this test, so just add a mock
peers.put("peer_" + i, mock(ReplicationPeerDescription.class));
}
- for (Future<?> future : manager.migrateQueuesFromZk(UTIL.getZooKeeperWatcher(), EXECUTOR)) {
- future.get(1, TimeUnit.MINUTES);
- }
+ manager.migrateQueuesFromZk(UTIL.getZooKeeperWatcher(), EXECUTOR).get(1, TimeUnit.MINUTES);
// should have called initializer
verify(queueStorageInitializer).initialize();
List<ReplicationQueueData> queueDatas = queueStorage.listAllQueues();
[hbase] 04/11: HBASE-27215 Add support for sync replication (#4762)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit d24e7cf39f98beeaa94c997953171a71bbab2447
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Thu Sep 15 22:58:29 2022 +0800
HBASE-27215 Add support for sync replication (#4762)
Signed-off-by: Xiaolin Ha <ha...@apache.org>
---
.../regionserver/ReplicationSource.java | 2 +-
.../regionserver/ReplicationSourceManager.java | 53 +++++++++++-----------
.../TestDrainReplicationQueuesForStandBy.java | 3 --
3 files changed, 28 insertions(+), 30 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index e078722b157..0784a87711b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -465,7 +465,7 @@ public class ReplicationSource implements ReplicationSourceInterface {
t.getName());
manager.refreshSources(peerId);
break;
- } catch (IOException e1) {
+ } catch (IOException | ReplicationException e1) {
LOG.error("Replication sources refresh failed.", e1);
sleepForRetries("Sleeping before try refreshing sources again", maxRetriesMultiplier);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 03569be86fc..f3d07315240 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -403,38 +403,44 @@ public class ReplicationSourceManager {
// TODO: use empty initial offsets for now, revisit when adding support for sync replication
ReplicationSourceInterface src =
createSource(new ReplicationQueueData(queueId, ImmutableMap.of()), peer);
- // synchronized here to avoid race with preLogRoll where we add new log to source and also
+ // synchronized here to avoid race with postLogRoll where we add new log to source and also
// walsById.
ReplicationSourceInterface toRemove;
- Map<String, NavigableSet<String>> wals = new HashMap<>();
+ ReplicationQueueData queueData;
synchronized (latestPaths) {
+ // Here we make a copy of all the remaining wal files and then delete them from the
+ // replication queue storage after releasing the lock. It is not safe to just remove the old
+ // map from walsById since later we may fail to update the replication queue storage, and when
+ // we retry next time, we can not know the wal files that needs to be set to the replication
+ // queue storage
+ ImmutableMap.Builder<String, ReplicationGroupOffset> builder = ImmutableMap.builder();
+ synchronized (walsById) {
+ walsById.get(queueId).forEach((group, wals) -> {
+ if (!wals.isEmpty()) {
+ builder.put(group, new ReplicationGroupOffset(wals.last(), -1));
+ }
+ });
+ }
+ queueData = new ReplicationQueueData(queueId, builder.build());
+ src = createSource(queueData, peer);
toRemove = sources.put(peerId, src);
if (toRemove != null) {
LOG.info("Terminate replication source for " + toRemove.getPeerId());
toRemove.terminate(terminateMessage);
toRemove.getSourceMetrics().clear();
}
- // Here we make a copy of all the remaining wal files and then delete them from the
- // replication queue storage after releasing the lock. It is not safe to just remove the old
- // map from walsById since later we may fail to delete them from the replication queue
- // storage, and when we retry next time, we can not know the wal files that need to be deleted
- // from the replication queue storage.
- walsById.get(queueId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));
+ }
+ for (Map.Entry<String, ReplicationGroupOffset> entry : queueData.getOffsets().entrySet()) {
+ queueStorage.setOffset(queueId, entry.getKey(), entry.getValue(), Collections.emptyMap());
}
LOG.info("Startup replication source for " + src.getPeerId());
src.startup();
- for (NavigableSet<String> walsByGroup : wals.values()) {
- // TODO: just need to reset the replication offset
- // for (String wal : walsByGroup) {
- // queueStorage.removeWAL(server.getServerName(), peerId, wal);
- // }
- }
synchronized (walsById) {
- Map<String, NavigableSet<String>> oldWals = walsById.get(queueId);
- wals.forEach((k, v) -> {
- NavigableSet<String> walsByGroup = oldWals.get(k);
+ Map<String, NavigableSet<String>> wals = walsById.get(queueId);
+ queueData.getOffsets().forEach((group, offset) -> {
+ NavigableSet<String> walsByGroup = wals.get(group);
if (walsByGroup != null) {
- walsByGroup.removeAll(v);
+ walsByGroup.headSet(offset.getWal(), true).clear();
}
});
}
@@ -457,13 +463,8 @@ public class ReplicationSourceManager {
}
private ReplicationSourceInterface createRefreshedSource(ReplicationQueueId queueId,
- ReplicationPeer peer) throws IOException {
- Map<String, ReplicationGroupOffset> offsets;
- try {
- offsets = queueStorage.getOffsets(queueId);
- } catch (ReplicationException e) {
- throw new IOException(e);
- }
+ ReplicationPeer peer) throws IOException, ReplicationException {
+ Map<String, ReplicationGroupOffset> offsets = queueStorage.getOffsets(queueId);
return createSource(new ReplicationQueueData(queueId, ImmutableMap.copyOf(offsets)), peer);
}
@@ -473,7 +474,7 @@ public class ReplicationSourceManager {
* replication queue storage and only to enqueue all logs to the new replication source
* @param peerId the id of the replication peer
*/
- public void refreshSources(String peerId) throws IOException {
+ public void refreshSources(String peerId) throws ReplicationException, IOException {
String terminateMessage = "Peer " + peerId
+ " state or config changed. Will close the previous replication source and open a new one";
ReplicationPeer peer = replicationPeers.getPeer(peerId);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java
index 8918f8422e1..0189d475575 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java
@@ -35,12 +35,9 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-// TODO: revisit later
-@Ignore
@Category({ ReplicationTests.class, MediumTests.class })
public class TestDrainReplicationQueuesForStandBy extends SyncReplicationTestBase {
[hbase] 02/11: HBASE-27213 Add support for claim queue operation (#4708)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a38d57448450bed4478a2b101caca1632003688e
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Sat Aug 20 23:10:58 2022 +0800
HBASE-27213 Add support for claim queue operation (#4708)
Signed-off-by: Xin Sun <dd...@gmail.com>
---
.../protobuf/server/master/MasterProcedure.proto | 6 +--
.../AssignReplicationQueuesProcedure.java | 13 ++---
.../master/replication/ModifyPeerProcedure.java | 2 +-
.../master/replication/RemovePeerProcedure.java | 41 +++++++++++++-
.../regionserver/ReplicationSourceManager.java | 37 +++++++++----
.../replication/TestClaimReplicationQueue.java | 2 +-
...java => TestRemovePeerProcedureWaitForSCP.java} | 63 +++++++++++++---------
.../replication/TestSerialReplicationFailover.java | 3 --
8 files changed, 116 insertions(+), 51 deletions(-)
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
index 2e0da0deb84..76a1d676487 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
@@ -515,6 +515,7 @@ message UpdatePeerConfigStateData {
message RemovePeerStateData {
optional ReplicationPeer peer_config = 1;
+ repeated int64 ongoing_assign_replication_queues_proc_ids = 2;
}
message EnablePeerStateData {
@@ -714,9 +715,8 @@ message ModifyColumnFamilyStoreFileTrackerStateData {
}
enum AssignReplicationQueuesState {
- ASSIGN_REPLICATION_QUEUES_PRE_CHECK = 1;
- ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES = 2;
- ASSIGN_REPLICATION_QUEUES_CLAIM = 3;
+ ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES = 1;
+ ASSIGN_REPLICATION_QUEUES_CLAIM = 2;
}
message AssignReplicationQueuesStateData {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
index e7fb5e51715..d33259dd436 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
@@ -23,6 +23,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface;
@@ -102,8 +103,12 @@ public class AssignReplicationQueuesProcedure
}
private Flow claimQueues(MasterProcedureEnv env) throws ReplicationException {
+ Set<String> existingPeerIds = env.getReplicationPeerManager().listPeers(null).stream()
+ .map(ReplicationPeerDescription::getPeerId).collect(Collectors.toSet());
ReplicationQueueStorage storage = env.getReplicationPeerManager().getQueueStorage();
- List<ReplicationQueueId> queueIds = storage.listAllQueueIds(crashedServer);
+ // filter out replication queue for deleted peers
+ List<ReplicationQueueId> queueIds = storage.listAllQueueIds(crashedServer).stream()
+ .filter(q -> existingPeerIds.contains(q.getPeerId())).collect(Collectors.toList());
if (queueIds.isEmpty()) {
LOG.debug("Finish claiming replication queues for {}", crashedServer);
// we are done
@@ -130,10 +135,6 @@ public class AssignReplicationQueuesProcedure
throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
try {
switch (state) {
- case ASSIGN_REPLICATION_QUEUES_PRE_CHECK:
- // TODO: reserved for implementing the fencing logic with Add/Remove/UpdatePeerProcedure
- setNextState(AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES);
- return Flow.HAS_MORE_STATE;
case ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES:
addMissingQueues(env);
retryCounter = null;
@@ -183,7 +184,7 @@ public class AssignReplicationQueuesProcedure
@Override
protected AssignReplicationQueuesState getInitialState() {
- return AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_PRE_CHECK;
+ return AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES;
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index 67d70a166be..78b97620c01 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -74,7 +74,7 @@ public abstract class ModifyPeerProcedure extends AbstractPeerProcedure<PeerModi
* update the peer storage.
*/
protected abstract void postPeerModification(MasterProcedureEnv env)
- throws IOException, ReplicationException;
+ throws IOException, ReplicationException, ProcedureSuspendedException;
protected void releaseLatch(MasterProcedureEnv env) {
ProcedurePrepareLatch.releaseLatch(latch, this);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
index 2042e846849..2fadc3fd664 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/RemovePeerProcedure.java
@@ -18,10 +18,17 @@
package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.yetus.audience.InterfaceAudience;
@@ -40,6 +47,8 @@ public class RemovePeerProcedure extends ModifyPeerProcedure {
private ReplicationPeerConfig peerConfig;
+ private List<Long> ongoingAssignReplicationQueuesProcIds = Collections.emptyList();
+
public RemovePeerProcedure() {
}
@@ -64,15 +73,43 @@ public class RemovePeerProcedure extends ModifyPeerProcedure {
@Override
protected void updatePeerStorage(MasterProcedureEnv env) throws ReplicationException {
env.getReplicationPeerManager().removePeer(peerId);
+ // record ongoing AssignReplicationQueuesProcedures after we update the peer storage
+ ongoingAssignReplicationQueuesProcIds = env.getMasterServices().getMasterProcedureExecutor()
+ .getProcedures().stream().filter(p -> p instanceof AssignReplicationQueuesProcedure)
+ .filter(p -> !p.isFinished()).map(Procedure::getProcId).collect(Collectors.toList());
}
private void removeRemoteWALs(MasterProcedureEnv env) throws IOException {
env.getMasterServices().getSyncReplicationReplayWALManager().removePeerRemoteWALs(peerId);
}
+ private void checkAssignReplicationQueuesFinished(MasterProcedureEnv env)
+ throws ProcedureSuspendedException {
+ if (ongoingAssignReplicationQueuesProcIds.isEmpty()) {
+ LOG.info("No ongoing assign replication queues procedures when removing peer {}, move on",
+ peerId);
+ }
+ ProcedureExecutor<MasterProcedureEnv> procExec =
+ env.getMasterServices().getMasterProcedureExecutor();
+ long[] unfinishedProcIds =
+ ongoingAssignReplicationQueuesProcIds.stream().map(procExec::getProcedure)
+ .filter(p -> p != null && !p.isFinished()).mapToLong(Procedure::getProcId).toArray();
+ if (unfinishedProcIds.length == 0) {
+ LOG.info(
+ "All assign replication queues procedures are finished when removing peer {}, move on",
+ peerId);
+ } else {
+ throw suspend(env.getMasterConfiguration(), backoff -> LOG.info(
+ "There are still {} pending assign replication queues procedures {} when removing peer {}, sleep {} secs",
+ unfinishedProcIds.length, Arrays.toString(unfinishedProcIds), peerId, backoff / 1000));
+ }
+ }
+
@Override
protected void postPeerModification(MasterProcedureEnv env)
- throws IOException, ReplicationException {
+ throws IOException, ReplicationException, ProcedureSuspendedException {
+ checkAssignReplicationQueuesFinished(env);
+
if (peerConfig.isSyncReplication()) {
removeRemoteWALs(env);
}
@@ -94,6 +131,7 @@ public class RemovePeerProcedure extends ModifyPeerProcedure {
if (peerConfig != null) {
builder.setPeerConfig(ReplicationPeerConfigUtil.convert(peerConfig));
}
+ builder.addAllOngoingAssignReplicationQueuesProcIds(ongoingAssignReplicationQueuesProcIds);
serializer.serialize(builder.build());
}
@@ -104,5 +142,6 @@ public class RemovePeerProcedure extends ModifyPeerProcedure {
if (data.hasPeerConfig()) {
this.peerConfig = ReplicationPeerConfigUtil.convert(data.getPeerConfig());
}
+ ongoingAssignReplicationQueuesProcIds = data.getOngoingAssignReplicationQueuesProcIdsList();
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index 309ae522e2c..fff98ae68be 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -236,7 +236,7 @@ public class ReplicationSourceManager {
*/
void init() throws IOException {
for (String id : this.replicationPeers.getAllPeerIds()) {
- addSource(id);
+ addSource(id, true);
}
}
@@ -256,7 +256,7 @@ public class ReplicationSourceManager {
throw new IOException(e);
}
if (added) {
- addSource(peerId);
+ addSource(peerId, false);
}
}
@@ -322,11 +322,16 @@ public class ReplicationSourceManager {
/**
* Add a normal source for the given peer on this region server. Meanwhile, add new replication
* queue to storage. For the newly added peer, we only need to enqueue the latest log of each wal
- * group and do replication
+ * group and do replication.
+ * <p/>
+ * We add a {@code init} parameter to indicate whether this is part of the initialization process.
+ * If so, we should skip adding the replication queues as this may introduce dead lock on region
+ * server start up and hbase:replication table online.
* @param peerId the id of the replication peer
+ * @param init whether this call is part of the initialization process
* @return the source that was created
*/
- void addSource(String peerId) throws IOException {
+ void addSource(String peerId, boolean init) throws IOException {
ReplicationPeer peer = replicationPeers.getPeer(peerId);
if (
ReplicationUtils.LEGACY_REGION_REPLICATION_ENDPOINT_NAME
@@ -351,11 +356,16 @@ public class ReplicationSourceManager {
NavigableSet<String> wals = new TreeSet<>();
wals.add(walPath.getName());
walsByGroup.put(walPrefixAndPath.getKey(), wals);
- // Abort RS and throw exception to make add peer failed
- // TODO: can record the length of the current wal file so we could replicate less data
- abortAndThrowIOExceptionWhenFail(
- () -> this.queueStorage.setOffset(queueId, walPrefixAndPath.getKey(),
- new ReplicationGroupOffset(walPath.getName(), 0), Collections.emptyMap()));
+ if (!init) {
+ // Abort RS and throw exception to make add peer failed
+ // Ideally we'd better use the current file size as offset so we can skip replicating
+ // the data before adding replication peer, but the problem is that the file may not end
+ // at a valid entry's ending, and the current WAL Reader implementation can not deal
+ // with reading from the middle of a WAL entry. Can improve later.
+ abortAndThrowIOExceptionWhenFail(
+ () -> this.queueStorage.setOffset(queueId, walPrefixAndPath.getKey(),
+ new ReplicationGroupOffset(walPath.getName(), 0), Collections.emptyMap()));
+ }
src.enqueueLog(walPath);
LOG.trace("Enqueued {} to source {} during source creation.", walPath, src.getQueueId());
}
@@ -794,9 +804,15 @@ public class ReplicationSourceManager {
* @return {@code true} means we should replicate the given {@code wal}, otherwise {@code false}.
*/
private boolean shouldReplicate(ReplicationGroupOffset offset, String wal) {
- if (offset == null || offset == ReplicationGroupOffset.BEGIN) {
+ // skip replicating meta wals
+ if (AbstractFSWALProvider.isMetaFile(wal)) {
return false;
}
+ // if no offset or the offset is just a place marker, replicate
+ if (offset == null || offset == ReplicationGroupOffset.BEGIN) {
+ return true;
+ }
+ // otherwise, compare the timestamp
long walTs = AbstractFSWALProvider.getTimestamp(wal);
long startWalTs = AbstractFSWALProvider.getTimestamp(offset.getWal());
if (walTs < startWalTs) {
@@ -891,7 +907,6 @@ public class ReplicationSourceManager {
LOG.debug("Skip enqueuing log {} because it is before the start offset {}", file.getName(),
groupOffset);
}
- walFilesPQ.add(file);
}
// the method is a bit long, so assign it to null here to avoid later we reuse it again by
// mistake, we should use the sorted walFilesPQ instead
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java
index a12081a7636..de226b13e8f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java
@@ -156,7 +156,7 @@ public class TestClaimReplicationQueue extends TestReplicationBase {
hbaseAdmin.enableReplicationPeer(PEER_ID3);
EMPTY = false;
- // wait until the SCP finished, ClaimReplicationQueuesProcedure is a sub procedure of SCP
+ // wait until the SCP finished, AssignReplicationQueuesProcedure is a sub procedure of SCP
UTIL1.waitFor(30000, () -> master.getProcedures().stream()
.filter(p -> p instanceof ServerCrashProcedure).allMatch(Procedure::isSuccess));
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestRemovePeerProcedureWaitForSCP.java
similarity index 73%
copy from hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java
copy to hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestRemovePeerProcedureWaitForSCP.java
index a12081a7636..e93fa3b01e8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestRemovePeerProcedureWaitForSCP.java
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.hbase.replication;
+import static org.hamcrest.MatcherAssert.*;
+import static org.hamcrest.Matchers.*;
+import static org.junit.Assert.assertEquals;
+
import java.io.IOException;
import java.util.Collections;
import java.util.List;
@@ -32,6 +36,7 @@ import org.apache.hadoop.hbase.master.RegionServerList;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
import org.apache.hadoop.hbase.master.replication.AssignReplicationQueuesProcedure;
+import org.apache.hadoop.hbase.master.replication.RemovePeerProcedure;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
@@ -43,18 +48,20 @@ import org.junit.experimental.categories.Category;
import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationState;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
/**
- * In HBASE-26029, we reimplement the claim queue operation with proc-v2 and make it a step in SCP,
- * this is a UT to make sure the {@link AssignReplicationQueuesProcedure} works correctly.
+ * Make sure we will wait until all the SCPs finished in RemovePeerProcedure.
+ * <p/>
+ * See HBASE-27109 for more details.
*/
@Category({ ReplicationTests.class, LargeTests.class })
-public class TestClaimReplicationQueue extends TestReplicationBase {
+public class TestRemovePeerProcedureWaitForSCP extends TestReplicationBase {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestClaimReplicationQueue.class);
+ HBaseClassTestRule.forClass(TestRemovePeerProcedureWaitForSCP.class);
private static final TableName tableName3 = TableName.valueOf("test3");
@@ -62,8 +69,6 @@ public class TestClaimReplicationQueue extends TestReplicationBase {
private static Table table3;
- private static Table table4;
-
private static volatile boolean EMPTY = false;
public static final class ServerManagerForTest extends ServerManager {
@@ -106,14 +111,6 @@ public class TestClaimReplicationQueue extends TestReplicationBase {
TestReplicationBase.setUpBeforeClass();
createTable(tableName3);
table3 = connection1.getTable(tableName3);
- table4 = connection2.getTable(tableName3);
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- Closeables.close(table3, true);
- Closeables.close(table4, true);
- TestReplicationBase.tearDownAfterClass();
}
@Override
@@ -130,15 +127,21 @@ public class TestClaimReplicationQueue extends TestReplicationBase {
removePeer(PEER_ID3);
}
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ Closeables.close(table3, true);
+ TestReplicationBase.tearDownAfterClass();
+ }
+
@Test
- public void testClaim() throws Exception {
+ public void testWait() throws Exception {
// disable the peers
hbaseAdmin.disableReplicationPeer(PEER_ID2);
hbaseAdmin.disableReplicationPeer(PEER_ID3);
// put some data
- int count1 = UTIL1.loadTable(htable1, famName);
- int count2 = UTIL1.loadTable(table3, famName);
+ UTIL1.loadTable(htable1, famName);
+ UTIL1.loadTable(table3, famName);
EMPTY = true;
UTIL1.getMiniHBaseCluster().stopRegionServer(0).join();
@@ -152,16 +155,26 @@ public class TestClaimReplicationQueue extends TestReplicationBase {
.filter(p -> p instanceof AssignReplicationQueuesProcedure)
.anyMatch(p -> p.getState() == ProcedureState.WAITING_TIMEOUT));
- hbaseAdmin.enableReplicationPeer(PEER_ID2);
- hbaseAdmin.enableReplicationPeer(PEER_ID3);
-
+ // call remove replication peer, and make sure it will be stuck in the POST_PEER_MODIFICATION
+ // state.
+ hbaseAdmin.removeReplicationPeerAsync(PEER_ID3);
+ UTIL1.waitFor(30000,
+ () -> master.getProcedures().stream().filter(p -> p instanceof RemovePeerProcedure)
+ .anyMatch(p -> ((RemovePeerProcedure) p).getCurrentStateId()
+ == PeerModificationState.POST_PEER_MODIFICATION_VALUE));
+ Thread.sleep(5000);
+ assertEquals(PeerModificationState.POST_PEER_MODIFICATION_VALUE,
+ ((RemovePeerProcedure) master.getProcedures().stream()
+ .filter(p -> p instanceof RemovePeerProcedure).findFirst().get()).getCurrentStateId());
EMPTY = false;
- // wait until the SCP finished, ClaimReplicationQueuesProcedure is a sub procedure of SCP
+ // wait until the SCP finished, AssignReplicationQueuesProcedure is a sub procedure of SCP
UTIL1.waitFor(30000, () -> master.getProcedures().stream()
.filter(p -> p instanceof ServerCrashProcedure).allMatch(Procedure::isSuccess));
-
- // we should get all the data in the target cluster
- waitForReplication(htable2, count1, NB_RETRIES);
- waitForReplication(table4, count2, NB_RETRIES);
+ // the RemovePeerProcedure should have also finished
+ UTIL1.waitFor(30000, () -> master.getProcedures().stream()
+ .filter(p -> p instanceof RemovePeerProcedure).allMatch(Procedure::isSuccess));
+ // make sure there is no remaining replication queues for PEER_ID3
+ assertThat(master.getReplicationPeerManager().getQueueStorage().listAllQueueIds(PEER_ID3),
+ empty());
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java
index 6906db4cd46..1295ea14abc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java
@@ -32,12 +32,9 @@ import org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.junit.Before;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-// revisit later when we reviewing the implementation for serial replication
-@Ignore
@Category({ ReplicationTests.class, MediumTests.class })
public class TestSerialReplicationFailover extends SerialReplicationTestBase {
[hbase] 10/11: HBASE-27430 Should disable replication log cleaner when migrating replication queue data (#4901)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit b59ec9baf7c6c8a83fe368098020d1b92ccdd977
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Sat Dec 3 20:51:40 2022 +0800
HBASE-27430 Should disable replication log cleaner when migrating replication queue data (#4901)
Signed-off-by: Liangjun He <he...@apache.org>
---
.../protobuf/server/master/MasterProcedure.proto | 12 +++---
...rateReplicationQueueFromZkToTableProcedure.java | 47 +++++++++++++++++++++-
...rateReplicationQueueFromZkToTableProcedure.java | 29 ++++++++++++-
3 files changed, 80 insertions(+), 8 deletions(-)
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
index b6f5d7e50bb..14d07c17c88 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
@@ -724,11 +724,13 @@ message AssignReplicationQueuesStateData {
}
enum MigrateReplicationQueueFromZkToTableState {
- MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE = 1;
- MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER = 2;
- MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE = 3;
- MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING = 4;
- MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER = 5;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_CLEANER = 1;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE = 2;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER = 3;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE = 4;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING = 5;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER = 6;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_CLEANER = 7;
}
message MigrateReplicationQueueFromZkToTableStateData {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
index 93ff27db3f7..b7c4e33ef85 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
@@ -17,7 +17,9 @@
*/
package org.apache.hadoop.hbase.master.replication;
+import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_CLEANER;
import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER;
+import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_CLEANER;
import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER;
import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE;
import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE;
@@ -111,6 +113,26 @@ public class MigrateReplicationQueueFromZkToTableProcedure
}
}
+ private void disableReplicationLogCleaner(MasterProcedureEnv env)
+ throws ProcedureSuspendedException {
+ if (!env.getReplicationPeerManager().getReplicationLogCleanerBarrier().disable()) {
+ // it is not likely that we can reach here as we will schedule this procedure immediately
+ // after master restarting, where ReplicationLogCleaner should have not started its first run
+ // yet. But anyway, let's make the code more robust. And it is safe to wait a bit here since
+ // there will be no data in the new replication queue storage before we execute this procedure
+ // so ReplicationLogCleaner will quit immediately without doing anything.
+ throw suspend(env.getMasterConfiguration(),
+ backoff -> LOG.info(
+ "Can not disable replication log cleaner, sleep {} secs and retry later",
+ backoff / 1000));
+ }
+ resetRetry();
+ }
+
+ private void enableReplicationLogCleaner(MasterProcedureEnv env) {
+ env.getReplicationPeerManager().getReplicationLogCleanerBarrier().enable();
+ }
+
private void waitUntilNoPeerProcedure(MasterProcedureEnv env) throws ProcedureSuspendedException {
long peerProcCount;
try {
@@ -136,6 +158,10 @@ public class MigrateReplicationQueueFromZkToTableProcedure
MigrateReplicationQueueFromZkToTableState state)
throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
switch (state) {
+ case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_CLEANER:
+ disableReplicationLogCleaner(env);
+ setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE);
+ return Flow.HAS_MORE_STATE;
case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE:
waitUntilNoPeerProcedure(env);
List<ReplicationPeerDescription> peers = env.getReplicationPeerManager().listPeers(null);
@@ -152,7 +178,8 @@ public class MigrateReplicationQueueFromZkToTableProcedure
"failed to delete old replication queue data, sleep {} secs and retry later",
backoff / 1000, e));
}
- return Flow.NO_MORE_STATE;
+ setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_CLEANER);
+ return Flow.HAS_MORE_STATE;
}
// here we do not care the peers which have already been disabled, as later we do not need
// to enable them
@@ -232,6 +259,10 @@ public class MigrateReplicationQueueFromZkToTableProcedure
for (String peerId : disabledPeerIds) {
addChildProcedure(new EnablePeerProcedure(peerId));
}
+ setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_CLEANER);
+ return Flow.HAS_MORE_STATE;
+ case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_CLEANER:
+ enableReplicationLogCleaner(env);
return Flow.NO_MORE_STATE;
default:
throw new UnsupportedOperationException("unhandled state=" + state);
@@ -263,7 +294,19 @@ public class MigrateReplicationQueueFromZkToTableProcedure
@Override
protected MigrateReplicationQueueFromZkToTableState getInitialState() {
- return MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE;
+ return MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_CLEANER;
+ }
+
+ @Override
+ protected void afterReplay(MasterProcedureEnv env) {
+ if (getCurrentState() == getInitialState()) {
+ // do not need to disable log cleaner or acquire lock if we are in the initial state, later
+ // when executing the procedure we will try to disable and acquire.
+ return;
+ }
+ if (!env.getReplicationPeerManager().getReplicationLogCleanerBarrier().disable()) {
+ throw new IllegalStateException("can not disable log cleaner, this should not happen");
+ }
}
@Override
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedure.java
index 752abc380b8..cb795edcd62 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedure.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedure.java
@@ -17,8 +17,11 @@
*/
package org.apache.hadoop.hbase.master.replication;
+import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_CLEANER;
import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -48,6 +51,7 @@ import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.master.ReplicationLogCleanerBarrier;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -102,6 +106,8 @@ public class TestMigrateReplicationQueueFromZkToTableProcedure {
@BeforeClass
public static void setupCluster() throws Exception {
+ // one hour, to make sure it will not run during the test
+ UTIL.getConfiguration().setInt(HMaster.HBASE_MASTER_CLEANER_INTERVAL, 60 * 60 * 1000);
UTIL.startMiniCluster(
StartTestingClusterOption.builder().masterClass(HMasterForTest.class).build());
}
@@ -193,8 +199,10 @@ public class TestMigrateReplicationQueueFromZkToTableProcedure {
UTIL.waitFor(30000, () -> proc.isSuccess());
}
+ // make sure we will disable replication peers while migrating
+ // and also tests disable/enable replication log cleaner and wait for region server upgrading
@Test
- public void testDisablePeerAndWaitUpgrading() throws Exception {
+ public void testDisablePeerAndWaitStates() throws Exception {
String peerId = "2";
ReplicationPeerConfig rpc = ReplicationPeerConfig.newBuilder()
.setClusterKey(UTIL.getZkCluster().getAddress().toString() + ":/testhbase")
@@ -206,11 +214,22 @@ public class TestMigrateReplicationQueueFromZkToTableProcedure {
EXTRA_REGION_SERVERS
.put(ServerName.valueOf("localhost", 54321, EnvironmentEdgeManager.currentTime()), metrics);
+ ReplicationLogCleanerBarrier barrier = UTIL.getHBaseCluster().getMaster()
+ .getReplicationPeerManager().getReplicationLogCleanerBarrier();
+ assertTrue(barrier.start());
+
ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
MigrateReplicationQueueFromZkToTableProcedure proc =
new MigrateReplicationQueueFromZkToTableProcedure();
procExec.submitProcedure(proc);
+
+ Thread.sleep(5000);
+ // make sure we are still waiting for replication log cleaner quit
+ assertEquals(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_CLEANER.getNumber(),
+ proc.getCurrentStateId());
+ barrier.stop();
+
// wait until we reach the wait upgrading state
UTIL.waitFor(30000,
() -> proc.getCurrentStateId()
@@ -218,9 +237,17 @@ public class TestMigrateReplicationQueueFromZkToTableProcedure {
&& proc.getState() == ProcedureState.WAITING_TIMEOUT);
// make sure the peer is disabled for migrating
assertFalse(UTIL.getAdmin().isReplicationPeerEnabled(peerId));
+ // make sure the replication log cleaner is disabled
+ assertFalse(barrier.start());
// the procedure should finish successfully
EXTRA_REGION_SERVERS.clear();
UTIL.waitFor(30000, () -> proc.isSuccess());
+
+ // make sure the peer is enabled again
+ assertTrue(UTIL.getAdmin().isReplicationPeerEnabled(peerId));
+ // make sure the replication log cleaner is enabled again
+ assertTrue(barrier.start());
+ barrier.stop();
}
}
[hbase] 07/11: HBASE-27218 Support rolling upgrading (#4808)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 1158247dbbf55a00bcee7ac3d28de5b1f912847b
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Sun Nov 6 16:57:11 2022 +0800
HBASE-27218 Support rolling upgrading (#4808)
Signed-off-by: Yu Li <li...@apache.org>
---
.../apache/hadoop/hbase/zookeeper/ZNodePaths.java | 8 +-
.../apache/hadoop/hbase/procedure2/Procedure.java | 15 +
.../protobuf/server/master/MasterProcedure.proto | 12 +
hbase-replication/pom.xml | 10 +
.../hbase/replication/ReplicationQueueStorage.java | 19 ++
.../replication/TableReplicationQueueStorage.java | 65 +++-
.../ZKReplicationQueueStorageForMigration.java | 351 +++++++++++++++++++++
.../replication/TestZKReplicationQueueStorage.java | 317 +++++++++++++++++++
hbase-server/pom.xml | 6 +
.../org/apache/hadoop/hbase/master/HMaster.java | 13 +
.../master/procedure/ServerCrashProcedure.java | 19 ++
.../replication/AbstractPeerNoLockProcedure.java | 5 +-
...rateReplicationQueueFromZkToTableProcedure.java | 244 ++++++++++++++
.../master/replication/ModifyPeerProcedure.java | 26 ++
.../master/replication/ReplicationPeerManager.java | 104 +++++-
.../TransitPeerSyncReplicationStateProcedure.java | 14 +
.../replication/TestMigrateReplicationQueue.java | 126 ++++++++
...rateReplicationQueueFromZkToTableProcedure.java | 226 +++++++++++++
...icationQueueFromZkToTableProcedureRecovery.java | 128 ++++++++
...tReplicationPeerManagerMigrateQueuesFromZk.java | 216 +++++++++++++
.../hbase/replication/TestReplicationBase.java | 2 +-
pom.xml | 7 +-
22 files changed, 1917 insertions(+), 16 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
index d19d2100466..3f66c7cdc0c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
@@ -220,7 +220,11 @@ public class ZNodePaths {
* @param suffix ending of znode name
* @return result of properly joining prefix with suffix
*/
- public static String joinZNode(String prefix, String suffix) {
- return prefix + ZNodePaths.ZNODE_PATH_SEPARATOR + suffix;
+ public static String joinZNode(String prefix, String... suffix) {
+ StringBuilder sb = new StringBuilder(prefix);
+ for (String s : suffix) {
+ sb.append(ZNodePaths.ZNODE_PATH_SEPARATOR).append(s);
+ }
+ return sb.toString();
}
}
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index 34c74d92c16..43adba2bc21 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
import org.apache.hadoop.hbase.metrics.Counter;
import org.apache.hadoop.hbase.metrics.Histogram;
@@ -33,6 +34,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
/**
@@ -1011,6 +1013,19 @@ public abstract class Procedure<TEnvironment> implements Comparable<Procedure<TE
releaseLock(env);
}
+ protected final ProcedureSuspendedException suspend(int timeoutMillis, boolean jitter)
+ throws ProcedureSuspendedException {
+ if (jitter) {
+ // 10% possible jitter
+ double add = (double) timeoutMillis * ThreadLocalRandom.current().nextDouble(0.1);
+ timeoutMillis += add;
+ }
+ setTimeout(timeoutMillis);
+ setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
+ skipPersistence();
+ throw new ProcedureSuspendedException();
+ }
+
@Override
public int compareTo(final Procedure<TEnvironment> other) {
return Long.compare(getProcId(), other.getProcId());
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
index 76a1d676487..b6f5d7e50bb 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
@@ -722,3 +722,15 @@ enum AssignReplicationQueuesState {
message AssignReplicationQueuesStateData {
required ServerName crashed_server = 1;
}
+
+enum MigrateReplicationQueueFromZkToTableState {
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE = 1;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER = 2;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE = 3;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING = 4;
+ MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER = 5;
+}
+
+message MigrateReplicationQueueFromZkToTableStateData {
+ repeated string disabled_peer_id = 1;
+}
diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml
index dad93578609..d294cfdbe01 100644
--- a/hbase-replication/pom.xml
+++ b/hbase-replication/pom.xml
@@ -98,6 +98,16 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-library</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
index 6f6aee38cc8..1e36bbeb78f 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
@@ -22,6 +22,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration.ZkLastPushedSeqId;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
@@ -184,4 +185,22 @@ public interface ReplicationQueueStorage {
* @return Whether the replication queue table exists
*/
boolean hasData() throws ReplicationException;
+
+ // the below 3 methods are used for migrating
+ /**
+ * Update the replication queue datas for a given region server.
+ */
+ void batchUpdateQueues(ServerName serverName, List<ReplicationQueueData> datas)
+ throws ReplicationException;
+
+ /**
+ * Update last pushed sequence id for the given regions and peers.
+ */
+ void batchUpdateLastSequenceIds(List<ZkLastPushedSeqId> lastPushedSeqIds)
+ throws ReplicationException;
+
+ /**
+ * Add the given hfile refs to the given peer.
+ */
+ void batchUpdateHFileRefs(String peerId, List<String> hfileRefs) throws ReplicationException;
}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
index 392a3692d66..f3870f4d09d 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
@@ -21,12 +21,14 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Set;
import java.util.function.Supplier;
+import java.util.stream.Collectors;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
@@ -46,6 +48,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Scan.ReadType;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration.ZkLastPushedSeqId;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FutureUtils;
import org.apache.hadoop.hbase.util.Pair;
@@ -74,12 +77,6 @@ public class TableReplicationQueueStorage implements ReplicationQueueStorage {
private final TableName tableName;
- @FunctionalInterface
- private interface TableCreator {
-
- void create() throws IOException;
- }
-
public TableReplicationQueueStorage(Connection conn, TableName tableName) {
this.conn = conn;
this.tableName = tableName;
@@ -541,4 +538,60 @@ public class TableReplicationQueueStorage implements ReplicationQueueStorage {
throw new ReplicationException("failed to get replication queue table", e);
}
}
+
+ @Override
+ public void batchUpdateQueues(ServerName serverName, List<ReplicationQueueData> datas)
+ throws ReplicationException {
+ List<Put> puts = new ArrayList<>();
+ for (ReplicationQueueData data : datas) {
+ if (data.getOffsets().isEmpty()) {
+ continue;
+ }
+ Put put = new Put(Bytes.toBytes(data.getId().toString()));
+ data.getOffsets().forEach((walGroup, offset) -> {
+ put.addColumn(QUEUE_FAMILY, Bytes.toBytes(walGroup), Bytes.toBytes(offset.toString()));
+ });
+ puts.add(put);
+ }
+ try (Table table = conn.getTable(tableName)) {
+ table.put(puts);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to batch update queues", e);
+ }
+ }
+
+ @Override
+ public void batchUpdateLastSequenceIds(List<ZkLastPushedSeqId> lastPushedSeqIds)
+ throws ReplicationException {
+ Map<String, Put> peerId2Put = new HashMap<>();
+ for (ZkLastPushedSeqId lastPushedSeqId : lastPushedSeqIds) {
+ peerId2Put
+ .computeIfAbsent(lastPushedSeqId.getPeerId(), peerId -> new Put(Bytes.toBytes(peerId)))
+ .addColumn(LAST_SEQUENCE_ID_FAMILY, Bytes.toBytes(lastPushedSeqId.getEncodedRegionName()),
+ Bytes.toBytes(lastPushedSeqId.getLastPushedSeqId()));
+ }
+ try (Table table = conn.getTable(tableName)) {
+ table
+ .put(peerId2Put.values().stream().filter(p -> !p.isEmpty()).collect(Collectors.toList()));
+ } catch (IOException e) {
+ throw new ReplicationException("failed to batch update last pushed sequence ids", e);
+ }
+ }
+
+ @Override
+ public void batchUpdateHFileRefs(String peerId, List<String> hfileRefs)
+ throws ReplicationException {
+ if (hfileRefs.isEmpty()) {
+ return;
+ }
+ Put put = new Put(Bytes.toBytes(peerId));
+ for (String ref : hfileRefs) {
+ put.addColumn(HFILE_REF_FAMILY, Bytes.toBytes(ref), HConstants.EMPTY_BYTE_ARRAY);
+ }
+ try (Table table = conn.getTable(tableName)) {
+ table.put(put);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to batch update hfile references", e);
+ }
+ }
}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorageForMigration.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorageForMigration.java
new file mode 100644
index 00000000000..22cc1314522
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorageForMigration.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import com.google.errorprone.annotations.RestrictedApi;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.zookeeper.KeeperException;
+
+import org.apache.hbase.thirdparty.com.google.common.base.Splitter;
+
+/**
+ * Just retain a small set of the methods for the old zookeeper based replication queue storage, for
+ * migrating.
+ */
+@InterfaceAudience.Private
+public class ZKReplicationQueueStorageForMigration extends ZKReplicationStorageBase {
+
+ public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY =
+ "zookeeper.znode.replication.hfile.refs";
+ public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT = "hfile-refs";
+
+ public static final String ZOOKEEPER_ZNODE_REPLICATION_REGIONS_KEY =
+ "zookeeper.znode.replication.regions";
+ public static final String ZOOKEEPER_ZNODE_REPLICATION_REGIONS_DEFAULT = "regions";
+
+ /**
+ * The name of the znode that contains all replication queues
+ */
+ private final String queuesZNode;
+
+ /**
+ * The name of the znode that contains queues of hfile references to be replicated
+ */
+ private final String hfileRefsZNode;
+
+ private final String regionsZNode;
+
+ public ZKReplicationQueueStorageForMigration(ZKWatcher zookeeper, Configuration conf) {
+ super(zookeeper, conf);
+ String queuesZNodeName = conf.get("zookeeper.znode.replication.rs", "rs");
+ String hfileRefsZNodeName = conf.get(ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
+ ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
+ this.queuesZNode = ZNodePaths.joinZNode(replicationZNode, queuesZNodeName);
+ this.hfileRefsZNode = ZNodePaths.joinZNode(replicationZNode, hfileRefsZNodeName);
+ this.regionsZNode = ZNodePaths.joinZNode(replicationZNode, conf
+ .get(ZOOKEEPER_ZNODE_REPLICATION_REGIONS_KEY, ZOOKEEPER_ZNODE_REPLICATION_REGIONS_DEFAULT));
+ }
+
+ public interface MigrationIterator<T> {
+
+ T next() throws Exception;
+ }
+
+ @SuppressWarnings("rawtypes")
+ private static final MigrationIterator EMPTY_ITER = new MigrationIterator() {
+
+ @Override
+ public Object next() {
+ return null;
+ }
+ };
+
+ public static final class ZkReplicationQueueData {
+
+ private final ReplicationQueueId queueId;
+
+ private final Map<String, Long> walOffsets;
+
+ public ZkReplicationQueueData(ReplicationQueueId queueId, Map<String, Long> walOffsets) {
+ this.queueId = queueId;
+ this.walOffsets = walOffsets;
+ }
+
+ public ReplicationQueueId getQueueId() {
+ return queueId;
+ }
+
+ public Map<String, Long> getWalOffsets() {
+ return walOffsets;
+ }
+ }
+
+ private String getRsNode(ServerName serverName) {
+ return ZNodePaths.joinZNode(queuesZNode, serverName.getServerName());
+ }
+
+ private String getQueueNode(ServerName serverName, String queueId) {
+ return ZNodePaths.joinZNode(getRsNode(serverName), queueId);
+ }
+
+ private String getFileNode(String queueNode, String fileName) {
+ return ZNodePaths.joinZNode(queueNode, fileName);
+ }
+
+ private String getFileNode(ServerName serverName, String queueId, String fileName) {
+ return getFileNode(getQueueNode(serverName, queueId), fileName);
+ }
+
+ @SuppressWarnings("unchecked")
+ public MigrationIterator<Pair<ServerName, List<ZkReplicationQueueData>>> listAllQueues()
+ throws KeeperException {
+ List<String> replicators = ZKUtil.listChildrenNoWatch(zookeeper, queuesZNode);
+ if (replicators == null || replicators.isEmpty()) {
+ ZKUtil.deleteNodeRecursively(zookeeper, queuesZNode);
+ return EMPTY_ITER;
+ }
+ Iterator<String> iter = replicators.iterator();
+ return new MigrationIterator<Pair<ServerName, List<ZkReplicationQueueData>>>() {
+
+ private ServerName previousServerName;
+
+ @Override
+ public Pair<ServerName, List<ZkReplicationQueueData>> next() throws Exception {
+ if (previousServerName != null) {
+ ZKUtil.deleteNodeRecursively(zookeeper, getRsNode(previousServerName));
+ }
+ if (!iter.hasNext()) {
+ ZKUtil.deleteNodeRecursively(zookeeper, queuesZNode);
+ return null;
+ }
+ String replicator = iter.next();
+ ServerName serverName = ServerName.parseServerName(replicator);
+ previousServerName = serverName;
+ List<String> queueIdList = ZKUtil.listChildrenNoWatch(zookeeper, getRsNode(serverName));
+ if (queueIdList == null || queueIdList.isEmpty()) {
+ return Pair.newPair(serverName, Collections.emptyList());
+ }
+ List<ZkReplicationQueueData> queueDataList = new ArrayList<>(queueIdList.size());
+ for (String queueIdStr : queueIdList) {
+ ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueIdStr);
+ ReplicationQueueId queueId;
+ if (queueInfo.getDeadRegionServers().isEmpty()) {
+ queueId = new ReplicationQueueId(serverName, queueInfo.getPeerId());
+ } else {
+ queueId = new ReplicationQueueId(serverName, queueInfo.getPeerId(),
+ queueInfo.getDeadRegionServers().get(0));
+ }
+ List<String> wals =
+ ZKUtil.listChildrenNoWatch(zookeeper, getQueueNode(serverName, queueIdStr));
+ ZkReplicationQueueData queueData;
+ if (wals == null || wals.isEmpty()) {
+ queueData = new ZkReplicationQueueData(queueId, Collections.emptyMap());
+ } else {
+ Map<String, Long> walOffsets = new HashMap<>();
+ for (String wal : wals) {
+ byte[] data = ZKUtil.getData(zookeeper, getFileNode(serverName, queueIdStr, wal));
+ if (data == null || data.length == 0) {
+ walOffsets.put(wal, 0L);
+ } else {
+ walOffsets.put(wal, ZKUtil.parseWALPositionFrom(data));
+ }
+ }
+ queueData = new ZkReplicationQueueData(queueId, walOffsets);
+ }
+ queueDataList.add(queueData);
+ }
+ return Pair.newPair(serverName, queueDataList);
+ }
+ };
+ }
+
+ public static final class ZkLastPushedSeqId {
+
+ private final String encodedRegionName;
+
+ private final String peerId;
+
+ private final long lastPushedSeqId;
+
+ ZkLastPushedSeqId(String encodedRegionName, String peerId, long lastPushedSeqId) {
+ this.encodedRegionName = encodedRegionName;
+ this.peerId = peerId;
+ this.lastPushedSeqId = lastPushedSeqId;
+ }
+
+ public String getEncodedRegionName() {
+ return encodedRegionName;
+ }
+
+ public String getPeerId() {
+ return peerId;
+ }
+
+ public long getLastPushedSeqId() {
+ return lastPushedSeqId;
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ public MigrationIterator<List<ZkLastPushedSeqId>> listAllLastPushedSeqIds()
+ throws KeeperException {
+ List<String> level1Prefixs = ZKUtil.listChildrenNoWatch(zookeeper, regionsZNode);
+ if (level1Prefixs == null || level1Prefixs.isEmpty()) {
+ ZKUtil.deleteNodeRecursively(zookeeper, regionsZNode);
+ return EMPTY_ITER;
+ }
+ Iterator<String> level1Iter = level1Prefixs.iterator();
+ return new MigrationIterator<List<ZkLastPushedSeqId>>() {
+
+ private String level1Prefix;
+
+ private Iterator<String> level2Iter;
+
+ private String level2Prefix;
+
+ @Override
+ public List<ZkLastPushedSeqId> next() throws Exception {
+ for (;;) {
+ if (level2Iter == null || !level2Iter.hasNext()) {
+ if (!level1Iter.hasNext()) {
+ ZKUtil.deleteNodeRecursively(zookeeper, regionsZNode);
+ return null;
+ }
+ if (level1Prefix != null) {
+ // this will also delete the previous level2Prefix which is under this level1Prefix
+ ZKUtil.deleteNodeRecursively(zookeeper,
+ ZNodePaths.joinZNode(regionsZNode, level1Prefix));
+ }
+ level1Prefix = level1Iter.next();
+ List<String> level2Prefixes = ZKUtil.listChildrenNoWatch(zookeeper,
+ ZNodePaths.joinZNode(regionsZNode, level1Prefix));
+ if (level2Prefixes != null) {
+ level2Iter = level2Prefixes.iterator();
+ // reset level2Prefix as we have switched level1Prefix, otherwise the below delete
+ // level2Prefix section will delete the znode with this level2Prefix under the new
+ // level1Prefix
+ level2Prefix = null;
+ }
+ } else {
+ if (level2Prefix != null) {
+ ZKUtil.deleteNodeRecursively(zookeeper,
+ ZNodePaths.joinZNode(regionsZNode, level1Prefix, level2Prefix));
+ }
+ level2Prefix = level2Iter.next();
+ List<String> encodedRegionNameAndPeerIds = ZKUtil.listChildrenNoWatch(zookeeper,
+ ZNodePaths.joinZNode(regionsZNode, level1Prefix, level2Prefix));
+ if (encodedRegionNameAndPeerIds == null || encodedRegionNameAndPeerIds.isEmpty()) {
+ return Collections.emptyList();
+ }
+ List<ZkLastPushedSeqId> lastPushedSeqIds = new ArrayList<>();
+ for (String encodedRegionNameAndPeerId : encodedRegionNameAndPeerIds) {
+ byte[] data = ZKUtil.getData(zookeeper, ZNodePaths.joinZNode(regionsZNode,
+ level1Prefix, level2Prefix, encodedRegionNameAndPeerId));
+ long lastPushedSeqId = ZKUtil.parseWALPositionFrom(data);
+ Iterator<String> iter = Splitter.on('-').split(encodedRegionNameAndPeerId).iterator();
+ String encodedRegionName = level1Prefix + level2Prefix + iter.next();
+ String peerId = iter.next();
+ lastPushedSeqIds
+ .add(new ZkLastPushedSeqId(encodedRegionName, peerId, lastPushedSeqId));
+ }
+ return Collections.unmodifiableList(lastPushedSeqIds);
+ }
+ }
+ }
+ };
+ }
+
+ private String getHFileRefsPeerNode(String peerId) {
+ return ZNodePaths.joinZNode(hfileRefsZNode, peerId);
+ }
+
+ /**
+ * Pair<PeerId, List<HFileRefs>>
+ */
+ @SuppressWarnings("unchecked")
+ public MigrationIterator<Pair<String, List<String>>> listAllHFileRefs() throws KeeperException {
+ List<String> peerIds = ZKUtil.listChildrenNoWatch(zookeeper, hfileRefsZNode);
+ if (peerIds == null || peerIds.isEmpty()) {
+ ZKUtil.deleteNodeRecursively(zookeeper, hfileRefsZNode);
+ return EMPTY_ITER;
+ }
+ Iterator<String> iter = peerIds.iterator();
+ return new MigrationIterator<Pair<String, List<String>>>() {
+
+ private String previousPeerId;
+
+ @Override
+ public Pair<String, List<String>> next() throws KeeperException {
+ if (previousPeerId != null) {
+ ZKUtil.deleteNodeRecursively(zookeeper, getHFileRefsPeerNode(previousPeerId));
+ }
+ if (!iter.hasNext()) {
+ ZKUtil.deleteNodeRecursively(zookeeper, hfileRefsZNode);
+ return null;
+ }
+ String peerId = iter.next();
+ List<String> refs = ZKUtil.listChildrenNoWatch(zookeeper, getHFileRefsPeerNode(peerId));
+ previousPeerId = peerId;
+ return Pair.newPair(peerId, refs != null ? refs : Collections.emptyList());
+ }
+ };
+ }
+
+ public boolean hasData() throws KeeperException {
+ return ZKUtil.checkExists(zookeeper, queuesZNode) != -1
+ || ZKUtil.checkExists(zookeeper, regionsZNode) != -1
+ || ZKUtil.checkExists(zookeeper, hfileRefsZNode) != -1;
+ }
+
+ public void deleteAllData() throws KeeperException {
+ ZKUtil.deleteNodeRecursively(zookeeper, queuesZNode);
+ ZKUtil.deleteNodeRecursively(zookeeper, regionsZNode);
+ ZKUtil.deleteNodeRecursively(zookeeper, hfileRefsZNode);
+ }
+
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ String getQueuesZNode() {
+ return queuesZNode;
+ }
+
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ String getHfileRefsZNode() {
+ return hfileRefsZNode;
+ }
+
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ String getRegionsZNode() {
+ return regionsZNode;
+ }
+}
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
new file mode 100644
index 00000000000..e38b7b134e9
--- /dev/null
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.empty;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseZKTestingUtil;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration.MigrationIterator;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration.ZkLastPushedSeqId;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration.ZkReplicationQueueData;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.zookeeper.KeeperException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import org.apache.hbase.thirdparty.com.google.common.base.Splitter;
+import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
+import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
+
+@Category({ ReplicationTests.class, MediumTests.class })
+public class TestZKReplicationQueueStorage {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestZKReplicationQueueStorage.class);
+
+ private static final HBaseZKTestingUtil UTIL = new HBaseZKTestingUtil();
+
+ private ZKWatcher zk;
+
+ private ZKReplicationQueueStorageForMigration storage;
+
+ @Rule
+ public final TestName name = new TestName();
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ UTIL.startMiniZKCluster();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws IOException {
+ UTIL.shutdownMiniZKCluster();
+ }
+
+ @Before
+ public void setUp() throws IOException {
+ Configuration conf = UTIL.getConfiguration();
+ conf.set(ZKReplicationStorageBase.REPLICATION_ZNODE, name.getMethodName());
+ zk = new ZKWatcher(conf, name.getMethodName(), null);
+ storage = new ZKReplicationQueueStorageForMigration(zk, conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ ZKUtil.deleteNodeRecursively(zk, storage.replicationZNode);
+ Closeables.close(zk, true);
+ }
+
+ public static void mockQueuesData(ZKReplicationQueueStorageForMigration storage, int nServers,
+ String peerId, ServerName deadServer) throws KeeperException {
+ ZKWatcher zk = storage.zookeeper;
+ for (int i = 0; i < nServers; i++) {
+ ServerName sn =
+ ServerName.valueOf("test-hbase-" + i, 12345, EnvironmentEdgeManager.currentTime());
+ String rsZNode = ZNodePaths.joinZNode(storage.getQueuesZNode(), sn.toString());
+ String peerZNode = ZNodePaths.joinZNode(rsZNode, peerId);
+ ZKUtil.createWithParents(zk, peerZNode);
+ for (int j = 0; j < i; j++) {
+ String wal = ZNodePaths.joinZNode(peerZNode, sn.toString() + "." + j);
+ ZKUtil.createSetData(zk, wal, ZKUtil.positionToByteArray(j));
+ }
+ String deadServerPeerZNode = ZNodePaths.joinZNode(rsZNode, peerId + "-" + deadServer);
+ ZKUtil.createWithParents(zk, deadServerPeerZNode);
+ for (int j = 0; j < i; j++) {
+ String wal = ZNodePaths.joinZNode(deadServerPeerZNode, deadServer.toString() + "." + j);
+ if (j > 0) {
+ ZKUtil.createSetData(zk, wal, ZKUtil.positionToByteArray(j));
+ } else {
+ ZKUtil.createWithParents(zk, wal);
+ }
+ }
+ }
+ ZKUtil.createWithParents(zk,
+ ZNodePaths.joinZNode(storage.getQueuesZNode(), deadServer.toString()));
+ }
+
+ private static String getLastPushedSeqIdZNode(String regionsZNode, String encodedName,
+ String peerId) {
+ return ZNodePaths.joinZNode(regionsZNode, encodedName.substring(0, 2),
+ encodedName.substring(2, 4), encodedName.substring(4) + "-" + peerId);
+ }
+
+ public static Map<String, Set<String>> mockLastPushedSeqIds(
+ ZKReplicationQueueStorageForMigration storage, String peerId1, String peerId2, int nRegions,
+ int emptyLevel1Count, int emptyLevel2Count) throws KeeperException {
+ ZKWatcher zk = storage.zookeeper;
+ Map<String, Set<String>> name2PeerIds = new HashMap<>();
+ byte[] bytes = new byte[32];
+ for (int i = 0; i < nRegions; i++) {
+ ThreadLocalRandom.current().nextBytes(bytes);
+ String encodeName = MD5Hash.getMD5AsHex(bytes);
+ String znode1 = getLastPushedSeqIdZNode(storage.getRegionsZNode(), encodeName, peerId1);
+ ZKUtil.createSetData(zk, znode1, ZKUtil.positionToByteArray(1));
+ String znode2 = getLastPushedSeqIdZNode(storage.getRegionsZNode(), encodeName, peerId2);
+ ZKUtil.createSetData(zk, znode2, ZKUtil.positionToByteArray(2));
+ name2PeerIds.put(encodeName, Sets.newHashSet(peerId1, peerId2));
+ }
+ int addedEmptyZNodes = 0;
+ for (int i = 0; i < 256; i++) {
+ String level1ZNode =
+ ZNodePaths.joinZNode(storage.getRegionsZNode(), String.format("%02x", i));
+ if (ZKUtil.checkExists(zk, level1ZNode) == -1) {
+ ZKUtil.createWithParents(zk, level1ZNode);
+ addedEmptyZNodes++;
+ if (addedEmptyZNodes <= emptyLevel2Count) {
+ ZKUtil.createWithParents(zk, ZNodePaths.joinZNode(level1ZNode, "ab"));
+ }
+ if (addedEmptyZNodes >= emptyLevel1Count + emptyLevel2Count) {
+ break;
+ }
+ }
+ }
+ return name2PeerIds;
+ }
+
+ public static void mockHFileRefs(ZKReplicationQueueStorageForMigration storage, int nPeers)
+ throws KeeperException {
+ ZKWatcher zk = storage.zookeeper;
+ for (int i = 0; i < nPeers; i++) {
+ String peerId = "peer_" + i;
+ ZKUtil.createWithParents(zk, ZNodePaths.joinZNode(storage.getHfileRefsZNode(), peerId));
+ for (int j = 0; j < i; j++) {
+ ZKUtil.createWithParents(zk,
+ ZNodePaths.joinZNode(storage.getHfileRefsZNode(), peerId, "hfile-" + j));
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteAllData() throws Exception {
+ assertFalse(storage.hasData());
+ ZKUtil.createWithParents(zk, storage.getQueuesZNode());
+ assertTrue(storage.hasData());
+ storage.deleteAllData();
+ assertFalse(storage.hasData());
+ }
+
+ @Test
+ public void testEmptyIter() throws Exception {
+ ZKUtil.createWithParents(zk, storage.getQueuesZNode());
+ ZKUtil.createWithParents(zk, storage.getRegionsZNode());
+ ZKUtil.createWithParents(zk, storage.getHfileRefsZNode());
+ assertNull(storage.listAllQueues().next());
+ assertEquals(-1, ZKUtil.checkExists(zk, storage.getQueuesZNode()));
+ assertNull(storage.listAllLastPushedSeqIds().next());
+ assertEquals(-1, ZKUtil.checkExists(zk, storage.getRegionsZNode()));
+ assertNull(storage.listAllHFileRefs().next());
+ assertEquals(-1, ZKUtil.checkExists(zk, storage.getHfileRefsZNode()));
+ }
+
+ @Test
+ public void testListAllQueues() throws Exception {
+ String peerId = "1";
+ ServerName deadServer =
+ ServerName.valueOf("test-hbase-dead", 12345, EnvironmentEdgeManager.currentTime());
+ int nServers = 10;
+ mockQueuesData(storage, nServers, peerId, deadServer);
+ MigrationIterator<Pair<ServerName, List<ZkReplicationQueueData>>> iter =
+ storage.listAllQueues();
+ ServerName previousServerName = null;
+ for (int i = 0; i < nServers + 1; i++) {
+ Pair<ServerName, List<ZkReplicationQueueData>> pair = iter.next();
+ assertNotNull(pair);
+ if (previousServerName != null) {
+ assertEquals(-1, ZKUtil.checkExists(zk,
+ ZNodePaths.joinZNode(storage.getQueuesZNode(), previousServerName.toString())));
+ }
+ ServerName sn = pair.getFirst();
+ previousServerName = sn;
+ if (sn.equals(deadServer)) {
+ assertThat(pair.getSecond(), empty());
+ } else {
+ assertEquals(2, pair.getSecond().size());
+ int n = Integer.parseInt(Iterables.getLast(Splitter.on('-').split(sn.getHostname())));
+ ZkReplicationQueueData data0 = pair.getSecond().get(0);
+ assertEquals(peerId, data0.getQueueId().getPeerId());
+ assertEquals(sn, data0.getQueueId().getServerName());
+ assertEquals(n, data0.getWalOffsets().size());
+ for (int j = 0; j < n; j++) {
+ assertEquals(j,
+ data0.getWalOffsets().get(
+ (data0.getQueueId().isRecovered() ? deadServer.toString() : sn.toString()) + "." + j)
+ .intValue());
+ }
+ ZkReplicationQueueData data1 = pair.getSecond().get(1);
+ assertEquals(peerId, data1.getQueueId().getPeerId());
+ assertEquals(sn, data1.getQueueId().getServerName());
+ assertEquals(n, data1.getWalOffsets().size());
+ for (int j = 0; j < n; j++) {
+ assertEquals(j,
+ data1.getWalOffsets().get(
+ (data1.getQueueId().isRecovered() ? deadServer.toString() : sn.toString()) + "." + j)
+ .intValue());
+ }
+ // the order of the returned result is undetermined
+ if (data0.getQueueId().getSourceServerName().isPresent()) {
+ assertEquals(deadServer, data0.getQueueId().getSourceServerName().get());
+ assertFalse(data1.getQueueId().getSourceServerName().isPresent());
+ } else {
+ assertEquals(deadServer, data1.getQueueId().getSourceServerName().get());
+ }
+ }
+ }
+ assertNull(iter.next());
+ assertEquals(-1, ZKUtil.checkExists(zk, storage.getQueuesZNode()));
+ }
+
+ @Test
+ public void testListAllLastPushedSeqIds() throws Exception {
+ String peerId1 = "1";
+ String peerId2 = "2";
+ Map<String, Set<String>> name2PeerIds =
+ mockLastPushedSeqIds(storage, peerId1, peerId2, 100, 10, 10);
+ MigrationIterator<List<ZkLastPushedSeqId>> iter = storage.listAllLastPushedSeqIds();
+ int emptyListCount = 0;
+ for (;;) {
+ List<ZkLastPushedSeqId> list = iter.next();
+ if (list == null) {
+ break;
+ }
+ if (list.isEmpty()) {
+ emptyListCount++;
+ continue;
+ }
+ for (ZkLastPushedSeqId seqId : list) {
+ name2PeerIds.get(seqId.getEncodedRegionName()).remove(seqId.getPeerId());
+ if (seqId.getPeerId().equals(peerId1)) {
+ assertEquals(1, seqId.getLastPushedSeqId());
+ } else {
+ assertEquals(2, seqId.getLastPushedSeqId());
+ }
+ }
+ }
+ assertEquals(10, emptyListCount);
+ name2PeerIds.forEach((encodedRegionName, peerIds) -> {
+ assertThat(encodedRegionName + " still has unmigrated peers", peerIds, empty());
+ });
+ assertEquals(-1, ZKUtil.checkExists(zk, storage.getRegionsZNode()));
+ }
+
+ @Test
+ public void testListAllHFileRefs() throws Exception {
+ int nPeers = 10;
+ mockHFileRefs(storage, nPeers);
+ MigrationIterator<Pair<String, List<String>>> iter = storage.listAllHFileRefs();
+ String previousPeerId = null;
+ for (int i = 0; i < nPeers; i++) {
+ Pair<String, List<String>> pair = iter.next();
+ if (previousPeerId != null) {
+ assertEquals(-1, ZKUtil.checkExists(zk,
+ ZNodePaths.joinZNode(storage.getHfileRefsZNode(), previousPeerId)));
+ }
+ String peerId = pair.getFirst();
+ previousPeerId = peerId;
+ int index = Integer.parseInt(Iterables.getLast(Splitter.on('_').split(peerId)));
+ assertEquals(index, pair.getSecond().size());
+ }
+ assertNull(iter.next());
+ assertEquals(-1, ZKUtil.checkExists(zk, storage.getHfileRefsZNode()));
+ }
+}
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index 0dba4aa9833..b61b0252a05 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -102,6 +102,12 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-replication</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-replication</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-balancer</artifactId>
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 118457648de..67d0f889d64 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -171,6 +171,7 @@ import org.apache.hadoop.hbase.master.replication.AbstractPeerProcedure;
import org.apache.hadoop.hbase.master.replication.AddPeerProcedure;
import org.apache.hadoop.hbase.master.replication.DisablePeerProcedure;
import org.apache.hadoop.hbase.master.replication.EnablePeerProcedure;
+import org.apache.hadoop.hbase.master.replication.MigrateReplicationQueueFromZkToTableProcedure;
import org.apache.hadoop.hbase.master.replication.RemovePeerProcedure;
import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
import org.apache.hadoop.hbase.master.replication.SyncReplicationReplayWALManager;
@@ -221,6 +222,7 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration;
import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner;
import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
import org.apache.hadoop.hbase.replication.master.ReplicationSinkTrackerTableCreator;
@@ -1050,6 +1052,17 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
this.balancer.initialize();
this.balancer.updateClusterMetrics(getClusterMetricsWithoutCoprocessor());
+ // try migrate replication data
+ ZKReplicationQueueStorageForMigration oldReplicationQueueStorage =
+ new ZKReplicationQueueStorageForMigration(zooKeeper, conf);
+ // check whether there are something to migrate and we haven't scheduled a migration procedure
+ // yet
+ if (
+ oldReplicationQueueStorage.hasData() && procedureExecutor.getProcedures().stream()
+ .allMatch(p -> !(p instanceof MigrateReplicationQueueFromZkToTableProcedure))
+ ) {
+ procedureExecutor.submitProcedure(new MigrateReplicationQueueFromZkToTableProcedure());
+ }
// start up all service threads.
startupTaskGroup.addTask("Initializing master service threads");
startServiceThreads();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
index 487c45e5c5c..97976756d82 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
import org.apache.hadoop.hbase.master.replication.AssignReplicationQueuesProcedure;
+import org.apache.hadoop.hbase.master.replication.MigrateReplicationQueueFromZkToTableProcedure;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.procedure2.Procedure;
@@ -52,6 +53,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.ServerCrashState;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
/**
* Handle crashed server. This is a port to ProcedureV2 of what used to be euphemistically called
@@ -266,6 +268,16 @@ public class ServerCrashProcedure extends
}
break;
case SERVER_CRASH_CLAIM_REPLICATION_QUEUES:
+ if (
+ env.getMasterServices().getProcedures().stream()
+ .filter(p -> p instanceof MigrateReplicationQueueFromZkToTableProcedure)
+ .anyMatch(p -> !p.isFinished())
+ ) {
+ LOG.info("There is a pending {}, will retry claim replication queue later",
+ MigrateReplicationQueueFromZkToTableProcedure.class.getSimpleName());
+ suspend(10_000, true);
+ return Flow.NO_MORE_STATE;
+ }
addChildProcedure(new AssignReplicationQueuesProcedure(serverName));
setNextState(ServerCrashState.SERVER_CRASH_FINISH);
break;
@@ -431,6 +443,13 @@ public class ServerCrashProcedure extends
env.getProcedureScheduler().wakeServerExclusiveLock(this, getServerName());
}
+ @Override
+ protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
+ setState(ProcedureProtos.ProcedureState.RUNNABLE);
+ env.getProcedureScheduler().addFront(this);
+ return false;
+ }
+
@Override
public void toStringClassDetails(StringBuilder sb) {
sb.append(getProcName());
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerNoLockProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerNoLockProcedure.java
index 660f9968573..1f0a89f2076 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerNoLockProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AbstractPeerNoLockProcedure.java
@@ -98,10 +98,7 @@ public abstract class AbstractPeerNoLockProcedure<TState>
}
long backoff = retryCounter.getBackoffTimeAndIncrementAttempts();
backoffConsumer.accept(backoff);
- setTimeout(Math.toIntExact(backoff));
- setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
- skipPersistence();
- throw new ProcedureSuspendedException();
+ throw suspend(Math.toIntExact(backoff), false);
}
protected final void resetRetry() {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
new file mode 100644
index 00000000000..536f232338e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/MigrateReplicationQueueFromZkToTableProcedure.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER;
+import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER;
+import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE;
+import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE;
+import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+import org.apache.hadoop.hbase.master.procedure.GlobalProcedureInterface;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration;
+import org.apache.hadoop.hbase.util.VersionInfo;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableStateData;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+
+/**
+ * A procedure for migrating replication queue data from zookeeper to hbase:replication table.
+ */
+@InterfaceAudience.Private
+public class MigrateReplicationQueueFromZkToTableProcedure
+ extends StateMachineProcedure<MasterProcedureEnv, MigrateReplicationQueueFromZkToTableState>
+ implements GlobalProcedureInterface {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MigrateReplicationQueueFromZkToTableProcedure.class);
+
+ private static final int MIN_MAJOR_VERSION = 3;
+
+ private List<String> disabledPeerIds;
+
+ private List<Future<?>> futures;
+
+ private ExecutorService executor;
+
+ @Override
+ public String getGlobalId() {
+ return getClass().getSimpleName();
+ }
+
+ private ExecutorService getExecutorService() {
+ if (executor == null) {
+ executor = Executors.newFixedThreadPool(3, new ThreadFactoryBuilder()
+ .setNameFormat(getClass().getSimpleName() + "-%d").setDaemon(true).build());
+ }
+ return executor;
+ }
+
+ private void shutdownExecutorService() {
+ if (executor != null) {
+ executor.shutdown();
+ executor = null;
+ }
+ }
+
+ private void waitUntilNoPeerProcedure(MasterProcedureEnv env) throws ProcedureSuspendedException {
+ long peerProcCount;
+ try {
+ peerProcCount = env.getMasterServices().getProcedures().stream()
+ .filter(p -> p instanceof PeerProcedureInterface).filter(p -> !p.isFinished()).count();
+ } catch (IOException e) {
+ LOG.warn("failed to check peer procedure status", e);
+ throw suspend(5000, true);
+ }
+ if (peerProcCount > 0) {
+ LOG.info("There are still {} pending peer procedures, will sleep and check later",
+ peerProcCount);
+ throw suspend(10_000, true);
+ }
+ LOG.info("No pending peer procedures found, continue...");
+ }
+
+ @Override
+ protected Flow executeFromState(MasterProcedureEnv env,
+ MigrateReplicationQueueFromZkToTableState state)
+ throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+ switch (state) {
+ case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE:
+ waitUntilNoPeerProcedure(env);
+ List<ReplicationPeerDescription> peers = env.getReplicationPeerManager().listPeers(null);
+ if (peers.isEmpty()) {
+ LOG.info("No active replication peer found, delete old replication queue data and quit");
+ ZKReplicationQueueStorageForMigration oldStorage =
+ new ZKReplicationQueueStorageForMigration(env.getMasterServices().getZooKeeper(),
+ env.getMasterConfiguration());
+ try {
+ oldStorage.deleteAllData();
+ } catch (KeeperException e) {
+ LOG.warn("failed to delete old replication queue data, sleep and retry later", e);
+ suspend(10_000, true);
+ }
+ return Flow.NO_MORE_STATE;
+ }
+ // here we do not care the peers which have already been disabled, as later we do not need
+ // to enable them
+ disabledPeerIds = peers.stream().filter(ReplicationPeerDescription::isEnabled)
+ .map(ReplicationPeerDescription::getPeerId).collect(Collectors.toList());
+ setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER);
+ return Flow.HAS_MORE_STATE;
+ case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_DISABLE_PEER:
+ for (String peerId : disabledPeerIds) {
+ addChildProcedure(new DisablePeerProcedure(peerId));
+ }
+ setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE);
+ return Flow.HAS_MORE_STATE;
+ case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_MIGRATE:
+ if (futures != null) {
+ // wait until all futures done
+ long notDone = futures.stream().filter(f -> !f.isDone()).count();
+ if (notDone == 0) {
+ boolean succ = true;
+ for (Future<?> future : futures) {
+ try {
+ future.get();
+ } catch (Exception e) {
+ succ = false;
+ LOG.warn("Failed to migrate", e);
+ }
+ }
+ if (succ) {
+ shutdownExecutorService();
+ setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING);
+ return Flow.HAS_MORE_STATE;
+ }
+ // reschedule to retry migration again
+ futures = null;
+ } else {
+ LOG.info("There still {} pending migration tasks, will sleep and check later", notDone);
+ throw suspend(10_000, true);
+ }
+ }
+ try {
+ futures = env.getReplicationPeerManager()
+ .migrateQueuesFromZk(env.getMasterServices().getZooKeeper(), getExecutorService());
+ } catch (IOException e) {
+ LOG.warn("failed to submit migration tasks", e);
+ throw suspend(10_000, true);
+ }
+ throw suspend(10_000, true);
+ case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING:
+ long rsWithLowerVersion =
+ env.getMasterServices().getServerManager().getOnlineServers().values().stream()
+ .filter(sm -> VersionInfo.getMajorVersion(sm.getVersion()) < MIN_MAJOR_VERSION).count();
+ if (rsWithLowerVersion == 0) {
+ setNextState(MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER);
+ return Flow.HAS_MORE_STATE;
+ } else {
+ LOG.info("There are still {} region servers which have a major version less than {}, "
+ + "will sleep and check later", rsWithLowerVersion, MIN_MAJOR_VERSION);
+ throw suspend(10_000, true);
+ }
+ case MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_ENABLE_PEER:
+ for (String peerId : disabledPeerIds) {
+ addChildProcedure(new EnablePeerProcedure(peerId));
+ }
+ return Flow.NO_MORE_STATE;
+ default:
+ throw new UnsupportedOperationException("unhandled state=" + state);
+ }
+ }
+
+ @Override
+ protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
+ setState(ProcedureProtos.ProcedureState.RUNNABLE);
+ env.getProcedureScheduler().addFront(this);
+ return false;
+ }
+
+ @Override
+ protected void rollbackState(MasterProcedureEnv env,
+ MigrateReplicationQueueFromZkToTableState state) throws IOException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected MigrateReplicationQueueFromZkToTableState getState(int stateId) {
+ return MigrateReplicationQueueFromZkToTableState.forNumber(stateId);
+ }
+
+ @Override
+ protected int getStateId(MigrateReplicationQueueFromZkToTableState state) {
+ return state.getNumber();
+ }
+
+ @Override
+ protected MigrateReplicationQueueFromZkToTableState getInitialState() {
+ return MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_PREPARE;
+ }
+
+ @Override
+ protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.serializeStateData(serializer);
+ MigrateReplicationQueueFromZkToTableStateData.Builder builder =
+ MigrateReplicationQueueFromZkToTableStateData.newBuilder();
+ if (disabledPeerIds != null) {
+ builder.addAllDisabledPeerId(disabledPeerIds);
+ }
+ serializer.serialize(builder.build());
+ }
+
+ @Override
+ protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.deserializeStateData(serializer);
+ MigrateReplicationQueueFromZkToTableStateData data =
+ serializer.deserialize(MigrateReplicationQueueFromZkToTableStateData.class);
+ disabledPeerIds = data.getDisabledPeerIdList().stream().collect(Collectors.toList());
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
index 78b97620c01..c358ec164e7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ModifyPeerProcedure.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
import java.io.InterruptedIOException;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.TableDescriptor;
@@ -27,6 +28,7 @@ import org.apache.hadoop.hbase.master.TableStateManager;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
import org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure;
+import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -152,12 +154,36 @@ public abstract class ModifyPeerProcedure extends AbstractPeerProcedure<PeerModi
}
}
+ private boolean shouldFailForMigrating(MasterProcedureEnv env) throws IOException {
+ long parentProcId = getParentProcId();
+ if (
+ parentProcId != Procedure.NO_PROC_ID && env.getMasterServices().getMasterProcedureExecutor()
+ .getProcedure(parentProcId) instanceof MigrateReplicationQueueFromZkToTableProcedure
+ ) {
+ // this is scheduled by MigrateReplicationQueueFromZkToTableProcedure, should not fail it
+ return false;
+ }
+ return env.getMasterServices().getProcedures().stream()
+ .filter(p -> p instanceof MigrateReplicationQueueFromZkToTableProcedure)
+ .anyMatch(p -> !p.isFinished());
+ }
+
@Override
protected Flow executeFromState(MasterProcedureEnv env, PeerModificationState state)
throws ProcedureSuspendedException, InterruptedException {
switch (state) {
case PRE_PEER_MODIFICATION:
try {
+ if (shouldFailForMigrating(env)) {
+ LOG.info("There is a pending {}, give up execution of {}",
+ MigrateReplicationQueueFromZkToTableProcedure.class.getSimpleName(),
+ getClass().getName());
+ setFailure("master-" + getPeerOperationType().name().toLowerCase() + "-peer",
+ new DoNotRetryIOException("There is a pending "
+ + MigrateReplicationQueueFromZkToTableProcedure.class.getSimpleName()));
+ releaseLatch(env);
+ return Flow.NO_MORE_STATE;
+ }
prePeerModification(env);
} catch (IOException e) {
LOG.warn("{} failed to call pre CP hook or the pre check is failed for peer {}, "
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index 0a1dbf848bd..81f569c3f9e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -21,14 +21,18 @@ import com.google.errorprone.annotations.RestrictedApi;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
@@ -39,6 +43,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.master.MasterServices;
@@ -49,17 +54,24 @@ import org.apache.hadoop.hbase.replication.BaseReplicationEndpoint;
import org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfigBuilder;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration.MigrationIterator;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration.ZkLastPushedSeqId;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration.ZkReplicationQueueData;
import org.apache.hadoop.hbase.replication.master.ReplicationLogCleanerBarrier;
import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -106,7 +118,7 @@ public class ReplicationPeerManager {
private final Configuration conf;
@FunctionalInterface
- private interface ReplicationQueueStorageInitializer {
+ interface ReplicationQueueStorageInitializer {
void initialize() throws IOException;
}
@@ -138,6 +150,10 @@ public class ReplicationPeerManager {
}
}
+ private void initializeQueueStorage() throws IOException {
+ queueStorageInitializer.initialize();
+ }
+
void preAddPeer(String peerId, ReplicationPeerConfig peerConfig)
throws ReplicationException, IOException {
if (peerId.contains("-")) {
@@ -152,7 +168,7 @@ public class ReplicationPeerManager {
}
// lazy create table
- queueStorageInitializer.initialize();
+ initializeQueueStorage();
// make sure that there is no queues with the same peer id. This may happen when we create a
// peer with the same id with a old deleted peer. If the replication queues for the old peer
// have not been cleaned up yet then we should not create the new peer, otherwise the old wal
@@ -699,4 +715,88 @@ public class ReplicationPeerManager {
public ReplicationLogCleanerBarrier getReplicationLogCleanerBarrier() {
return replicationLogCleanerBarrier;
}
+
+ private ReplicationQueueData convert(ZkReplicationQueueData zkData) {
+ Map<String, ReplicationGroupOffset> groupOffsets = new HashMap<>();
+ zkData.getWalOffsets().forEach((wal, offset) -> {
+ String walGroup = AbstractFSWALProvider.getWALPrefixFromWALName(wal);
+ groupOffsets.compute(walGroup, (k, oldOffset) -> {
+ if (oldOffset == null) {
+ return new ReplicationGroupOffset(wal, offset);
+ }
+ // we should record the first wal's offset
+ long oldWalTs = AbstractFSWALProvider.getTimestamp(oldOffset.getWal());
+ long walTs = AbstractFSWALProvider.getTimestamp(wal);
+ if (walTs < oldWalTs) {
+ return new ReplicationGroupOffset(wal, offset);
+ }
+ return oldOffset;
+ });
+ });
+ return new ReplicationQueueData(zkData.getQueueId(), ImmutableMap.copyOf(groupOffsets));
+ }
+
+ private void migrateQueues(ZKReplicationQueueStorageForMigration oldQueueStorage)
+ throws Exception {
+ MigrationIterator<Pair<ServerName, List<ZkReplicationQueueData>>> iter =
+ oldQueueStorage.listAllQueues();
+ for (;;) {
+ Pair<ServerName, List<ZkReplicationQueueData>> pair = iter.next();
+ if (pair == null) {
+ return;
+ }
+ queueStorage.batchUpdateQueues(pair.getFirst(),
+ pair.getSecond().stream().filter(data -> peers.containsKey(data.getQueueId().getPeerId()))
+ .map(this::convert).collect(Collectors.toList()));
+ }
+ }
+
+ private void migrateLastPushedSeqIds(ZKReplicationQueueStorageForMigration oldQueueStorage)
+ throws Exception {
+ MigrationIterator<List<ZkLastPushedSeqId>> iter = oldQueueStorage.listAllLastPushedSeqIds();
+ for (;;) {
+ List<ZkLastPushedSeqId> list = iter.next();
+ if (list == null) {
+ return;
+ }
+ queueStorage.batchUpdateLastSequenceIds(list.stream()
+ .filter(data -> peers.containsKey(data.getPeerId())).collect(Collectors.toList()));
+ }
+ }
+
+ private void migrateHFileRefs(ZKReplicationQueueStorageForMigration oldQueueStorage)
+ throws Exception {
+ MigrationIterator<Pair<String, List<String>>> iter = oldQueueStorage.listAllHFileRefs();
+ for (;;) {
+ Pair<String, List<String>> pair = iter.next();
+ if (pair == null) {
+ return;
+ }
+ if (peers.containsKey(pair.getFirst())) {
+ queueStorage.batchUpdateHFileRefs(pair.getFirst(), pair.getSecond());
+ }
+ }
+ }
+
+ /**
+ * Submit the migration tasks to the given {@code executor} and return the futures.
+ */
+ List<Future<?>> migrateQueuesFromZk(ZKWatcher zookeeper, ExecutorService executor)
+ throws IOException {
+ // the replication queue table creation is asynchronous and will be triggered by addPeer, so
+ // here we need to manually initialize it since we will not call addPeer.
+ initializeQueueStorage();
+ ZKReplicationQueueStorageForMigration oldStorage =
+ new ZKReplicationQueueStorageForMigration(zookeeper, conf);
+ return Arrays.asList(executor.submit(() -> {
+ migrateQueues(oldStorage);
+ return null;
+ }), executor.submit(() -> {
+ migrateLastPushedSeqIds(oldStorage);
+ return null;
+ }), executor.submit(() -> {
+ migrateHFileRefs(oldStorage);
+ return null;
+ }));
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
index 2de10cb2778..89658903538 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/TransitPeerSyncReplicationStateProcedure.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure.Flow;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
@@ -236,6 +237,19 @@ public class TransitPeerSyncReplicationStateProcedure
switch (state) {
case PRE_PEER_SYNC_REPLICATION_STATE_TRANSITION:
try {
+ if (
+ env.getMasterServices().getProcedures().stream()
+ .filter(p -> p instanceof MigrateReplicationQueueFromZkToTableProcedure)
+ .anyMatch(p -> !p.isFinished())
+ ) {
+ LOG.info("There is a pending {}, give up execution of {}",
+ MigrateReplicationQueueFromZkToTableProcedure.class.getSimpleName(),
+ getClass().getSimpleName());
+ setFailure("master-transit-peer-sync-replication-state",
+ new DoNotRetryIOException("There is a pending "
+ + MigrateReplicationQueueFromZkToTableProcedure.class.getSimpleName()));
+ return Flow.NO_MORE_STATE;
+ }
preTransit(env);
} catch (IOException e) {
LOG.warn("Failed to call pre CP hook or the pre check is failed for peer {} "
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueue.java
new file mode 100644
index 00000000000..1b0f727a072
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueue.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.replication.TestReplicationBase;
+import org.apache.hadoop.hbase.replication.ZKReplicationStorageBase;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+
+@Category({ MasterTests.class, LargeTests.class })
+public class TestMigrateReplicationQueue extends TestReplicationBase {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMigrateReplicationQueue.class);
+
+ private int disableAndInsert() throws Exception {
+ UTIL1.getAdmin().disableReplicationPeer(PEER_ID2);
+ return UTIL1.loadTable(htable1, famName);
+ }
+
+ private String getQueuesZNode() throws IOException {
+ Configuration conf = UTIL1.getConfiguration();
+ ZKWatcher zk = UTIL1.getZooKeeperWatcher();
+ String replicationZNode = ZNodePaths.joinZNode(zk.getZNodePaths().baseZNode,
+ conf.get(ZKReplicationStorageBase.REPLICATION_ZNODE,
+ ZKReplicationStorageBase.REPLICATION_ZNODE_DEFAULT));
+ return ZNodePaths.joinZNode(replicationZNode, conf.get("zookeeper.znode.replication.rs", "rs"));
+ }
+
+ private void mockData() throws Exception {
+ // delete the replication queue table to simulate upgrading from an older version of hbase
+ TableName replicationQueueTableName = TableName
+ .valueOf(UTIL1.getConfiguration().get(ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME,
+ ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME_DEFAULT.getNameAsString()));
+ List<ReplicationQueueData> queueDatas = UTIL1.getMiniHBaseCluster().getMaster()
+ .getReplicationPeerManager().getQueueStorage().listAllQueues();
+ assertEquals(UTIL1.getMiniHBaseCluster().getRegionServerThreads().size(), queueDatas.size());
+ UTIL1.getAdmin().disableTable(replicationQueueTableName);
+ UTIL1.getAdmin().deleteTable(replicationQueueTableName);
+ // shutdown the hbase cluster
+ UTIL1.shutdownMiniHBaseCluster();
+ ZKWatcher zk = UTIL1.getZooKeeperWatcher();
+ String queuesZNode = getQueuesZNode();
+ for (ReplicationQueueData queueData : queueDatas) {
+ String replicatorZNode =
+ ZNodePaths.joinZNode(queuesZNode, queueData.getId().getServerName().toString());
+ String queueZNode = ZNodePaths.joinZNode(replicatorZNode, queueData.getId().getPeerId());
+ assertEquals(1, queueData.getOffsets().size());
+ ReplicationGroupOffset offset = Iterables.getOnlyElement(queueData.getOffsets().values());
+ String walZNode = ZNodePaths.joinZNode(queueZNode, offset.getWal());
+ ZKUtil.createSetData(zk, walZNode, ZKUtil.positionToByteArray(offset.getOffset()));
+ }
+ }
+
+ @Test
+ public void testMigrate() throws Exception {
+ int count = disableAndInsert();
+ mockData();
+ restartSourceCluster(1);
+ UTIL1.waitFor(60000,
+ () -> UTIL1.getMiniHBaseCluster().getMaster().getProcedures().stream()
+ .filter(p -> p instanceof MigrateReplicationQueueFromZkToTableProcedure).findAny()
+ .map(Procedure::isSuccess).orElse(false));
+ TableName replicationQueueTableName = TableName
+ .valueOf(UTIL1.getConfiguration().get(ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME,
+ ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME_DEFAULT.getNameAsString()));
+ assertTrue(UTIL1.getAdmin().tableExists(replicationQueueTableName));
+ ZKWatcher zk = UTIL1.getZooKeeperWatcher();
+ assertEquals(-1, ZKUtil.checkExists(zk, getQueuesZNode()));
+ // wait until SCP finishes, which means we can finish the claim queue operation
+ UTIL1.waitFor(60000, () -> UTIL1.getMiniHBaseCluster().getMaster().getProcedures().stream()
+ .filter(p -> p instanceof ServerCrashProcedure).allMatch(Procedure::isSuccess));
+ List<ReplicationQueueData> queueDatas = UTIL1.getMiniHBaseCluster().getMaster()
+ .getReplicationPeerManager().getQueueStorage().listAllQueues();
+ assertEquals(1, queueDatas.size());
+ // should have 1 recovered queue, as we haven't replicated anything out so there is no queue
+ // data for the new alive region server
+ assertTrue(queueDatas.get(0).getId().isRecovered());
+ assertEquals(1, queueDatas.get(0).getOffsets().size());
+ // the peer is still disabled, so no data has been replicated
+ assertFalse(UTIL1.getAdmin().isReplicationPeerEnabled(PEER_ID2));
+ assertEquals(0, HBaseTestingUtil.countRows(htable2));
+ // enable peer, and make sure the replication can continue correctly
+ UTIL1.getAdmin().enableReplicationPeer(PEER_ID2);
+ waitForReplication(count, 100);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedure.java
new file mode 100644
index 00000000000..752abc380b8
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedure.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import static org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MigrateReplicationQueueFromZkToTableState.MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.ServerMetrics;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.StartTestingClusterOption;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.RegionServerList;
+import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.PeerProcedureInterface;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestMigrateReplicationQueueFromZkToTableProcedure {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMigrateReplicationQueueFromZkToTableProcedure.class);
+
+ private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
+
+ public static final class HMasterForTest extends HMaster {
+
+ public HMasterForTest(Configuration conf) throws IOException {
+ super(conf);
+ }
+
+ @Override
+ protected ServerManager createServerManager(MasterServices master, RegionServerList storage)
+ throws IOException {
+ setupClusterConnection();
+ return new ServerManagerForTest(master, storage);
+ }
+ }
+
+ private static final ConcurrentMap<ServerName, ServerMetrics> EXTRA_REGION_SERVERS =
+ new ConcurrentHashMap<>();
+
+ public static final class ServerManagerForTest extends ServerManager {
+
+ public ServerManagerForTest(MasterServices master, RegionServerList storage) {
+ super(master, storage);
+ }
+
+ @Override
+ public Map<ServerName, ServerMetrics> getOnlineServers() {
+ Map<ServerName, ServerMetrics> map = new HashMap<>(super.getOnlineServers());
+ map.putAll(EXTRA_REGION_SERVERS);
+ return map;
+ }
+ }
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ UTIL.startMiniCluster(
+ StartTestingClusterOption.builder().masterClass(HMasterForTest.class).build());
+ }
+
+ @AfterClass
+ public static void cleanupTest() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
+ return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ Admin admin = UTIL.getAdmin();
+ for (ReplicationPeerDescription pd : admin.listReplicationPeers()) {
+ admin.removeReplicationPeer(pd.getPeerId());
+ }
+ }
+
+ private static CountDownLatch PEER_PROC_ARRIVE;
+
+ private static CountDownLatch PEER_PROC_RESUME;
+
+ public static final class FakePeerProcedure extends Procedure<MasterProcedureEnv>
+ implements PeerProcedureInterface {
+
+ private String peerId;
+
+ public FakePeerProcedure() {
+ }
+
+ public FakePeerProcedure(String peerId) {
+ this.peerId = peerId;
+ }
+
+ @Override
+ public String getPeerId() {
+ return peerId;
+ }
+
+ @Override
+ public PeerOperationType getPeerOperationType() {
+ return PeerOperationType.UPDATE_CONFIG;
+ }
+
+ @Override
+ protected Procedure<MasterProcedureEnv>[] execute(MasterProcedureEnv env)
+ throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
+ PEER_PROC_ARRIVE.countDown();
+ PEER_PROC_RESUME.await();
+ return null;
+ }
+
+ @Override
+ protected void rollback(MasterProcedureEnv env) throws IOException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected boolean abort(MasterProcedureEnv env) {
+ return false;
+ }
+
+ @Override
+ protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ }
+
+ @Override
+ protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ }
+ }
+
+ @Test
+ public void testWaitUntilNoPeerProcedure() throws Exception {
+ PEER_PROC_ARRIVE = new CountDownLatch(1);
+ PEER_PROC_RESUME = new CountDownLatch(1);
+ ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ procExec.submitProcedure(new FakePeerProcedure("1"));
+ PEER_PROC_ARRIVE.await();
+ MigrateReplicationQueueFromZkToTableProcedure proc =
+ new MigrateReplicationQueueFromZkToTableProcedure();
+ procExec.submitProcedure(proc);
+ // make sure we will wait until there is no peer related procedures before proceeding
+ UTIL.waitFor(30000, () -> proc.getState() == ProcedureState.WAITING_TIMEOUT);
+ // continue and make sure we can finish successfully
+ PEER_PROC_RESUME.countDown();
+ UTIL.waitFor(30000, () -> proc.isSuccess());
+ }
+
+ @Test
+ public void testDisablePeerAndWaitUpgrading() throws Exception {
+ String peerId = "2";
+ ReplicationPeerConfig rpc = ReplicationPeerConfig.newBuilder()
+ .setClusterKey(UTIL.getZkCluster().getAddress().toString() + ":/testhbase")
+ .setReplicateAllUserTables(true).build();
+ UTIL.getAdmin().addReplicationPeer(peerId, rpc);
+ // put a fake region server to simulate that there are still region servers with older version
+ ServerMetrics metrics = mock(ServerMetrics.class);
+ when(metrics.getVersion()).thenReturn("2.5.0");
+ EXTRA_REGION_SERVERS
+ .put(ServerName.valueOf("localhost", 54321, EnvironmentEdgeManager.currentTime()), metrics);
+
+ ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ MigrateReplicationQueueFromZkToTableProcedure proc =
+ new MigrateReplicationQueueFromZkToTableProcedure();
+ procExec.submitProcedure(proc);
+ // wait until we reach the wait upgrading state
+ UTIL.waitFor(30000,
+ () -> proc.getCurrentStateId()
+ == MIGRATE_REPLICATION_QUEUE_FROM_ZK_TO_TABLE_WAIT_UPGRADING.getNumber()
+ && proc.getState() == ProcedureState.WAITING_TIMEOUT);
+ // make sure the peer is disabled for migrating
+ assertFalse(UTIL.getAdmin().isReplicationPeerEnabled(peerId));
+
+ // the procedure should finish successfully
+ EXTRA_REGION_SERVERS.clear();
+ UTIL.waitFor(30000, () -> proc.isSuccess());
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedureRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedureRecovery.java
new file mode 100644
index 00000000000..8d1a975400f
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestMigrateReplicationQueueFromZkToTableProcedureRecovery.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration;
+import org.apache.hadoop.hbase.replication.ZKReplicationStorageBase;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestMigrateReplicationQueueFromZkToTableProcedureRecovery {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMigrateReplicationQueueFromZkToTableProcedureRecovery.class);
+
+ private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ UTIL.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
+ UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void cleanupTest() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
+ return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+ }
+
+ @Before
+ public void setup() throws Exception {
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
+ }
+
+ private String getHFileRefsZNode() throws IOException {
+ Configuration conf = UTIL.getConfiguration();
+ ZKWatcher zk = UTIL.getZooKeeperWatcher();
+ String replicationZNode = ZNodePaths.joinZNode(zk.getZNodePaths().baseZNode,
+ conf.get(ZKReplicationStorageBase.REPLICATION_ZNODE,
+ ZKReplicationStorageBase.REPLICATION_ZNODE_DEFAULT));
+ return ZNodePaths.joinZNode(replicationZNode,
+ conf.get(ZKReplicationQueueStorageForMigration.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
+ ZKReplicationQueueStorageForMigration.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT));
+ }
+
+ @Test
+ public void testRecoveryAndDoubleExecution() throws Exception {
+ String peerId = "2";
+ ReplicationPeerConfig rpc = ReplicationPeerConfig.newBuilder()
+ .setClusterKey(UTIL.getZkCluster().getAddress().toString() + ":/testhbase")
+ .setReplicateAllUserTables(true).build();
+ UTIL.getAdmin().addReplicationPeer(peerId, rpc);
+
+ // here we only test a simple migration, more complicated migration will be tested in other UTs,
+ // such as TestMigrateReplicationQueue and TestReplicationPeerManagerMigrateFromZk
+ String hfileRefsZNode = getHFileRefsZNode();
+ String hfile = "hfile";
+ String hfileZNode = ZNodePaths.joinZNode(hfileRefsZNode, peerId, hfile);
+ ZKUtil.createWithParents(UTIL.getZooKeeperWatcher(), hfileZNode);
+
+ ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+
+ ProcedureTestingUtility.waitNoProcedureRunning(procExec);
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
+
+ // Start the migration procedure && kill the executor
+ long procId = procExec.submitProcedure(new MigrateReplicationQueueFromZkToTableProcedure());
+ // Restart the executor and execute the step twice
+ MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId);
+ // Validate the migration result
+ ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
+ ReplicationQueueStorage queueStorage =
+ UTIL.getMiniHBaseCluster().getMaster().getReplicationPeerManager().getQueueStorage();
+ List<String> hfiles = queueStorage.getReplicableHFiles(peerId);
+ assertThat(hfiles, Matchers.<List<String>> both(hasItem(hfile)).and(hasSize(1)));
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestReplicationPeerManagerMigrateQueuesFromZk.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestReplicationPeerManagerMigrateQueuesFromZk.java
new file mode 100644
index 00000000000..73915e856ea
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/replication/TestReplicationPeerManagerMigrateQueuesFromZk.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.empty;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNameTestRule;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.ReplicationQueueStorageInitializer;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.replication.TableReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.TestZKReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+@Category({ MasterTests.class, MediumTests.class })
+public class TestReplicationPeerManagerMigrateQueuesFromZk {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReplicationPeerManagerMigrateQueuesFromZk.class);
+
+ private static HBaseTestingUtil UTIL = new HBaseTestingUtil();
+
+ private static ExecutorService EXECUTOR;
+
+ ConcurrentMap<String, ReplicationPeerDescription> peers;
+
+ private ReplicationPeerStorage peerStorage;
+
+ private ReplicationQueueStorage queueStorage;
+
+ private ReplicationQueueStorageInitializer queueStorageInitializer;
+
+ private ReplicationPeerManager manager;
+
+ private int nServers = 10;
+
+ private int nPeers = 10;
+
+ private int nRegions = 100;
+
+ private ServerName deadServerName;
+
+ @Rule
+ public final TableNameTestRule tableNameRule = new TableNameTestRule();
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ UTIL.startMiniCluster(1);
+ EXECUTOR = Executors.newFixedThreadPool(3,
+ new ThreadFactoryBuilder().setDaemon(true)
+ .setNameFormat(TestReplicationPeerManagerMigrateQueuesFromZk.class.getSimpleName() + "-%d")
+ .build());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ EXECUTOR.shutdownNow();
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void setUp() throws IOException {
+ Configuration conf = UTIL.getConfiguration();
+ peerStorage = mock(ReplicationPeerStorage.class);
+ TableName tableName = tableNameRule.getTableName();
+ UTIL.getAdmin()
+ .createTable(ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName));
+ queueStorage = new TableReplicationQueueStorage(UTIL.getConnection(), tableName);
+ queueStorageInitializer = mock(ReplicationQueueStorageInitializer.class);
+ peers = new ConcurrentHashMap<>();
+ deadServerName =
+ ServerName.valueOf("test-hbase-dead", 12345, EnvironmentEdgeManager.currentTime());
+ manager = new ReplicationPeerManager(peerStorage, queueStorage, peers, conf, "cluster",
+ queueStorageInitializer);
+ }
+
+ private Map<String, Set<String>> prepareData() throws Exception {
+ ZKReplicationQueueStorageForMigration storage = new ZKReplicationQueueStorageForMigration(
+ UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
+ TestZKReplicationQueueStorage.mockQueuesData(storage, 10, "peer_0", deadServerName);
+ Map<String, Set<String>> encodedName2PeerIds = TestZKReplicationQueueStorage
+ .mockLastPushedSeqIds(storage, "peer_1", "peer_2", nRegions, 10, 10);
+ TestZKReplicationQueueStorage.mockHFileRefs(storage, 10);
+ return encodedName2PeerIds;
+ }
+
+ @Test
+ public void testNoPeers() throws Exception {
+ prepareData();
+ for (Future<?> future : manager.migrateQueuesFromZk(UTIL.getZooKeeperWatcher(), EXECUTOR)) {
+ future.get(1, TimeUnit.MINUTES);
+ }
+ // should have called initializer
+ verify(queueStorageInitializer).initialize();
+ // should have not migrated any data since there is no peer
+ try (Table table = UTIL.getConnection().getTable(tableNameRule.getTableName())) {
+ assertEquals(0, HBaseTestingUtil.countRows(table));
+ }
+ }
+
+ @Test
+ public void testMigrate() throws Exception {
+ Map<String, Set<String>> encodedName2PeerIds = prepareData();
+ // add all peers so we will migrate them all
+ for (int i = 0; i < nPeers; i++) {
+ // value is not used in this test, so just add a mock
+ peers.put("peer_" + i, mock(ReplicationPeerDescription.class));
+ }
+ for (Future<?> future : manager.migrateQueuesFromZk(UTIL.getZooKeeperWatcher(), EXECUTOR)) {
+ future.get(1, TimeUnit.MINUTES);
+ }
+ // should have called initializer
+ verify(queueStorageInitializer).initialize();
+ List<ReplicationQueueData> queueDatas = queueStorage.listAllQueues();
+ // there should be two empty queues so minus 2
+ assertEquals(2 * nServers - 2, queueDatas.size());
+ for (ReplicationQueueData queueData : queueDatas) {
+ assertEquals("peer_0", queueData.getId().getPeerId());
+ assertEquals(1, queueData.getOffsets().size());
+ String walGroup = queueData.getId().getServerWALsBelongTo().toString();
+ ReplicationGroupOffset offset = queueData.getOffsets().get(walGroup);
+ assertEquals(0, offset.getOffset());
+ assertEquals(queueData.getId().getServerWALsBelongTo().toString() + ".0", offset.getWal());
+ }
+ // there is no method in ReplicationQueueStorage can list all the last pushed sequence ids
+ try (Table table = UTIL.getConnection().getTable(tableNameRule.getTableName());
+ ResultScanner scanner =
+ table.getScanner(TableReplicationQueueStorage.LAST_SEQUENCE_ID_FAMILY)) {
+ for (int i = 0; i < 2; i++) {
+ Result result = scanner.next();
+ String peerId = Bytes.toString(result.getRow());
+ assertEquals(nRegions, result.size());
+ for (Cell cell : result.rawCells()) {
+ String encodedRegionName = Bytes.toString(cell.getQualifierArray(),
+ cell.getQualifierOffset(), cell.getQualifierLength());
+ encodedName2PeerIds.get(encodedRegionName).remove(peerId);
+ long seqId =
+ Bytes.toLong(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+ assertEquals(i + 1, seqId);
+ }
+ }
+ encodedName2PeerIds.forEach((encodedRegionName, peerIds) -> {
+ assertThat(encodedRegionName + " still has unmigrated peers", peerIds, empty());
+ });
+ assertNull(scanner.next());
+ }
+ for (int i = 0; i < nPeers; i++) {
+ List<String> refs = queueStorage.getReplicableHFiles("peer_" + i);
+ assertEquals(i, refs.size());
+ Set<String> refsSet = new HashSet<>(refs);
+ for (int j = 0; j < i; j++) {
+ assertTrue(refsSet.remove("hfile-" + j));
+ }
+ assertThat(refsSet, empty());
+ }
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
index b6157ac0f18..27477527277 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java
@@ -216,7 +216,7 @@ public class TestReplicationBase {
conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);
}
- static void restartSourceCluster(int numSlaves) throws Exception {
+ protected static void restartSourceCluster(int numSlaves) throws Exception {
Closeables.close(hbaseAdmin, true);
Closeables.close(htable1, true);
UTIL1.shutdownMiniHBaseCluster();
diff --git a/pom.xml b/pom.xml
index 44a26739b45..c215329af5e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1037,13 +1037,18 @@
<artifactId>hbase-hadoop-compat</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-replication</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-replication</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-balancer</artifactId>
[hbase] 11/11: HBASE-27216 Revisit the ReplicationSyncUp tool (#4966)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 49589ef28532060d5561b81f30ea3adb1eec327c
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Sat Mar 18 21:38:53 2023 +0800
HBASE-27216 Revisit the ReplicationSyncUp tool (#4966)
Signed-off-by: Liangjun He <he...@apache.org>
---
.../org/apache/hadoop/hbase/util/JsonMapper.java | 4 +
.../protobuf/server/master/MasterProcedure.proto | 1 +
.../hbase/replication/ReplicationQueueStorage.java | 21 ++
.../replication/ReplicationStorageFactory.java | 27 +-
.../replication/TableReplicationQueueStorage.java | 20 ++
.../org/apache/hadoop/hbase/master/HMaster.java | 41 +++
.../AssignReplicationQueuesProcedure.java | 48 ++-
.../ClaimReplicationQueueRemoteProcedure.java | 32 ++
.../OfflineTableReplicationQueueStorage.java | 382 +++++++++++++++++++++
.../master/replication/ReplicationPeerManager.java | 2 +-
.../regionserver/ReplicationSourceManager.java | 188 +++++-----
.../regionserver/ReplicationSyncUp.java | 195 +++++++++--
.../hadoop/hbase/wal/AbstractFSWALProvider.java | 4 +
.../hbase/master/cleaner/TestLogsCleaner.java | 4 +-
.../replication/TestReplicationSyncUpTool.java | 185 +++++-----
.../replication/TestReplicationSyncUpToolBase.java | 3 +-
...estReplicationSyncUpToolWithBulkLoadedData.java | 58 ++--
.../TestTableReplicationQueueStorage.java | 51 +++
...tReplicationSyncUpToolWithMultipleAsyncWAL.java | 3 -
.../TestReplicationSyncUpToolWithMultipleWAL.java | 3 -
.../regionserver/TestReplicationSourceManager.java | 14 +-
.../regionserver/TestSerialReplicationChecker.java | 4 +-
22 files changed, 1025 insertions(+), 265 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/JsonMapper.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/JsonMapper.java
index 0ff131f23bf..f2c4585a6a8 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/util/JsonMapper.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/util/JsonMapper.java
@@ -40,4 +40,8 @@ public final class JsonMapper {
public static String writeObjectAsString(Object object) throws IOException {
return GSON.toJson(object);
}
+
+ public static <T> T fromJson(String json, Class<T> clazz) {
+ return GSON.fromJson(json, clazz);
+ }
}
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
index 14d07c17c88..901abf6bd0c 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
@@ -717,6 +717,7 @@ message ModifyColumnFamilyStoreFileTrackerStateData {
enum AssignReplicationQueuesState {
ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES = 1;
ASSIGN_REPLICATION_QUEUES_CLAIM = 2;
+ ASSIGN_REPLICATION_QUEUES_REMOVE_QUEUES = 3;
}
message AssignReplicationQueuesStateData {
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
index 1e36bbeb78f..b5bc64eb55a 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
@@ -203,4 +203,25 @@ public interface ReplicationQueueStorage {
* Add the given hfile refs to the given peer.
*/
void batchUpdateHFileRefs(String peerId, List<String> hfileRefs) throws ReplicationException;
+
+ // the below method is for clean up stale data after running ReplicatoinSyncUp
+ /**
+ * Remove all the last sequence ids and hfile references data which are written before the given
+ * timestamp.
+ * <p/>
+ * The data of these two types are not used by replication directly.
+ * <p/>
+ * For last sequence ids, we will check it in serial replication, to make sure that we will
+ * replicate all edits in order, so if there are stale data, the worst case is that we will stop
+ * replicating as we think we still need to finish previous ranges first, although actually we
+ * have already replicated them out.
+ * <p/>
+ * For hfile references, it is just used by hfile cleaner to not remove these hfiles before we
+ * replicate them out, so if there are stale data, the worst case is that we can not remove these
+ * hfiles, although actually they have already been replicated out.
+ * <p/>
+ * So it is OK for us to just bring up the cluster first, and then use this method to delete the
+ * stale data, i.e, the data which are written before a specific timestamp.
+ */
+ void removeLastSequenceIdsAndHFileRefsBefore(long ts) throws ReplicationException;
}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
index dc4317feaa4..4d5fcb45634 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.replication;
import java.io.IOException;
+import java.lang.reflect.Constructor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
@@ -27,8 +28,11 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.CoprocessorDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Used to create replication storage(peer, queue) classes.
@@ -36,11 +40,15 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public final class ReplicationStorageFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(ReplicationStorageFactory.class);
+
public static final String REPLICATION_QUEUE_TABLE_NAME = "hbase.replication.queue.table.name";
public static final TableName REPLICATION_QUEUE_TABLE_NAME_DEFAULT =
TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "replication");
+ public static final String REPLICATION_QUEUE_IMPL = "hbase.replication.queue.impl";
+
public static TableDescriptor createReplicationQueueTableDescriptor(TableName tableName)
throws IOException {
return TableDescriptorBuilder.newBuilder(tableName)
@@ -72,15 +80,26 @@ public final class ReplicationStorageFactory {
*/
public static ReplicationQueueStorage getReplicationQueueStorage(Connection conn,
Configuration conf) {
- return getReplicationQueueStorage(conn, TableName.valueOf(conf.get(REPLICATION_QUEUE_TABLE_NAME,
- REPLICATION_QUEUE_TABLE_NAME_DEFAULT.getNameAsString())));
+ return getReplicationQueueStorage(conn, conf, TableName.valueOf(conf
+ .get(REPLICATION_QUEUE_TABLE_NAME, REPLICATION_QUEUE_TABLE_NAME_DEFAULT.getNameAsString())));
}
/**
* Create a new {@link ReplicationQueueStorage}.
*/
public static ReplicationQueueStorage getReplicationQueueStorage(Connection conn,
- TableName tableName) {
- return new TableReplicationQueueStorage(conn, tableName);
+ Configuration conf, TableName tableName) {
+ Class<? extends ReplicationQueueStorage> clazz = conf.getClass(REPLICATION_QUEUE_IMPL,
+ TableReplicationQueueStorage.class, ReplicationQueueStorage.class);
+ try {
+ Constructor<? extends ReplicationQueueStorage> c =
+ clazz.getConstructor(Connection.class, TableName.class);
+ return c.newInstance(conn, tableName);
+ } catch (Exception e) {
+ LOG.debug(
+ "failed to create ReplicationQueueStorage with Connection, try creating with Configuration",
+ e);
+ return ReflectionUtils.newInstance(clazz, conf, tableName);
+ }
}
}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
index f3870f4d09d..e59edd52f79 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
@@ -594,4 +594,24 @@ public class TableReplicationQueueStorage implements ReplicationQueueStorage {
throw new ReplicationException("failed to batch update hfile references", e);
}
}
+
+ @Override
+ public void removeLastSequenceIdsAndHFileRefsBefore(long ts) throws ReplicationException {
+ try (Table table = conn.getTable(tableName);
+ ResultScanner scanner = table.getScanner(new Scan().addFamily(LAST_SEQUENCE_ID_FAMILY)
+ .addFamily(HFILE_REF_FAMILY).setFilter(new KeyOnlyFilter()))) {
+ for (;;) {
+ Result r = scanner.next();
+ if (r == null) {
+ break;
+ }
+ Delete delete = new Delete(r.getRow()).addFamily(LAST_SEQUENCE_ID_FAMILY, ts)
+ .addFamily(HFILE_REF_FAMILY, ts);
+ table.delete(delete);
+ }
+ } catch (IOException e) {
+ throw new ReplicationException(
+ "failed to remove last sequence ids and hfile references before timestamp " + ts, e);
+ }
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 67d0f889d64..f2086393e63 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -34,6 +34,9 @@ import java.lang.reflect.InvocationTargetException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -59,6 +62,7 @@ import java.util.stream.Collectors;
import javax.servlet.http.HttpServlet;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CatalogFamilyFormat;
@@ -226,6 +230,8 @@ import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration
import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner;
import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
import org.apache.hadoop.hbase.replication.master.ReplicationSinkTrackerTableCreator;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSyncUp;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSyncUp.ReplicationSyncUpToolInfo;
import org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint;
import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer;
import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager;
@@ -246,6 +252,7 @@ import org.apache.hadoop.hbase.util.HBaseFsck;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.hbase.util.IdLock;
import org.apache.hadoop.hbase.util.JVMClusterUtil;
+import org.apache.hadoop.hbase.util.JsonMapper;
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.RetryCounter;
@@ -267,7 +274,9 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
+import org.apache.hbase.thirdparty.com.google.common.io.ByteStreams;
import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
+import org.apache.hbase.thirdparty.com.google.gson.JsonParseException;
import org.apache.hbase.thirdparty.com.google.protobuf.Descriptors;
import org.apache.hbase.thirdparty.com.google.protobuf.Service;
import org.apache.hbase.thirdparty.org.eclipse.jetty.server.Server;
@@ -1278,6 +1287,38 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
status.setStatus("Initializing MOB Cleaner");
initMobCleaner();
+ // delete the stale data for replication sync up tool if necessary
+ status.setStatus("Cleanup ReplicationSyncUp status if necessary");
+ Path replicationSyncUpInfoFile =
+ new Path(new Path(dataRootDir, ReplicationSyncUp.INFO_DIR), ReplicationSyncUp.INFO_FILE);
+ if (dataFs.exists(replicationSyncUpInfoFile)) {
+ // info file is available, load the timestamp and use it to clean up stale data in replication
+ // queue storage.
+ byte[] data;
+ try (FSDataInputStream in = dataFs.open(replicationSyncUpInfoFile)) {
+ data = ByteStreams.toByteArray(in);
+ }
+ ReplicationSyncUpToolInfo info = null;
+ try {
+ info = JsonMapper.fromJson(Bytes.toString(data), ReplicationSyncUpToolInfo.class);
+ } catch (JsonParseException e) {
+ // usually this should be a partial file, which means the ReplicationSyncUp tool did not
+ // finish properly, so not a problem. Here we do not clean up the status as we do not know
+ // the reason why the tool did not finish properly, so let users clean the status up
+ // manually
+ LOG.warn("failed to parse replication sync up info file, ignore and continue...", e);
+ }
+ if (info != null) {
+ LOG.info("Remove last sequence ids and hfile references which are written before {}({})",
+ info.getStartTimeMs(), DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.systemDefault())
+ .format(Instant.ofEpochMilli(info.getStartTimeMs())));
+ replicationPeerManager.getQueueStorage()
+ .removeLastSequenceIdsAndHFileRefsBefore(info.getStartTimeMs());
+ // delete the file after removing the stale data, so next time we do not need to do this
+ // again.
+ dataFs.delete(replicationSyncUpInfoFile, false);
+ }
+ }
status.setStatus("Calling postStartMaster coprocessors");
if (this.cpHost != null) {
// don't let cp initialization errors kill the master
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
index d33259dd436..b547c87009d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
@@ -24,7 +24,9 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
@@ -37,6 +39,7 @@ import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSyncUp;
import org.apache.hadoop.hbase.util.RetryCounter;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -102,7 +105,7 @@ public class AssignReplicationQueuesProcedure
}
}
- private Flow claimQueues(MasterProcedureEnv env) throws ReplicationException {
+ private Flow claimQueues(MasterProcedureEnv env) throws ReplicationException, IOException {
Set<String> existingPeerIds = env.getReplicationPeerManager().listPeers(null).stream()
.map(ReplicationPeerDescription::getPeerId).collect(Collectors.toSet());
ReplicationQueueStorage storage = env.getReplicationPeerManager().getQueueStorage();
@@ -130,18 +133,51 @@ public class AssignReplicationQueuesProcedure
return Flow.HAS_MORE_STATE;
}
+ // check whether ReplicationSyncUp has already done the work for us, if so, we should skip
+ // claiming the replication queues and deleting them instead.
+ private boolean shouldSkip(MasterProcedureEnv env) throws IOException {
+ MasterFileSystem mfs = env.getMasterFileSystem();
+ Path syncUpDir = new Path(mfs.getRootDir(), ReplicationSyncUp.INFO_DIR);
+ return mfs.getFileSystem().exists(new Path(syncUpDir, crashedServer.getServerName()));
+ }
+
+ private void removeQueues(MasterProcedureEnv env) throws ReplicationException, IOException {
+ ReplicationQueueStorage storage = env.getReplicationPeerManager().getQueueStorage();
+ for (ReplicationQueueId queueId : storage.listAllQueueIds(crashedServer)) {
+ storage.removeQueue(queueId);
+ }
+ MasterFileSystem mfs = env.getMasterFileSystem();
+ Path syncUpDir = new Path(mfs.getRootDir(), ReplicationSyncUp.INFO_DIR);
+ // remove the region server record file
+ mfs.getFileSystem().delete(new Path(syncUpDir, crashedServer.getServerName()), false);
+ }
+
@Override
protected Flow executeFromState(MasterProcedureEnv env, AssignReplicationQueuesState state)
throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
try {
switch (state) {
case ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES:
- addMissingQueues(env);
- retryCounter = null;
- setNextState(AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_CLAIM);
- return Flow.HAS_MORE_STATE;
+ if (shouldSkip(env)) {
+ setNextState(AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_REMOVE_QUEUES);
+ return Flow.HAS_MORE_STATE;
+ } else {
+ addMissingQueues(env);
+ retryCounter = null;
+ setNextState(AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_CLAIM);
+ return Flow.HAS_MORE_STATE;
+ }
case ASSIGN_REPLICATION_QUEUES_CLAIM:
- return claimQueues(env);
+ if (shouldSkip(env)) {
+ retryCounter = null;
+ setNextState(AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_REMOVE_QUEUES);
+ return Flow.HAS_MORE_STATE;
+ } else {
+ return claimQueues(env);
+ }
+ case ASSIGN_REPLICATION_QUEUES_REMOVE_QUEUES:
+ removeQueues(env);
+ return Flow.NO_MORE_STATE;
default:
throw new UnsupportedOperationException("unhandled state=" + state);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
index 7b637384398..d3aeeba541a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
@@ -19,16 +19,22 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
import java.util.Optional;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher.ServerOperation;
import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface;
import org.apache.hadoop.hbase.master.procedure.ServerRemoteProcedure;
+import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteOperation;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure;
import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.regionserver.ClaimReplicationQueueCallable;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSyncUp;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,6 +60,32 @@ public class ClaimReplicationQueueRemoteProcedure extends ServerRemoteProcedure
this.targetServer = targetServer;
}
+ // check whether ReplicationSyncUp has already done the work for us, if so, we should skip
+ // claiming the replication queues and deleting them instead.
+ private boolean shouldSkip(MasterProcedureEnv env) throws IOException {
+ MasterFileSystem mfs = env.getMasterFileSystem();
+ Path syncUpDir = new Path(mfs.getRootDir(), ReplicationSyncUp.INFO_DIR);
+ return mfs.getFileSystem().exists(new Path(syncUpDir, getServerName().getServerName()));
+ }
+
+ @Override
+ protected synchronized Procedure<MasterProcedureEnv>[] execute(MasterProcedureEnv env)
+ throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
+ try {
+ if (shouldSkip(env)) {
+ LOG.info("Skip claiming {} because replication sync up has already done it for us",
+ getServerName());
+ return null;
+ }
+ } catch (IOException e) {
+ LOG.warn("failed to check whether we should skip claiming {} due to replication sync up",
+ getServerName(), e);
+ // just finish the procedure here, as the AssignReplicationQueuesProcedure will reschedule
+ return null;
+ }
+ return super.execute(env);
+ }
+
@Override
public Optional<RemoteOperation> remoteCallBuild(MasterProcedureEnv env, ServerName remote) {
assert targetServer.equals(remote);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/OfflineTableReplicationQueueStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/OfflineTableReplicationQueueStorage.java
new file mode 100644
index 00000000000..9faca74f710
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/OfflineTableReplicationQueueStorage.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ClientSideRegionScanner;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.replication.TableReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ZKReplicationQueueStorageForMigration;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
+@InterfaceAudience.Private
+public class OfflineTableReplicationQueueStorage implements ReplicationQueueStorage {
+
+ private final Map<ReplicationQueueId, Map<String, ReplicationGroupOffset>> offsets =
+ new HashMap<>();
+
+ private final Map<String, Map<String, Long>> lastSequenceIds = new HashMap<>();
+
+ private final Map<String, Set<String>> hfileRefs = new HashMap<>();
+
+ private void loadRegionInfo(FileSystem fs, Path regionDir,
+ NavigableMap<byte[], RegionInfo> startKey2RegionInfo) throws IOException {
+ RegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
+ // TODO: we consider that the there will not be too many regions for hbase:replication table, so
+ // here we just iterate over all the regions to find out the overlapped ones. Can be optimized
+ // later.
+ Iterator<Map.Entry<byte[], RegionInfo>> iter = startKey2RegionInfo.entrySet().iterator();
+ while (iter.hasNext()) {
+ Map.Entry<byte[], RegionInfo> entry = iter.next();
+ if (hri.isOverlap(entry.getValue())) {
+ if (hri.getRegionId() > entry.getValue().getRegionId()) {
+ // we are newer, remove the old hri, we can not break here as if hri is a merged region,
+ // we need to remove all its parent regions.
+ iter.remove();
+ } else {
+ // we are older, just return, skip the below add
+ return;
+ }
+ }
+
+ }
+ startKey2RegionInfo.put(hri.getStartKey(), hri);
+ }
+
+ private void loadOffsets(Result result) {
+ NavigableMap<byte[], byte[]> map =
+ result.getFamilyMap(TableReplicationQueueStorage.QUEUE_FAMILY);
+ if (map == null || map.isEmpty()) {
+ return;
+ }
+ Map<String, ReplicationGroupOffset> offsetMap = new HashMap<>();
+ map.forEach((k, v) -> {
+ String walGroup = Bytes.toString(k);
+ ReplicationGroupOffset offset = ReplicationGroupOffset.parse(Bytes.toString(v));
+ offsetMap.put(walGroup, offset);
+ });
+ ReplicationQueueId queueId = ReplicationQueueId.parse(Bytes.toString(result.getRow()));
+ offsets.put(queueId, offsetMap);
+ }
+
+ private void loadLastSequenceIds(Result result) {
+ NavigableMap<byte[], byte[]> map =
+ result.getFamilyMap(TableReplicationQueueStorage.LAST_SEQUENCE_ID_FAMILY);
+ if (map == null || map.isEmpty()) {
+ return;
+ }
+ Map<String, Long> lastSeqIdMap = new HashMap<>();
+ map.forEach((k, v) -> {
+ String encodedRegionName = Bytes.toString(k);
+ long lastSeqId = Bytes.toLong(v);
+ lastSeqIdMap.put(encodedRegionName, lastSeqId);
+ });
+ String peerId = Bytes.toString(result.getRow());
+ lastSequenceIds.put(peerId, lastSeqIdMap);
+ }
+
+ private void loadHFileRefs(Result result) {
+ NavigableMap<byte[], byte[]> map =
+ result.getFamilyMap(TableReplicationQueueStorage.HFILE_REF_FAMILY);
+ if (map == null || map.isEmpty()) {
+ return;
+ }
+ Set<String> refs = new HashSet<>();
+ map.keySet().forEach(ref -> refs.add(Bytes.toString(ref)));
+ String peerId = Bytes.toString(result.getRow());
+ hfileRefs.put(peerId, refs);
+ }
+
+ private void loadReplicationQueueData(Configuration conf, TableName tableName)
+ throws IOException {
+ Path rootDir = CommonFSUtils.getRootDir(conf);
+ Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
+ FileSystem fs = tableDir.getFileSystem(conf);
+ FileStatus[] regionDirs =
+ CommonFSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
+ if (regionDirs == null) {
+ return;
+ }
+ NavigableMap<byte[], RegionInfo> startKey2RegionInfo = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ for (FileStatus regionDir : regionDirs) {
+ loadRegionInfo(fs, regionDir.getPath(), startKey2RegionInfo);
+ }
+ TableDescriptor td = ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName);
+ for (RegionInfo hri : startKey2RegionInfo.values()) {
+ try (ClientSideRegionScanner scanner =
+ new ClientSideRegionScanner(conf, fs, rootDir, td, hri, new Scan(), null)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ loadOffsets(result);
+ loadLastSequenceIds(result);
+ loadHFileRefs(result);
+ }
+ }
+ }
+ }
+
+ public OfflineTableReplicationQueueStorage(Configuration conf, TableName tableName)
+ throws IOException {
+ loadReplicationQueueData(conf, tableName);
+ }
+
+ @Override
+ public synchronized void setOffset(ReplicationQueueId queueId, String walGroup,
+ ReplicationGroupOffset offset, Map<String, Long> lastSeqIds) throws ReplicationException {
+ Map<String, ReplicationGroupOffset> offsetMap = offsets.get(queueId);
+ if (offsetMap == null) {
+ offsetMap = new HashMap<>();
+ offsets.put(queueId, offsetMap);
+ }
+ offsetMap.put(walGroup, offset);
+ Map<String, Long> lastSeqIdsMap = lastSequenceIds.get(queueId.getPeerId());
+ if (lastSeqIdsMap == null) {
+ lastSeqIdsMap = new HashMap<>();
+ lastSequenceIds.put(queueId.getPeerId(), lastSeqIdsMap);
+ }
+ for (Map.Entry<String, Long> entry : lastSeqIds.entrySet()) {
+ Long oldSeqId = lastSeqIdsMap.get(entry.getKey());
+ if (oldSeqId == null || oldSeqId < entry.getValue()) {
+ lastSeqIdsMap.put(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+
+ @Override
+ public synchronized Map<String, ReplicationGroupOffset> getOffsets(ReplicationQueueId queueId)
+ throws ReplicationException {
+ Map<String, ReplicationGroupOffset> offsetMap = offsets.get(queueId);
+ if (offsetMap == null) {
+ return Collections.emptyMap();
+ }
+ return ImmutableMap.copyOf(offsetMap);
+ }
+
+ @Override
+ public synchronized List<ReplicationQueueId> listAllQueueIds(String peerId)
+ throws ReplicationException {
+ return offsets.keySet().stream().filter(rqi -> rqi.getPeerId().equals(peerId))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized List<ReplicationQueueId> listAllQueueIds(ServerName serverName)
+ throws ReplicationException {
+ return offsets.keySet().stream().filter(rqi -> rqi.getServerName().equals(serverName))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized List<ReplicationQueueId> listAllQueueIds(String peerId, ServerName serverName)
+ throws ReplicationException {
+ return offsets.keySet().stream()
+ .filter(rqi -> rqi.getPeerId().equals(peerId) && rqi.getServerName().equals(serverName))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized List<ReplicationQueueData> listAllQueues() throws ReplicationException {
+ return offsets.entrySet().stream()
+ .map(e -> new ReplicationQueueData(e.getKey(), ImmutableMap.copyOf(e.getValue())))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized List<ServerName> listAllReplicators() throws ReplicationException {
+ return offsets.keySet().stream().map(ReplicationQueueId::getServerName).distinct()
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized Map<String, ReplicationGroupOffset> claimQueue(ReplicationQueueId queueId,
+ ServerName targetServerName) throws ReplicationException {
+ Map<String, ReplicationGroupOffset> offsetMap = offsets.remove(queueId);
+ if (offsetMap == null) {
+ return Collections.emptyMap();
+ }
+ offsets.put(queueId.claim(targetServerName), offsetMap);
+ return ImmutableMap.copyOf(offsetMap);
+ }
+
+ @Override
+ public synchronized void removeQueue(ReplicationQueueId queueId) throws ReplicationException {
+ offsets.remove(queueId);
+ }
+
+ @Override
+ public synchronized void removeAllQueues(String peerId) throws ReplicationException {
+ Iterator<ReplicationQueueId> iter = offsets.keySet().iterator();
+ while (iter.hasNext()) {
+ if (iter.next().getPeerId().equals(peerId)) {
+ iter.remove();
+ }
+ }
+ }
+
+ @Override
+ public synchronized long getLastSequenceId(String encodedRegionName, String peerId)
+ throws ReplicationException {
+ Map<String, Long> lastSeqIdMap = lastSequenceIds.get(peerId);
+ if (lastSeqIdMap == null) {
+ return HConstants.NO_SEQNUM;
+ }
+ Long lastSeqId = lastSeqIdMap.get(encodedRegionName);
+ return lastSeqId != null ? lastSeqId.longValue() : HConstants.NO_SEQNUM;
+ }
+
+ @Override
+ public synchronized void setLastSequenceIds(String peerId, Map<String, Long> lastSeqIds)
+ throws ReplicationException {
+ Map<String, Long> lastSeqIdMap = lastSequenceIds.get(peerId);
+ if (lastSeqIdMap == null) {
+ lastSeqIdMap = new HashMap<>();
+ lastSequenceIds.put(peerId, lastSeqIdMap);
+ }
+ lastSeqIdMap.putAll(lastSeqIds);
+ }
+
+ @Override
+ public synchronized void removeLastSequenceIds(String peerId) throws ReplicationException {
+ lastSequenceIds.remove(peerId);
+ }
+
+ @Override
+ public synchronized void removeLastSequenceIds(String peerId, List<String> encodedRegionNames)
+ throws ReplicationException {
+ Map<String, Long> lastSeqIdMap = lastSequenceIds.get(peerId);
+ if (lastSeqIdMap == null) {
+ return;
+ }
+ for (String encodedRegionName : encodedRegionNames) {
+ lastSeqIdMap.remove(encodedRegionName);
+ }
+ }
+
+ @Override
+ public synchronized void removePeerFromHFileRefs(String peerId) throws ReplicationException {
+ hfileRefs.remove(peerId);
+ }
+
+ @Override
+ public synchronized void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs)
+ throws ReplicationException {
+ Set<String> refs = hfileRefs.get(peerId);
+ if (refs == null) {
+ refs = new HashSet<>();
+ hfileRefs.put(peerId, refs);
+ }
+ for (Pair<Path, Path> pair : pairs) {
+ refs.add(pair.getSecond().getName());
+ }
+ }
+
+ @Override
+ public synchronized void removeHFileRefs(String peerId, List<String> files)
+ throws ReplicationException {
+ Set<String> refs = hfileRefs.get(peerId);
+ if (refs == null) {
+ return;
+ }
+ refs.removeAll(files);
+ }
+
+ @Override
+ public synchronized List<String> getAllPeersFromHFileRefsQueue() throws ReplicationException {
+ return ImmutableList.copyOf(hfileRefs.keySet());
+ }
+
+ @Override
+ public synchronized List<String> getReplicableHFiles(String peerId) throws ReplicationException {
+ Set<String> refs = hfileRefs.get(peerId);
+ if (refs == null) {
+ return Collections.emptyList();
+ }
+ return ImmutableList.copyOf(refs);
+ }
+
+ @Override
+ public synchronized Set<String> getAllHFileRefs() throws ReplicationException {
+ return hfileRefs.values().stream().flatMap(Set::stream).collect(Collectors.toSet());
+ }
+
+ @Override
+ public boolean hasData() throws ReplicationException {
+ return true;
+ }
+
+ @Override
+ public void batchUpdateQueues(ServerName serverName, List<ReplicationQueueData> datas)
+ throws ReplicationException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void batchUpdateLastSequenceIds(
+ List<ZKReplicationQueueStorageForMigration.ZkLastPushedSeqId> lastPushedSeqIds)
+ throws ReplicationException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void batchUpdateHFileRefs(String peerId, List<String> hfileRefs)
+ throws ReplicationException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void removeLastSequenceIdsAndHFileRefsBefore(long ts) throws ReplicationException {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index d8c1b5c64c5..bb170be64af 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -641,7 +641,7 @@ public class ReplicationPeerManager {
};
}
return Pair.newPair(ReplicationStorageFactory.getReplicationQueueStorage(
- services.getConnection(), replicationQueueTableName), initializer);
+ services.getConnection(), conf, replicationQueueTableName), initializer);
}
public static ReplicationPeerManager create(MasterServices services, String clusterId)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index f3d07315240..f34f0d194e6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -25,7 +25,6 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -125,6 +124,7 @@ import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFacto
*/
@InterfaceAudience.Private
public class ReplicationSourceManager {
+
private static final Logger LOG = LoggerFactory.getLogger(ReplicationSourceManager.class);
// all the sources that read this RS's logs and every peer only has one replication source
private final ConcurrentMap<String, ReplicationSourceInterface> sources;
@@ -146,13 +146,15 @@ public class ReplicationSourceManager {
// All logs we are currently tracking
// Index structure of the map is: queue_id->logPrefix/logGroup->logs
- // For normal replication source, the peer id is same with the queue id
private final ConcurrentMap<ReplicationQueueId, Map<String, NavigableSet<String>>> walsById;
// Logs for recovered sources we are currently tracking
// the map is: queue_id->logPrefix/logGroup->logs
- // For recovered source, the queue id's format is peer_id-servername-*
+ // for recovered source, the WAL files should already been moved to oldLogDir, and we have
+ // different layout of old WAL files, for example, with server name sub directories or not, so
+ // here we record the full path instead of just the name, so when refreshing we can enqueue the
+ // WAL file again, without trying to guess the real path of the WAL files.
private final ConcurrentMap<ReplicationQueueId,
- Map<String, NavigableSet<String>>> walsByIdRecoveredQueues;
+ Map<String, NavigableSet<Path>>> walsByIdRecoveredQueues;
private final SyncReplicationPeerMappingManager syncReplicationPeerMappingManager;
@@ -514,9 +516,9 @@ public class ReplicationSourceManager {
ReplicationSourceInterface recoveredReplicationSource =
createRefreshedSource(oldSourceQueueId, peer);
this.oldsources.add(recoveredReplicationSource);
- for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(oldSourceQueueId)
+ for (NavigableSet<Path> walsByGroup : walsByIdRecoveredQueues.get(oldSourceQueueId)
.values()) {
- walsByGroup.forEach(wal -> recoveredReplicationSource.enqueueLog(new Path(wal)));
+ walsByGroup.forEach(wal -> recoveredReplicationSource.enqueueLog(wal));
}
toStartup.add(recoveredReplicationSource);
}
@@ -656,9 +658,11 @@ public class ReplicationSourceManager {
void cleanOldLogs(String log, boolean inclusive, ReplicationSourceInterface source) {
String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(log);
if (source.isRecovered()) {
- NavigableSet<String> wals = walsByIdRecoveredQueues.get(source.getQueueId()).get(logPrefix);
+ NavigableSet<Path> wals = walsByIdRecoveredQueues.get(source.getQueueId()).get(logPrefix);
if (wals != null) {
- NavigableSet<String> walsToRemove = wals.headSet(log, inclusive);
+ // here we just want to compare the timestamp, so it is OK to just create a fake WAL path
+ NavigableSet<String> walsToRemove = wals.headSet(new Path(oldLogDir, log), inclusive)
+ .stream().map(Path::getName).collect(Collectors.toCollection(TreeSet::new));
if (walsToRemove.isEmpty()) {
return;
}
@@ -814,6 +818,93 @@ public class ReplicationSourceManager {
}
void claimQueue(ReplicationQueueId queueId) {
+ claimQueue(queueId, false);
+ }
+
+ // sorted from oldest to newest
+ private PriorityQueue<Path> getWALFilesToReplicate(ServerName sourceRS, boolean syncUp,
+ Map<String, ReplicationGroupOffset> offsets) throws IOException {
+ List<Path> walFiles = AbstractFSWALProvider.getArchivedWALFiles(conf, sourceRS,
+ URLEncoder.encode(sourceRS.toString(), StandardCharsets.UTF_8.name()));
+ if (syncUp) {
+ // we also need to list WALs directory for ReplicationSyncUp
+ walFiles.addAll(AbstractFSWALProvider.getWALFiles(conf, sourceRS));
+ }
+ PriorityQueue<Path> walFilesPQ =
+ new PriorityQueue<>(AbstractFSWALProvider.TIMESTAMP_COMPARATOR);
+ // sort the wal files and also filter out replicated files
+ for (Path file : walFiles) {
+ String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(file.getName());
+ ReplicationGroupOffset groupOffset = offsets.get(walGroupId);
+ if (shouldReplicate(groupOffset, file.getName())) {
+ walFilesPQ.add(file);
+ } else {
+ LOG.debug("Skip enqueuing log {} because it is before the start offset {}", file.getName(),
+ groupOffset);
+ }
+ }
+ return walFilesPQ;
+ }
+
+ private void addRecoveredSource(ReplicationSourceInterface src, ReplicationPeerImpl oldPeer,
+ ReplicationQueueId claimedQueueId, PriorityQueue<Path> walFiles) {
+ ReplicationPeerImpl peer = replicationPeers.getPeer(src.getPeerId());
+ if (peer == null || peer != oldPeer) {
+ src.terminate("Recovered queue doesn't belong to any current peer");
+ deleteQueue(claimedQueueId);
+ return;
+ }
+ // Do not setup recovered queue if a sync replication peer is in STANDBY state, or is
+ // transiting to STANDBY state. The only exception is we are in STANDBY state and
+ // transiting to DA, under this state we will replay the remote WAL and they need to be
+ // replicated back.
+ if (peer.getPeerConfig().isSyncReplication()) {
+ Pair<SyncReplicationState, SyncReplicationState> stateAndNewState =
+ peer.getSyncReplicationStateAndNewState();
+ if (
+ (stateAndNewState.getFirst().equals(SyncReplicationState.STANDBY)
+ && stateAndNewState.getSecond().equals(SyncReplicationState.NONE))
+ || stateAndNewState.getSecond().equals(SyncReplicationState.STANDBY)
+ ) {
+ src.terminate("Sync replication peer is in STANDBY state");
+ deleteQueue(claimedQueueId);
+ return;
+ }
+ }
+ // track sources in walsByIdRecoveredQueues
+ Map<String, NavigableSet<Path>> walsByGroup = new HashMap<>();
+ walsByIdRecoveredQueues.put(claimedQueueId, walsByGroup);
+ for (Path wal : walFiles) {
+ String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal.getName());
+ NavigableSet<Path> wals = walsByGroup.get(walPrefix);
+ if (wals == null) {
+ wals = new TreeSet<>(AbstractFSWALProvider.TIMESTAMP_COMPARATOR);
+ walsByGroup.put(walPrefix, wals);
+ }
+ wals.add(wal);
+ }
+ oldsources.add(src);
+ LOG.info("Added source for recovered queue {}, number of wals to replicate: {}", claimedQueueId,
+ walFiles.size());
+ for (Path wal : walFiles) {
+ LOG.debug("Enqueueing log {} from recovered queue for source: {}", wal, claimedQueueId);
+ src.enqueueLog(wal);
+ }
+ src.startup();
+ }
+
+ /**
+ * Claim a replication queue.
+ * <p/>
+ * We add a flag to indicate whether we are called by ReplicationSyncUp. For normal claiming queue
+ * operation, we are the last step of a SCP, so we can assume that all the WAL files are under
+ * oldWALs directory. But for ReplicationSyncUp, we may want to claim the replication queue for a
+ * region server which has not been processed by SCP yet, so we still need to look at its WALs
+ * directory.
+ * @param queueId the replication queue id we want to claim
+ * @param syncUp whether we are called by ReplicationSyncUp
+ */
+ void claimQueue(ReplicationQueueId queueId, boolean syncUp) {
// Wait a bit before transferring the queues, we may be shutting down.
// This sleep may not be enough in some cases.
try {
@@ -872,76 +963,17 @@ public class ReplicationSourceManager {
server.abort("Failed to create replication source after claiming queue.", e);
return;
}
- List<Path> walFiles;
+ PriorityQueue<Path> walFiles;
try {
- walFiles = AbstractFSWALProvider.getArchivedWALFiles(conf, sourceRS,
- URLEncoder.encode(sourceRS.toString(), StandardCharsets.UTF_8.name()));
+ walFiles = getWALFilesToReplicate(sourceRS, syncUp, offsets);
} catch (IOException e) {
- LOG.error("Can not list all wal files for peer {} and queue {}", peerId, queueId, e);
- server.abort("Can not list all wal files after claiming queue.", e);
+ LOG.error("Can not list wal files for peer {} and queue {}", peerId, queueId, e);
+ server.abort("Can not list wal files after claiming queue.", e);
return;
}
- PriorityQueue<Path> walFilesPQ = new PriorityQueue<>(
- Comparator.<Path, Long> comparing(p -> AbstractFSWALProvider.getTimestamp(p.getName()))
- .thenComparing(Path::getName));
- // sort the wal files and also filter out replicated files
- for (Path file : walFiles) {
- String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(file.getName());
- ReplicationGroupOffset groupOffset = offsets.get(walGroupId);
- if (shouldReplicate(groupOffset, file.getName())) {
- walFilesPQ.add(file);
- } else {
- LOG.debug("Skip enqueuing log {} because it is before the start offset {}", file.getName(),
- groupOffset);
- }
- }
- // the method is a bit long, so assign it to null here to avoid later we reuse it again by
- // mistake, we should use the sorted walFilesPQ instead
- walFiles = null;
// synchronized on oldsources to avoid adding recovered source for the to-be-removed peer
synchronized (oldsources) {
- peer = replicationPeers.getPeer(src.getPeerId());
- if (peer == null || peer != oldPeer) {
- src.terminate("Recovered queue doesn't belong to any current peer");
- deleteQueue(claimedQueueId);
- return;
- }
- // Do not setup recovered queue if a sync replication peer is in STANDBY state, or is
- // transiting to STANDBY state. The only exception is we are in STANDBY state and
- // transiting to DA, under this state we will replay the remote WAL and they need to be
- // replicated back.
- if (peer.getPeerConfig().isSyncReplication()) {
- Pair<SyncReplicationState, SyncReplicationState> stateAndNewState =
- peer.getSyncReplicationStateAndNewState();
- if (
- (stateAndNewState.getFirst().equals(SyncReplicationState.STANDBY)
- && stateAndNewState.getSecond().equals(SyncReplicationState.NONE))
- || stateAndNewState.getSecond().equals(SyncReplicationState.STANDBY)
- ) {
- src.terminate("Sync replication peer is in STANDBY state");
- deleteQueue(claimedQueueId);
- return;
- }
- }
- // track sources in walsByIdRecoveredQueues
- Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();
- walsByIdRecoveredQueues.put(claimedQueueId, walsByGroup);
- for (Path wal : walFilesPQ) {
- String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal.getName());
- NavigableSet<String> wals = walsByGroup.get(walPrefix);
- if (wals == null) {
- wals = new TreeSet<>();
- walsByGroup.put(walPrefix, wals);
- }
- wals.add(wal.getName());
- }
- oldsources.add(src);
- LOG.info("Added source for recovered queue {}", claimedQueueId);
- for (Path wal : walFilesPQ) {
- LOG.debug("Enqueueing log {} from recovered queue for source: {}", wal, claimedQueueId);
- src.enqueueLog(new Path(oldLogDir, wal));
- }
- src.startup();
+ addRecoveredSource(src, oldPeer, claimedQueueId, walFiles);
}
}
@@ -970,16 +1002,6 @@ public class ReplicationSourceManager {
return Collections.unmodifiableMap(walsById);
}
- /**
- * Get a copy of the wals of the recovered sources on this rs
- * @return a sorted set of wal names
- */
- @RestrictedApi(explanation = "Should only be called in tests", link = "",
- allowedOnPath = ".*/src/test/.*")
- Map<ReplicationQueueId, Map<String, NavigableSet<String>>> getWalsByIdRecoveredQueues() {
- return Collections.unmodifiableMap(walsByIdRecoveredQueues);
- }
-
/**
* Get a list of all the normal sources of this rs
* @return list of all normal sources
@@ -1099,8 +1121,6 @@ public class ReplicationSourceManager {
return this.globalMetrics;
}
- @RestrictedApi(explanation = "Should only be called in tests", link = "",
- allowedOnPath = ".*/src/test/.*")
ReplicationQueueStorage getQueueStorage() {
return queueStorage;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index b63ad473719..f071cf6f1f8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -17,13 +17,17 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
@@ -35,11 +39,18 @@ import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.AsyncClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.master.replication.OfflineTableReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.JsonMapper;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.hbase.wal.WALFactory;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -59,6 +70,31 @@ import org.apache.zookeeper.KeeperException;
@InterfaceAudience.Private
public class ReplicationSyncUp extends Configured implements Tool {
+ public static class ReplicationSyncUpToolInfo {
+
+ private long startTimeMs;
+
+ public ReplicationSyncUpToolInfo() {
+ }
+
+ public ReplicationSyncUpToolInfo(long startTimeMs) {
+ this.startTimeMs = startTimeMs;
+ }
+
+ public long getStartTimeMs() {
+ return startTimeMs;
+ }
+
+ public void setStartTimeMs(long startTimeMs) {
+ this.startTimeMs = startTimeMs;
+ }
+ }
+
+ // For storing the information used to skip replicating some wals after the cluster is back online
+ public static final String INFO_DIR = "ReplicationSyncUp";
+
+ public static final String INFO_FILE = "info";
+
private static final long SLEEP_TIME = 10000;
/**
@@ -69,41 +105,116 @@ public class ReplicationSyncUp extends Configured implements Tool {
System.exit(ret);
}
- private Set<ServerName> getLiveRegionServers(ZKWatcher zkw) throws KeeperException {
- List<String> rsZNodes = ZKUtil.listChildrenNoWatch(zkw, zkw.getZNodePaths().rsZNode);
- return rsZNodes == null
- ? Collections.emptySet()
- : rsZNodes.stream().map(ServerName::parseServerName).collect(Collectors.toSet());
+ // Find region servers under wal directory
+ // Here we only care about the region servers which may still be alive, as we need to add
+ // replications for them if missing. The dead region servers which have already been processed
+ // fully do not need to add their replication queues again, as the operation has already been done
+ // in SCP.
+ private Set<ServerName> listRegionServers(FileSystem walFs, Path walDir) throws IOException {
+ FileStatus[] statuses;
+ try {
+ statuses = walFs.listStatus(walDir);
+ } catch (FileNotFoundException e) {
+ System.out.println("WAL directory " + walDir + " does not exists, ignore");
+ return Collections.emptySet();
+ }
+ Set<ServerName> regionServers = new HashSet<>();
+ for (FileStatus status : statuses) {
+ // All wal files under the walDir is within its region server's directory
+ if (!status.isDirectory()) {
+ continue;
+ }
+ ServerName sn = AbstractFSWALProvider.getServerNameFromWALDirectoryName(status.getPath());
+ if (sn != null) {
+ regionServers.add(sn);
+ }
+ }
+ return regionServers;
+ }
+
+ private void addMissingReplicationQueues(ReplicationQueueStorage storage, ServerName regionServer,
+ Set<String> peerIds) throws ReplicationException {
+ Set<String> existingQueuePeerIds = new HashSet<>();
+ List<ReplicationQueueId> queueIds = storage.listAllQueueIds(regionServer);
+ for (Iterator<ReplicationQueueId> iter = queueIds.iterator(); iter.hasNext();) {
+ ReplicationQueueId queueId = iter.next();
+ if (!queueId.isRecovered()) {
+ existingQueuePeerIds.add(queueId.getPeerId());
+ }
+ }
+
+ for (String peerId : peerIds) {
+ if (!existingQueuePeerIds.contains(peerId)) {
+ ReplicationQueueId queueId = new ReplicationQueueId(regionServer, peerId);
+ System.out.println("Add replication queue " + queueId + " for claiming");
+ storage.setOffset(queueId, regionServer.toString(), ReplicationGroupOffset.BEGIN,
+ Collections.emptyMap());
+ }
+ }
+ }
+
+ private void addMissingReplicationQueues(ReplicationQueueStorage storage,
+ Set<ServerName> regionServers, Set<String> peerIds) throws ReplicationException {
+ for (ServerName regionServer : regionServers) {
+ addMissingReplicationQueues(storage, regionServer, peerIds);
+ }
}
// When using this tool, usually the source cluster is unhealthy, so we should try to claim the
// replication queues for the dead region servers first and then replicate the data out.
- private void claimReplicationQueues(ZKWatcher zkw, ReplicationSourceManager mgr)
- throws ReplicationException, KeeperException {
- // TODO: reimplement this tool
- // List<ServerName> replicators = mgr.getQueueStorage().getListOfReplicators();
- // Set<ServerName> liveRegionServers = getLiveRegionServers(zkw);
- // for (ServerName sn : replicators) {
- // if (!liveRegionServers.contains(sn)) {
- // List<String> replicationQueues = mgr.getQueueStorage().getAllQueues(sn);
- // System.out.println(sn + " is dead, claim its replication queues: " + replicationQueues);
- // for (String queue : replicationQueues) {
- // mgr.claimQueue(sn, queue);
- // }
- // }
- // }
+ private void claimReplicationQueues(ReplicationSourceManager mgr, Set<ServerName> regionServers)
+ throws ReplicationException, KeeperException, IOException {
+ // union the region servers from both places, i.e, from the wal directory, and the records in
+ // replication queue storage.
+ Set<ServerName> replicators = new HashSet<>(regionServers);
+ ReplicationQueueStorage queueStorage = mgr.getQueueStorage();
+ replicators.addAll(queueStorage.listAllReplicators());
+ FileSystem fs = CommonFSUtils.getCurrentFileSystem(getConf());
+ Path infoDir = new Path(CommonFSUtils.getRootDir(getConf()), INFO_DIR);
+ for (ServerName sn : replicators) {
+ List<ReplicationQueueId> replicationQueues = queueStorage.listAllQueueIds(sn);
+ System.out.println(sn + " is dead, claim its replication queues: " + replicationQueues);
+ // record the rs name, so when master restarting, we will skip claiming its replication queue
+ fs.createNewFile(new Path(infoDir, sn.getServerName()));
+ for (ReplicationQueueId queueId : replicationQueues) {
+ mgr.claimQueue(queueId, true);
+ }
+ }
+ }
+
+ private void writeInfoFile(FileSystem fs) throws IOException {
+ // Record the info of this run. Currently only record the time we run the job. We will use this
+ // timestamp to clean up the data for last sequence ids and hfile refs in replication queue
+ // storage. See ReplicationQueueStorage.removeLastSequenceIdsAndHFileRefsBefore.
+ ReplicationSyncUpToolInfo info =
+ new ReplicationSyncUpToolInfo(EnvironmentEdgeManager.currentTime());
+ String json = JsonMapper.writeObjectAsString(info);
+ Path infoDir = new Path(CommonFSUtils.getRootDir(getConf()), INFO_DIR);
+ try (FSDataOutputStream out = fs.create(new Path(infoDir, INFO_FILE), false)) {
+ out.write(Bytes.toBytes(json));
+ }
}
@Override
public int run(String[] args) throws Exception {
Abortable abortable = new Abortable() {
+
+ private volatile boolean abort = false;
+
@Override
public void abort(String why, Throwable e) {
+ if (isAborted()) {
+ return;
+ }
+ abort = true;
+ System.err.println("Aborting because of " + why);
+ e.printStackTrace();
+ System.exit(1);
}
@Override
public boolean isAborted() {
- return false;
+ return abort;
}
};
Configuration conf = getConf();
@@ -114,16 +225,24 @@ public class ReplicationSyncUp extends Configured implements Tool {
Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
Path logDir = new Path(walRootDir, HConstants.HREGION_LOGDIR_NAME);
- System.out.println("Start Replication Server start");
+ System.out.println("Start Replication Server");
+ writeInfoFile(fs);
Replication replication = new Replication();
- replication.initialize(new DummyServer(zkw), fs, logDir, oldLogDir,
+ // use offline table replication queue storage
+ getConf().setClass(ReplicationStorageFactory.REPLICATION_QUEUE_IMPL,
+ OfflineTableReplicationQueueStorage.class, ReplicationQueueStorage.class);
+ DummyServer server = new DummyServer(getConf(), zkw);
+ replication.initialize(server, fs, new Path(logDir, server.toString()), oldLogDir,
new WALFactory(conf,
ServerName
.valueOf(getClass().getSimpleName() + ",16010," + EnvironmentEdgeManager.currentTime()),
null, false));
ReplicationSourceManager manager = replication.getReplicationManager();
manager.init();
- claimReplicationQueues(zkw, manager);
+ Set<ServerName> regionServers = listRegionServers(fs, logDir);
+ addMissingReplicationQueues(manager.getQueueStorage(), regionServers,
+ manager.getReplicationPeers().getAllPeerIds());
+ claimReplicationQueues(manager, regionServers);
while (manager.activeFailoverTaskCount() > 0) {
Thread.sleep(SLEEP_TIME);
}
@@ -138,23 +257,22 @@ public class ReplicationSyncUp extends Configured implements Tool {
return 0;
}
- class DummyServer implements Server {
- String hostname;
- ZKWatcher zkw;
+ private static final class DummyServer implements Server {
+ private final Configuration conf;
+ private final String hostname;
+ private final ZKWatcher zkw;
+ private volatile boolean abort = false;
- DummyServer(ZKWatcher zkw) {
+ DummyServer(Configuration conf, ZKWatcher zkw) {
// a unique name in case the first run fails
hostname = EnvironmentEdgeManager.currentTime() + ".SyncUpTool.replication.org";
+ this.conf = conf;
this.zkw = zkw;
}
- DummyServer(String hostname) {
- this.hostname = hostname;
- }
-
@Override
public Configuration getConfiguration() {
- return getConf();
+ return conf;
}
@Override
@@ -174,11 +292,18 @@ public class ReplicationSyncUp extends Configured implements Tool {
@Override
public void abort(String why, Throwable e) {
+ if (isAborted()) {
+ return;
+ }
+ abort = true;
+ System.err.println("Aborting because of " + why);
+ e.printStackTrace();
+ System.exit(1);
}
@Override
public boolean isAborted() {
- return false;
+ return abort;
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
index dce58dbfae4..129636275e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
@@ -311,6 +311,10 @@ public abstract class AbstractFSWALProvider<T extends AbstractFSWAL<?>> implemen
return matcher.matches() ? Long.parseLong(matcher.group(2)) : NO_TIMESTAMP;
}
+ public static final Comparator<Path> TIMESTAMP_COMPARATOR =
+ Comparator.<Path, Long> comparing(p -> AbstractFSWALProvider.getTimestamp(p.getName()))
+ .thenComparing(Path::getName);
+
/**
* Construct the directory name for all WALs on a given server. Dir names currently look like this
* for WALs: <code>hbase//WALs/kalashnikov.att.net,61634,1486865297088</code>.
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
index d7ba6c227c6..5d474bc2164 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
@@ -127,8 +127,8 @@ public class TestLogsCleaner {
TableDescriptor td = ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName);
TEST_UTIL.getAdmin().createTable(td);
TEST_UTIL.waitTableAvailable(tableName);
- queueStorage =
- ReplicationStorageFactory.getReplicationQueueStorage(TEST_UTIL.getConnection(), tableName);
+ queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(TEST_UTIL.getConnection(),
+ conf, tableName);
masterServices = mock(MasterServices.class);
when(masterServices.getConnection()).thenReturn(TEST_UTIL.getConnection());
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
index 7a89af15902..38225613b9d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
@@ -22,24 +22,28 @@ import static org.apache.hadoop.hbase.replication.TestReplicationBase.NB_RETRIES
import static org.apache.hadoop.hbase.replication.TestReplicationBase.NB_ROWS_IN_BATCH;
import static org.apache.hadoop.hbase.replication.TestReplicationBase.SLEEP_TIME;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSyncUp;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-// revisit later when we implement the new ReplicationSyncUpTool
-@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpTool extends TestReplicationSyncUpToolBase {
@@ -55,39 +59,70 @@ public class TestReplicationSyncUpTool extends TestReplicationSyncUpToolBase {
*/
@Test
public void testSyncUpTool() throws Exception {
-
- /**
- * Set up Replication: on Master and one Slave Table: t1_syncup and t2_syncup columnfamily:
- * 'cf1' : replicated 'norep': not replicated
- */
+ // Set up Replication: on Master and one Slave
+ // Table: t1_syncup and t2_syncup
+ // columnfamily:
+ // 'cf1' : replicated
+ // 'norep': not replicated
setupReplication();
- /**
- * at Master: t1_syncup: put 100 rows into cf1, and 1 rows into norep t2_syncup: put 200 rows
- * into cf1, and 1 rows into norep verify correctly replicated to slave
- */
+ //
+ // at Master:
+ // t1_syncup: put 100 rows into cf1, and 1 rows into norep
+ // t2_syncup: put 200 rows into cf1, and 1 rows into norep
+ //
+ // verify correctly replicated to slave
putAndReplicateRows();
- /**
- * Verify delete works step 1: stop hbase on Slave step 2: at Master: t1_syncup: delete 50 rows
- * from cf1 t2_syncup: delete 100 rows from cf1 no change on 'norep' step 3: stop hbase on
- * master, restart hbase on Slave step 4: verify Slave still have the rows before delete
- * t1_syncup: 100 rows from cf1 t2_syncup: 200 rows from cf1 step 5: run syncup tool on Master
- * step 6: verify that delete show up on Slave t1_syncup: 50 rows from cf1 t2_syncup: 100 rows
- * from cf1 verify correctly replicated to Slave
- */
+ // Verify delete works
+ //
+ // step 1: stop hbase on Slave
+ //
+ // step 2: at Master:
+ // t1_syncup: delete 50 rows from cf1
+ // t2_syncup: delete 100 rows from cf1
+ // no change on 'norep'
+ //
+ // step 3: stop hbase on master, restart hbase on Slave
+ //
+ // step 4: verify Slave still have the rows before delete
+ // t1_syncup: 100 rows from cf1
+ // t2_syncup: 200 rows from cf1
+ //
+ // step 5: run syncup tool on Master
+ //
+ // step 6: verify that delete show up on Slave
+ // t1_syncup: 50 rows from cf1
+ // t2_syncup: 100 rows from cf1
+ //
+ // verify correctly replicated to Slave
mimicSyncUpAfterDelete();
- /**
- * Verify put works step 1: stop hbase on Slave step 2: at Master: t1_syncup: put 100 rows from
- * cf1 t2_syncup: put 200 rows from cf1 and put another row on 'norep' ATTN: put to 'cf1' will
- * overwrite existing rows, so end count will be 100 and 200 respectively put to 'norep' will
- * add a new row. step 3: stop hbase on master, restart hbase on Slave step 4: verify Slave
- * still has the rows before put t1_syncup: 50 rows from cf1 t2_syncup: 100 rows from cf1 step
- * 5: run syncup tool on Master step 6: verify that put show up on Slave and 'norep' does not
- * t1_syncup: 100 rows from cf1 t2_syncup: 200 rows from cf1 verify correctly replicated to
- * Slave
- */
+ // Verify put works
+ //
+ // step 1: stop hbase on Slave
+ //
+ // step 2: at Master:
+ // t1_syncup: put 100 rows from cf1
+ // t2_syncup: put 200 rows from cf1
+ // and put another row on 'norep'
+ // ATTN:
+ // put to 'cf1' will overwrite existing rows, so end count will be 100 and 200 respectively
+ // put to 'norep' will add a new row.
+ //
+ // step 3: stop hbase on master, restart hbase on Slave
+ //
+ // step 4: verify Slave still has the rows before put
+ // t1_syncup: 50 rows from cf1
+ // t2_syncup: 100 rows from cf1
+ //
+ // step 5: run syncup tool on Master
+ //
+ // step 6: verify that put show up on Slave and 'norep' does not
+ // t1_syncup: 100 rows from cf1
+ // t2_syncup: 200 rows from cf1
+ //
+ // verify correctly replicated to Slave
mimicSyncUpAfterPut();
}
@@ -172,7 +207,8 @@ public class TestReplicationSyncUpTool extends TestReplicationSyncUpToolBase {
int rowCount_ht2Source = countRows(ht2Source);
assertEquals("t2_syncup has 101 rows on source, after remove 100 of the replicated colfam", 101,
rowCount_ht2Source);
-
+ List<ServerName> sourceRses = UTIL1.getHBaseCluster().getRegionServerThreads().stream()
+ .map(rst -> rst.getRegionServer().getServerName()).collect(Collectors.toList());
shutDownSourceHBaseCluster();
restartTargetHBaseCluster(1);
@@ -184,40 +220,33 @@ public class TestReplicationSyncUpTool extends TestReplicationSyncUpToolBase {
assertEquals("@Peer1 t1_syncup should still have 100 rows", 100, rowCountHt1TargetAtPeer1);
assertEquals("@Peer1 t2_syncup should still have 200 rows", 200, rowCountHt2TargetAtPeer1);
+ syncUp(UTIL1);
+
// After sync up
- for (int i = 0; i < NB_RETRIES; i++) {
- syncUp(UTIL1);
- rowCountHt1TargetAtPeer1 = countRows(ht1TargetAtPeer1);
- rowCountHt2TargetAtPeer1 = countRows(ht2TargetAtPeer1);
- if (i == NB_RETRIES - 1) {
- if (rowCountHt1TargetAtPeer1 != 50 || rowCountHt2TargetAtPeer1 != 100) {
- // syncUP still failed. Let's look at the source in case anything wrong there
- restartSourceHBaseCluster(1);
- rowCount_ht1Source = countRows(ht1Source);
- LOG.debug("t1_syncup should have 51 rows at source, and it is " + rowCount_ht1Source);
- rowCount_ht2Source = countRows(ht2Source);
- LOG.debug("t2_syncup should have 101 rows at source, and it is " + rowCount_ht2Source);
- }
- assertEquals("@Peer1 t1_syncup should be sync up and have 50 rows", 50,
- rowCountHt1TargetAtPeer1);
- assertEquals("@Peer1 t2_syncup should be sync up and have 100 rows", 100,
- rowCountHt2TargetAtPeer1);
- }
- if (rowCountHt1TargetAtPeer1 == 50 && rowCountHt2TargetAtPeer1 == 100) {
- LOG.info("SyncUpAfterDelete succeeded at retry = " + i);
- break;
- } else {
- LOG.debug("SyncUpAfterDelete failed at retry = " + i + ", with rowCount_ht1TargetPeer1 ="
- + rowCountHt1TargetAtPeer1 + " and rowCount_ht2TargetAtPeer1 ="
- + rowCountHt2TargetAtPeer1);
- }
- Thread.sleep(SLEEP_TIME);
+ rowCountHt1TargetAtPeer1 = countRows(ht1TargetAtPeer1);
+ rowCountHt2TargetAtPeer1 = countRows(ht2TargetAtPeer1);
+ assertEquals("@Peer1 t1_syncup should be sync up and have 50 rows", 50,
+ rowCountHt1TargetAtPeer1);
+ assertEquals("@Peer1 t2_syncup should be sync up and have 100 rows", 100,
+ rowCountHt2TargetAtPeer1);
+
+ // check we have recorded the dead region servers and also have an info file
+ Path rootDir = CommonFSUtils.getRootDir(UTIL1.getConfiguration());
+ Path syncUpInfoDir = new Path(rootDir, ReplicationSyncUp.INFO_DIR);
+ FileSystem fs = UTIL1.getTestFileSystem();
+ for (ServerName sn : sourceRses) {
+ assertTrue(fs.exists(new Path(syncUpInfoDir, sn.getServerName())));
}
+ assertTrue(fs.exists(new Path(syncUpInfoDir, ReplicationSyncUp.INFO_FILE)));
+ assertEquals(sourceRses.size() + 1, fs.listStatus(syncUpInfoDir).length);
+
+ restartSourceHBaseCluster(1);
+ // should finally removed all the records after restart
+ UTIL1.waitFor(60000, () -> fs.listStatus(syncUpInfoDir).length == 0);
}
private void mimicSyncUpAfterPut() throws Exception {
LOG.debug("mimicSyncUpAfterPut");
- restartSourceHBaseCluster(1);
shutDownTargetHBaseCluster();
Put p;
@@ -261,34 +290,14 @@ public class TestReplicationSyncUpTool extends TestReplicationSyncUpToolBase {
assertEquals("@Peer1 t2_syncup should be NOT sync up and have 100 rows", 100,
rowCountHt2TargetAtPeer1);
- // after syun up
- for (int i = 0; i < NB_RETRIES; i++) {
- syncUp(UTIL1);
- rowCountHt1TargetAtPeer1 = countRows(ht1TargetAtPeer1);
- rowCountHt2TargetAtPeer1 = countRows(ht2TargetAtPeer1);
- if (i == NB_RETRIES - 1) {
- if (rowCountHt1TargetAtPeer1 != 100 || rowCountHt2TargetAtPeer1 != 200) {
- // syncUP still failed. Let's look at the source in case anything wrong there
- restartSourceHBaseCluster(1);
- rowCount_ht1Source = countRows(ht1Source);
- LOG.debug("t1_syncup should have 102 rows at source, and it is " + rowCount_ht1Source);
- rowCount_ht2Source = countRows(ht2Source);
- LOG.debug("t2_syncup should have 202 rows at source, and it is " + rowCount_ht2Source);
- }
- assertEquals("@Peer1 t1_syncup should be sync up and have 100 rows", 100,
- rowCountHt1TargetAtPeer1);
- assertEquals("@Peer1 t2_syncup should be sync up and have 200 rows", 200,
- rowCountHt2TargetAtPeer1);
- }
- if (rowCountHt1TargetAtPeer1 == 100 && rowCountHt2TargetAtPeer1 == 200) {
- LOG.info("SyncUpAfterPut succeeded at retry = " + i);
- break;
- } else {
- LOG.debug("SyncUpAfterPut failed at retry = " + i + ", with rowCount_ht1TargetPeer1 ="
- + rowCountHt1TargetAtPeer1 + " and rowCount_ht2TargetAtPeer1 ="
- + rowCountHt2TargetAtPeer1);
- }
- Thread.sleep(SLEEP_TIME);
- }
+ syncUp(UTIL1);
+
+ // after sync up
+ rowCountHt1TargetAtPeer1 = countRows(ht1TargetAtPeer1);
+ rowCountHt2TargetAtPeer1 = countRows(ht2TargetAtPeer1);
+ assertEquals("@Peer1 t1_syncup should be sync up and have 100 rows", 100,
+ rowCountHt1TargetAtPeer1);
+ assertEquals("@Peer1 t2_syncup should be sync up and have 200 rows", 200,
+ rowCountHt2TargetAtPeer1);
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolBase.java
index d3142106362..8a28db3b185 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolBase.java
@@ -136,7 +136,8 @@ public abstract class TestReplicationSyncUpToolBase {
}
final void syncUp(HBaseTestingUtil util) throws Exception {
- ToolRunner.run(util.getConfiguration(), new ReplicationSyncUp(), new String[0]);
+ ToolRunner.run(new Configuration(util.getConfiguration()), new ReplicationSyncUp(),
+ new String[0]);
}
// Utilities that manager shutdown / restart of source / sink clusters. They take care of
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java
index b5de8e6324f..afed0483388 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Table;
@@ -45,14 +46,11 @@ import org.apache.hadoop.hbase.tool.BulkLoadHFiles;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.HFileTestUtil;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-//revisit later when we implement the new ReplicationSyncUpTool
-@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpToolWithBulkLoadedData extends TestReplicationSyncUpToolBase {
@@ -74,40 +72,50 @@ public class TestReplicationSyncUpToolWithBulkLoadedData extends TestReplication
@Test
public void testSyncUpTool() throws Exception {
- /**
- * Set up Replication: on Master and one Slave Table: t1_syncup and t2_syncup columnfamily:
- * 'cf1' : replicated 'norep': not replicated
- */
+ // Set up Replication:
+ // on Master and one Slave Table: t1_syncup and t2_syncup
+ // columnfamily:
+ // 'cf1' : replicated
+ // 'norep': not replicated
setupReplication();
- /**
- * Prepare 24 random hfile ranges required for creating hfiles
- */
+ // Prepare 24 random hfile ranges required for creating hfiles
Iterator<String> randomHFileRangeListIterator = null;
Set<String> randomHFileRanges = new HashSet<>(24);
for (int i = 0; i < 24; i++) {
- randomHFileRanges.add(UTIL1.getRandomUUID().toString());
+ randomHFileRanges.add(HBaseTestingUtil.getRandomUUID().toString());
}
List<String> randomHFileRangeList = new ArrayList<>(randomHFileRanges);
Collections.sort(randomHFileRangeList);
randomHFileRangeListIterator = randomHFileRangeList.iterator();
- /**
- * at Master: t1_syncup: Load 50 rows into cf1, and 50 rows from other hdfs into cf1, and 3 rows
- * into norep t2_syncup: Load 100 rows into cf1, and 100 rows from other hdfs into cf1, and 3
- * rows into norep verify correctly replicated to slave
- */
+ // at Master:
+ // t1_syncup: Load 50 rows into cf1, and 50 rows from other hdfs into cf1, and 3 rows into norep
+ // t2_syncup: Load 100 rows into cf1, and 100 rows from other hdfs into cf1, and 3 rows into
+ // norep
+ // verify correctly replicated to slave
loadAndReplicateHFiles(true, randomHFileRangeListIterator);
- /**
- * Verify hfile load works step 1: stop hbase on Slave step 2: at Master: t1_syncup: Load
- * another 100 rows into cf1 and 3 rows into norep t2_syncup: Load another 200 rows into cf1 and
- * 3 rows into norep step 3: stop hbase on master, restart hbase on Slave step 4: verify Slave
- * still has the rows before load t1_syncup: 100 rows from cf1 t2_syncup: 200 rows from cf1 step
- * 5: run syncup tool on Master step 6: verify that hfiles show up on Slave and 'norep' does not
- * t1_syncup: 200 rows from cf1 t2_syncup: 400 rows from cf1 verify correctly replicated to
- * Slave
- */
+ // Verify hfile load works
+ //
+ // step 1: stop hbase on Slave
+ //
+ // step 2: at Master:
+ // t1_syncup: Load another 100 rows into cf1 and 3 rows into norep
+ // t2_syncup: Load another 200 rows into cf1 and 3 rows into norep
+ //
+ // step 3: stop hbase on master, restart hbase on Slave
+ //
+ // step 4: verify Slave still has the rows before load
+ // t1_syncup: 100 rows from cf1
+ // t2_syncup: 200 rows from cf1
+ //
+ // step 5: run syncup tool on Master
+ //
+ // step 6: verify that hfiles show up on Slave and 'norep' does not
+ // t1_syncup: 200 rows from cf1
+ // t2_syncup: 400 rows from cf1
+ // verify correctly replicated to Slave
mimicSyncUpAfterBulkLoad(randomHFileRangeListIterator);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestTableReplicationQueueStorage.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestTableReplicationQueueStorage.java
index 4148c1c1a2c..9041831d0e8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestTableReplicationQueueStorage.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestTableReplicationQueueStorage.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.MD5Hash;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.zookeeper.KeeperException;
@@ -420,4 +421,54 @@ public class TestTableReplicationQueueStorage {
assertEquals(0, storage.getAllPeersFromHFileRefsQueue().size());
assertTrue(storage.getReplicableHFiles(peerId2).isEmpty());
}
+
+ private void addLastSequenceIdsAndHFileRefs(String peerId1, String peerId2)
+ throws ReplicationException {
+ for (int i = 0; i < 100; i++) {
+ String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
+ storage.setLastSequenceIds(peerId1, ImmutableMap.of(encodedRegionName, (long) i));
+ }
+
+ List<Pair<Path, Path>> files1 = new ArrayList<>(3);
+ files1.add(new Pair<>(null, new Path("file_1")));
+ files1.add(new Pair<>(null, new Path("file_2")));
+ files1.add(new Pair<>(null, new Path("file_3")));
+ storage.addHFileRefs(peerId2, files1);
+ }
+
+ @Test
+ public void testRemoveLastSequenceIdsAndHFileRefsBefore()
+ throws ReplicationException, InterruptedException {
+ String peerId1 = "1";
+ String peerId2 = "2";
+ addLastSequenceIdsAndHFileRefs(peerId1, peerId2);
+ // make sure we have write these out
+ for (int i = 0; i < 100; i++) {
+ String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
+ assertEquals(i, storage.getLastSequenceId(encodedRegionName, peerId1));
+ }
+ assertEquals(1, storage.getAllPeersFromHFileRefsQueue().size());
+ assertEquals(3, storage.getReplicableHFiles(peerId2).size());
+
+ // should have nothing after removal
+ long ts = EnvironmentEdgeManager.currentTime();
+ storage.removeLastSequenceIdsAndHFileRefsBefore(ts);
+ for (int i = 0; i < 100; i++) {
+ String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
+ assertEquals(HConstants.NO_SEQNUM, storage.getLastSequenceId(encodedRegionName, peerId1));
+ }
+ assertEquals(0, storage.getAllPeersFromHFileRefsQueue().size());
+
+ Thread.sleep(100);
+ // add again and remove with the old timestamp
+ addLastSequenceIdsAndHFileRefs(peerId1, peerId2);
+ storage.removeLastSequenceIdsAndHFileRefsBefore(ts);
+ // make sure we do not delete the data which are written after the give timestamp
+ for (int i = 0; i < 100; i++) {
+ String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
+ assertEquals(i, storage.getLastSequenceId(encodedRegionName, peerId1));
+ }
+ assertEquals(1, storage.getAllPeersFromHFileRefsQueue().size());
+ assertEquals(3, storage.getReplicableHFiles(peerId2).size());
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java
index 28779be4399..83cd41773ca 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java
@@ -25,11 +25,8 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.wal.RegionGroupingProvider;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.experimental.categories.Category;
-//revisit later when we implement the new ReplicationSyncUpTool
-@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpToolWithMultipleAsyncWAL extends TestReplicationSyncUpTool {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java
index f495f433bc9..673b841430e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java
@@ -25,11 +25,8 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.wal.RegionGroupingProvider;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.experimental.categories.Category;
-//revisit later when we implement the new ReplicationSyncUpTool
-@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpToolWithMultipleWAL extends TestReplicationSyncUpTool {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index b7564ed9168..1bb9a3e2949 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -45,11 +45,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter;
import org.apache.hadoop.hbase.replication.DummyReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
@@ -125,8 +122,6 @@ public class TestReplicationSourceManager {
private static final TableName TABLE_NAME = TableName.valueOf("test");
- private static TableDescriptor TD;
-
private static RegionInfo RI;
private static NavigableMap<byte[], Integer> SCOPES;
@@ -152,10 +147,6 @@ public class TestReplicationSourceManager {
FS = UTIL.getTestFileSystem();
CONF = new Configuration(UTIL.getConfiguration());
CONF.setLong("replication.sleep.before.failover", 0);
- TD = TableDescriptorBuilder.newBuilder(TABLE_NAME)
- .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(F1)
- .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
- .setColumnFamily(ColumnFamilyDescriptorBuilder.of(F2)).build();
RI = RegionInfoBuilder.newBuilder(TABLE_NAME).build();
SCOPES = new TreeMap<>(Bytes.BYTES_COMPARATOR);
@@ -176,7 +167,8 @@ public class TestReplicationSourceManager {
when(server.getConfiguration()).thenReturn(CONF);
when(server.getZooKeeper()).thenReturn(UTIL.getZooKeeperWatcher());
when(server.getConnection()).thenReturn(UTIL.getConnection());
- when(server.getServerName()).thenReturn(ServerName.valueOf("hostname.example.org", 1234, 1));
+ ServerName sn = ServerName.valueOf("hostname.example.org", 1234, 1);
+ when(server.getServerName()).thenReturn(sn);
oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
FS.mkdirs(oldLogDir);
logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
@@ -189,7 +181,7 @@ public class TestReplicationSourceManager {
CONF.set(ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME, tableName.getNameAsString());
replication = new Replication();
- replication.initialize(server, FS, logDir, oldLogDir,
+ replication.initialize(server, FS, new Path(logDir, sn.toString()), oldLogDir,
new WALFactory(CONF, server.getServerName(), null, false));
manager = replication.getReplicationManager();
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java
index 1544265435c..8731adbe4c2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java
@@ -99,8 +99,8 @@ public class TestSerialReplicationChecker {
TableName repTable = TableName.valueOf("test_serial_rep");
UTIL.getAdmin()
.createTable(ReplicationStorageFactory.createReplicationQueueTableDescriptor(repTable));
- QUEUE_STORAGE =
- ReplicationStorageFactory.getReplicationQueueStorage(UTIL.getConnection(), repTable);
+ QUEUE_STORAGE = ReplicationStorageFactory.getReplicationQueueStorage(UTIL.getConnection(),
+ UTIL.getConfiguration(), repTable);
}
@AfterClass
[hbase] 08/11: HBASE-27217 Revisit the DumpReplicationQueues tool (#4810)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 96f15b3d2d3fddd0f257c627473c293a95e6505a
Author: LiangJun He <20...@163.com>
AuthorDate: Sun Nov 13 22:03:36 2022 +0800
HBASE-27217 Revisit the DumpReplicationQueues tool (#4810)
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../regionserver/DumpReplicationQueues.java | 240 +++++++++++++--------
.../hadoop/hbase/wal/AbstractFSWALProvider.java | 20 ++
.../regionserver/TestDumpReplicationQueues.java | 159 +++++++++-----
3 files changed, 284 insertions(+), 135 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
index 98d0a55fbc4..b284e3f6837 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -19,8 +19,12 @@ package org.apache.hadoop.hbase.replication.regionserver;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -31,7 +35,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Admin;
@@ -40,28 +44,33 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.replication.TableCFs;
import org.apache.hadoop.hbase.io.WALLink;
import org.apache.hadoop.hbase.procedure2.util.StringUtils;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationOffsetUtil;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.zookeeper.ZKDump;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.AtomicLongMap;
/**
- * TODO: reimplement this tool
* <p/>
* Provides information about the existing states of replication, replication peers and queues.
* Usage: hbase org.apache.hadoop.hbase.replication.regionserver.DumpReplicationQueues [args]
* Arguments: --distributed Polls each RS to dump information about the queue --hdfs Reports HDFS
- * usage by the replication queues (note: can be overestimated).
+ * usage by the replication queues (note: can be overestimated). In the new version, we
+ * reimplemented the DumpReplicationQueues tool to support obtaining information from replication
+ * table.
*/
@InterfaceAudience.Private
public class DumpReplicationQueues extends Configured implements Tool {
@@ -185,7 +194,7 @@ public class DumpReplicationQueues extends Configured implements Tool {
System.err.println("General Options:");
System.err.println(" -h|--h|--help Show this help and exit.");
System.err.println(" --distributed Poll each RS and print its own replication queue. "
- + "Default only polls ZooKeeper");
+ + "Default only polls replication table.");
System.err.println(" --hdfs Use HDFS to calculate usage of WALs by replication."
+ " It could be overestimated if replicating to multiple peers."
+ " --distributed flag is also needed.");
@@ -201,13 +210,7 @@ public class DumpReplicationQueues extends Configured implements Tool {
Connection connection = ConnectionFactory.createConnection(conf);
Admin admin = connection.getAdmin();
- ZKWatcher zkw =
- new ZKWatcher(conf, "DumpReplicationQueues" + EnvironmentEdgeManager.currentTime(),
- new WarnOnlyAbortable(), true);
-
try {
- // Our zk watcher
- LOG.info("Our Quorum: " + zkw.getQuorum());
List<TableCFs> replicatedTableCFs = admin.listReplicatedTableCFs();
if (replicatedTableCFs.isEmpty()) {
LOG.info("No tables with a configured replication peer were found.");
@@ -229,21 +232,72 @@ public class DumpReplicationQueues extends Configured implements Tool {
LOG.info("Found [--distributed], will poll each RegionServer.");
Set<String> peerIds =
peers.stream().map((peer) -> peer.getPeerId()).collect(Collectors.toSet());
- System.out.println(dumpQueues(zkw, peerIds, opts.isHdfs()));
+ System.out.println(dumpQueues(connection, peerIds, opts.isHdfs(), conf));
System.out.println(dumpReplicationSummary());
} else {
- // use ZK instead
- System.out.print("Dumping replication znodes via ZooKeeper:");
- System.out.println(ZKDump.getReplicationZnodesDump(zkw));
+ // use replication table instead
+ System.out.println("Dumping replication info via replication table.");
+ System.out.println(dumpReplicationViaTable(connection, conf));
}
return (0);
} catch (IOException e) {
return (-1);
} finally {
- zkw.close();
+ connection.close();
}
}
+ public String dumpReplicationViaTable(Connection connection, Configuration conf)
+ throws ReplicationException, IOException {
+ StringBuilder sb = new StringBuilder();
+ ReplicationQueueStorage queueStorage =
+ ReplicationStorageFactory.getReplicationQueueStorage(connection, conf);
+
+ // The dump info format is as follows:
+ // peers:
+ // peers/1: zk1:2181:/hbase
+ // peers/1/peer-state: ENABLED
+ // rs:
+ // rs/rs1,16020,1664092120094/1/rs1%2C16020%2C1664092120094.1664096778778: 123
+ // rs/rs2,16020,1664092120094/2/rs1%2C16020%2C1664092120094.1664096778778: 321
+ // hfile-refs:
+ // hfile-refs/1/hfile1,hfile2
+ // hfile-refs/2/hfile3,hfile4
+ String peersKey = "peers";
+ sb.append(peersKey).append(": ").append("\n");
+ List<ReplicationPeerDescription> repPeerDescs = connection.getAdmin().listReplicationPeers();
+ for (ReplicationPeerDescription repPeerDesc : repPeerDescs) {
+ sb.append(peersKey).append("/").append(repPeerDesc.getPeerId()).append(": ")
+ .append(repPeerDesc.getPeerConfig().getClusterKey()).append("\n");
+ sb.append(peersKey).append("/").append(repPeerDesc.getPeerId()).append("/peer-state: ")
+ .append(repPeerDesc.isEnabled() ? "ENABLED" : "DISABLED").append("\n");
+ }
+
+ List<ReplicationQueueData> repQueueDataList = queueStorage.listAllQueues();
+ String rsKey = "rs";
+ sb.append(rsKey).append(": ").append("\n");
+ for (ReplicationQueueData repQueueData : repQueueDataList) {
+ String peerId = repQueueData.getId().getPeerId();
+ for (ImmutableMap.Entry<String, ReplicationGroupOffset> entry : repQueueData.getOffsets()
+ .entrySet()) {
+ sb.append(rsKey).append("/").append(entry.getKey()).append("/").append(peerId).append("/")
+ .append(entry.getValue().getWal()).append(": ").append(entry.getValue().getOffset())
+ .append("\n");
+ }
+ }
+
+ List<String> peerIds = queueStorage.getAllPeersFromHFileRefsQueue();
+ String hfileKey = "hfile-refs";
+ sb.append(hfileKey).append(": ").append("\n");
+ for (String peerId : peerIds) {
+ List<String> hfiles = queueStorage.getReplicableHFiles(peerId);
+ sb.append(hfileKey).append("/").append(peerId).append("/").append(String.join(",", hfiles))
+ .append("\n");
+ }
+
+ return sb.toString();
+ }
+
public String dumpReplicationSummary() {
StringBuilder sb = new StringBuilder();
if (!deletedQueues.isEmpty()) {
@@ -294,71 +348,103 @@ public class DumpReplicationQueues extends Configured implements Tool {
return sb.toString();
}
- public String dumpQueues(ZKWatcher zkw, Set<String> peerIds, boolean hdfs) throws Exception {
- ReplicationQueueStorage queueStorage;
+ public String dumpQueues(Connection connection, Set<String> peerIds, boolean hdfs,
+ Configuration conf) throws Exception {
StringBuilder sb = new StringBuilder();
+ ReplicationQueueStorage queueStorage =
+ ReplicationStorageFactory.getReplicationQueueStorage(connection, conf);
+
+ Set<ServerName> liveRegionServers =
+ connection.getAdmin().getClusterMetrics().getLiveServerMetrics().keySet();
- // queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
- // Set<ServerName> liveRegionServers = ZKUtil.listChildrenNoWatch(zkw,
- // zkw.getZNodePaths().rsZNode)
- // .stream().map(ServerName::parseServerName).collect(Collectors.toSet());
- //
- // Loops each peer on each RS and dumps the queues
- // List<ServerName> regionservers = queueStorage.getListOfReplicators();
- // if (regionservers == null || regionservers.isEmpty()) {
- // return sb.toString();
- // }
- // for (ServerName regionserver : regionservers) {
- // List<String> queueIds = queueStorage.getAllQueues(regionserver);
- // if (!liveRegionServers.contains(regionserver)) {
- // deadRegionServers.add(regionserver.getServerName());
- // }
- // for (String queueId : queueIds) {
- // ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- // List<String> wals = queueStorage.getWALsInQueue(regionserver, queueId);
- // Collections.sort(wals);
- // if (!peerIds.contains(queueInfo.getPeerId())) {
- // deletedQueues.add(regionserver + "/" + queueId);
- // sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, true, hdfs));
- // } else {
- // sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, false, hdfs));
- // }
- // }
- // }
+ List<ServerName> regionServers = queueStorage.listAllReplicators();
+ if (regionServers == null || regionServers.isEmpty()) {
+ return sb.toString();
+ }
+ for (ServerName regionServer : regionServers) {
+ List<ReplicationQueueId> queueIds = queueStorage.listAllQueueIds(regionServer);
+
+ if (!liveRegionServers.contains(regionServer)) {
+ deadRegionServers.add(regionServer.getServerName());
+ }
+ for (ReplicationQueueId queueId : queueIds) {
+ List<String> tmpWals = new ArrayList<>();
+ // wals
+ AbstractFSWALProvider
+ .getWALFiles(connection.getConfiguration(), queueId.getServerWALsBelongTo()).stream()
+ .map(Path::toString).forEach(tmpWals::add);
+
+ // old wals
+ AbstractFSWALProvider.getArchivedWALFiles(connection.getConfiguration(),
+ queueId.getServerWALsBelongTo(), URLEncoder
+ .encode(queueId.getServerWALsBelongTo().toString(), StandardCharsets.UTF_8.name()))
+ .stream().map(Path::toString).forEach(tmpWals::add);
+
+ Map<String, ReplicationGroupOffset> offsets = queueStorage.getOffsets(queueId);
+ // filter out the wal files that should replicate
+ List<String> wals = new ArrayList<>();
+ for (Map.Entry<String, ReplicationGroupOffset> entry : offsets.entrySet()) {
+ ReplicationGroupOffset offset = entry.getValue();
+ for (String wal : tmpWals) {
+ if (ReplicationOffsetUtil.shouldReplicate(offset, wal)) {
+ wals.add(wal);
+ }
+ }
+ }
+ Collections.sort(wals, Comparator.comparingLong(AbstractFSWALProvider::getTimestamp));
+ if (!peerIds.contains(queueId.getPeerId())) {
+ deletedQueues.add(regionServer + "/" + queueId);
+ sb.append(formatQueue(regionServer, offsets, wals, queueId, true, hdfs));
+ } else {
+ sb.append(formatQueue(regionServer, offsets, wals, queueId, false, hdfs));
+ }
+ }
+ }
return sb.toString();
}
- private String formatQueue(ServerName regionserver, ReplicationQueueStorage queueStorage,
- ReplicationQueueInfo queueInfo, String queueId, List<String> wals, boolean isDeleted,
- boolean hdfs) throws Exception {
+ private String formatQueue(ServerName regionServer, Map<String, ReplicationGroupOffset> offsets,
+ List<String> wals, ReplicationQueueId queueId, boolean isDeleted, boolean hdfs)
+ throws Exception {
StringBuilder sb = new StringBuilder();
- List<ServerName> deadServers;
-
- sb.append("Dumping replication queue info for RegionServer: [" + regionserver + "]" + "\n");
- sb.append(" Queue znode: " + queueId + "\n");
- sb.append(" PeerID: " + queueInfo.getPeerId() + "\n");
- sb.append(" Recovered: " + queueInfo.isQueueRecovered() + "\n");
- deadServers = queueInfo.getDeadRegionServers();
- if (deadServers.isEmpty()) {
- sb.append(" No dead RegionServers found in this queue." + "\n");
+ sb.append("Dumping replication queue info for RegionServer: [" + regionServer + "]" + "\n");
+ sb.append(" Queue id: " + queueId + "\n");
+ sb.append(" PeerID: " + queueId.getPeerId() + "\n");
+ sb.append(" Recovered: " + queueId.isRecovered() + "\n");
+ // In new version, we only record the first dead RegionServer in queueId.
+ if (queueId.getSourceServerName().isPresent()) {
+ sb.append(" Dead RegionServer: " + queueId.getSourceServerName().get() + "\n");
} else {
- sb.append(" Dead RegionServers: " + deadServers + "\n");
+ sb.append(" No dead RegionServer found in this queue." + "\n");
}
sb.append(" Was deleted: " + isDeleted + "\n");
sb.append(" Number of WALs in replication queue: " + wals.size() + "\n");
- peersQueueSize.addAndGet(queueInfo.getPeerId(), wals.size());
-
- for (String wal : wals) {
- // long position = queueStorage.getWALPosition(regionserver, queueInfo.getPeerId(), wal);
- // sb.append(" Replication position for " + wal + ": "
- // + (position > 0 ? position : "0" + " (not started or nothing to replicate)") + "\n");
+ peersQueueSize.addAndGet(queueId.getPeerId(), wals.size());
+
+ for (Map.Entry<String, ReplicationGroupOffset> entry : offsets.entrySet()) {
+ String walGroup = entry.getKey();
+ ReplicationGroupOffset offset = entry.getValue();
+ for (String wal : wals) {
+ long position = 0;
+ if (offset.getWal().equals(wal)) {
+ position = offset.getOffset();
+ }
+ sb.append(
+ " Replication position for " + (walGroup != null ? walGroup + "/" + wal : wal) + ": ");
+ if (position == 0) {
+ sb.append("0 (not started or nothing to replicate)");
+ } else if (position > 0) {
+ sb.append(position);
+ }
+ sb.append("\n");
+ }
}
if (hdfs) {
FileSystem fs = FileSystem.get(getConf());
sb.append(" Total size of WALs on HDFS for this queue: "
- + StringUtils.humanSize(getTotalWALSize(fs, wals, regionserver)) + "\n");
+ + StringUtils.humanSize(getTotalWALSize(fs, wals, regionServer)) + "\n");
}
return sb.toString();
}
@@ -366,8 +452,7 @@ public class DumpReplicationQueues extends Configured implements Tool {
/**
* return total size in bytes from a list of WALs
*/
- private long getTotalWALSize(FileSystem fs, List<String> wals, ServerName server)
- throws IOException {
+ private long getTotalWALSize(FileSystem fs, List<String> wals, ServerName server) {
long size = 0;
FileStatus fileStatus;
@@ -389,19 +474,4 @@ public class DumpReplicationQueues extends Configured implements Tool {
totalSizeOfWALs += size;
return size;
}
-
- private static class WarnOnlyAbortable implements Abortable {
- @Override
- public void abort(String why, Throwable e) {
- LOG.warn("DumpReplicationQueue received abort, ignoring. Reason: " + why);
- if (LOG.isDebugEnabled()) {
- LOG.debug(e.toString(), e);
- }
- }
-
- @Override
- public boolean isAborted() {
- return false;
- }
- }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
index 1b387f33ecc..dce58dbfae4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
@@ -379,6 +379,26 @@ public abstract class AbstractFSWALProvider<T extends AbstractFSWAL<?>> implemen
return archivedWalFiles;
}
+ /**
+ * List all the wal files for a logPrefix.
+ */
+ public static List<Path> getWALFiles(Configuration c, ServerName serverName) throws IOException {
+ Path walRoot = new Path(CommonFSUtils.getWALRootDir(c), HConstants.HREGION_LOGDIR_NAME);
+ FileSystem fs = walRoot.getFileSystem(c);
+ List<Path> walFiles = new ArrayList<>();
+ Path walDir = new Path(walRoot, serverName.toString());
+ try {
+ for (FileStatus status : fs.listStatus(walDir)) {
+ if (status.isFile()) {
+ walFiles.add(status.getPath());
+ }
+ }
+ } catch (FileNotFoundException e) {
+ LOG.info("WAL dir {} not exists", walDir);
+ }
+ return walFiles;
+ }
+
/**
* Pulls a ServerName out of a Path generated according to our layout rules. In the below layouts,
* this method ignores the format of the logfile component. Current format: [base directory for
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java
index 3475ae5c192..3e1dc624fe7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java
@@ -17,34 +17,43 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfigBuilder;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.After;
+import org.junit.Before;
import org.junit.ClassRule;
-import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
/**
* Tests for DumpReplicationQueues tool
*/
-// TODO: reimplement
-@Ignore
@Category({ ReplicationTests.class, SmallTests.class })
public class TestDumpReplicationQueues {
@@ -52,49 +61,99 @@ public class TestDumpReplicationQueues {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestDumpReplicationQueues.class);
- /**
- * Makes sure dumpQueues returns wals znodes ordered chronologically.
- * @throws Exception if dumpqueues finds any error while handling list of znodes.
- */
+ private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
+ private static Configuration CONF;
+ private static FileSystem FS = null;
+ private Path root;
+ private Path logDir;
+ @Rule
+ public final TestName name = new TestName();
+
+ @Before
+ public void setup() throws Exception {
+ UTIL.startMiniCluster(3);
+ CONF = UTIL.getConfiguration();
+ TableName tableName = TableName.valueOf("replication_" + name.getMethodName());
+ UTIL.getAdmin()
+ .createTable(ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName));
+ CONF.set(ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME, tableName.getNameAsString());
+ FS = FileSystem.get(CONF);
+ root = UTIL.getDataTestDirOnTestFS("hbase");
+ logDir = new Path(root, HConstants.HREGION_LOGDIR_NAME);
+ FS.mkdirs(logDir);
+ CommonFSUtils.setRootDir(CONF, root);
+ CommonFSUtils.setWALRootDir(CONF, root);
+ }
+
@Test
- public void testDumpReplicationReturnsWalSorted() throws Exception {
- Configuration config = HBaseConfiguration.create();
- ZKWatcher zkWatcherMock = mock(ZKWatcher.class);
- ZNodePaths zNodePath = new ZNodePaths(config);
- RecoverableZooKeeper recoverableZooKeeperMock = mock(RecoverableZooKeeper.class);
- when(zkWatcherMock.getRecoverableZooKeeper()).thenReturn(recoverableZooKeeperMock);
- when(zkWatcherMock.getZNodePaths()).thenReturn(zNodePath);
- List<String> nodes = new ArrayList<>();
- String server = "rs1,60030," + EnvironmentEdgeManager.currentTime();
- nodes.add(server);
- when(recoverableZooKeeperMock.getChildren("/hbase/rs", null)).thenReturn(nodes);
- when(recoverableZooKeeperMock.getChildren("/hbase/replication/rs", null)).thenReturn(nodes);
- List<String> queuesIds = new ArrayList<>();
- queuesIds.add("1");
- when(recoverableZooKeeperMock.getChildren("/hbase/replication/rs/" + server, null))
- .thenReturn(queuesIds);
- List<String> wals = new ArrayList<>();
- wals.add("rs1%2C60964%2C1549394085556.1549394101427");
- wals.add("rs1%2C60964%2C1549394085556.1549394101426");
- wals.add("rs1%2C60964%2C1549394085556.1549394101428");
- when(recoverableZooKeeperMock.getChildren("/hbase/replication/rs/" + server + "/1", null))
- .thenReturn(wals);
+ public void testDumpReplication() throws Exception {
+ String peerId = "1";
+ String serverNameStr = "rs1,12345,123";
+ addPeer(peerId, "hbase");
+ ServerName serverName = ServerName.valueOf(serverNameStr);
+ String walName = "rs1%2C12345%2C123.10";
+ Path walPath = new Path(logDir, serverNameStr + "/" + walName);
+ FS.createNewFile(walPath);
+
+ ReplicationQueueId queueId = new ReplicationQueueId(serverName, peerId);
+ ReplicationQueueStorage queueStorage =
+ ReplicationStorageFactory.getReplicationQueueStorage(UTIL.getConnection(), CONF);
+ queueStorage.setOffset(queueId, "wal-group",
+ new ReplicationGroupOffset(FS.listStatus(walPath)[0].getPath().toString(), 123),
+ Collections.emptyMap());
+
DumpReplicationQueues dumpQueues = new DumpReplicationQueues();
Set<String> peerIds = new HashSet<>();
- peerIds.add("1");
- dumpQueues.setConf(config);
- String dump = dumpQueues.dumpQueues(zkWatcherMock, peerIds, false);
+ peerIds.add(peerId);
+ List<String> wals = new ArrayList<>();
+ wals.add("rs1%2C12345%2C123.12");
+ wals.add("rs1%2C12345%2C123.15");
+ wals.add("rs1%2C12345%2C123.11");
+ for (String wal : wals) {
+ Path wPath = new Path(logDir, serverNameStr + "/" + wal);
+ FS.createNewFile(wPath);
+ }
+
+ String dump = dumpQueues.dumpQueues(UTIL.getConnection(), peerIds, false, CONF);
+ assertTrue(dump.indexOf("Queue id: 1-rs1,12345,123") > 0);
+ assertTrue(dump.indexOf("Number of WALs in replication queue: 4") > 0);
+ // test for 'Returns wal sorted'
String[] parsedDump = dump.split("Replication position for");
- assertEquals("Parsed dump should have 4 parts.", 4, parsedDump.length);
- assertTrue(
- "First wal should be rs1%2C60964%2C1549394085556.1549394101426, but got: " + parsedDump[1],
- parsedDump[1].indexOf("rs1%2C60964%2C1549394085556.1549394101426") >= 0);
- assertTrue(
- "Second wal should be rs1%2C60964%2C1549394085556.1549394101427, but got: " + parsedDump[2],
- parsedDump[2].indexOf("rs1%2C60964%2C1549394085556.1549394101427") >= 0);
- assertTrue(
- "Third wal should be rs1%2C60964%2C1549394085556.1549394101428, but got: " + parsedDump[3],
- parsedDump[3].indexOf("rs1%2C60964%2C1549394085556.1549394101428") >= 0);
+ assertTrue("First wal should be rs1%2C12345%2C123.10: 123, but got: " + parsedDump[1],
+ parsedDump[1].indexOf("rs1%2C12345%2C123.10: 123") >= 0);
+ assertTrue("Second wal should be rs1%2C12345%2C123.11: 0, but got: " + parsedDump[2],
+ parsedDump[2].indexOf("rs1%2C12345%2C123.11: 0 (not started or nothing to replicate)") >= 0);
+ assertTrue("Third wal should be rs1%2C12345%2C123.12: 0, but got: " + parsedDump[3],
+ parsedDump[3].indexOf("rs1%2C12345%2C123.12: 0 (not started or nothing to replicate)") >= 0);
+ assertTrue("Fourth wal should be rs1%2C12345%2C123.15: 0, but got: " + parsedDump[4],
+ parsedDump[4].indexOf("rs1%2C12345%2C123.15: 0 (not started or nothing to replicate)") >= 0);
+
+ Path file1 = new Path("testHFile1");
+ Path file2 = new Path("testHFile2");
+ List<Pair<Path, Path>> files = new ArrayList<>(1);
+ files.add(new Pair<>(null, file1));
+ files.add(new Pair<>(null, file2));
+ queueStorage.addHFileRefs(peerId, files);
+ // test for 'Dump Replication via replication table'
+ String dump2 = dumpQueues.dumpReplicationViaTable(UTIL.getConnection(), CONF);
+ assertTrue(dump2.indexOf("peers/1/peer-state: ENABLED") > 0);
+ assertTrue(dump2.indexOf("rs1,12345,123/rs1%2C12345%2C123.10: 123") >= 0);
+ assertTrue(dump2.indexOf("hfile-refs/1/testHFile1,testHFile2") >= 0);
+ }
+
+ /**
+ * Add a peer
+ */
+ private void addPeer(String peerId, String clusterKey) throws IOException {
+ ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder()
+ .setClusterKey(UTIL.getZkCluster().getAddress().toString() + ":/" + clusterKey)
+ .setReplicationEndpointImpl(
+ TestReplicationSourceManager.ReplicationEndpointForTest.class.getName());
+ UTIL.getAdmin().addReplicationPeer(peerId, builder.build(), true);
}
+ @After
+ public void tearDown() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
}
[hbase] 05/11: HBASE-27392 Add a new procedure type for implementing some global operations such as migration (#4803)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 31c8f8c40b01862ada9fc7b2a11bb21530f2c90b
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Thu Sep 29 10:08:02 2022 +0800
HBASE-27392 Add a new procedure type for implementing some global operations such as migration (#4803)
Signed-off-by: Xin Sun <dd...@gmail.com>
---
.../hbase/procedure2/LockedResourceType.java | 3 +-
.../master/procedure/GlobalProcedureInterface.java | 15 ++-
.../hadoop/hbase/master/procedure/GlobalQueue.java | 21 ++--
.../master/procedure/MasterProcedureScheduler.java | 119 ++++++++++++++++++++-
.../hbase/master/procedure/SchemaLocking.java | 18 +++-
.../procedure/TestMasterProcedureScheduler.java | 48 +++++++++
6 files changed, 202 insertions(+), 22 deletions(-)
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
index 12f899d7565..40141017009 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
+++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
@@ -26,5 +26,6 @@ public enum LockedResourceType {
TABLE,
REGION,
PEER,
- META
+ META,
+ GLOBAL
}
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalProcedureInterface.java
similarity index 82%
copy from hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
copy to hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalProcedureInterface.java
index 12f899d7565..1ef168abfd8 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalProcedureInterface.java
@@ -15,16 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hbase.procedure2;
+package org.apache.hadoop.hbase.master.procedure;
import org.apache.yetus.audience.InterfaceAudience;
+/**
+ * Procedure interface for global operations, such as migration.
+ */
@InterfaceAudience.Private
-public enum LockedResourceType {
- SERVER,
- NAMESPACE,
- TABLE,
- REGION,
- PEER,
- META
+public interface GlobalProcedureInterface {
+
+ String getGlobalId();
}
diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalQueue.java
similarity index 69%
copy from hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
copy to hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalQueue.java
index 12f899d7565..1633dc4856e 100644
--- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/LockedResourceType.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/GlobalQueue.java
@@ -15,16 +15,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hbase.procedure2;
+package org.apache.hadoop.hbase.master.procedure;
+import org.apache.hadoop.hbase.procedure2.LockStatus;
+import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
-public enum LockedResourceType {
- SERVER,
- NAMESPACE,
- TABLE,
- REGION,
- PEER,
- META
+public class GlobalQueue extends Queue<String> {
+
+ public GlobalQueue(String globalId, LockStatus lockStatus) {
+ super(globalId, lockStatus);
+ }
+
+ @Override
+ boolean requireExclusiveLock(Procedure<?> proc) {
+ return true;
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
index 866f2f6f403..fbf0eb8abf3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
import java.util.function.Supplier;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
@@ -95,16 +96,20 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
(n, k) -> n.compareKey((String) k);
private final static AvlKeyComparator<MetaQueue> META_QUEUE_KEY_COMPARATOR =
(n, k) -> n.compareKey((TableName) k);
+ private final static AvlKeyComparator<GlobalQueue> GLOBAL_QUEUE_KEY_COMPARATOR =
+ (n, k) -> n.compareKey((String) k);
private final FairQueue<ServerName> serverRunQueue = new FairQueue<>();
private final FairQueue<TableName> tableRunQueue = new FairQueue<>();
private final FairQueue<String> peerRunQueue = new FairQueue<>();
private final FairQueue<TableName> metaRunQueue = new FairQueue<>();
+ private final FairQueue<String> globalRunQueue = new FairQueue<>();
private final ServerQueue[] serverBuckets = new ServerQueue[128];
private TableQueue tableMap = null;
private PeerQueue peerMap = null;
private MetaQueue metaMap = null;
+ private GlobalQueue globalMap = null;
private final SchemaLocking locking;
@@ -128,6 +133,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
doAdd(serverRunQueue, getServerQueue(spi.getServerName(), spi), proc, addFront);
} else if (isPeerProcedure(proc)) {
doAdd(peerRunQueue, getPeerQueue(getPeerId(proc)), proc, addFront);
+ } else if (isGlobalProcedure(proc)) {
+ doAdd(globalRunQueue, getGlobalQueue(getGlobalId(proc)), proc, addFront);
} else {
// TODO: at the moment we only have Table and Server procedures
// if you are implementing a non-table/non-server procedure, you have two options: create
@@ -163,14 +170,19 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
@Override
protected boolean queueHasRunnables() {
- return metaRunQueue.hasRunnables() || tableRunQueue.hasRunnables()
- || serverRunQueue.hasRunnables() || peerRunQueue.hasRunnables();
+ return globalRunQueue.hasRunnables() || metaRunQueue.hasRunnables()
+ || tableRunQueue.hasRunnables() || serverRunQueue.hasRunnables()
+ || peerRunQueue.hasRunnables();
}
@Override
protected Procedure dequeue() {
- // meta procedure is always the first priority
- Procedure<?> pollResult = doPoll(metaRunQueue);
+ // pull global first
+ Procedure<?> pollResult = doPoll(globalRunQueue);
+ // then meta procedure
+ if (pollResult == null) {
+ pollResult = doPoll(metaRunQueue);
+ }
// For now, let server handling have precedence over table handling; presumption is that it
// is more important handling crashed servers than it is running the
// enabling/disabling tables, etc.
@@ -268,6 +280,14 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
clear(peerMap, peerRunQueue, PEER_QUEUE_KEY_COMPARATOR);
peerMap = null;
+ // Remove Meta
+ clear(metaMap, metaRunQueue, META_QUEUE_KEY_COMPARATOR);
+ metaMap = null;
+
+ // Remove Global
+ clear(globalMap, globalRunQueue, GLOBAL_QUEUE_KEY_COMPARATOR);
+ globalMap = null;
+
assert size() == 0 : "expected queue size to be 0, got " + size();
}
@@ -300,6 +320,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
count += queueSize(tableMap);
count += queueSize(peerMap);
count += queueSize(metaMap);
+ count += queueSize(globalMap);
return count;
}
@@ -502,6 +523,51 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
return proc instanceof MetaProcedureInterface;
}
+ // ============================================================================
+ // Global Queue Lookup Helpers
+ // ============================================================================
+ private GlobalQueue getGlobalQueue(String globalId) {
+ GlobalQueue node = AvlTree.get(globalMap, globalId, GLOBAL_QUEUE_KEY_COMPARATOR);
+ if (node != null) {
+ return node;
+ }
+ node = new GlobalQueue(globalId, locking.getGlobalLock(globalId));
+ globalMap = AvlTree.insert(globalMap, node);
+ return node;
+ }
+
+ private void removeGlobalQueue(String globalId) {
+ globalMap = AvlTree.remove(globalMap, globalId, GLOBAL_QUEUE_KEY_COMPARATOR);
+ locking.removeGlobalLock(globalId);
+ }
+
+ private void tryCleanupGlobalQueue(String globalId, Procedure<?> procedure) {
+ schedLock();
+ try {
+ GlobalQueue queue = AvlTree.get(globalMap, globalId, GLOBAL_QUEUE_KEY_COMPARATOR);
+ if (queue == null) {
+ return;
+ }
+
+ final LockAndQueue lock = locking.getGlobalLock(globalId);
+ if (queue.isEmpty() && lock.tryExclusiveLock(procedure)) {
+ removeFromRunQueue(globalRunQueue, queue,
+ () -> "clean up global queue after " + procedure + " completed");
+ removeGlobalQueue(globalId);
+ }
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ private static boolean isGlobalProcedure(Procedure<?> proc) {
+ return proc instanceof GlobalProcedureInterface;
+ }
+
+ private static String getGlobalId(Procedure<?> proc) {
+ return ((GlobalProcedureInterface) proc).getGlobalId();
+ }
+
// ============================================================================
// Table Locking Helpers
// ============================================================================
@@ -1006,6 +1072,51 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler {
}
}
+ // ============================================================================
+ // Global Locking Helpers
+ // ============================================================================
+ /**
+ * Try to acquire the share lock on global.
+ * @see #wakeGlobalExclusiveLock(Procedure, String)
+ * @param procedure the procedure trying to acquire the lock
+ * @return true if the procedure has to wait for global to be available
+ */
+ public boolean waitGlobalExclusiveLock(Procedure<?> procedure, String globalId) {
+ schedLock();
+ try {
+ final LockAndQueue lock = locking.getGlobalLock(globalId);
+ if (lock.tryExclusiveLock(procedure)) {
+ removeFromRunQueue(globalRunQueue, getGlobalQueue(globalId),
+ () -> procedure + " held shared lock");
+ return false;
+ }
+ waitProcedure(lock, procedure);
+ logLockedResource(LockedResourceType.GLOBAL, HConstants.EMPTY_STRING);
+ return true;
+ } finally {
+ schedUnlock();
+ }
+ }
+
+ /**
+ * Wake the procedures waiting for global.
+ * @see #waitGlobalExclusiveLock(Procedure, String)
+ * @param procedure the procedure releasing the lock
+ */
+ public void wakeGlobalExclusiveLock(Procedure<?> procedure, String globalId) {
+ schedLock();
+ try {
+ final LockAndQueue lock = locking.getGlobalLock(globalId);
+ lock.releaseExclusiveLock(procedure);
+ addToRunQueue(globalRunQueue, getGlobalQueue(globalId),
+ () -> procedure + " released shared lock");
+ int waitingCount = wakeWaitingProcedures(lock);
+ wakePollIfNeeded(waitingCount);
+ } finally {
+ schedUnlock();
+ }
+ }
+
/**
* For debugging. Expensive.
*/
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SchemaLocking.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SchemaLocking.java
index 13419ac455c..853d13b0c93 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SchemaLocking.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/SchemaLocking.java
@@ -53,6 +53,7 @@ class SchemaLocking {
// Single map for all regions irrespective of tables. Key is encoded region name.
private final Map<String, LockAndQueue> regionLocks = new HashMap<>();
private final Map<String, LockAndQueue> peerLocks = new HashMap<>();
+ private final Map<String, LockAndQueue> globalLocks = new HashMap<>();
private final LockAndQueue metaLock;
public SchemaLocking(Function<Long, Procedure<?>> procedureRetriever) {
@@ -94,6 +95,10 @@ class SchemaLocking {
return metaLock;
}
+ LockAndQueue getGlobalLock(String globalId) {
+ return getLock(globalLocks, globalId);
+ }
+
LockAndQueue removeRegionLock(String encodedRegionName) {
return regionLocks.remove(encodedRegionName);
}
@@ -114,6 +119,10 @@ class SchemaLocking {
return peerLocks.remove(peerId);
}
+ LockAndQueue removeGlobalLock(String globalId) {
+ return globalLocks.remove(globalId);
+ }
+
private LockedResource createLockedResource(LockedResourceType resourceType, String resourceName,
LockAndQueue queue) {
LockType lockType;
@@ -164,6 +173,8 @@ class SchemaLocking {
addToLockedResources(lockedResources, peerLocks, Function.identity(), LockedResourceType.PEER);
addToLockedResources(lockedResources, ImmutableMap.of(TableName.META_TABLE_NAME, metaLock),
tn -> tn.getNameAsString(), LockedResourceType.META);
+ addToLockedResources(lockedResources, globalLocks, Function.identity(),
+ LockedResourceType.GLOBAL);
return lockedResources;
}
@@ -191,6 +202,10 @@ class SchemaLocking {
break;
case META:
queue = metaLock;
+ break;
+ case GLOBAL:
+ queue = globalLocks.get(resourceName);
+ break;
default:
queue = null;
break;
@@ -216,7 +231,8 @@ class SchemaLocking {
+ filterUnlocked(this.namespaceLocks) + ", tableLocks=" + filterUnlocked(this.tableLocks)
+ ", regionLocks=" + filterUnlocked(this.regionLocks) + ", peerLocks="
+ filterUnlocked(this.peerLocks) + ", metaLocks="
- + filterUnlocked(ImmutableMap.of(TableName.META_TABLE_NAME, metaLock));
+ + filterUnlocked(ImmutableMap.of(TableName.META_TABLE_NAME, metaLock)) + ", globalLocks="
+ + filterUnlocked(globalLocks);
}
private String filterUnlocked(Map<?, LockAndQueue> locks) {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
index f0edf73715e..0cf34126a94 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureScheduler.java
@@ -940,6 +940,21 @@ public class TestMasterProcedureScheduler {
}
}
+ public static class TestGlobalProcedure extends TestProcedure
+ implements GlobalProcedureInterface {
+ private final String globalId;
+
+ public TestGlobalProcedure(long procId, String globalId) {
+ super(procId);
+ this.globalId = globalId;
+ }
+
+ @Override
+ public String getGlobalId() {
+ return globalId;
+ }
+ }
+
private static LockProcedure createLockProcedure(LockType lockType, long procId)
throws Exception {
LockProcedure procedure = new LockProcedure();
@@ -1093,6 +1108,39 @@ public class TestMasterProcedureScheduler {
assertEquals(1, resource.getWaitingProcedures().size());
}
+ @Test
+ public void testListLocksGlobal() throws Exception {
+ String globalId = "1";
+ LockProcedure procedure = createExclusiveLockProcedure(4);
+ queue.waitGlobalExclusiveLock(procedure, globalId);
+
+ List<LockedResource> locks = queue.getLocks();
+ assertEquals(1, locks.size());
+
+ LockedResource resource = locks.get(0);
+ assertLockResource(resource, LockedResourceType.GLOBAL, globalId);
+ assertExclusiveLock(resource, procedure);
+ assertTrue(resource.getWaitingProcedures().isEmpty());
+
+ // Try to acquire the exclusive lock again with same procedure
+ assertFalse(queue.waitGlobalExclusiveLock(procedure, globalId));
+
+ // Try to acquire the exclusive lock again with new procedure
+ LockProcedure procedure2 = createExclusiveLockProcedure(5);
+ assertTrue(queue.waitGlobalExclusiveLock(procedure2, globalId));
+
+ // Same peerId, still only has 1 LockedResource
+ locks = queue.getLocks();
+ assertEquals(1, locks.size());
+
+ resource = locks.get(0);
+ assertLockResource(resource, LockedResourceType.GLOBAL, globalId);
+ // LockedResource owner still is the origin procedure
+ assertExclusiveLock(resource, procedure);
+ // The new procedure should in the waiting list
+ assertEquals(1, resource.getWaitingProcedures().size());
+ }
+
@Test
public void testListLocksWaiting() throws Exception {
LockProcedure procedure1 = createExclusiveLockProcedure(1);
[hbase] 06/11: HBASE-27405 Fix the replication hfile/log cleaner report that the replication table does not exist (#4811)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-27109/table_based_rqs
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 26d6ba2d0ad5430ff52f6481ac13cecbf1515f1e
Author: LiangJun He <20...@163.com>
AuthorDate: Wed Oct 12 14:40:05 2022 +0800
HBASE-27405 Fix the replication hfile/log cleaner report that the replication table does not exist (#4811)
Signed-off-by: Duo Zhang <zh...@apache.org>
---
.../apache/hadoop/hbase/replication/ReplicationQueueStorage.java | 6 ++++++
.../hadoop/hbase/replication/TableReplicationQueueStorage.java | 9 +++++++++
.../hadoop/hbase/replication/master/ReplicationLogCleaner.java | 8 ++++++++
.../hbase/replication/master/TestReplicationLogCleaner.java | 1 +
4 files changed, 24 insertions(+)
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
index c4204f0e8c4..6f6aee38cc8 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
@@ -178,4 +178,10 @@ public interface ReplicationQueueStorage {
* created hfile references during the call may not be included.
*/
Set<String> getAllHFileRefs() throws ReplicationException;
+
+ /**
+ * Whether the replication queue table exists.
+ * @return Whether the replication queue table exists
+ */
+ boolean hasData() throws ReplicationException;
}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
index 0c9553f4fd8..392a3692d66 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
@@ -532,4 +532,13 @@ public class TableReplicationQueueStorage implements ReplicationQueueStorage {
throw new ReplicationException("failed to getAllHFileRefs", e);
}
}
+
+ @Override
+ public boolean hasData() throws ReplicationException {
+ try {
+ return conn.getAdmin().getDescriptor(tableName) != null;
+ } catch (IOException e) {
+ throw new ReplicationException("failed to get replication queue table", e);
+ }
+ }
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index f1fd8f8d6b3..3ab52da6158 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -76,6 +76,14 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
if (this.getConf() == null) {
return;
}
+ try {
+ if (!rpm.getQueueStorage().hasData()) {
+ return;
+ }
+ } catch (ReplicationException e) {
+ LOG.error("Error occurred while executing queueStorage.hasData()", e);
+ return;
+ }
canFilter = rpm.getReplicationLogCleanerBarrier().start();
if (canFilter) {
notFullyDeadServers = getNotFullyDeadServers.get();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java
index 7a227fb0603..7edadae03b1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/master/TestReplicationLogCleaner.java
@@ -86,6 +86,7 @@ public class TestReplicationLogCleaner {
when(rpm.listPeers(null)).thenReturn(new ArrayList<>());
ReplicationQueueStorage rqs = mock(ReplicationQueueStorage.class);
when(rpm.getQueueStorage()).thenReturn(rqs);
+ when(rpm.getQueueStorage().hasData()).thenReturn(true);
when(rqs.listAllQueues()).thenReturn(new ArrayList<>());
ServerManager sm = mock(ServerManager.class);
when(services.getServerManager()).thenReturn(sm);