You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2023/05/15 14:54:01 UTC
[hbase] 01/16: HBASE-27212 Implement a new table based replication queue storage and make the minimum replication system work (#4672)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit e2d8db6dfc1ab70e97560cc7bcaff10c45ba62c4
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Mon Aug 15 10:46:27 2022 +0800
HBASE-27212 Implement a new table based replication queue storage and make the minimum replication system work (#4672)
Signed-off-by: Xin Sun <dd...@gmail.com>
---
.../protobuf/server/master/MasterProcedure.proto | 17 +-
.../hbase/replication/ReplicationGroupOffset.java | 57 ++
.../hbase/replication/ReplicationQueueData.java | 47 ++
.../hbase/replication/ReplicationQueueId.java | 141 ++++
.../hbase/replication/ReplicationQueueStorage.java | 154 ++--
.../replication/ReplicationStorageFactory.java | 43 +-
.../hadoop/hbase/replication/ReplicationUtils.java | 15 -
.../replication/TableReplicationQueueStorage.java | 535 ++++++++++++
.../replication/ZKReplicationQueueStorage.java | 689 ---------------
.../replication/TestReplicationStateBasic.java | 243 +-----
.../replication/TestReplicationStateZKImpl.java | 1 -
.../replication/TestZKReplicationQueueStorage.java | 341 --------
.../org/apache/hadoop/hbase/master/HMaster.java | 3 +-
.../master/procedure/ServerCrashProcedure.java | 26 +-
.../AssignReplicationQueuesProcedure.java | 204 +++++
.../ClaimReplicationQueueRemoteProcedure.java | 44 +-
.../ClaimReplicationQueuesProcedure.java | 44 +-
.../master/replication/ReplicationPeerManager.java | 128 ++-
.../hbase/regionserver/wal/AbstractFSWAL.java | 6 +-
.../master/ReplicationHFileCleaner.java | 71 +-
.../replication/master/ReplicationLogCleaner.java | 25 +-
.../ClaimReplicationQueueCallable.java | 17 +-
.../regionserver/DumpReplicationQueues.java | 62 +-
.../regionserver/PeerProcedureHandler.java | 5 +-
.../regionserver/PeerProcedureHandlerImpl.java | 6 +-
.../regionserver/RecoveredReplicationSource.java | 138 +--
.../RecoveredReplicationSourceShipper.java | 64 +-
.../replication/regionserver/Replication.java | 2 +-
.../replication/regionserver/ReplicationLoad.java | 4 +-
.../regionserver/ReplicationSource.java | 70 +-
.../regionserver/ReplicationSourceFactory.java | 7 +-
.../regionserver/ReplicationSourceInterface.java | 26 +-
.../regionserver/ReplicationSourceLogQueue.java | 4 +-
.../regionserver/ReplicationSourceManager.java | 302 ++++---
.../regionserver/ReplicationSourceShipper.java | 6 +-
.../ReplicationSourceWALActionListener.java | 5 -
.../regionserver/ReplicationSyncUp.java | 23 +-
.../org/apache/hadoop/hbase/util/HBaseFsck.java | 4 +-
.../hadoop/hbase/util/hbck/ReplicationChecker.java | 47 +-
.../hadoop/hbase/wal/AbstractFSWALProvider.java | 40 +
.../hbase/client/TestAsyncReplicationAdminApi.java | 10 +-
.../master/assignment/MockMasterServices.java | 2 +-
.../hbase/master/cleaner/TestLogsCleaner.java | 42 +-
.../cleaner/TestReplicationHFileCleaner.java | 93 +-
.../TestBulkLoadReplicationHFileRefs.java | 2 +-
...StartupWithLegacyRegionReplicationEndpoint.java | 35 +-
.../hbase/replication/ReplicationSourceDummy.java | 13 +-
.../TestAddToSerialReplicationPeer.java | 5 +-
.../replication/TestClaimReplicationQueue.java | 8 +-
...TestNamespaceReplicationWithBulkLoadedData.java | 4 +-
.../TestReplicationEmptyWALRecovery.java | 1 -
.../replication/TestReplicationSyncUpTool.java | 3 +
...estReplicationSyncUpToolWithBulkLoadedData.java | 3 +
.../replication/TestSerialReplicationFailover.java | 3 +
.../TestTableReplicationQueueStorage.java | 423 ++++++++++
...tReplicationSyncUpToolWithMultipleAsyncWAL.java | 3 +
.../TestReplicationSyncUpToolWithMultipleWAL.java | 3 +
.../TestDrainReplicationQueuesForStandBy.java | 3 +
.../regionserver/TestDumpReplicationQueues.java | 3 +
.../regionserver/TestReplicationSource.java | 82 +-
.../regionserver/TestReplicationSourceManager.java | 933 +++++----------------
.../TestReplicationSourceManagerZkImpl.java | 107 ---
.../regionserver/TestReplicationWALEdits.java | 147 ++++
.../regionserver/TestSerialReplicationChecker.java | 17 +-
.../regionserver/TestWALEntrySinkFilter.java | 23 -
.../TestHBaseFsckCleanReplicationBarriers.java | 20 +-
.../hadoop/hbase/util/TestHBaseFsckEncryption.java | 3 +
.../apache/hadoop/hbase/util/TestHBaseFsckMOB.java | 3 +
.../hbase/util/TestHBaseFsckReplication.java | 103 ++-
69 files changed, 2708 insertions(+), 3055 deletions(-)
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
index 59bb031589a..2e0da0deb84 100644
--- a/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/server/master/MasterProcedure.proto
@@ -679,16 +679,13 @@ message ClaimReplicationQueueRemoteStateData {
required ServerName crashed_server = 1;
required string queue = 2;
required ServerName target_server = 3;
+ optional ServerName source_server = 4;
}
message ClaimReplicationQueueRemoteParameter {
required ServerName crashed_server = 1;
required string queue = 2;
-}
-
-enum ClaimReplicationQueuesState {
- CLAIM_REPLICATION_QUEUES_DISPATCH = 1;
- CLAIM_REPLICATION_QUEUES_FINISH = 2;
+ optional ServerName source_server = 3;
}
enum ModifyTableDescriptorState {
@@ -715,3 +712,13 @@ message ModifyStoreFileTrackerStateData {
message ModifyColumnFamilyStoreFileTrackerStateData {
required bytes family = 1;
}
+
+enum AssignReplicationQueuesState {
+ ASSIGN_REPLICATION_QUEUES_PRE_CHECK = 1;
+ ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES = 2;
+ ASSIGN_REPLICATION_QUEUES_CLAIM = 3;
+}
+
+message AssignReplicationQueuesStateData {
+ required ServerName crashed_server = 1;
+}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationGroupOffset.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationGroupOffset.java
new file mode 100644
index 00000000000..bd13594b99a
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationGroupOffset.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class ReplicationGroupOffset {
+
+ public static final ReplicationGroupOffset BEGIN = new ReplicationGroupOffset("", 0L);
+
+ private final String wal;
+
+ private final long offset;
+
+ public ReplicationGroupOffset(String wal, long offset) {
+ this.wal = wal;
+ this.offset = offset;
+ }
+
+ public String getWal() {
+ return wal;
+ }
+
+ /**
+ * A negative value means this file has already been fully replicated out
+ */
+ public long getOffset() {
+ return offset;
+ }
+
+ @Override
+ public String toString() {
+ return wal + ":" + offset;
+ }
+
+ public static ReplicationGroupOffset parse(String str) {
+ int index = str.lastIndexOf(':');
+ return new ReplicationGroupOffset(str.substring(0, index),
+ Long.parseLong(str.substring(index + 1)));
+ }
+}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueData.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueData.java
new file mode 100644
index 00000000000..794ae9d3a55
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueData.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
+/**
+ * Representing all the information for a replication queue.
+ */
+@InterfaceAudience.Private
+public class ReplicationQueueData {
+
+ private final ReplicationQueueId id;
+
+ private final ImmutableMap<String, ReplicationGroupOffset> offsets;
+
+ public ReplicationQueueData(ReplicationQueueId id,
+ ImmutableMap<String, ReplicationGroupOffset> offsets) {
+ this.id = id;
+ this.offsets = offsets;
+ }
+
+ public ReplicationQueueId getId() {
+ return id;
+ }
+
+ public ImmutableMap<String, ReplicationGroupOffset> getOffsets() {
+ return offsets;
+ }
+}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueId.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueId.java
new file mode 100644
index 00000000000..73633dda9a4
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueId.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.Objects;
+import java.util.Optional;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class ReplicationQueueId {
+
+ private final ServerName serverName;
+
+ private final String peerId;
+
+ private final Optional<ServerName> sourceServerName;
+
+ // we do not allow '-' in peer names so it is safe to use it as the separator for peer id and
+ // server name
+ private static final char PEER_ID_SEPARATOR = '-';
+
+ // The '/' character is not valid for a hostname or a nodename(FQDN, so it is safe to use it as
+ // the separator for server names)
+ private static final char SERVER_NAME_SEPARATOR = '/';
+
+ public ReplicationQueueId(ServerName serverName, String peerId) {
+ this.serverName = Objects.requireNonNull(serverName);
+ this.peerId = Objects.requireNonNull(peerId);
+ this.sourceServerName = Optional.empty();
+ }
+
+ public ReplicationQueueId(ServerName serverName, String peerId, ServerName sourceServerName) {
+ this.serverName = Objects.requireNonNull(serverName);
+ this.peerId = Objects.requireNonNull(peerId);
+ this.sourceServerName = Optional.of(sourceServerName);
+ }
+
+ public ServerName getServerName() {
+ return serverName;
+ }
+
+ public String getPeerId() {
+ return peerId;
+ }
+
+ public Optional<ServerName> getSourceServerName() {
+ return sourceServerName;
+ }
+
+ public ServerName getServerWALsBelongTo() {
+ return sourceServerName.orElse(serverName);
+ }
+
+ public boolean isRecovered() {
+ return sourceServerName.isPresent();
+ }
+
+ public ReplicationQueueId claim(ServerName targetServerName) {
+ ServerName newSourceServerName = sourceServerName.orElse(serverName);
+ return new ReplicationQueueId(targetServerName, peerId, newSourceServerName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(peerId, serverName, sourceServerName);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!(obj instanceof ReplicationQueueId)) {
+ return false;
+ }
+ ReplicationQueueId other = (ReplicationQueueId) obj;
+ return Objects.equals(peerId, other.peerId) && Objects.equals(serverName, other.serverName)
+ && Objects.equals(sourceServerName, other.sourceServerName);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb =
+ new StringBuilder().append(peerId).append(PEER_ID_SEPARATOR).append(serverName);
+ sourceServerName.ifPresent(s -> sb.append(SERVER_NAME_SEPARATOR).append(s.toString()));
+ return sb.toString();
+ }
+
+ public static ReplicationQueueId parse(String str) {
+ int dashIndex = str.indexOf(PEER_ID_SEPARATOR);
+ String peerId = str.substring(0, dashIndex);
+ int slashIndex = str.indexOf(SERVER_NAME_SEPARATOR, dashIndex + 1);
+ if (slashIndex < 0) {
+ String serverName = str.substring(dashIndex + 1);
+ return new ReplicationQueueId(ServerName.valueOf(serverName), peerId);
+ } else {
+ String serverName = str.substring(dashIndex + 1, slashIndex);
+ String sourceServerName = str.substring(slashIndex + 1);
+ return new ReplicationQueueId(ServerName.valueOf(serverName), peerId,
+ ServerName.valueOf(sourceServerName));
+ }
+ }
+
+ public static String getPeerId(String str) {
+ int dashIndex = str.indexOf(PEER_ID_SEPARATOR);
+ return str.substring(0, dashIndex);
+ }
+
+ public static byte[] getScanPrefix(ServerName serverName, String peerId) {
+ return Bytes.toBytes(peerId + PEER_ID_SEPARATOR + serverName.toString());
+ }
+
+ public static byte[] getScanPrefix(String peerId) {
+ return Bytes.toBytes(peerId + PEER_ID_SEPARATOR);
+ }
+
+ private static char getNextChar(char c) {
+ return (char) ((int) c + 1);
+ }
+
+ public static byte[] getScanStartRowForNextPeerId(String peerId) {
+ return Bytes.toBytes(peerId + getNextChar(PEER_ID_SEPARATOR));
+ }
+}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
index 0f95c04b254..c4204f0e8c4 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueStorage.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.replication;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.SortedSet;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.util.Pair;
@@ -33,40 +32,79 @@ import org.apache.yetus.audience.InterfaceAudience;
public interface ReplicationQueueStorage {
/**
- * Remove a replication queue for a given regionserver.
- * @param serverName the name of the regionserver
- * @param queueId a String that identifies the queue.
+ * Set the current offset for a specific WAL group in a given queue.
+ * @param queueId the id of the queue
+ * @param walGroup the group of the WAL, can be empty if multi wal is not enabled
+ * @param offset the current offset of replication progress
+ * @param lastSeqIds map with {encodedRegionName, sequenceId} pairs for serial replication.
+ */
+ void setOffset(ReplicationQueueId queueId, String walGroup, ReplicationGroupOffset offset,
+ Map<String, Long> lastSeqIds) throws ReplicationException;
+
+ /**
+ * Get the current offset of all the WAL groups for a queue
+ * @param queueId the id of the queue
+ * @return a map of all offsets of the WAL groups. The key the is WAL group and the value is the
+ * position.
+ */
+ Map<String, ReplicationGroupOffset> getOffsets(ReplicationQueueId queueId)
+ throws ReplicationException;
+
+ /**
+ * Get a list of all queues for the specific peer.
+ * @param peerId the id of the peer
+ * @return a list of queueIds
*/
- void removeQueue(ServerName serverName, String queueId) throws ReplicationException;
+ List<ReplicationQueueId> listAllQueueIds(String peerId) throws ReplicationException;
/**
- * Add a new WAL file to the given queue for a given regionserver. If the queue does not exist it
- * is created.
- * @param serverName the name of the regionserver
- * @param queueId a String that identifies the queue.
- * @param fileName name of the WAL
+ * Get a list of all queues for the specific region server.
+ * @param serverName the server name of the region server that owns the set of queues
+ * @return a list of queueIds
*/
- void addWAL(ServerName serverName, String queueId, String fileName) throws ReplicationException;
+ List<ReplicationQueueId> listAllQueueIds(ServerName serverName) throws ReplicationException;
/**
- * Remove an WAL file from the given queue for a given regionserver.
- * @param serverName the name of the regionserver
- * @param queueId a String that identifies the queue.
- * @param fileName name of the WAL
+ * Get a list of all queues for the specific region server and the specific peer
+ * @param peerId the id of the peer
+ * @param serverName the server name of the region server that owns the set of queues
+ * @return a list of queueIds
*/
- void removeWAL(ServerName serverName, String queueId, String fileName)
+ List<ReplicationQueueId> listAllQueueIds(String peerId, ServerName serverName)
throws ReplicationException;
/**
- * Set the current position for a specific WAL in a given queue for a given regionserver.
- * @param serverName the name of the regionserver
- * @param queueId a String that identifies the queue
- * @param fileName name of the WAL
- * @param position the current position in the file. Will ignore if less than or equal to 0.
- * @param lastSeqIds map with {encodedRegionName, sequenceId} pairs for serial replication.
+ * Get a list of all queues and the offsets.
*/
- void setWALPosition(ServerName serverName, String queueId, String fileName, long position,
- Map<String, Long> lastSeqIds) throws ReplicationException;
+ List<ReplicationQueueData> listAllQueues() throws ReplicationException;
+
+ /**
+ * Get a list of all region servers that have outstanding replication queues. These servers could
+ * be alive, dead or from a previous run of the cluster.
+ * @return a list of server names
+ */
+ List<ServerName> listAllReplicators() throws ReplicationException;
+
+ /**
+ * Change ownership for the queue identified by queueId and belongs to a dead region server.
+ * @param queueId the id of the queue
+ * @param targetServerName the name of the target region server
+ * @return the new PeerId and A SortedSet of WALs in its queue
+ */
+ Map<String, ReplicationGroupOffset> claimQueue(ReplicationQueueId queueId,
+ ServerName targetServerName) throws ReplicationException;
+
+ /**
+ * Remove a replication queue
+ * @param queueId the id of the queue to remove
+ */
+ void removeQueue(ReplicationQueueId queueId) throws ReplicationException;
+
+ /**
+ * Remove all the replication queues for the given peer. Usually used when removing a peer.
+ * @param peerId the id of the peer
+ */
+ void removeAllQueues(String peerId) throws ReplicationException;
/**
* Read the max sequence id of the specific region for a given peer. For serial replication, we
@@ -99,67 +137,6 @@ public interface ReplicationQueueStorage {
void removeLastSequenceIds(String peerId, List<String> encodedRegionNames)
throws ReplicationException;
- /**
- * Get the current position for a specific WAL in a given queue for a given regionserver.
- * @param serverName the name of the regionserver
- * @param queueId a String that identifies the queue
- * @param fileName name of the WAL
- * @return the current position in the file
- */
- long getWALPosition(ServerName serverName, String queueId, String fileName)
- throws ReplicationException;
-
- /**
- * Get a list of all WALs in the given queue on the given region server.
- * @param serverName the server name of the region server that owns the queue
- * @param queueId a String that identifies the queue
- * @return a list of WALs
- */
- List<String> getWALsInQueue(ServerName serverName, String queueId) throws ReplicationException;
-
- /**
- * Get a list of all queues for the specified region server.
- * @param serverName the server name of the region server that owns the set of queues
- * @return a list of queueIds
- */
- List<String> getAllQueues(ServerName serverName) throws ReplicationException;
-
- /**
- * Change ownership for the queue identified by queueId and belongs to a dead region server.
- * @param sourceServerName the name of the dead region server
- * @param destServerName the name of the target region server
- * @param queueId the id of the queue
- * @return the new PeerId and A SortedSet of WALs in its queue
- */
- Pair<String, SortedSet<String>> claimQueue(ServerName sourceServerName, String queueId,
- ServerName destServerName) throws ReplicationException;
-
- /**
- * Remove the record of region server if the queue is empty.
- */
- void removeReplicatorIfQueueIsEmpty(ServerName serverName) throws ReplicationException;
-
- /**
- * Get a list of all region servers that have outstanding replication queues. These servers could
- * be alive, dead or from a previous run of the cluster.
- * @return a list of server names
- */
- List<ServerName> getListOfReplicators() throws ReplicationException;
-
- /**
- * Load all wals in all replication queues. This method guarantees to return a snapshot which
- * contains all WALs at the start of this call even there is concurrent queue failover. However,
- * some newly created WALs during the call may not be included.
- */
- Set<String> getAllWALs() throws ReplicationException;
-
- /**
- * Add a peer to hfile reference queue if peer does not exist.
- * @param peerId peer cluster id to be added
- * @throws ReplicationException if fails to add a peer id to hfile reference queue
- */
- void addPeerToHFileRefs(String peerId) throws ReplicationException;
-
/**
* Remove a peer from hfile reference queue.
* @param peerId peer cluster id to be removed
@@ -201,11 +178,4 @@ public interface ReplicationQueueStorage {
* created hfile references during the call may not be included.
*/
Set<String> getAllHFileRefs() throws ReplicationException;
-
- /**
- * Get full znode name for given region server
- * @param serverName the name of the region server
- * @return full znode name
- */
- String getRsNode(ServerName serverName);
}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
index 0124dbdd113..d0c204f9934 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStorageFactory.java
@@ -17,9 +17,18 @@
*/
package org.apache.hadoop.hbase.replication;
+import java.io.IOException;
import java.lang.reflect.Constructor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.Coprocessor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.CoprocessorDescriptorBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -37,6 +46,27 @@ public final class ReplicationStorageFactory {
public static final ReplicationPeerStorageType DEFAULT_REPLICATION_PEER_STORAGE_IMPL =
ReplicationPeerStorageType.ZOOKEEPER;
+ public static final String REPLICATION_QUEUE_TABLE_NAME = "hbase.replication.queue.table.name";
+
+ public static final TableName REPLICATION_QUEUE_TABLE_NAME_DEFAULT =
+ TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "replication");
+
+ public static TableDescriptor createReplicationQueueTableDescriptor(TableName tableName)
+ throws IOException {
+ return TableDescriptorBuilder.newBuilder(tableName)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.of(TableReplicationQueueStorage.QUEUE_FAMILY))
+ .setColumnFamily(
+ ColumnFamilyDescriptorBuilder.of(TableReplicationQueueStorage.LAST_SEQUENCE_ID_FAMILY))
+ .setColumnFamily(
+ ColumnFamilyDescriptorBuilder.of(TableReplicationQueueStorage.HFILE_REF_FAMILY))
+ .setValue("hbase.regionserver.region.split_restriction.type", "DelimitedKeyPrefix")
+ .setValue("hbase.regionserver.region.split_restriction.delimiter", "-")
+ .setCoprocessor(CoprocessorDescriptorBuilder
+ .newBuilder("org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint")
+ .setPriority(Coprocessor.PRIORITY_SYSTEM).build())
+ .build();
+ }
+
private ReplicationStorageFactory() {
}
@@ -76,8 +106,17 @@ public final class ReplicationStorageFactory {
/**
* Create a new {@link ReplicationQueueStorage}.
*/
- public static ReplicationQueueStorage getReplicationQueueStorage(ZKWatcher zk,
+ public static ReplicationQueueStorage getReplicationQueueStorage(Connection conn,
Configuration conf) {
- return new ZKReplicationQueueStorage(zk, conf);
+ return getReplicationQueueStorage(conn, TableName.valueOf(conf.get(REPLICATION_QUEUE_TABLE_NAME,
+ REPLICATION_QUEUE_TABLE_NAME_DEFAULT.getNameAsString())));
+ }
+
+ /**
+ * Create a new {@link ReplicationQueueStorage}.
+ */
+ public static ReplicationQueueStorage getReplicationQueueStorage(Connection conn,
+ TableName tableName) {
+ return new TableReplicationQueueStorage(conn, tableName);
}
}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
index d1bca8b4b04..ae78781a313 100644
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationUtils.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CompoundConfiguration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -81,20 +80,6 @@ public final class ReplicationUtils {
return otherConf;
}
- public static void removeAllQueues(ReplicationQueueStorage queueStorage, String peerId)
- throws ReplicationException {
- for (ServerName replicator : queueStorage.getListOfReplicators()) {
- List<String> queueIds = queueStorage.getAllQueues(replicator);
- for (String queueId : queueIds) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (queueInfo.getPeerId().equals(peerId)) {
- queueStorage.removeQueue(replicator, queueId);
- }
- }
- queueStorage.removeReplicatorIfQueueIsEmpty(replicator);
- }
- }
-
private static boolean isCollectionEqual(Collection<String> c1, Collection<String> c2) {
if (c1 == null) {
return c2 == null;
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
new file mode 100644
index 00000000000..0c9553f4fd8
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableReplicationQueueStorage.java
@@ -0,0 +1,535 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.function.Supplier;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellScanner;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.CompareOperator;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.AsyncTable;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Scan.ReadType;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FutureUtils;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos;
+
+/**
+ * HBase table based replication queue storage.
+ */
+@InterfaceAudience.Private
+public class TableReplicationQueueStorage implements ReplicationQueueStorage {
+
+ public static final byte[] QUEUE_FAMILY = Bytes.toBytes("queue");
+
+ public static final byte[] LAST_SEQUENCE_ID_FAMILY = Bytes.toBytes("sid");
+
+ public static final byte[] HFILE_REF_FAMILY = Bytes.toBytes("hfileref");
+
+ private final Connection conn;
+
+ private final TableName tableName;
+
+ @FunctionalInterface
+ private interface TableCreator {
+
+ void create() throws IOException;
+ }
+
+ public TableReplicationQueueStorage(Connection conn, TableName tableName) {
+ this.conn = conn;
+ this.tableName = tableName;
+ }
+
+ private void addLastSeqIdsPut(MultiRowMutationProtos.MutateRowsRequest.Builder builder,
+ String peerId, Map<String, Long> lastSeqIds, AsyncTable<?> table) throws IOException {
+ // get the previous sequence ids first
+ byte[] row = Bytes.toBytes(peerId);
+ Get get = new Get(row);
+ lastSeqIds.keySet().forEach(encodedRegionName -> get.addColumn(LAST_SEQUENCE_ID_FAMILY,
+ Bytes.toBytes(encodedRegionName)));
+ Result result = FutureUtils.get(table.get(get));
+ Put put = new Put(row);
+ for (Map.Entry<String, Long> entry : lastSeqIds.entrySet()) {
+ String encodedRegionName = entry.getKey();
+ long lastSeqId = entry.getValue();
+ byte[] encodedRegionNameAsBytes = Bytes.toBytes(encodedRegionName);
+ byte[] previousLastSeqIdAsBytes =
+ result.getValue(LAST_SEQUENCE_ID_FAMILY, encodedRegionNameAsBytes);
+ if (previousLastSeqIdAsBytes != null) {
+ long previousLastSeqId = Bytes.toLong(previousLastSeqIdAsBytes);
+ if (lastSeqId > previousLastSeqId) {
+ // update last seq id when it is greater, and use CAS to make sure we do not overwrite
+ // other's value.
+ put.addColumn(LAST_SEQUENCE_ID_FAMILY, encodedRegionNameAsBytes,
+ Bytes.toBytes(lastSeqId));
+ builder.addCondition(ProtobufUtil.toCondition(row, LAST_SEQUENCE_ID_FAMILY,
+ encodedRegionNameAsBytes, CompareOperator.EQUAL, previousLastSeqIdAsBytes, null));
+ }
+ } else {
+ // also update last seq id when there is no value yet, and use CAS to make sure we do not
+ // overwrite
+ // other's value.
+ put.addColumn(LAST_SEQUENCE_ID_FAMILY, encodedRegionNameAsBytes, Bytes.toBytes(lastSeqId));
+ builder.addCondition(ProtobufUtil.toCondition(row, LAST_SEQUENCE_ID_FAMILY,
+ encodedRegionNameAsBytes, CompareOperator.EQUAL, null, null));
+ }
+ }
+ if (!put.isEmpty()) {
+ builder.addMutationRequest(ProtobufUtil.toMutation(MutationType.PUT, put));
+ }
+ }
+
+ @Override
+ public void setOffset(ReplicationQueueId queueId, String walGroup, ReplicationGroupOffset offset,
+ Map<String, Long> lastSeqIds) throws ReplicationException {
+ Put put = new Put(Bytes.toBytes(queueId.toString())).addColumn(QUEUE_FAMILY,
+ Bytes.toBytes(walGroup), Bytes.toBytes(offset.toString()));
+ AsyncTable<?> asyncTable = conn.toAsyncConnection().getTable(tableName);
+ try {
+ if (lastSeqIds.isEmpty()) {
+ FutureUtils.get(asyncTable.put(put));
+ } else {
+ for (;;) {
+ MultiRowMutationProtos.MutateRowsRequest.Builder builder =
+ MultiRowMutationProtos.MutateRowsRequest.newBuilder();
+ addLastSeqIdsPut(builder, queueId.getPeerId(), lastSeqIds, asyncTable);
+ if (builder.getMutationRequestCount() > 0) {
+ // use MultiRowMutationService to atomically update offset and last sequence ids
+ MultiRowMutationProtos.MutateRowsRequest request =
+ builder.addMutationRequest(ProtobufUtil.toMutation(MutationType.PUT, put)).build();
+ MultiRowMutationProtos.MutateRowsResponse responose =
+ FutureUtils.get(asyncTable.<MultiRowMutationProtos.MultiRowMutationService.Interface,
+ MultiRowMutationProtos.MutateRowsResponse> coprocessorService(
+ MultiRowMutationProtos.MultiRowMutationService::newStub,
+ (stub, controller, done) -> stub.mutateRows(controller, request, done),
+ put.getRow()));
+ if (responose.getProcessed()) {
+ break;
+ }
+ } else {
+ // we do not need to update last seq id, fallback to single put
+ FutureUtils.get(asyncTable.put(put));
+ break;
+ }
+ }
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to setOffset, queueId=" + queueId + ", walGroup="
+ + walGroup + ", offset=" + offset + ", lastSeqIds=" + lastSeqIds, e);
+ }
+ }
+
+ private ImmutableMap<String, ReplicationGroupOffset> parseOffsets(Result result) {
+ ImmutableMap.Builder<String, ReplicationGroupOffset> builder =
+ ImmutableMap.builderWithExpectedSize(result.size());
+ NavigableMap<byte[], byte[]> map = result.getFamilyMap(QUEUE_FAMILY);
+ if (map != null) {
+ map.forEach((k, v) -> {
+ String walGroup = Bytes.toString(k);
+ ReplicationGroupOffset offset = ReplicationGroupOffset.parse(Bytes.toString(v));
+ builder.put(walGroup, offset);
+ });
+ }
+ return builder.build();
+ }
+
+ private Map<String, ReplicationGroupOffset> getOffsets0(Table table, ReplicationQueueId queueId)
+ throws IOException {
+ Result result = table.get(new Get(Bytes.toBytes(queueId.toString())).addFamily(QUEUE_FAMILY));
+ return parseOffsets(result);
+ }
+
+ @Override
+ public Map<String, ReplicationGroupOffset> getOffsets(ReplicationQueueId queueId)
+ throws ReplicationException {
+ try (Table table = conn.getTable(tableName)) {
+ return getOffsets0(table, queueId);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to getOffsets, queueId=" + queueId, e);
+ }
+ }
+
+ private void listAllQueueIds(Table table, Scan scan, List<ReplicationQueueId> queueIds)
+ throws IOException {
+ try (ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ ReplicationQueueId queueId = ReplicationQueueId.parse(Bytes.toString(result.getRow()));
+ queueIds.add(queueId);
+ }
+ }
+ }
+
+ private void listAllQueueIds(Table table, String peerId, ServerName serverName,
+ List<ReplicationQueueId> queueIds) throws IOException {
+ listAllQueueIds(table,
+ new Scan().setStartStopRowForPrefixScan(ReplicationQueueId.getScanPrefix(serverName, peerId))
+ .addFamily(QUEUE_FAMILY).setFilter(new KeyOnlyFilter()),
+ queueIds);
+ }
+
+ @Override
+ public List<ReplicationQueueId> listAllQueueIds(String peerId) throws ReplicationException {
+ Scan scan = new Scan().setStartStopRowForPrefixScan(ReplicationQueueId.getScanPrefix(peerId))
+ .addFamily(QUEUE_FAMILY).setFilter(new KeyOnlyFilter());
+ List<ReplicationQueueId> queueIds = new ArrayList<>();
+ try (Table table = conn.getTable(tableName)) {
+ listAllQueueIds(table, scan, queueIds);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to listAllQueueIds, peerId=" + peerId, e);
+ }
+ return queueIds;
+ }
+
+ @Override
+ public List<ReplicationQueueId> listAllQueueIds(ServerName serverName)
+ throws ReplicationException {
+ List<ReplicationQueueId> queueIds = new ArrayList<>();
+ try (Table table = conn.getTable(tableName)) {
+ KeyOnlyFilter keyOnlyFilter = new KeyOnlyFilter();
+ String previousPeerId = null;
+ for (;;) {
+ // first, get the next peerId
+ Scan peerScan =
+ new Scan().addFamily(QUEUE_FAMILY).setOneRowLimit().setFilter(keyOnlyFilter);
+ if (previousPeerId != null) {
+ peerScan.withStartRow(ReplicationQueueId.getScanStartRowForNextPeerId(previousPeerId));
+ }
+ String peerId;
+ try (ResultScanner scanner = table.getScanner(peerScan)) {
+ Result result = scanner.next();
+ if (result == null) {
+ // no more peers, break
+ break;
+ }
+ peerId = ReplicationQueueId.getPeerId(Bytes.toString(result.getRow()));
+ }
+ listAllQueueIds(table, peerId, serverName, queueIds);
+ previousPeerId = peerId;
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to listAllQueueIds, serverName=" + serverName, e);
+ }
+ return queueIds;
+ }
+
+ @Override
+ public List<ReplicationQueueId> listAllQueueIds(String peerId, ServerName serverName)
+ throws ReplicationException {
+ List<ReplicationQueueId> queueIds = new ArrayList<>();
+ try (Table table = conn.getTable(tableName)) {
+ listAllQueueIds(table, peerId, serverName, queueIds);
+ } catch (IOException e) {
+ throw new ReplicationException(
+ "failed to listAllQueueIds, peerId=" + peerId + ", serverName=" + serverName, e);
+ }
+ return queueIds;
+ }
+
+ @Override
+ public List<ReplicationQueueData> listAllQueues() throws ReplicationException {
+ List<ReplicationQueueData> queues = new ArrayList<>();
+ Scan scan = new Scan().addFamily(QUEUE_FAMILY).setReadType(ReadType.STREAM);
+ try (Table table = conn.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ ReplicationQueueId queueId = ReplicationQueueId.parse(Bytes.toString(result.getRow()));
+ ReplicationQueueData queueData = new ReplicationQueueData(queueId, parseOffsets(result));
+ queues.add(queueData);
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to listAllQueues", e);
+ }
+ return queues;
+ }
+
+ @Override
+ public List<ServerName> listAllReplicators() throws ReplicationException {
+ Set<ServerName> replicators = new HashSet<>();
+ Scan scan = new Scan().addFamily(QUEUE_FAMILY).setFilter(new KeyOnlyFilter())
+ .setReadType(ReadType.STREAM);
+ try (Table table = conn.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ ReplicationQueueId queueId = ReplicationQueueId.parse(Bytes.toString(result.getRow()));
+ replicators.add(queueId.getServerName());
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to listAllReplicators", e);
+ }
+ return new ArrayList<>(replicators);
+ }
+
+ @Override
+ public Map<String, ReplicationGroupOffset> claimQueue(ReplicationQueueId queueId,
+ ServerName targetServerName) throws ReplicationException {
+ ReplicationQueueId newQueueId = queueId.claim(targetServerName);
+ byte[] coprocessorRow = ReplicationQueueId.getScanPrefix(queueId.getPeerId());
+ AsyncTable<?> asyncTable = conn.toAsyncConnection().getTable(tableName);
+ try (Table table = conn.getTable(tableName)) {
+ for (;;) {
+ Map<String, ReplicationGroupOffset> offsets = getOffsets0(table, queueId);
+ if (offsets.isEmpty()) {
+ return Collections.emptyMap();
+ }
+ Map.Entry<String, ReplicationGroupOffset> entry = offsets.entrySet().iterator().next();
+ ClientProtos.Condition condition = ProtobufUtil.toCondition(
+ Bytes.toBytes(queueId.toString()), QUEUE_FAMILY, Bytes.toBytes(entry.getKey()),
+ CompareOperator.EQUAL, Bytes.toBytes(entry.getValue().toString()), null);
+ Delete delete = new Delete(Bytes.toBytes(queueId.toString())).addFamily(QUEUE_FAMILY);
+ Put put = new Put(Bytes.toBytes(newQueueId.toString()));
+ offsets.forEach((walGroup, offset) -> put.addColumn(QUEUE_FAMILY, Bytes.toBytes(walGroup),
+ Bytes.toBytes(offset.toString())));
+ MultiRowMutationProtos.MutateRowsRequest request =
+ MultiRowMutationProtos.MutateRowsRequest.newBuilder().addCondition(condition)
+ .addMutationRequest(ProtobufUtil.toMutation(MutationType.DELETE, delete))
+ .addMutationRequest(ProtobufUtil.toMutation(MutationType.PUT, put)).build();
+ MultiRowMutationProtos.MutateRowsResponse resp =
+ FutureUtils.get(asyncTable.<MultiRowMutationProtos.MultiRowMutationService.Interface,
+ MultiRowMutationProtos.MutateRowsResponse> coprocessorService(
+ MultiRowMutationProtos.MultiRowMutationService::newStub,
+ (stub, controller, done) -> stub.mutateRows(controller, request, done),
+ coprocessorRow));
+ if (resp.getProcessed()) {
+ return offsets;
+ }
+ // if the multi is not processed, which usually the queue has already been claimed by
+ // others, for safety, let's try claiming again, usually the next get operation above will
+ // return an empty map and we will quit the loop.
+ }
+ } catch (IOException e) {
+ throw new ReplicationException(
+ "failed to claimQueue, queueId=" + queueId + ", targetServerName=" + targetServerName, e);
+ }
+ }
+
+ @Override
+ public void removeQueue(ReplicationQueueId queueId) throws ReplicationException {
+ try (Table table = conn.getTable(tableName)) {
+ table.delete(new Delete(Bytes.toBytes(queueId.toString())).addFamily(QUEUE_FAMILY));
+ } catch (IOException e) {
+ throw new ReplicationException("failed to removeQueue, queueId=" + queueId, e);
+ }
+ }
+
+ @Override
+ public void removeAllQueues(String peerId) throws ReplicationException {
+ Scan scan = new Scan().setStartStopRowForPrefixScan(ReplicationQueueId.getScanPrefix(peerId))
+ .addFamily(QUEUE_FAMILY).setFilter(new KeyOnlyFilter());
+ try (Table table = conn.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ table.delete(new Delete(result.getRow()));
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to listAllQueueIds, peerId=" + peerId, e);
+ }
+ }
+
+ @Override
+ public long getLastSequenceId(String encodedRegionName, String peerId)
+ throws ReplicationException {
+ byte[] qual = Bytes.toBytes(encodedRegionName);
+ try (Table table = conn.getTable(tableName)) {
+ Result result =
+ table.get(new Get(Bytes.toBytes(peerId)).addColumn(LAST_SEQUENCE_ID_FAMILY, qual));
+ byte[] lastSeqId = result.getValue(LAST_SEQUENCE_ID_FAMILY, qual);
+ return lastSeqId != null ? Bytes.toLong(lastSeqId) : HConstants.NO_SEQNUM;
+ } catch (IOException e) {
+ throw new ReplicationException("failed to getLastSequenceId, encodedRegionName="
+ + encodedRegionName + ", peerId=" + peerId, e);
+ }
+ }
+
+ @Override
+ public void setLastSequenceIds(String peerId, Map<String, Long> lastSeqIds)
+ throws ReplicationException {
+ // No need CAS and retry here, because it'll call setLastSequenceIds() for disabled peers
+ // only, so no conflict happen.
+ Put put = new Put(Bytes.toBytes(peerId));
+ lastSeqIds.forEach((encodedRegionName, lastSeqId) -> put.addColumn(LAST_SEQUENCE_ID_FAMILY,
+ Bytes.toBytes(encodedRegionName), Bytes.toBytes(lastSeqId)));
+ try (Table table = conn.getTable(tableName)) {
+ table.put(put);
+ } catch (IOException e) {
+ throw new ReplicationException(
+ "failed to setLastSequenceIds, peerId=" + peerId + ", lastSeqIds=" + lastSeqIds, e);
+ }
+ }
+
+ @Override
+ public void removeLastSequenceIds(String peerId) throws ReplicationException {
+ Delete delete = new Delete(Bytes.toBytes(peerId)).addFamily(LAST_SEQUENCE_ID_FAMILY);
+ try (Table table = conn.getTable(tableName)) {
+ table.delete(delete);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to removeLastSequenceIds, peerId=" + peerId, e);
+ }
+ }
+
+ @Override
+ public void removeLastSequenceIds(String peerId, List<String> encodedRegionNames)
+ throws ReplicationException {
+ Delete delete = new Delete(Bytes.toBytes(peerId));
+ encodedRegionNames.forEach(n -> delete.addColumns(LAST_SEQUENCE_ID_FAMILY, Bytes.toBytes(n)));
+ try (Table table = conn.getTable(tableName)) {
+ table.delete(delete);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to removeLastSequenceIds, peerId=" + peerId
+ + ", encodedRegionNames=" + encodedRegionNames, e);
+ }
+ }
+
+ @Override
+ public void removePeerFromHFileRefs(String peerId) throws ReplicationException {
+ try (Table table = conn.getTable(tableName)) {
+ table.delete(new Delete(Bytes.toBytes(peerId)).addFamily(HFILE_REF_FAMILY));
+ } catch (IOException e) {
+ throw new ReplicationException("failed to removePeerFromHFileRefs, peerId=" + peerId, e);
+ }
+ }
+
+ @Override
+ public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs)
+ throws ReplicationException {
+ Put put = new Put(Bytes.toBytes(peerId));
+ pairs.forEach(p -> put.addColumn(HFILE_REF_FAMILY, Bytes.toBytes(p.getSecond().getName()),
+ HConstants.EMPTY_BYTE_ARRAY));
+ try (Table table = conn.getTable(tableName)) {
+ table.put(put);
+ } catch (IOException e) {
+ throw new ReplicationException(
+ "failed to addHFileRefs, peerId=" + peerId + ", pairs=" + pairs, e);
+ }
+ }
+
+ @Override
+ public void removeHFileRefs(String peerId, List<String> files) throws ReplicationException {
+ Delete delete = new Delete(Bytes.toBytes(peerId));
+ files.forEach(f -> delete.addColumns(HFILE_REF_FAMILY, Bytes.toBytes(f)));
+ try (Table table = conn.getTable(tableName)) {
+ table.delete(delete);
+ } catch (IOException e) {
+ throw new ReplicationException(
+ "failed to removeHFileRefs, peerId=" + peerId + ", files=" + files, e);
+ }
+ }
+
+ @Override
+ public List<String> getAllPeersFromHFileRefsQueue() throws ReplicationException {
+ List<String> peerIds = new ArrayList<>();
+ Scan scan = new Scan().addFamily(HFILE_REF_FAMILY).setReadType(ReadType.STREAM)
+ .setFilter(new KeyOnlyFilter());
+ try (Table table = conn.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ peerIds.add(Bytes.toString(result.getRow()));
+ }
+ } catch (IOException e) {
+ throw new ReplicationException("failed to getAllPeersFromHFileRefsQueue", e);
+ }
+ return peerIds;
+ }
+
+ private <T extends Collection<String>> T scanHFiles(Scan scan, Supplier<T> creator)
+ throws IOException {
+ T files = creator.get();
+ try (Table table = conn.getTable(tableName); ResultScanner scanner = table.getScanner(scan)) {
+ for (;;) {
+ Result result = scanner.next();
+ if (result == null) {
+ break;
+ }
+ CellScanner cellScanner = result.cellScanner();
+ while (cellScanner.advance()) {
+ Cell cell = cellScanner.current();
+ files.add(Bytes.toString(CellUtil.cloneQualifier(cell)));
+ }
+ }
+ }
+ return files;
+ }
+
+ @Override
+ public List<String> getReplicableHFiles(String peerId) throws ReplicationException {
+ // use scan to avoid getting a too large row one time, which may cause a very huge memory usage.
+ Scan scan = new Scan().addFamily(HFILE_REF_FAMILY)
+ .setStartStopRowForPrefixScan(Bytes.toBytes(peerId)).setAllowPartialResults(true);
+ try {
+ return scanHFiles(scan, ArrayList::new);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to getReplicableHFiles, peerId=" + peerId, e);
+ }
+ }
+
+ @Override
+ public Set<String> getAllHFileRefs() throws ReplicationException {
+ Scan scan = new Scan().addFamily(HFILE_REF_FAMILY).setReadType(ReadType.STREAM)
+ .setAllowPartialResults(true);
+ try {
+ return scanHFiles(scan, HashSet::new);
+ } catch (IOException e) {
+ throw new ReplicationException("failed to getAllHFileRefs", e);
+ }
+ }
+}
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
deleted file mode 100644
index f3506ad3555..00000000000
--- a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ZKReplicationQueueStorage.java
+++ /dev/null
@@ -1,689 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import static java.util.stream.Collectors.toList;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.stream.Collectors;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.KeeperException.BadVersionException;
-import org.apache.zookeeper.KeeperException.NoNodeException;
-import org.apache.zookeeper.KeeperException.NodeExistsException;
-import org.apache.zookeeper.KeeperException.NotEmptyException;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
-
-/**
- * ZK based replication queue storage.
- * <p>
- * The base znode for each regionserver is the regionserver name. For example:
- *
- * <pre>
- * /hbase/replication/rs/hostname.example.org,6020,1234
- * </pre>
- *
- * Within this znode, the region server maintains a set of WAL replication queues. These queues are
- * represented by child znodes named using there give queue id. For example:
- *
- * <pre>
- * /hbase/replication/rs/hostname.example.org,6020,1234/1
- * /hbase/replication/rs/hostname.example.org,6020,1234/2
- * </pre>
- *
- * Each queue has one child znode for every WAL that still needs to be replicated. The value of
- * these WAL child znodes is the latest position that has been replicated. This position is updated
- * every time a WAL entry is replicated. For example:
- *
- * <pre>
- * /hbase/replication/rs/hostname.example.org,6020,1234/1/23522342.23422 [VALUE: 254]
- * </pre>
- */
-@InterfaceAudience.Private
-class ZKReplicationQueueStorage extends ZKReplicationStorageBase
- implements ReplicationQueueStorage {
-
- private static final Logger LOG = LoggerFactory.getLogger(ZKReplicationQueueStorage.class);
-
- public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY =
- "zookeeper.znode.replication.hfile.refs";
- public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT = "hfile-refs";
-
- public static final String ZOOKEEPER_ZNODE_REPLICATION_REGIONS_KEY =
- "zookeeper.znode.replication.regions";
- public static final String ZOOKEEPER_ZNODE_REPLICATION_REGIONS_DEFAULT = "regions";
-
- /**
- * The name of the znode that contains all replication queues
- */
- private final String queuesZNode;
-
- /**
- * The name of the znode that contains queues of hfile references to be replicated
- */
- private final String hfileRefsZNode;
-
- final String regionsZNode;
-
- public ZKReplicationQueueStorage(ZKWatcher zookeeper, Configuration conf) {
- super(zookeeper, conf);
-
- String queuesZNodeName = conf.get("zookeeper.znode.replication.rs", "rs");
- String hfileRefsZNodeName = conf.get(ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
- ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
- this.queuesZNode = ZNodePaths.joinZNode(replicationZNode, queuesZNodeName);
- this.hfileRefsZNode = ZNodePaths.joinZNode(replicationZNode, hfileRefsZNodeName);
- this.regionsZNode = ZNodePaths.joinZNode(replicationZNode, conf
- .get(ZOOKEEPER_ZNODE_REPLICATION_REGIONS_KEY, ZOOKEEPER_ZNODE_REPLICATION_REGIONS_DEFAULT));
- }
-
- @Override
- public String getRsNode(ServerName serverName) {
- return ZNodePaths.joinZNode(queuesZNode, serverName.getServerName());
- }
-
- private String getQueueNode(ServerName serverName, String queueId) {
- return ZNodePaths.joinZNode(getRsNode(serverName), queueId);
- }
-
- private String getFileNode(String queueNode, String fileName) {
- return ZNodePaths.joinZNode(queueNode, fileName);
- }
-
- private String getFileNode(ServerName serverName, String queueId, String fileName) {
- return getFileNode(getQueueNode(serverName, queueId), fileName);
- }
-
- /**
- * <p>
- * Put all regions under /hbase/replication/regions znode will lead to too many children because
- * of the huge number of regions in real production environment. So here we will distribute the
- * znodes to multiple directories.
- * </p>
- * <p>
- * So the final znode path will be format like this:
- *
- * <pre>
- * /hbase/replication/regions/dd/04/e76a6966d4ffa908ed0586764767-100
- * </pre>
- *
- * Here the full encoded region name is dd04e76a6966d4ffa908ed0586764767, and we use the first two
- * characters 'dd' as the first level directory name, and use the next two characters '04' as the
- * second level directory name, and the rest part as the prefix of the znode, and the suffix '100'
- * is the peer id.
- * </p>
- * @param encodedRegionName the encoded region name.
- * @param peerId peer id for replication.
- * @return ZNode path to persist the max sequence id that we've pushed for the given region and
- * peer.
- */
- String getSerialReplicationRegionPeerNode(String encodedRegionName, String peerId) {
- if (encodedRegionName == null || encodedRegionName.length() != RegionInfo.MD5_HEX_LENGTH) {
- throw new IllegalArgumentException(
- "Invalid encoded region name: " + encodedRegionName + ", length should be 32.");
- }
- return new StringBuilder(regionsZNode).append(ZNodePaths.ZNODE_PATH_SEPARATOR)
- .append(encodedRegionName, 0, 2).append(ZNodePaths.ZNODE_PATH_SEPARATOR)
- .append(encodedRegionName, 2, 4).append(ZNodePaths.ZNODE_PATH_SEPARATOR)
- .append(encodedRegionName, 4, encodedRegionName.length()).append("-").append(peerId)
- .toString();
- }
-
- @Override
- public void removeQueue(ServerName serverName, String queueId) throws ReplicationException {
- try {
- ZKUtil.deleteNodeRecursively(zookeeper, getQueueNode(serverName, queueId));
- } catch (KeeperException e) {
- throw new ReplicationException(
- "Failed to delete queue (serverName=" + serverName + ", queueId=" + queueId + ")", e);
- }
- }
-
- @Override
- public void addWAL(ServerName serverName, String queueId, String fileName)
- throws ReplicationException {
- try {
- ZKUtil.createWithParents(zookeeper, getFileNode(serverName, queueId, fileName));
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to add wal to queue (serverName=" + serverName
- + ", queueId=" + queueId + ", fileName=" + fileName + ")", e);
- }
- }
-
- @Override
- public void removeWAL(ServerName serverName, String queueId, String fileName)
- throws ReplicationException {
- String fileNode = getFileNode(serverName, queueId, fileName);
- try {
- ZKUtil.deleteNode(zookeeper, fileNode);
- } catch (NoNodeException e) {
- LOG.warn("{} already deleted when removing log", fileNode);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to remove wal from queue (serverName=" + serverName
- + ", queueId=" + queueId + ", fileName=" + fileName + ")", e);
- }
- }
-
- private void addLastSeqIdsToOps(String queueId, Map<String, Long> lastSeqIds,
- List<ZKUtilOp> listOfOps) throws KeeperException, ReplicationException {
- String peerId = new ReplicationQueueInfo(queueId).getPeerId();
- for (Entry<String, Long> lastSeqEntry : lastSeqIds.entrySet()) {
- String path = getSerialReplicationRegionPeerNode(lastSeqEntry.getKey(), peerId);
- Pair<Long, Integer> p = getLastSequenceIdWithVersion(lastSeqEntry.getKey(), peerId);
- byte[] data = ZKUtil.positionToByteArray(lastSeqEntry.getValue());
- if (p.getSecond() < 0) { // ZNode does not exist.
- ZKUtil.createWithParents(zookeeper,
- path.substring(0, path.lastIndexOf(ZNodePaths.ZNODE_PATH_SEPARATOR)));
- listOfOps.add(ZKUtilOp.createAndFailSilent(path, data));
- continue;
- }
- // Perform CAS in a specific version v0 (HBASE-20138)
- int v0 = p.getSecond();
- long lastPushedSeqId = p.getFirst();
- if (lastSeqEntry.getValue() <= lastPushedSeqId) {
- continue;
- }
- listOfOps.add(ZKUtilOp.setData(path, data, v0));
- }
- }
-
- @Override
- public void setWALPosition(ServerName serverName, String queueId, String fileName, long position,
- Map<String, Long> lastSeqIds) throws ReplicationException {
- try {
- for (int retry = 0;; retry++) {
- List<ZKUtilOp> listOfOps = new ArrayList<>();
- if (position > 0) {
- listOfOps.add(ZKUtilOp.setData(getFileNode(serverName, queueId, fileName),
- ZKUtil.positionToByteArray(position)));
- }
- // Persist the max sequence id(s) of regions for serial replication atomically.
- addLastSeqIdsToOps(queueId, lastSeqIds, listOfOps);
- if (listOfOps.isEmpty()) {
- return;
- }
- try {
- ZKUtil.multiOrSequential(zookeeper, listOfOps, false);
- return;
- } catch (KeeperException.BadVersionException | KeeperException.NodeExistsException e) {
- LOG.warn(
- "Bad version(or node exist) when persist the last pushed sequence id to zookeeper "
- + "storage, Retry = " + retry + ", serverName=" + serverName + ", queueId=" + queueId
- + ", fileName=" + fileName);
- }
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to set log position (serverName=" + serverName
- + ", queueId=" + queueId + ", fileName=" + fileName + ", position=" + position + ")", e);
- }
- }
-
- /**
- * Return the {lastPushedSequenceId, ZNodeDataVersion} pair. if ZNodeDataVersion is -1, it means
- * that the ZNode does not exist.
- */
- protected Pair<Long, Integer> getLastSequenceIdWithVersion(String encodedRegionName,
- String peerId) throws KeeperException {
- Stat stat = new Stat();
- String path = getSerialReplicationRegionPeerNode(encodedRegionName, peerId);
- byte[] data = ZKUtil.getDataNoWatch(zookeeper, path, stat);
- if (data == null) {
- // ZNode does not exist, so just return version -1 to indicate that no node exist.
- return Pair.newPair(HConstants.NO_SEQNUM, -1);
- }
- try {
- return Pair.newPair(ZKUtil.parseWALPositionFrom(data), stat.getVersion());
- } catch (DeserializationException de) {
- LOG.warn("Failed to parse log position (region=" + encodedRegionName + ", peerId=" + peerId
- + "), data=" + Bytes.toStringBinary(data));
- }
- return Pair.newPair(HConstants.NO_SEQNUM, stat.getVersion());
- }
-
- @Override
- public long getLastSequenceId(String encodedRegionName, String peerId)
- throws ReplicationException {
- try {
- return getLastSequenceIdWithVersion(encodedRegionName, peerId).getFirst();
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get last pushed sequence id (encodedRegionName="
- + encodedRegionName + ", peerId=" + peerId + ")", e);
- }
- }
-
- @Override
- public void setLastSequenceIds(String peerId, Map<String, Long> lastSeqIds)
- throws ReplicationException {
- try {
- // No need CAS and retry here, because it'll call setLastSequenceIds() for disabled peers
- // only, so no conflict happen.
- List<ZKUtilOp> listOfOps = new ArrayList<>();
- for (Entry<String, Long> lastSeqEntry : lastSeqIds.entrySet()) {
- String path = getSerialReplicationRegionPeerNode(lastSeqEntry.getKey(), peerId);
- ZKUtil.createWithParents(zookeeper, path);
- listOfOps.add(ZKUtilOp.setData(path, ZKUtil.positionToByteArray(lastSeqEntry.getValue())));
- }
- if (!listOfOps.isEmpty()) {
- ZKUtil.multiOrSequential(zookeeper, listOfOps, true);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to set last sequence ids, peerId=" + peerId
- + ", size of lastSeqIds=" + lastSeqIds.size(), e);
- }
- }
-
- @Override
- public void removeLastSequenceIds(String peerId) throws ReplicationException {
- String suffix = "-" + peerId;
- try {
- StringBuilder sb = new StringBuilder(regionsZNode);
- int regionsZNodeLength = regionsZNode.length();
- int levelOneLength = regionsZNodeLength + 3;
- int levelTwoLength = levelOneLength + 3;
- List<String> levelOneDirs = ZKUtil.listChildrenNoWatch(zookeeper, regionsZNode);
- // it is possible that levelOneDirs is null if we haven't write any last pushed sequence ids
- // yet, so we need an extra check here.
- if (CollectionUtils.isEmpty(levelOneDirs)) {
- return;
- }
- for (String levelOne : levelOneDirs) {
- sb.append(ZNodePaths.ZNODE_PATH_SEPARATOR).append(levelOne);
- for (String levelTwo : ZKUtil.listChildrenNoWatch(zookeeper, sb.toString())) {
- sb.append(ZNodePaths.ZNODE_PATH_SEPARATOR).append(levelTwo);
- for (String znode : ZKUtil.listChildrenNoWatch(zookeeper, sb.toString())) {
- if (znode.endsWith(suffix)) {
- sb.append(ZNodePaths.ZNODE_PATH_SEPARATOR).append(znode);
- ZKUtil.deleteNode(zookeeper, sb.toString());
- sb.setLength(levelTwoLength);
- }
- }
- sb.setLength(levelOneLength);
- }
- sb.setLength(regionsZNodeLength);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to remove all last sequence ids, peerId=" + peerId, e);
- }
- }
-
- @Override
- public void removeLastSequenceIds(String peerId, List<String> encodedRegionNames)
- throws ReplicationException {
- try {
- List<ZKUtilOp> listOfOps =
- encodedRegionNames.stream().map(n -> getSerialReplicationRegionPeerNode(n, peerId))
- .map(ZKUtilOp::deleteNodeFailSilent).collect(Collectors.toList());
- ZKUtil.multiOrSequential(zookeeper, listOfOps, true);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to remove last sequence ids, peerId=" + peerId
- + ", encodedRegionNames.size=" + encodedRegionNames.size(), e);
- }
- }
-
- @Override
- public long getWALPosition(ServerName serverName, String queueId, String fileName)
- throws ReplicationException {
- byte[] bytes;
- try {
- bytes = ZKUtil.getData(zookeeper, getFileNode(serverName, queueId, fileName));
- } catch (KeeperException | InterruptedException e) {
- throw new ReplicationException("Failed to get log position (serverName=" + serverName
- + ", queueId=" + queueId + ", fileName=" + fileName + ")", e);
- }
- try {
- return ZKUtil.parseWALPositionFrom(bytes);
- } catch (DeserializationException de) {
- LOG.warn("Failed parse log position (serverName={}, queueId={}, fileName={})", serverName,
- queueId, fileName);
- }
- // if we can not parse the position, start at the beginning of the wal file again
- return 0;
- }
-
- /**
- * This implement must update the cversion of root {@link #queuesZNode}. The optimistic lock of
- * the {@link #getAllWALs()} method is based on the cversion of root {@link #queuesZNode}.
- * @see #getAllWALs() to show the usage of the cversion of root {@link #queuesZNode} .
- */
- @Override
- public Pair<String, SortedSet<String>> claimQueue(ServerName sourceServerName, String queueId,
- ServerName destServerName) throws ReplicationException {
- LOG.info("Atomically moving {}/{}'s WALs to {}", sourceServerName, queueId, destServerName);
- try {
- ZKUtil.createWithParents(zookeeper, getRsNode(destServerName));
- } catch (KeeperException e) {
- throw new ReplicationException("Claim queue queueId=" + queueId + " from " + sourceServerName
- + " to " + destServerName + " failed when creating the node for " + destServerName, e);
- }
- String newQueueId = queueId + "-" + sourceServerName;
- try {
- String oldQueueNode = getQueueNode(sourceServerName, queueId);
- List<String> wals = ZKUtil.listChildrenNoWatch(zookeeper, oldQueueNode);
- if (CollectionUtils.isEmpty(wals)) {
- ZKUtil.deleteNodeFailSilent(zookeeper, oldQueueNode);
- LOG.info("Removed empty {}/{}", sourceServerName, queueId);
- return new Pair<>(newQueueId, Collections.emptySortedSet());
- }
- String newQueueNode = getQueueNode(destServerName, newQueueId);
- List<ZKUtilOp> listOfOps = new ArrayList<>();
- SortedSet<String> logQueue = new TreeSet<>();
- // create the new cluster znode
- listOfOps.add(ZKUtilOp.createAndFailSilent(newQueueNode, HConstants.EMPTY_BYTE_ARRAY));
- // get the offset of the logs and set it to new znodes
- for (String wal : wals) {
- String oldWalNode = getFileNode(oldQueueNode, wal);
- byte[] logOffset = ZKUtil.getData(this.zookeeper, oldWalNode);
- LOG.debug("Creating {} with data {}", wal, Bytes.toStringBinary(logOffset));
- String newWalNode = getFileNode(newQueueNode, wal);
- listOfOps.add(ZKUtilOp.createAndFailSilent(newWalNode, logOffset));
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldWalNode));
- logQueue.add(wal);
- }
- // add delete op for peer
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldQueueNode));
- // Append new queue id for prevent lock competition in zookeeper server.
- String claimLockZNode = ZNodePaths.joinZNode(queuesZNode, "cversion_" + newQueueId);
- // A trick for update the cversion of root queuesZNode .
- // The optimistic lock of the getAllWALs() method is based on the cversion of root queuesZNode
- listOfOps.add(ZKUtilOp.createAndFailSilent(claimLockZNode, HConstants.EMPTY_BYTE_ARRAY));
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(claimLockZNode));
-
- LOG.trace("The multi list size is {}", listOfOps.size());
- ZKUtil.multiOrSequential(zookeeper, listOfOps, false);
-
- LOG.info("Atomically moved {}/{}'s WALs to {}", sourceServerName, queueId, destServerName);
- return new Pair<>(newQueueId, logQueue);
- } catch (NoNodeException | NodeExistsException | NotEmptyException | BadVersionException e) {
- // Multi call failed; it looks like some other regionserver took away the logs.
- // These exceptions mean that zk tells us the request can not be execute. So return an empty
- // queue to tell the upper layer that claim nothing. For other types of exception should be
- // thrown out to notify the upper layer.
- LOG.info("Claim queue queueId={} from {} to {} failed with {}, someone else took the log?",
- queueId, sourceServerName, destServerName, e.toString());
- return new Pair<>(newQueueId, Collections.emptySortedSet());
- } catch (KeeperException | InterruptedException e) {
- throw new ReplicationException("Claim queue queueId=" + queueId + " from " + sourceServerName
- + " to " + destServerName + " failed", e);
- }
- }
-
- @Override
- public void removeReplicatorIfQueueIsEmpty(ServerName serverName) throws ReplicationException {
- try {
- ZKUtil.deleteNodeFailSilent(zookeeper, getRsNode(serverName));
- } catch (NotEmptyException e) {
- // keep silence to avoid logging too much.
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to remove replicator for " + serverName, e);
- }
- }
-
- private List<ServerName> getListOfReplicators0() throws KeeperException {
- List<String> children = ZKUtil.listChildrenNoWatch(zookeeper, queuesZNode);
- if (children == null) {
- children = Collections.emptyList();
- }
- return children.stream().map(ServerName::parseServerName).collect(toList());
- }
-
- @Override
- public List<ServerName> getListOfReplicators() throws ReplicationException {
- try {
- return getListOfReplicators0();
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get list of replicators", e);
- }
- }
-
- private List<String> getWALsInQueue0(ServerName serverName, String queueId)
- throws KeeperException {
- List<String> children =
- ZKUtil.listChildrenNoWatch(zookeeper, getQueueNode(serverName, queueId));
- return children != null ? children : Collections.emptyList();
- }
-
- @Override
- public List<String> getWALsInQueue(ServerName serverName, String queueId)
- throws ReplicationException {
- try {
- return getWALsInQueue0(serverName, queueId);
- } catch (KeeperException e) {
- throw new ReplicationException(
- "Failed to get wals in queue (serverName=" + serverName + ", queueId=" + queueId + ")", e);
- }
- }
-
- private List<String> getAllQueues0(ServerName serverName) throws KeeperException {
- List<String> children = ZKUtil.listChildrenNoWatch(zookeeper, getRsNode(serverName));
- return children != null ? children : Collections.emptyList();
- }
-
- @Override
- public List<String> getAllQueues(ServerName serverName) throws ReplicationException {
- try {
- return getAllQueues0(serverName);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get all queues (serverName=" + serverName + ")", e);
- }
- }
-
- // will be overridden in UTs
- protected int getQueuesZNodeCversion() throws KeeperException {
- Stat stat = new Stat();
- ZKUtil.getDataNoWatch(this.zookeeper, this.queuesZNode, stat);
- return stat.getCversion();
- }
-
- /**
- * The optimistic lock of this implement is based on the cversion of root {@link #queuesZNode}.
- * Therefore, we must update the cversion of root {@link #queuesZNode} when migrate wal nodes to
- * other queues.
- * @see #claimQueue(ServerName, String, ServerName) as an example of updating root
- * {@link #queuesZNode} cversion.
- */
- @Override
- public Set<String> getAllWALs() throws ReplicationException {
- try {
- for (int retry = 0;; retry++) {
- int v0 = getQueuesZNodeCversion();
- List<ServerName> rss = getListOfReplicators0();
- if (rss.isEmpty()) {
- LOG.debug("Didn't find a RegionServer that replicates, won't prevent deletions.");
- return Collections.emptySet();
- }
- Set<String> wals = new HashSet<>();
- for (ServerName rs : rss) {
- for (String queueId : getAllQueues0(rs)) {
- wals.addAll(getWALsInQueue0(rs, queueId));
- }
- }
- int v1 = getQueuesZNodeCversion();
- if (v0 == v1) {
- return wals;
- }
- LOG.info("Replication queue node cversion changed from %d to %d, retry = %d", v0, v1,
- retry);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get all wals", e);
- }
- }
-
- private String getHFileRefsPeerNode(String peerId) {
- return ZNodePaths.joinZNode(hfileRefsZNode, peerId);
- }
-
- private String getHFileNode(String peerNode, String fileName) {
- return ZNodePaths.joinZNode(peerNode, fileName);
- }
-
- @Override
- public void addPeerToHFileRefs(String peerId) throws ReplicationException {
- String peerNode = getHFileRefsPeerNode(peerId);
- try {
- if (ZKUtil.checkExists(zookeeper, peerNode) == -1) {
- LOG.info("Adding peer {} to hfile reference queue.", peerId);
- ZKUtil.createWithParents(zookeeper, peerNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to add peer " + peerId + " to hfile reference queue.",
- e);
- }
- }
-
- @Override
- public void removePeerFromHFileRefs(String peerId) throws ReplicationException {
- String peerNode = getHFileRefsPeerNode(peerId);
- try {
- if (ZKUtil.checkExists(zookeeper, peerNode) == -1) {
- LOG.debug("Peer {} not found in hfile reference queue.", peerNode);
- } else {
- LOG.info("Removing peer {} from hfile reference queue.", peerNode);
- ZKUtil.deleteNodeRecursively(zookeeper, peerNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException(
- "Failed to remove peer " + peerId + " from hfile reference queue.", e);
- }
- }
-
- @Override
- public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs)
- throws ReplicationException {
- String peerNode = getHFileRefsPeerNode(peerId);
- LOG.debug("Adding hfile references {} in queue {}", pairs, peerNode);
- List<ZKUtilOp> listOfOps =
- pairs.stream().map(p -> p.getSecond().getName()).map(n -> getHFileNode(peerNode, n))
- .map(f -> ZKUtilOp.createAndFailSilent(f, HConstants.EMPTY_BYTE_ARRAY)).collect(toList());
- LOG.debug("The multi list size for adding hfile references in zk for node {} is {}", peerNode,
- listOfOps.size());
- try {
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to add hfile reference to peer " + peerId, e);
- }
- }
-
- @Override
- public void removeHFileRefs(String peerId, List<String> files) throws ReplicationException {
- String peerNode = getHFileRefsPeerNode(peerId);
- LOG.debug("Removing hfile references {} from queue {}", files, peerNode);
-
- List<ZKUtilOp> listOfOps = files.stream().map(n -> getHFileNode(peerNode, n))
- .map(ZKUtilOp::deleteNodeFailSilent).collect(toList());
- LOG.debug("The multi list size for removing hfile references in zk for node {} is {}", peerNode,
- listOfOps.size());
- try {
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to remove hfile reference from peer " + peerId, e);
- }
- }
-
- private List<String> getAllPeersFromHFileRefsQueue0() throws KeeperException {
- List<String> children = ZKUtil.listChildrenNoWatch(zookeeper, hfileRefsZNode);
- return children != null ? children : Collections.emptyList();
- }
-
- @Override
- public List<String> getAllPeersFromHFileRefsQueue() throws ReplicationException {
- try {
- return getAllPeersFromHFileRefsQueue0();
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get list of all peers in hfile references node.",
- e);
- }
- }
-
- private List<String> getReplicableHFiles0(String peerId) throws KeeperException {
- List<String> children =
- ZKUtil.listChildrenNoWatch(this.zookeeper, getHFileRefsPeerNode(peerId));
- return children != null ? children : Collections.emptyList();
- }
-
- @Override
- public List<String> getReplicableHFiles(String peerId) throws ReplicationException {
- try {
- return getReplicableHFiles0(peerId);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get list of hfile references for peer " + peerId,
- e);
- }
- }
-
- // will be overridden in UTs
- protected int getHFileRefsZNodeCversion() throws ReplicationException {
- Stat stat = new Stat();
- try {
- ZKUtil.getDataNoWatch(zookeeper, hfileRefsZNode, stat);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get stat of replication hfile references node.", e);
- }
- return stat.getCversion();
- }
-
- @Override
- public Set<String> getAllHFileRefs() throws ReplicationException {
- try {
- for (int retry = 0;; retry++) {
- int v0 = getHFileRefsZNodeCversion();
- List<String> peers = getAllPeersFromHFileRefsQueue();
- if (peers.isEmpty()) {
- LOG.debug("Didn't find any peers with hfile references, won't prevent deletions.");
- return Collections.emptySet();
- }
- Set<String> hfileRefs = new HashSet<>();
- for (String peer : peers) {
- hfileRefs.addAll(getReplicableHFiles0(peer));
- }
- int v1 = getHFileRefsZNodeCversion();
- if (v0 == v1) {
- return hfileRefs;
- }
- LOG.debug("Replication hfile references node cversion changed from %d to %d, retry = %d",
- v0, v1, retry);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to get all hfile refs", e);
- }
- }
-}
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
index 15cf5b1f1f6..dc46e4f1c7c 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateBasic.java
@@ -17,30 +17,18 @@
*/
package org.apache.hadoop.hbase.replication;
-import static org.hamcrest.CoreMatchers.hasItems;
-import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
-import org.apache.zookeeper.KeeperException;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
-
/**
* White box testing for replication state interfaces. Implementations should extend this class, and
* initialize the interfaces properly.
@@ -49,7 +37,6 @@ public abstract class TestReplicationStateBasic {
private static final Logger LOG = LoggerFactory.getLogger(TestReplicationStateBasic.class);
- protected ReplicationQueueStorage rqs;
protected ServerName server1 = ServerName.valueOf("hostname1.example.org", 1234, 12345);
protected ServerName server2 = ServerName.valueOf("hostname2.example.org", 1234, 12345);
protected ServerName server3 = ServerName.valueOf("hostname3.example.org", 1234, 12345);
@@ -66,161 +53,6 @@ public abstract class TestReplicationStateBasic {
protected static final int ZK_MAX_COUNT = 300;
protected static final int ZK_SLEEP_INTERVAL = 100; // millis
- @Test
- public void testReplicationQueueStorage() throws ReplicationException {
- // Test methods with empty state
- assertEquals(0, rqs.getListOfReplicators().size());
- assertTrue(rqs.getWALsInQueue(server1, "qId1").isEmpty());
- assertTrue(rqs.getAllQueues(server1).isEmpty());
-
- /*
- * Set up data Two replicators: -- server1: three queues with 0, 1 and 2 log files each --
- * server2: zero queues
- */
- rqs.addWAL(server1, "qId1", "trash");
- rqs.removeWAL(server1, "qId1", "trash");
- rqs.addWAL(server1, "qId2", "filename1");
- rqs.addWAL(server1, "qId3", "filename2");
- rqs.addWAL(server1, "qId3", "filename3");
- rqs.addWAL(server2, "trash", "trash");
- rqs.removeQueue(server2, "trash");
-
- List<ServerName> reps = rqs.getListOfReplicators();
- assertEquals(2, reps.size());
- assertTrue(server1.getServerName(), reps.contains(server1));
- assertTrue(server2.getServerName(), reps.contains(server2));
-
- assertTrue(rqs.getWALsInQueue(ServerName.valueOf("bogus", 12345, 12345), "bogus").isEmpty());
- assertTrue(rqs.getWALsInQueue(server1, "bogus").isEmpty());
- assertEquals(0, rqs.getWALsInQueue(server1, "qId1").size());
- assertEquals(1, rqs.getWALsInQueue(server1, "qId2").size());
- assertEquals("filename1", rqs.getWALsInQueue(server1, "qId2").get(0));
-
- assertTrue(rqs.getAllQueues(ServerName.valueOf("bogus", 12345, -1L)).isEmpty());
- assertEquals(0, rqs.getAllQueues(server2).size());
- List<String> list = rqs.getAllQueues(server1);
- assertEquals(3, list.size());
- assertTrue(list.contains("qId2"));
- assertTrue(list.contains("qId3"));
- }
-
- private void removeAllQueues(ServerName serverName) throws ReplicationException {
- for (String queue : rqs.getAllQueues(serverName)) {
- rqs.removeQueue(serverName, queue);
- }
- }
-
- @Test
- public void testReplicationQueues() throws ReplicationException {
- // Initialize ReplicationPeer so we can add peers (we don't transfer lone queues)
- rp.init();
-
- rqs.removeQueue(server1, "bogus");
- rqs.removeWAL(server1, "bogus", "bogus");
- removeAllQueues(server1);
- assertEquals(0, rqs.getAllQueues(server1).size());
- assertEquals(0, rqs.getWALPosition(server1, "bogus", "bogus"));
- assertTrue(rqs.getWALsInQueue(server1, "bogus").isEmpty());
- assertTrue(rqs.getAllQueues(ServerName.valueOf("bogus", 1234, 12345)).isEmpty());
-
- populateQueues();
-
- assertEquals(3, rqs.getListOfReplicators().size());
- assertEquals(0, rqs.getWALsInQueue(server2, "qId1").size());
- assertEquals(5, rqs.getWALsInQueue(server3, "qId5").size());
- assertEquals(0, rqs.getWALPosition(server3, "qId1", "filename0"));
- rqs.setWALPosition(server3, "qId5", "filename4", 354L, Collections.emptyMap());
- assertEquals(354L, rqs.getWALPosition(server3, "qId5", "filename4"));
-
- assertEquals(5, rqs.getWALsInQueue(server3, "qId5").size());
- assertEquals(0, rqs.getWALsInQueue(server2, "qId1").size());
- assertEquals(0, rqs.getAllQueues(server1).size());
- assertEquals(1, rqs.getAllQueues(server2).size());
- assertEquals(5, rqs.getAllQueues(server3).size());
-
- assertEquals(0, rqs.getAllQueues(server1).size());
- rqs.removeReplicatorIfQueueIsEmpty(server1);
- assertEquals(2, rqs.getListOfReplicators().size());
-
- List<String> queues = rqs.getAllQueues(server3);
- assertEquals(5, queues.size());
- for (String queue : queues) {
- rqs.claimQueue(server3, queue, server2);
- }
- rqs.removeReplicatorIfQueueIsEmpty(server3);
- assertEquals(1, rqs.getListOfReplicators().size());
-
- assertEquals(6, rqs.getAllQueues(server2).size());
- removeAllQueues(server2);
- rqs.removeReplicatorIfQueueIsEmpty(server2);
- assertEquals(0, rqs.getListOfReplicators().size());
- }
-
- @Test
- public void testHfileRefsReplicationQueues() throws ReplicationException, KeeperException {
- rp.init();
-
- List<Pair<Path, Path>> files1 = new ArrayList<>(3);
- files1.add(new Pair<>(null, new Path("file_1")));
- files1.add(new Pair<>(null, new Path("file_2")));
- files1.add(new Pair<>(null, new Path("file_3")));
- assertTrue(rqs.getReplicableHFiles(ID_ONE).isEmpty());
- assertEquals(0, rqs.getAllPeersFromHFileRefsQueue().size());
- rp.getPeerStorage().addPeer(ID_ONE,
- ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build(), true,
- SyncReplicationState.NONE);
- rqs.addPeerToHFileRefs(ID_ONE);
- rqs.addHFileRefs(ID_ONE, files1);
- assertEquals(1, rqs.getAllPeersFromHFileRefsQueue().size());
- assertEquals(3, rqs.getReplicableHFiles(ID_ONE).size());
- List<String> hfiles2 = new ArrayList<>(files1.size());
- for (Pair<Path, Path> p : files1) {
- hfiles2.add(p.getSecond().getName());
- }
- String removedString = hfiles2.remove(0);
- rqs.removeHFileRefs(ID_ONE, hfiles2);
- assertEquals(1, rqs.getReplicableHFiles(ID_ONE).size());
- hfiles2 = new ArrayList<>(1);
- hfiles2.add(removedString);
- rqs.removeHFileRefs(ID_ONE, hfiles2);
- assertEquals(0, rqs.getReplicableHFiles(ID_ONE).size());
- rp.getPeerStorage().removePeer(ID_ONE);
- }
-
- @Test
- public void testRemovePeerForHFileRefs() throws ReplicationException, KeeperException {
- rp.init();
- rp.getPeerStorage().addPeer(ID_ONE,
- ReplicationPeerConfig.newBuilder().setClusterKey(KEY_ONE).build(), true,
- SyncReplicationState.NONE);
- rqs.addPeerToHFileRefs(ID_ONE);
- rp.getPeerStorage().addPeer(ID_TWO,
- ReplicationPeerConfig.newBuilder().setClusterKey(KEY_TWO).build(), true,
- SyncReplicationState.NONE);
- rqs.addPeerToHFileRefs(ID_TWO);
-
- List<Pair<Path, Path>> files1 = new ArrayList<>(3);
- files1.add(new Pair<>(null, new Path("file_1")));
- files1.add(new Pair<>(null, new Path("file_2")));
- files1.add(new Pair<>(null, new Path("file_3")));
- rqs.addHFileRefs(ID_ONE, files1);
- rqs.addHFileRefs(ID_TWO, files1);
- assertEquals(2, rqs.getAllPeersFromHFileRefsQueue().size());
- assertEquals(3, rqs.getReplicableHFiles(ID_ONE).size());
- assertEquals(3, rqs.getReplicableHFiles(ID_TWO).size());
-
- rp.getPeerStorage().removePeer(ID_ONE);
- rqs.removePeerFromHFileRefs(ID_ONE);
- assertEquals(1, rqs.getAllPeersFromHFileRefsQueue().size());
- assertTrue(rqs.getReplicableHFiles(ID_ONE).isEmpty());
- assertEquals(3, rqs.getReplicableHFiles(ID_TWO).size());
-
- rp.getPeerStorage().removePeer(ID_TWO);
- rqs.removePeerFromHFileRefs(ID_TWO);
- assertEquals(0, rqs.getAllPeersFromHFileRefsQueue().size());
- assertTrue(rqs.getReplicableHFiles(ID_TWO).isEmpty());
- }
-
@Test
public void testReplicationPeers() throws Exception {
rp.init();
@@ -286,55 +118,7 @@ public abstract class TestReplicationStateBasic {
assertNumberOfPeers(2);
}
- private String getFileName(String base, int i) {
- return String.format(base + "-%04d", i);
- }
-
- @Test
- public void testPersistLogPositionAndSeqIdAtomically() throws Exception {
- ServerName serverName1 = ServerName.valueOf("127.0.0.1", 8000, 10000);
- assertTrue(rqs.getAllQueues(serverName1).isEmpty());
- String queue1 = "1";
- String region0 = "6b2c8f8555335cc9af74455b94516cbe",
- region1 = "6ecd2e9e010499f8ddef97ee8f70834f";
- for (int i = 0; i < 10; i++) {
- rqs.addWAL(serverName1, queue1, getFileName("file1", i));
- }
- List<String> queueIds = rqs.getAllQueues(serverName1);
- assertEquals(1, queueIds.size());
- assertThat(queueIds, hasItems("1"));
-
- List<String> wals1 = rqs.getWALsInQueue(serverName1, queue1);
- assertEquals(10, wals1.size());
- for (int i = 0; i < 10; i++) {
- assertThat(wals1, hasItems(getFileName("file1", i)));
- }
-
- for (int i = 0; i < 10; i++) {
- assertEquals(0, rqs.getWALPosition(serverName1, queue1, getFileName("file1", i)));
- }
- assertEquals(HConstants.NO_SEQNUM, rqs.getLastSequenceId(region0, queue1));
- assertEquals(HConstants.NO_SEQNUM, rqs.getLastSequenceId(region1, queue1));
-
- for (int i = 0; i < 10; i++) {
- rqs.setWALPosition(serverName1, queue1, getFileName("file1", i), (i + 1) * 100,
- ImmutableMap.of(region0, i * 100L, region1, (i + 1) * 100L));
- }
-
- for (int i = 0; i < 10; i++) {
- assertEquals((i + 1) * 100, rqs.getWALPosition(serverName1, queue1, getFileName("file1", i)));
- }
- assertEquals(900L, rqs.getLastSequenceId(region0, queue1));
- assertEquals(1000L, rqs.getLastSequenceId(region1, queue1));
-
- // Try to decrease the last pushed id by setWALPosition method.
- rqs.setWALPosition(serverName1, queue1, getFileName("file1", 0), 11 * 100,
- ImmutableMap.of(region0, 899L, region1, 1001L));
- assertEquals(900L, rqs.getLastSequenceId(region0, queue1));
- assertEquals(1001L, rqs.getLastSequenceId(region1, queue1));
- }
-
- protected void assertConnectedPeerStatus(boolean status, String peerId) throws Exception {
+ private void assertConnectedPeerStatus(boolean status, String peerId) throws Exception {
// we can first check if the value was changed in the store, if it wasn't then fail right away
if (status != rp.getPeerStorage().isPeerEnabled(peerId)) {
fail("ConnectedPeerStatus was " + !status + " but expected " + status + " in ZK");
@@ -353,30 +137,7 @@ public abstract class TestReplicationStateBasic {
}
}
- protected void assertNumberOfPeers(int total) throws ReplicationException {
+ private void assertNumberOfPeers(int total) throws ReplicationException {
assertEquals(total, rp.getPeerStorage().listPeerIds().size());
}
-
- /*
- * three replicators: rq1 has 0 queues, rq2 has 1 queue with no logs, rq3 has 5 queues with 1, 2,
- * 3, 4, 5 log files respectively
- */
- protected void populateQueues() throws ReplicationException {
- rqs.addWAL(server1, "trash", "trash");
- rqs.removeQueue(server1, "trash");
-
- rqs.addWAL(server2, "qId1", "trash");
- rqs.removeWAL(server2, "qId1", "trash");
-
- for (int i = 1; i < 6; i++) {
- for (int j = 0; j < i; j++) {
- rqs.addWAL(server3, "qId" + i, "filename" + j);
- }
- // Add peers for the corresponding queues so they are not orphans
- rp.getPeerStorage().addPeer("qId" + i,
- ReplicationPeerConfig.newBuilder()
- .setClusterKey(MiniZooKeeperCluster.HOST + ":2818:/bogus" + i).build(),
- true, SyncReplicationState.NONE);
- }
- }
}
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
index d2540987906..c5c8769282a 100644
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
+++ b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationStateZKImpl.java
@@ -80,7 +80,6 @@ public class TestReplicationStateZKImpl extends TestReplicationStateBasic {
@Before
public void setUp() throws IOException {
zkTimeoutCount = 0;
- rqs = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
rp =
ReplicationFactory.getReplicationPeers(FileSystem.get(utility.getConfiguration()), zkw, conf);
OUR_KEY = ZKConfig.getZooKeeperClusterKey(conf);
diff --git a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java b/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
deleted file mode 100644
index ccd3c17f3bc..00000000000
--- a/hbase-replication/src/test/java/org/apache/hadoop/hbase/replication/TestZKReplicationQueueStorage.java
+++ /dev/null
@@ -1,341 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import static org.hamcrest.CoreMatchers.hasItems;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.SortedSet;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseZKTestingUtil;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.ReplicationTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.MD5Hash;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.zookeeper.KeeperException;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
-
-@Category({ ReplicationTests.class, MediumTests.class })
-public class TestZKReplicationQueueStorage {
-
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestZKReplicationQueueStorage.class);
-
- private static final HBaseZKTestingUtil UTIL = new HBaseZKTestingUtil();
-
- private static ZKReplicationQueueStorage STORAGE;
-
- @BeforeClass
- public static void setUp() throws Exception {
- UTIL.startMiniZKCluster();
- STORAGE = new ZKReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
- }
-
- @AfterClass
- public static void tearDown() throws IOException {
- UTIL.shutdownMiniZKCluster();
- }
-
- @After
- public void tearDownAfterTest() throws ReplicationException, KeeperException, IOException {
- for (ServerName serverName : STORAGE.getListOfReplicators()) {
- for (String queue : STORAGE.getAllQueues(serverName)) {
- STORAGE.removeQueue(serverName, queue);
- }
- STORAGE.removeReplicatorIfQueueIsEmpty(serverName);
- }
- for (String peerId : STORAGE.getAllPeersFromHFileRefsQueue()) {
- STORAGE.removePeerFromHFileRefs(peerId);
- }
- }
-
- private ServerName getServerName(int i) {
- return ServerName.valueOf("127.0.0.1", 8000 + i, 10000 + i);
- }
-
- @Test
- public void testReplicator() throws ReplicationException {
- assertTrue(STORAGE.getListOfReplicators().isEmpty());
- String queueId = "1";
- for (int i = 0; i < 10; i++) {
- STORAGE.addWAL(getServerName(i), queueId, "file" + i);
- }
- List<ServerName> replicators = STORAGE.getListOfReplicators();
- assertEquals(10, replicators.size());
- for (int i = 0; i < 10; i++) {
- assertThat(replicators, hasItems(getServerName(i)));
- }
- for (int i = 0; i < 5; i++) {
- STORAGE.removeQueue(getServerName(i), queueId);
- }
- for (int i = 0; i < 10; i++) {
- STORAGE.removeReplicatorIfQueueIsEmpty(getServerName(i));
- }
- replicators = STORAGE.getListOfReplicators();
- assertEquals(5, replicators.size());
- for (int i = 5; i < 10; i++) {
- assertThat(replicators, hasItems(getServerName(i)));
- }
- }
-
- private String getFileName(String base, int i) {
- return String.format(base + "-%04d", i);
- }
-
- @Test
- public void testAddRemoveLog() throws ReplicationException {
- ServerName serverName1 = ServerName.valueOf("127.0.0.1", 8000, 10000);
- assertTrue(STORAGE.getAllQueues(serverName1).isEmpty());
- String queue1 = "1";
- String queue2 = "2";
- for (int i = 0; i < 10; i++) {
- STORAGE.addWAL(serverName1, queue1, getFileName("file1", i));
- STORAGE.addWAL(serverName1, queue2, getFileName("file2", i));
- }
- List<String> queueIds = STORAGE.getAllQueues(serverName1);
- assertEquals(2, queueIds.size());
- assertThat(queueIds, hasItems("1", "2"));
-
- List<String> wals1 = STORAGE.getWALsInQueue(serverName1, queue1);
- List<String> wals2 = STORAGE.getWALsInQueue(serverName1, queue2);
- assertEquals(10, wals1.size());
- assertEquals(10, wals2.size());
- for (int i = 0; i < 10; i++) {
- assertThat(wals1, hasItems(getFileName("file1", i)));
- assertThat(wals2, hasItems(getFileName("file2", i)));
- }
-
- for (int i = 0; i < 10; i++) {
- assertEquals(0, STORAGE.getWALPosition(serverName1, queue1, getFileName("file1", i)));
- assertEquals(0, STORAGE.getWALPosition(serverName1, queue2, getFileName("file2", i)));
- STORAGE.setWALPosition(serverName1, queue1, getFileName("file1", i), (i + 1) * 100,
- Collections.emptyMap());
- STORAGE.setWALPosition(serverName1, queue2, getFileName("file2", i), (i + 1) * 100 + 10,
- Collections.emptyMap());
- }
-
- for (int i = 0; i < 10; i++) {
- assertEquals((i + 1) * 100,
- STORAGE.getWALPosition(serverName1, queue1, getFileName("file1", i)));
- assertEquals((i + 1) * 100 + 10,
- STORAGE.getWALPosition(serverName1, queue2, getFileName("file2", i)));
- }
-
- for (int i = 0; i < 10; i++) {
- if (i % 2 == 0) {
- STORAGE.removeWAL(serverName1, queue1, getFileName("file1", i));
- } else {
- STORAGE.removeWAL(serverName1, queue2, getFileName("file2", i));
- }
- }
-
- queueIds = STORAGE.getAllQueues(serverName1);
- assertEquals(2, queueIds.size());
- assertThat(queueIds, hasItems("1", "2"));
-
- ServerName serverName2 = ServerName.valueOf("127.0.0.1", 8001, 10001);
- Pair<String, SortedSet<String>> peer1 = STORAGE.claimQueue(serverName1, "1", serverName2);
-
- assertEquals("1-" + serverName1.getServerName(), peer1.getFirst());
- assertEquals(5, peer1.getSecond().size());
- int i = 1;
- for (String wal : peer1.getSecond()) {
- assertEquals(getFileName("file1", i), wal);
- assertEquals((i + 1) * 100,
- STORAGE.getWALPosition(serverName2, peer1.getFirst(), getFileName("file1", i)));
- i += 2;
- }
-
- queueIds = STORAGE.getAllQueues(serverName1);
- assertEquals(1, queueIds.size());
- assertThat(queueIds, hasItems("2"));
- wals2 = STORAGE.getWALsInQueue(serverName1, queue2);
- assertEquals(5, wals2.size());
- for (i = 0; i < 10; i += 2) {
- assertThat(wals2, hasItems(getFileName("file2", i)));
- }
-
- queueIds = STORAGE.getAllQueues(serverName2);
- assertEquals(1, queueIds.size());
- assertThat(queueIds, hasItems(peer1.getFirst()));
- wals1 = STORAGE.getWALsInQueue(serverName2, peer1.getFirst());
- assertEquals(5, wals1.size());
- for (i = 1; i < 10; i += 2) {
- assertThat(wals1, hasItems(getFileName("file1", i)));
- }
-
- Set<String> allWals = STORAGE.getAllWALs();
- assertEquals(10, allWals.size());
- for (i = 0; i < 10; i++) {
- assertThat(allWals, hasItems(i % 2 == 0 ? getFileName("file2", i) : getFileName("file1", i)));
- }
- }
-
- // For HBASE-12865, HBASE-26482
- @Test
- public void testClaimQueueChangeCversion() throws ReplicationException, KeeperException {
- ServerName serverName1 = ServerName.valueOf("127.0.0.1", 8000, 10000);
- STORAGE.addWAL(serverName1, "1", "file");
- STORAGE.addWAL(serverName1, "2", "file");
-
- ServerName serverName2 = ServerName.valueOf("127.0.0.1", 8001, 10001);
- // Avoid claimQueue update cversion for prepare server2 rsNode.
- STORAGE.addWAL(serverName2, "1", "file");
- STORAGE.addWAL(serverName2, "2", "file");
-
- int v0 = STORAGE.getQueuesZNodeCversion();
-
- STORAGE.claimQueue(serverName1, "1", serverName2);
- int v1 = STORAGE.getQueuesZNodeCversion();
- // cversion should be increased by claimQueue method.
- assertTrue(v1 > v0);
-
- STORAGE.claimQueue(serverName1, "2", serverName2);
- int v2 = STORAGE.getQueuesZNodeCversion();
- // cversion should be increased by claimQueue method.
- assertTrue(v2 > v1);
- }
-
- private ZKReplicationQueueStorage createWithUnstableVersion() throws IOException {
- return new ZKReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration()) {
-
- private int called = 0;
- private int getLastSeqIdOpIndex = 0;
-
- @Override
- protected int getQueuesZNodeCversion() throws KeeperException {
- if (called < 4) {
- called++;
- }
- return called;
- }
-
- @Override
- protected Pair<Long, Integer> getLastSequenceIdWithVersion(String encodedRegionName,
- String peerId) throws KeeperException {
- Pair<Long, Integer> oldPair = super.getLastSequenceIdWithVersion(encodedRegionName, peerId);
- if (getLastSeqIdOpIndex < 100) {
- // Let the ZNode version increase.
- String path = getSerialReplicationRegionPeerNode(encodedRegionName, peerId);
- ZKUtil.createWithParents(zookeeper, path);
- ZKUtil.setData(zookeeper, path, ZKUtil.positionToByteArray(100L));
- }
- getLastSeqIdOpIndex++;
- return oldPair;
- }
- };
- }
-
- @Test
- public void testGetAllWALsCversionChange() throws IOException, ReplicationException {
- ZKReplicationQueueStorage storage = createWithUnstableVersion();
- storage.addWAL(getServerName(0), "1", "file");
- // This should return eventually when cversion stabilizes
- Set<String> allWals = storage.getAllWALs();
- assertEquals(1, allWals.size());
- assertThat(allWals, hasItems("file"));
- }
-
- // For HBASE-14621
- @Test
- public void testGetAllHFileRefsCversionChange() throws IOException, ReplicationException {
- ZKReplicationQueueStorage storage = createWithUnstableVersion();
- storage.addPeerToHFileRefs("1");
- Path p = new Path("/test");
- storage.addHFileRefs("1", Arrays.asList(Pair.newPair(p, p)));
- // This should return eventually when cversion stabilizes
- Set<String> allHFileRefs = storage.getAllHFileRefs();
- assertEquals(1, allHFileRefs.size());
- assertThat(allHFileRefs, hasItems("test"));
- }
-
- // For HBASE-20138
- @Test
- public void testSetWALPositionBadVersion() throws IOException, ReplicationException {
- ZKReplicationQueueStorage storage = createWithUnstableVersion();
- ServerName serverName1 = ServerName.valueOf("128.0.0.1", 8000, 10000);
- assertTrue(storage.getAllQueues(serverName1).isEmpty());
- String queue1 = "1";
- String fileName = getFileName("file1", 0);
- String encodedRegionName = "31d9792f4435b99d9fb1016f6fbc8dc6";
- storage.addWAL(serverName1, queue1, fileName);
-
- List<String> wals1 = storage.getWALsInQueue(serverName1, queue1);
- assertEquals(1, wals1.size());
-
- assertEquals(0, storage.getWALPosition(serverName1, queue1, fileName));
- // This should return eventually when data version stabilizes
- storage.setWALPosition(serverName1, queue1, fileName, 100,
- ImmutableMap.of(encodedRegionName, 120L));
-
- assertEquals(100, storage.getWALPosition(serverName1, queue1, fileName));
- assertEquals(120L, storage.getLastSequenceId(encodedRegionName, queue1));
- }
-
- @Test
- public void testRegionsZNodeLayout() throws Exception {
- String peerId = "1";
- String encodedRegionName = "31d9792f4435b99d9fb1016f6fbc8dc7";
- String expectedPath = "/hbase/replication/regions/31/d9/792f4435b99d9fb1016f6fbc8dc7-" + peerId;
- String path = STORAGE.getSerialReplicationRegionPeerNode(encodedRegionName, peerId);
- assertEquals(expectedPath, path);
- }
-
- @Test
- public void testRemoveAllLastPushedSeqIdsForPeer() throws Exception {
- String peerId = "1";
- String peerIdToDelete = "2";
- for (int i = 0; i < 100; i++) {
- String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
- STORAGE.setLastSequenceIds(peerId, ImmutableMap.of(encodedRegionName, (long) i));
- STORAGE.setLastSequenceIds(peerIdToDelete, ImmutableMap.of(encodedRegionName, (long) i));
- }
- for (int i = 0; i < 100; i++) {
- String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
- assertEquals(i, STORAGE.getLastSequenceId(encodedRegionName, peerId));
- assertEquals(i, STORAGE.getLastSequenceId(encodedRegionName, peerIdToDelete));
- }
- STORAGE.removeLastSequenceIds(peerIdToDelete);
- for (int i = 0; i < 100; i++) {
- String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
- assertEquals(i, STORAGE.getLastSequenceId(encodedRegionName, peerId));
- assertEquals(HConstants.NO_SEQNUM,
- STORAGE.getLastSequenceId(encodedRegionName, peerIdToDelete));
- }
- }
-}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f3d4dfb292d..896f9a5d086 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -788,8 +788,7 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
}
this.rsGroupInfoManager = RSGroupInfoManager.create(this);
- this.replicationPeerManager =
- ReplicationPeerManager.create(fileSystemManager.getFileSystem(), zooKeeper, conf, clusterId);
+ this.replicationPeerManager = ReplicationPeerManager.create(this, clusterId);
this.configurationManager.registerObserver(replicationPeerManager);
this.replicationPeerModificationStateStore =
new ReplicationPeerModificationStateStore(masterRegion);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
index 0bd0f3ba0c7..487c45e5c5c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.master.SplitWALManager;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
-import org.apache.hadoop.hbase.master.replication.ClaimReplicationQueuesProcedure;
+import org.apache.hadoop.hbase.master.replication.AssignReplicationQueuesProcedure;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.monitoring.TaskMonitor;
import org.apache.hadoop.hbase.procedure2.Procedure;
@@ -240,15 +240,33 @@ public class ServerCrashProcedure extends
}
assignRegions(env, regionsOnCrashedServer);
}
- setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
+ // If there is no replication peer, we do not need to enter the claim queues stage.
+ // This is also very important that now we will later initialize ReplicationQueueStorage
+ // so if there is no replication peer added yet, the storage can not be accessed.
+ // And there will be no race because:
+ // 1. For adding replication peer, if the peer storage has not been updated yet, the crash
+ // region server will not have any replication queues for this peer, so it is safe to skip
+ // claiming.
+ // 2. For removing replication peer, it it has already updated the peer storage, then
+ // there is no way to rollback and region servers are already started to close and delete
+ // replication queues, so it is also safe to skip claiming.
+ if (env.getReplicationPeerManager().listPeers(null).isEmpty()) {
+ setNextState(ServerCrashState.SERVER_CRASH_FINISH);
+ } else {
+ setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
+ }
break;
case SERVER_CRASH_HANDLE_RIT2:
// Noop. Left in place because we used to call handleRIT here for a second time
// but no longer necessary since HBASE-20634.
- setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
+ if (env.getReplicationPeerManager().listPeers(null).isEmpty()) {
+ setNextState(ServerCrashState.SERVER_CRASH_FINISH);
+ } else {
+ setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
+ }
break;
case SERVER_CRASH_CLAIM_REPLICATION_QUEUES:
- addChildProcedure(new ClaimReplicationQueuesProcedure(serverName));
+ addChildProcedure(new AssignReplicationQueuesProcedure(serverName));
setNextState(ServerCrashState.SERVER_CRASH_FINISH);
break;
case SERVER_CRASH_FINISH:
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
new file mode 100644
index 00000000000..e7fb5e51715
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/AssignReplicationQueuesProcedure.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.replication;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface;
+import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
+import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
+import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.util.RetryCounter;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.AssignReplicationQueuesState;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.AssignReplicationQueuesStateData;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+
+@InterfaceAudience.Private
+public class AssignReplicationQueuesProcedure
+ extends StateMachineProcedure<MasterProcedureEnv, AssignReplicationQueuesState>
+ implements ServerProcedureInterface {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AssignReplicationQueuesProcedure.class);
+
+ private ServerName crashedServer;
+
+ private RetryCounter retryCounter;
+
+ public AssignReplicationQueuesProcedure() {
+ }
+
+ public AssignReplicationQueuesProcedure(ServerName crashedServer) {
+ this.crashedServer = crashedServer;
+ }
+
+ @Override
+ public ServerName getServerName() {
+ return crashedServer;
+ }
+
+ @Override
+ public boolean hasMetaTableRegion() {
+ return false;
+ }
+
+ @Override
+ public ServerOperationType getServerOperationType() {
+ return ServerOperationType.CLAIM_REPLICATION_QUEUES;
+ }
+
+ private void addMissingQueues(MasterProcedureEnv env) throws ReplicationException {
+ ReplicationQueueStorage storage = env.getReplicationPeerManager().getQueueStorage();
+
+ Set<String> existingQueuePeerIds = new HashSet<>();
+ List<ReplicationQueueId> queueIds = storage.listAllQueueIds(crashedServer);
+ for (Iterator<ReplicationQueueId> iter = queueIds.iterator(); iter.hasNext();) {
+ ReplicationQueueId queueId = iter.next();
+ if (!queueId.isRecovered()) {
+ existingQueuePeerIds.add(queueId.getPeerId());
+ }
+ }
+ List<ReplicationPeerDescription> peers = env.getReplicationPeerManager().listPeers(null);
+ for (ReplicationPeerDescription peer : peers) {
+ if (!existingQueuePeerIds.contains(peer.getPeerId())) {
+ ReplicationQueueId queueId = new ReplicationQueueId(crashedServer, peer.getPeerId());
+ LOG.debug("Add replication queue {} for claiming", queueId);
+ env.getReplicationPeerManager().getQueueStorage().setOffset(queueId,
+ crashedServer.toString(), ReplicationGroupOffset.BEGIN, Collections.emptyMap());
+ }
+ }
+ }
+
+ private Flow claimQueues(MasterProcedureEnv env) throws ReplicationException {
+ ReplicationQueueStorage storage = env.getReplicationPeerManager().getQueueStorage();
+ List<ReplicationQueueId> queueIds = storage.listAllQueueIds(crashedServer);
+ if (queueIds.isEmpty()) {
+ LOG.debug("Finish claiming replication queues for {}", crashedServer);
+ // we are done
+ return Flow.NO_MORE_STATE;
+ }
+ LOG.debug("There are {} replication queues need to be claimed for {}", queueIds.size(),
+ crashedServer);
+ List<ServerName> targetServers =
+ env.getMasterServices().getServerManager().getOnlineServersList();
+ if (targetServers.isEmpty()) {
+ throw new ReplicationException("no region server available");
+ }
+ Collections.shuffle(targetServers);
+ for (int i = 0, n = Math.min(queueIds.size(), targetServers.size()); i < n; i++) {
+ addChildProcedure(
+ new ClaimReplicationQueueRemoteProcedure(queueIds.get(i), targetServers.get(i)));
+ }
+ retryCounter = null;
+ return Flow.HAS_MORE_STATE;
+ }
+
+ @Override
+ protected Flow executeFromState(MasterProcedureEnv env, AssignReplicationQueuesState state)
+ throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+ try {
+ switch (state) {
+ case ASSIGN_REPLICATION_QUEUES_PRE_CHECK:
+ // TODO: reserved for implementing the fencing logic with Add/Remove/UpdatePeerProcedure
+ setNextState(AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES);
+ return Flow.HAS_MORE_STATE;
+ case ASSIGN_REPLICATION_QUEUES_ADD_MISSING_QUEUES:
+ addMissingQueues(env);
+ retryCounter = null;
+ setNextState(AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_CLAIM);
+ return Flow.HAS_MORE_STATE;
+ case ASSIGN_REPLICATION_QUEUES_CLAIM:
+ return claimQueues(env);
+ default:
+ throw new UnsupportedOperationException("unhandled state=" + state);
+ }
+ } catch (Exception e) {
+ if (retryCounter == null) {
+ retryCounter = ProcedureUtil.createRetryCounter(env.getMasterConfiguration());
+ }
+ long backoff = retryCounter.getBackoffTimeAndIncrementAttempts();
+ LOG.warn("Failed to claim replication queues for {}, suspend {}secs {}; {};", crashedServer,
+ backoff / 1000, e);
+ setTimeout(Math.toIntExact(backoff));
+ setState(ProcedureProtos.ProcedureState.WAITING_TIMEOUT);
+ skipPersistence();
+ throw new ProcedureSuspendedException();
+ }
+ }
+
+ @Override
+ protected synchronized boolean setTimeoutFailure(MasterProcedureEnv env) {
+ setState(ProcedureProtos.ProcedureState.RUNNABLE);
+ env.getProcedureScheduler().addFront(this);
+ return false;
+ }
+
+ @Override
+ protected void rollbackState(MasterProcedureEnv env, AssignReplicationQueuesState state)
+ throws IOException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected AssignReplicationQueuesState getState(int stateId) {
+ return AssignReplicationQueuesState.forNumber(stateId);
+ }
+
+ @Override
+ protected int getStateId(AssignReplicationQueuesState state) {
+ return state.getNumber();
+ }
+
+ @Override
+ protected AssignReplicationQueuesState getInitialState() {
+ return AssignReplicationQueuesState.ASSIGN_REPLICATION_QUEUES_PRE_CHECK;
+ }
+
+ @Override
+ protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.serializeStateData(serializer);
+ serializer.serialize(AssignReplicationQueuesStateData.newBuilder()
+ .setCrashedServer(ProtobufUtil.toServerName(crashedServer)).build());
+ }
+
+ @Override
+ protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
+ super.deserializeStateData(serializer);
+ AssignReplicationQueuesStateData proto =
+ serializer.deserialize(AssignReplicationQueuesStateData.class);
+ crashedServer = ProtobufUtil.toServerName(proto.getCrashedServer());
+ }
+
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
index 9ef97d1fff6..7b637384398 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueueRemoteProcedure.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.master.procedure.ServerRemoteProcedure;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteOperation;
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteProcedure;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.regionserver.ClaimReplicationQueueCallable;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -43,34 +44,33 @@ public class ClaimReplicationQueueRemoteProcedure extends ServerRemoteProcedure
private static final Logger LOG =
LoggerFactory.getLogger(ClaimReplicationQueueRemoteProcedure.class);
- private ServerName crashedServer;
-
- private String queue;
+ private ReplicationQueueId queueId;
public ClaimReplicationQueueRemoteProcedure() {
}
- public ClaimReplicationQueueRemoteProcedure(ServerName crashedServer, String queue,
- ServerName targetServer) {
- this.crashedServer = crashedServer;
- this.queue = queue;
+ public ClaimReplicationQueueRemoteProcedure(ReplicationQueueId queueId, ServerName targetServer) {
+ this.queueId = queueId;
this.targetServer = targetServer;
}
@Override
public Optional<RemoteOperation> remoteCallBuild(MasterProcedureEnv env, ServerName remote) {
assert targetServer.equals(remote);
+ ClaimReplicationQueueRemoteParameter.Builder builder = ClaimReplicationQueueRemoteParameter
+ .newBuilder().setCrashedServer(ProtobufUtil.toServerName(queueId.getServerName()))
+ .setQueue(queueId.getPeerId());
+ queueId.getSourceServerName()
+ .ifPresent(sourceServer -> builder.setSourceServer(ProtobufUtil.toServerName(sourceServer)));
return Optional.of(new ServerOperation(this, getProcId(), ClaimReplicationQueueCallable.class,
- ClaimReplicationQueueRemoteParameter.newBuilder()
- .setCrashedServer(ProtobufUtil.toServerName(crashedServer)).setQueue(queue).build()
- .toByteArray()));
+ builder.build().toByteArray()));
}
@Override
public ServerName getServerName() {
// return crashed server here, as we are going to recover its replication queues so we should
// use its scheduler queue instead of the one for the target server.
- return crashedServer;
+ return queueId.getServerName();
}
@Override
@@ -86,8 +86,7 @@ public class ClaimReplicationQueueRemoteProcedure extends ServerRemoteProcedure
@Override
protected void complete(MasterProcedureEnv env, Throwable error) {
if (error != null) {
- LOG.warn("Failed to claim replication queue {} of crashed server on server {} ", queue,
- crashedServer, targetServer, error);
+ LOG.warn("Failed to claim replication queue {} on server {} ", queueId, targetServer, error);
this.succ = false;
} else {
this.succ = true;
@@ -111,17 +110,26 @@ public class ClaimReplicationQueueRemoteProcedure extends ServerRemoteProcedure
@Override
protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
- serializer.serialize(ClaimReplicationQueueRemoteStateData.newBuilder()
- .setCrashedServer(ProtobufUtil.toServerName(crashedServer)).setQueue(queue)
- .setTargetServer(ProtobufUtil.toServerName(targetServer)).build());
+ ClaimReplicationQueueRemoteStateData.Builder builder = ClaimReplicationQueueRemoteStateData
+ .newBuilder().setCrashedServer(ProtobufUtil.toServerName(queueId.getServerName()))
+ .setQueue(queueId.getPeerId()).setTargetServer(ProtobufUtil.toServerName(targetServer));
+ queueId.getSourceServerName()
+ .ifPresent(sourceServer -> builder.setSourceServer(ProtobufUtil.toServerName(sourceServer)));
+ serializer.serialize(builder.build());
}
@Override
protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
ClaimReplicationQueueRemoteStateData data =
serializer.deserialize(ClaimReplicationQueueRemoteStateData.class);
- crashedServer = ProtobufUtil.toServerName(data.getCrashedServer());
- queue = data.getQueue();
targetServer = ProtobufUtil.toServerName(data.getTargetServer());
+ ServerName crashedServer = ProtobufUtil.toServerName(data.getCrashedServer());
+ String queue = data.getQueue();
+ if (data.hasSourceServer()) {
+ queueId = new ReplicationQueueId(crashedServer, queue,
+ ProtobufUtil.toServerName(data.getSourceServer()));
+ } else {
+ queueId = new ReplicationQueueId(crashedServer, queue);
+ }
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueuesProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueuesProcedure.java
index 5eb6608f4ee..747d352d2aa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueuesProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ClaimReplicationQueuesProcedure.java
@@ -19,8 +19,10 @@ package org.apache.hadoop.hbase.master.replication;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ServerProcedureInterface;
@@ -30,7 +32,9 @@ import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.util.RetryCounter;
import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
@@ -44,7 +48,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
/**
* Used to assign the replication queues of a dead server to other region servers.
+ * @deprecated Use {@link AssignReplicationQueuesProcedure} instead, kept only for keeping
+ * compatibility.
*/
+@Deprecated
@InterfaceAudience.Private
public class ClaimReplicationQueuesProcedure extends Procedure<MasterProcedureEnv>
implements ServerProcedureInterface {
@@ -82,22 +89,36 @@ public class ClaimReplicationQueuesProcedure extends Procedure<MasterProcedureEn
throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {
ReplicationQueueStorage storage = env.getReplicationPeerManager().getQueueStorage();
try {
- List<String> queues = storage.getAllQueues(crashedServer);
+ List<ReplicationQueueId> queues = storage.listAllQueueIds(crashedServer);
+ Set<String> existQueuePeerIds = new HashSet<>();
// this is for upgrading to the new region replication framework, where we will delete the
- // legacy region_replica_replication peer directly, without deleting the replication queues,
- // as it may still be used by region servers which have not been upgraded yet.
- for (Iterator<String> iter = queues.iterator(); iter.hasNext();) {
- ReplicationQueueInfo queue = new ReplicationQueueInfo(iter.next());
- if (queue.getPeerId().equals(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER)) {
+ // legacy region_replica_replication peer directly, without deleting the replication queues
+ for (Iterator<ReplicationQueueId> iter = queues.iterator(); iter.hasNext();) {
+ ReplicationQueueId queueId = iter.next();
+ if (queueId.getPeerId().equals(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER)) {
LOG.info("Found replication queue {} for legacy region replication peer, "
- + "skipping claiming and removing...", queue.getQueueId());
+ + "skipping claiming and removing...", queueId);
iter.remove();
- storage.removeQueue(crashedServer, queue.getQueueId());
+ storage.removeQueue(queueId);
+ } else if (!queueId.isRecovered()) {
+ existQueuePeerIds.add(queueId.getPeerId());
+ }
+ }
+ List<ReplicationPeerDescription> peers = env.getReplicationPeerManager().listPeers(null);
+ // TODO: the implementation is not enough yet, if there are retries, we need to know whether
+ // the replication queue for the given peer has been claimed or not, otherwise this logic will
+ // introduce redundant replication queues for the same peer. Add this logic to make some UTs
+ // pass first.
+ for (ReplicationPeerDescription peer : peers) {
+ if (!existQueuePeerIds.contains(peer.getPeerId())) {
+ ReplicationQueueId queueId = new ReplicationQueueId(crashedServer, peer.getPeerId());
+ env.getReplicationPeerManager().getQueueStorage().setOffset(queueId,
+ crashedServer.toString(), ReplicationGroupOffset.BEGIN, Collections.emptyMap());
+ queues.add(queueId);
}
}
if (queues.isEmpty()) {
LOG.debug("Finish claiming replication queues for {}", crashedServer);
- storage.removeReplicatorIfQueueIsEmpty(crashedServer);
// we are done
return null;
}
@@ -112,8 +133,7 @@ public class ClaimReplicationQueuesProcedure extends Procedure<MasterProcedureEn
ClaimReplicationQueueRemoteProcedure[] procs =
new ClaimReplicationQueueRemoteProcedure[Math.min(queues.size(), targetServers.size())];
for (int i = 0; i < procs.length; i++) {
- procs[i] = new ClaimReplicationQueueRemoteProcedure(crashedServer, queues.get(i),
- targetServers.get(i));
+ procs[i] = new ClaimReplicationQueueRemoteProcedure(queues.get(i), targetServers.get(i));
}
return procs;
} catch (ReplicationException e) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
index bfb7b7c10c0..53270bcbb04 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/replication/ReplicationPeerManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.master.replication;
+import com.google.errorprone.annotations.RestrictedApi;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -29,6 +30,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
@@ -38,10 +40,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil;
import org.apache.hadoop.hbase.conf.ConfigurationObserver;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.replication.BaseReplicationEndpoint;
import org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
@@ -50,11 +55,12 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfigBuilder;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
import org.apache.hadoop.hbase.zookeeper.ZKConfig;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
@@ -105,9 +111,20 @@ public class ReplicationPeerManager implements ConfigurationObserver {
private final ZKWatcher zk;
+ @FunctionalInterface
+ private interface ReplicationQueueStorageInitializer {
+
+ void initialize() throws IOException;
+ }
+
+ private final ReplicationQueueStorageInitializer queueStorageInitializer;
+
+ // we will mock this class in UT so leave the constructor as package private and not mark the
+ // class as final, since mockito can not mock a final class
ReplicationPeerManager(FileSystem fs, ZKWatcher zk, ReplicationPeerStorage peerStorage,
ReplicationQueueStorage queueStorage, ConcurrentMap<String, ReplicationPeerDescription> peers,
- Configuration conf, String clusterId) {
+ Configuration conf, String clusterId,
+ ReplicationQueueStorageInitializer queueStorageInitializer) {
this.fs = fs;
this.zk = zk;
this.peerStorage = peerStorage;
@@ -115,19 +132,15 @@ public class ReplicationPeerManager implements ConfigurationObserver {
this.peers = peers;
this.conf = conf;
this.clusterId = clusterId;
+ this.queueStorageInitializer = queueStorageInitializer;
}
private void checkQueuesDeleted(String peerId)
throws ReplicationException, DoNotRetryIOException {
- for (ServerName replicator : queueStorage.getListOfReplicators()) {
- List<String> queueIds = queueStorage.getAllQueues(replicator);
- for (String queueId : queueIds) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (queueInfo.getPeerId().equals(peerId)) {
- throw new DoNotRetryIOException("undeleted queue for peerId: " + peerId + ", replicator: "
- + replicator + ", queueId: " + queueId);
- }
- }
+ List<ReplicationQueueId> queueIds = queueStorage.listAllQueueIds(peerId);
+ if (!queueIds.isEmpty()) {
+ throw new DoNotRetryIOException("There are still " + queueIds.size()
+ + " undeleted queue(s) for peerId: " + peerId + ", first is " + queueIds.get(0));
}
if (queueStorage.getAllPeersFromHFileRefsQueue().contains(peerId)) {
throw new DoNotRetryIOException("Undeleted queue for peer " + peerId + " in hfile-refs");
@@ -135,7 +148,7 @@ public class ReplicationPeerManager implements ConfigurationObserver {
}
void preAddPeer(String peerId, ReplicationPeerConfig peerConfig)
- throws DoNotRetryIOException, ReplicationException {
+ throws ReplicationException, IOException {
if (peerId.contains("-")) {
throw new DoNotRetryIOException("Found invalid peer name: " + peerId);
}
@@ -146,6 +159,9 @@ public class ReplicationPeerManager implements ConfigurationObserver {
if (peers.containsKey(peerId)) {
throw new DoNotRetryIOException("Replication peer " + peerId + " already exists");
}
+
+ // lazy create table
+ queueStorageInitializer.initialize();
// make sure that there is no queues with the same peer id. This may happen when we create a
// peer with the same id with a old deleted peer. If the replication queues for the old peer
// have not been cleaned up yet then we should not create the new peer, otherwise the old wal
@@ -365,8 +381,8 @@ public class ReplicationPeerManager implements ConfigurationObserver {
// claimed once after the refresh peer procedure done(as the next claim queue will just delete
// it), so we can make sure that a two pass scan will finally find the queue and remove it,
// unless it has already been removed by others.
- ReplicationUtils.removeAllQueues(queueStorage, peerId);
- ReplicationUtils.removeAllQueues(queueStorage, peerId);
+ queueStorage.removeAllQueues(peerId);
+ queueStorage.removeAllQueues(peerId);
}
public void removeAllQueuesAndHFileRefs(String peerId) throws ReplicationException {
@@ -568,14 +584,69 @@ public class ReplicationPeerManager implements ConfigurationObserver {
.collect(Collectors.toList());
}
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ public ReplicationPeerStorage getPeerStorage() {
+ return peerStorage;
+ }
+
public ReplicationQueueStorage getQueueStorage() {
return queueStorage;
}
- public static ReplicationPeerManager create(FileSystem fs, ZKWatcher zk, Configuration conf,
- String clusterId) throws ReplicationException {
+ private static Pair<ReplicationQueueStorage, ReplicationQueueStorageInitializer>
+ createReplicationQueueStorage(MasterServices services) throws IOException {
+ Configuration conf = services.getConfiguration();
+ TableName replicationQueueTableName =
+ TableName.valueOf(conf.get(ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME,
+ ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME_DEFAULT.getNameAsString()));
+ ReplicationQueueStorageInitializer initializer;
+ if (services.getTableDescriptors().exists(replicationQueueTableName)) {
+ // no need to create the table
+ initializer = () -> {
+ };
+ } else {
+ // lazy create the replication table.
+ initializer = new ReplicationQueueStorageInitializer() {
+
+ private volatile boolean created = false;
+
+ @Override
+ public void initialize() throws IOException {
+ if (created) {
+ return;
+ }
+ synchronized (this) {
+ if (created) {
+ return;
+ }
+ if (services.getTableDescriptors().exists(replicationQueueTableName)) {
+ created = true;
+ return;
+ }
+ long procId = services.createSystemTable(ReplicationStorageFactory
+ .createReplicationQueueTableDescriptor(replicationQueueTableName));
+ ProcedureExecutor<MasterProcedureEnv> procExec = services.getMasterProcedureExecutor();
+ ProcedureSyncWait.waitFor(procExec.getEnvironment(), TimeUnit.MINUTES.toMillis(1),
+ "Creating table " + replicationQueueTableName, () -> procExec.isFinished(procId));
+ }
+ }
+ };
+ }
+ return Pair.newPair(ReplicationStorageFactory.getReplicationQueueStorage(
+ services.getConnection(), replicationQueueTableName), initializer);
+ }
+
+ public static ReplicationPeerManager create(MasterServices services, String clusterId)
+ throws ReplicationException, IOException {
+ Configuration conf = services.getConfiguration();
+ FileSystem fs = services.getMasterFileSystem().getFileSystem();
+ ZKWatcher zk = services.getZooKeeper();
ReplicationPeerStorage peerStorage =
ReplicationStorageFactory.getReplicationPeerStorage(fs, zk, conf);
+ Pair<ReplicationQueueStorage, ReplicationQueueStorageInitializer> pair =
+ createReplicationQueueStorage(services);
+ ReplicationQueueStorage queueStorage = pair.getFirst();
ConcurrentMap<String, ReplicationPeerDescription> peers = new ConcurrentHashMap<>();
for (String peerId : peerStorage.listPeerIds()) {
ReplicationPeerConfig peerConfig = peerStorage.getPeerConfig(peerId);
@@ -585,7 +656,24 @@ public class ReplicationPeerManager implements ConfigurationObserver {
) {
// we do not use this endpoint for region replication any more, see HBASE-26233
LOG.info("Legacy region replication peer found, removing: {}", peerConfig);
- peerStorage.removePeer(peerId);
+ // do it asynchronous to not block the start up of HMaster
+ new Thread("Remove legacy replication peer " + peerId) {
+
+ @Override
+ public void run() {
+ try {
+ // need to delete two times to make sure we delete all the queues, see the comments in
+ // above
+ // removeAllQueues method for more details.
+ queueStorage.removeAllQueues(peerId);
+ queueStorage.removeAllQueues(peerId);
+ // delete queue first and then peer, because we use peer as a flag.
+ peerStorage.removePeer(peerId);
+ } catch (Exception e) {
+ LOG.warn("Failed to delete legacy replication peer {}", peerId);
+ }
+ }
+ }.start();
continue;
}
peerConfig = ReplicationPeerConfigUtil.updateReplicationBasePeerConfigs(conf, peerConfig);
@@ -594,8 +682,8 @@ public class ReplicationPeerManager implements ConfigurationObserver {
SyncReplicationState state = peerStorage.getPeerSyncReplicationState(peerId);
peers.put(peerId, new ReplicationPeerDescription(peerId, enabled, peerConfig, state));
}
- return new ReplicationPeerManager(fs, zk, peerStorage,
- ReplicationStorageFactory.getReplicationQueueStorage(zk, conf), peers, conf, clusterId);
+ return new ReplicationPeerManager(fs, zk, peerStorage, queueStorage, peers, conf, clusterId,
+ pair.getSecond());
}
/**
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index dc4c3a0a6bc..8df65487c67 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -32,6 +32,7 @@ import java.io.IOException;
import java.io.InterruptedIOException;
import java.lang.management.MemoryType;
import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -441,8 +442,9 @@ public abstract class AbstractFSWAL<W extends WriterBase> implements WAL {
}
// If prefix is null||empty then just name it wal
- this.walFilePrefix =
- prefix == null || prefix.isEmpty() ? "wal" : URLEncoder.encode(prefix, "UTF8");
+ this.walFilePrefix = prefix == null || prefix.isEmpty()
+ ? "wal"
+ : URLEncoder.encode(prefix, StandardCharsets.UTF_8.name());
// we only correctly differentiate suffices when numeric ones start with '.'
if (suffix != null && !(suffix.isEmpty()) && !(suffix.startsWith(WAL_FILE_NAME_DELIMITER))) {
throw new IllegalArgumentException("WAL suffix must start with '" + WAL_FILE_NAME_DELIMITER
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java
index 819e4c5e54a..00e875f8be5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationHFileCleaner.java
@@ -19,23 +19,28 @@ package org.apache.hadoop.hbase.replication.master;
import java.io.IOException;
import java.util.Collections;
+import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.base.Predicate;
import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hbase.thirdparty.org.apache.commons.collections4.MapUtils;
/**
* Implementation of a file cleaner that checks if a hfile is still scheduled for replication before
@@ -44,15 +49,20 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class ReplicationHFileCleaner extends BaseHFileCleanerDelegate {
private static final Logger LOG = LoggerFactory.getLogger(ReplicationHFileCleaner.class);
- private ZKWatcher zkw;
+ private Connection conn;
+ private boolean shareConn;
private ReplicationQueueStorage rqs;
private boolean stopped = false;
@Override
public Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
- // all members of this class are null if replication is disabled,
- // so we cannot filter the files
- if (this.getConf() == null) {
+ if (
+ !(getConf().getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
+ HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT))
+ ) {
+ LOG.warn(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY + " is not enabled. Better to remove "
+ + ReplicationHFileCleaner.class + " from " + HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS
+ + " configuration.");
return files;
}
@@ -88,51 +98,34 @@ public class ReplicationHFileCleaner extends BaseHFileCleanerDelegate {
}
@Override
- public void setConf(Configuration config) {
- // If either replication or replication of bulk load hfiles is disabled, keep all members null
- if (
- !(config.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
- HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT))
- ) {
- LOG.warn(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY + " is not enabled. Better to remove "
- + ReplicationHFileCleaner.class + " from " + HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS
- + " configuration.");
- return;
- }
- // Make my own Configuration. Then I'll have my own connection to zk that
- // I can close myself when time comes.
- Configuration conf = new Configuration(config);
+ public void init(Map<String, Object> params) {
+ super.init(params);
try {
- setConf(conf, new ZKWatcher(conf, "replicationHFileCleaner", null));
+ if (MapUtils.isNotEmpty(params)) {
+ Object master = params.get(HMaster.MASTER);
+ if (master != null && master instanceof Server) {
+ conn = ((Server) master).getConnection();
+ shareConn = true;
+ }
+ }
+ if (conn == null) {
+ conn = ConnectionFactory.createConnection(getConf());
+ }
+ this.rqs = ReplicationStorageFactory.getReplicationQueueStorage(conn, getConf());
} catch (IOException e) {
LOG.error("Error while configuring " + this.getClass().getName(), e);
}
}
- @InterfaceAudience.Private
- public void setConf(Configuration conf, ZKWatcher zk) {
- super.setConf(conf);
- try {
- initReplicationQueueStorage(conf, zk);
- } catch (Exception e) {
- LOG.error("Error while configuring " + this.getClass().getName(), e);
- }
- }
-
- private void initReplicationQueueStorage(Configuration conf, ZKWatcher zk) {
- this.zkw = zk;
- this.rqs = ReplicationStorageFactory.getReplicationQueueStorage(zk, conf);
- }
-
@Override
public void stop(String why) {
if (this.stopped) {
return;
}
this.stopped = true;
- if (this.zkw != null) {
- LOG.info("Stopping " + this.zkw);
- this.zkw.close();
+ if (!shareConn && this.conn != null) {
+ LOG.info("Stopping " + this.conn);
+ IOUtils.closeQuietly(conn);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
index 54e600e09ad..7135ca9a9b2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/master/ReplicationLogCleaner.java
@@ -26,9 +26,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
-import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -56,14 +54,15 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
@Override
public void preClean() {
readZKTimestamp = EnvironmentEdgeManager.currentTime();
- try {
- // The concurrently created new WALs may not be included in the return list,
- // but they won't be deleted because they're not in the checking set.
- wals = queueStorage.getAllWALs();
- } catch (ReplicationException e) {
- LOG.warn("Failed to read zookeeper, skipping checking deletable files");
- wals = null;
- }
+ // TODO: revisit the implementation
+ // try {
+ // // The concurrently created new WALs may not be included in the return list,
+ // // but they won't be deleted because they're not in the checking set.
+ // wals = queueStorage.getAllWALs();
+ // } catch (ReplicationException e) {
+ // LOG.warn("Failed to read zookeeper, skipping checking deletable files");
+ // wals = null;
+ // }
}
@Override
@@ -115,7 +114,8 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
if (zkw == null) {
zkw = new ZKWatcher(getConf(), "replicationLogCleaner", null);
}
- this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
+ // TODO: revisit the implementation
+ // this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
} catch (IOException e) {
LOG.error("Error while configuring " + this.getClass().getName(), e);
}
@@ -126,7 +126,8 @@ public class ReplicationLogCleaner extends BaseLogCleanerDelegate {
super.setConf(conf);
try {
this.zkw = zk;
- this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zk, conf);
+ // TODO: revisit the implementation
+ // this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zk, conf);
} catch (Exception e) {
LOG.error("Error while configuring " + this.getClass().getName(), e);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ClaimReplicationQueueCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ClaimReplicationQueueCallable.java
index b9a7be813af..2b7e14f9f7a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ClaimReplicationQueueCallable.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ClaimReplicationQueueCallable.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.replication.regionserver;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.executor.EventType;
import org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
@@ -30,9 +31,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.C
@InterfaceAudience.Private
public class ClaimReplicationQueueCallable extends BaseRSProcedureCallable {
- private ServerName crashedServer;
-
- private String queue;
+ private ReplicationQueueId queueId;
@Override
public EventType getEventType() {
@@ -42,14 +41,20 @@ public class ClaimReplicationQueueCallable extends BaseRSProcedureCallable {
@Override
protected void doCall() throws Exception {
PeerProcedureHandler handler = rs.getReplicationSourceService().getPeerProcedureHandler();
- handler.claimReplicationQueue(crashedServer, queue);
+ handler.claimReplicationQueue(queueId);
}
@Override
protected void initParameter(byte[] parameter) throws InvalidProtocolBufferException {
ClaimReplicationQueueRemoteParameter param =
ClaimReplicationQueueRemoteParameter.parseFrom(parameter);
- crashedServer = ProtobufUtil.toServerName(param.getCrashedServer());
- queue = param.getQueue();
+ ServerName crashedServer = ProtobufUtil.toServerName(param.getCrashedServer());
+ String queue = param.getQueue();
+ if (param.hasSourceServer()) {
+ ServerName sourceServer = ProtobufUtil.toServerName(param.getSourceServer());
+ queueId = new ReplicationQueueId(crashedServer, queue, sourceServer);
+ } else {
+ queueId = new ReplicationQueueId(crashedServer, queue);
+ }
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
index 4636e239904..98d0a55fbc4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -21,7 +21,6 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -45,10 +44,8 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.zookeeper.ZKDump;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -59,6 +56,8 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.AtomicLongMap;
/**
+ * TODO: reimplement this tool
+ * <p/>
* Provides information about the existing states of replication, replication peers and queues.
* Usage: hbase org.apache.hadoop.hbase.replication.regionserver.DumpReplicationQueues [args]
* Arguments: --distributed Polls each RS to dump information about the queue --hdfs Reports HDFS
@@ -299,32 +298,33 @@ public class DumpReplicationQueues extends Configured implements Tool {
ReplicationQueueStorage queueStorage;
StringBuilder sb = new StringBuilder();
- queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
- Set<ServerName> liveRegionServers = ZKUtil.listChildrenNoWatch(zkw, zkw.getZNodePaths().rsZNode)
- .stream().map(ServerName::parseServerName).collect(Collectors.toSet());
-
+ // queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
+ // Set<ServerName> liveRegionServers = ZKUtil.listChildrenNoWatch(zkw,
+ // zkw.getZNodePaths().rsZNode)
+ // .stream().map(ServerName::parseServerName).collect(Collectors.toSet());
+ //
// Loops each peer on each RS and dumps the queues
- List<ServerName> regionservers = queueStorage.getListOfReplicators();
- if (regionservers == null || regionservers.isEmpty()) {
- return sb.toString();
- }
- for (ServerName regionserver : regionservers) {
- List<String> queueIds = queueStorage.getAllQueues(regionserver);
- if (!liveRegionServers.contains(regionserver)) {
- deadRegionServers.add(regionserver.getServerName());
- }
- for (String queueId : queueIds) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- List<String> wals = queueStorage.getWALsInQueue(regionserver, queueId);
- Collections.sort(wals);
- if (!peerIds.contains(queueInfo.getPeerId())) {
- deletedQueues.add(regionserver + "/" + queueId);
- sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, true, hdfs));
- } else {
- sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, false, hdfs));
- }
- }
- }
+ // List<ServerName> regionservers = queueStorage.getListOfReplicators();
+ // if (regionservers == null || regionservers.isEmpty()) {
+ // return sb.toString();
+ // }
+ // for (ServerName regionserver : regionservers) {
+ // List<String> queueIds = queueStorage.getAllQueues(regionserver);
+ // if (!liveRegionServers.contains(regionserver)) {
+ // deadRegionServers.add(regionserver.getServerName());
+ // }
+ // for (String queueId : queueIds) {
+ // ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
+ // List<String> wals = queueStorage.getWALsInQueue(regionserver, queueId);
+ // Collections.sort(wals);
+ // if (!peerIds.contains(queueInfo.getPeerId())) {
+ // deletedQueues.add(regionserver + "/" + queueId);
+ // sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, true, hdfs));
+ // } else {
+ // sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, false, hdfs));
+ // }
+ // }
+ // }
return sb.toString();
}
@@ -350,9 +350,9 @@ public class DumpReplicationQueues extends Configured implements Tool {
peersQueueSize.addAndGet(queueInfo.getPeerId(), wals.size());
for (String wal : wals) {
- long position = queueStorage.getWALPosition(regionserver, queueInfo.getPeerId(), wal);
- sb.append(" Replication position for " + wal + ": "
- + (position > 0 ? position : "0" + " (not started or nothing to replicate)") + "\n");
+ // long position = queueStorage.getWALPosition(regionserver, queueInfo.getPeerId(), wal);
+ // sb.append(" Replication position for " + wal + ": "
+ // + (position > 0 ? position : "0" + " (not started or nothing to replicate)") + "\n");
}
if (hdfs) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
index 3df78c1d831..b2cffd59fd4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandler.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.yetus.audience.InterfaceAudience;
/**
@@ -42,6 +42,5 @@ public interface PeerProcedureHandler {
void transitSyncReplicationPeerState(String peerId, int stage, HRegionServer rs)
throws ReplicationException, IOException;
- void claimReplicationQueue(ServerName crashedServer, String queue)
- throws ReplicationException, IOException;
+ void claimReplicationQueue(ReplicationQueueId queueId) throws ReplicationException, IOException;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
index 0187de14f80..cd3db44d8fa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/PeerProcedureHandlerImpl.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.replication.regionserver;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.concurrent.locks.Lock;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.LogRoller;
import org.apache.hadoop.hbase.replication.ReplicationException;
@@ -28,6 +27,7 @@ import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
import org.apache.hadoop.hbase.util.KeyLocker;
@@ -226,8 +226,8 @@ public class PeerProcedureHandlerImpl implements PeerProcedureHandler {
}
@Override
- public void claimReplicationQueue(ServerName crashedServer, String queue)
+ public void claimReplicationQueue(ReplicationQueueId queueId)
throws ReplicationException, IOException {
- replicationSourceManager.claimQueue(crashedServer, queue);
+ replicationSourceManager.claimQueue(queueId);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
index 024248a3f8c..e740a01dc4f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSource.java
@@ -17,23 +17,7 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
-import java.io.IOException;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.PriorityBlockingQueue;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.replication.ReplicationPeer;
-import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.util.CommonFSUtils;
-import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Class that handles the recovered source of a replication stream, which is transfered from another
@@ -42,124 +26,14 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public class RecoveredReplicationSource extends ReplicationSource {
- private static final Logger LOG = LoggerFactory.getLogger(RecoveredReplicationSource.class);
-
- private String actualPeerId;
-
- @Override
- public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueueStorage queueStorage, ReplicationPeer replicationPeer, Server server,
- String peerClusterZnode, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
- MetricsSource metrics) throws IOException {
- super.init(conf, fs, manager, queueStorage, replicationPeer, server, peerClusterZnode,
- clusterId, walFileLengthProvider, metrics);
- this.actualPeerId = this.replicationQueueInfo.getPeerId();
- }
-
@Override
protected RecoveredReplicationSourceShipper createNewShipper(String walGroupId) {
- return new RecoveredReplicationSourceShipper(conf, walGroupId, logQueue, this, queueStorage);
- }
-
- public void locateRecoveredPaths(String walGroupId) throws IOException {
- boolean hasPathChanged = false;
- PriorityBlockingQueue<Path> queue = logQueue.getQueue(walGroupId);
- PriorityBlockingQueue<Path> newPaths = new PriorityBlockingQueue<Path>(queueSizePerGroup,
- new AbstractFSWALProvider.WALStartTimeComparator());
- pathsLoop: for (Path path : queue) {
- if (fs.exists(path)) { // still in same location, don't need to do anything
- newPaths.add(path);
- continue;
- }
- // Path changed - try to find the right path.
- hasPathChanged = true;
- if (server instanceof ReplicationSyncUp.DummyServer) {
- // In the case of disaster/recovery, HMaster may be shutdown/crashed before flush data
- // from .logs to .oldlogs. Loop into .logs folders and check whether a match exists
- Path newPath = getReplSyncUpPath(path);
- newPaths.add(newPath);
- continue;
- } else {
- // See if Path exists in the dead RS folder (there could be a chain of failures
- // to look at)
- List<ServerName> deadRegionServers = this.replicationQueueInfo.getDeadRegionServers();
- LOG.info("NB dead servers : " + deadRegionServers.size());
- final Path walDir = CommonFSUtils.getWALRootDir(conf);
- for (ServerName curDeadServerName : deadRegionServers) {
- final Path deadRsDirectory = new Path(walDir,
- AbstractFSWALProvider.getWALDirectoryName(curDeadServerName.getServerName()));
- Path[] locs = new Path[] { new Path(deadRsDirectory, path.getName()),
- new Path(deadRsDirectory.suffix(AbstractFSWALProvider.SPLITTING_EXT), path.getName()) };
- for (Path possibleLogLocation : locs) {
- LOG.info("Possible location " + possibleLogLocation.toUri().toString());
- if (manager.getFs().exists(possibleLogLocation)) {
- // We found the right new location
- LOG.info("Log " + path + " still exists at " + possibleLogLocation);
- newPaths.add(possibleLogLocation);
- continue pathsLoop;
- }
- }
- }
- // didn't find a new location
- LOG.error(
- String.format("WAL Path %s doesn't exist and couldn't find its new location", path));
- newPaths.add(path);
- }
- }
-
- if (hasPathChanged) {
- if (newPaths.size() != queue.size()) { // this shouldn't happen
- LOG.error("Recovery queue size is incorrect");
- throw new IOException("Recovery queue size error");
- }
- // put the correct locations in the queue
- // since this is a recovered queue with no new incoming logs,
- // there shouldn't be any concurrency issues
- logQueue.clear(walGroupId);
- for (Path path : newPaths) {
- logQueue.enqueueLog(path, walGroupId);
- }
- }
- }
-
- // N.B. the ReplicationSyncUp tool sets the manager.getWALDir to the root of the wal
- // area rather than to the wal area for a particular region server.
- private Path getReplSyncUpPath(Path path) throws IOException {
- FileStatus[] rss = fs.listStatus(manager.getLogDir());
- for (FileStatus rs : rss) {
- Path p = rs.getPath();
- FileStatus[] logs = fs.listStatus(p);
- for (FileStatus log : logs) {
- p = new Path(p, log.getPath().getName());
- if (p.getName().equals(path.getName())) {
- LOG.info("Log " + p.getName() + " found at " + p);
- return p;
+ return new RecoveredReplicationSourceShipper(conf, walGroupId, logQueue, this, queueStorage,
+ () -> {
+ if (workerThreads.isEmpty()) {
+ this.getSourceMetrics().clear();
+ manager.finishRecoveredSource(this);
}
- }
- }
- LOG.error("Didn't find path for: " + path.getName());
- return path;
- }
-
- void tryFinish() {
- if (workerThreads.isEmpty()) {
- this.getSourceMetrics().clear();
- manager.finishRecoveredSource(this);
- }
- }
-
- @Override
- public String getPeerId() {
- return this.actualPeerId;
- }
-
- @Override
- public ServerName getServerWALsBelongTo() {
- return this.replicationQueueInfo.getDeadRegionServers().get(0);
- }
-
- @Override
- public boolean isRecovered() {
- return true;
+ });
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
index 4f2bafcf156..2bb3a7c3591 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/RecoveredReplicationSourceShipper.java
@@ -17,83 +17,27 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
-import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.util.Threads;
import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Used by a {@link RecoveredReplicationSource}.
*/
@InterfaceAudience.Private
public class RecoveredReplicationSourceShipper extends ReplicationSourceShipper {
- private static final Logger LOG =
- LoggerFactory.getLogger(RecoveredReplicationSourceShipper.class);
- protected final RecoveredReplicationSource source;
- private final ReplicationQueueStorage replicationQueues;
+ private final Runnable tryFinish;
public RecoveredReplicationSourceShipper(Configuration conf, String walGroupId,
ReplicationSourceLogQueue logQueue, RecoveredReplicationSource source,
- ReplicationQueueStorage queueStorage) {
+ ReplicationQueueStorage queueStorage, Runnable tryFinish) {
super(conf, walGroupId, logQueue, source);
- this.source = source;
- this.replicationQueues = queueStorage;
+ this.tryFinish = tryFinish;
}
@Override
protected void postFinish() {
- source.tryFinish();
- }
-
- @Override
- public long getStartPosition() {
- long startPosition = getRecoveredQueueStartPos();
- int numRetries = 0;
- while (numRetries <= maxRetriesMultiplier) {
- try {
- source.locateRecoveredPaths(walGroupId);
- break;
- } catch (IOException e) {
- LOG.error("Error while locating recovered queue paths, attempt #" + numRetries, e);
- numRetries++;
- }
- }
- return startPosition;
- }
-
- // If this is a recovered queue, the queue is already full and the first log
- // normally has a position (unless the RS failed between 2 logs)
- private long getRecoveredQueueStartPos() {
- long startPosition = 0;
- String peerClusterZNode = source.getQueueId();
- try {
- startPosition = this.replicationQueues.getWALPosition(source.getServer().getServerName(),
- peerClusterZNode, this.logQueue.getQueue(walGroupId).peek().getName());
- LOG.trace("Recovered queue started with log {} at position {}",
- this.logQueue.getQueue(walGroupId).peek(), startPosition);
- } catch (ReplicationException e) {
- terminate("Couldn't get the position of this recovered queue " + peerClusterZNode, e);
- }
- return startPosition;
- }
-
- private void terminate(String reason, Exception cause) {
- if (cause == null) {
- LOG.info("Closing worker for wal group {} because: {}", this.walGroupId, reason);
- } else {
- LOG.error(
- "Closing worker for wal group " + this.walGroupId + " because an error occurred: " + reason,
- cause);
- }
- entryReader.interrupt();
- Threads.shutdown(entryReader, sleepForRetries);
- this.interrupt();
- Threads.shutdown(this, sleepForRetries);
- LOG.info("ReplicationSourceWorker {} terminated", this.getName());
+ tryFinish.run();
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
index 338718bd8ea..6279c4b9596 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java
@@ -100,7 +100,7 @@ public class Replication implements ReplicationSourceService, PropagatingConfigu
try {
this.queueStorage =
- ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
+ ReplicationStorageFactory.getReplicationQueueStorage(server.getConnection(), conf);
this.replicationPeers = ReplicationFactory.getReplicationPeers(server.getFileSystem(),
server.getZooKeeper(), this.conf);
this.replicationPeers.init();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java
index 00306dd1702..7e8ca88b730 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java
@@ -89,8 +89,8 @@ public class ReplicationLoad {
rLoadSourceBuild.setOPsShipped(oPsShipped);
if (source instanceof ReplicationSource) {
ReplicationSource replSource = (ReplicationSource) source;
- rLoadSourceBuild.setRecovered(replSource.getReplicationQueueInfo().isQueueRecovered());
- rLoadSourceBuild.setQueueId(replSource.getReplicationQueueInfo().getQueueId());
+ rLoadSourceBuild.setRecovered(replSource.getQueueId().isRecovered());
+ rLoadSourceBuild.setQueueId(replSource.getQueueId().toString());
rLoadSourceBuild.setRunning(replSource.isWorkerRunning());
rLoadSourceBuild.setEditsSinceRestart(timeStampOfNextToReplicate > 0);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
index 03420445638..a49bfd7b623 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.replication.regionserver;
import static org.apache.hadoop.hbase.wal.AbstractFSWALProvider.findArchivedLog;
+import com.google.errorprone.annotations.RestrictedApi;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
@@ -52,8 +53,10 @@ import org.apache.hadoop.hbase.replication.ChainWALEntryFilter;
import org.apache.hadoop.hbase.replication.ClusterMarkingEntryFilter;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.SystemTableWALEntryFilter;
import org.apache.hadoop.hbase.replication.WALEntryFilter;
@@ -67,6 +70,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
/**
@@ -90,7 +94,6 @@ public class ReplicationSource implements ReplicationSourceInterface {
protected ReplicationPeer replicationPeer;
protected Configuration conf;
- protected ReplicationQueueInfo replicationQueueInfo;
// The manager of all sources to which we ping back our progress
protected ReplicationSourceManager manager;
@@ -103,8 +106,11 @@ public class ReplicationSource implements ReplicationSourceInterface {
private UUID clusterId;
// total number of edits we replicated
private AtomicLong totalReplicatedEdits = new AtomicLong(0);
- // The znode we currently play with
- protected String queueId;
+ // The id of the replication queue
+ protected ReplicationQueueId queueId;
+ // The start offsets. Usually only recovered replication queue needs this, but probably when we
+ // update the peer config and restart the replication peer, we also need this?
+ protected ImmutableMap<String, ReplicationGroupOffset> startOffsets;
// Maximum number of retries before taking bold actions
private int maxRetriesMultiplier;
// Indicates if this particular source is running
@@ -184,14 +190,14 @@ public class ReplicationSource implements ReplicationSourceInterface {
* @param fs file system to use
* @param manager replication manager to ping to
* @param server the server for this region server
- * @param queueId the id of our replication queue
+ * @param queueData the id and offsets of our replication queue
* @param clusterId unique UUID for the cluster
* @param metrics metrics for replication source
*/
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
ReplicationQueueStorage queueStorage, ReplicationPeer replicationPeer, Server server,
- String queueId, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
+ ReplicationQueueData queueData, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
MetricsSource metrics) throws IOException {
this.server = server;
this.conf = HBaseConfiguration.create(conf);
@@ -211,8 +217,8 @@ public class ReplicationSource implements ReplicationSourceInterface {
this.metrics = metrics;
this.clusterId = clusterId;
- this.queueId = queueId;
- this.replicationQueueInfo = new ReplicationQueueInfo(queueId);
+ this.queueId = queueData.getId();
+ this.startOffsets = queueData.getOffsets();
// A defaultBandwidth of '0' means no bandwidth; i.e. no throttling.
defaultBandwidth = this.conf.getLong("replication.source.per.peer.node.bandwidth", 0);
@@ -240,24 +246,24 @@ public class ReplicationSource implements ReplicationSourceInterface {
return;
}
// Use WAL prefix as the WALGroupId for this peer.
- String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal.getName());
- boolean queueExists = logQueue.enqueueLog(wal, walPrefix);
+ String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(wal.getName());
+ boolean queueExists = logQueue.enqueueLog(wal, walGroupId);
if (!queueExists) {
if (this.isSourceActive() && this.walEntryFilter != null) {
// new wal group observed after source startup, start a new worker thread to track it
// notice: it's possible that wal enqueued when this.running is set but worker thread
// still not launched, so it's necessary to check workerThreads before start the worker
- tryStartNewShipper(walPrefix);
+ tryStartNewShipper(walGroupId);
}
}
if (LOG.isTraceEnabled()) {
- LOG.trace("{} Added wal {} to queue of source {}.", logPeerId(), walPrefix,
- this.replicationQueueInfo.getQueueId());
+ LOG.trace("{} Added wal {} to queue of source {}.", logPeerId(), walGroupId, queueId);
}
}
- @InterfaceAudience.Private
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
public Map<String, PriorityBlockingQueue<Path>> getQueues() {
return logQueue.getQueues();
}
@@ -330,6 +336,30 @@ public class ReplicationSource implements ReplicationSourceInterface {
this.walEntryFilter = new ChainWALEntryFilter(filters);
}
+ private long getStartOffset(String walGroupId) {
+ ReplicationGroupOffset startOffset = startOffsets.get(walGroupId);
+ if (startOffset == null || startOffset == ReplicationGroupOffset.BEGIN) {
+ return 0L;
+ }
+ // this method will only be called when start new shipper, and we will only start new shipper
+ // when there is a new queue, so here the queue for walGroupId will never be null.
+ Path first = logQueue.getQueue(walGroupId).peek();
+ if (!startOffset.getWal().equals(first.getName())) {
+ return 0L;
+ }
+ // Usually, if we arrive here, the start offset should never be -1, as it means this file has
+ // been fully replicated so we should have filtered it out in upper layer, usually in
+ // ReplicationSourceManager. Add a warn message for safety, as usually replicate more data will
+ // not cause big problems.
+ if (startOffset.getOffset() < 0) {
+ LOG.warn("Should have already replicated wal {}, return start offset as 0",
+ startOffset.getWal());
+ return 0L;
+ } else {
+ return startOffset.getOffset();
+ }
+ }
+
private void tryStartNewShipper(String walGroupId) {
workerThreads.compute(walGroupId, (key, value) -> {
if (value != null) {
@@ -339,7 +369,7 @@ public class ReplicationSource implements ReplicationSourceInterface {
LOG.debug("{} starting shipping worker for walGroupId={}", logPeerId(), walGroupId);
ReplicationSourceShipper worker = createNewShipper(walGroupId);
ReplicationSourceWALReader walReader =
- createNewWALReader(walGroupId, worker.getStartPosition());
+ createNewWALReader(walGroupId, getStartOffset(walGroupId));
Threads.setDaemonThreadRunning(
walReader, Thread.currentThread().getName() + ".replicationSource.wal-reader."
+ walGroupId + "," + queueId,
@@ -568,7 +598,7 @@ public class ReplicationSource implements ReplicationSourceInterface {
}
LOG.info("{} queueId={} (queues={}) is replicating from cluster={} to cluster={}", logPeerId(),
- this.replicationQueueInfo.getQueueId(), logQueue.getNumQueues(), clusterId, peerClusterId);
+ queueId, logQueue.getNumQueues(), clusterId, peerClusterId);
initializeWALEntryFilter(peerClusterId);
// Start workers
for (String walGroupId : logQueue.getQueues().keySet()) {
@@ -715,7 +745,7 @@ public class ReplicationSource implements ReplicationSourceInterface {
}
@Override
- public String getQueueId() {
+ public ReplicationQueueId getQueueId() {
return this.queueId;
}
@@ -735,10 +765,6 @@ public class ReplicationSource implements ReplicationSourceInterface {
return !this.server.isStopped() && this.sourceRunning;
}
- public ReplicationQueueInfo getReplicationQueueInfo() {
- return replicationQueueInfo;
- }
-
public boolean isWorkerRunning() {
for (ReplicationSourceShipper worker : this.workerThreads.values()) {
if (worker.isActive()) {
@@ -791,7 +817,7 @@ public class ReplicationSource implements ReplicationSourceInterface {
@Override
public ServerName getServerWALsBelongTo() {
- return server.getServerName();
+ return queueId.getServerWALsBelongTo();
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java
index 331f2269cf9..225c6fd4d74 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceFactory.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hbase.replication.regionserver;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,9 +34,8 @@ public final class ReplicationSourceFactory {
private ReplicationSourceFactory() {
}
- static ReplicationSourceInterface create(Configuration conf, String queueId) {
- ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queueId);
- boolean isQueueRecovered = replicationQueueInfo.isQueueRecovered();
+ static ReplicationSourceInterface create(Configuration conf, ReplicationQueueId queueId) {
+ boolean isQueueRecovered = queueId.isRecovered();
ReplicationSourceInterface src;
try {
String defaultReplicationSourceImpl = isQueueRecovered
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
index fa026b919f3..69ad2887064 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceInterface.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.WAL.Entry;
@@ -43,14 +45,22 @@ import org.apache.yetus.audience.InterfaceAudience;
public interface ReplicationSourceInterface {
/**
* Initializer for the source
- * @param conf the configuration to use
- * @param fs the file system to use
- * @param manager the manager to use
- * @param server the server for this region server
+ * @param conf the configuration to use
+ * @param fs the file system to use
+ * @param manager the manager to use
+ * @param queueStorage the replication queue storage
+ * @param replicationPeer the replication peer
+ * @param server the server for this region server
+ * @param queueData the existing replication queue data, contains the queue id and
+ * replication start offsets
+ * @param clusterId the cluster id
+ * @param walFileLengthProvider for getting the length of the WAL file which is currently being
+ * written
+ * @param metrics the replication metrics
*/
void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
ReplicationQueueStorage queueStorage, ReplicationPeer replicationPeer, Server server,
- String queueId, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
+ ReplicationQueueData queueData, UUID clusterId, WALFileLengthProvider walFileLengthProvider,
MetricsSource metrics) throws IOException;
/**
@@ -106,14 +116,14 @@ public interface ReplicationSourceInterface {
* Get the queue id that the source is replicating to
* @return queue id
*/
- String getQueueId();
+ ReplicationQueueId getQueueId();
/**
* Get the id that the source is replicating to.
* @return peer id
*/
default String getPeerId() {
- return getPeer().getId();
+ return getQueueId().getPeerId();
}
/**
@@ -183,7 +193,7 @@ public interface ReplicationSourceInterface {
/** Returns whether this is a replication source for recovery. */
default boolean isRecovered() {
- return false;
+ return getQueueId().isRecovered();
}
/** Returns The instance of queueStorage used by this ReplicationSource. */
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceLogQueue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceLogQueue.java
index a0e6f1b8d1f..93a28b60d27 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceLogQueue.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceLogQueue.java
@@ -97,7 +97,7 @@ public class ReplicationSourceLogQueue {
* @param walGroupId walGroupId
*/
public int getQueueSize(String walGroupId) {
- Queue queue = queues.get(walGroupId);
+ Queue<Path> queue = queues.get(walGroupId);
if (queue == null) {
return 0;
}
@@ -117,7 +117,7 @@ public class ReplicationSourceLogQueue {
/**
* Return queue for the given walGroupId Please don't add or remove elements from the returned
- * queue. Use @enqueueLog and @remove methods respectively.
+ * queue. Use {@link #enqueueLog(Path, String)} and {@link #remove(String)} methods respectively.
* @param walGroupId walGroupId
*/
public PriorityBlockingQueue<Path> getQueue(String walGroupId) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
index eb991b7f349..c16ba8b133c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceManager.java
@@ -17,17 +17,22 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
+import com.google.errorprone.annotations.RestrictedApi;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
import java.util.OptionalLong;
+import java.util.PriorityQueue;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -49,12 +54,13 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
-import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
@@ -68,6 +74,7 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -90,26 +97,25 @@ import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFacto
* operations.</li>
* <li>Need synchronized on {@link #walsById}. There are four methods which modify it,
* {@link #addPeer(String)}, {@link #removePeer(String)},
- * {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)} and {@link #preLogRoll(Path)}.
- * {@link #walsById} is a ConcurrentHashMap and there is a Lock for peer id in
- * {@link PeerProcedureHandlerImpl}. So there is no race between {@link #addPeer(String)} and
+ * {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)} and
+ * {@link #postLogRoll(Path)}. {@link #walsById} is a ConcurrentHashMap and there is a Lock for peer
+ * id in {@link PeerProcedureHandlerImpl}. So there is no race between {@link #addPeer(String)} and
* {@link #removePeer(String)}. {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)}
* is called by {@link ReplicationSourceInterface}. So no race with {@link #addPeer(String)}.
* {@link #removePeer(String)} will terminate the {@link ReplicationSourceInterface} firstly, then
* remove the wals from {@link #walsById}. So no race with {@link #removePeer(String)}. The only
* case need synchronized is {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)} and
- * {@link #preLogRoll(Path)}.</li>
+ * {@link #postLogRoll(Path)}.</li>
* <li>No need synchronized on {@link #walsByIdRecoveredQueues}. There are three methods which
* modify it, {@link #removePeer(String)} ,
* {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)} and
- * {@link ReplicationSourceManager#claimQueue(ServerName, String)}.
+ * {@link #claimQueue(ReplicationQueueId)}.
* {@link #cleanOldLogs(String, boolean, ReplicationSourceInterface)} is called by
* {@link ReplicationSourceInterface}. {@link #removePeer(String)} will terminate the
* {@link ReplicationSourceInterface} firstly, then remove the wals from
- * {@link #walsByIdRecoveredQueues}. And
- * {@link ReplicationSourceManager#claimQueue(ServerName, String)} will add the wals to
- * {@link #walsByIdRecoveredQueues} firstly, then start up a {@link ReplicationSourceInterface}. So
- * there is no race here. For {@link ReplicationSourceManager#claimQueue(ServerName, String)} and
+ * {@link #walsByIdRecoveredQueues}. And {@link #claimQueue(ReplicationQueueId)} will add the wals
+ * to {@link #walsByIdRecoveredQueues} firstly, then start up a {@link ReplicationSourceInterface}.
+ * So there is no race here. For {@link #claimQueue(ReplicationQueueId)} and
* {@link #removePeer(String)}, there is already synchronized on {@link #oldsources}. So no need
* synchronized on {@link #walsByIdRecoveredQueues}.</li>
* <li>Need synchronized on {@link #latestPaths} to avoid the new open source miss new log.</li>
@@ -141,11 +147,12 @@ public class ReplicationSourceManager {
// All logs we are currently tracking
// Index structure of the map is: queue_id->logPrefix/logGroup->logs
// For normal replication source, the peer id is same with the queue id
- private final ConcurrentMap<String, Map<String, NavigableSet<String>>> walsById;
+ private final ConcurrentMap<ReplicationQueueId, Map<String, NavigableSet<String>>> walsById;
// Logs for recovered sources we are currently tracking
// the map is: queue_id->logPrefix/logGroup->logs
// For recovered source, the queue id's format is peer_id-servername-*
- private final ConcurrentMap<String, Map<String, NavigableSet<String>>> walsByIdRecoveredQueues;
+ private final ConcurrentMap<ReplicationQueueId,
+ Map<String, NavigableSet<String>>> walsByIdRecoveredQueues;
private final SyncReplicationPeerMappingManager syncReplicationPeerMappingManager;
@@ -163,8 +170,6 @@ public class ReplicationSourceManager {
// Homemade executer service for replication
private final ThreadPoolExecutor executor;
- private final boolean replicationForBulkLoadDataEnabled;
-
private AtomicLong totalBufferUsed = new AtomicLong();
// How long should we sleep for each retry when deleting remote wal files for sync replication
@@ -219,8 +224,6 @@ public class ReplicationSourceManager {
tfb.setDaemon(true);
this.executor.setThreadFactory(tfb.build());
this.latestPaths = new HashMap<>();
- this.replicationForBulkLoadDataEnabled = conf.getBoolean(
- HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
this.sleepForRetries = this.conf.getLong("replication.source.sync.sleepforretries", 1000);
this.maxRetriesMultiplier =
this.conf.getInt("replication.source.sync.maxretriesmultiplier", 60);
@@ -235,11 +238,6 @@ public class ReplicationSourceManager {
void init() throws IOException {
for (String id : this.replicationPeers.getAllPeerIds()) {
addSource(id);
- if (replicationForBulkLoadDataEnabled) {
- // Check if peer exists in hfile-refs queue, if not add it. This can happen in the case
- // when a peer was added before replication for bulk loaded data was enabled.
- throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(id));
- }
}
}
@@ -260,9 +258,6 @@ public class ReplicationSourceManager {
}
if (added) {
addSource(peerId);
- if (replicationForBulkLoadDataEnabled) {
- throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(peerId));
- }
}
}
@@ -293,26 +288,17 @@ public class ReplicationSourceManager {
removeRecoveredSource(src);
}
}
- LOG
- .info("Number of deleted recovered sources for " + peerId + ": " + oldSourcesToDelete.size());
+ LOG.info("Number of deleted recovered sources for {}: {}", peerId, oldSourcesToDelete.size());
// Now close the normal source for this peer
ReplicationSourceInterface srcToRemove = this.sources.get(peerId);
if (srcToRemove != null) {
srcToRemove.terminate(terminateMessage);
removeSource(srcToRemove);
- } else {
- // This only happened in unit test TestReplicationSourceManager#testPeerRemovalCleanup
- // Delete queue from storage and memory and queue id is same with peer id for normal
- // source
- deleteQueue(peerId);
- this.walsById.remove(peerId);
}
ReplicationPeerConfig peerConfig = peer.getPeerConfig();
if (peerConfig.isSyncReplication()) {
syncReplicationPeerMappingManager.remove(peerId, peerConfig);
}
- // Remove HFile Refs
- abortWhenFail(() -> this.queueStorage.removePeerFromHFileRefs(peerId));
}
/**
@@ -320,17 +306,17 @@ public class ReplicationSourceManager {
* @param queueId the id of the replication queue to associate the ReplicationSource with.
* @see #createCatalogReplicationSource(RegionInfo) for creating a ReplicationSource for meta.
*/
- private ReplicationSourceInterface createSource(String queueId, ReplicationPeer replicationPeer)
- throws IOException {
- ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, queueId);
+ private ReplicationSourceInterface createSource(ReplicationQueueData queueData,
+ ReplicationPeer replicationPeer) throws IOException {
+ ReplicationSourceInterface src = ReplicationSourceFactory.create(conf, queueData.getId());
// Init the just created replication source. Pass the default walProvider's wal file length
// provider. Presumption is we replicate user-space Tables only. For hbase:meta region replica
// replication, see #createCatalogReplicationSource().
WALFileLengthProvider walFileLengthProvider = this.walFactory.getWALProvider() != null
? this.walFactory.getWALProvider().getWALFileLengthProvider()
: p -> OptionalLong.empty();
- src.init(conf, fs, this, queueStorage, replicationPeer, server, queueId, clusterId,
- walFileLengthProvider, new MetricsSource(queueId));
+ src.init(conf, fs, this, queueStorage, replicationPeer, server, queueData, clusterId,
+ walFileLengthProvider, new MetricsSource(queueData.getId().toString()));
return src;
}
@@ -351,12 +337,14 @@ public class ReplicationSourceManager {
LOG.info("Legacy region replication peer found, skip adding: {}", peer.getPeerConfig());
return;
}
- ReplicationSourceInterface src = createSource(peerId, peer);
+ ReplicationQueueId queueId = new ReplicationQueueId(server.getServerName(), peerId);
+ ReplicationSourceInterface src =
+ createSource(new ReplicationQueueData(queueId, ImmutableMap.of()), peer);
// synchronized on latestPaths to avoid missing the new log
synchronized (this.latestPaths) {
this.sources.put(peerId, src);
Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();
- this.walsById.put(peerId, walsByGroup);
+ this.walsById.put(queueId, walsByGroup);
// Add the latest wal to that source's queue
if (!latestPaths.isEmpty()) {
for (Map.Entry<String, Path> walPrefixAndPath : latestPaths.entrySet()) {
@@ -365,8 +353,10 @@ public class ReplicationSourceManager {
wals.add(walPath.getName());
walsByGroup.put(walPrefixAndPath.getKey(), wals);
// Abort RS and throw exception to make add peer failed
+ // TODO: can record the length of the current wal file so we could replicate less data
abortAndThrowIOExceptionWhenFail(
- () -> this.queueStorage.addWAL(server.getServerName(), peerId, walPath.getName()));
+ () -> this.queueStorage.setOffset(queueId, walPrefixAndPath.getKey(),
+ new ReplicationGroupOffset(walPath.getName(), 0), Collections.emptyMap()));
src.enqueueLog(walPath);
LOG.trace("Enqueued {} to source {} during source creation.", walPath, src.getQueueId());
}
@@ -399,7 +389,10 @@ public class ReplicationSourceManager {
+ " is transiting to STANDBY. Will close the previous replication source and open a new one";
ReplicationPeer peer = replicationPeers.getPeer(peerId);
assert peer.getPeerConfig().isSyncReplication();
- ReplicationSourceInterface src = createSource(peerId, peer);
+ ReplicationQueueId queueId = new ReplicationQueueId(server.getServerName(), peerId);
+ // TODO: use empty initial offsets for now, revisit when adding support for sync replication
+ ReplicationSourceInterface src =
+ createSource(new ReplicationQueueData(queueId, ImmutableMap.of()), peer);
// synchronized here to avoid race with preLogRoll where we add new log to source and also
// walsById.
ReplicationSourceInterface toRemove;
@@ -416,17 +409,18 @@ public class ReplicationSourceManager {
// map from walsById since later we may fail to delete them from the replication queue
// storage, and when we retry next time, we can not know the wal files that need to be deleted
// from the replication queue storage.
- walsById.get(peerId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));
+ walsById.get(queueId).forEach((k, v) -> wals.put(k, new TreeSet<>(v)));
}
LOG.info("Startup replication source for " + src.getPeerId());
src.startup();
for (NavigableSet<String> walsByGroup : wals.values()) {
- for (String wal : walsByGroup) {
- queueStorage.removeWAL(server.getServerName(), peerId, wal);
- }
+ // TODO: just need to reset the replication offset
+ // for (String wal : walsByGroup) {
+ // queueStorage.removeWAL(server.getServerName(), peerId, wal);
+ // }
}
synchronized (walsById) {
- Map<String, NavigableSet<String>> oldWals = walsById.get(peerId);
+ Map<String, NavigableSet<String>> oldWals = walsById.get(queueId);
wals.forEach((k, v) -> {
NavigableSet<String> walsByGroup = oldWals.get(k);
if (walsByGroup != null) {
@@ -441,17 +435,28 @@ public class ReplicationSourceManager {
for (Iterator<ReplicationSourceInterface> iter = oldsources.iterator(); iter.hasNext();) {
ReplicationSourceInterface oldSource = iter.next();
if (oldSource.getPeerId().equals(peerId)) {
- String queueId = oldSource.getQueueId();
+ ReplicationQueueId oldSourceQueueId = oldSource.getQueueId();
oldSource.terminate(terminateMessage);
oldSource.getSourceMetrics().clear();
- queueStorage.removeQueue(server.getServerName(), queueId);
- walsByIdRecoveredQueues.remove(queueId);
+ queueStorage.removeQueue(oldSourceQueueId);
+ walsByIdRecoveredQueues.remove(oldSourceQueueId);
iter.remove();
}
}
}
}
+ private ReplicationSourceInterface createRefreshedSource(ReplicationQueueId queueId,
+ ReplicationPeer peer) throws IOException {
+ Map<String, ReplicationGroupOffset> offsets;
+ try {
+ offsets = queueStorage.getOffsets(queueId);
+ } catch (ReplicationException e) {
+ throw new IOException(e);
+ }
+ return createSource(new ReplicationQueueData(queueId, ImmutableMap.copyOf(offsets)), peer);
+ }
+
/**
* Close the previous replication sources of this peer id and open new sources to trigger the new
* replication state changes or new replication config changes. Here we don't need to change
@@ -462,6 +467,7 @@ public class ReplicationSourceManager {
String terminateMessage = "Peer " + peerId
+ " state or config changed. Will close the previous replication source and open a new one";
ReplicationPeer peer = replicationPeers.getPeer(peerId);
+ ReplicationQueueId queueId = new ReplicationQueueId(server.getServerName(), peerId);
ReplicationSourceInterface src;
// synchronized on latestPaths to avoid missing the new log
synchronized (this.latestPaths) {
@@ -471,9 +477,9 @@ public class ReplicationSourceManager {
// Do not clear metrics
toRemove.terminate(terminateMessage, null, false);
}
- src = createSource(peerId, peer);
+ src = createRefreshedSource(queueId, peer);
this.sources.put(peerId, src);
- for (NavigableSet<String> walsByGroup : walsById.get(peerId).values()) {
+ for (NavigableSet<String> walsByGroup : walsById.get(queueId).values()) {
walsByGroup.forEach(wal -> src.enqueueLog(new Path(this.logDir, wal)));
}
}
@@ -483,20 +489,22 @@ public class ReplicationSourceManager {
List<ReplicationSourceInterface> toStartup = new ArrayList<>();
// synchronized on oldsources to avoid race with NodeFailoverWorker
synchronized (this.oldsources) {
- List<String> previousQueueIds = new ArrayList<>();
+ List<ReplicationQueueId> oldSourceQueueIds = new ArrayList<>();
for (Iterator<ReplicationSourceInterface> iter = this.oldsources.iterator(); iter
.hasNext();) {
ReplicationSourceInterface oldSource = iter.next();
if (oldSource.getPeerId().equals(peerId)) {
- previousQueueIds.add(oldSource.getQueueId());
+ oldSourceQueueIds.add(oldSource.getQueueId());
oldSource.terminate(terminateMessage);
iter.remove();
}
}
- for (String queueId : previousQueueIds) {
- ReplicationSourceInterface recoveredReplicationSource = createSource(queueId, peer);
+ for (ReplicationQueueId oldSourceQueueId : oldSourceQueueIds) {
+ ReplicationSourceInterface recoveredReplicationSource =
+ createRefreshedSource(oldSourceQueueId, peer);
this.oldsources.add(recoveredReplicationSource);
- for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(queueId).values()) {
+ for (SortedSet<String> walsByGroup : walsByIdRecoveredQueues.get(oldSourceQueueId)
+ .values()) {
walsByGroup.forEach(wal -> recoveredReplicationSource.enqueueLog(new Path(wal)));
}
toStartup.add(recoveredReplicationSource);
@@ -549,8 +557,8 @@ public class ReplicationSourceManager {
* Delete a complete queue of wals associated with a replication source
* @param queueId the id of replication queue to delete
*/
- private void deleteQueue(String queueId) {
- abortWhenFail(() -> this.queueStorage.removeQueue(server.getServerName(), queueId));
+ private void deleteQueue(ReplicationQueueId queueId) {
+ abortWhenFail(() -> this.queueStorage.removeQueue(queueId));
}
@FunctionalInterface
@@ -616,10 +624,15 @@ public class ReplicationSourceManager {
*/
public void logPositionAndCleanOldLogs(ReplicationSourceInterface source,
WALEntryBatch entryBatch) {
- String fileName = entryBatch.getLastWalPath().getName();
- interruptOrAbortWhenFail(() -> this.queueStorage.setWALPosition(server.getServerName(),
- source.getQueueId(), fileName, entryBatch.getLastWalPosition(), entryBatch.getLastSeqIds()));
- cleanOldLogs(fileName, entryBatch.isEndOfFile(), source);
+ String walName = entryBatch.getLastWalPath().getName();
+ String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(walName);
+ // if end of file, we just set the offset to -1 so we know that this file has already been fully
+ // replicated, otherwise we need to compare the file length
+ ReplicationGroupOffset offset = new ReplicationGroupOffset(walName,
+ entryBatch.isEndOfFile() ? -1 : entryBatch.getLastWalPosition());
+ interruptOrAbortWhenFail(() -> this.queueStorage.setOffset(source.getQueueId(), walPrefix,
+ offset, entryBatch.getLastSeqIds()));
+ cleanOldLogs(walName, entryBatch.isEndOfFile(), source);
}
/**
@@ -644,7 +657,7 @@ public class ReplicationSourceManager {
} else {
NavigableSet<String> wals;
NavigableSet<String> walsToRemove;
- // synchronized on walsById to avoid race with preLogRoll
+ // synchronized on walsById to avoid race with postLogRoll
synchronized (this.walsById) {
wals = walsById.get(source.getQueueId()).get(logPrefix);
if (wals == null) {
@@ -726,33 +739,21 @@ public class ReplicationSourceManager {
}
}
}
- String queueId = source.getQueueId();
- for (String wal : wals) {
- interruptOrAbortWhenFail(
- () -> this.queueStorage.removeWAL(server.getServerName(), queueId, wal));
- }
}
// public because of we call it in TestReplicationEmptyWALRecovery
- public void preLogRoll(Path newLog) throws IOException {
+ public void postLogRoll(Path newLog) throws IOException {
String logName = newLog.getName();
String logPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(logName);
// synchronized on latestPaths to avoid the new open source miss the new log
synchronized (this.latestPaths) {
- // Add log to queue storage
- for (ReplicationSourceInterface source : this.sources.values()) {
- // If record log to queue storage failed, abort RS and throw exception to make log roll
- // failed
- abortAndThrowIOExceptionWhenFail(
- () -> this.queueStorage.addWAL(server.getServerName(), source.getQueueId(), logName));
- }
-
// synchronized on walsById to avoid race with cleanOldLogs
synchronized (this.walsById) {
// Update walsById map
- for (Map.Entry<String, Map<String, NavigableSet<String>>> entry : this.walsById
+ for (Map.Entry<ReplicationQueueId, Map<String, NavigableSet<String>>> entry : this.walsById
.entrySet()) {
- String peerId = entry.getKey();
+ ReplicationQueueId queueId = entry.getKey();
+ String peerId = queueId.getPeerId();
Map<String, NavigableSet<String>> walsByPrefix = entry.getValue();
boolean existingPrefix = false;
for (Map.Entry<String, NavigableSet<String>> walsEntry : walsByPrefix.entrySet()) {
@@ -780,10 +781,6 @@ public class ReplicationSourceManager {
// Add to latestPaths
latestPaths.put(logPrefix, newLog);
}
- }
-
- // public because of we call it in TestReplicationEmptyWALRecovery
- public void postLogRoll(Path newLog) throws IOException {
// This only updates the sources we own, not the recovered ones
for (ReplicationSourceInterface source : this.sources.values()) {
source.enqueueLog(newLog);
@@ -792,7 +789,29 @@ public class ReplicationSourceManager {
}
}
- void claimQueue(ServerName deadRS, String queue) {
+ /**
+ * Check whether we should replicate the given {@code wal}.
+ * @param wal the file name of the wal
+ * @return {@code true} means we should replicate the given {@code wal}, otherwise {@code false}.
+ */
+ private boolean shouldReplicate(ReplicationGroupOffset offset, String wal) {
+ if (offset == null || offset == ReplicationGroupOffset.BEGIN) {
+ return false;
+ }
+ long walTs = AbstractFSWALProvider.getTimestamp(wal);
+ long startWalTs = AbstractFSWALProvider.getTimestamp(offset.getWal());
+ if (walTs < startWalTs) {
+ return false;
+ } else if (walTs > startWalTs) {
+ return true;
+ }
+ // if the timestamp equals, usually it means we should include this wal but there is a special
+ // case, a negative offset means the wal has already been fully replicated, so here we should
+ // check the offset.
+ return offset.getOffset() >= 0;
+ }
+
+ void claimQueue(ReplicationQueueId queueId) {
// Wait a bit before transferring the queues, we may be shutting down.
// This sleep may not be enough in some cases.
try {
@@ -807,66 +826,83 @@ public class ReplicationSourceManager {
LOG.info("Not transferring queue since we are shutting down");
return;
}
- // After claim the queues from dead region server, wewill skip to start the
+ // After claim the queues from dead region server, we will skip to start the
// RecoveredReplicationSource if the peer has been removed. but there's possible that remove a
// peer with peerId = 2 and add a peer with peerId = 2 again during failover. So we need to get
// a copy of the replication peer first to decide whether we should start the
// RecoveredReplicationSource. If the latest peer is not the old peer, we should also skip to
// start the RecoveredReplicationSource, Otherwise the rs will abort (See HBASE-20475).
- String peerId = new ReplicationQueueInfo(queue).getPeerId();
+ String peerId = queueId.getPeerId();
ReplicationPeerImpl oldPeer = replicationPeers.getPeer(peerId);
if (oldPeer == null) {
LOG.info("Not transferring queue since the replication peer {} for queue {} does not exist",
- peerId, queue);
+ peerId, queueId);
return;
}
- Pair<String, SortedSet<String>> claimedQueue;
+ Map<String, ReplicationGroupOffset> offsets;
try {
- claimedQueue = queueStorage.claimQueue(deadRS, queue, server.getServerName());
+ offsets = queueStorage.claimQueue(queueId, server.getServerName());
} catch (ReplicationException e) {
- LOG.error(
- "ReplicationException: cannot claim dead region ({})'s " + "replication queue. Znode : ({})"
- + " Possible solution: check if znode size exceeds jute.maxBuffer value. "
- + " If so, increase it for both client and server side.",
- deadRS, queueStorage.getRsNode(deadRS), e);
+ LOG.error("ReplicationException: cannot claim dead region ({})'s replication queue",
+ queueId.getServerName(), e);
server.abort("Failed to claim queue from dead regionserver.", e);
return;
}
- if (claimedQueue.getSecond().isEmpty()) {
+ if (offsets.isEmpty()) {
+ // someone else claimed the queue
return;
}
- String queueId = claimedQueue.getFirst();
- Set<String> walsSet = claimedQueue.getSecond();
+ ServerName sourceRS = queueId.getServerWALsBelongTo();
+ ReplicationQueueId claimedQueueId = queueId.claim(server.getServerName());
ReplicationPeerImpl peer = replicationPeers.getPeer(peerId);
if (peer == null || peer != oldPeer) {
- LOG.warn("Skipping failover for peer {} of node {}, peer is null", peerId, deadRS);
- abortWhenFail(() -> queueStorage.removeQueue(server.getServerName(), queueId));
+ LOG.warn("Skipping failover for peer {} of node {}, peer is null", peerId, sourceRS);
+ deleteQueue(claimedQueueId);
return;
}
- if (
- server instanceof ReplicationSyncUp.DummyServer
- && peer.getPeerState().equals(PeerState.DISABLED)
- ) {
- LOG.warn(
- "Peer {} is disabled. ReplicationSyncUp tool will skip " + "replicating data to this peer.",
- peerId);
- return;
- }
-
ReplicationSourceInterface src;
try {
- src = createSource(queueId, peer);
+ src =
+ createSource(new ReplicationQueueData(claimedQueueId, ImmutableMap.copyOf(offsets)), peer);
} catch (IOException e) {
- LOG.error("Can not create replication source for peer {} and queue {}", peerId, queueId, e);
+ LOG.error("Can not create replication source for peer {} and queue {}", peerId,
+ claimedQueueId, e);
server.abort("Failed to create replication source after claiming queue.", e);
return;
}
+ List<Path> walFiles;
+ try {
+ walFiles = AbstractFSWALProvider.getArchivedWALFiles(conf, sourceRS,
+ URLEncoder.encode(sourceRS.toString(), StandardCharsets.UTF_8.name()));
+ } catch (IOException e) {
+ LOG.error("Can not list all wal files for peer {} and queue {}", peerId, queueId, e);
+ server.abort("Can not list all wal files after claiming queue.", e);
+ return;
+ }
+ PriorityQueue<Path> walFilesPQ = new PriorityQueue<>(
+ Comparator.<Path, Long> comparing(p -> AbstractFSWALProvider.getTimestamp(p.getName()))
+ .thenComparing(Path::getName));
+ // sort the wal files and also filter out replicated files
+ for (Path file : walFiles) {
+ String walGroupId = AbstractFSWALProvider.getWALPrefixFromWALName(file.getName());
+ ReplicationGroupOffset groupOffset = offsets.get(walGroupId);
+ if (shouldReplicate(groupOffset, file.getName())) {
+ walFilesPQ.add(file);
+ } else {
+ LOG.debug("Skip enqueuing log {} because it is before the start offset {}", file.getName(),
+ groupOffset);
+ }
+ walFilesPQ.add(file);
+ }
+ // the method is a bit long, so assign it to null here to avoid later we reuse it again by
+ // mistake, we should use the sorted walFilesPQ instead
+ walFiles = null;
// synchronized on oldsources to avoid adding recovered source for the to-be-removed peer
synchronized (oldsources) {
peer = replicationPeers.getPeer(src.getPeerId());
if (peer == null || peer != oldPeer) {
src.terminate("Recovered queue doesn't belong to any current peer");
- deleteQueue(queueId);
+ deleteQueue(claimedQueueId);
return;
}
// Do not setup recovered queue if a sync replication peer is in STANDBY state, or is
@@ -882,26 +918,26 @@ public class ReplicationSourceManager {
|| stateAndNewState.getSecond().equals(SyncReplicationState.STANDBY)
) {
src.terminate("Sync replication peer is in STANDBY state");
- deleteQueue(queueId);
+ deleteQueue(claimedQueueId);
return;
}
}
// track sources in walsByIdRecoveredQueues
Map<String, NavigableSet<String>> walsByGroup = new HashMap<>();
- walsByIdRecoveredQueues.put(queueId, walsByGroup);
- for (String wal : walsSet) {
- String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal);
+ walsByIdRecoveredQueues.put(claimedQueueId, walsByGroup);
+ for (Path wal : walFilesPQ) {
+ String walPrefix = AbstractFSWALProvider.getWALPrefixFromWALName(wal.getName());
NavigableSet<String> wals = walsByGroup.get(walPrefix);
if (wals == null) {
wals = new TreeSet<>();
walsByGroup.put(walPrefix, wals);
}
- wals.add(wal);
+ wals.add(wal.getName());
}
oldsources.add(src);
- LOG.info("Added source for recovered queue {}", src.getQueueId());
- for (String wal : walsSet) {
- LOG.trace("Enqueueing log from recovered queue for source: " + src.getQueueId());
+ LOG.info("Added source for recovered queue {}", claimedQueueId);
+ for (Path wal : walFilesPQ) {
+ LOG.debug("Enqueueing log {} from recovered queue for source: {}", wal, claimedQueueId);
src.enqueueLog(new Path(oldLogDir, wal));
}
src.startup();
@@ -927,7 +963,9 @@ public class ReplicationSourceManager {
* Get a copy of the wals of the normal sources on this rs
* @return a sorted set of wal names
*/
- public Map<String, Map<String, NavigableSet<String>>> getWALs() {
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ public Map<ReplicationQueueId, Map<String, NavigableSet<String>>> getWALs() {
return Collections.unmodifiableMap(walsById);
}
@@ -935,7 +973,9 @@ public class ReplicationSourceManager {
* Get a copy of the wals of the recovered sources on this rs
* @return a sorted set of wal names
*/
- Map<String, Map<String, NavigableSet<String>>> getWalsByIdRecoveredQueues() {
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
+ Map<ReplicationQueueId, Map<String, NavigableSet<String>>> getWalsByIdRecoveredQueues() {
return Collections.unmodifiableMap(walsByIdRecoveredQueues);
}
@@ -963,16 +1003,6 @@ public class ReplicationSourceManager {
return this.sources.get(peerId);
}
- List<String> getAllQueues() throws IOException {
- List<String> allQueues = Collections.emptyList();
- try {
- allQueues = queueStorage.getAllQueues(server.getServerName());
- } catch (ReplicationException e) {
- throw new IOException(e);
- }
- return allQueues;
- }
-
int getSizeOfLatestPath() {
synchronized (latestPaths) {
return latestPaths.size();
@@ -1068,6 +1098,8 @@ public class ReplicationSourceManager {
return this.globalMetrics;
}
+ @RestrictedApi(explanation = "Should only be called in tests", link = "",
+ allowedOnPath = ".*/src/test/.*")
ReplicationQueueStorage getQueueStorage() {
return queueStorage;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java
index 0733c61bc20..7b863dc35ae 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java
@@ -57,7 +57,7 @@ public class ReplicationSourceShipper extends Thread {
private final Configuration conf;
protected final String walGroupId;
protected final ReplicationSourceLogQueue logQueue;
- private final ReplicationSource source;
+ protected final ReplicationSource source;
// Last position in the log that we sent to ZooKeeper
// It will be accessed by the stats thread so make it volatile
@@ -299,10 +299,6 @@ public class ReplicationSourceShipper extends Thread {
this.entryReader = entryReader;
}
- long getStartPosition() {
- return 0;
- }
-
protected boolean isActive() {
return source.isSourceActive() && state == WorkerState.RUNNING && !isInterrupted();
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java
index 7337694addb..9380c6b6305 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALActionListener.java
@@ -43,11 +43,6 @@ class ReplicationSourceWALActionListener implements WALActionsListener {
this.manager = manager;
}
- @Override
- public void preLogRoll(Path oldPath, Path newPath) throws IOException {
- manager.preLogRoll(newPath);
- }
-
@Override
public void postLogRoll(Path oldPath, Path newPath) throws IOException {
manager.postLogRoll(newPath);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
index daf9081234d..50ffd6df1af 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSyncUp.java
@@ -80,17 +80,18 @@ public class ReplicationSyncUp extends Configured implements Tool {
// replication queues for the dead region servers first and then replicate the data out.
private void claimReplicationQueues(ZKWatcher zkw, ReplicationSourceManager mgr)
throws ReplicationException, KeeperException {
- List<ServerName> replicators = mgr.getQueueStorage().getListOfReplicators();
- Set<ServerName> liveRegionServers = getLiveRegionServers(zkw);
- for (ServerName sn : replicators) {
- if (!liveRegionServers.contains(sn)) {
- List<String> replicationQueues = mgr.getQueueStorage().getAllQueues(sn);
- System.out.println(sn + " is dead, claim its replication queues: " + replicationQueues);
- for (String queue : replicationQueues) {
- mgr.claimQueue(sn, queue);
- }
- }
- }
+ // TODO: reimplement this tool
+ // List<ServerName> replicators = mgr.getQueueStorage().getListOfReplicators();
+ // Set<ServerName> liveRegionServers = getLiveRegionServers(zkw);
+ // for (ServerName sn : replicators) {
+ // if (!liveRegionServers.contains(sn)) {
+ // List<String> replicationQueues = mgr.getQueueStorage().getAllQueues(sn);
+ // System.out.println(sn + " is dead, claim its replication queues: " + replicationQueues);
+ // for (String queue : replicationQueues) {
+ // mgr.claimQueue(sn, queue);
+ // }
+ // }
+ // }
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
index 7e10fd786a4..36acffa9664 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
@@ -2558,7 +2558,7 @@ public class HBaseFsck extends Configured implements Closeable {
}
private void checkAndFixReplication() throws ReplicationException, IOException {
- ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, errors);
+ ReplicationChecker checker = new ReplicationChecker(getConf(), zkw, connection, errors);
checker.checkUnDeletedQueues();
if (checker.hasUnDeletedQueues() && this.fixReplication) {
@@ -3831,7 +3831,7 @@ public class HBaseFsck extends Configured implements Closeable {
return;
}
ReplicationQueueStorage queueStorage =
- ReplicationStorageFactory.getReplicationQueueStorage(zkw, getConf());
+ ReplicationStorageFactory.getReplicationQueueStorage(connection, getConf());
List<ReplicationPeerDescription> peerDescriptions = admin.listReplicationPeers();
if (peerDescriptions != null && peerDescriptions.size() > 0) {
List<String> peers = peerDescriptions.stream()
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
index 7e7a46573b8..497304a3111 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/ReplicationChecker.java
@@ -27,9 +27,11 @@ import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.util.HbckErrorReporter;
@@ -48,18 +50,18 @@ public class ReplicationChecker {
private final HbckErrorReporter errorReporter;
// replicator with its queueIds for removed peers
- private Map<ServerName, List<String>> undeletedQueueIds = new HashMap<>();
+ private Map<ServerName, List<ReplicationQueueId>> undeletedQueueIds = new HashMap<>();
// replicator with its undeleted queueIds for removed peers in hfile-refs queue
private Set<String> undeletedHFileRefsPeerIds = new HashSet<>();
private final ReplicationPeerStorage peerStorage;
private final ReplicationQueueStorage queueStorage;
- public ReplicationChecker(Configuration conf, ZKWatcher zkw, HbckErrorReporter errorReporter)
- throws IOException {
+ public ReplicationChecker(Configuration conf, ZKWatcher zkw, Connection conn,
+ HbckErrorReporter errorReporter) throws IOException {
this.peerStorage =
ReplicationStorageFactory.getReplicationPeerStorage(FileSystem.get(conf), zkw, conf);
- this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(zkw, conf);
+ this.queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(conn, conf);
this.errorReporter = errorReporter;
}
@@ -68,19 +70,19 @@ public class ReplicationChecker {
.contains(HbckErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE);
}
- private Map<ServerName, List<String>> getUnDeletedQueues() throws ReplicationException {
- Map<ServerName, List<String>> undeletedQueues = new HashMap<>();
+ private Map<ServerName, List<ReplicationQueueId>> getUnDeletedQueues()
+ throws ReplicationException {
+ Map<ServerName, List<ReplicationQueueId>> undeletedQueues = new HashMap<>();
Set<String> peerIds = new HashSet<>(peerStorage.listPeerIds());
- for (ServerName replicator : queueStorage.getListOfReplicators()) {
- for (String queueId : queueStorage.getAllQueues(replicator)) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (!peerIds.contains(queueInfo.getPeerId())) {
- undeletedQueues.computeIfAbsent(replicator, key -> new ArrayList<>()).add(queueId);
- LOG.debug(
- "Undeleted replication queue for removed peer found: "
- + "[removedPeerId={}, replicator={}, queueId={}]",
- queueInfo.getPeerId(), replicator, queueId);
- }
+ for (ReplicationQueueData queueData : queueStorage.listAllQueues()) {
+ ReplicationQueueId queueId = queueData.getId();
+ if (!peerIds.contains(queueId.getPeerId())) {
+ undeletedQueues.computeIfAbsent(queueId.getServerName(), key -> new ArrayList<>())
+ .add(queueId);
+ LOG.debug(
+ "Undeleted replication queue for removed peer found: "
+ + "[removedPeerId={}, replicator={}, queueId={}]",
+ queueId.getPeerId(), queueId.getServerName(), queueId);
}
}
return undeletedQueues;
@@ -103,9 +105,8 @@ public class ReplicationChecker {
undeletedQueueIds = getUnDeletedQueues();
undeletedQueueIds.forEach((replicator, queueIds) -> {
queueIds.forEach(queueId -> {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
String msg = "Undeleted replication queue for removed peer found: "
- + String.format("[removedPeerId=%s, replicator=%s, queueId=%s]", queueInfo.getPeerId(),
+ + String.format("[removedPeerId=%s, replicator=%s, queueId=%s]", queueId.getPeerId(),
replicator, queueId);
errorReporter.reportError(HbckErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE, msg);
});
@@ -118,12 +119,12 @@ public class ReplicationChecker {
}
public void fixUnDeletedQueues() throws ReplicationException {
- for (Map.Entry<ServerName, List<String>> replicatorAndQueueIds : undeletedQueueIds.entrySet()) {
+ for (Map.Entry<ServerName, List<ReplicationQueueId>> replicatorAndQueueIds : undeletedQueueIds
+ .entrySet()) {
ServerName replicator = replicatorAndQueueIds.getKey();
- for (String queueId : replicatorAndQueueIds.getValue()) {
- queueStorage.removeQueue(replicator, queueId);
+ for (ReplicationQueueId queueId : replicatorAndQueueIds.getValue()) {
+ queueStorage.removeQueue(queueId);
}
- queueStorage.removeReplicatorIfQueueIsEmpty(replicator);
}
for (String peerId : undeletedHFileRefsPeerIds) {
queueStorage.removePeerFromHFileRefs(peerId);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
index 4ca32e83e63..db39a8ba023 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvider.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.wal;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
@@ -28,6 +29,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Abortable;
@@ -335,6 +337,44 @@ public abstract class AbstractFSWALProvider<T extends AbstractFSWAL<?>> implemen
return dirName.toString();
}
+ /**
+ * List all the old wal files for a dead region server.
+ * <p/>
+ * Initially added for supporting replication, where we need to get the wal files to replicate for
+ * a dead region server.
+ */
+ public static List<Path> getArchivedWALFiles(Configuration conf, ServerName serverName,
+ String logPrefix) throws IOException {
+ Path walRootDir = CommonFSUtils.getWALRootDir(conf);
+ FileSystem fs = walRootDir.getFileSystem(conf);
+ List<Path> archivedWalFiles = new ArrayList<>();
+ // list both the root old wal dir and the separate old wal dir, so we will not miss any files if
+ // the SEPARATE_OLDLOGDIR config is changed
+ Path oldWalDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ try {
+ for (FileStatus status : fs.listStatus(oldWalDir, p -> p.getName().startsWith(logPrefix))) {
+ if (status.isFile()) {
+ archivedWalFiles.add(status.getPath());
+ }
+ }
+ } catch (FileNotFoundException e) {
+ LOG.info("Old WAL dir {} not exists", oldWalDir);
+ return Collections.emptyList();
+ }
+ Path separatedOldWalDir = new Path(oldWalDir, serverName.toString());
+ try {
+ for (FileStatus status : fs.listStatus(separatedOldWalDir,
+ p -> p.getName().startsWith(logPrefix))) {
+ if (status.isFile()) {
+ archivedWalFiles.add(status.getPath());
+ }
+ }
+ } catch (FileNotFoundException e) {
+ LOG.info("Seprated old WAL dir {} not exists", separatedOldWalDir);
+ }
+ return archivedWalFiles;
+ }
+
/**
* Pulls a ServerName out of a Path generated according to our layout rules. In the below layouts,
* this method ignores the format of the logfile component. Current format: [base directory for
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
index f942f4ed99d..157277d8302 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncReplicationAdminApi.java
@@ -43,12 +43,12 @@ import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
-import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfigBuilder;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint;
@@ -102,11 +102,9 @@ public class TestAsyncReplicationAdminApi extends TestAsyncAdminBase {
} catch (Exception e) {
}
ReplicationQueueStorage queueStorage = ReplicationStorageFactory
- .getReplicationQueueStorage(TEST_UTIL.getZooKeeperWatcher(), TEST_UTIL.getConfiguration());
- for (ServerName serverName : queueStorage.getListOfReplicators()) {
- for (String queue : queueStorage.getAllQueues(serverName)) {
- queueStorage.removeQueue(serverName, queue);
- }
+ .getReplicationQueueStorage(TEST_UTIL.getConnection(), TEST_UTIL.getConfiguration());
+ for (ReplicationQueueData queueData : queueStorage.listAllQueues()) {
+ queueStorage.removeQueue(queueData.getId());
}
admin.replicationPeerModificationSwitch(true).join();
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
index 11f882eb45a..c601425e5f0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
@@ -151,7 +151,7 @@ public class MockMasterServices extends MockNoopMasterServices {
CommonFSUtils.setRootDir(getConfiguration(), rootdir);
this.rpm = mock(ReplicationPeerManager.class);
ReplicationQueueStorage rqs = mock(ReplicationQueueStorage.class);
- when(rqs.getAllQueues(any())).thenReturn(Collections.emptyList());
+ when(rqs.listAllQueueIds(any(ServerName.class))).thenReturn(Collections.emptyList());
when(rpm.getQueueStorage()).thenReturn(rqs);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
index e9e5f973cf8..1a0537bcbaf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestLogsCleaner.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.master.cleaner;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
@@ -45,8 +44,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
@@ -62,13 +61,14 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+// revisit later after we implement new replication log cleaner
+@Ignore
@Category({ MasterTests.class, MediumTests.class })
public class TestLogsCleaner {
@@ -131,8 +131,8 @@ public class TestLogsCleaner {
HMaster.decorateMasterConfiguration(conf);
Server server = new DummyServer();
- ReplicationQueueStorage queueStorage =
- ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
+ ReplicationQueueStorage queueStorage = ReplicationStorageFactory
+ .getReplicationQueueStorage(ConnectionFactory.createConnection(conf), conf);
String fakeMachineName =
URLEncoder.encode(server.getServerName().toString(), StandardCharsets.UTF_8.name());
@@ -162,7 +162,7 @@ public class TestLogsCleaner {
// Case 4: put 3 WALs in ZK indicating that they are scheduled for replication so these
// files would pass TimeToLiveLogCleaner but would be rejected by ReplicationLogCleaner
if (i % (30 / 3) == 0) {
- queueStorage.addWAL(server.getServerName(), fakeMachineName, fileName.getName());
+ // queueStorage.addWAL(server.getServerName(), fakeMachineName, fileName.getName());
LOG.info("Replication log file: " + fileName);
}
}
@@ -222,20 +222,20 @@ public class TestLogsCleaner {
try {
faultyZK.init(false);
- ReplicationQueueStorage queueStorage =
- spy(ReplicationStorageFactory.getReplicationQueueStorage(faultyZK, conf));
- doAnswer(new Answer<Object>() {
- @Override
- public Object answer(InvocationOnMock invocation) throws Throwable {
- try {
- return invocation.callRealMethod();
- } catch (ReplicationException e) {
- LOG.debug("Caught Exception", e);
- getListOfReplicatorsFailed.set(true);
- throw e;
- }
- }
- }).when(queueStorage).getAllWALs();
+ ReplicationQueueStorage queueStorage = spy(ReplicationStorageFactory
+ .getReplicationQueueStorage(ConnectionFactory.createConnection(conf), conf));
+ // doAnswer(new Answer<Object>() {
+ // @Override
+ // public Object answer(InvocationOnMock invocation) throws Throwable {
+ // try {
+ // return invocation.callRealMethod();
+ // } catch (ReplicationException e) {
+ // LOG.debug("Caught Exception", e);
+ // getListOfReplicatorsFailed.set(true);
+ // throw e;
+ // }
+ // }
+ // }).when(queueStorage).getAllWALs();
cleaner.setConf(conf, faultyZK, queueStorage);
// should keep all files due to a ConnectionLossException getting the queues znodes
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
index 87d21e583dd..2409b081cce 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestReplicationHFileCleaner.java
@@ -17,14 +17,12 @@
*/
package org.apache.hadoop.hbase.master.cleaner;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.spy;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -32,12 +30,11 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ZooKeeperConnectionException;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
@@ -49,25 +46,21 @@ import org.apache.hadoop.hbase.replication.SyncReplicationState;
import org.apache.hadoop.hbase.replication.master.ReplicationHFileCleaner;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.MockServer;
import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.data.Stat;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-
+// TODO: revisit later
+@Ignore
@Category({ MasterTests.class, SmallTests.class })
public class TestReplicationHFileCleaner {
@@ -87,14 +80,14 @@ public class TestReplicationHFileCleaner {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
- TEST_UTIL.startMiniZKCluster();
+ TEST_UTIL.startMiniCluster();
server = new DummyServer();
conf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
HMaster.decorateMasterConfiguration(conf);
rp =
ReplicationFactory.getReplicationPeers(server.getFileSystem(), server.getZooKeeper(), conf);
rp.init();
- rq = ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
+ rq = ReplicationStorageFactory.getReplicationQueueStorage(server.getConnection(), conf);
fs = FileSystem.get(conf);
}
@@ -109,7 +102,6 @@ public class TestReplicationHFileCleaner {
rp.getPeerStorage().addPeer(peerId,
ReplicationPeerConfig.newBuilder().setClusterKey(TEST_UTIL.getClusterKey()).build(), true,
SyncReplicationState.NONE);
- rq.addPeerToHFileRefs(peerId);
}
@After
@@ -184,47 +176,6 @@ public class TestReplicationHFileCleaner {
assertTrue(deletableFilesIterator.next().getPath().equals(deletablefile));
}
- /**
- * ReplicationHFileCleaner should be able to ride over ZooKeeper errors without aborting.
- */
- @Test
- public void testZooKeeperAbort() throws Exception {
- ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner();
-
- List<FileStatus> dummyFiles = Lists.newArrayList(
- new FileStatus(100, false, 3, 100, EnvironmentEdgeManager.currentTime(), new Path("hfile1")),
- new FileStatus(100, false, 3, 100, EnvironmentEdgeManager.currentTime(), new Path("hfile2")));
-
- FaultyZooKeeperWatcher faultyZK =
- new FaultyZooKeeperWatcher(conf, "testZooKeeperAbort-faulty", null);
- try {
- faultyZK.init();
- cleaner.setConf(conf, faultyZK);
- // should keep all files due to a ConnectionLossException getting the queues znodes
- Iterable<FileStatus> toDelete = cleaner.getDeletableFiles(dummyFiles);
- assertFalse(toDelete.iterator().hasNext());
- assertFalse(cleaner.isStopped());
- } finally {
- faultyZK.close();
- }
-
- // when zk is working both files should be returned
- cleaner = new ReplicationHFileCleaner();
- ZKWatcher zkw = new ZKWatcher(conf, "testZooKeeperAbort-normal", null);
- try {
- cleaner.setConf(conf, zkw);
- Iterable<FileStatus> filesToDelete = cleaner.getDeletableFiles(dummyFiles);
- Iterator<FileStatus> iter = filesToDelete.iterator();
- assertTrue(iter.hasNext());
- assertEquals(new Path("hfile1"), iter.next().getPath());
- assertTrue(iter.hasNext());
- assertEquals(new Path("hfile2"), iter.next().getPath());
- assertFalse(iter.hasNext());
- } finally {
- zkw.close();
- }
- }
-
static class DummyServer extends MockServer {
@Override
@@ -233,13 +184,12 @@ public class TestReplicationHFileCleaner {
}
@Override
- public ZKWatcher getZooKeeper() {
+ public Connection getConnection() {
try {
- return new ZKWatcher(getConfiguration(), "dummy server", this);
+ return TEST_UTIL.getConnection();
} catch (IOException e) {
- LOG.error("Can not get ZKWatcher", e);
+ throw new UncheckedIOException(e);
}
- return null;
}
@Override
@@ -247,29 +197,8 @@ public class TestReplicationHFileCleaner {
try {
return TEST_UTIL.getTestFileSystem();
} catch (IOException e) {
- LOG.error("Can not get FileSystem", e);
+ throw new UncheckedIOException(e);
}
- return null;
- }
- }
-
- static class FaultyZooKeeperWatcher extends ZKWatcher {
- private RecoverableZooKeeper zk;
-
- public FaultyZooKeeperWatcher(Configuration conf, String identifier, Abortable abortable)
- throws ZooKeeperConnectionException, IOException {
- super(conf, identifier, abortable);
- }
-
- public void init() throws Exception {
- this.zk = spy(super.getRecoverableZooKeeper());
- doThrow(new KeeperException.ConnectionLossException()).when(zk)
- .getData("/hbase/replication/hfile-refs", null, new Stat());
- }
-
- @Override
- public RecoverableZooKeeper getRecoverableZooKeeper() {
- return zk;
}
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplicationHFileRefs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplicationHFileRefs.java
index 70a6e88552b..787784c8ec4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplicationHFileRefs.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBulkLoadReplicationHFileRefs.java
@@ -114,7 +114,7 @@ public class TestBulkLoadReplicationHFileRefs extends TestReplicationBase {
admin1 = UTIL1.getConnection().getAdmin();
admin2 = UTIL2.getConnection().getAdmin();
- queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(UTIL1.getZooKeeperWatcher(),
+ queueStorage = ReplicationStorageFactory.getReplicationQueueStorage(UTIL1.getConnection(),
UTIL1.getConfiguration());
admin1.createNamespace(NamespaceDescriptor.create(REPLICATE_NAMESPACE).build());
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/regionreplication/TestStartupWithLegacyRegionReplicationEndpoint.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/regionreplication/TestStartupWithLegacyRegionReplicationEndpoint.java
index 66eaff0493e..5af9edb8efc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/regionreplication/TestStartupWithLegacyRegionReplicationEndpoint.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/regionreplication/TestStartupWithLegacyRegionReplicationEndpoint.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.Collections;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
@@ -29,9 +30,13 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.replication.ReplicationPeerManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
+import org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceInterface;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
@@ -57,6 +62,10 @@ public class TestStartupWithLegacyRegionReplicationEndpoint {
@BeforeClass
public static void setUp() throws Exception {
UTIL.startMiniCluster(1);
+ // add a peer to force initialize the replication storage
+ UTIL.getAdmin().addReplicationPeer("1", ReplicationPeerConfig.newBuilder()
+ .setClusterKey(UTIL.getZkCluster().getAddress().toString() + ":/1").build());
+ UTIL.getAdmin().removeReplicationPeer("1");
}
@AfterClass
@@ -66,40 +75,42 @@ public class TestStartupWithLegacyRegionReplicationEndpoint {
@Test
public void test() throws Exception {
+ String peerId = "legacy";
ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
.setClusterKey("127.0.0.1:2181:/hbase")
.setReplicationEndpointImpl(ReplicationUtils.LEGACY_REGION_REPLICATION_ENDPOINT_NAME).build();
SingleProcessHBaseCluster cluster = UTIL.getMiniHBaseCluster();
HMaster master = cluster.getMaster();
// can not use Admin.addPeer as it will fail with ClassNotFound
- master.getReplicationPeerManager().addPeer("legacy", peerConfig, true);
+ master.getReplicationPeerManager().addPeer(peerId, peerConfig, true);
// add a wal file to the queue
ServerName rsName = cluster.getRegionServer(0).getServerName();
- master.getReplicationPeerManager().getQueueStorage().addWAL(rsName,
- ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER, "test-wal-file");
+ master.getReplicationPeerManager().getQueueStorage().setOffset(
+ new ReplicationQueueId(rsName, ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_PEER), "",
+ new ReplicationGroupOffset("test-wal-file", 0), Collections.emptyMap());
cluster.stopRegionServer(0);
RegionServerThread rst = cluster.startRegionServer();
// we should still have this peer
- assertNotNull(UTIL.getAdmin().getReplicationPeerConfig("legacy"));
+ assertNotNull(UTIL.getAdmin().getReplicationPeerConfig(peerId));
// but at RS side, we should not have this peer loaded as replication source
- assertTrue(rst.getRegionServer().getReplicationSourceService().getReplicationManager()
- .getSources().isEmpty());
+ assertTrue(
+ rst.getRegionServer().getReplicationSourceService().getReplicationManager().getSources()
+ .stream().map(ReplicationSourceInterface::getPeerId).noneMatch(p -> p.equals(peerId)));
UTIL.shutdownMiniHBaseCluster();
UTIL.restartHBaseCluster(1);
// now we should have removed the peer
assertThrows(ReplicationPeerNotFoundException.class,
() -> UTIL.getAdmin().getReplicationPeerConfig("legacy"));
- // at rs side, we should not have the peer this time, not only for not having replication source
- assertTrue(UTIL.getMiniHBaseCluster().getRegionServer(0).getReplicationSourceService()
- .getReplicationManager().getReplicationPeers().getAllPeerIds().isEmpty());
- // make sure that we can finish the SCP and delete the test-wal-file
+ // make sure that we can finish the SCP
UTIL.waitFor(15000,
() -> UTIL.getMiniHBaseCluster().getMaster().getProcedures().stream()
.filter(p -> p instanceof ServerCrashProcedure).map(p -> (ServerCrashProcedure) p)
.allMatch(Procedure::isSuccess));
- assertTrue(UTIL.getMiniHBaseCluster().getMaster().getReplicationPeerManager().getQueueStorage()
- .getAllQueues(rsName).isEmpty());
+ // the deletion is async, so wait until they get deleted
+ ReplicationPeerManager ppm = UTIL.getMiniHBaseCluster().getMaster().getReplicationPeerManager();
+ UTIL.waitFor(15000, () -> !ppm.getPeerStorage().listPeerIds().contains(peerId)
+ && ppm.getQueueStorage().listAllQueueIds(peerId, rsName).isEmpty());
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
index 52ea2cd6050..da0868be885 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/ReplicationSourceDummy.java
@@ -41,7 +41,7 @@ public class ReplicationSourceDummy implements ReplicationSourceInterface {
private ReplicationSourceManager manager;
private ReplicationPeer replicationPeer;
- private String peerClusterId;
+ private ReplicationQueueId queueId;
private Path currentPath;
private MetricsSource metrics;
private WALFileLengthProvider walFileLengthProvider;
@@ -49,11 +49,11 @@ public class ReplicationSourceDummy implements ReplicationSourceInterface {
@Override
public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueueStorage rq, ReplicationPeer rp, Server server, String peerClusterId,
+ ReplicationQueueStorage rq, ReplicationPeer rp, Server server, ReplicationQueueData queueData,
UUID clusterId, WALFileLengthProvider walFileLengthProvider, MetricsSource metrics)
throws IOException {
this.manager = manager;
- this.peerClusterId = peerClusterId;
+ this.queueId = queueData.getId();
this.metrics = metrics;
this.walFileLengthProvider = walFileLengthProvider;
this.replicationPeer = rp;
@@ -100,14 +100,13 @@ public class ReplicationSourceDummy implements ReplicationSourceInterface {
}
@Override
- public String getQueueId() {
- return peerClusterId;
+ public ReplicationQueueId getQueueId() {
+ return queueId;
}
@Override
public String getPeerId() {
- String[] parts = peerClusterId.split("-", 2);
- return parts.length != 1 ? parts[0] : peerClusterId;
+ return queueId.getPeerId();
}
@Override
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestAddToSerialReplicationPeer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestAddToSerialReplicationPeer.java
index 229da6b0712..5e764ebb0ef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestAddToSerialReplicationPeer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestAddToSerialReplicationPeer.java
@@ -79,8 +79,9 @@ public class TestAddToSerialReplicationPeer extends SerialReplicationTestBase {
ReplicationSourceManager manager =
((Replication) rs.getReplicationSourceService()).getReplicationManager();
// Make sure replication moves to the new file.
- return (manager.getWALs().get(PEER_ID).get(logPrefix).size() == 1)
- && !oldWalName.equals(manager.getWALs().get(PEER_ID).get(logPrefix).first());
+ ReplicationQueueId queueId = new ReplicationQueueId(rs.getServerName(), PEER_ID);
+ return (manager.getWALs().get(queueId).get(logPrefix).size() == 1)
+ && !oldWalName.equals(manager.getWALs().get(queueId).get(logPrefix).first());
}
@Override
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java
index b8718f3526b..a12081a7636 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestClaimReplicationQueue.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionServerList;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
-import org.apache.hadoop.hbase.master.replication.ClaimReplicationQueuesProcedure;
+import org.apache.hadoop.hbase.master.replication.AssignReplicationQueuesProcedure;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
@@ -47,7 +47,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.Procedu
/**
* In HBASE-26029, we reimplement the claim queue operation with proc-v2 and make it a step in SCP,
- * this is a UT to make sure the {@link ClaimReplicationQueuesProcedure} works correctly.
+ * this is a UT to make sure the {@link AssignReplicationQueuesProcedure} works correctly.
*/
@Category({ ReplicationTests.class, LargeTests.class })
public class TestClaimReplicationQueue extends TestReplicationBase {
@@ -77,7 +77,7 @@ public class TestClaimReplicationQueue extends TestReplicationBase {
// return no region server to make the procedure hang
if (EMPTY) {
for (StackTraceElement e : Thread.currentThread().getStackTrace()) {
- if (e.getClassName().equals(ClaimReplicationQueuesProcedure.class.getName())) {
+ if (e.getClassName().equals(AssignReplicationQueuesProcedure.class.getName())) {
return Collections.emptyList();
}
}
@@ -149,7 +149,7 @@ public class TestClaimReplicationQueue extends TestReplicationBase {
HMaster master = UTIL1.getMiniHBaseCluster().getMaster();
UTIL1.waitFor(30000,
() -> master.getProcedures().stream()
- .filter(p -> p instanceof ClaimReplicationQueuesProcedure)
+ .filter(p -> p instanceof AssignReplicationQueuesProcedure)
.anyMatch(p -> p.getState() == ProcedureState.WAITING_TIMEOUT));
hbaseAdmin.enableReplicationPeer(PEER_ID2);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplicationWithBulkLoadedData.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplicationWithBulkLoadedData.java
index 180991c4a7f..5fc48b2d729 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplicationWithBulkLoadedData.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestNamespaceReplicationWithBulkLoadedData.java
@@ -259,8 +259,8 @@ public final class TestNamespaceReplicationWithBulkLoadedData extends TestBulkLo
MiniZooKeeperCluster zkCluster = UTIL1.getZkCluster();
ZKWatcher watcher = new ZKWatcher(UTIL1.getConfiguration(), "TestZnodeHFiles-refs", null);
RecoverableZooKeeper zk = RecoverableZooKeeper.connect(UTIL1.getConfiguration(), watcher);
- ZKReplicationQueueStorage replicationQueueStorage =
- new ZKReplicationQueueStorage(watcher, UTIL1.getConfiguration());
+ ReplicationQueueStorage replicationQueueStorage = ReplicationStorageFactory
+ .getReplicationQueueStorage(UTIL1.getConnection(), UTIL1.getConfiguration());
Set<String> hfiles = replicationQueueStorage.getAllHFileRefs();
assertTrue(hfiles.isEmpty());
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java
index 63cbfe3119c..67546febab7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationEmptyWALRecovery.java
@@ -332,7 +332,6 @@ public class TestReplicationEmptyWALRecovery extends TestReplicationBase {
for (int i = 0; i < numRs; i++) {
HRegionServer hrs = UTIL1.getHBaseCluster().getRegionServer(i);
Replication replicationService = (Replication) hrs.getReplicationSourceService();
- replicationService.getReplicationManager().preLogRoll(emptyWalPaths.get(i));
replicationService.getReplicationManager().postLogRoll(emptyWalPaths.get(i));
RegionInfo regionInfo =
UTIL1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
index bf65d4db82e..7a89af15902 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpTool.java
@@ -32,11 +32,14 @@ import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+// revisit later when we implement the new ReplicationSyncUpTool
+@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpTool extends TestReplicationSyncUpToolBase {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java
index fbf8ac6b3c9..b5de8e6324f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSyncUpToolWithBulkLoadedData.java
@@ -45,11 +45,14 @@ import org.apache.hadoop.hbase.tool.BulkLoadHFiles;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.HFileTestUtil;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+//revisit later when we implement the new ReplicationSyncUpTool
+@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpToolWithBulkLoadedData extends TestReplicationSyncUpToolBase {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java
index 1295ea14abc..6906db4cd46 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestSerialReplicationFailover.java
@@ -32,9 +32,12 @@ import org.apache.hadoop.hbase.util.CommonFSUtils.StreamLacksCapabilityException
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.junit.Before;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+// revisit later when we reviewing the implementation for serial replication
+@Ignore
@Category({ ReplicationTests.class, MediumTests.class })
public class TestSerialReplicationFailover extends SerialReplicationTestBase {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestTableReplicationQueueStorage.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestTableReplicationQueueStorage.java
new file mode 100644
index 00000000000..4148c1c1a2c
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestTableReplicationQueueStorage.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNameTestRule;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.MD5Hash;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.zookeeper.KeeperException;
+import org.hamcrest.Matchers;
+import org.hamcrest.collection.IsEmptyCollection;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+
+@Category({ ReplicationTests.class, MediumTests.class })
+public class TestTableReplicationQueueStorage {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestTableReplicationQueueStorage.class);
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestTableReplicationQueueStorage.class);
+
+ private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
+
+ @Rule
+ public TableNameTestRule tableNameRule = new TableNameTestRule();
+
+ private TableReplicationQueueStorage storage;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UTIL.startMiniCluster();
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void setUpBeforeTest() throws Exception {
+ TableName tableName = tableNameRule.getTableName();
+ TableDescriptor td = ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName);
+ UTIL.getAdmin().createTable(td);
+ UTIL.waitTableAvailable(tableName);
+ storage = new TableReplicationQueueStorage(UTIL.getConnection(), tableName);
+ }
+
+ private ServerName getServerName(int i) {
+ return ServerName.valueOf("127.0.0.1", 8000 + i, 10000 + i);
+ }
+
+ private String getFileName(String base, int i) {
+ return String.format(base + "-%04d", i);
+ }
+
+ @Test
+ public void testReplicator() throws ReplicationException {
+ assertTrue(storage.listAllReplicators().isEmpty());
+ String peerId = "1";
+ for (int i = 0; i < 10; i++) {
+ ReplicationQueueId queueId = new ReplicationQueueId(getServerName(i), peerId);
+ storage.setOffset(queueId, "group-" + i, new ReplicationGroupOffset("file-" + i, i * 100),
+ Collections.emptyMap());
+ }
+ List<ServerName> replicators = storage.listAllReplicators();
+ assertEquals(10, replicators.size());
+ for (int i = 0; i < 10; i++) {
+ assertThat(replicators, hasItem(getServerName(i)));
+ }
+ for (int i = 0; i < 5; i++) {
+ ReplicationQueueId queueId = new ReplicationQueueId(getServerName(i), peerId);
+ storage.removeQueue(queueId);
+ }
+ replicators = storage.listAllReplicators();
+ assertEquals(5, replicators.size());
+ for (int i = 0; i < 5; i++) {
+ assertThat(replicators, not(hasItem(getServerName(i))));
+ }
+ for (int i = 5; i < 10; i++) {
+ assertThat(replicators, hasItem(getServerName(i)));
+ }
+ }
+
+ @Test
+ public void testGetSetOffset() {
+
+ }
+
+ private void assertQueueId(String peerId, ServerName serverName, ReplicationQueueId queueId) {
+ assertEquals(peerId, queueId.getPeerId());
+ assertEquals(serverName, queueId.getServerName());
+ assertFalse(queueId.getSourceServerName().isPresent());
+ }
+
+ @Test
+ public void testPersistLogPositionAndSeqIdAtomically() throws Exception {
+ ServerName serverName1 = ServerName.valueOf("127.0.0.1", 8000, 10000);
+ assertTrue(storage.listAllQueueIds(serverName1).isEmpty());
+ String peerId1 = "1";
+ String region0 = "6b2c8f8555335cc9af74455b94516cbe";
+ String region1 = "6ecd2e9e010499f8ddef97ee8f70834f";
+
+ for (int i = 0; i < 10; i++) {
+ ReplicationQueueId queueId = new ReplicationQueueId(serverName1, peerId1);
+ assertTrue(storage.getOffsets(queueId).isEmpty());
+ }
+ assertEquals(HConstants.NO_SEQNUM, storage.getLastSequenceId(region0, peerId1));
+ assertEquals(HConstants.NO_SEQNUM, storage.getLastSequenceId(region1, peerId1));
+
+ for (int i = 0; i < 10; i++) {
+ ReplicationQueueId queueId = new ReplicationQueueId(serverName1, peerId1);
+ storage.setOffset(queueId, "group1-" + i,
+ new ReplicationGroupOffset(getFileName("file1", i), (i + 1) * 100),
+ ImmutableMap.of(region0, i * 100L, region1, (i + 1) * 100L));
+ }
+
+ List<ReplicationQueueId> queueIds = storage.listAllQueueIds(serverName1);
+ assertEquals(1, queueIds.size());
+ assertQueueId(peerId1, serverName1, queueIds.get(0));
+
+ Map<String, ReplicationGroupOffset> offsets =
+ storage.getOffsets(new ReplicationQueueId(serverName1, peerId1));
+ for (int i = 0; i < 10; i++) {
+ ReplicationGroupOffset offset = offsets.get("group1-" + i);
+ assertEquals(getFileName("file1", i), offset.getWal());
+ assertEquals((i + 1) * 100, offset.getOffset());
+ }
+ assertEquals(900L, storage.getLastSequenceId(region0, peerId1));
+ assertEquals(1000L, storage.getLastSequenceId(region1, peerId1));
+
+ // Try to decrease the last pushed id by setWALPosition method.
+ storage.setOffset(new ReplicationQueueId(serverName1, peerId1), "group1-0",
+ new ReplicationGroupOffset(getFileName("file1", 0), 11 * 100),
+ ImmutableMap.of(region0, 899L, region1, 1001L));
+ assertEquals(900L, storage.getLastSequenceId(region0, peerId1));
+ assertEquals(1001L, storage.getLastSequenceId(region1, peerId1));
+ }
+
+ private void assertGroupOffset(String wal, long offset, ReplicationGroupOffset groupOffset) {
+ assertEquals(wal, groupOffset.getWal());
+ assertEquals(offset, groupOffset.getOffset());
+ }
+
+ @Test
+ public void testClaimQueue() throws Exception {
+ String peerId = "1";
+ ServerName serverName1 = getServerName(1);
+ ReplicationQueueId queueId = new ReplicationQueueId(serverName1, peerId);
+ for (int i = 0; i < 10; i++) {
+ storage.setOffset(queueId, "group-" + i, new ReplicationGroupOffset("wal-" + i, i),
+ Collections.emptyMap());
+ }
+
+ ServerName serverName2 = getServerName(2);
+ Map<String, ReplicationGroupOffset> offsets2 = storage.claimQueue(queueId, serverName2);
+ assertEquals(10, offsets2.size());
+ for (int i = 0; i < 10; i++) {
+ assertGroupOffset("wal-" + i, i, offsets2.get("group-" + i));
+ }
+ ReplicationQueueId claimedQueueId2 = new ReplicationQueueId(serverName2, peerId, serverName1);
+ assertThat(storage.listAllQueueIds(peerId, serverName1), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId, serverName2),
+ Matchers.<List<ReplicationQueueId>> both(hasItem(claimedQueueId2)).and(hasSize(1)));
+ offsets2 = storage.getOffsets(claimedQueueId2);
+ assertEquals(10, offsets2.size());
+ for (int i = 0; i < 10; i++) {
+ assertGroupOffset("wal-" + i, i, offsets2.get("group-" + i));
+ }
+
+ ServerName serverName3 = getServerName(3);
+ Map<String, ReplicationGroupOffset> offsets3 = storage.claimQueue(claimedQueueId2, serverName3);
+ assertEquals(10, offsets3.size());
+ for (int i = 0; i < 10; i++) {
+ assertGroupOffset("wal-" + i, i, offsets3.get("group-" + i));
+ }
+ ReplicationQueueId claimedQueueId3 = new ReplicationQueueId(serverName3, peerId, serverName1);
+ assertThat(storage.listAllQueueIds(peerId, serverName1), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId, serverName2), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId, serverName3),
+ Matchers.<List<ReplicationQueueId>> both(hasItem(claimedQueueId3)).and(hasSize(1)));
+ offsets3 = storage.getOffsets(claimedQueueId3);
+ assertEquals(10, offsets3.size());
+ for (int i = 0; i < 10; i++) {
+ assertGroupOffset("wal-" + i, i, offsets3.get("group-" + i));
+ }
+ storage.removeQueue(claimedQueueId3);
+ assertThat(storage.listAllQueueIds(peerId), IsEmptyCollection.empty());
+ }
+
+ @Test
+ public void testClaimQueueMultiThread() throws Exception {
+ String peerId = "3";
+ String walGroup = "group";
+ ReplicationGroupOffset groupOffset = new ReplicationGroupOffset("wal", 123);
+ ServerName sourceServerName = getServerName(100);
+ ReplicationQueueId queueId = new ReplicationQueueId(sourceServerName, peerId);
+ storage.setOffset(queueId, walGroup, groupOffset, Collections.emptyMap());
+ List<ServerName> serverNames =
+ IntStream.range(0, 10).mapToObj(this::getServerName).collect(Collectors.toList());
+ for (int i = 0; i < 10; i++) {
+ final ReplicationQueueId toClaim = queueId;
+ List<Thread> threads = new ArrayList<>();
+ Map<ServerName, Map<String, ReplicationGroupOffset>> claimed = new ConcurrentHashMap<>();
+ Set<ServerName> failed = ConcurrentHashMap.newKeySet();
+ for (ServerName serverName : serverNames) {
+ if (serverName.equals(queueId.getServerName())) {
+ continue;
+ }
+ threads.add(new Thread("Claim-" + i + "-" + serverName) {
+
+ @Override
+ public void run() {
+ try {
+ Map<String, ReplicationGroupOffset> offsets = storage.claimQueue(toClaim, serverName);
+ if (!offsets.isEmpty()) {
+ claimed.put(serverName, offsets);
+ }
+ } catch (ReplicationException e) {
+ LOG.error("failed to claim queue", e);
+ failed.add(serverName);
+ }
+ }
+ });
+ }
+ LOG.info("Claim round {}, there are {} threads to claim {}", i, threads.size(), toClaim);
+ for (Thread thread : threads) {
+ thread.start();
+ }
+ for (Thread thread : threads) {
+ thread.join(30000);
+ assertFalse(thread.isAlive());
+ }
+ LOG.info("Finish claim round {}, claimed={}, failed={}", i, claimed, failed);
+ assertThat(failed, IsEmptyCollection.empty());
+ assertEquals(1, claimed.size());
+ Map<String, ReplicationGroupOffset> offsets = Iterables.getOnlyElement(claimed.values());
+ assertEquals(1, offsets.size());
+ assertGroupOffset("wal", 123, offsets.get("group"));
+ queueId = new ReplicationQueueId(Iterables.getOnlyElement(claimed.keySet()), peerId,
+ sourceServerName);
+ assertThat(storage.listAllQueueIds(peerId),
+ Matchers.<List<ReplicationQueueId>> both(hasItem(queueId)).and(hasSize(1)));
+ }
+ }
+
+ @Test
+ public void testListRemovePeerAllQueues() throws Exception {
+ String peerId1 = "1";
+ String peerId2 = "2";
+ for (int i = 0; i < 100; i++) {
+ ServerName serverName = getServerName(i);
+ String group = "group";
+ ReplicationGroupOffset offset = new ReplicationGroupOffset("wal", i);
+ ReplicationQueueId queueId1 = new ReplicationQueueId(serverName, peerId1);
+ ReplicationQueueId queueId2 = new ReplicationQueueId(serverName, peerId2);
+ storage.setOffset(queueId1, group, offset, Collections.emptyMap());
+ storage.setOffset(queueId2, group, offset, Collections.emptyMap());
+ }
+ List<ReplicationQueueData> queueDatas = storage.listAllQueues();
+ assertThat(queueDatas, hasSize(200));
+ for (int i = 0; i < 100; i++) {
+ ReplicationQueueData peerId1Data = queueDatas.get(i);
+ ReplicationQueueData peerId2Data = queueDatas.get(i + 100);
+ ServerName serverName = getServerName(i);
+ assertEquals(new ReplicationQueueId(serverName, peerId1), peerId1Data.getId());
+ assertEquals(new ReplicationQueueId(serverName, peerId2), peerId2Data.getId());
+ assertEquals(1, peerId1Data.getOffsets().size());
+ assertEquals(1, peerId2Data.getOffsets().size());
+ assertGroupOffset("wal", i, peerId1Data.getOffsets().get("group"));
+ assertGroupOffset("wal", i, peerId2Data.getOffsets().get("group"));
+ }
+ List<ReplicationQueueId> queueIds1 = storage.listAllQueueIds(peerId1);
+ assertThat(queueIds1, hasSize(100));
+ for (int i = 0; i < 100; i++) {
+ ServerName serverName = getServerName(i);
+ assertEquals(new ReplicationQueueId(serverName, peerId1), queueIds1.get(i));
+ }
+ List<ReplicationQueueId> queueIds2 = storage.listAllQueueIds(peerId2);
+ assertThat(queueIds2, hasSize(100));
+ for (int i = 0; i < 100; i++) {
+ ServerName serverName = getServerName(i);
+ assertEquals(new ReplicationQueueId(serverName, peerId2), queueIds2.get(i));
+ }
+
+ storage.removeAllQueues(peerId1);
+ assertThat(storage.listAllQueues(), hasSize(100));
+ assertThat(storage.listAllQueueIds(peerId1), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId2), hasSize(100));
+
+ storage.removeAllQueues(peerId2);
+ assertThat(storage.listAllQueues(), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId1), IsEmptyCollection.empty());
+ assertThat(storage.listAllQueueIds(peerId2), IsEmptyCollection.empty());
+ }
+
+ @Test
+ public void testRemoveAllLastPushedSeqIdsForPeer() throws Exception {
+ String peerId = "1";
+ String peerIdToDelete = "2";
+ for (int i = 0; i < 100; i++) {
+ String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
+ storage.setLastSequenceIds(peerId, ImmutableMap.of(encodedRegionName, (long) i));
+ storage.setLastSequenceIds(peerIdToDelete, ImmutableMap.of(encodedRegionName, (long) i));
+ }
+ for (int i = 0; i < 100; i++) {
+ String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
+ assertEquals(i, storage.getLastSequenceId(encodedRegionName, peerId));
+ assertEquals(i, storage.getLastSequenceId(encodedRegionName, peerIdToDelete));
+ }
+ storage.removeLastSequenceIds(peerIdToDelete);
+ for (int i = 0; i < 100; i++) {
+ String encodedRegionName = MD5Hash.getMD5AsHex(Bytes.toBytes(i));
+ assertEquals(i, storage.getLastSequenceId(encodedRegionName, peerId));
+ assertEquals(HConstants.NO_SEQNUM,
+ storage.getLastSequenceId(encodedRegionName, peerIdToDelete));
+ }
+ }
+
+ @Test
+ public void testHfileRefsReplicationQueues() throws ReplicationException, KeeperException {
+ String peerId1 = "1";
+
+ List<Pair<Path, Path>> files1 = new ArrayList<>(3);
+ files1.add(new Pair<>(null, new Path("file_1")));
+ files1.add(new Pair<>(null, new Path("file_2")));
+ files1.add(new Pair<>(null, new Path("file_3")));
+ assertTrue(storage.getReplicableHFiles(peerId1).isEmpty());
+ assertEquals(0, storage.getAllPeersFromHFileRefsQueue().size());
+
+ storage.addHFileRefs(peerId1, files1);
+ assertEquals(1, storage.getAllPeersFromHFileRefsQueue().size());
+ assertEquals(3, storage.getReplicableHFiles(peerId1).size());
+ List<String> hfiles2 = new ArrayList<>(files1.size());
+ for (Pair<Path, Path> p : files1) {
+ hfiles2.add(p.getSecond().getName());
+ }
+ String removedString = hfiles2.remove(0);
+ storage.removeHFileRefs(peerId1, hfiles2);
+ assertEquals(1, storage.getReplicableHFiles(peerId1).size());
+ hfiles2 = new ArrayList<>(1);
+ hfiles2.add(removedString);
+ storage.removeHFileRefs(peerId1, hfiles2);
+ assertEquals(0, storage.getReplicableHFiles(peerId1).size());
+ }
+
+ @Test
+ public void testRemovePeerForHFileRefs() throws ReplicationException, KeeperException {
+ String peerId1 = "1";
+ String peerId2 = "2";
+
+ List<Pair<Path, Path>> files1 = new ArrayList<>(3);
+ files1.add(new Pair<>(null, new Path("file_1")));
+ files1.add(new Pair<>(null, new Path("file_2")));
+ files1.add(new Pair<>(null, new Path("file_3")));
+ storage.addHFileRefs(peerId1, files1);
+ storage.addHFileRefs(peerId2, files1);
+ assertEquals(2, storage.getAllPeersFromHFileRefsQueue().size());
+ assertEquals(3, storage.getReplicableHFiles(peerId1).size());
+ assertEquals(3, storage.getReplicableHFiles(peerId2).size());
+
+ storage.removePeerFromHFileRefs(peerId1);
+ assertEquals(1, storage.getAllPeersFromHFileRefsQueue().size());
+ assertTrue(storage.getReplicableHFiles(peerId1).isEmpty());
+ assertEquals(3, storage.getReplicableHFiles(peerId2).size());
+
+ storage.removePeerFromHFileRefs(peerId2);
+ assertEquals(0, storage.getAllPeersFromHFileRefsQueue().size());
+ assertTrue(storage.getReplicableHFiles(peerId2).isEmpty());
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java
index 83cd41773ca..28779be4399 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleAsyncWAL.java
@@ -25,8 +25,11 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.wal.RegionGroupingProvider;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.experimental.categories.Category;
+//revisit later when we implement the new ReplicationSyncUpTool
+@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpToolWithMultipleAsyncWAL extends TestReplicationSyncUpTool {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java
index 673b841430e..f495f433bc9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/multiwal/TestReplicationSyncUpToolWithMultipleWAL.java
@@ -25,8 +25,11 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.wal.RegionGroupingProvider;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.experimental.categories.Category;
+//revisit later when we implement the new ReplicationSyncUpTool
+@Ignore
@Category({ ReplicationTests.class, LargeTests.class })
public class TestReplicationSyncUpToolWithMultipleWAL extends TestReplicationSyncUpTool {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java
index 0189d475575..8918f8422e1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDrainReplicationQueuesForStandBy.java
@@ -35,9 +35,12 @@ import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+// TODO: revisit later
+@Ignore
@Category({ ReplicationTests.class, MediumTests.class })
public class TestDrainReplicationQueuesForStandBy extends SyncReplicationTestBase {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java
index d78a45ca6b9..3475ae5c192 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java
@@ -36,12 +36,15 @@ import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
/**
* Tests for DumpReplicationQueues tool
*/
+// TODO: reimplement
+@Ignore
@Category({ ReplicationTests.class, SmallTests.class })
public class TestDumpReplicationQueues {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java
index e80f0ea35ba..707bab875d2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSource.java
@@ -23,7 +23,6 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -57,7 +56,8 @@ import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hadoop.hbase.replication.ReplicationEndpoint;
import org.apache.hadoop.hbase.replication.ReplicationPeer;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
+import org.apache.hadoop.hbase.replication.ReplicationQueueData;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.WALEntryFilter;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
@@ -79,6 +79,8 @@ import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
+
@Category({ ReplicationTests.class, MediumTests.class })
public class TestReplicationSource {
@@ -134,11 +136,13 @@ public class TestReplicationSource {
ReplicationSourceManager manager = Mockito.mock(ReplicationSourceManager.class);
Mockito.when(manager.getGlobalMetrics())
.thenReturn(mock(MetricsReplicationGlobalSourceSource.class));
- String queueId = "qid";
+
RegionServerServices rss =
TEST_UTIL.createMockRegionServerService(ServerName.parseServerName("a.b.c,1,1"));
- rs.init(conf, null, manager, null, mockPeer, rss, queueId, null, p -> OptionalLong.empty(),
- new MetricsSource(queueId));
+ ReplicationQueueId queueId = new ReplicationQueueId(rss.getServerName(), "qid");
+ rs.init(conf, null, manager, null, mockPeer, rss,
+ new ReplicationQueueData(queueId, ImmutableMap.of()), null, p -> OptionalLong.empty(),
+ new MetricsSource(queueId.toString()));
try {
rs.startup();
assertTrue(rs.isSourceActive());
@@ -171,11 +175,12 @@ public class TestReplicationSource {
.thenReturn(DoNothingReplicationEndpoint.class.getName());
Mockito.when(mockPeer.getPeerConfig()).thenReturn(peerConfig);
ReplicationSourceManager manager = Mockito.mock(ReplicationSourceManager.class);
- String queueId = "qid";
RegionServerServices rss =
TEST_UTIL.createMockRegionServerService(ServerName.parseServerName("a.b.c,1,1"));
- rs.init(conf, null, manager, null, mockPeer, rss, queueId, uuid, p -> OptionalLong.empty(),
- new MetricsSource(queueId));
+ ReplicationQueueId queueId = new ReplicationQueueId(rss.getServerName(), "qid");
+ rs.init(conf, null, manager, null, mockPeer, rss,
+ new ReplicationQueueData(queueId, ImmutableMap.of()), uuid, p -> OptionalLong.empty(),
+ new MetricsSource(queueId.toString()));
try {
rs.startup();
TEST_UTIL.waitFor(30000, () -> rs.getWalEntryFilter() != null);
@@ -257,8 +262,11 @@ public class TestReplicationSource {
Configuration testConf = HBaseConfiguration.create();
testConf.setInt("replication.source.maxretriesmultiplier", 1);
ReplicationSourceManager manager = Mockito.mock(ReplicationSourceManager.class);
- source.init(testConf, null, manager, null, mockPeer, null, "testPeer", null,
- p -> OptionalLong.empty(), null);
+ ReplicationQueueId queueId =
+ new ReplicationQueueId(ServerName.valueOf("test,123,123"), "testPeer");
+ source.init(testConf, null, manager, null, mockPeer, null,
+ new ReplicationQueueData(queueId, ImmutableMap.of()), null, p -> OptionalLong.empty(),
+ null);
ExecutorService executor = Executors.newSingleThreadExecutor();
Future<?> future = executor.submit(() -> source.terminate("testing source termination"));
long sleepForRetries = testConf.getLong("replication.source.sleepforretries", 1000);
@@ -276,8 +284,11 @@ public class TestReplicationSource {
ReplicationPeer mockPeer = mock(ReplicationPeer.class);
Mockito.when(mockPeer.getPeerBandwidth()).thenReturn(0L);
Configuration testConf = HBaseConfiguration.create();
- source.init(testConf, null, mockManager, null, mockPeer, Mockito.mock(Server.class), "testPeer",
- null, p -> OptionalLong.empty(), mock(MetricsSource.class));
+ ReplicationQueueId queueId =
+ new ReplicationQueueId(ServerName.valueOf("test,123,123"), "testPeer");
+ source.init(testConf, null, mockManager, null, mockPeer, Mockito.mock(Server.class),
+ new ReplicationQueueData(queueId, ImmutableMap.of()), null, p -> OptionalLong.empty(),
+ mock(MetricsSource.class));
ReplicationSourceWALReader reader =
new ReplicationSourceWALReader(null, conf, null, 0, null, source, null);
ReplicationSourceShipper shipper = new ReplicationSourceShipper(conf, null, null, source);
@@ -480,35 +491,6 @@ public class TestReplicationSource {
}
- /**
- * Test HBASE-20497 Moved here from TestReplicationSource because doesn't need cluster.
- */
- @Test
- public void testRecoveredReplicationSourceShipperGetPosition() throws Exception {
- String walGroupId = "fake-wal-group-id";
- ServerName serverName = ServerName.valueOf("www.example.com", 12006, 1524679704418L);
- ServerName deadServer = ServerName.valueOf("www.deadServer.com", 12006, 1524679704419L);
- RecoveredReplicationSource source = mock(RecoveredReplicationSource.class);
- Server server = mock(Server.class);
- Mockito.when(server.getServerName()).thenReturn(serverName);
- Mockito.when(source.getServer()).thenReturn(server);
- Mockito.when(source.getServerWALsBelongTo()).thenReturn(deadServer);
- ReplicationQueueStorage storage = mock(ReplicationQueueStorage.class);
- Mockito.when(storage.getWALPosition(Mockito.eq(serverName), Mockito.any(), Mockito.any()))
- .thenReturn(1001L);
- Mockito.when(storage.getWALPosition(Mockito.eq(deadServer), Mockito.any(), Mockito.any()))
- .thenReturn(-1L);
- Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
- conf.setInt("replication.source.maxretriesmultiplier", -1);
- MetricsSource metricsSource = mock(MetricsSource.class);
- doNothing().when(metricsSource).incrSizeOfLogQueue();
- ReplicationSourceLogQueue logQueue = new ReplicationSourceLogQueue(conf, metricsSource, source);
- logQueue.enqueueLog(new Path("/www/html/test"), walGroupId);
- RecoveredReplicationSourceShipper shipper =
- new RecoveredReplicationSourceShipper(conf, walGroupId, logQueue, source, storage);
- assertEquals(1001L, shipper.getStartPosition());
- }
-
private RegionServerServices setupForAbortTests(ReplicationSource rs, Configuration conf,
String endpointName) throws IOException {
conf.setInt("replication.source.maxretriesmultiplier", 1);
@@ -522,11 +504,12 @@ public class TestReplicationSource {
ReplicationSourceManager manager = Mockito.mock(ReplicationSourceManager.class);
Mockito.when(manager.getGlobalMetrics())
.thenReturn(mock(MetricsReplicationGlobalSourceSource.class));
- String queueId = "qid";
RegionServerServices rss =
TEST_UTIL.createMockRegionServerService(ServerName.parseServerName("a.b.c,1,1"));
- rs.init(conf, null, manager, null, mockPeer, rss, queueId, null, p -> OptionalLong.empty(),
- new MetricsSource(queueId));
+ ReplicationQueueId queueId = new ReplicationQueueId(rss.getServerName(), "qid");
+ rs.init(conf, null, manager, null, mockPeer, rss,
+ new ReplicationQueueData(queueId, ImmutableMap.of()), null, p -> OptionalLong.empty(),
+ new MetricsSource(queueId.toString()));
return rss;
}
@@ -624,8 +607,8 @@ public class TestReplicationSource {
ManualEnvironmentEdge manualEdge = new ManualEnvironmentEdge();
EnvironmentEdgeManager.injectEdge(manualEdge);
- String id = "1";
- MetricsSource metrics = new MetricsSource(id);
+ String peerId = "1";
+ MetricsSource metrics = new MetricsSource(peerId);
Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
conf.setInt("replication.source.maxretriesmultiplier", 1);
ReplicationPeer mockPeer = Mockito.mock(ReplicationPeer.class);
@@ -640,16 +623,17 @@ public class TestReplicationSource {
.thenReturn(mock(MetricsReplicationGlobalSourceSource.class));
RegionServerServices rss =
TEST_UTIL.createMockRegionServerService(ServerName.parseServerName("a.b.c,1,1"));
-
+ ReplicationQueueId queueId = new ReplicationQueueId(rss.getServerName(), peerId);
ReplicationSource source = new ReplicationSource();
- source.init(conf, null, manager, null, mockPeer, rss, id, null, p -> OptionalLong.empty(),
+ source.init(conf, null, manager, null, mockPeer, rss,
+ new ReplicationQueueData(queueId, ImmutableMap.of()), null, p -> OptionalLong.empty(),
metrics);
final Path log1 = new Path(logDir, "log-walgroup-a.8");
manualEdge.setValue(10);
// Diff of current time (10) and log-walgroup-a.8 timestamp will be 2.
source.enqueueLog(log1);
- MetricsReplicationSourceSource metricsSource1 = getSourceMetrics(id);
+ MetricsReplicationSourceSource metricsSource1 = getSourceMetrics(peerId);
assertEquals(2, metricsSource1.getOldestWalAge());
final Path log2 = new Path(logDir, "log-walgroup-b.4");
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
index c48dbc39a03..6aba327d791 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
@@ -17,578 +17,326 @@
*/
package org.apache.hadoop.hbase.replication.regionserver;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
-import java.lang.reflect.Field;
-import java.net.URLEncoder;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.Collections;
import java.util.NavigableMap;
-import java.util.NavigableSet;
import java.util.Set;
-import java.util.SortedSet;
import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.ChoreService;
-import org.apache.hadoop.hbase.ClusterId;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
+import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
-import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
-import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
-import org.apache.hadoop.hbase.regionserver.RegionServerServices;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationPeer;
+import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter;
+import org.apache.hadoop.hbase.replication.DummyReplicationEndpoint;
+import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationPeerConfigBuilder;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.replication.ReplicationUtils;
import org.apache.hadoop.hbase.replication.SyncReplicationState;
-import org.apache.hadoop.hbase.replication.ZKReplicationPeerStorage;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.JVMClusterUtil;
-import org.apache.hadoop.hbase.util.MockServer;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALEdit;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.hbase.wal.WALKeyImpl;
-import org.apache.hadoop.hbase.zookeeper.ZKClusterId;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
+import org.hamcrest.Matchers;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
-import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
-
-/**
- * An abstract class that tests ReplicationSourceManager. Classes that extend this class should set
- * up the proper config for this class and initialize the proper cluster using HBaseTestingUtility.
- */
@Category({ ReplicationTests.class, MediumTests.class })
-public abstract class TestReplicationSourceManager {
+public class TestReplicationSourceManager {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestReplicationSourceManager.class);
- protected static final Logger LOG = LoggerFactory.getLogger(TestReplicationSourceManager.class);
-
- protected static Configuration conf;
+ public static final class ReplicationEndpointForTest extends DummyReplicationEndpoint {
- protected static HBaseTestingUtil utility;
+ private String clusterKey;
- protected static Replication replication;
-
- protected static ReplicationSourceManager manager;
-
- protected static ReplicationSourceManager managerOfCluster;
+ @Override
+ public boolean replicate(ReplicateContext replicateContext) {
+ // if you want to block the replication, for example, do not want the recovered source to be
+ // removed
+ if (clusterKey.endsWith("error")) {
+ throw new RuntimeException("Inject error");
+ }
+ return true;
+ }
- protected static ZKWatcher zkw;
+ @Override
+ public void init(Context context) throws IOException {
+ super.init(context);
+ this.clusterKey = context.getReplicationPeer().getPeerConfig().getClusterKey();
+ }
- protected static TableDescriptor htd;
+ }
- protected static RegionInfo hri;
+ private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
- protected static final byte[] r1 = Bytes.toBytes("r1");
+ private static Configuration CONF;
- protected static final byte[] r2 = Bytes.toBytes("r2");
+ private static FileSystem FS;
- protected static final byte[] f1 = Bytes.toBytes("f1");
+ private static final byte[] F1 = Bytes.toBytes("f1");
- protected static final byte[] f2 = Bytes.toBytes("f2");
+ private static final byte[] F2 = Bytes.toBytes("f2");
- protected static final TableName test = TableName.valueOf("test");
+ private static final TableName TABLE_NAME = TableName.valueOf("test");
- protected static final String slaveId = "1";
+ private static TableDescriptor TD;
- protected static FileSystem fs;
+ private static RegionInfo RI;
- protected static Path oldLogDir;
+ private static NavigableMap<byte[], Integer> SCOPES;
- protected static Path logDir;
+ @Rule
+ public final TestName name = new TestName();
- protected static Path remoteLogDir;
+ private Path oldLogDir;
- protected static CountDownLatch latch;
+ private Path logDir;
- protected static List<String> files = new ArrayList<>();
- protected static NavigableMap<byte[], Integer> scopes;
+ private Path remoteLogDir;
- protected static void setupZkAndReplication() throws Exception {
- // The implementing class should set up the conf
- assertNotNull(conf);
- zkw = new ZKWatcher(conf, "test", null);
- ZKUtil.createWithParents(zkw, "/hbase/replication");
- ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1");
- ZKUtil.setData(zkw, "/hbase/replication/peers/1",
- Bytes.toBytes(conf.get(HConstants.ZOOKEEPER_QUORUM) + ":"
- + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":/1"));
- ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/peer-state");
- ZKUtil.setData(zkw, "/hbase/replication/peers/1/peer-state",
- ZKReplicationPeerStorage.ENABLED_ZNODE_BYTES);
- ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/sync-rep-state");
- ZKUtil.setData(zkw, "/hbase/replication/peers/1/sync-rep-state",
- ZKReplicationPeerStorage.NONE_STATE_ZNODE_BYTES);
- ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/new-sync-rep-state");
- ZKUtil.setData(zkw, "/hbase/replication/peers/1/new-sync-rep-state",
- ZKReplicationPeerStorage.NONE_STATE_ZNODE_BYTES);
- ZKUtil.createWithParents(zkw, "/hbase/replication/state");
- ZKUtil.setData(zkw, "/hbase/replication/state", ZKReplicationPeerStorage.ENABLED_ZNODE_BYTES);
+ private Server server;
- ZKClusterId.setClusterId(zkw, new ClusterId());
- CommonFSUtils.setRootDir(utility.getConfiguration(), utility.getDataTestDir());
- fs = FileSystem.get(conf);
- oldLogDir = utility.getDataTestDir(HConstants.HREGION_OLDLOGDIR_NAME);
- logDir = utility.getDataTestDir(HConstants.HREGION_LOGDIR_NAME);
- remoteLogDir = utility.getDataTestDir(ReplicationUtils.REMOTE_WAL_DIR_NAME);
- replication = new Replication();
- replication.initialize(new DummyServer(), fs, logDir, oldLogDir,
- new WALFactory(conf, "test", null, false));
- managerOfCluster = getManagerFromCluster();
- if (managerOfCluster != null) {
- // After replication procedure, we need to add peer by hand (other than by receiving
- // notification from zk)
- managerOfCluster.addPeer(slaveId);
- }
+ private Replication replication;
- manager = replication.getReplicationManager();
- manager.addSource(slaveId);
- if (managerOfCluster != null) {
- waitPeer(slaveId, managerOfCluster, true);
- }
- waitPeer(slaveId, manager, true);
+ private ReplicationSourceManager manager;
- htd = TableDescriptorBuilder.newBuilder(test)
- .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(f1)
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ UTIL.startMiniCluster(1);
+ FS = UTIL.getTestFileSystem();
+ CONF = new Configuration(UTIL.getConfiguration());
+ CONF.setLong("replication.sleep.before.failover", 0);
+ TD = TableDescriptorBuilder.newBuilder(TABLE_NAME)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(F1)
.setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
- .setColumnFamily(ColumnFamilyDescriptorBuilder.of(f2)).build();
-
- scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (byte[] fam : htd.getColumnFamilyNames()) {
- scopes.put(fam, 0);
- }
- hri = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(r1).setEndKey(r2).build();
- }
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.of(F2)).build();
- private static ReplicationSourceManager getManagerFromCluster() {
- // TestReplicationSourceManagerZkImpl won't start the mini hbase cluster.
- if (utility.getMiniHBaseCluster() == null) {
- return null;
- }
- return utility.getMiniHBaseCluster().getRegionServerThreads().stream()
- .map(JVMClusterUtil.RegionServerThread::getRegionServer).findAny()
- .map(RegionServerServices::getReplicationSourceService).map(r -> (Replication) r)
- .map(Replication::getReplicationManager).get();
+ RI = RegionInfoBuilder.newBuilder(TABLE_NAME).build();
+ SCOPES = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ SCOPES.put(F1, 1);
+ SCOPES.put(F2, 0);
}
@AfterClass
- public static void tearDownAfterClass() throws Exception {
- if (manager != null) {
- manager.join();
- }
- utility.shutdownMiniCluster();
- }
-
- @Rule
- public TestName testName = new TestName();
-
- private void cleanLogDir() throws IOException {
- fs.delete(logDir, true);
- fs.delete(oldLogDir, true);
- fs.delete(remoteLogDir, true);
+ public static void tearDownAfterClass() throws IOException {
+ UTIL.shutdownMiniCluster();
}
@Before
public void setUp() throws Exception {
- LOG.info("Start " + testName.getMethodName());
- cleanLogDir();
- }
-
- @After
- public void tearDown() throws Exception {
- LOG.info("End " + testName.getMethodName());
- cleanLogDir();
- List<String> ids = manager.getSources().stream().map(ReplicationSourceInterface::getPeerId)
- .collect(Collectors.toList());
- for (String id : ids) {
- if (slaveId.equals(id)) {
- continue;
- }
- removePeerAndWait(id);
- }
- }
-
- @Test
- public void testLogRoll() throws Exception {
- long baseline = 1000;
- long time = baseline;
- MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
- KeyValue kv = new KeyValue(r1, f1, r1);
- WALEdit edit = new WALEdit();
- edit.add(kv);
-
- WALFactory wals =
- new WALFactory(utility.getConfiguration(), URLEncoder.encode("regionserver:60020", "UTF8"));
- ReplicationSourceManager replicationManager = replication.getReplicationManager();
- wals.getWALProvider()
- .addWALActionsListener(new ReplicationSourceWALActionListener(conf, replicationManager));
- final WAL wal = wals.getWAL(hri);
- manager.init();
- TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf("tableame"))
- .setColumnFamily(ColumnFamilyDescriptorBuilder.of(f1)).build();
- NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- for (byte[] fam : htd.getColumnFamilyNames()) {
- scopes.put(fam, 0);
- }
- // Testing normal log rolling every 20
- for (long i = 1; i < 101; i++) {
- if (i > 1 && i % 20 == 0) {
- wal.rollWriter();
- }
- LOG.info(Long.toString(i));
- final long txid = wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test,
- EnvironmentEdgeManager.currentTime(), mvcc, scopes), edit);
- wal.sync(txid);
- }
+ Path rootDir = UTIL.getDataTestDirOnTestFS(name.getMethodName());
+ CommonFSUtils.setRootDir(CONF, rootDir);
+ server = mock(Server.class);
+ when(server.getConfiguration()).thenReturn(CONF);
+ when(server.getZooKeeper()).thenReturn(UTIL.getZooKeeperWatcher());
+ when(server.getConnection()).thenReturn(UTIL.getConnection());
+ when(server.getServerName()).thenReturn(ServerName.valueOf("hostname.example.org", 1234, 1));
+ oldLogDir = new Path(rootDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ FS.mkdirs(oldLogDir);
+ logDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
+ FS.mkdirs(logDir);
+ remoteLogDir = new Path(rootDir, ReplicationUtils.REMOTE_WAL_DIR_NAME);
+ FS.mkdirs(remoteLogDir);
+ TableName tableName = TableName.valueOf("replication_" + name.getMethodName());
+ UTIL.getAdmin()
+ .createTable(ReplicationStorageFactory.createReplicationQueueTableDescriptor(tableName));
+ CONF.set(ReplicationStorageFactory.REPLICATION_QUEUE_TABLE_NAME, tableName.getNameAsString());
- // Simulate a rapid insert that's followed
- // by a report that's still not totally complete (missing last one)
- LOG.info(baseline + " and " + time);
- baseline += 101;
- time = baseline;
- LOG.info(baseline + " and " + time);
-
- for (int i = 0; i < 3; i++) {
- wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test,
- EnvironmentEdgeManager.currentTime(), mvcc, scopes), edit);
- }
- wal.sync();
-
- int logNumber = 0;
- for (Map.Entry<String, NavigableSet<String>> entry : manager.getWALs().get(slaveId)
- .entrySet()) {
- logNumber += entry.getValue().size();
- }
- assertEquals(6, logNumber);
-
- wal.rollWriter();
-
- ReplicationSourceInterface source = mock(ReplicationSourceInterface.class);
- when(source.getQueueId()).thenReturn("1");
- when(source.isRecovered()).thenReturn(false);
- when(source.isSyncReplication()).thenReturn(false);
- manager.logPositionAndCleanOldLogs(source,
- new WALEntryBatch(0, manager.getSources().get(0).getCurrentPath()));
-
- wal.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), test,
- EnvironmentEdgeManager.currentTime(), mvcc, scopes), edit);
- wal.sync();
-
- assertEquals(1, manager.getWALs().size());
-
- // TODO Need a case with only 2 WALs and we only want to delete the first one
- }
-
- @Test
- public void testClaimQueues() throws Exception {
- Server server = new DummyServer("hostname0.example.org");
- ReplicationQueueStorage rq = ReplicationStorageFactory
- .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
- // populate some znodes in the peer znode
- files.add("log1");
- files.add("log2");
- for (String file : files) {
- rq.addWAL(server.getServerName(), "1", file);
- }
- // create 3 DummyServers
- Server s1 = new DummyServer("dummyserver1.example.org");
- Server s2 = new DummyServer("dummyserver2.example.org");
- Server s3 = new DummyServer("dummyserver3.example.org");
-
- // create 3 DummyNodeFailoverWorkers
- DummyNodeFailoverWorker w1 = new DummyNodeFailoverWorker(server.getServerName(), s1);
- DummyNodeFailoverWorker w2 = new DummyNodeFailoverWorker(server.getServerName(), s2);
- DummyNodeFailoverWorker w3 = new DummyNodeFailoverWorker(server.getServerName(), s3);
-
- latch = new CountDownLatch(3);
- // start the threads
- w1.start();
- w2.start();
- w3.start();
- // make sure only one is successful
- int populatedMap = 0;
- // wait for result now... till all the workers are done.
- latch.await();
- populatedMap +=
- w1.isLogZnodesMapPopulated() + w2.isLogZnodesMapPopulated() + w3.isLogZnodesMapPopulated();
- assertEquals(1, populatedMap);
- server.abort("", null);
+ replication = new Replication();
+ replication.initialize(server, FS, logDir, oldLogDir,
+ new WALFactory(CONF, "test", null, false));
+ manager = replication.getReplicationManager();
}
- @Test
- public void testCleanupFailoverQueues() throws Exception {
- Server server = new DummyServer("hostname1.example.org");
- ReplicationQueueStorage rq = ReplicationStorageFactory
- .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
- // populate some znodes in the peer znode
- SortedSet<String> files = new TreeSet<>();
- String group = "testgroup";
- String file1 = group + "." + EnvironmentEdgeManager.currentTime() + ".log1";
- String file2 = group + "." + EnvironmentEdgeManager.currentTime() + ".log2";
- files.add(file1);
- files.add(file2);
- for (String file : files) {
- rq.addWAL(server.getServerName(), "1", file);
- }
- Server s1 = new DummyServer("dummyserver1.example.org");
- ReplicationPeers rp1 = ReplicationFactory.getReplicationPeers(s1.getFileSystem(),
- s1.getZooKeeper(), s1.getConfiguration());
- rp1.init();
- manager.claimQueue(server.getServerName(), "1");
- assertEquals(1, manager.getWalsByIdRecoveredQueues().size());
- String id = "1-" + server.getServerName().getServerName();
- assertEquals(files, manager.getWalsByIdRecoveredQueues().get(id).get(group));
- ReplicationSourceInterface source = mock(ReplicationSourceInterface.class);
- when(source.getQueueId()).thenReturn(id);
- when(source.isRecovered()).thenReturn(true);
- when(source.isSyncReplication()).thenReturn(false);
- manager.cleanOldLogs(file2, false, source);
- // log1 should be deleted
- assertEquals(Sets.newHashSet(file2), manager.getWalsByIdRecoveredQueues().get(id).get(group));
+ @After
+ public void tearDown() {
+ replication.stopReplicationService();
}
- @Test
- public void testCleanupUnknownPeerZNode() throws Exception {
- Server server = new DummyServer("hostname2.example.org");
- ReplicationQueueStorage rq = ReplicationStorageFactory
- .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
- // populate some znodes in the peer znode
- // add log to an unknown peer
- String group = "testgroup";
- rq.addWAL(server.getServerName(), "2", group + ".log1");
- rq.addWAL(server.getServerName(), "2", group + ".log2");
-
- manager.claimQueue(server.getServerName(), "2");
-
- // The log of the unknown peer should be removed from zk
- for (String peer : manager.getAllQueues()) {
- assertTrue(peer.startsWith("1"));
- }
+ /**
+ * Add a peer and wait for it to initialize
+ */
+ private void addPeerAndWait(String peerId, String clusterKey, boolean syncRep)
+ throws ReplicationException, IOException {
+ ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder()
+ .setClusterKey(UTIL.getZkCluster().getAddress().toString() + ":/" + clusterKey)
+ .setReplicationEndpointImpl(ReplicationEndpointForTest.class.getName());
+ if (syncRep) {
+ builder.setTableCFsMap(ImmutableMap.of(TABLE_NAME, Collections.emptyList()))
+ .setRemoteWALDir(FS.makeQualified(remoteLogDir).toString());
+ }
+
+ manager.getReplicationPeers().getPeerStorage().addPeer(peerId, builder.build(), true,
+ syncRep ? SyncReplicationState.DOWNGRADE_ACTIVE : SyncReplicationState.NONE);
+ manager.addPeer(peerId);
+ UTIL.waitFor(20000, () -> {
+ ReplicationSourceInterface rs = manager.getSource(peerId);
+ return rs != null && rs.isSourceActive();
+ });
}
/**
- * Test for HBASE-9038, Replication.scopeWALEdits would NPE if it wasn't filtering out the
- * compaction WALEdit.
+ * Remove a peer and wait for it to get cleaned up
*/
- @Test
- public void testCompactionWALEdits() throws Exception {
- TableName tableName = TableName.valueOf("testCompactionWALEdits");
- WALProtos.CompactionDescriptor compactionDescriptor =
- WALProtos.CompactionDescriptor.getDefaultInstance();
- RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(HConstants.EMPTY_START_ROW)
- .setEndKey(HConstants.EMPTY_END_ROW).build();
- WALEdit edit = WALEdit.createCompaction(hri, compactionDescriptor);
- ReplicationSourceWALActionListener.scopeWALEdits(new WALKeyImpl(), edit, conf);
+ private void removePeerAndWait(String peerId) throws Exception {
+ ReplicationPeers rp = manager.getReplicationPeers();
+ rp.getPeerStorage().removePeer(peerId);
+ manager.removePeer(peerId);
+ UTIL.waitFor(20000, () -> {
+ if (rp.getPeer(peerId) != null) {
+ return false;
+ }
+ if (manager.getSource(peerId) != null) {
+ return false;
+ }
+ return manager.getOldSources().stream().noneMatch(rs -> rs.getPeerId().equals(peerId));
+ });
}
- @Test
- public void testBulkLoadWALEditsWithoutBulkLoadReplicationEnabled() throws Exception {
- NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- // 1. Get the bulk load wal edit event
- WALEdit logEdit = getBulkLoadWALEdit(scope);
- // 2. Create wal key
- WALKeyImpl logKey = new WALKeyImpl(scope);
-
- // 3. Get the scopes for the key
- ReplicationSourceWALActionListener.scopeWALEdits(logKey, logEdit, conf);
-
- // 4. Assert that no bulk load entry scopes are added if bulk load hfile replication is disabled
- assertNull("No bulk load entries scope should be added if bulk load replication is disabled.",
- logKey.getReplicationScopes());
+ private void createWALFile(Path file) throws Exception {
+ ProtobufLogWriter writer = new ProtobufLogWriter();
+ try {
+ writer.init(FS, file, CONF, false, FS.getDefaultBlockSize(file), null);
+ WALKeyImpl key = new WALKeyImpl(RI.getEncodedNameAsBytes(), TABLE_NAME,
+ EnvironmentEdgeManager.currentTime(), SCOPES);
+ WALEdit edit = new WALEdit();
+ edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(F1).setFamily(F1)
+ .setQualifier(F1).setType(Cell.Type.Put).setValue(F1).build());
+ edit.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(F2).setFamily(F2)
+ .setQualifier(F2).setType(Cell.Type.Put).setValue(F2).build());
+ writer.append(new WAL.Entry(key, edit));
+ writer.sync(false);
+ } finally {
+ writer.close();
+ }
}
@Test
- public void testBulkLoadWALEdits() throws Exception {
- // 1. Get the bulk load wal edit event
- NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
- WALEdit logEdit = getBulkLoadWALEdit(scope);
- // 2. Create wal key
- WALKeyImpl logKey = new WALKeyImpl(scope);
- // 3. Enable bulk load hfile replication
- Configuration bulkLoadConf = HBaseConfiguration.create(conf);
- bulkLoadConf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
-
- // 4. Get the scopes for the key
- ReplicationSourceWALActionListener.scopeWALEdits(logKey, logEdit, bulkLoadConf);
-
- NavigableMap<byte[], Integer> scopes = logKey.getReplicationScopes();
- // Assert family with replication scope global is present in the key scopes
- assertTrue("This family scope is set to global, should be part of replication key scopes.",
- scopes.containsKey(f1));
- // Assert family with replication scope local is not present in the key scopes
- assertFalse("This family scope is set to local, should not be part of replication key scopes",
- scopes.containsKey(f2));
+ public void testClaimQueue() throws Exception {
+ String peerId = "1";
+ addPeerAndWait(peerId, "error", false);
+ ServerName serverName = ServerName.valueOf("hostname0.example.org", 12345, 123);
+ String walName1 = serverName.toString() + ".1";
+ createWALFile(new Path(oldLogDir, walName1));
+ ReplicationQueueId queueId = new ReplicationQueueId(serverName, peerId);
+ ReplicationQueueStorage queueStorage = manager.getQueueStorage();
+ queueStorage.setOffset(queueId, "", new ReplicationGroupOffset(peerId, 0),
+ Collections.emptyMap());
+ manager.claimQueue(queueId);
+ assertThat(manager.getOldSources(), hasSize(1));
}
- /**
- * Test whether calling removePeer() on a ReplicationSourceManager that failed on initializing the
- * corresponding ReplicationSourceInterface correctly cleans up the corresponding replication
- * queue and ReplicationPeer. See HBASE-16096.
- */
@Test
- public void testPeerRemovalCleanup() throws Exception {
- String replicationSourceImplName = conf.get("replication.replicationsource.implementation");
- final String peerId = "FakePeer";
- final ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
- .setClusterKey(utility.getZkCluster().getAddress().toString() + ":/hbase").build();
- try {
- DummyServer server = new DummyServer();
- ReplicationQueueStorage rq = ReplicationStorageFactory
- .getReplicationQueueStorage(server.getZooKeeper(), server.getConfiguration());
- // Purposely fail ReplicationSourceManager.addSource() by causing ReplicationSourceInterface
- // initialization to throw an exception.
- conf.set("replication.replicationsource.implementation",
- FailInitializeDummyReplicationSource.class.getName());
- manager.getReplicationPeers();
- // Set up the znode and ReplicationPeer for the fake peer
- // Don't wait for replication source to initialize, we know it won't.
- addPeerAndWait(peerId, peerConfig, false);
-
- // Sanity check
- assertNull(manager.getSource(peerId));
-
- // Create a replication queue for the fake peer
- rq.addWAL(server.getServerName(), peerId, "FakeFile");
- // Unregister peer, this should remove the peer and clear all queues associated with it
- // Need to wait for the ReplicationTracker to pick up the changes and notify listeners.
- removePeerAndWait(peerId);
- assertFalse(rq.getAllQueues(server.getServerName()).contains(peerId));
- } finally {
- conf.set("replication.replicationsource.implementation", replicationSourceImplName);
- removePeerAndWait(peerId);
- }
- }
+ public void testSameWALPrefix() throws IOException {
+ String walName1 = "localhost,8080,12345-45678-Peer.34567";
+ String walName2 = "localhost,8080,12345.56789";
+ manager.postLogRoll(new Path(walName1));
+ manager.postLogRoll(new Path(walName2));
- private static MetricsReplicationSourceSource getGlobalSource() throws Exception {
- ReplicationSourceInterface source = manager.getSource(slaveId);
- // Retrieve the global replication metrics source
- Field f = MetricsSource.class.getDeclaredField("globalSourceSource");
- f.setAccessible(true);
- return (MetricsReplicationSourceSource) f.get(source.getSourceMetrics());
+ Set<String> latestWals =
+ manager.getLastestPath().stream().map(Path::getName).collect(Collectors.toSet());
+ assertThat(latestWals,
+ Matchers.<Set<String>> both(hasSize(2)).and(hasItems(walName1, walName2)));
}
- private static long getSizeOfLatestPath() {
- // If no mini cluster is running, there are extra replication manager influencing the metrics.
- if (utility.getMiniHBaseCluster() == null) {
- return 0;
- }
- return utility.getMiniHBaseCluster().getRegionServerThreads().stream()
- .map(JVMClusterUtil.RegionServerThread::getRegionServer)
- .map(RegionServerServices::getReplicationSourceService).map(r -> (Replication) r)
- .map(Replication::getReplicationManager)
- .mapToLong(ReplicationSourceManager::getSizeOfLatestPath).sum();
+ private MetricsReplicationSourceSource getGlobalSource() {
+ return CompatibilitySingletonFactory.getInstance(MetricsReplicationSourceFactory.class)
+ .getGlobalSource();
}
@Test
public void testRemovePeerMetricsCleanup() throws Exception {
- final String peerId = "DummyPeer";
- final ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
- .setClusterKey(utility.getZkCluster().getAddress().toString() + ":/hbase").build();
- try {
- MetricsReplicationSourceSource globalSource = getGlobalSource();
- final int globalLogQueueSizeInitial = globalSource.getSizeOfLogQueue();
- final long sizeOfLatestPath = getSizeOfLatestPath();
- addPeerAndWait(peerId, peerConfig, true);
- assertEquals(sizeOfLatestPath + globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
- ReplicationSourceInterface source = manager.getSource(peerId);
- // Sanity check
- assertNotNull(source);
- final int sizeOfSingleLogQueue = source.getSourceMetrics().getSizeOfLogQueue();
- // Enqueue log and check if metrics updated
- source.enqueueLog(new Path("abc"));
- assertEquals(1 + sizeOfSingleLogQueue, source.getSourceMetrics().getSizeOfLogQueue());
- assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
- globalSource.getSizeOfLogQueue());
-
- // Removing the peer should reset the global metrics
- removePeerAndWait(peerId);
- assertEquals(globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
-
- // Adding the same peer back again should reset the single source metrics
- addPeerAndWait(peerId, peerConfig, true);
- source = manager.getSource(peerId);
- assertNotNull(source);
- assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
- globalSource.getSizeOfLogQueue());
- } finally {
- removePeerAndWait(peerId);
- }
+ MetricsReplicationSourceSource globalSource = getGlobalSource();
+ int globalLogQueueSizeInitial = globalSource.getSizeOfLogQueue();
+ String peerId = "DummyPeer";
+ addPeerAndWait(peerId, "hbase", false);
+ // there is no latestPaths so the size of log queue should not change
+ assertEquals(globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
+
+ ReplicationSourceInterface source = manager.getSource(peerId);
+ // Sanity check
+ assertNotNull(source);
+ int sizeOfSingleLogQueue = source.getSourceMetrics().getSizeOfLogQueue();
+ // Enqueue log and check if metrics updated
+ Path serverLogDir = new Path(logDir, server.getServerName().toString());
+ source.enqueueLog(new Path(serverLogDir, server.getServerName() + ".1"));
+ assertEquals(1 + sizeOfSingleLogQueue, source.getSourceMetrics().getSizeOfLogQueue());
+ assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
+ globalSource.getSizeOfLogQueue());
+
+ // Removing the peer should reset the global metrics
+ removePeerAndWait(peerId);
+ assertEquals(globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
+
+ // Adding the same peer back again should reset the single source metrics
+ addPeerAndWait(peerId, "hbase", false);
+ source = manager.getSource(peerId);
+ assertNotNull(source);
+ assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
+ globalSource.getSizeOfLogQueue());
}
@Test
public void testDisablePeerMetricsCleanup() throws Exception {
final String peerId = "DummyPeer";
- final ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
- .setClusterKey(utility.getZkCluster().getAddress().toString() + ":/hbase").build();
try {
MetricsReplicationSourceSource globalSource = getGlobalSource();
final int globalLogQueueSizeInitial = globalSource.getSizeOfLogQueue();
- final long sizeOfLatestPath = getSizeOfLatestPath();
- addPeerAndWait(peerId, peerConfig, true);
- assertEquals(sizeOfLatestPath + globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
+ addPeerAndWait(peerId, "hbase", false);
+ assertEquals(globalLogQueueSizeInitial, globalSource.getSizeOfLogQueue());
ReplicationSourceInterface source = manager.getSource(peerId);
// Sanity check
assertNotNull(source);
final int sizeOfSingleLogQueue = source.getSourceMetrics().getSizeOfLogQueue();
// Enqueue log and check if metrics updated
- source.enqueueLog(new Path("abc"));
+ Path serverLogDir = new Path(logDir, server.getServerName().toString());
+ source.enqueueLog(new Path(serverLogDir, server.getServerName() + ".1"));
assertEquals(1 + sizeOfSingleLogQueue, source.getSourceMetrics().getSizeOfLogQueue());
assertEquals(source.getSourceMetrics().getSizeOfLogQueue() + globalLogQueueSizeInitial,
globalSource.getSizeOfLogQueue());
@@ -607,274 +355,27 @@ public abstract class TestReplicationSourceManager {
}
}
- private ReplicationSourceInterface mockReplicationSource(String peerId) {
- ReplicationSourceInterface source = mock(ReplicationSourceInterface.class);
- when(source.getPeerId()).thenReturn(peerId);
- when(source.getQueueId()).thenReturn(peerId);
- when(source.isRecovered()).thenReturn(false);
- when(source.isSyncReplication()).thenReturn(true);
- ReplicationPeerConfig config = mock(ReplicationPeerConfig.class);
- when(config.getRemoteWALDir())
- .thenReturn(remoteLogDir.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString());
- ReplicationPeer peer = mock(ReplicationPeer.class);
- when(peer.getPeerConfig()).thenReturn(config);
- when(source.getPeer()).thenReturn(peer);
- return source;
- }
-
@Test
public void testRemoveRemoteWALs() throws Exception {
- String peerId2 = slaveId + "_2";
- addPeerAndWait(peerId2,
- ReplicationPeerConfig.newBuilder()
- .setClusterKey("localhost:" + utility.getZkCluster().getClientPort() + ":/hbase").build(),
- true);
- try {
- // make sure that we can deal with files which does not exist
- String walNameNotExists =
- "remoteWAL-12345-" + slaveId + ".12345" + ReplicationUtils.SYNC_WAL_SUFFIX;
- Path wal = new Path(logDir, walNameNotExists);
- manager.preLogRoll(wal);
- manager.postLogRoll(wal);
-
- Path remoteLogDirForPeer = new Path(remoteLogDir, slaveId);
- fs.mkdirs(remoteLogDirForPeer);
- String walName = "remoteWAL-12345-" + slaveId + ".23456" + ReplicationUtils.SYNC_WAL_SUFFIX;
- Path remoteWAL =
- new Path(remoteLogDirForPeer, walName).makeQualified(fs.getUri(), fs.getWorkingDirectory());
- fs.create(remoteWAL).close();
- wal = new Path(logDir, walName);
- manager.preLogRoll(wal);
- manager.postLogRoll(wal);
-
- ReplicationSourceInterface source = mockReplicationSource(peerId2);
- manager.cleanOldLogs(walName, true, source);
- // still there if peer id does not match
- assertTrue(fs.exists(remoteWAL));
-
- source = mockReplicationSource(slaveId);
- manager.cleanOldLogs(walName, true, source);
- assertFalse(fs.exists(remoteWAL));
- } finally {
- removePeerAndWait(peerId2);
- }
- }
-
- @Test
- public void testSameWALPrefix() throws IOException {
- Set<String> latestWalsBefore =
- manager.getLastestPath().stream().map(Path::getName).collect(Collectors.toSet());
- String walName1 = "localhost,8080,12345-45678-Peer.34567";
- String walName2 = "localhost,8080,12345.56789";
- manager.preLogRoll(new Path(walName1));
- manager.preLogRoll(new Path(walName2));
-
- Set<String> latestWals = manager.getLastestPath().stream().map(Path::getName)
- .filter(n -> !latestWalsBefore.contains(n)).collect(Collectors.toSet());
- assertEquals(2, latestWals.size());
- assertTrue(latestWals.contains(walName1));
- assertTrue(latestWals.contains(walName2));
- }
-
- /**
- * Add a peer and wait for it to initialize
- * @param waitForSource Whether to wait for replication source to initialize
- */
- private void addPeerAndWait(final String peerId, final ReplicationPeerConfig peerConfig,
- final boolean waitForSource) throws Exception {
- final ReplicationPeers rp = manager.getReplicationPeers();
- rp.getPeerStorage().addPeer(peerId, peerConfig, true, SyncReplicationState.NONE);
- try {
- manager.addPeer(peerId);
- } catch (Exception e) {
- // ignore the failed exception, because we'll test both success & failed case.
- }
- waitPeer(peerId, manager, waitForSource);
- if (managerOfCluster != null) {
- managerOfCluster.addPeer(peerId);
- waitPeer(peerId, managerOfCluster, waitForSource);
- }
- }
-
- private static void waitPeer(final String peerId, ReplicationSourceManager manager,
- final boolean waitForSource) {
- ReplicationPeers rp = manager.getReplicationPeers();
- Waiter.waitFor(conf, 20000, () -> {
- if (waitForSource) {
- ReplicationSourceInterface rs = manager.getSource(peerId);
- if (rs == null) {
- return false;
- }
- if (rs instanceof ReplicationSourceDummy) {
- return ((ReplicationSourceDummy) rs).isStartup();
- }
- return true;
- } else {
- return (rp.getPeer(peerId) != null);
- }
- });
- }
-
- /**
- * Remove a peer and wait for it to get cleaned up
- */
- private void removePeerAndWait(final String peerId) throws Exception {
- final ReplicationPeers rp = manager.getReplicationPeers();
- if (rp.getPeerStorage().listPeerIds().contains(peerId)) {
- rp.getPeerStorage().removePeer(peerId);
- try {
- manager.removePeer(peerId);
- } catch (Exception e) {
- // ignore the failed exception and continue.
- }
- }
- Waiter.waitFor(conf, 20000, new Waiter.Predicate<Exception>() {
- @Override
- public boolean evaluate() throws Exception {
- Collection<String> peers = rp.getPeerStorage().listPeerIds();
- return (!manager.getAllQueues().contains(peerId)) && (rp.getPeer(peerId) == null)
- && (!peers.contains(peerId)) && manager.getSource(peerId) == null;
- }
- });
- }
-
- private WALEdit getBulkLoadWALEdit(NavigableMap<byte[], Integer> scope) {
- // 1. Create store files for the families
- Map<byte[], List<Path>> storeFiles = new HashMap<>(1);
- Map<String, Long> storeFilesSize = new HashMap<>(1);
- List<Path> p = new ArrayList<>(1);
- Path hfilePath1 = new Path(Bytes.toString(f1));
- p.add(hfilePath1);
- try {
- storeFilesSize.put(hfilePath1.getName(), fs.getFileStatus(hfilePath1).getLen());
- } catch (IOException e) {
- LOG.debug("Failed to calculate the size of hfile " + hfilePath1);
- storeFilesSize.put(hfilePath1.getName(), 0L);
- }
- storeFiles.put(f1, p);
- scope.put(f1, 1);
- p = new ArrayList<>(1);
- Path hfilePath2 = new Path(Bytes.toString(f2));
- p.add(hfilePath2);
- try {
- storeFilesSize.put(hfilePath2.getName(), fs.getFileStatus(hfilePath2).getLen());
- } catch (IOException e) {
- LOG.debug("Failed to calculate the size of hfile " + hfilePath2);
- storeFilesSize.put(hfilePath2.getName(), 0L);
- }
- storeFiles.put(f2, p);
- // 2. Create bulk load descriptor
- BulkLoadDescriptor desc = ProtobufUtil.toBulkLoadDescriptor(hri.getTable(),
- UnsafeByteOperations.unsafeWrap(hri.getEncodedNameAsBytes()), storeFiles, storeFilesSize, 1);
-
- // 3. create bulk load wal edit event
- WALEdit logEdit = WALEdit.createBulkLoadEvent(hri, desc);
- return logEdit;
- }
-
- static class DummyNodeFailoverWorker extends Thread {
- private Map<String, Set<String>> logZnodesMap;
- Server server;
- private ServerName deadRS;
- ReplicationQueueStorage rq;
-
- public DummyNodeFailoverWorker(ServerName deadRS, Server s) throws Exception {
- this.deadRS = deadRS;
- this.server = s;
- this.rq = ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(),
- server.getConfiguration());
- }
-
- @Override
- public void run() {
- try {
- logZnodesMap = new HashMap<>();
- List<String> queues = rq.getAllQueues(deadRS);
- for (String queue : queues) {
- Pair<String, SortedSet<String>> pair =
- rq.claimQueue(deadRS, queue, server.getServerName());
- if (pair != null) {
- logZnodesMap.put(pair.getFirst(), pair.getSecond());
- }
- }
- server.abort("Done with testing", null);
- } catch (Exception e) {
- LOG.error("Got exception while running NodeFailoverWorker", e);
- } finally {
- latch.countDown();
- }
- }
-
- /** Returns 1 when the map is not empty. */
- private int isLogZnodesMapPopulated() {
- Collection<Set<String>> sets = logZnodesMap.values();
- if (sets.size() > 1) {
- throw new RuntimeException("unexpected size of logZnodesMap: " + sets.size());
- }
- if (sets.size() == 1) {
- Set<String> s = sets.iterator().next();
- for (String file : files) {
- // at least one file was missing
- if (!s.contains(file)) {
- return 0;
- }
- }
- return 1; // we found all the files
- }
- return 0;
- }
- }
-
- static class FailInitializeDummyReplicationSource extends ReplicationSourceDummy {
-
- @Override
- public void init(Configuration conf, FileSystem fs, ReplicationSourceManager manager,
- ReplicationQueueStorage rq, ReplicationPeer rp, Server server, String peerClusterId,
- UUID clusterId, WALFileLengthProvider walFileLengthProvider, MetricsSource metrics)
- throws IOException {
- throw new IOException("Failing deliberately");
- }
- }
-
- static class DummyServer extends MockServer {
- String hostname;
-
- DummyServer() {
- hostname = "hostname.example.org";
- }
-
- DummyServer(String hostname) {
- this.hostname = hostname;
- }
-
- @Override
- public Configuration getConfiguration() {
- return conf;
- }
-
- @Override
- public ZKWatcher getZooKeeper() {
- return zkw;
- }
-
- @Override
- public FileSystem getFileSystem() {
- return fs;
- }
-
- @Override
- public Connection getConnection() {
- return null;
- }
-
- @Override
- public ChoreService getChoreService() {
- return null;
- }
-
- @Override
- public ServerName getServerName() {
- return ServerName.valueOf(hostname, 1234, 1L);
- }
+ String peerId = "2";
+ addPeerAndWait(peerId, "hbase", true);
+ // make sure that we can deal with files which does not exist
+ String walNameNotExists =
+ "remoteWAL-12345-" + peerId + ".12345" + ReplicationUtils.SYNC_WAL_SUFFIX;
+ Path wal = new Path(logDir, walNameNotExists);
+ manager.postLogRoll(wal);
+
+ Path remoteLogDirForPeer = new Path(remoteLogDir, peerId);
+ FS.mkdirs(remoteLogDirForPeer);
+ String walName = "remoteWAL-12345-" + peerId + ".23456" + ReplicationUtils.SYNC_WAL_SUFFIX;
+ Path remoteWAL =
+ new Path(remoteLogDirForPeer, walName).makeQualified(FS.getUri(), FS.getWorkingDirectory());
+ FS.create(remoteWAL).close();
+ wal = new Path(logDir, walName);
+ manager.postLogRoll(wal);
+
+ ReplicationSourceInterface source = manager.getSource(peerId);
+ manager.cleanOldLogs(walName, true, source);
+ assertFalse(FS.exists(remoteWAL));
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
deleted file mode 100644
index b26505a6270..00000000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManagerZkImpl.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication.regionserver;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.List;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtil;
-import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.replication.ReplicationQueueInfo;
-import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
-import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.ReplicationTests;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Tests the ReplicationSourceManager with ReplicationQueueZkImpl's and
- * ReplicationQueuesClientZkImpl. Also includes extra tests outside of those in
- * TestReplicationSourceManager that test ReplicationQueueZkImpl-specific behaviors.
- */
-@Category({ ReplicationTests.class, MediumTests.class })
-public class TestReplicationSourceManagerZkImpl extends TestReplicationSourceManager {
-
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestReplicationSourceManagerZkImpl.class);
-
- @BeforeClass
- public static void setUpBeforeClass() throws Exception {
- conf = HBaseConfiguration.create();
- conf.set("replication.replicationsource.implementation",
- ReplicationSourceDummy.class.getCanonicalName());
- conf.setLong("replication.sleep.before.failover", 2000);
- conf.setInt("replication.source.maxretriesmultiplier", 10);
- utility = new HBaseTestingUtil(conf);
- utility.startMiniZKCluster();
- setupZkAndReplication();
- }
-
- // Tests the naming convention of adopted queues for ReplicationQueuesZkImpl
- @Test
- public void testNodeFailoverDeadServerParsing() throws Exception {
- Server server = new DummyServer("ec2-54-234-230-108.compute-1.amazonaws.com");
- ReplicationQueueStorage queueStorage =
- ReplicationStorageFactory.getReplicationQueueStorage(server.getZooKeeper(), conf);
- // populate some znodes in the peer znode
- files.add("log1");
- files.add("log2");
- for (String file : files) {
- queueStorage.addWAL(server.getServerName(), "1", file);
- }
-
- // create 3 DummyServers
- Server s1 = new DummyServer("ip-10-8-101-114.ec2.internal");
- Server s2 = new DummyServer("ec2-107-20-52-47.compute-1.amazonaws.com");
- Server s3 = new DummyServer("ec2-23-20-187-167.compute-1.amazonaws.com");
-
- // simulate three servers fail sequentially
- ServerName serverName = server.getServerName();
- List<String> unclaimed = queueStorage.getAllQueues(serverName);
- queueStorage.claimQueue(serverName, unclaimed.get(0), s1.getServerName());
- queueStorage.removeReplicatorIfQueueIsEmpty(serverName);
-
- serverName = s1.getServerName();
- unclaimed = queueStorage.getAllQueues(serverName);
- queueStorage.claimQueue(serverName, unclaimed.get(0), s2.getServerName());
- queueStorage.removeReplicatorIfQueueIsEmpty(serverName);
-
- serverName = s2.getServerName();
- unclaimed = queueStorage.getAllQueues(serverName);
- String queue3 =
- queueStorage.claimQueue(serverName, unclaimed.get(0), s3.getServerName()).getFirst();
- queueStorage.removeReplicatorIfQueueIsEmpty(serverName);
-
- ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(queue3);
- List<ServerName> result = replicationQueueInfo.getDeadRegionServers();
- // verify
- assertTrue(result.contains(server.getServerName()));
- assertTrue(result.contains(s1.getServerName()));
- assertTrue(result.contains(s2.getServerName()));
-
- server.stop("");
- }
-}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALEdits.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALEdits.java
new file mode 100644
index 00000000000..bc885db0df5
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALEdits.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication.regionserver;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.testclassification.ReplicationTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.wal.WALEdit;
+import org.apache.hadoop.hbase.wal.WALKeyImpl;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
+
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor;
+
+@Category({ ReplicationTests.class, SmallTests.class })
+public class TestReplicationWALEdits {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestReplicationWALEdits.class);
+
+ private static final Configuration CONF = HBaseConfiguration.create();
+
+ private static final TableName TABLE_NAME = TableName.valueOf("test");
+
+ private static final byte[] F1 = Bytes.toBytes("f1");
+
+ private static final byte[] F2 = Bytes.toBytes("f2");
+
+ private static final RegionInfo RI = RegionInfoBuilder.newBuilder(TABLE_NAME).build();
+
+ /**
+ * Test for HBASE-9038, Replication.scopeWALEdits would NPE if it wasn't filtering out the
+ * compaction WALEdit.
+ */
+ @Test
+ public void testCompactionWALEdits() throws Exception {
+ TableName tableName = TableName.valueOf("testCompactionWALEdits");
+ WALProtos.CompactionDescriptor compactionDescriptor =
+ WALProtos.CompactionDescriptor.getDefaultInstance();
+ RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(HConstants.EMPTY_START_ROW)
+ .setEndKey(HConstants.EMPTY_END_ROW).build();
+ WALEdit edit = WALEdit.createCompaction(hri, compactionDescriptor);
+ ReplicationSourceWALActionListener.scopeWALEdits(new WALKeyImpl(), edit, CONF);
+ }
+
+ private WALEdit getBulkLoadWALEdit(NavigableMap<byte[], Integer> scope) {
+ // 1. Create store files for the families
+ Map<byte[], List<Path>> storeFiles = new HashMap<>(1);
+ Map<String, Long> storeFilesSize = new HashMap<>(1);
+ List<Path> p = new ArrayList<>(1);
+ Path hfilePath1 = new Path(Bytes.toString(F1));
+ p.add(hfilePath1);
+ storeFilesSize.put(hfilePath1.getName(), 0L);
+ storeFiles.put(F1, p);
+ scope.put(F1, 1);
+ p = new ArrayList<>(1);
+ Path hfilePath2 = new Path(Bytes.toString(F2));
+ p.add(hfilePath2);
+ storeFilesSize.put(hfilePath2.getName(), 0L);
+ storeFiles.put(F2, p);
+ // 2. Create bulk load descriptor
+ BulkLoadDescriptor desc = ProtobufUtil.toBulkLoadDescriptor(RI.getTable(),
+ UnsafeByteOperations.unsafeWrap(RI.getEncodedNameAsBytes()), storeFiles, storeFilesSize, 1);
+
+ // 3. create bulk load wal edit event
+ WALEdit logEdit = WALEdit.createBulkLoadEvent(RI, desc);
+ return logEdit;
+ }
+
+ @Test
+ public void testBulkLoadWALEditsWithoutBulkLoadReplicationEnabled() throws Exception {
+ NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ // 1. Get the bulk load wal edit event
+ WALEdit logEdit = getBulkLoadWALEdit(scope);
+ // 2. Create wal key
+ WALKeyImpl logKey = new WALKeyImpl(scope);
+
+ // 3. Get the scopes for the key
+ ReplicationSourceWALActionListener.scopeWALEdits(logKey, logEdit, CONF);
+
+ // 4. Assert that no bulk load entry scopes are added if bulk load hfile replication is disabled
+ assertNull("No bulk load entries scope should be added if bulk load replication is disabled.",
+ logKey.getReplicationScopes());
+ }
+
+ @Test
+ public void testBulkLoadWALEdits() throws Exception {
+ // 1. Get the bulk load wal edit event
+ NavigableMap<byte[], Integer> scope = new TreeMap<>(Bytes.BYTES_COMPARATOR);
+ WALEdit logEdit = getBulkLoadWALEdit(scope);
+ // 2. Create wal key
+ WALKeyImpl logKey = new WALKeyImpl(scope);
+ // 3. Enable bulk load hfile replication
+ Configuration bulkLoadConf = HBaseConfiguration.create(CONF);
+ bulkLoadConf.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
+
+ // 4. Get the scopes for the key
+ ReplicationSourceWALActionListener.scopeWALEdits(logKey, logEdit, bulkLoadConf);
+
+ NavigableMap<byte[], Integer> scopes = logKey.getReplicationScopes();
+ // Assert family with replication scope global is present in the key scopes
+ assertTrue("This family scope is set to global, should be part of replication key scopes.",
+ scopes.containsKey(F1));
+ // Assert family with replication scope local is not present in the key scopes
+ assertFalse("This family scope is set to local, should not be part of replication key scopes",
+ scopes.containsKey(F2));
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java
index 9da36769429..1544265435c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestSerialReplicationChecker.java
@@ -46,6 +46,8 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.replication.ReplicationBarrierFamilyFormat;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -94,10 +96,11 @@ public class TestSerialReplicationChecker {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
UTIL.startMiniCluster(1);
- QUEUE_STORAGE = ReplicationStorageFactory.getReplicationQueueStorage(UTIL.getZooKeeperWatcher(),
- UTIL.getConfiguration());
- QUEUE_STORAGE.addWAL(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(), PEER_ID,
- WAL_FILE_NAME);
+ TableName repTable = TableName.valueOf("test_serial_rep");
+ UTIL.getAdmin()
+ .createTable(ReplicationStorageFactory.createReplicationQueueTableDescriptor(repTable));
+ QUEUE_STORAGE =
+ ReplicationStorageFactory.getReplicationQueueStorage(UTIL.getConnection(), repTable);
}
@AfterClass
@@ -174,8 +177,10 @@ public class TestSerialReplicationChecker {
}
private void updatePushedSeqId(RegionInfo region, long seqId) throws ReplicationException {
- QUEUE_STORAGE.setWALPosition(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(),
- PEER_ID, WAL_FILE_NAME, 10, ImmutableMap.of(region.getEncodedName(), seqId));
+ ReplicationQueueId queueId = new ReplicationQueueId(
+ UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(), PEER_ID);
+ QUEUE_STORAGE.setOffset(queueId, "", new ReplicationGroupOffset(WAL_FILE_NAME, 10),
+ ImmutableMap.of(region.getEncodedName(), seqId));
}
private void addParents(RegionInfo region, List<RegionInfo> parents) throws IOException {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java
index d66aef492ff..93fa22c00fd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntrySinkFilter.java
@@ -25,7 +25,6 @@ import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
@@ -35,7 +34,6 @@ import org.apache.hadoop.hbase.CellBuilderType;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.AdvancedScanResultConsumer;
import org.apache.hadoop.hbase.client.AsyncClusterConnection;
@@ -55,8 +53,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
@@ -72,31 +68,12 @@ public class TestWALEntrySinkFilter {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALEntrySinkFilter.class);
- private static final Logger LOG = LoggerFactory.getLogger(TestReplicationSink.class);
@Rule
public TestName name = new TestName();
static final int BOUNDARY = 5;
static final AtomicInteger UNFILTERED = new AtomicInteger();
static final AtomicInteger FILTERED = new AtomicInteger();
- /**
- * Implemetentation of Stoppable to pass into ReplicationSink.
- */
- private static Stoppable STOPPABLE = new Stoppable() {
- private final AtomicBoolean stop = new AtomicBoolean(false);
-
- @Override
- public boolean isStopped() {
- return this.stop.get();
- }
-
- @Override
- public void stop(String why) {
- LOG.info("STOPPING BECAUSE: " + why);
- this.stop.set(true);
- }
- };
-
/**
* Test filter. Filter will filter out any write time that is <= 5 (BOUNDARY). We count how many
* items we filter out and we count how many cells make it through for distribution way down below
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java
index d4f0ec664e3..20ed3796dbd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckCleanReplicationBarriers.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MetaTableAccessor;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put;
@@ -43,7 +44,9 @@ import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.master.RegionState;
import org.apache.hadoop.hbase.replication.ReplicationBarrierFamilyFormat;
import org.apache.hadoop.hbase.replication.ReplicationException;
+import org.apache.hadoop.hbase.replication.ReplicationGroupOffset;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationQueueId;
import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -78,13 +81,9 @@ public class TestHBaseFsckCleanReplicationBarriers {
@BeforeClass
public static void setUp() throws Exception {
UTIL.startMiniCluster(1);
- QUEUE_STORAGE = ReplicationStorageFactory.getReplicationQueueStorage(UTIL.getZooKeeperWatcher(),
+ QUEUE_STORAGE = ReplicationStorageFactory.getReplicationQueueStorage(UTIL.getConnection(),
UTIL.getConfiguration());
createPeer();
- QUEUE_STORAGE.addWAL(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(), PEER_1,
- WAL_FILE_NAME);
- QUEUE_STORAGE.addWAL(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(), PEER_2,
- WAL_FILE_NAME);
}
@AfterClass
@@ -205,9 +204,12 @@ public class TestHBaseFsckCleanReplicationBarriers {
}
private void updatePushedSeqId(RegionInfo region, long seqId) throws ReplicationException {
- QUEUE_STORAGE.setWALPosition(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(),
- PEER_1, WAL_FILE_NAME, 10, ImmutableMap.of(region.getEncodedName(), seqId));
- QUEUE_STORAGE.setWALPosition(UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(),
- PEER_2, WAL_FILE_NAME, 10, ImmutableMap.of(region.getEncodedName(), seqId));
+ ServerName sn = UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName();
+ QUEUE_STORAGE.setOffset(new ReplicationQueueId(sn, PEER_1), "",
+ new ReplicationGroupOffset(WAL_FILE_NAME, 10),
+ ImmutableMap.of(region.getEncodedName(), seqId));
+ QUEUE_STORAGE.setOffset(new ReplicationQueueId(sn, PEER_2), "",
+ new ReplicationGroupOffset(WAL_FILE_NAME, 10),
+ ImmutableMap.of(region.getEncodedName(), seqId));
}
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java
index 66fd10bd156..aeed1a9a483 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckEncryption.java
@@ -55,9 +55,12 @@ import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+//revisit later
+@Ignore
@Category({ MiscTests.class, MediumTests.class })
public class TestHBaseFsckEncryption {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
index 70ea559e330..b24b721762d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckMOB.java
@@ -40,11 +40,14 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
+// revisit later
+@Ignore
@Category({ MiscTests.class, MediumTests.class })
public class TestHBaseFsckMOB extends BaseTestHBaseFsck {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
index e44e00d2d37..fdf0d2d6a25 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckReplication.java
@@ -17,25 +17,14 @@
*/
package org.apache.hadoop.hbase.util;
-import static org.junit.Assert.assertEquals;
-
-import java.util.List;
-import java.util.stream.Stream;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
-import org.apache.hadoop.hbase.replication.ReplicationPeerStorage;
-import org.apache.hadoop.hbase.replication.ReplicationQueueStorage;
-import org.apache.hadoop.hbase.replication.ReplicationStorageFactory;
-import org.apache.hadoop.hbase.replication.SyncReplicationState;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.ReplicationTests;
-import org.apache.hadoop.hbase.util.HbckErrorReporter.ERROR_CODE;
-import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -59,51 +48,53 @@ public class TestHBaseFsckReplication {
UTIL.shutdownMiniCluster();
}
+ // TODO: reimplement
+ @Ignore
@Test
public void test() throws Exception {
- ReplicationPeerStorage peerStorage = ReplicationStorageFactory.getReplicationPeerStorage(
- UTIL.getTestFileSystem(), UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
- ReplicationQueueStorage queueStorage = ReplicationStorageFactory
- .getReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
-
- String peerId1 = "1";
- String peerId2 = "2";
- peerStorage.addPeer(peerId1, ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
- true, SyncReplicationState.NONE);
- peerStorage.addPeer(peerId2, ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
- true, SyncReplicationState.NONE);
- for (int i = 0; i < 10; i++) {
- queueStorage.addWAL(ServerName.valueOf("localhost", 10000 + i, 100000 + i), peerId1,
- "file-" + i);
- }
- queueStorage.addWAL(ServerName.valueOf("localhost", 10000, 100000), peerId2, "file");
- HBaseFsck fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), true);
- HbckTestingUtil.assertNoErrors(fsck);
-
- // should not remove anything since the replication peer is still alive
- assertEquals(10, queueStorage.getListOfReplicators().size());
- peerStorage.removePeer(peerId1);
- // there should be orphan queues
- assertEquals(10, queueStorage.getListOfReplicators().size());
- fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), false);
- HbckTestingUtil.assertErrors(fsck, Stream.generate(() -> {
- return ERROR_CODE.UNDELETED_REPLICATION_QUEUE;
- }).limit(10).toArray(ERROR_CODE[]::new));
-
- // should not delete anything when fix is false
- assertEquals(10, queueStorage.getListOfReplicators().size());
-
- fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), true);
- HbckTestingUtil.assertErrors(fsck, Stream.generate(() -> {
- return ERROR_CODE.UNDELETED_REPLICATION_QUEUE;
- }).limit(10).toArray(ERROR_CODE[]::new));
-
- List<ServerName> replicators = queueStorage.getListOfReplicators();
- // should not remove the server with queue for peerId2
- assertEquals(1, replicators.size());
- assertEquals(ServerName.valueOf("localhost", 10000, 100000), replicators.get(0));
- for (String queueId : queueStorage.getAllQueues(replicators.get(0))) {
- assertEquals(peerId2, queueId);
- }
+ // ReplicationPeerStorage peerStorage = ReplicationStorageFactory
+ // .getReplicationPeerStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
+ // ReplicationQueueStorage queueStorage = ReplicationStorageFactory
+ // .getReplicationQueueStorage(UTIL.getZooKeeperWatcher(), UTIL.getConfiguration());
+ //
+ // String peerId1 = "1";
+ // String peerId2 = "2";
+ // peerStorage.addPeer(peerId1, ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
+ // true, SyncReplicationState.NONE);
+ // peerStorage.addPeer(peerId2, ReplicationPeerConfig.newBuilder().setClusterKey("key").build(),
+ // true, SyncReplicationState.NONE);
+ // for (int i = 0; i < 10; i++) {
+ // queueStorage.addWAL(ServerName.valueOf("localhost", 10000 + i, 100000 + i), peerId1,
+ // "file-" + i);
+ // }
+ // queueStorage.addWAL(ServerName.valueOf("localhost", 10000, 100000), peerId2, "file");
+ // HBaseFsck fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), true);
+ // HbckTestingUtil.assertNoErrors(fsck);
+ //
+ // // should not remove anything since the replication peer is still alive
+ // assertEquals(10, queueStorage.getListOfReplicators().size());
+ // peerStorage.removePeer(peerId1);
+ // // there should be orphan queues
+ // assertEquals(10, queueStorage.getListOfReplicators().size());
+ // fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), false);
+ // HbckTestingUtil.assertErrors(fsck, Stream.generate(() -> {
+ // return ERROR_CODE.UNDELETED_REPLICATION_QUEUE;
+ // }).limit(10).toArray(ERROR_CODE[]::new));
+ //
+ // // should not delete anything when fix is false
+ // assertEquals(10, queueStorage.getListOfReplicators().size());
+ //
+ // fsck = HbckTestingUtil.doFsck(UTIL.getConfiguration(), true);
+ // HbckTestingUtil.assertErrors(fsck, Stream.generate(() -> {
+ // return ERROR_CODE.UNDELETED_REPLICATION_QUEUE;
+ // }).limit(10).toArray(ERROR_CODE[]::new));
+ //
+ // List<ServerName> replicators = queueStorage.getListOfReplicators();
+ // // should not remove the server with queue for peerId2
+ // assertEquals(1, replicators.size());
+ // assertEquals(ServerName.valueOf("localhost", 10000, 100000), replicators.get(0));
+ // for (String queueId : queueStorage.getAllQueues(replicators.get(0))) {
+ // assertEquals(peerId2, queueId);
+ // }
}
}