You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2017/08/23 21:42:42 UTC
[1/4] hbase git commit: HBASE-17442 Move most of the replication
related classes from hbase-client to new hbase-replication package. (Guanghao
Zhang).
Repository: hbase
Updated Branches:
refs/heads/master ae052e454 -> 26e6c2ceb
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
new file mode 100644
index 0000000..4733706
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
@@ -0,0 +1,407 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * This class provides an implementation of the
+ * interface using ZooKeeper. The
+ * base znode that this class works at is the myQueuesZnode. The myQueuesZnode contains a list of
+ * all outstanding WAL files on this region server that need to be replicated. The myQueuesZnode is
+ * the regionserver name (a concatenation of the region server’s hostname, client port and start
+ * code). For example:
+ *
+ * /hbase/replication/rs/hostname.example.org,6020,1234
+ *
+ * Within this znode, the region server maintains a set of WAL replication queues. These queues are
+ * represented by child znodes named using there give queue id. For example:
+ *
+ * /hbase/replication/rs/hostname.example.org,6020,1234/1
+ * /hbase/replication/rs/hostname.example.org,6020,1234/2
+ *
+ * Each queue has one child znode for every WAL that still needs to be replicated. The value of
+ * these WAL child znodes is the latest position that has been replicated. This position is updated
+ * every time a WAL entry is replicated. For example:
+ *
+ * /hbase/replication/rs/hostname.example.org,6020,1234/1/23522342.23422 [VALUE: 254]
+ */
+@InterfaceAudience.Private
+public class ReplicationQueuesZKImpl extends ReplicationStateZKBase implements ReplicationQueues {
+
+ /** Znode containing all replication queues for this region server. */
+ private String myQueuesZnode;
+
+ private static final Log LOG = LogFactory.getLog(ReplicationQueuesZKImpl.class);
+
+ public ReplicationQueuesZKImpl(ReplicationQueuesArguments args) {
+ this(args.getZk(), args.getConf(), args.getAbortable());
+ }
+
+ public ReplicationQueuesZKImpl(final ZooKeeperWatcher zk, Configuration conf,
+ Abortable abortable) {
+ super(zk, conf, abortable);
+ }
+
+ @Override
+ public void init(String serverName) throws ReplicationException {
+ this.myQueuesZnode = ZKUtil.joinZNode(this.queuesZNode, serverName);
+ try {
+ if (ZKUtil.checkExists(this.zookeeper, this.myQueuesZnode) < 0) {
+ ZKUtil.createWithParents(this.zookeeper, this.myQueuesZnode);
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException("Could not initialize replication queues.", e);
+ }
+ if (conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
+ HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT)) {
+ try {
+ if (ZKUtil.checkExists(this.zookeeper, this.hfileRefsZNode) < 0) {
+ ZKUtil.createWithParents(this.zookeeper, this.hfileRefsZNode);
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException("Could not initialize hfile references replication queue.",
+ e);
+ }
+ }
+ }
+
+ @Override
+ public void removeQueue(String queueId) {
+ try {
+ ZKUtil.deleteNodeRecursively(this.zookeeper, ZKUtil.joinZNode(this.myQueuesZnode, queueId));
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to delete queue (queueId=" + queueId + ")", e);
+ }
+ }
+
+ @Override
+ public void addLog(String queueId, String filename) throws ReplicationException {
+ String znode = ZKUtil.joinZNode(this.myQueuesZnode, queueId);
+ znode = ZKUtil.joinZNode(znode, filename);
+ try {
+ ZKUtil.createWithParents(this.zookeeper, znode);
+ } catch (KeeperException e) {
+ throw new ReplicationException(
+ "Could not add log because znode could not be created. queueId=" + queueId
+ + ", filename=" + filename);
+ }
+ }
+
+ @Override
+ public void removeLog(String queueId, String filename) {
+ try {
+ String znode = ZKUtil.joinZNode(this.myQueuesZnode, queueId);
+ znode = ZKUtil.joinZNode(znode, filename);
+ ZKUtil.deleteNode(this.zookeeper, znode);
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to remove wal from queue (queueId=" + queueId + ", filename="
+ + filename + ")", e);
+ }
+ }
+
+ @Override
+ public void setLogPosition(String queueId, String filename, long position) {
+ try {
+ String znode = ZKUtil.joinZNode(this.myQueuesZnode, queueId);
+ znode = ZKUtil.joinZNode(znode, filename);
+ // Why serialize String of Long and not Long as bytes?
+ ZKUtil.setData(this.zookeeper, znode, ZKUtil.positionToByteArray(position));
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to write replication wal position (filename=" + filename
+ + ", position=" + position + ")", e);
+ }
+ }
+
+ @Override
+ public long getLogPosition(String queueId, String filename) throws ReplicationException {
+ String clusterZnode = ZKUtil.joinZNode(this.myQueuesZnode, queueId);
+ String znode = ZKUtil.joinZNode(clusterZnode, filename);
+ byte[] bytes = null;
+ try {
+ bytes = ZKUtil.getData(this.zookeeper, znode);
+ } catch (KeeperException e) {
+ throw new ReplicationException("Internal Error: could not get position in log for queueId="
+ + queueId + ", filename=" + filename, e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return 0;
+ }
+ try {
+ return ZKUtil.parseWALPositionFrom(bytes);
+ } catch (DeserializationException de) {
+ LOG.warn("Failed to parse WALPosition for queueId=" + queueId + " and wal=" + filename
+ + " znode content, continuing.");
+ }
+ // if we can not parse the position, start at the beginning of the wal file
+ // again
+ return 0;
+ }
+
+ @Override
+ public boolean isThisOurRegionServer(String regionserver) {
+ return ZKUtil.joinZNode(this.queuesZNode, regionserver).equals(this.myQueuesZnode);
+ }
+
+ @Override
+ public List<String> getUnClaimedQueueIds(String regionserver) {
+ if (isThisOurRegionServer(regionserver)) {
+ return null;
+ }
+ String rsZnodePath = ZKUtil.joinZNode(this.queuesZNode, regionserver);
+ List<String> queues = null;
+ try {
+ queues = ZKUtil.listChildrenNoWatch(this.zookeeper, rsZnodePath);
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to getUnClaimedQueueIds for RS" + regionserver, e);
+ }
+ return queues;
+ }
+
+ @Override
+ public Pair<String, SortedSet<String>> claimQueue(String regionserver, String queueId) {
+ LOG.info("Atomically moving " + regionserver + "/" + queueId + "'s WALs to my queue");
+ return moveQueueUsingMulti(regionserver, queueId);
+ }
+
+ @Override
+ public void removeReplicatorIfQueueIsEmpty(String regionserver) {
+ String rsPath = ZKUtil.joinZNode(this.queuesZNode, regionserver);
+ try {
+ List<String> list = ZKUtil.listChildrenNoWatch(this.zookeeper, rsPath);
+ if (list != null && list.isEmpty()){
+ ZKUtil.deleteNode(this.zookeeper, rsPath);
+ }
+ } catch (KeeperException e) {
+ LOG.warn("Got error while removing replicator", e);
+ }
+ }
+
+ @Override
+ public void removeAllQueues() {
+ try {
+ ZKUtil.deleteNodeRecursively(this.zookeeper, this.myQueuesZnode);
+ } catch (KeeperException e) {
+ // if the znode is already expired, don't bother going further
+ if (e instanceof KeeperException.SessionExpiredException) {
+ return;
+ }
+ this.abortable.abort("Failed to delete replication queues for region server: "
+ + this.myQueuesZnode, e);
+ }
+ }
+
+ @Override
+ public List<String> getLogsInQueue(String queueId) {
+ String znode = ZKUtil.joinZNode(this.myQueuesZnode, queueId);
+ List<String> result = null;
+ try {
+ result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to get list of wals for queueId=" + queueId, e);
+ }
+ return result;
+ }
+
+ @Override
+ public List<String> getAllQueues() {
+ List<String> listOfQueues = null;
+ try {
+ listOfQueues = ZKUtil.listChildrenNoWatch(this.zookeeper, this.myQueuesZnode);
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to get a list of queues for region server: "
+ + this.myQueuesZnode, e);
+ }
+ return listOfQueues == null ? new ArrayList<>() : listOfQueues;
+ }
+
+ /**
+ * It "atomically" copies one peer's wals queue from another dead region server and returns them
+ * all sorted. The new peer id is equal to the old peer id appended with the dead server's znode.
+ * @param znode pertaining to the region server to copy the queues from
+ * @peerId peerId pertaining to the queue need to be copied
+ */
+ private Pair<String, SortedSet<String>> moveQueueUsingMulti(String znode, String peerId) {
+ try {
+ // hbase/replication/rs/deadrs
+ String deadRSZnodePath = ZKUtil.joinZNode(this.queuesZNode, znode);
+ List<ZKUtilOp> listOfOps = new ArrayList<>();
+ ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(peerId);
+
+ String newPeerId = peerId + "-" + znode;
+ String newPeerZnode = ZKUtil.joinZNode(this.myQueuesZnode, newPeerId);
+ // check the logs queue for the old peer cluster
+ String oldClusterZnode = ZKUtil.joinZNode(deadRSZnodePath, peerId);
+ List<String> wals = ZKUtil.listChildrenNoWatch(this.zookeeper, oldClusterZnode);
+
+ if (!peerExists(replicationQueueInfo.getPeerId())) {
+ LOG.warn("Peer " + replicationQueueInfo.getPeerId() +
+ " didn't exist, will move its queue to avoid the failure of multi op");
+ for (String wal : wals) {
+ String oldWalZnode = ZKUtil.joinZNode(oldClusterZnode, wal);
+ listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldWalZnode));
+ }
+ listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode));
+ ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false);
+ return null;
+ }
+
+ SortedSet<String> logQueue = new TreeSet<>();
+ if (wals == null || wals.isEmpty()) {
+ listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode));
+ } else {
+ // create the new cluster znode
+ ZKUtilOp op = ZKUtilOp.createAndFailSilent(newPeerZnode, HConstants.EMPTY_BYTE_ARRAY);
+ listOfOps.add(op);
+ // get the offset of the logs and set it to new znodes
+ for (String wal : wals) {
+ String oldWalZnode = ZKUtil.joinZNode(oldClusterZnode, wal);
+ byte[] logOffset = ZKUtil.getData(this.zookeeper, oldWalZnode);
+ LOG.debug("Creating " + wal + " with data " + Bytes.toString(logOffset));
+ String newLogZnode = ZKUtil.joinZNode(newPeerZnode, wal);
+ listOfOps.add(ZKUtilOp.createAndFailSilent(newLogZnode, logOffset));
+ listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldWalZnode));
+ logQueue.add(wal);
+ }
+ // add delete op for peer
+ listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode));
+
+ if (LOG.isTraceEnabled())
+ LOG.trace(" The multi list size is: " + listOfOps.size());
+ }
+ ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false);
+
+ LOG.info("Atomically moved " + znode + "/" + peerId + "'s WALs to my queue");
+ return new Pair<>(newPeerId, logQueue);
+ } catch (KeeperException e) {
+ // Multi call failed; it looks like some other regionserver took away the logs.
+ LOG.warn("Got exception in copyQueuesFromRSUsingMulti: ", e);
+ } catch (InterruptedException e) {
+ LOG.warn("Got exception in copyQueuesFromRSUsingMulti: ", e);
+ Thread.currentThread().interrupt();
+ }
+ return null;
+ }
+
+ @Override
+ public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs)
+ throws ReplicationException {
+ String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
+ boolean debugEnabled = LOG.isDebugEnabled();
+ if (debugEnabled) {
+ LOG.debug("Adding hfile references " + pairs + " in queue " + peerZnode);
+ }
+
+ int size = pairs.size();
+ List<ZKUtilOp> listOfOps = new ArrayList<>(size);
+
+ for (int i = 0; i < size; i++) {
+ listOfOps.add(ZKUtilOp.createAndFailSilent(
+ ZKUtil.joinZNode(peerZnode, pairs.get(i).getSecond().getName()),
+ HConstants.EMPTY_BYTE_ARRAY));
+ }
+ if (debugEnabled) {
+ LOG.debug(" The multi list size for adding hfile references in zk for node " + peerZnode
+ + " is " + listOfOps.size());
+ }
+ try {
+ ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to create hfile reference znode=" + e.getPath(), e);
+ }
+ }
+
+ @Override
+ public void removeHFileRefs(String peerId, List<String> files) {
+ String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
+ boolean debugEnabled = LOG.isDebugEnabled();
+ if (debugEnabled) {
+ LOG.debug("Removing hfile references " + files + " from queue " + peerZnode);
+ }
+
+ int size = files.size();
+ List<ZKUtilOp> listOfOps = new ArrayList<>(size);
+
+ for (int i = 0; i < size; i++) {
+ listOfOps.add(ZKUtilOp.deleteNodeFailSilent(ZKUtil.joinZNode(peerZnode, files.get(i))));
+ }
+ if (debugEnabled) {
+ LOG.debug(" The multi list size for removing hfile references in zk for node " + peerZnode
+ + " is " + listOfOps.size());
+ }
+ try {
+ ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
+ } catch (KeeperException e) {
+ LOG.error("Failed to remove hfile reference znode=" + e.getPath(), e);
+ }
+ }
+
+ @Override
+ public void addPeerToHFileRefs(String peerId) throws ReplicationException {
+ String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
+ try {
+ if (ZKUtil.checkExists(this.zookeeper, peerZnode) == -1) {
+ LOG.info("Adding peer " + peerId + " to hfile reference queue.");
+ ZKUtil.createWithParents(this.zookeeper, peerZnode);
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException("Failed to add peer " + peerId + " to hfile reference queue.",
+ e);
+ }
+ }
+
+ @Override
+ public void removePeerFromHFileRefs(String peerId) {
+ final String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
+ try {
+ if (ZKUtil.checkExists(this.zookeeper, peerZnode) == -1) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Peer " + peerZnode + " not found in hfile reference queue.");
+ }
+ return;
+ } else {
+ LOG.info("Removing peer " + peerZnode + " from hfile reference queue.");
+ ZKUtil.deleteNodeRecursively(this.zookeeper, peerZnode);
+ }
+ } catch (KeeperException e) {
+ LOG.error("Ignoring the exception to remove peer " + peerId + " from hfile reference queue.",
+ e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
new file mode 100644
index 0000000..47b780d
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
@@ -0,0 +1,155 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+
+/**
+ * This is a base class for maintaining replication state in zookeeper.
+ */
+@InterfaceAudience.Private
+public abstract class ReplicationStateZKBase {
+
+ /**
+ * The name of the znode that contains the replication status of a remote slave (i.e. peer)
+ * cluster.
+ */
+ protected final String peerStateNodeName;
+ /** The name of the base znode that contains all replication state. */
+ protected final String replicationZNode;
+ /** The name of the znode that contains a list of all remote slave (i.e. peer) clusters. */
+ protected final String peersZNode;
+ /** The name of the znode that contains all replication queues */
+ protected final String queuesZNode;
+ /** The name of the znode that contains queues of hfile references to be replicated */
+ protected final String hfileRefsZNode;
+ /** The cluster key of the local cluster */
+ protected final String ourClusterKey;
+ /** The name of the znode that contains tableCFs */
+ protected final String tableCFsNodeName;
+
+ protected final ZooKeeperWatcher zookeeper;
+ protected final Configuration conf;
+ protected final Abortable abortable;
+
+ // Public for testing
+ public static final byte[] ENABLED_ZNODE_BYTES =
+ toByteArray(ReplicationProtos.ReplicationState.State.ENABLED);
+ public static final byte[] DISABLED_ZNODE_BYTES =
+ toByteArray(ReplicationProtos.ReplicationState.State.DISABLED);
+ public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY =
+ "zookeeper.znode.replication.hfile.refs";
+ public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT = "hfile-refs";
+
+ public ReplicationStateZKBase(ZooKeeperWatcher zookeeper, Configuration conf,
+ Abortable abortable) {
+ this.zookeeper = zookeeper;
+ this.conf = conf;
+ this.abortable = abortable;
+
+ String replicationZNodeName = conf.get("zookeeper.znode.replication", "replication");
+ String peersZNodeName = conf.get("zookeeper.znode.replication.peers", "peers");
+ String queuesZNodeName = conf.get("zookeeper.znode.replication.rs", "rs");
+ String hfileRefsZNodeName = conf.get(ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
+ ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
+ this.peerStateNodeName = conf.get("zookeeper.znode.replication.peers.state", "peer-state");
+ this.tableCFsNodeName = conf.get("zookeeper.znode.replication.peers.tableCFs", "tableCFs");
+ this.ourClusterKey = ZKConfig.getZooKeeperClusterKey(this.conf);
+ this.replicationZNode = ZKUtil.joinZNode(this.zookeeper.znodePaths.baseZNode,
+ replicationZNodeName);
+ this.peersZNode = ZKUtil.joinZNode(replicationZNode, peersZNodeName);
+ this.queuesZNode = ZKUtil.joinZNode(replicationZNode, queuesZNodeName);
+ this.hfileRefsZNode = ZKUtil.joinZNode(replicationZNode, hfileRefsZNodeName);
+ }
+
+ public List<String> getListOfReplicators() {
+ List<String> result = null;
+ try {
+ result = ZKUtil.listChildrenNoWatch(this.zookeeper, this.queuesZNode);
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to get list of replicators", e);
+ }
+ return result;
+ }
+
+ /**
+ * @param state
+ * @return Serialized protobuf of <code>state</code> with pb magic prefix prepended suitable for
+ * use as content of a peer-state znode under a peer cluster id as in
+ * /hbase/replication/peers/PEER_ID/peer-state.
+ */
+ protected static byte[] toByteArray(final ReplicationProtos.ReplicationState.State state) {
+ ReplicationProtos.ReplicationState msg =
+ ReplicationProtos.ReplicationState.newBuilder().setState(state).build();
+ // There is no toByteArray on this pb Message?
+ // 32 bytes is default which seems fair enough here.
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ CodedOutputStream cos = CodedOutputStream.newInstance(baos, 16);
+ msg.writeTo(cos);
+ cos.flush();
+ baos.flush();
+ return ProtobufUtil.prependPBMagic(baos.toByteArray());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected boolean peerExists(String id) throws KeeperException {
+ return ZKUtil.checkExists(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id)) >= 0;
+ }
+
+ /**
+ * Determine if a ZK path points to a peer node.
+ * @param path path to be checked
+ * @return true if the path points to a peer node, otherwise false
+ */
+ protected boolean isPeerPath(String path) {
+ return path.split("/").length == peersZNode.split("/").length + 1;
+ }
+
+ @VisibleForTesting
+ protected String getTableCFsNode(String id) {
+ return ZKUtil.joinZNode(this.peersZNode, ZKUtil.joinZNode(id, this.tableCFsNodeName));
+ }
+
+ @VisibleForTesting
+ protected String getPeerStateNode(String id) {
+ return ZKUtil.joinZNode(this.peersZNode, ZKUtil.joinZNode(id, this.peerStateNodeName));
+ }
+ @VisibleForTesting
+ protected String getPeerNode(String id) {
+ return ZKUtil.joinZNode(this.peersZNode, id);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
new file mode 100644
index 0000000..fa1d2d6
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
@@ -0,0 +1,441 @@
+/*
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.shaded.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.RetryCounter;
+import org.apache.hadoop.hbase.util.RetryCounterFactory;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/*
+ * Abstract class that provides an interface to the Replication Table. Which is currently
+ * being used for WAL offset tracking.
+ * The basic schema of this table will store each individual queue as a
+ * seperate row. The row key will be a unique identifier of the creating server's name and the
+ * queueId. Each queue must have the following two columns:
+ * COL_QUEUE_OWNER: tracks which server is currently responsible for tracking the queue
+ * COL_QUEUE_OWNER_HISTORY: a "|" delimited list of the previous server's that have owned this
+ * queue. The most recent previous owner is the leftmost entry.
+ * They will also have columns mapping [WAL filename : offset]
+ * The most flexible method of interacting with the Replication Table is by calling
+ * getOrBlockOnReplicationTable() which will return a new copy of the Replication Table. It is up
+ * to the caller to close the returned table.
+ */
+@InterfaceAudience.Private
+abstract class ReplicationTableBase {
+
+ /** Name of the HBase Table used for tracking replication*/
+ public static final TableName REPLICATION_TABLE_NAME =
+ TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "replication");
+
+ // Column family and column names for Queues in the Replication Table
+ public static final byte[] CF_QUEUE = Bytes.toBytes("q");
+ public static final byte[] COL_QUEUE_OWNER = Bytes.toBytes("o");
+ public static final byte[] COL_QUEUE_OWNER_HISTORY = Bytes.toBytes("h");
+
+ // Column Descriptor for the Replication Table
+ private static final HColumnDescriptor REPLICATION_COL_DESCRIPTOR =
+ new HColumnDescriptor(CF_QUEUE).setMaxVersions(1)
+ .setInMemory(true)
+ .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
+ // TODO: Figure out which bloom filter to use
+ .setBloomFilterType(BloomType.NONE);
+
+ // The value used to delimit the queueId and server name inside of a queue's row key. Currently a
+ // hyphen, because it is guaranteed that queueId (which is a cluster id) cannot contain hyphens.
+ // See HBASE-11394.
+ public static final String ROW_KEY_DELIMITER = "-";
+
+ // The value used to delimit server names in the queue history list
+ public static final String QUEUE_HISTORY_DELIMITER = "|";
+
+ /*
+ * Make sure that HBase table operations for replication have a high number of retries. This is
+ * because the server is aborted if any HBase table operation fails. Each RPC will be attempted
+ * 3600 times before exiting. This provides each operation with 2 hours of retries
+ * before the server is aborted.
+ */
+ private static final int CLIENT_RETRIES = 3600;
+ private static final int RPC_TIMEOUT = 2000;
+ private static final int OPERATION_TIMEOUT = CLIENT_RETRIES * RPC_TIMEOUT;
+
+ // We only need a single thread to initialize the Replication Table
+ private static final int NUM_INITIALIZE_WORKERS = 1;
+
+ protected final Configuration conf;
+ protected final Abortable abortable;
+ private final Connection connection;
+ private final Executor executor;
+ private volatile CountDownLatch replicationTableInitialized;
+
+ public ReplicationTableBase(Configuration conf, Abortable abort) throws IOException {
+ this.conf = new Configuration(conf);
+ this.abortable = abort;
+ decorateConf();
+ this.connection = ConnectionFactory.createConnection(this.conf);
+ this.executor = setUpExecutor();
+ this.replicationTableInitialized = new CountDownLatch(1);
+ createReplicationTableInBackground();
+ }
+
+ /**
+ * Modify the connection's config so that operations run on the Replication Table have longer and
+ * a larger number of retries
+ */
+ private void decorateConf() {
+ this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, CLIENT_RETRIES);
+ }
+
+ /**
+ * Sets up the thread pool executor used to build the Replication Table in the background
+ * @return the configured executor
+ */
+ private Executor setUpExecutor() {
+ ThreadPoolExecutor tempExecutor = new ThreadPoolExecutor(NUM_INITIALIZE_WORKERS,
+ NUM_INITIALIZE_WORKERS, 100, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
+ ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
+ tfb.setNameFormat("ReplicationTableExecutor-%d");
+ tfb.setDaemon(true);
+ tempExecutor.setThreadFactory(tfb.build());
+ return tempExecutor;
+ }
+
+ /**
+ * Get whether the Replication Table has been successfully initialized yet
+ * @return whether the Replication Table is initialized
+ */
+ public boolean getInitializationStatus() {
+ return replicationTableInitialized.getCount() == 0;
+ }
+
+ /**
+ * Increases the RPC and operations timeouts for the Replication Table
+ */
+ private Table setReplicationTableTimeOuts(Table replicationTable) {
+ replicationTable.setRpcTimeout(RPC_TIMEOUT);
+ replicationTable.setOperationTimeout(OPERATION_TIMEOUT);
+ return replicationTable;
+ }
+
+ /**
+ * Build the row key for the given queueId. This will uniquely identify it from all other queues
+ * in the cluster.
+ * @param serverName The owner of the queue
+ * @param queueId String identifier of the queue
+ * @return String representation of the queue's row key
+ */
+ protected String buildQueueRowKey(String serverName, String queueId) {
+ return queueId + ROW_KEY_DELIMITER + serverName;
+ }
+
+ /**
+ * Parse the original queueId from a row key
+ * @param rowKey String representation of a queue's row key
+ * @return the original queueId
+ */
+ protected String getRawQueueIdFromRowKey(String rowKey) {
+ return rowKey.split(ROW_KEY_DELIMITER)[0];
+ }
+
+ /**
+ * Returns a queue's row key given either its raw or reclaimed queueId
+ *
+ * @param queueId queueId of the queue
+ * @return byte representation of the queue's row key
+ */
+ protected byte[] queueIdToRowKey(String serverName, String queueId) {
+ // Cluster id's are guaranteed to have no hyphens, so if the passed in queueId has no hyphen
+ // then this is not a reclaimed queue.
+ if (!queueId.contains(ROW_KEY_DELIMITER)) {
+ return Bytes.toBytes(buildQueueRowKey(serverName, queueId));
+ // If the queueId contained some hyphen it was reclaimed. In this case, the queueId is the
+ // queue's row key
+ } else {
+ return Bytes.toBytes(queueId);
+ }
+ }
+
+ /**
+ * Creates a "|" delimited record of the queue's past region server owners.
+ *
+ * @param originalHistory the queue's original owner history
+ * @param oldServer the name of the server that used to own the queue
+ * @return the queue's new owner history
+ */
+ protected String buildClaimedQueueHistory(String originalHistory, String oldServer) {
+ return oldServer + QUEUE_HISTORY_DELIMITER + originalHistory;
+ }
+
+ /**
+ * Get a list of all region servers that have outstanding replication queues. These servers could
+ * be alive, dead or from a previous run of the cluster.
+ * @return a list of server names
+ */
+ protected List<String> getListOfReplicators() {
+ // scan all of the queues and return a list of all unique OWNER values
+ Set<String> peerServers = new HashSet<>();
+ ResultScanner allQueuesInCluster = null;
+ try (Table replicationTable = getOrBlockOnReplicationTable()){
+ Scan scan = new Scan();
+ scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER);
+ allQueuesInCluster = replicationTable.getScanner(scan);
+ for (Result queue : allQueuesInCluster) {
+ peerServers.add(Bytes.toString(queue.getValue(CF_QUEUE, COL_QUEUE_OWNER)));
+ }
+ } catch (IOException e) {
+ String errMsg = "Failed getting list of replicators";
+ abortable.abort(errMsg, e);
+ } finally {
+ if (allQueuesInCluster != null) {
+ allQueuesInCluster.close();
+ }
+ }
+ return new ArrayList<>(peerServers);
+ }
+
+ protected List<String> getAllQueues(String serverName) {
+ List<String> allQueues = new ArrayList<>();
+ ResultScanner queueScanner = null;
+ try {
+ queueScanner = getQueuesBelongingToServer(serverName);
+ for (Result queue : queueScanner) {
+ String rowKey = Bytes.toString(queue.getRow());
+ // If the queue does not have a Owner History, then we must be its original owner. So we
+ // want to return its queueId in raw form
+ if (Bytes.toString(queue.getValue(CF_QUEUE, COL_QUEUE_OWNER_HISTORY)).length() == 0) {
+ allQueues.add(getRawQueueIdFromRowKey(rowKey));
+ } else {
+ allQueues.add(rowKey);
+ }
+ }
+ return allQueues;
+ } catch (IOException e) {
+ String errMsg = "Failed getting list of all replication queues for serverName=" + serverName;
+ abortable.abort(errMsg, e);
+ return null;
+ } finally {
+ if (queueScanner != null) {
+ queueScanner.close();
+ }
+ }
+ }
+
+ protected List<String> getLogsInQueue(String serverName, String queueId) {
+ String rowKey = queueId;
+ if (!queueId.contains(ROW_KEY_DELIMITER)) {
+ rowKey = buildQueueRowKey(serverName, queueId);
+ }
+ return getLogsInQueue(Bytes.toBytes(rowKey));
+ }
+
+ protected List<String> getLogsInQueue(byte[] rowKey) {
+ String errMsg = "Failed getting logs in queue queueId=" + Bytes.toString(rowKey);
+ try (Table replicationTable = getOrBlockOnReplicationTable()) {
+ Get getQueue = new Get(rowKey);
+ Result queue = replicationTable.get(getQueue);
+ if (queue == null || queue.isEmpty()) {
+ abortable.abort(errMsg, new ReplicationException(errMsg));
+ return null;
+ }
+ return readWALsFromResult(queue);
+ } catch (IOException e) {
+ abortable.abort(errMsg, e);
+ return null;
+ }
+ }
+
+ /**
+ * Read all of the WAL's from a queue into a list
+ *
+ * @param queue HBase query result containing the queue
+ * @return a list of all the WAL filenames
+ */
+ protected List<String> readWALsFromResult(Result queue) {
+ List<String> wals = new ArrayList<>();
+ Map<byte[], byte[]> familyMap = queue.getFamilyMap(CF_QUEUE);
+ for (byte[] cQualifier : familyMap.keySet()) {
+ // Ignore the meta data fields of the queue
+ if (Arrays.equals(cQualifier, COL_QUEUE_OWNER) || Arrays.equals(cQualifier,
+ COL_QUEUE_OWNER_HISTORY)) {
+ continue;
+ }
+ wals.add(Bytes.toString(cQualifier));
+ }
+ return wals;
+ }
+
+ /**
+ * Get the queue id's and meta data (Owner and History) for the queues belonging to the named
+ * server
+ *
+ * @param server name of the server
+ * @return a ResultScanner over the QueueIds belonging to the server
+ * @throws IOException
+ */
+ protected ResultScanner getQueuesBelongingToServer(String server) throws IOException {
+ Scan scan = new Scan();
+ SingleColumnValueFilter filterMyQueues = new SingleColumnValueFilter(CF_QUEUE, COL_QUEUE_OWNER,
+ CompareFilter.CompareOp.EQUAL, Bytes.toBytes(server));
+ scan.setFilter(filterMyQueues);
+ scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER);
+ scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER_HISTORY);
+ try (Table replicationTable = getOrBlockOnReplicationTable()) {
+ ResultScanner results = replicationTable.getScanner(scan);
+ return results;
+ }
+ }
+
+ /**
+ * Attempts to acquire the Replication Table. This operation will block until it is assigned by
+ * the CreateReplicationWorker thread. It is up to the caller of this method to close the
+ * returned Table
+ * @return the Replication Table when it is created
+ * @throws IOException
+ */
+ protected Table getOrBlockOnReplicationTable() throws IOException {
+ // Sleep until the Replication Table becomes available
+ try {
+ replicationTableInitialized.await();
+ } catch (InterruptedException e) {
+ String errMsg = "Unable to acquire the Replication Table due to InterruptedException: " +
+ e.getMessage();
+ throw new InterruptedIOException(errMsg);
+ }
+ return getAndSetUpReplicationTable();
+ }
+
+ /**
+ * Creates a new copy of the Replication Table and sets up the proper Table time outs for it
+ *
+ * @return the Replication Table
+ * @throws IOException
+ */
+ private Table getAndSetUpReplicationTable() throws IOException {
+ Table replicationTable = connection.getTable(REPLICATION_TABLE_NAME);
+ setReplicationTableTimeOuts(replicationTable);
+ return replicationTable;
+ }
+
+ /**
+ * Builds the Replication Table in a background thread. Any method accessing the Replication Table
+ * should do so through getOrBlockOnReplicationTable()
+ *
+ * @return the Replication Table
+ * @throws IOException if the Replication Table takes too long to build
+ */
+ private void createReplicationTableInBackground() throws IOException {
+ executor.execute(new CreateReplicationTableWorker());
+ }
+
+ /**
+ * Attempts to build the Replication Table. Will continue blocking until we have a valid
+ * Table for the Replication Table.
+ */
+ private class CreateReplicationTableWorker implements Runnable {
+
+ private Admin admin;
+
+ @Override
+ public void run() {
+ try {
+ admin = connection.getAdmin();
+ if (!replicationTableExists()) {
+ createReplicationTable();
+ }
+ int maxRetries = conf.getInt("hbase.replication.queues.createtable.retries.number",
+ CLIENT_RETRIES);
+ RetryCounterFactory counterFactory = new RetryCounterFactory(maxRetries, RPC_TIMEOUT);
+ RetryCounter retryCounter = counterFactory.create();
+ while (!replicationTableExists()) {
+ retryCounter.sleepUntilNextRetry();
+ if (!retryCounter.shouldRetry()) {
+ throw new IOException("Unable to acquire the Replication Table");
+ }
+ }
+ replicationTableInitialized.countDown();
+ } catch (IOException | InterruptedException e) {
+ abortable.abort("Failed building Replication Table", e);
+ }
+ }
+
+ /**
+ * Create the replication table with the provided HColumnDescriptor REPLICATION_COL_DESCRIPTOR
+ * in TableBasedReplicationQueuesImpl
+ *
+ * @throws IOException
+ */
+ private void createReplicationTable() throws IOException {
+ HTableDescriptor replicationTableDescriptor = new HTableDescriptor(REPLICATION_TABLE_NAME);
+ replicationTableDescriptor.addFamily(REPLICATION_COL_DESCRIPTOR);
+ try {
+ admin.createTable(replicationTableDescriptor);
+ } catch (TableExistsException e) {
+ // In this case we can just continue as normal
+ }
+ }
+
+ /**
+ * Checks whether the Replication Table exists yet
+ *
+ * @return whether the Replication Table exists
+ * @throws IOException
+ */
+ private boolean replicationTableExists() {
+ try {
+ return admin.tableExists(REPLICATION_TABLE_NAME);
+ } catch (IOException e) {
+ return false;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java
new file mode 100644
index 0000000..51d7473
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java
@@ -0,0 +1,49 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * This is the interface for a Replication Tracker. A replication tracker provides the facility to
+ * subscribe and track events that reflect a change in replication state. These events are used by
+ * the ReplicationSourceManager to coordinate replication tasks such as addition/deletion of queues
+ * and queue failover. These events are defined in the ReplicationListener interface. If a class
+ * would like to listen to replication events it must implement the ReplicationListener interface
+ * and register itself with a Replication Tracker.
+ */
+@InterfaceAudience.Private
+public interface ReplicationTracker {
+
+ /**
+ * Register a replication listener to receive replication events.
+ * @param listener
+ */
+ public void registerListener(ReplicationListener listener);
+
+ public void removeListener(ReplicationListener listener);
+
+ /**
+ * Returns a list of other live region servers in the cluster.
+ * @return List of region servers.
+ */
+ public List<String> getListOfRegionServers();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
new file mode 100644
index 0000000..9865d83
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
@@ -0,0 +1,250 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * This class is a ZooKeeper implementation of the ReplicationTracker interface. This class is
+ * responsible for handling replication events that are defined in the ReplicationListener
+ * interface.
+ */
+@InterfaceAudience.Private
+public class ReplicationTrackerZKImpl extends ReplicationStateZKBase implements ReplicationTracker {
+
+ private static final Log LOG = LogFactory.getLog(ReplicationTrackerZKImpl.class);
+ // All about stopping
+ private final Stoppable stopper;
+ // listeners to be notified
+ private final List<ReplicationListener> listeners = new CopyOnWriteArrayList<>();
+ // List of all the other region servers in this cluster
+ private final ArrayList<String> otherRegionServers = new ArrayList<>();
+ private final ReplicationPeers replicationPeers;
+
+ public ReplicationTrackerZKImpl(ZooKeeperWatcher zookeeper,
+ final ReplicationPeers replicationPeers, Configuration conf, Abortable abortable,
+ Stoppable stopper) {
+ super(zookeeper, conf, abortable);
+ this.replicationPeers = replicationPeers;
+ this.stopper = stopper;
+ this.zookeeper.registerListener(new OtherRegionServerWatcher(this.zookeeper));
+ this.zookeeper.registerListener(new PeersWatcher(this.zookeeper));
+ }
+
+ @Override
+ public void registerListener(ReplicationListener listener) {
+ listeners.add(listener);
+ }
+
+ @Override
+ public void removeListener(ReplicationListener listener) {
+ listeners.remove(listener);
+ }
+
+ /**
+ * Return a snapshot of the current region servers.
+ */
+ @Override
+ public List<String> getListOfRegionServers() {
+ refreshOtherRegionServersList();
+
+ List<String> list = null;
+ synchronized (otherRegionServers) {
+ list = new ArrayList<>(otherRegionServers);
+ }
+ return list;
+ }
+
+ /**
+ * Watcher used to be notified of the other region server's death in the local cluster. It
+ * initiates the process to transfer the queues if it is able to grab the lock.
+ */
+ public class OtherRegionServerWatcher extends ZooKeeperListener {
+
+ /**
+ * Construct a ZooKeeper event listener.
+ */
+ public OtherRegionServerWatcher(ZooKeeperWatcher watcher) {
+ super(watcher);
+ }
+
+ /**
+ * Called when a new node has been created.
+ * @param path full path of the new node
+ */
+ public void nodeCreated(String path) {
+ refreshListIfRightPath(path);
+ }
+
+ /**
+ * Called when a node has been deleted
+ * @param path full path of the deleted node
+ */
+ public void nodeDeleted(String path) {
+ if (stopper.isStopped()) {
+ return;
+ }
+ boolean cont = refreshListIfRightPath(path);
+ if (!cont) {
+ return;
+ }
+ LOG.info(path + " znode expired, triggering replicatorRemoved event");
+ for (ReplicationListener rl : listeners) {
+ rl.regionServerRemoved(getZNodeName(path));
+ }
+ }
+
+ /**
+ * Called when an existing node has a child node added or removed.
+ * @param path full path of the node whose children have changed
+ */
+ public void nodeChildrenChanged(String path) {
+ if (stopper.isStopped()) {
+ return;
+ }
+ refreshListIfRightPath(path);
+ }
+
+ private boolean refreshListIfRightPath(String path) {
+ if (!path.startsWith(this.watcher.znodePaths.rsZNode)) {
+ return false;
+ }
+ return refreshOtherRegionServersList();
+ }
+ }
+
+ /**
+ * Watcher used to follow the creation and deletion of peer clusters.
+ */
+ public class PeersWatcher extends ZooKeeperListener {
+
+ /**
+ * Construct a ZooKeeper event listener.
+ */
+ public PeersWatcher(ZooKeeperWatcher watcher) {
+ super(watcher);
+ }
+
+ /**
+ * Called when a node has been deleted
+ * @param path full path of the deleted node
+ */
+ public void nodeDeleted(String path) {
+ List<String> peers = refreshPeersList(path);
+ if (peers == null) {
+ return;
+ }
+ if (isPeerPath(path)) {
+ String id = getZNodeName(path);
+ LOG.info(path + " znode expired, triggering peerRemoved event");
+ for (ReplicationListener rl : listeners) {
+ rl.peerRemoved(id);
+ }
+ }
+ }
+
+ /**
+ * Called when an existing node has a child node added or removed.
+ * @param path full path of the node whose children have changed
+ */
+ public void nodeChildrenChanged(String path) {
+ List<String> peers = refreshPeersList(path);
+ if (peers == null) {
+ return;
+ }
+ LOG.info(path + " znode expired, triggering peerListChanged event");
+ for (ReplicationListener rl : listeners) {
+ rl.peerListChanged(peers);
+ }
+ }
+ }
+
+ /**
+ * Verify if this event is meant for us, and if so then get the latest peers' list from ZK. Also
+ * reset the watches.
+ * @param path path to check against
+ * @return A list of peers' identifiers if the event concerns this watcher, else null.
+ */
+ private List<String> refreshPeersList(String path) {
+ if (!path.startsWith(getPeersZNode())) {
+ return null;
+ }
+ return this.replicationPeers.getAllPeerIds();
+ }
+
+ private String getPeersZNode() {
+ return this.peersZNode;
+ }
+
+ /**
+ * Extracts the znode name of a peer cluster from a ZK path
+ * @param fullPath Path to extract the id from
+ * @return the id or an empty string if path is invalid
+ */
+ private String getZNodeName(String fullPath) {
+ String[] parts = fullPath.split("/");
+ return parts.length > 0 ? parts[parts.length - 1] : "";
+ }
+
+ /**
+ * Reads the list of region servers from ZK and atomically clears our local view of it and
+ * replaces it with the updated list.
+ * @return true if the local list of the other region servers was updated with the ZK data (even
+ * if it was empty), false if the data was missing in ZK
+ */
+ private boolean refreshOtherRegionServersList() {
+ List<String> newRsList = getRegisteredRegionServers();
+ if (newRsList == null) {
+ return false;
+ } else {
+ synchronized (otherRegionServers) {
+ otherRegionServers.clear();
+ otherRegionServers.addAll(newRsList);
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Get a list of all the other region servers in this cluster and set a watch
+ * @return a list of server nanes
+ */
+ private List<String> getRegisteredRegionServers() {
+ List<String> result = null;
+ try {
+ result = ZKUtil.listChildrenAndWatchThem(this.zookeeper, this.zookeeper.znodePaths.rsZNode);
+ } catch (KeeperException e) {
+ this.abortable.abort("Get list of registered region servers", e);
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesClientImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesClientImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesClientImpl.java
new file mode 100644
index 0000000..3507547
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesClientImpl.java
@@ -0,0 +1,112 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.zookeeper.KeeperException;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Implements the ReplicationQueuesClient interface on top of the Replication Table. It utilizes
+ * the ReplicationTableBase to access the Replication Table.
+ */
+@InterfaceAudience.Private
+public class TableBasedReplicationQueuesClientImpl extends ReplicationTableBase
+ implements ReplicationQueuesClient {
+
+ public TableBasedReplicationQueuesClientImpl(ReplicationQueuesClientArguments args)
+ throws IOException {
+ super(args.getConf(), args.getAbortable());
+ }
+ public TableBasedReplicationQueuesClientImpl(Configuration conf,
+ Abortable abortable) throws IOException {
+ super(conf, abortable);
+ }
+
+ @Override
+ public void init() throws ReplicationException{
+ // no-op
+ }
+
+ @Override
+ public List<String> getListOfReplicators() {
+ return super.getListOfReplicators();
+ }
+
+ @Override
+ public List<String> getLogsInQueue(String serverName, String queueId) {
+ return super.getLogsInQueue(serverName, queueId);
+ }
+
+ @Override
+ public List<String> getAllQueues(String serverName) {
+ return super.getAllQueues(serverName);
+ }
+
+ @Override
+ public Set<String> getAllWALs() {
+ Set<String> allWals = new HashSet<>();
+ ResultScanner allQueues = null;
+ try (Table replicationTable = getOrBlockOnReplicationTable()) {
+ allQueues = replicationTable.getScanner(new Scan());
+ for (Result queue : allQueues) {
+ for (String wal : readWALsFromResult(queue)) {
+ allWals.add(wal);
+ }
+ }
+ } catch (IOException e) {
+ String errMsg = "Failed getting all WAL's in Replication Table";
+ abortable.abort(errMsg, e);
+ } finally {
+ if (allQueues != null) {
+ allQueues.close();
+ }
+ }
+ return allWals;
+ }
+
+ @Override
+ public int getHFileRefsNodeChangeVersion() throws KeeperException {
+ // TODO
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public List<String> getAllPeersFromHFileRefsQueue() throws KeeperException {
+ // TODO
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public List<String> getReplicableHFiles(String peerId) throws KeeperException {
+ // TODO
+ throw new NotImplementedException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java
new file mode 100644
index 0000000..bf55e8c
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java
@@ -0,0 +1,450 @@
+/*
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.RowMutations;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+/**
+ * This class provides an implementation of the ReplicationQueues interface using an HBase table
+ * "Replication Table". It utilizes the ReplicationTableBase to access the Replication Table.
+ */
+@InterfaceAudience.Private
+public class TableBasedReplicationQueuesImpl extends ReplicationTableBase
+ implements ReplicationQueues {
+
+ private static final Log LOG = LogFactory.getLog(TableBasedReplicationQueuesImpl.class);
+
+ // Common byte values used in replication offset tracking
+ private static final byte[] INITIAL_OFFSET_BYTES = Bytes.toBytes(0L);
+ private static final byte[] EMPTY_STRING_BYTES = Bytes.toBytes("");
+
+ private String serverName = null;
+ private byte[] serverNameBytes = null;
+
+ // TODO: Only use this variable temporarily. Eventually we want to use HBase to store all
+ // TODO: replication information
+ private ReplicationStateZKBase replicationState;
+
+ public TableBasedReplicationQueuesImpl(ReplicationQueuesArguments args) throws IOException {
+ this(args.getConf(), args.getAbortable(), args.getZk());
+ }
+
+ public TableBasedReplicationQueuesImpl(Configuration conf, Abortable abort, ZooKeeperWatcher zkw)
+ throws IOException {
+ super(conf, abort);
+ replicationState = new ReplicationStateZKBase(zkw, conf, abort) {};
+ }
+
+ @Override
+ public void init(String serverName) throws ReplicationException {
+ this.serverName = serverName;
+ this.serverNameBytes = Bytes.toBytes(serverName);
+ }
+
+ @Override
+ public List<String> getListOfReplicators() {
+ return super.getListOfReplicators();
+ }
+
+ @Override
+ public void removeQueue(String queueId) {
+ try {
+ byte[] rowKey = queueIdToRowKey(queueId);
+ if (checkQueueExists(queueId)) {
+ Delete deleteQueue = new Delete(rowKey);
+ safeQueueUpdate(deleteQueue);
+ } else {
+ LOG.info("No logs were registered for queue id=" + queueId + " so no rows were removed " +
+ "from the replication table while removing the queue");
+ }
+ } catch (IOException | ReplicationException e) {
+ String errMsg = "Failed removing queue queueId=" + queueId;
+ abortable.abort(errMsg, e);
+ }
+ }
+
+ @Override
+ public void addLog(String queueId, String filename) throws ReplicationException {
+ try (Table replicationTable = getOrBlockOnReplicationTable()) {
+ if (!checkQueueExists(queueId)) {
+ // Each queue will have an Owner, OwnerHistory, and a collection of [WAL:offset] key values
+ Put putNewQueue = new Put(Bytes.toBytes(buildQueueRowKey(queueId)));
+ putNewQueue.addColumn(CF_QUEUE, COL_QUEUE_OWNER, serverNameBytes);
+ putNewQueue.addColumn(CF_QUEUE, COL_QUEUE_OWNER_HISTORY, EMPTY_STRING_BYTES);
+ putNewQueue.addColumn(CF_QUEUE, Bytes.toBytes(filename), INITIAL_OFFSET_BYTES);
+ replicationTable.put(putNewQueue);
+ } else {
+ // Otherwise simply add the new log and offset as a new column
+ Put putNewLog = new Put(queueIdToRowKey(queueId));
+ putNewLog.addColumn(CF_QUEUE, Bytes.toBytes(filename), INITIAL_OFFSET_BYTES);
+ safeQueueUpdate(putNewLog);
+ }
+ } catch (IOException | ReplicationException e) {
+ String errMsg = "Failed adding log queueId=" + queueId + " filename=" + filename;
+ abortable.abort(errMsg, e);
+ }
+ }
+
+ @Override
+ public void removeLog(String queueId, String filename) {
+ try {
+ byte[] rowKey = queueIdToRowKey(queueId);
+ Delete delete = new Delete(rowKey);
+ delete.addColumns(CF_QUEUE, Bytes.toBytes(filename));
+ safeQueueUpdate(delete);
+ } catch (IOException | ReplicationException e) {
+ String errMsg = "Failed removing log queueId=" + queueId + " filename=" + filename;
+ abortable.abort(errMsg, e);
+ }
+ }
+
+ @Override
+ public void setLogPosition(String queueId, String filename, long position) {
+ try (Table replicationTable = getOrBlockOnReplicationTable()) {
+ byte[] rowKey = queueIdToRowKey(queueId);
+ // Check that the log exists. addLog() must have been called before setLogPosition().
+ Get checkLogExists = new Get(rowKey);
+ checkLogExists.addColumn(CF_QUEUE, Bytes.toBytes(filename));
+ if (!replicationTable.exists(checkLogExists)) {
+ String errMsg = "Could not set position of non-existent log from queueId=" + queueId +
+ ", filename=" + filename;
+ abortable.abort(errMsg, new ReplicationException(errMsg));
+ return;
+ }
+ // Update the log offset if it exists
+ Put walAndOffset = new Put(rowKey);
+ walAndOffset.addColumn(CF_QUEUE, Bytes.toBytes(filename), Bytes.toBytes(position));
+ safeQueueUpdate(walAndOffset);
+ } catch (IOException | ReplicationException e) {
+ String errMsg = "Failed writing log position queueId=" + queueId + "filename=" +
+ filename + " position=" + position;
+ abortable.abort(errMsg, e);
+ }
+ }
+
+ @Override
+ public long getLogPosition(String queueId, String filename) throws ReplicationException {
+ try {
+ byte[] rowKey = queueIdToRowKey(queueId);
+ Get getOffset = new Get(rowKey);
+ getOffset.addColumn(CF_QUEUE, Bytes.toBytes(filename));
+ Result result = getResultIfOwner(getOffset);
+ if (result == null || !result.containsColumn(CF_QUEUE, Bytes.toBytes(filename))) {
+ throw new ReplicationException("Could not read empty result while getting log position " +
+ "queueId=" + queueId + ", filename=" + filename);
+ }
+ return Bytes.toLong(result.getValue(CF_QUEUE, Bytes.toBytes(filename)));
+ } catch (IOException e) {
+ throw new ReplicationException("Could not get position in log for queueId=" + queueId +
+ ", filename=" + filename);
+ }
+ }
+
+ @Override
+ public void removeAllQueues() {
+ List<String> myQueueIds = getAllQueues();
+ for (String queueId : myQueueIds) {
+ removeQueue(queueId);
+ }
+ }
+
+ @Override
+ public List<String> getLogsInQueue(String queueId) {
+ String errMsg = "Failed getting logs in queue queueId=" + queueId;
+ byte[] rowKey = queueIdToRowKey(queueId);
+ List<String> logs = new ArrayList<>();
+ try {
+ Get getQueue = new Get(rowKey);
+ Result queue = getResultIfOwner(getQueue);
+ if (queue == null || queue.isEmpty()) {
+ String errMsgLostOwnership = "Failed getting logs for queue queueId=" +
+ Bytes.toString(rowKey) + " because the queue was missing or we lost ownership";
+ abortable.abort(errMsg, new ReplicationException(errMsgLostOwnership));
+ return null;
+ }
+ Map<byte[], byte[]> familyMap = queue.getFamilyMap(CF_QUEUE);
+ for(byte[] cQualifier : familyMap.keySet()) {
+ if (Arrays.equals(cQualifier, COL_QUEUE_OWNER) || Arrays.equals(cQualifier,
+ COL_QUEUE_OWNER_HISTORY)) {
+ continue;
+ }
+ logs.add(Bytes.toString(cQualifier));
+ }
+ } catch (IOException e) {
+ abortable.abort(errMsg, e);
+ return null;
+ }
+ return logs;
+ }
+
+ @Override
+ public List<String> getAllQueues() {
+ return getAllQueues(serverName);
+ }
+
+ @Override public List<String> getUnClaimedQueueIds(String regionserver) {
+ if (isThisOurRegionServer(regionserver)) {
+ return null;
+ }
+ try (ResultScanner queuesToClaim = getQueuesBelongingToServer(regionserver)) {
+ List<String> res = new ArrayList<>();
+ for (Result queue : queuesToClaim) {
+ String rowKey = Bytes.toString(queue.getRow());
+ res.add(rowKey);
+ }
+ return res.isEmpty() ? null : res;
+ } catch (IOException e) {
+ String errMsg = "Failed getUnClaimedQueueIds";
+ abortable.abort(errMsg, e);
+ }
+ return null;
+ }
+
+ @Override public void removeReplicatorIfQueueIsEmpty(String regionserver) {
+ // Do nothing here
+ }
+
+ @Override
+ public Pair<String, SortedSet<String>> claimQueue(String regionserver, String queueId) {
+ if (isThisOurRegionServer(regionserver)) {
+ return null;
+ }
+
+ try (ResultScanner queuesToClaim = getQueuesBelongingToServer(regionserver)){
+ for (Result queue : queuesToClaim) {
+ String rowKey = Bytes.toString(queue.getRow());
+ if (!rowKey.equals(queueId)){
+ continue;
+ }
+ if (attemptToClaimQueue(queue, regionserver)) {
+ ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(rowKey);
+ if (replicationState.peerExists(replicationQueueInfo.getPeerId())) {
+ SortedSet<String> sortedLogs = new TreeSet<>();
+ List<String> logs = getLogsInQueue(queue.getRow());
+ for (String log : logs) {
+ sortedLogs.add(log);
+ }
+ LOG.info(serverName + " has claimed queue " + rowKey + " from " + regionserver);
+ return new Pair<>(rowKey, sortedLogs);
+ } else {
+ // Delete orphaned queues
+ removeQueue(Bytes.toString(queue.getRow()));
+ LOG.info(serverName + " has deleted abandoned queue " + queueId + " from " +
+ regionserver);
+ }
+ }
+ }
+ } catch (IOException | KeeperException e) {
+ String errMsg = "Failed claiming queues for regionserver=" + regionserver;
+ abortable.abort(errMsg, e);
+ }
+ return null;
+ }
+
+ @Override
+ public boolean isThisOurRegionServer(String regionserver) {
+ return this.serverName.equals(regionserver);
+ }
+
+ @Override
+ public void addPeerToHFileRefs(String peerId) throws ReplicationException {
+ // TODO
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void removePeerFromHFileRefs(String peerId) {
+ // TODO
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs)
+ throws ReplicationException {
+ // TODO
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public void removeHFileRefs(String peerId, List<String> files) {
+ // TODO
+ throw new NotImplementedException();
+ }
+
+ private String buildQueueRowKey(String queueId) {
+ return buildQueueRowKey(serverName, queueId);
+ }
+
+ /**
+ * Convenience method that gets the row key of the queue specified by queueId
+ * @param queueId queueId of a queue in this server
+ * @return the row key of the queue in the Replication Table
+ */
+ private byte[] queueIdToRowKey(String queueId) {
+ return queueIdToRowKey(serverName, queueId);
+ }
+
+ /**
+ * See safeQueueUpdate(RowMutations mutate)
+ *
+ * @param put Row mutation to perform on the queue
+ */
+ private void safeQueueUpdate(Put put) throws ReplicationException, IOException {
+ RowMutations mutations = new RowMutations(put.getRow());
+ mutations.add(put);
+ safeQueueUpdate(mutations);
+ }
+
+ /**
+ * See safeQueueUpdate(RowMutations mutate)
+ *
+ * @param delete Row mutation to perform on the queue
+ */
+ private void safeQueueUpdate(Delete delete) throws ReplicationException,
+ IOException{
+ RowMutations mutations = new RowMutations(delete.getRow());
+ mutations.add(delete);
+ safeQueueUpdate(mutations);
+ }
+
+ /**
+ * Attempt to mutate a given queue in the Replication Table with a checkAndPut on the OWNER column
+ * of the queue. Abort the server if this checkAndPut fails: which means we have somehow lost
+ * ownership of the column or an IO Exception has occurred during the transaction.
+ *
+ * @param mutate Mutation to perform on a given queue
+ */
+ private void safeQueueUpdate(RowMutations mutate) throws ReplicationException, IOException{
+ try (Table replicationTable = getOrBlockOnReplicationTable()) {
+ boolean updateSuccess = replicationTable.checkAndMutate(mutate.getRow(),
+ CF_QUEUE, COL_QUEUE_OWNER, CompareFilter.CompareOp.EQUAL, serverNameBytes, mutate);
+ if (!updateSuccess) {
+ throw new ReplicationException("Failed to update Replication Table because we lost queue " +
+ " ownership");
+ }
+ }
+ }
+
+ /**
+ * Check if the queue specified by queueId is stored in HBase
+ *
+ * @param queueId Either raw or reclaimed format of the queueId
+ * @return Whether the queue is stored in HBase
+ * @throws IOException
+ */
+ private boolean checkQueueExists(String queueId) throws IOException {
+ try (Table replicationTable = getOrBlockOnReplicationTable()) {
+ byte[] rowKey = queueIdToRowKey(queueId);
+ return replicationTable.exists(new Get(rowKey));
+ }
+ }
+
+ /**
+ * Attempt to claim the given queue with a checkAndPut on the OWNER column. We check that the
+ * recently killed server is still the OWNER before we claim it.
+ *
+ * @param queue The queue that we are trying to claim
+ * @param originalServer The server that originally owned the queue
+ * @return Whether we successfully claimed the queue
+ * @throws IOException
+ */
+ private boolean attemptToClaimQueue (Result queue, String originalServer) throws IOException{
+ Put putQueueNameAndHistory = new Put(queue.getRow());
+ putQueueNameAndHistory.addColumn(CF_QUEUE, COL_QUEUE_OWNER, Bytes.toBytes(serverName));
+ String newOwnerHistory = buildClaimedQueueHistory(Bytes.toString(queue.getValue(CF_QUEUE,
+ COL_QUEUE_OWNER_HISTORY)), originalServer);
+ putQueueNameAndHistory.addColumn(CF_QUEUE, COL_QUEUE_OWNER_HISTORY,
+ Bytes.toBytes(newOwnerHistory));
+ RowMutations claimAndRenameQueue = new RowMutations(queue.getRow());
+ claimAndRenameQueue.add(putQueueNameAndHistory);
+ // Attempt to claim ownership for this queue by checking if the current OWNER is the original
+ // server. If it is not then another RS has already claimed it. If it is we set ourselves as the
+ // new owner and update the queue's history
+ try (Table replicationTable = getOrBlockOnReplicationTable()) {
+ boolean success = replicationTable.checkAndMutate(queue.getRow(),
+ CF_QUEUE, COL_QUEUE_OWNER, CompareFilter.CompareOp.EQUAL, Bytes.toBytes(originalServer),
+ claimAndRenameQueue);
+ return success;
+ }
+ }
+
+ /**
+ * Attempts to run a Get on some queue. Will only return a non-null result if we currently own
+ * the queue.
+ *
+ * @param get The Get that we want to query
+ * @return The result of the Get if this server is the owner of the queue. Else it returns null.
+ * @throws IOException
+ */
+ private Result getResultIfOwner(Get get) throws IOException {
+ Scan scan = new Scan(get);
+ // Check if the Get currently contains all columns or only specific columns
+ if (scan.getFamilyMap().size() > 0) {
+ // Add the OWNER column if the scan is already only over specific columns
+ scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER);
+ }
+ scan.setMaxResultSize(1);
+ SingleColumnValueFilter checkOwner = new SingleColumnValueFilter(CF_QUEUE, COL_QUEUE_OWNER,
+ CompareFilter.CompareOp.EQUAL, serverNameBytes);
+ scan.setFilter(checkOwner);
+ ResultScanner scanner = null;
+ try (Table replicationTable = getOrBlockOnReplicationTable()) {
+ scanner = replicationTable.getScanner(scan);
+ Result result = scanner.next();
+ return (result == null || result.isEmpty()) ? null : result;
+ } finally {
+ if (scanner != null) {
+ scanner.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-server/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index 890ba2b..e1ef882 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -378,6 +378,10 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-replication</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
<artifactId>hbase-prefix-tree</artifactId>
<scope>runtime</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
index 0e619ea..ae4e7cc 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/BaseReplicationEndpoint.java
@@ -27,8 +27,6 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
import org.apache.hadoop.hbase.shaded.com.google.common.util.concurrent.AbstractService;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
-
/**
* A Base implementation for {@link ReplicationEndpoint}s. Users should consider extending this
* class rather than implementing {@link ReplicationEndpoint} directly for better backwards
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 31ef241..4d10324 100755
--- a/pom.xml
+++ b/pom.xml
@@ -63,6 +63,7 @@
<modules>
<module>hbase-build-support</module>
<module>hbase-build-configuration</module>
+ <module>hbase-replication</module>
<module>hbase-resource-bundle</module>
<module>hbase-server</module>
<module>hbase-thrift</module>
@@ -1604,6 +1605,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <artifactId>hbase-replication</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<artifactId>hbase-server</artifactId>
<groupId>org.apache.hbase</groupId>
<version>${project.version}</version>
[4/4] hbase git commit: HBASE-17442 Move most of the replication
related classes from hbase-client to new hbase-replication package. (Guanghao
Zhang).
Posted by ap...@apache.org.
HBASE-17442 Move most of the replication related classes from hbase-client to new hbase-replication package. (Guanghao Zhang).
Change-Id: Ie0e24cc617ab4bf56de8b1747062d1b78a5d4669
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/26e6c2ce
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/26e6c2ce
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/26e6c2ce
Branch: refs/heads/master
Commit: 26e6c2ceb4db80e561ec321dba52673e7f285d1b
Parents: ae052e4
Author: Apekshit Sharma <ap...@apache.org>
Authored: Thu Aug 17 20:59:35 2017 -0700
Committer: Apekshit Sharma <ap...@apache.org>
Committed: Wed Aug 23 14:41:58 2017 -0700
----------------------------------------------------------------------
.../client/replication/ReplicationAdmin.java | 102 +---
.../hbase/replication/ReplicationFactory.java | 66 ---
.../hbase/replication/ReplicationListener.java | 51 --
.../hbase/replication/ReplicationPeer.java | 89 ---
.../ReplicationPeerConfigListener.java | 33 --
.../replication/ReplicationPeerZKImpl.java | 318 -----------
.../hbase/replication/ReplicationPeers.java | 177 ------
.../replication/ReplicationPeersZKImpl.java | 546 -------------------
.../hbase/replication/ReplicationQueueInfo.java | 130 -----
.../hbase/replication/ReplicationQueues.java | 160 ------
.../replication/ReplicationQueuesArguments.java | 70 ---
.../replication/ReplicationQueuesClient.java | 93 ----
.../ReplicationQueuesClientArguments.java | 40 --
.../ReplicationQueuesClientZKImpl.java | 175 ------
.../replication/ReplicationQueuesZKImpl.java | 407 --------------
.../replication/ReplicationStateZKBase.java | 155 ------
.../hbase/replication/ReplicationTableBase.java | 441 ---------------
.../hbase/replication/ReplicationTracker.java | 49 --
.../replication/ReplicationTrackerZKImpl.java | 250 ---------
.../TableBasedReplicationQueuesClientImpl.java | 112 ----
.../TableBasedReplicationQueuesImpl.java | 450 ---------------
.../apache/hadoop/hbase/zookeeper/ZKUtil.java | 14 +-
.../hadoop/hbase/zookeeper/ZNodePaths.java | 22 +-
hbase-replication/pom.xml | 264 +++++++++
.../hbase/replication/ReplicationFactory.java | 66 +++
.../hbase/replication/ReplicationListener.java | 51 ++
.../hbase/replication/ReplicationPeer.java | 89 +++
.../ReplicationPeerConfigListener.java | 33 ++
.../replication/ReplicationPeerZKImpl.java | 318 +++++++++++
.../hbase/replication/ReplicationPeers.java | 177 ++++++
.../replication/ReplicationPeersZKImpl.java | 546 +++++++++++++++++++
.../hbase/replication/ReplicationQueueInfo.java | 130 +++++
.../hbase/replication/ReplicationQueues.java | 160 ++++++
.../replication/ReplicationQueuesArguments.java | 70 +++
.../replication/ReplicationQueuesClient.java | 93 ++++
.../ReplicationQueuesClientArguments.java | 40 ++
.../ReplicationQueuesClientZKImpl.java | 175 ++++++
.../replication/ReplicationQueuesZKImpl.java | 407 ++++++++++++++
.../replication/ReplicationStateZKBase.java | 155 ++++++
.../hbase/replication/ReplicationTableBase.java | 441 +++++++++++++++
.../hbase/replication/ReplicationTracker.java | 49 ++
.../replication/ReplicationTrackerZKImpl.java | 250 +++++++++
.../TableBasedReplicationQueuesClientImpl.java | 112 ++++
.../TableBasedReplicationQueuesImpl.java | 450 +++++++++++++++
hbase-server/pom.xml | 4 +
.../replication/BaseReplicationEndpoint.java | 2 -
pom.xml | 6 +
47 files changed, 4113 insertions(+), 3925 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
index 752d18c..615a79d 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
@@ -26,37 +26,22 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.ReplicationPeerNotFoundException;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.replication.ReplicationException;
-import org.apache.hadoop.hbase.replication.ReplicationFactory;
-import org.apache.hadoop.hbase.replication.ReplicationPeer;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
-import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
-import org.apache.hadoop.hbase.replication.ReplicationPeers;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
-import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
@@ -101,16 +86,6 @@ public class ReplicationAdmin implements Closeable {
Integer.toString(HConstants.REPLICATION_SCOPE_SERIAL);
private final Connection connection;
- // TODO: replication should be managed by master. All the classes except ReplicationAdmin should
- // be moved to hbase-server. Resolve it in HBASE-11392.
- private final ReplicationQueuesClient replicationQueuesClient;
- private final ReplicationPeers replicationPeers;
- /**
- * A watcher used by replicationPeers and replicationQueuesClient. Keep reference so can dispose
- * on {@link #close()}.
- */
- private final ZooKeeperWatcher zkw;
-
private Admin admin;
/**
@@ -122,49 +97,6 @@ public class ReplicationAdmin implements Closeable {
public ReplicationAdmin(Configuration conf) throws IOException {
this.connection = ConnectionFactory.createConnection(conf);
admin = connection.getAdmin();
- try {
- zkw = createZooKeeperWatcher();
- try {
- this.replicationQueuesClient =
- ReplicationFactory.getReplicationQueuesClient(new ReplicationQueuesClientArguments(conf,
- this.connection, zkw));
- this.replicationQueuesClient.init();
- this.replicationPeers = ReplicationFactory.getReplicationPeers(zkw, conf,
- this.replicationQueuesClient, this.connection);
- this.replicationPeers.init();
- } catch (Exception exception) {
- if (zkw != null) {
- zkw.close();
- }
- throw exception;
- }
- } catch (Exception exception) {
- connection.close();
- if (exception instanceof IOException) {
- throw (IOException) exception;
- } else if (exception instanceof RuntimeException) {
- throw (RuntimeException) exception;
- } else {
- throw new IOException("Error initializing the replication admin client.", exception);
- }
- }
- }
-
- private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
- // This Abortable doesn't 'abort'... it just logs.
- return new ZooKeeperWatcher(connection.getConfiguration(), "ReplicationAdmin", new Abortable() {
- @Override
- public void abort(String why, Throwable e) {
- LOG.error(why, e);
- // We used to call system.exit here but this script can be embedded by other programs that
- // want to do replication stuff... so inappropriate calling System.exit. Just log for now.
- }
-
- @Override
- public boolean isAborted() {
- return false;
- }
- });
}
/**
@@ -452,9 +384,6 @@ public class ReplicationAdmin implements Closeable {
@Override
public void close() throws IOException {
- if (this.zkw != null) {
- this.zkw.close();
- }
if (this.connection != null) {
this.connection.close();
}
@@ -518,40 +447,13 @@ public class ReplicationAdmin implements Closeable {
admin.disableTableReplication(tableName);
}
- @VisibleForTesting
- @Deprecated
- public void peerAdded(String id) throws ReplicationException {
- this.replicationPeers.peerConnected(id);
- }
-
/**
* @deprecated use {@link org.apache.hadoop.hbase.client.Admin#listReplicationPeers()} instead
*/
@VisibleForTesting
@Deprecated
- List<ReplicationPeer> listReplicationPeers() throws IOException {
- Map<String, ReplicationPeerConfig> peers = listPeerConfigs();
- if (peers == null || peers.size() <= 0) {
- return null;
- }
- List<ReplicationPeer> listOfPeers = new ArrayList<>(peers.size());
- for (Entry<String, ReplicationPeerConfig> peerEntry : peers.entrySet()) {
- String peerId = peerEntry.getKey();
- try {
- Pair<ReplicationPeerConfig, Configuration> pair = this.replicationPeers.getPeerConf(peerId);
- Configuration peerConf = pair.getSecond();
- ReplicationPeer peer = new ReplicationPeerZKImpl(zkw, pair.getSecond(),
- peerId, pair.getFirst(), this.connection);
- listOfPeers.add(peer);
- } catch (ReplicationException e) {
- LOG.warn("Failed to get valid replication peers. "
- + "Error connecting to peer cluster with peerId=" + peerId + ". Error message="
- + e.getMessage());
- LOG.debug("Failure details to get valid replication peers.", e);
- continue;
- }
- }
- return listOfPeers;
+ List<ReplicationPeerDescription> listReplicationPeers() throws IOException {
+ return admin.listReplicationPeers();
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
deleted file mode 100644
index 8506cbb..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import org.apache.commons.lang.reflect.ConstructorUtils;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.Stoppable;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-
-/**
- * A factory class for instantiating replication objects that deal with replication state.
- */
-@InterfaceAudience.Private
-public class ReplicationFactory {
-
- public static final Class defaultReplicationQueueClass = ReplicationQueuesZKImpl.class;
-
- public static ReplicationQueues getReplicationQueues(ReplicationQueuesArguments args)
- throws Exception {
- Class<?> classToBuild = args.getConf().getClass("hbase.region.replica." +
- "replication.replicationQueues.class", defaultReplicationQueueClass);
- return (ReplicationQueues) ConstructorUtils.invokeConstructor(classToBuild, args);
- }
-
- public static ReplicationQueuesClient getReplicationQueuesClient(
- ReplicationQueuesClientArguments args) throws Exception {
- Class<?> classToBuild = args.getConf().getClass(
- "hbase.region.replica.replication.replicationQueuesClient.class",
- ReplicationQueuesClientZKImpl.class);
- return (ReplicationQueuesClient) ConstructorUtils.invokeConstructor(classToBuild, args);
- }
-
- public static ReplicationPeers getReplicationPeers(final ZooKeeperWatcher zk, Configuration conf,
- Abortable abortable) {
- return getReplicationPeers(zk, conf, null, abortable);
- }
-
- public static ReplicationPeers getReplicationPeers(final ZooKeeperWatcher zk, Configuration conf,
- final ReplicationQueuesClient queuesClient, Abortable abortable) {
- return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable);
- }
-
- public static ReplicationTracker getReplicationTracker(ZooKeeperWatcher zookeeper,
- final ReplicationPeers replicationPeers, Configuration conf, Abortable abortable,
- Stoppable stopper) {
- return new ReplicationTrackerZKImpl(zookeeper, replicationPeers, conf, abortable, stopper);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java
deleted file mode 100644
index dfb5fdc..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.List;
-
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-/**
- * The replication listener interface can be implemented if a class needs to subscribe to events
- * generated by the ReplicationTracker. These events include things like addition/deletion of peer
- * clusters or failure of a local region server. To receive events, the class also needs to register
- * itself with a Replication Tracker.
- */
-@InterfaceAudience.Private
-public interface ReplicationListener {
-
- /**
- * A region server has been removed from the local cluster
- * @param regionServer the removed region server
- */
- public void regionServerRemoved(String regionServer);
-
- /**
- * A peer cluster has been removed (i.e. unregistered) from replication.
- * @param peerId The peer id of the cluster that has been removed
- */
- public void peerRemoved(String peerId);
-
- /**
- * The list of registered peer clusters has changed.
- * @param peerIds A list of all currently registered peer clusters
- */
- public void peerListChanged(List<String> peerIds);
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
deleted file mode 100644
index 4f18048..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseInterfaceAudience;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-
-/**
- * ReplicationPeer manages enabled / disabled state for the peer.
- */
-@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION)
-public interface ReplicationPeer {
-
- /**
- * State of the peer, whether it is enabled or not
- */
- @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION)
- enum PeerState {
- ENABLED,
- DISABLED
- }
-
- /**
- * Get the identifier of this peer
- * @return string representation of the id
- */
- String getId();
-
- /**
- * Get the peer config object
- * @return the ReplicationPeerConfig for this peer
- */
- public ReplicationPeerConfig getPeerConfig();
-
- /**
- * Returns the state of the peer
- * @return the enabled state
- */
- PeerState getPeerState();
-
- /**
- * Get the configuration object required to communicate with this peer
- * @return configuration object
- */
- public Configuration getConfiguration();
-
- /**
- * Get replicable (table, cf-list) map of this peer
- * @return the replicable (table, cf-list) map
- */
- public Map<TableName, List<String>> getTableCFs();
-
- /**
- * Get replicable namespace set of this peer
- * @return the replicable namespaces set
- */
- public Set<String> getNamespaces();
-
- /**
- * Get the per node bandwidth upper limit for this peer
- * @return the bandwidth upper limit
- */
- public long getPeerBandwidth();
-
- void trackPeerConfigChanges(ReplicationPeerConfigListener listener);
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigListener.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigListener.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigListener.java
deleted file mode 100644
index 4e04186..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigListener.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.replication;
-
-import org.apache.hadoop.hbase.HBaseInterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION)
-public interface ReplicationPeerConfigListener {
- /** Callback method for when users update the ReplicationPeerConfig for this peer
- *
- * @param rpc The updated ReplicationPeerConfig
- */
- void peerConfigUpdated(ReplicationPeerConfig rpc);
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
deleted file mode 100644
index 3973be9..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
+++ /dev/null
@@ -1,318 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.replication.ReplicationSerDeHelper;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.KeeperException.NodeExistsException;
-
-@InterfaceAudience.Private
-public class ReplicationPeerZKImpl extends ReplicationStateZKBase
- implements ReplicationPeer, Abortable, Closeable {
- private static final Log LOG = LogFactory.getLog(ReplicationPeerZKImpl.class);
-
- private ReplicationPeerConfig peerConfig;
- private final String id;
- private volatile PeerState peerState;
- private volatile Map<TableName, List<String>> tableCFs = new HashMap<>();
- private final Configuration conf;
- private PeerStateTracker peerStateTracker;
- private PeerConfigTracker peerConfigTracker;
-
-
- /**
- * Constructor that takes all the objects required to communicate with the specified peer, except
- * for the region server addresses.
- * @param conf configuration object to this peer
- * @param id string representation of this peer's identifier
- * @param peerConfig configuration for the replication peer
- */
- public ReplicationPeerZKImpl(ZooKeeperWatcher zkWatcher, Configuration conf,
- String id, ReplicationPeerConfig peerConfig,
- Abortable abortable)
- throws ReplicationException {
- super(zkWatcher, conf, abortable);
- this.conf = conf;
- this.peerConfig = peerConfig;
- this.id = id;
- }
-
- /**
- * start a state tracker to check whether this peer is enabled or not
- *
- * @param peerStateNode path to zk node which stores peer state
- * @throws KeeperException
- */
- public void startStateTracker(String peerStateNode)
- throws KeeperException {
- ensurePeerEnabled(peerStateNode);
- this.peerStateTracker = new PeerStateTracker(peerStateNode, zookeeper, this);
- this.peerStateTracker.start();
- try {
- this.readPeerStateZnode();
- } catch (DeserializationException e) {
- throw ZKUtil.convert(e);
- }
- }
-
- private void readPeerStateZnode() throws DeserializationException {
- this.peerState =
- isStateEnabled(this.peerStateTracker.getData(false))
- ? PeerState.ENABLED
- : PeerState.DISABLED;
- }
-
- /**
- * start a table-cfs tracker to listen the (table, cf-list) map change
- * @param peerConfigNode path to zk node which stores table-cfs
- * @throws KeeperException
- */
- public void startPeerConfigTracker(String peerConfigNode)
- throws KeeperException {
- this.peerConfigTracker = new PeerConfigTracker(peerConfigNode, zookeeper,
- this);
- this.peerConfigTracker.start();
- this.readPeerConfig();
- }
-
- private ReplicationPeerConfig readPeerConfig() {
- try {
- byte[] data = peerConfigTracker.getData(false);
- if (data != null) {
- this.peerConfig = ReplicationSerDeHelper.parsePeerFrom(data);
- }
- } catch (DeserializationException e) {
- LOG.error("", e);
- }
- return this.peerConfig;
- }
-
- @Override
- public PeerState getPeerState() {
- return peerState;
- }
-
- /**
- * Get the identifier of this peer
- * @return string representation of the id (short)
- */
- @Override
- public String getId() {
- return id;
- }
-
- /**
- * Get the peer config object
- * @return the ReplicationPeerConfig for this peer
- */
- @Override
- public ReplicationPeerConfig getPeerConfig() {
- return peerConfig;
- }
-
- /**
- * Get the configuration object required to communicate with this peer
- * @return configuration object
- */
- @Override
- public Configuration getConfiguration() {
- return conf;
- }
-
- /**
- * Get replicable (table, cf-list) map of this peer
- * @return the replicable (table, cf-list) map
- */
- @Override
- public Map<TableName, List<String>> getTableCFs() {
- this.tableCFs = peerConfig.getTableCFsMap();
- return this.tableCFs;
- }
-
- /**
- * Get replicable namespace set of this peer
- * @return the replicable namespaces set
- */
- @Override
- public Set<String> getNamespaces() {
- return this.peerConfig.getNamespaces();
- }
-
- @Override
- public long getPeerBandwidth() {
- return this.peerConfig.getBandwidth();
- }
-
- @Override
- public void trackPeerConfigChanges(ReplicationPeerConfigListener listener) {
- if (this.peerConfigTracker != null){
- this.peerConfigTracker.setListener(listener);
- }
- }
-
- @Override
- public void abort(String why, Throwable e) {
- LOG.fatal("The ReplicationPeer corresponding to peer " + peerConfig
- + " was aborted for the following reason(s):" + why, e);
- }
-
- @Override
- public boolean isAborted() {
- // Currently the replication peer is never "Aborted", we just log when the
- // abort method is called.
- return false;
- }
-
- @Override
- public void close() throws IOException {
- // TODO: stop zkw?
- }
-
- /**
- * Parse the raw data from ZK to get a peer's state
- * @param bytes raw ZK data
- * @return True if the passed in <code>bytes</code> are those of a pb serialized ENABLED state.
- * @throws DeserializationException
- */
- public static boolean isStateEnabled(final byte[] bytes) throws DeserializationException {
- ReplicationProtos.ReplicationState.State state = parseStateFrom(bytes);
- return ReplicationProtos.ReplicationState.State.ENABLED == state;
- }
-
- /**
- * @param bytes Content of a state znode.
- * @return State parsed from the passed bytes.
- * @throws DeserializationException
- */
- private static ReplicationProtos.ReplicationState.State parseStateFrom(final byte[] bytes)
- throws DeserializationException {
- ProtobufUtil.expectPBMagicPrefix(bytes);
- int pblen = ProtobufUtil.lengthOfPBMagic();
- ReplicationProtos.ReplicationState.Builder builder =
- ReplicationProtos.ReplicationState.newBuilder();
- ReplicationProtos.ReplicationState state;
- try {
- ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
- state = builder.build();
- return state.getState();
- } catch (IOException e) {
- throw new DeserializationException(e);
- }
- }
-
- /**
- * Utility method to ensure an ENABLED znode is in place; if not present, we create it.
- * @param path Path to znode to check
- * @return True if we created the znode.
- * @throws NodeExistsException
- * @throws KeeperException
- */
- private boolean ensurePeerEnabled(final String path)
- throws NodeExistsException, KeeperException {
- if (ZKUtil.checkExists(zookeeper, path) == -1) {
- // There is a race b/w PeerWatcher and ReplicationZookeeper#add method to create the
- // peer-state znode. This happens while adding a peer.
- // The peer state data is set as "ENABLED" by default.
- ZKUtil.createNodeIfNotExistsAndWatch(zookeeper, path,
- ReplicationStateZKBase.ENABLED_ZNODE_BYTES);
- return true;
- }
- return false;
- }
-
- /**
- * Tracker for state of this peer
- */
- public class PeerStateTracker extends ZooKeeperNodeTracker {
-
- public PeerStateTracker(String peerStateZNode, ZooKeeperWatcher watcher,
- Abortable abortable) {
- super(watcher, peerStateZNode, abortable);
- }
-
- @Override
- public synchronized void nodeDataChanged(String path) {
- if (path.equals(node)) {
- super.nodeDataChanged(path);
- try {
- readPeerStateZnode();
- } catch (DeserializationException e) {
- LOG.warn("Failed deserializing the content of " + path, e);
- }
- }
- }
- }
-
- /**
- * Tracker for PeerConfigNode of this peer
- */
- public class PeerConfigTracker extends ZooKeeperNodeTracker {
-
- ReplicationPeerConfigListener listener;
-
- public PeerConfigTracker(String peerConfigNode, ZooKeeperWatcher watcher,
- Abortable abortable) {
- super(watcher, peerConfigNode, abortable);
- }
-
- public synchronized void setListener(ReplicationPeerConfigListener listener){
- this.listener = listener;
- }
-
- @Override
- public synchronized void nodeCreated(String path) {
- if (path.equals(node)) {
- super.nodeCreated(path);
- ReplicationPeerConfig config = readPeerConfig();
- if (listener != null){
- listener.peerConfigUpdated(config);
- }
- }
- }
-
- @Override
- public synchronized void nodeDataChanged(String path) {
- //superclass calls nodeCreated
- if (path.equals(node)) {
- super.nodeDataChanged(path);
- }
-
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
deleted file mode 100644
index 2a7963a..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.util.Pair;
-
-/**
- * This provides an interface for maintaining a set of peer clusters. These peers are remote slave
- * clusters that data is replicated to. A peer cluster can be in three different states:
- *
- * 1. Not-Registered - There is no notion of the peer cluster.
- * 2. Registered - The peer has an id and is being tracked but there is no connection.
- * 3. Connected - There is an active connection to the remote peer.
- *
- * In the registered or connected state, a peer cluster can either be enabled or disabled.
- */
-@InterfaceAudience.Private
-public interface ReplicationPeers {
-
- /**
- * Initialize the ReplicationPeers interface.
- */
- void init() throws ReplicationException;
-
- /**
- * Add a new remote slave cluster for replication.
- * @param peerId a short that identifies the cluster
- * @param peerConfig configuration for the replication slave cluster
- */
- void registerPeer(String peerId, ReplicationPeerConfig peerConfig)
- throws ReplicationException;
-
- /**
- * Removes a remote slave cluster and stops the replication to it.
- * @param peerId a short that identifies the cluster
- */
- void unregisterPeer(String peerId) throws ReplicationException;
-
- /**
- * Method called after a peer has been connected. It will create a ReplicationPeer to track the
- * newly connected cluster.
- * @param peerId a short that identifies the cluster
- * @return whether a ReplicationPeer was successfully created
- * @throws ReplicationException
- */
- boolean peerConnected(String peerId) throws ReplicationException;
-
- /**
- * Method called after a peer has been disconnected. It will remove the ReplicationPeer that
- * tracked the disconnected cluster.
- * @param peerId a short that identifies the cluster
- */
- void peerDisconnected(String peerId);
-
- /**
- * Restart the replication to the specified remote slave cluster.
- * @param peerId a short that identifies the cluster
- */
- void enablePeer(String peerId) throws ReplicationException;
-
- /**
- * Stop the replication to the specified remote slave cluster.
- * @param peerId a short that identifies the cluster
- */
- void disablePeer(String peerId) throws ReplicationException;
-
- /**
- * Get the table and column-family list string of the peer from the underlying storage.
- * @param peerId a short that identifies the cluster
- */
- public Map<TableName, List<String>> getPeerTableCFsConfig(String peerId)
- throws ReplicationException;
-
- /**
- * Set the table and column-family list string of the peer to the underlying storage.
- * @param peerId a short that identifies the cluster
- * @param tableCFs the table and column-family list which will be replicated for this peer
- */
- public void setPeerTableCFsConfig(String peerId,
- Map<TableName, ? extends Collection<String>> tableCFs)
- throws ReplicationException;
-
- /**
- * Returns the ReplicationPeer for the specified connected peer. This ReplicationPeer will
- * continue to track changes to the Peer's state and config. This method returns null if no
- * peer has been connected with the given peerId.
- * @param peerId id for the peer
- * @return ReplicationPeer object
- */
- ReplicationPeer getConnectedPeer(String peerId);
-
- /**
- * Returns the set of peerIds of the clusters that have been connected and have an underlying
- * ReplicationPeer.
- * @return a Set of Strings for peerIds
- */
- public Set<String> getConnectedPeerIds();
-
- /**
- * Get the replication status for the specified connected remote slave cluster.
- * The value might be read from cache, so it is recommended to
- * use {@link #getStatusOfPeerFromBackingStore(String)}
- * if reading the state after enabling or disabling it.
- * @param peerId a short that identifies the cluster
- * @return true if replication is enabled, false otherwise.
- */
- boolean getStatusOfPeer(String peerId);
-
- /**
- * Get the replication status for the specified remote slave cluster, which doesn't
- * have to be connected. The state is read directly from the backing store.
- * @param peerId a short that identifies the cluster
- * @return true if replication is enabled, false otherwise.
- * @throws ReplicationException thrown if there's an error contacting the store
- */
- boolean getStatusOfPeerFromBackingStore(String peerId) throws ReplicationException;
-
- /**
- * List the cluster replication configs of all remote slave clusters (whether they are
- * enabled/disabled or connected/disconnected).
- * @return A map of peer ids to peer cluster keys
- */
- Map<String, ReplicationPeerConfig> getAllPeerConfigs();
-
- /**
- * List the peer ids of all remote slave clusters (whether they are enabled/disabled or
- * connected/disconnected).
- * @return A list of peer ids
- */
- List<String> getAllPeerIds();
-
- /**
- * Returns the configured ReplicationPeerConfig for this peerId
- * @param peerId a short name that identifies the cluster
- * @return ReplicationPeerConfig for the peer
- */
- ReplicationPeerConfig getReplicationPeerConfig(String peerId) throws ReplicationException;
-
- /**
- * Returns the configuration needed to talk to the remote slave cluster.
- * @param peerId a short that identifies the cluster
- * @return the configuration for the peer cluster, null if it was unable to get the configuration
- */
- Pair<ReplicationPeerConfig, Configuration> getPeerConf(String peerId) throws ReplicationException;
-
- /**
- * Update the peerConfig for the a given peer cluster
- * @param id a short that identifies the cluster
- * @param peerConfig new config for the peer cluster
- * @throws ReplicationException
- */
- void updatePeerConfig(String id, ReplicationPeerConfig peerConfig) throws ReplicationException;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
deleted file mode 100644
index 751e454..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
+++ /dev/null
@@ -1,546 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.CompoundConfiguration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.replication.ReplicationSerDeHelper;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
-import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZKConfig;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * This class provides an implementation of the ReplicationPeers interface using ZooKeeper. The
- * peers znode contains a list of all peer replication clusters and the current replication state of
- * those clusters. It has one child peer znode for each peer cluster. The peer znode is named with
- * the cluster id provided by the user in the HBase shell. The value of the peer znode contains the
- * peers cluster key provided by the user in the HBase Shell. The cluster key contains a list of
- * zookeeper quorum peers, the client port for the zookeeper quorum, and the base znode for HBase.
- * For example:
- *
- * /hbase/replication/peers/1 [Value: zk1.host.com,zk2.host.com,zk3.host.com:2181:/hbase]
- * /hbase/replication/peers/2 [Value: zk5.host.com,zk6.host.com,zk7.host.com:2181:/hbase]
- *
- * Each of these peer znodes has a child znode that indicates whether or not replication is enabled
- * on that peer cluster. These peer-state znodes do not have child znodes and simply contain a
- * boolean value (i.e. ENABLED or DISABLED). This value is read/maintained by the
- * ReplicationPeer.PeerStateTracker class. For example:
- *
- * /hbase/replication/peers/1/peer-state [Value: ENABLED]
- *
- * Each of these peer znodes has a child znode that indicates which data will be replicated
- * to the peer cluster. These peer-tableCFs znodes do not have child znodes and only have a
- * table/cf list config. This value is read/maintained by the ReplicationPeer.TableCFsTracker
- * class. For example:
- *
- * /hbase/replication/peers/1/tableCFs [Value: "table1; table2:cf1,cf3; table3:cfx,cfy"]
- */
-@InterfaceAudience.Private
-public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements ReplicationPeers {
-
- // Map of peer clusters keyed by their id
- private Map<String, ReplicationPeerZKImpl> peerClusters;
- private final ReplicationQueuesClient queuesClient;
- private Abortable abortable;
-
- private static final Log LOG = LogFactory.getLog(ReplicationPeersZKImpl.class);
-
- public ReplicationPeersZKImpl(final ZooKeeperWatcher zk, final Configuration conf,
- final ReplicationQueuesClient queuesClient, Abortable abortable) {
- super(zk, conf, abortable);
- this.abortable = abortable;
- this.peerClusters = new ConcurrentHashMap<>();
- this.queuesClient = queuesClient;
- }
-
- @Override
- public void init() throws ReplicationException {
- try {
- if (ZKUtil.checkExists(this.zookeeper, this.peersZNode) < 0) {
- ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Could not initialize replication peers", e);
- }
- addExistingPeers();
- }
-
- @Override
- public void registerPeer(String id, ReplicationPeerConfig peerConfig)
- throws ReplicationException {
- try {
- if (peerExists(id)) {
- throw new IllegalArgumentException("Cannot add a peer with id=" + id
- + " because that id already exists.");
- }
-
- if(id.contains("-")){
- throw new IllegalArgumentException("Found invalid peer name:" + id);
- }
-
- if (peerConfig.getClusterKey() != null) {
- try {
- ZKConfig.validateClusterKey(peerConfig.getClusterKey());
- } catch (IOException ioe) {
- throw new IllegalArgumentException(ioe.getMessage());
- }
- }
-
- checkQueuesDeleted(id);
-
- ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
-
- List<ZKUtilOp> listOfOps = new ArrayList<>(2);
- ZKUtilOp op1 = ZKUtilOp.createAndFailSilent(getPeerNode(id),
- ReplicationSerDeHelper.toByteArray(peerConfig));
- // b/w PeerWatcher and ReplicationZookeeper#add method to create the
- // peer-state znode. This happens while adding a peer
- // The peer state data is set as "ENABLED" by default.
- ZKUtilOp op2 = ZKUtilOp.createAndFailSilent(getPeerStateNode(id), ENABLED_ZNODE_BYTES);
- listOfOps.add(op1);
- listOfOps.add(op2);
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false);
- // A peer is enabled by default
- } catch (KeeperException e) {
- throw new ReplicationException("Could not add peer with id=" + id
- + ", peerConfif=>" + peerConfig, e);
- }
- }
-
- @Override
- public void unregisterPeer(String id) throws ReplicationException {
- try {
- if (!peerExists(id)) {
- throw new IllegalArgumentException("Cannot remove peer with id=" + id
- + " because that id does not exist.");
- }
- ZKUtil.deleteNodeRecursively(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id));
- } catch (KeeperException e) {
- throw new ReplicationException("Could not remove peer with id=" + id, e);
- }
- }
-
- @Override
- public void enablePeer(String id) throws ReplicationException {
- changePeerState(id, ReplicationProtos.ReplicationState.State.ENABLED);
- LOG.info("peer " + id + " is enabled");
- }
-
- @Override
- public void disablePeer(String id) throws ReplicationException {
- changePeerState(id, ReplicationProtos.ReplicationState.State.DISABLED);
- LOG.info("peer " + id + " is disabled");
- }
-
- @Override
- public Map<TableName, List<String>> getPeerTableCFsConfig(String id) throws ReplicationException {
- try {
- if (!peerExists(id)) {
- throw new IllegalArgumentException("peer " + id + " doesn't exist");
- }
- try {
- ReplicationPeerConfig rpc = getReplicationPeerConfig(id);
- if (rpc == null) {
- throw new ReplicationException("Unable to get tableCFs of the peer with id=" + id);
- }
- return rpc.getTableCFsMap();
- } catch (Exception e) {
- throw new ReplicationException(e);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Unable to get tableCFs of the peer with id=" + id, e);
- }
- }
-
- @Override
- public void setPeerTableCFsConfig(String id,
- Map<TableName, ? extends Collection<String>> tableCFs)
- throws ReplicationException {
- try {
- if (!peerExists(id)) {
- throw new IllegalArgumentException("Cannot set peer tableCFs because id=" + id
- + " does not exist.");
- }
- ReplicationPeerConfig rpc = getReplicationPeerConfig(id);
- if (rpc == null) {
- throw new ReplicationException("Unable to get tableCFs of the peer with id=" + id);
- }
- rpc.setTableCFsMap(tableCFs);
- ZKUtil.setData(this.zookeeper, getPeerNode(id),
- ReplicationSerDeHelper.toByteArray(rpc));
- LOG.info("Peer tableCFs with id= " + id + " is now " +
- ReplicationSerDeHelper.convertToString(tableCFs));
- } catch (KeeperException e) {
- throw new ReplicationException("Unable to change tableCFs of the peer with id=" + id, e);
- }
- }
-
- @Override
- public boolean getStatusOfPeer(String id) {
- ReplicationPeer replicationPeer = this.peerClusters.get(id);
- if (replicationPeer == null) {
- throw new IllegalArgumentException("Peer with id= " + id + " is not cached");
- }
- return replicationPeer.getPeerState() == PeerState.ENABLED;
- }
-
- @Override
- public boolean getStatusOfPeerFromBackingStore(String id) throws ReplicationException {
- try {
- if (!peerExists(id)) {
- throw new IllegalArgumentException("peer " + id + " doesn't exist");
- }
- String peerStateZNode = getPeerStateNode(id);
- try {
- return ReplicationPeerZKImpl.isStateEnabled(ZKUtil.getData(this.zookeeper, peerStateZNode));
- } catch (KeeperException e) {
- throw new ReplicationException(e);
- } catch (DeserializationException e) {
- throw new ReplicationException(e);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Unable to get status of the peer with id=" + id +
- " from backing store", e);
- } catch (InterruptedException e) {
- throw new ReplicationException(e);
- }
- }
-
- @Override
- public Map<String, ReplicationPeerConfig> getAllPeerConfigs() {
- Map<String, ReplicationPeerConfig> peers = new TreeMap<>();
- List<String> ids = null;
- try {
- ids = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
- for (String id : ids) {
- ReplicationPeerConfig peerConfig = getReplicationPeerConfig(id);
- if (peerConfig == null) {
- LOG.warn("Failed to get replication peer configuration of clusterid=" + id
- + " znode content, continuing.");
- continue;
- }
- peers.put(id, peerConfig);
- }
- } catch (KeeperException e) {
- this.abortable.abort("Cannot get the list of peers ", e);
- } catch (ReplicationException e) {
- this.abortable.abort("Cannot get the list of peers ", e);
- }
- return peers;
- }
-
- @Override
- public ReplicationPeer getConnectedPeer(String peerId) {
- return peerClusters.get(peerId);
- }
-
- @Override
- public Set<String> getConnectedPeerIds() {
- return peerClusters.keySet(); // this is not thread-safe
- }
-
- /**
- * Returns a ReplicationPeerConfig from the znode or null for the given peerId.
- */
- @Override
- public ReplicationPeerConfig getReplicationPeerConfig(String peerId)
- throws ReplicationException {
- String znode = getPeerNode(peerId);
- byte[] data = null;
- try {
- data = ZKUtil.getData(this.zookeeper, znode);
- } catch (InterruptedException e) {
- LOG.warn("Could not get configuration for peer because the thread " +
- "was interrupted. peerId=" + peerId);
- Thread.currentThread().interrupt();
- return null;
- } catch (KeeperException e) {
- throw new ReplicationException("Error getting configuration for peer with id="
- + peerId, e);
- }
- if (data == null) {
- LOG.error("Could not get configuration for peer because it doesn't exist. peerId=" + peerId);
- return null;
- }
-
- try {
- return ReplicationSerDeHelper.parsePeerFrom(data);
- } catch (DeserializationException e) {
- LOG.warn("Failed to parse cluster key from peerId=" + peerId
- + ", specifically the content from the following znode: " + znode);
- return null;
- }
- }
-
- @Override
- public Pair<ReplicationPeerConfig, Configuration> getPeerConf(String peerId)
- throws ReplicationException {
- ReplicationPeerConfig peerConfig = getReplicationPeerConfig(peerId);
-
- if (peerConfig == null) {
- return null;
- }
-
- Configuration otherConf;
- try {
- otherConf = HBaseConfiguration.createClusterConf(this.conf, peerConfig.getClusterKey());
- } catch (IOException e) {
- LOG.error("Can't get peer configuration for peerId=" + peerId + " because:", e);
- return null;
- }
-
- if (!peerConfig.getConfiguration().isEmpty()) {
- CompoundConfiguration compound = new CompoundConfiguration();
- compound.add(otherConf);
- compound.addStringMap(peerConfig.getConfiguration());
- return new Pair<>(peerConfig, compound);
- }
-
- return new Pair<>(peerConfig, otherConf);
- }
-
- @Override
- public void updatePeerConfig(String id, ReplicationPeerConfig newConfig)
- throws ReplicationException {
- ReplicationPeer peer = getConnectedPeer(id);
- if (peer == null){
- throw new ReplicationException("Could not find peer Id " + id + " in connected peers");
- }
- ReplicationPeerConfig existingConfig = peer.getPeerConfig();
- if (newConfig.getClusterKey() != null && !newConfig.getClusterKey().isEmpty() &&
- !newConfig.getClusterKey().equals(existingConfig.getClusterKey())){
- throw new ReplicationException("Changing the cluster key on an existing peer is not allowed."
- + " Existing key '" + existingConfig.getClusterKey() + "' does not match new key '"
- + newConfig.getClusterKey() +
- "'");
- }
- String existingEndpointImpl = existingConfig.getReplicationEndpointImpl();
- if (newConfig.getReplicationEndpointImpl() != null &&
- !newConfig.getReplicationEndpointImpl().isEmpty() &&
- !newConfig.getReplicationEndpointImpl().equals(existingEndpointImpl)){
- throw new ReplicationException("Changing the replication endpoint implementation class " +
- "on an existing peer is not allowed. Existing class '"
- + existingConfig.getReplicationEndpointImpl()
- + "' does not match new class '" + newConfig.getReplicationEndpointImpl() + "'");
- }
- //Update existingConfig's peer config and peer data with the new values, but don't touch config
- // or data that weren't explicitly changed
- existingConfig.getConfiguration().putAll(newConfig.getConfiguration());
- existingConfig.getPeerData().putAll(newConfig.getPeerData());
- existingConfig.setTableCFsMap(newConfig.getTableCFsMap());
- existingConfig.setNamespaces(newConfig.getNamespaces());
- existingConfig.setBandwidth(newConfig.getBandwidth());
-
- try {
- ZKUtil.setData(this.zookeeper, getPeerNode(id),
- ReplicationSerDeHelper.toByteArray(existingConfig));
- }
- catch(KeeperException ke){
- throw new ReplicationException("There was a problem trying to save changes to the " +
- "replication peer " + id, ke);
- }
- }
-
- /**
- * List all registered peer clusters and set a watch on their znodes.
- */
- @Override
- public List<String> getAllPeerIds() {
- List<String> ids = null;
- try {
- ids = ZKUtil.listChildrenAndWatchThem(this.zookeeper, this.peersZNode);
- } catch (KeeperException e) {
- this.abortable.abort("Cannot get the list of peers ", e);
- }
- return ids;
- }
-
- /**
- * A private method used during initialization. This method attempts to add all registered
- * peer clusters. This method does not set a watch on the peer cluster znodes.
- */
- private void addExistingPeers() throws ReplicationException {
- List<String> znodes = null;
- try {
- znodes = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
- } catch (KeeperException e) {
- throw new ReplicationException("Error getting the list of peer clusters.", e);
- }
- if (znodes != null) {
- for (String z : znodes) {
- createAndAddPeer(z);
- }
- }
- }
-
- @Override
- public boolean peerConnected(String peerId) throws ReplicationException {
- return createAndAddPeer(peerId);
- }
-
- @Override
- public void peerDisconnected(String peerId) {
- ReplicationPeer rp = this.peerClusters.get(peerId);
- if (rp != null) {
- ((ConcurrentMap<String, ReplicationPeerZKImpl>) peerClusters).remove(peerId, rp);
- }
- }
-
- /**
- * Attempt to connect to a new remote slave cluster.
- * @param peerId a short that identifies the cluster
- * @return true if a new connection was made, false if no new connection was made.
- */
- public boolean createAndAddPeer(String peerId) throws ReplicationException {
- if (peerClusters == null) {
- return false;
- }
- if (this.peerClusters.containsKey(peerId)) {
- return false;
- }
-
- ReplicationPeerZKImpl peer = null;
- try {
- peer = createPeer(peerId);
- } catch (Exception e) {
- throw new ReplicationException("Error adding peer with id=" + peerId, e);
- }
- if (peer == null) {
- return false;
- }
- ReplicationPeerZKImpl previous =
- ((ConcurrentMap<String, ReplicationPeerZKImpl>) peerClusters).putIfAbsent(peerId, peer);
- if (previous == null) {
- LOG.info("Added new peer cluster=" + peer.getPeerConfig().getClusterKey());
- } else {
- LOG.info("Peer already present, " + previous.getPeerConfig().getClusterKey() +
- ", new cluster=" + peer.getPeerConfig().getClusterKey());
- }
- return true;
- }
-
- /**
- * Update the state znode of a peer cluster.
- * @param id
- * @param state
- */
- private void changePeerState(String id, ReplicationProtos.ReplicationState.State state)
- throws ReplicationException {
- try {
- if (!peerExists(id)) {
- throw new IllegalArgumentException("Cannot enable/disable peer because id=" + id
- + " does not exist.");
- }
- String peerStateZNode = getPeerStateNode(id);
- byte[] stateBytes =
- (state == ReplicationProtos.ReplicationState.State.ENABLED) ? ENABLED_ZNODE_BYTES
- : DISABLED_ZNODE_BYTES;
- if (ZKUtil.checkExists(this.zookeeper, peerStateZNode) != -1) {
- ZKUtil.setData(this.zookeeper, peerStateZNode, stateBytes);
- } else {
- ZKUtil.createAndWatch(this.zookeeper, peerStateZNode, stateBytes);
- }
- LOG.info("Peer with id= " + id + " is now " + state.name());
- } catch (KeeperException e) {
- throw new ReplicationException("Unable to change state of the peer with id=" + id, e);
- }
- }
-
- /**
- * Helper method to connect to a peer
- * @param peerId peer's identifier
- * @return object representing the peer
- * @throws ReplicationException
- */
- private ReplicationPeerZKImpl createPeer(String peerId) throws ReplicationException {
- Pair<ReplicationPeerConfig, Configuration> pair = getPeerConf(peerId);
- if (pair == null) {
- return null;
- }
- Configuration peerConf = pair.getSecond();
-
- ReplicationPeerZKImpl peer = new ReplicationPeerZKImpl(zookeeper,
- peerConf, peerId, pair.getFirst(), abortable);
- try {
- peer.startStateTracker(this.getPeerStateNode(peerId));
- } catch (KeeperException e) {
- throw new ReplicationException("Error starting the peer state tracker for peerId=" +
- peerId, e);
- }
-
- try {
- peer.startPeerConfigTracker(this.getPeerNode(peerId));
- } catch (KeeperException e) {
- throw new ReplicationException("Error starting the peer tableCFs tracker for peerId=" +
- peerId, e);
- }
-
- return peer;
- }
-
- private void checkQueuesDeleted(String peerId) throws ReplicationException {
- if (queuesClient == null) return;
- try {
- List<String> replicators = queuesClient.getListOfReplicators();
- if (replicators == null || replicators.isEmpty()) {
- return;
- }
- for (String replicator : replicators) {
- List<String> queueIds = queuesClient.getAllQueues(replicator);
- for (String queueId : queueIds) {
- ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
- if (queueInfo.getPeerId().equals(peerId)) {
- throw new ReplicationException("undeleted queue for peerId: " + peerId
- + ", replicator: " + replicator + ", queueId: " + queueId);
- }
- }
- }
- // Check for hfile-refs queue
- if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode)
- && queuesClient.getAllPeersFromHFileRefsQueue().contains(peerId)) {
- throw new ReplicationException("Undeleted queue for peerId: " + peerId
- + ", found in hfile-refs node path " + hfileRefsZNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Could not check queues deleted with id=" + peerId, e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
deleted file mode 100644
index 1403f6d..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.ServerName;
-
-/**
- * This class is responsible for the parsing logic for a znode representing a queue.
- * It will extract the peerId if it's recovered as well as the dead region servers
- * that were part of the queue's history.
- */
-@InterfaceAudience.Private
-public class ReplicationQueueInfo {
- private static final Log LOG = LogFactory.getLog(ReplicationQueueInfo.class);
-
- private final String peerId;
- private final String peerClusterZnode;
- private boolean queueRecovered;
- // List of all the dead region servers that had this queue (if recovered)
- private List<String> deadRegionServers = new ArrayList<>();
-
- /**
- * The passed znode will be either the id of the peer cluster or
- * the handling story of that queue in the form of id-servername-*
- */
- public ReplicationQueueInfo(String znode) {
- this.peerClusterZnode = znode;
- String[] parts = znode.split("-", 2);
- this.queueRecovered = parts.length != 1;
- this.peerId = this.queueRecovered ?
- parts[0] : peerClusterZnode;
- if (parts.length >= 2) {
- // extract dead servers
- extractDeadServersFromZNodeString(parts[1], this.deadRegionServers);
- }
- }
-
- /**
- * Parse dead server names from znode string servername can contain "-" such as
- * "ip-10-46-221-101.ec2.internal", so we need skip some "-" during parsing for the following
- * cases: 2-ip-10-46-221-101.ec2.internal,52170,1364333181125-<server name>-...
- */
- private static void
- extractDeadServersFromZNodeString(String deadServerListStr, List<String> result) {
-
- if(deadServerListStr == null || result == null || deadServerListStr.isEmpty()) return;
-
- // valid server name delimiter "-" has to be after "," in a server name
- int seenCommaCnt = 0;
- int startIndex = 0;
- int len = deadServerListStr.length();
-
- for (int i = 0; i < len; i++) {
- switch (deadServerListStr.charAt(i)) {
- case ',':
- seenCommaCnt += 1;
- break;
- case '-':
- if(seenCommaCnt>=2) {
- if (i > startIndex) {
- String serverName = deadServerListStr.substring(startIndex, i);
- if(ServerName.isFullServerName(serverName)){
- result.add(serverName);
- } else {
- LOG.error("Found invalid server name:" + serverName);
- }
- startIndex = i + 1;
- }
- seenCommaCnt = 0;
- }
- break;
- default:
- break;
- }
- }
-
- // add tail
- if(startIndex < len - 1){
- String serverName = deadServerListStr.substring(startIndex, len);
- if(ServerName.isFullServerName(serverName)){
- result.add(serverName);
- } else {
- LOG.error("Found invalid server name at the end:" + serverName);
- }
- }
-
- LOG.debug("Found dead servers:" + result);
- }
-
- public List<String> getDeadRegionServers() {
- return Collections.unmodifiableList(this.deadRegionServers);
- }
-
- public String getPeerId() {
- return this.peerId;
- }
-
- public String getPeerClusterZnode() {
- return this.peerClusterZnode;
- }
-
- public boolean isQueueRecovered() {
- return queueRecovered;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
deleted file mode 100644
index be5a590..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.List;
-import java.util.SortedSet;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.util.Pair;
-
-/**
- * This provides an interface for maintaining a region server's replication queues. These queues
- * keep track of the WALs and HFile references (if hbase.replication.bulkload.enabled is enabled)
- * that still need to be replicated to remote clusters.
- */
-@InterfaceAudience.Private
-public interface ReplicationQueues {
-
- /**
- * Initialize the region server replication queue interface.
- * @param serverName The server name of the region server that owns the replication queues this
- * interface manages.
- */
- void init(String serverName) throws ReplicationException;
-
- /**
- * Remove a replication queue.
- * @param queueId a String that identifies the queue.
- */
- void removeQueue(String queueId);
-
- /**
- * Add a new WAL file to the given queue. If the queue does not exist it is created.
- * @param queueId a String that identifies the queue.
- * @param filename name of the WAL
- */
- void addLog(String queueId, String filename) throws ReplicationException;
-
- /**
- * Remove an WAL file from the given queue.
- * @param queueId a String that identifies the queue.
- * @param filename name of the WAL
- */
- void removeLog(String queueId, String filename);
-
- /**
- * Set the current position for a specific WAL in a given queue.
- * @param queueId a String that identifies the queue
- * @param filename name of the WAL
- * @param position the current position in the file
- */
- void setLogPosition(String queueId, String filename, long position);
-
- /**
- * Get the current position for a specific WAL in a given queue.
- * @param queueId a String that identifies the queue
- * @param filename name of the WAL
- * @return the current position in the file
- */
- long getLogPosition(String queueId, String filename) throws ReplicationException;
-
- /**
- * Remove all replication queues for this region server.
- */
- void removeAllQueues();
-
- /**
- * Get a list of all WALs in the given queue.
- * @param queueId a String that identifies the queue
- * @return a list of WALs, null if no such queue exists for this server
- */
- List<String> getLogsInQueue(String queueId);
-
- /**
- * Get a list of all queues for this region server.
- * @return a list of queueIds, an empty list if this region server is dead and has no outstanding queues
- */
- List<String> getAllQueues();
-
- /**
- * Get queueIds from a dead region server, whose queues has not been claimed by other region
- * servers.
- * @return empty if the queue exists but no children, null if the queue does not exist.
- */
- List<String> getUnClaimedQueueIds(String regionserver);
-
- /**
- * Take ownership for the queue identified by queueId and belongs to a dead region server.
- * @param regionserver the id of the dead region server
- * @param queueId the id of the queue
- * @return the new PeerId and A SortedSet of WALs in its queue, and null if no unclaimed queue.
- */
- Pair<String, SortedSet<String>> claimQueue(String regionserver, String queueId);
-
- /**
- * Remove the znode of region server if the queue is empty.
- * @param regionserver
- */
- void removeReplicatorIfQueueIsEmpty(String regionserver);
-
- /**
- * Get a list of all region servers that have outstanding replication queues. These servers could
- * be alive, dead or from a previous run of the cluster.
- * @return a list of server names
- */
- List<String> getListOfReplicators();
-
- /**
- * Checks if the provided znode is the same as this region server's
- * @param regionserver the id of the region server
- * @return if this is this rs's znode
- */
- boolean isThisOurRegionServer(String regionserver);
-
- /**
- * Add a peer to hfile reference queue if peer does not exist.
- * @param peerId peer cluster id to be added
- * @throws ReplicationException if fails to add a peer id to hfile reference queue
- */
- void addPeerToHFileRefs(String peerId) throws ReplicationException;
-
- /**
- * Remove a peer from hfile reference queue.
- * @param peerId peer cluster id to be removed
- */
- void removePeerFromHFileRefs(String peerId);
-
- /**
- * Add new hfile references to the queue.
- * @param peerId peer cluster id to which the hfiles need to be replicated
- * @param pairs list of pairs of { HFile location in staging dir, HFile path in region dir which
- * will be added in the queue }
- * @throws ReplicationException if fails to add a hfile reference
- */
- void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs) throws ReplicationException;
-
- /**
- * Remove hfile references from the queue.
- * @param peerId peer cluster id from which this hfile references needs to be removed
- * @param files list of hfile references to be removed
- */
- void removeHFileRefs(String peerId, List<String> files);
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
deleted file mode 100644
index 12fc6a1..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-
-/**
- * Wrapper around common arguments used to construct ReplicationQueues. Used to construct various
- * ReplicationQueues Implementations with different constructor arguments by reflection.
- */
-@InterfaceAudience.Private
-public class ReplicationQueuesArguments {
-
- private ZooKeeperWatcher zk;
- private Configuration conf;
- private Abortable abort;
-
- public ReplicationQueuesArguments(Configuration conf, Abortable abort) {
- this.conf = conf;
- this.abort = abort;
- }
-
- public ReplicationQueuesArguments(Configuration conf, Abortable abort, ZooKeeperWatcher zk) {
- this(conf, abort);
- setZk(zk);
- }
-
- public ZooKeeperWatcher getZk() {
- return zk;
- }
-
- public void setZk(ZooKeeperWatcher zk) {
- this.zk = zk;
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- public void setConf(Configuration conf) {
- this.conf = conf;
- }
-
- public Abortable getAbortable() {
- return abort;
- }
-
- public void setAbortable(Abortable abort) {
- this.abort = abort;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java
deleted file mode 100644
index 6d8900e..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * This provides an interface for clients of replication to view replication queues. These queues
- * keep track of the sources(WALs/HFile references) that still need to be replicated to remote
- * clusters.
- */
-@InterfaceAudience.Private
-public interface ReplicationQueuesClient {
-
- /**
- * Initialize the replication queue client interface.
- */
- public void init() throws ReplicationException;
-
- /**
- * Get a list of all region servers that have outstanding replication queues. These servers could
- * be alive, dead or from a previous run of the cluster.
- * @return a list of server names
- * @throws KeeperException zookeeper exception
- */
- List<String> getListOfReplicators() throws KeeperException;
-
- /**
- * Get a list of all WALs in the given queue on the given region server.
- * @param serverName the server name of the region server that owns the queue
- * @param queueId a String that identifies the queue
- * @return a list of WALs, null if this region server is dead and has no outstanding queues
- * @throws KeeperException zookeeper exception
- */
- List<String> getLogsInQueue(String serverName, String queueId) throws KeeperException;
-
- /**
- * Get a list of all queues for the specified region server.
- * @param serverName the server name of the region server that owns the set of queues
- * @return a list of queueIds, null if this region server is not a replicator.
- */
- List<String> getAllQueues(String serverName) throws KeeperException;
-
- /**
- * Load all wals in all replication queues from ZK. This method guarantees to return a
- * snapshot which contains all WALs in the zookeeper at the start of this call even there
- * is concurrent queue failover. However, some newly created WALs during the call may
- * not be included.
- */
- Set<String> getAllWALs() throws KeeperException;
-
- /**
- * Get the change version number of replication hfile references node. This can be used as
- * optimistic locking to get a consistent snapshot of the replication queues of hfile references.
- * @return change version number of hfile references node
- */
- int getHFileRefsNodeChangeVersion() throws KeeperException;
-
- /**
- * Get list of all peers from hfile reference queue.
- * @return a list of peer ids
- * @throws KeeperException zookeeper exception
- */
- List<String> getAllPeersFromHFileRefsQueue() throws KeeperException;
-
- /**
- * Get a list of all hfile references in the given peer.
- * @param peerId a String that identifies the peer
- * @return a list of hfile references, null if not found any
- * @throws KeeperException zookeeper exception
- */
- List<String> getReplicableHFiles(String peerId) throws KeeperException;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
deleted file mode 100644
index 834f831..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-
-/**
- * Wrapper around common arguments used to construct ReplicationQueuesClient. Used to construct
- * various ReplicationQueuesClient Implementations with different constructor arguments by
- * reflection.
- */
-@InterfaceAudience.Private
-public class ReplicationQueuesClientArguments extends ReplicationQueuesArguments {
- public ReplicationQueuesClientArguments(Configuration conf, Abortable abort,
- ZooKeeperWatcher zk) {
- super(conf, abort, zk);
- }
- public ReplicationQueuesClientArguments(Configuration conf, Abortable abort) {
- super(conf, abort);
- }
-}
[2/4] hbase git commit: HBASE-17442 Move most of the replication
related classes from hbase-client to new hbase-replication package. (Guanghao
Zhang).
Posted by ap...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-replication/pom.xml b/hbase-replication/pom.xml
new file mode 100644
index 0000000..858e9fc
--- /dev/null
+++ b/hbase-replication/pom.xml
@@ -0,0 +1,264 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ -->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>hbase-build-configuration</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <version>3.0.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+ <artifactId>hbase-replication</artifactId>
+ <name>Apache HBase - Replication</name>
+ <description>HBase Replication Support</description>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <!--Make it so assembly:single does nothing in here-->
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <skipAssembly>true</skipAssembly>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <!-- Always skip the second part executions, since we only run
+ simple unit tests in this module -->
+ <executions>
+ <execution>
+ <id>secondPartTestsExecution</id>
+ <phase>test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Make a jar and put the sources in the jar -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings
+ only. It has no influence on the Maven build itself.-->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <versionRange>[3.2,)</versionRange>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+ <dependencies>
+ <!-- Intra-project dependencies -->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-annotations</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-annotations</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <type>test-jar</type>
+ </dependency>
+ <!-- General dependencies -->
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+ <!-- profile against Hadoop 2.x: This is the default. -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <!--Below formatting for dev-support/generate-hadoopX-poms.sh-->
+ <!--h2--><name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>com.github.stephenc.findbugs</groupId>
+ <artifactId>findbugs-annotations</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <!--
+ profile for building against Hadoop 3.0.x. Activate using:
+ mvn -Dhadoop.profile=3.0
+ -->
+ <profile>
+ <id>hadoop-3.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>3.0</value>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>3.0-SNAPSHOT</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
new file mode 100644
index 0000000..8506cbb
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationFactory.java
@@ -0,0 +1,66 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.commons.lang.reflect.ConstructorUtils;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+
+/**
+ * A factory class for instantiating replication objects that deal with replication state.
+ */
+@InterfaceAudience.Private
+public class ReplicationFactory {
+
+ public static final Class defaultReplicationQueueClass = ReplicationQueuesZKImpl.class;
+
+ public static ReplicationQueues getReplicationQueues(ReplicationQueuesArguments args)
+ throws Exception {
+ Class<?> classToBuild = args.getConf().getClass("hbase.region.replica." +
+ "replication.replicationQueues.class", defaultReplicationQueueClass);
+ return (ReplicationQueues) ConstructorUtils.invokeConstructor(classToBuild, args);
+ }
+
+ public static ReplicationQueuesClient getReplicationQueuesClient(
+ ReplicationQueuesClientArguments args) throws Exception {
+ Class<?> classToBuild = args.getConf().getClass(
+ "hbase.region.replica.replication.replicationQueuesClient.class",
+ ReplicationQueuesClientZKImpl.class);
+ return (ReplicationQueuesClient) ConstructorUtils.invokeConstructor(classToBuild, args);
+ }
+
+ public static ReplicationPeers getReplicationPeers(final ZooKeeperWatcher zk, Configuration conf,
+ Abortable abortable) {
+ return getReplicationPeers(zk, conf, null, abortable);
+ }
+
+ public static ReplicationPeers getReplicationPeers(final ZooKeeperWatcher zk, Configuration conf,
+ final ReplicationQueuesClient queuesClient, Abortable abortable) {
+ return new ReplicationPeersZKImpl(zk, conf, queuesClient, abortable);
+ }
+
+ public static ReplicationTracker getReplicationTracker(ZooKeeperWatcher zookeeper,
+ final ReplicationPeers replicationPeers, Configuration conf, Abortable abortable,
+ Stoppable stopper) {
+ return new ReplicationTrackerZKImpl(zookeeper, replicationPeers, conf, abortable, stopper);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java
new file mode 100644
index 0000000..dfb5fdc
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationListener.java
@@ -0,0 +1,51 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.List;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+/**
+ * The replication listener interface can be implemented if a class needs to subscribe to events
+ * generated by the ReplicationTracker. These events include things like addition/deletion of peer
+ * clusters or failure of a local region server. To receive events, the class also needs to register
+ * itself with a Replication Tracker.
+ */
+@InterfaceAudience.Private
+public interface ReplicationListener {
+
+ /**
+ * A region server has been removed from the local cluster
+ * @param regionServer the removed region server
+ */
+ public void regionServerRemoved(String regionServer);
+
+ /**
+ * A peer cluster has been removed (i.e. unregistered) from replication.
+ * @param peerId The peer id of the cluster that has been removed
+ */
+ public void peerRemoved(String peerId);
+
+ /**
+ * The list of registered peer clusters has changed.
+ * @param peerIds A list of all currently registered peer clusters
+ */
+ public void peerListChanged(List<String> peerIds);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
new file mode 100644
index 0000000..4f18048
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+
+/**
+ * ReplicationPeer manages enabled / disabled state for the peer.
+ */
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION)
+public interface ReplicationPeer {
+
+ /**
+ * State of the peer, whether it is enabled or not
+ */
+ @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION)
+ enum PeerState {
+ ENABLED,
+ DISABLED
+ }
+
+ /**
+ * Get the identifier of this peer
+ * @return string representation of the id
+ */
+ String getId();
+
+ /**
+ * Get the peer config object
+ * @return the ReplicationPeerConfig for this peer
+ */
+ public ReplicationPeerConfig getPeerConfig();
+
+ /**
+ * Returns the state of the peer
+ * @return the enabled state
+ */
+ PeerState getPeerState();
+
+ /**
+ * Get the configuration object required to communicate with this peer
+ * @return configuration object
+ */
+ public Configuration getConfiguration();
+
+ /**
+ * Get replicable (table, cf-list) map of this peer
+ * @return the replicable (table, cf-list) map
+ */
+ public Map<TableName, List<String>> getTableCFs();
+
+ /**
+ * Get replicable namespace set of this peer
+ * @return the replicable namespaces set
+ */
+ public Set<String> getNamespaces();
+
+ /**
+ * Get the per node bandwidth upper limit for this peer
+ * @return the bandwidth upper limit
+ */
+ public long getPeerBandwidth();
+
+ void trackPeerConfigChanges(ReplicationPeerConfigListener listener);
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigListener.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigListener.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigListener.java
new file mode 100644
index 0000000..4e04186
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigListener.java
@@ -0,0 +1,33 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+
+@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.REPLICATION)
+public interface ReplicationPeerConfigListener {
+ /** Callback method for when users update the ReplicationPeerConfig for this peer
+ *
+ * @param rpc The updated ReplicationPeerConfig
+ */
+ void peerConfigUpdated(ReplicationPeerConfig rpc);
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
new file mode 100644
index 0000000..3973be9
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerZKImpl.java
@@ -0,0 +1,318 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.replication.ReplicationSerDeHelper;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NodeExistsException;
+
+@InterfaceAudience.Private
+public class ReplicationPeerZKImpl extends ReplicationStateZKBase
+ implements ReplicationPeer, Abortable, Closeable {
+ private static final Log LOG = LogFactory.getLog(ReplicationPeerZKImpl.class);
+
+ private ReplicationPeerConfig peerConfig;
+ private final String id;
+ private volatile PeerState peerState;
+ private volatile Map<TableName, List<String>> tableCFs = new HashMap<>();
+ private final Configuration conf;
+ private PeerStateTracker peerStateTracker;
+ private PeerConfigTracker peerConfigTracker;
+
+
+ /**
+ * Constructor that takes all the objects required to communicate with the specified peer, except
+ * for the region server addresses.
+ * @param conf configuration object to this peer
+ * @param id string representation of this peer's identifier
+ * @param peerConfig configuration for the replication peer
+ */
+ public ReplicationPeerZKImpl(ZooKeeperWatcher zkWatcher, Configuration conf,
+ String id, ReplicationPeerConfig peerConfig,
+ Abortable abortable)
+ throws ReplicationException {
+ super(zkWatcher, conf, abortable);
+ this.conf = conf;
+ this.peerConfig = peerConfig;
+ this.id = id;
+ }
+
+ /**
+ * start a state tracker to check whether this peer is enabled or not
+ *
+ * @param peerStateNode path to zk node which stores peer state
+ * @throws KeeperException
+ */
+ public void startStateTracker(String peerStateNode)
+ throws KeeperException {
+ ensurePeerEnabled(peerStateNode);
+ this.peerStateTracker = new PeerStateTracker(peerStateNode, zookeeper, this);
+ this.peerStateTracker.start();
+ try {
+ this.readPeerStateZnode();
+ } catch (DeserializationException e) {
+ throw ZKUtil.convert(e);
+ }
+ }
+
+ private void readPeerStateZnode() throws DeserializationException {
+ this.peerState =
+ isStateEnabled(this.peerStateTracker.getData(false))
+ ? PeerState.ENABLED
+ : PeerState.DISABLED;
+ }
+
+ /**
+ * start a table-cfs tracker to listen the (table, cf-list) map change
+ * @param peerConfigNode path to zk node which stores table-cfs
+ * @throws KeeperException
+ */
+ public void startPeerConfigTracker(String peerConfigNode)
+ throws KeeperException {
+ this.peerConfigTracker = new PeerConfigTracker(peerConfigNode, zookeeper,
+ this);
+ this.peerConfigTracker.start();
+ this.readPeerConfig();
+ }
+
+ private ReplicationPeerConfig readPeerConfig() {
+ try {
+ byte[] data = peerConfigTracker.getData(false);
+ if (data != null) {
+ this.peerConfig = ReplicationSerDeHelper.parsePeerFrom(data);
+ }
+ } catch (DeserializationException e) {
+ LOG.error("", e);
+ }
+ return this.peerConfig;
+ }
+
+ @Override
+ public PeerState getPeerState() {
+ return peerState;
+ }
+
+ /**
+ * Get the identifier of this peer
+ * @return string representation of the id (short)
+ */
+ @Override
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * Get the peer config object
+ * @return the ReplicationPeerConfig for this peer
+ */
+ @Override
+ public ReplicationPeerConfig getPeerConfig() {
+ return peerConfig;
+ }
+
+ /**
+ * Get the configuration object required to communicate with this peer
+ * @return configuration object
+ */
+ @Override
+ public Configuration getConfiguration() {
+ return conf;
+ }
+
+ /**
+ * Get replicable (table, cf-list) map of this peer
+ * @return the replicable (table, cf-list) map
+ */
+ @Override
+ public Map<TableName, List<String>> getTableCFs() {
+ this.tableCFs = peerConfig.getTableCFsMap();
+ return this.tableCFs;
+ }
+
+ /**
+ * Get replicable namespace set of this peer
+ * @return the replicable namespaces set
+ */
+ @Override
+ public Set<String> getNamespaces() {
+ return this.peerConfig.getNamespaces();
+ }
+
+ @Override
+ public long getPeerBandwidth() {
+ return this.peerConfig.getBandwidth();
+ }
+
+ @Override
+ public void trackPeerConfigChanges(ReplicationPeerConfigListener listener) {
+ if (this.peerConfigTracker != null){
+ this.peerConfigTracker.setListener(listener);
+ }
+ }
+
+ @Override
+ public void abort(String why, Throwable e) {
+ LOG.fatal("The ReplicationPeer corresponding to peer " + peerConfig
+ + " was aborted for the following reason(s):" + why, e);
+ }
+
+ @Override
+ public boolean isAborted() {
+ // Currently the replication peer is never "Aborted", we just log when the
+ // abort method is called.
+ return false;
+ }
+
+ @Override
+ public void close() throws IOException {
+ // TODO: stop zkw?
+ }
+
+ /**
+ * Parse the raw data from ZK to get a peer's state
+ * @param bytes raw ZK data
+ * @return True if the passed in <code>bytes</code> are those of a pb serialized ENABLED state.
+ * @throws DeserializationException
+ */
+ public static boolean isStateEnabled(final byte[] bytes) throws DeserializationException {
+ ReplicationProtos.ReplicationState.State state = parseStateFrom(bytes);
+ return ReplicationProtos.ReplicationState.State.ENABLED == state;
+ }
+
+ /**
+ * @param bytes Content of a state znode.
+ * @return State parsed from the passed bytes.
+ * @throws DeserializationException
+ */
+ private static ReplicationProtos.ReplicationState.State parseStateFrom(final byte[] bytes)
+ throws DeserializationException {
+ ProtobufUtil.expectPBMagicPrefix(bytes);
+ int pblen = ProtobufUtil.lengthOfPBMagic();
+ ReplicationProtos.ReplicationState.Builder builder =
+ ReplicationProtos.ReplicationState.newBuilder();
+ ReplicationProtos.ReplicationState state;
+ try {
+ ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
+ state = builder.build();
+ return state.getState();
+ } catch (IOException e) {
+ throw new DeserializationException(e);
+ }
+ }
+
+ /**
+ * Utility method to ensure an ENABLED znode is in place; if not present, we create it.
+ * @param path Path to znode to check
+ * @return True if we created the znode.
+ * @throws NodeExistsException
+ * @throws KeeperException
+ */
+ private boolean ensurePeerEnabled(final String path)
+ throws NodeExistsException, KeeperException {
+ if (ZKUtil.checkExists(zookeeper, path) == -1) {
+ // There is a race b/w PeerWatcher and ReplicationZookeeper#add method to create the
+ // peer-state znode. This happens while adding a peer.
+ // The peer state data is set as "ENABLED" by default.
+ ZKUtil.createNodeIfNotExistsAndWatch(zookeeper, path,
+ ReplicationStateZKBase.ENABLED_ZNODE_BYTES);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Tracker for state of this peer
+ */
+ public class PeerStateTracker extends ZooKeeperNodeTracker {
+
+ public PeerStateTracker(String peerStateZNode, ZooKeeperWatcher watcher,
+ Abortable abortable) {
+ super(watcher, peerStateZNode, abortable);
+ }
+
+ @Override
+ public synchronized void nodeDataChanged(String path) {
+ if (path.equals(node)) {
+ super.nodeDataChanged(path);
+ try {
+ readPeerStateZnode();
+ } catch (DeserializationException e) {
+ LOG.warn("Failed deserializing the content of " + path, e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Tracker for PeerConfigNode of this peer
+ */
+ public class PeerConfigTracker extends ZooKeeperNodeTracker {
+
+ ReplicationPeerConfigListener listener;
+
+ public PeerConfigTracker(String peerConfigNode, ZooKeeperWatcher watcher,
+ Abortable abortable) {
+ super(watcher, peerConfigNode, abortable);
+ }
+
+ public synchronized void setListener(ReplicationPeerConfigListener listener){
+ this.listener = listener;
+ }
+
+ @Override
+ public synchronized void nodeCreated(String path) {
+ if (path.equals(node)) {
+ super.nodeCreated(path);
+ ReplicationPeerConfig config = readPeerConfig();
+ if (listener != null){
+ listener.peerConfigUpdated(config);
+ }
+ }
+ }
+
+ @Override
+ public synchronized void nodeDataChanged(String path) {
+ //superclass calls nodeCreated
+ if (path.equals(node)) {
+ super.nodeDataChanged(path);
+ }
+
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
new file mode 100644
index 0000000..2a7963a
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeers.java
@@ -0,0 +1,177 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * This provides an interface for maintaining a set of peer clusters. These peers are remote slave
+ * clusters that data is replicated to. A peer cluster can be in three different states:
+ *
+ * 1. Not-Registered - There is no notion of the peer cluster.
+ * 2. Registered - The peer has an id and is being tracked but there is no connection.
+ * 3. Connected - There is an active connection to the remote peer.
+ *
+ * In the registered or connected state, a peer cluster can either be enabled or disabled.
+ */
+@InterfaceAudience.Private
+public interface ReplicationPeers {
+
+ /**
+ * Initialize the ReplicationPeers interface.
+ */
+ void init() throws ReplicationException;
+
+ /**
+ * Add a new remote slave cluster for replication.
+ * @param peerId a short that identifies the cluster
+ * @param peerConfig configuration for the replication slave cluster
+ */
+ void registerPeer(String peerId, ReplicationPeerConfig peerConfig)
+ throws ReplicationException;
+
+ /**
+ * Removes a remote slave cluster and stops the replication to it.
+ * @param peerId a short that identifies the cluster
+ */
+ void unregisterPeer(String peerId) throws ReplicationException;
+
+ /**
+ * Method called after a peer has been connected. It will create a ReplicationPeer to track the
+ * newly connected cluster.
+ * @param peerId a short that identifies the cluster
+ * @return whether a ReplicationPeer was successfully created
+ * @throws ReplicationException
+ */
+ boolean peerConnected(String peerId) throws ReplicationException;
+
+ /**
+ * Method called after a peer has been disconnected. It will remove the ReplicationPeer that
+ * tracked the disconnected cluster.
+ * @param peerId a short that identifies the cluster
+ */
+ void peerDisconnected(String peerId);
+
+ /**
+ * Restart the replication to the specified remote slave cluster.
+ * @param peerId a short that identifies the cluster
+ */
+ void enablePeer(String peerId) throws ReplicationException;
+
+ /**
+ * Stop the replication to the specified remote slave cluster.
+ * @param peerId a short that identifies the cluster
+ */
+ void disablePeer(String peerId) throws ReplicationException;
+
+ /**
+ * Get the table and column-family list string of the peer from the underlying storage.
+ * @param peerId a short that identifies the cluster
+ */
+ public Map<TableName, List<String>> getPeerTableCFsConfig(String peerId)
+ throws ReplicationException;
+
+ /**
+ * Set the table and column-family list string of the peer to the underlying storage.
+ * @param peerId a short that identifies the cluster
+ * @param tableCFs the table and column-family list which will be replicated for this peer
+ */
+ public void setPeerTableCFsConfig(String peerId,
+ Map<TableName, ? extends Collection<String>> tableCFs)
+ throws ReplicationException;
+
+ /**
+ * Returns the ReplicationPeer for the specified connected peer. This ReplicationPeer will
+ * continue to track changes to the Peer's state and config. This method returns null if no
+ * peer has been connected with the given peerId.
+ * @param peerId id for the peer
+ * @return ReplicationPeer object
+ */
+ ReplicationPeer getConnectedPeer(String peerId);
+
+ /**
+ * Returns the set of peerIds of the clusters that have been connected and have an underlying
+ * ReplicationPeer.
+ * @return a Set of Strings for peerIds
+ */
+ public Set<String> getConnectedPeerIds();
+
+ /**
+ * Get the replication status for the specified connected remote slave cluster.
+ * The value might be read from cache, so it is recommended to
+ * use {@link #getStatusOfPeerFromBackingStore(String)}
+ * if reading the state after enabling or disabling it.
+ * @param peerId a short that identifies the cluster
+ * @return true if replication is enabled, false otherwise.
+ */
+ boolean getStatusOfPeer(String peerId);
+
+ /**
+ * Get the replication status for the specified remote slave cluster, which doesn't
+ * have to be connected. The state is read directly from the backing store.
+ * @param peerId a short that identifies the cluster
+ * @return true if replication is enabled, false otherwise.
+ * @throws ReplicationException thrown if there's an error contacting the store
+ */
+ boolean getStatusOfPeerFromBackingStore(String peerId) throws ReplicationException;
+
+ /**
+ * List the cluster replication configs of all remote slave clusters (whether they are
+ * enabled/disabled or connected/disconnected).
+ * @return A map of peer ids to peer cluster keys
+ */
+ Map<String, ReplicationPeerConfig> getAllPeerConfigs();
+
+ /**
+ * List the peer ids of all remote slave clusters (whether they are enabled/disabled or
+ * connected/disconnected).
+ * @return A list of peer ids
+ */
+ List<String> getAllPeerIds();
+
+ /**
+ * Returns the configured ReplicationPeerConfig for this peerId
+ * @param peerId a short name that identifies the cluster
+ * @return ReplicationPeerConfig for the peer
+ */
+ ReplicationPeerConfig getReplicationPeerConfig(String peerId) throws ReplicationException;
+
+ /**
+ * Returns the configuration needed to talk to the remote slave cluster.
+ * @param peerId a short that identifies the cluster
+ * @return the configuration for the peer cluster, null if it was unable to get the configuration
+ */
+ Pair<ReplicationPeerConfig, Configuration> getPeerConf(String peerId) throws ReplicationException;
+
+ /**
+ * Update the peerConfig for the a given peer cluster
+ * @param id a short that identifies the cluster
+ * @param peerConfig new config for the peer cluster
+ * @throws ReplicationException
+ */
+ void updatePeerConfig(String id, ReplicationPeerConfig peerConfig) throws ReplicationException;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
new file mode 100644
index 0000000..751e454
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeersZKImpl.java
@@ -0,0 +1,546 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.CompoundConfiguration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.replication.ReplicationSerDeHelper;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
+import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.zookeeper.ZKConfig;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * This class provides an implementation of the ReplicationPeers interface using ZooKeeper. The
+ * peers znode contains a list of all peer replication clusters and the current replication state of
+ * those clusters. It has one child peer znode for each peer cluster. The peer znode is named with
+ * the cluster id provided by the user in the HBase shell. The value of the peer znode contains the
+ * peers cluster key provided by the user in the HBase Shell. The cluster key contains a list of
+ * zookeeper quorum peers, the client port for the zookeeper quorum, and the base znode for HBase.
+ * For example:
+ *
+ * /hbase/replication/peers/1 [Value: zk1.host.com,zk2.host.com,zk3.host.com:2181:/hbase]
+ * /hbase/replication/peers/2 [Value: zk5.host.com,zk6.host.com,zk7.host.com:2181:/hbase]
+ *
+ * Each of these peer znodes has a child znode that indicates whether or not replication is enabled
+ * on that peer cluster. These peer-state znodes do not have child znodes and simply contain a
+ * boolean value (i.e. ENABLED or DISABLED). This value is read/maintained by the
+ * ReplicationPeer.PeerStateTracker class. For example:
+ *
+ * /hbase/replication/peers/1/peer-state [Value: ENABLED]
+ *
+ * Each of these peer znodes has a child znode that indicates which data will be replicated
+ * to the peer cluster. These peer-tableCFs znodes do not have child znodes and only have a
+ * table/cf list config. This value is read/maintained by the ReplicationPeer.TableCFsTracker
+ * class. For example:
+ *
+ * /hbase/replication/peers/1/tableCFs [Value: "table1; table2:cf1,cf3; table3:cfx,cfy"]
+ */
+@InterfaceAudience.Private
+public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements ReplicationPeers {
+
+ // Map of peer clusters keyed by their id
+ private Map<String, ReplicationPeerZKImpl> peerClusters;
+ private final ReplicationQueuesClient queuesClient;
+ private Abortable abortable;
+
+ private static final Log LOG = LogFactory.getLog(ReplicationPeersZKImpl.class);
+
+ public ReplicationPeersZKImpl(final ZooKeeperWatcher zk, final Configuration conf,
+ final ReplicationQueuesClient queuesClient, Abortable abortable) {
+ super(zk, conf, abortable);
+ this.abortable = abortable;
+ this.peerClusters = new ConcurrentHashMap<>();
+ this.queuesClient = queuesClient;
+ }
+
+ @Override
+ public void init() throws ReplicationException {
+ try {
+ if (ZKUtil.checkExists(this.zookeeper, this.peersZNode) < 0) {
+ ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException("Could not initialize replication peers", e);
+ }
+ addExistingPeers();
+ }
+
+ @Override
+ public void registerPeer(String id, ReplicationPeerConfig peerConfig)
+ throws ReplicationException {
+ try {
+ if (peerExists(id)) {
+ throw new IllegalArgumentException("Cannot add a peer with id=" + id
+ + " because that id already exists.");
+ }
+
+ if(id.contains("-")){
+ throw new IllegalArgumentException("Found invalid peer name:" + id);
+ }
+
+ if (peerConfig.getClusterKey() != null) {
+ try {
+ ZKConfig.validateClusterKey(peerConfig.getClusterKey());
+ } catch (IOException ioe) {
+ throw new IllegalArgumentException(ioe.getMessage());
+ }
+ }
+
+ checkQueuesDeleted(id);
+
+ ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
+
+ List<ZKUtilOp> listOfOps = new ArrayList<>(2);
+ ZKUtilOp op1 = ZKUtilOp.createAndFailSilent(getPeerNode(id),
+ ReplicationSerDeHelper.toByteArray(peerConfig));
+ // b/w PeerWatcher and ReplicationZookeeper#add method to create the
+ // peer-state znode. This happens while adding a peer
+ // The peer state data is set as "ENABLED" by default.
+ ZKUtilOp op2 = ZKUtilOp.createAndFailSilent(getPeerStateNode(id), ENABLED_ZNODE_BYTES);
+ listOfOps.add(op1);
+ listOfOps.add(op2);
+ ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false);
+ // A peer is enabled by default
+ } catch (KeeperException e) {
+ throw new ReplicationException("Could not add peer with id=" + id
+ + ", peerConfif=>" + peerConfig, e);
+ }
+ }
+
+ @Override
+ public void unregisterPeer(String id) throws ReplicationException {
+ try {
+ if (!peerExists(id)) {
+ throw new IllegalArgumentException("Cannot remove peer with id=" + id
+ + " because that id does not exist.");
+ }
+ ZKUtil.deleteNodeRecursively(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id));
+ } catch (KeeperException e) {
+ throw new ReplicationException("Could not remove peer with id=" + id, e);
+ }
+ }
+
+ @Override
+ public void enablePeer(String id) throws ReplicationException {
+ changePeerState(id, ReplicationProtos.ReplicationState.State.ENABLED);
+ LOG.info("peer " + id + " is enabled");
+ }
+
+ @Override
+ public void disablePeer(String id) throws ReplicationException {
+ changePeerState(id, ReplicationProtos.ReplicationState.State.DISABLED);
+ LOG.info("peer " + id + " is disabled");
+ }
+
+ @Override
+ public Map<TableName, List<String>> getPeerTableCFsConfig(String id) throws ReplicationException {
+ try {
+ if (!peerExists(id)) {
+ throw new IllegalArgumentException("peer " + id + " doesn't exist");
+ }
+ try {
+ ReplicationPeerConfig rpc = getReplicationPeerConfig(id);
+ if (rpc == null) {
+ throw new ReplicationException("Unable to get tableCFs of the peer with id=" + id);
+ }
+ return rpc.getTableCFsMap();
+ } catch (Exception e) {
+ throw new ReplicationException(e);
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException("Unable to get tableCFs of the peer with id=" + id, e);
+ }
+ }
+
+ @Override
+ public void setPeerTableCFsConfig(String id,
+ Map<TableName, ? extends Collection<String>> tableCFs)
+ throws ReplicationException {
+ try {
+ if (!peerExists(id)) {
+ throw new IllegalArgumentException("Cannot set peer tableCFs because id=" + id
+ + " does not exist.");
+ }
+ ReplicationPeerConfig rpc = getReplicationPeerConfig(id);
+ if (rpc == null) {
+ throw new ReplicationException("Unable to get tableCFs of the peer with id=" + id);
+ }
+ rpc.setTableCFsMap(tableCFs);
+ ZKUtil.setData(this.zookeeper, getPeerNode(id),
+ ReplicationSerDeHelper.toByteArray(rpc));
+ LOG.info("Peer tableCFs with id= " + id + " is now " +
+ ReplicationSerDeHelper.convertToString(tableCFs));
+ } catch (KeeperException e) {
+ throw new ReplicationException("Unable to change tableCFs of the peer with id=" + id, e);
+ }
+ }
+
+ @Override
+ public boolean getStatusOfPeer(String id) {
+ ReplicationPeer replicationPeer = this.peerClusters.get(id);
+ if (replicationPeer == null) {
+ throw new IllegalArgumentException("Peer with id= " + id + " is not cached");
+ }
+ return replicationPeer.getPeerState() == PeerState.ENABLED;
+ }
+
+ @Override
+ public boolean getStatusOfPeerFromBackingStore(String id) throws ReplicationException {
+ try {
+ if (!peerExists(id)) {
+ throw new IllegalArgumentException("peer " + id + " doesn't exist");
+ }
+ String peerStateZNode = getPeerStateNode(id);
+ try {
+ return ReplicationPeerZKImpl.isStateEnabled(ZKUtil.getData(this.zookeeper, peerStateZNode));
+ } catch (KeeperException e) {
+ throw new ReplicationException(e);
+ } catch (DeserializationException e) {
+ throw new ReplicationException(e);
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException("Unable to get status of the peer with id=" + id +
+ " from backing store", e);
+ } catch (InterruptedException e) {
+ throw new ReplicationException(e);
+ }
+ }
+
+ @Override
+ public Map<String, ReplicationPeerConfig> getAllPeerConfigs() {
+ Map<String, ReplicationPeerConfig> peers = new TreeMap<>();
+ List<String> ids = null;
+ try {
+ ids = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
+ for (String id : ids) {
+ ReplicationPeerConfig peerConfig = getReplicationPeerConfig(id);
+ if (peerConfig == null) {
+ LOG.warn("Failed to get replication peer configuration of clusterid=" + id
+ + " znode content, continuing.");
+ continue;
+ }
+ peers.put(id, peerConfig);
+ }
+ } catch (KeeperException e) {
+ this.abortable.abort("Cannot get the list of peers ", e);
+ } catch (ReplicationException e) {
+ this.abortable.abort("Cannot get the list of peers ", e);
+ }
+ return peers;
+ }
+
+ @Override
+ public ReplicationPeer getConnectedPeer(String peerId) {
+ return peerClusters.get(peerId);
+ }
+
+ @Override
+ public Set<String> getConnectedPeerIds() {
+ return peerClusters.keySet(); // this is not thread-safe
+ }
+
+ /**
+ * Returns a ReplicationPeerConfig from the znode or null for the given peerId.
+ */
+ @Override
+ public ReplicationPeerConfig getReplicationPeerConfig(String peerId)
+ throws ReplicationException {
+ String znode = getPeerNode(peerId);
+ byte[] data = null;
+ try {
+ data = ZKUtil.getData(this.zookeeper, znode);
+ } catch (InterruptedException e) {
+ LOG.warn("Could not get configuration for peer because the thread " +
+ "was interrupted. peerId=" + peerId);
+ Thread.currentThread().interrupt();
+ return null;
+ } catch (KeeperException e) {
+ throw new ReplicationException("Error getting configuration for peer with id="
+ + peerId, e);
+ }
+ if (data == null) {
+ LOG.error("Could not get configuration for peer because it doesn't exist. peerId=" + peerId);
+ return null;
+ }
+
+ try {
+ return ReplicationSerDeHelper.parsePeerFrom(data);
+ } catch (DeserializationException e) {
+ LOG.warn("Failed to parse cluster key from peerId=" + peerId
+ + ", specifically the content from the following znode: " + znode);
+ return null;
+ }
+ }
+
+ @Override
+ public Pair<ReplicationPeerConfig, Configuration> getPeerConf(String peerId)
+ throws ReplicationException {
+ ReplicationPeerConfig peerConfig = getReplicationPeerConfig(peerId);
+
+ if (peerConfig == null) {
+ return null;
+ }
+
+ Configuration otherConf;
+ try {
+ otherConf = HBaseConfiguration.createClusterConf(this.conf, peerConfig.getClusterKey());
+ } catch (IOException e) {
+ LOG.error("Can't get peer configuration for peerId=" + peerId + " because:", e);
+ return null;
+ }
+
+ if (!peerConfig.getConfiguration().isEmpty()) {
+ CompoundConfiguration compound = new CompoundConfiguration();
+ compound.add(otherConf);
+ compound.addStringMap(peerConfig.getConfiguration());
+ return new Pair<>(peerConfig, compound);
+ }
+
+ return new Pair<>(peerConfig, otherConf);
+ }
+
+ @Override
+ public void updatePeerConfig(String id, ReplicationPeerConfig newConfig)
+ throws ReplicationException {
+ ReplicationPeer peer = getConnectedPeer(id);
+ if (peer == null){
+ throw new ReplicationException("Could not find peer Id " + id + " in connected peers");
+ }
+ ReplicationPeerConfig existingConfig = peer.getPeerConfig();
+ if (newConfig.getClusterKey() != null && !newConfig.getClusterKey().isEmpty() &&
+ !newConfig.getClusterKey().equals(existingConfig.getClusterKey())){
+ throw new ReplicationException("Changing the cluster key on an existing peer is not allowed."
+ + " Existing key '" + existingConfig.getClusterKey() + "' does not match new key '"
+ + newConfig.getClusterKey() +
+ "'");
+ }
+ String existingEndpointImpl = existingConfig.getReplicationEndpointImpl();
+ if (newConfig.getReplicationEndpointImpl() != null &&
+ !newConfig.getReplicationEndpointImpl().isEmpty() &&
+ !newConfig.getReplicationEndpointImpl().equals(existingEndpointImpl)){
+ throw new ReplicationException("Changing the replication endpoint implementation class " +
+ "on an existing peer is not allowed. Existing class '"
+ + existingConfig.getReplicationEndpointImpl()
+ + "' does not match new class '" + newConfig.getReplicationEndpointImpl() + "'");
+ }
+ //Update existingConfig's peer config and peer data with the new values, but don't touch config
+ // or data that weren't explicitly changed
+ existingConfig.getConfiguration().putAll(newConfig.getConfiguration());
+ existingConfig.getPeerData().putAll(newConfig.getPeerData());
+ existingConfig.setTableCFsMap(newConfig.getTableCFsMap());
+ existingConfig.setNamespaces(newConfig.getNamespaces());
+ existingConfig.setBandwidth(newConfig.getBandwidth());
+
+ try {
+ ZKUtil.setData(this.zookeeper, getPeerNode(id),
+ ReplicationSerDeHelper.toByteArray(existingConfig));
+ }
+ catch(KeeperException ke){
+ throw new ReplicationException("There was a problem trying to save changes to the " +
+ "replication peer " + id, ke);
+ }
+ }
+
+ /**
+ * List all registered peer clusters and set a watch on their znodes.
+ */
+ @Override
+ public List<String> getAllPeerIds() {
+ List<String> ids = null;
+ try {
+ ids = ZKUtil.listChildrenAndWatchThem(this.zookeeper, this.peersZNode);
+ } catch (KeeperException e) {
+ this.abortable.abort("Cannot get the list of peers ", e);
+ }
+ return ids;
+ }
+
+ /**
+ * A private method used during initialization. This method attempts to add all registered
+ * peer clusters. This method does not set a watch on the peer cluster znodes.
+ */
+ private void addExistingPeers() throws ReplicationException {
+ List<String> znodes = null;
+ try {
+ znodes = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
+ } catch (KeeperException e) {
+ throw new ReplicationException("Error getting the list of peer clusters.", e);
+ }
+ if (znodes != null) {
+ for (String z : znodes) {
+ createAndAddPeer(z);
+ }
+ }
+ }
+
+ @Override
+ public boolean peerConnected(String peerId) throws ReplicationException {
+ return createAndAddPeer(peerId);
+ }
+
+ @Override
+ public void peerDisconnected(String peerId) {
+ ReplicationPeer rp = this.peerClusters.get(peerId);
+ if (rp != null) {
+ ((ConcurrentMap<String, ReplicationPeerZKImpl>) peerClusters).remove(peerId, rp);
+ }
+ }
+
+ /**
+ * Attempt to connect to a new remote slave cluster.
+ * @param peerId a short that identifies the cluster
+ * @return true if a new connection was made, false if no new connection was made.
+ */
+ public boolean createAndAddPeer(String peerId) throws ReplicationException {
+ if (peerClusters == null) {
+ return false;
+ }
+ if (this.peerClusters.containsKey(peerId)) {
+ return false;
+ }
+
+ ReplicationPeerZKImpl peer = null;
+ try {
+ peer = createPeer(peerId);
+ } catch (Exception e) {
+ throw new ReplicationException("Error adding peer with id=" + peerId, e);
+ }
+ if (peer == null) {
+ return false;
+ }
+ ReplicationPeerZKImpl previous =
+ ((ConcurrentMap<String, ReplicationPeerZKImpl>) peerClusters).putIfAbsent(peerId, peer);
+ if (previous == null) {
+ LOG.info("Added new peer cluster=" + peer.getPeerConfig().getClusterKey());
+ } else {
+ LOG.info("Peer already present, " + previous.getPeerConfig().getClusterKey() +
+ ", new cluster=" + peer.getPeerConfig().getClusterKey());
+ }
+ return true;
+ }
+
+ /**
+ * Update the state znode of a peer cluster.
+ * @param id
+ * @param state
+ */
+ private void changePeerState(String id, ReplicationProtos.ReplicationState.State state)
+ throws ReplicationException {
+ try {
+ if (!peerExists(id)) {
+ throw new IllegalArgumentException("Cannot enable/disable peer because id=" + id
+ + " does not exist.");
+ }
+ String peerStateZNode = getPeerStateNode(id);
+ byte[] stateBytes =
+ (state == ReplicationProtos.ReplicationState.State.ENABLED) ? ENABLED_ZNODE_BYTES
+ : DISABLED_ZNODE_BYTES;
+ if (ZKUtil.checkExists(this.zookeeper, peerStateZNode) != -1) {
+ ZKUtil.setData(this.zookeeper, peerStateZNode, stateBytes);
+ } else {
+ ZKUtil.createAndWatch(this.zookeeper, peerStateZNode, stateBytes);
+ }
+ LOG.info("Peer with id= " + id + " is now " + state.name());
+ } catch (KeeperException e) {
+ throw new ReplicationException("Unable to change state of the peer with id=" + id, e);
+ }
+ }
+
+ /**
+ * Helper method to connect to a peer
+ * @param peerId peer's identifier
+ * @return object representing the peer
+ * @throws ReplicationException
+ */
+ private ReplicationPeerZKImpl createPeer(String peerId) throws ReplicationException {
+ Pair<ReplicationPeerConfig, Configuration> pair = getPeerConf(peerId);
+ if (pair == null) {
+ return null;
+ }
+ Configuration peerConf = pair.getSecond();
+
+ ReplicationPeerZKImpl peer = new ReplicationPeerZKImpl(zookeeper,
+ peerConf, peerId, pair.getFirst(), abortable);
+ try {
+ peer.startStateTracker(this.getPeerStateNode(peerId));
+ } catch (KeeperException e) {
+ throw new ReplicationException("Error starting the peer state tracker for peerId=" +
+ peerId, e);
+ }
+
+ try {
+ peer.startPeerConfigTracker(this.getPeerNode(peerId));
+ } catch (KeeperException e) {
+ throw new ReplicationException("Error starting the peer tableCFs tracker for peerId=" +
+ peerId, e);
+ }
+
+ return peer;
+ }
+
+ private void checkQueuesDeleted(String peerId) throws ReplicationException {
+ if (queuesClient == null) return;
+ try {
+ List<String> replicators = queuesClient.getListOfReplicators();
+ if (replicators == null || replicators.isEmpty()) {
+ return;
+ }
+ for (String replicator : replicators) {
+ List<String> queueIds = queuesClient.getAllQueues(replicator);
+ for (String queueId : queueIds) {
+ ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
+ if (queueInfo.getPeerId().equals(peerId)) {
+ throw new ReplicationException("undeleted queue for peerId: " + peerId
+ + ", replicator: " + replicator + ", queueId: " + queueId);
+ }
+ }
+ }
+ // Check for hfile-refs queue
+ if (-1 != ZKUtil.checkExists(zookeeper, hfileRefsZNode)
+ && queuesClient.getAllPeersFromHFileRefsQueue().contains(peerId)) {
+ throw new ReplicationException("Undeleted queue for peerId: " + peerId
+ + ", found in hfile-refs node path " + hfileRefsZNode);
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException("Could not check queues deleted with id=" + peerId, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
new file mode 100644
index 0000000..1403f6d
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueueInfo.java
@@ -0,0 +1,130 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.ServerName;
+
+/**
+ * This class is responsible for the parsing logic for a znode representing a queue.
+ * It will extract the peerId if it's recovered as well as the dead region servers
+ * that were part of the queue's history.
+ */
+@InterfaceAudience.Private
+public class ReplicationQueueInfo {
+ private static final Log LOG = LogFactory.getLog(ReplicationQueueInfo.class);
+
+ private final String peerId;
+ private final String peerClusterZnode;
+ private boolean queueRecovered;
+ // List of all the dead region servers that had this queue (if recovered)
+ private List<String> deadRegionServers = new ArrayList<>();
+
+ /**
+ * The passed znode will be either the id of the peer cluster or
+ * the handling story of that queue in the form of id-servername-*
+ */
+ public ReplicationQueueInfo(String znode) {
+ this.peerClusterZnode = znode;
+ String[] parts = znode.split("-", 2);
+ this.queueRecovered = parts.length != 1;
+ this.peerId = this.queueRecovered ?
+ parts[0] : peerClusterZnode;
+ if (parts.length >= 2) {
+ // extract dead servers
+ extractDeadServersFromZNodeString(parts[1], this.deadRegionServers);
+ }
+ }
+
+ /**
+ * Parse dead server names from znode string servername can contain "-" such as
+ * "ip-10-46-221-101.ec2.internal", so we need skip some "-" during parsing for the following
+ * cases: 2-ip-10-46-221-101.ec2.internal,52170,1364333181125-<server name>-...
+ */
+ private static void
+ extractDeadServersFromZNodeString(String deadServerListStr, List<String> result) {
+
+ if(deadServerListStr == null || result == null || deadServerListStr.isEmpty()) return;
+
+ // valid server name delimiter "-" has to be after "," in a server name
+ int seenCommaCnt = 0;
+ int startIndex = 0;
+ int len = deadServerListStr.length();
+
+ for (int i = 0; i < len; i++) {
+ switch (deadServerListStr.charAt(i)) {
+ case ',':
+ seenCommaCnt += 1;
+ break;
+ case '-':
+ if(seenCommaCnt>=2) {
+ if (i > startIndex) {
+ String serverName = deadServerListStr.substring(startIndex, i);
+ if(ServerName.isFullServerName(serverName)){
+ result.add(serverName);
+ } else {
+ LOG.error("Found invalid server name:" + serverName);
+ }
+ startIndex = i + 1;
+ }
+ seenCommaCnt = 0;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ // add tail
+ if(startIndex < len - 1){
+ String serverName = deadServerListStr.substring(startIndex, len);
+ if(ServerName.isFullServerName(serverName)){
+ result.add(serverName);
+ } else {
+ LOG.error("Found invalid server name at the end:" + serverName);
+ }
+ }
+
+ LOG.debug("Found dead servers:" + result);
+ }
+
+ public List<String> getDeadRegionServers() {
+ return Collections.unmodifiableList(this.deadRegionServers);
+ }
+
+ public String getPeerId() {
+ return this.peerId;
+ }
+
+ public String getPeerClusterZnode() {
+ return this.peerClusterZnode;
+ }
+
+ public boolean isQueueRecovered() {
+ return queueRecovered;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
new file mode 100644
index 0000000..be5a590
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueues.java
@@ -0,0 +1,160 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.List;
+import java.util.SortedSet;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.util.Pair;
+
+/**
+ * This provides an interface for maintaining a region server's replication queues. These queues
+ * keep track of the WALs and HFile references (if hbase.replication.bulkload.enabled is enabled)
+ * that still need to be replicated to remote clusters.
+ */
+@InterfaceAudience.Private
+public interface ReplicationQueues {
+
+ /**
+ * Initialize the region server replication queue interface.
+ * @param serverName The server name of the region server that owns the replication queues this
+ * interface manages.
+ */
+ void init(String serverName) throws ReplicationException;
+
+ /**
+ * Remove a replication queue.
+ * @param queueId a String that identifies the queue.
+ */
+ void removeQueue(String queueId);
+
+ /**
+ * Add a new WAL file to the given queue. If the queue does not exist it is created.
+ * @param queueId a String that identifies the queue.
+ * @param filename name of the WAL
+ */
+ void addLog(String queueId, String filename) throws ReplicationException;
+
+ /**
+ * Remove an WAL file from the given queue.
+ * @param queueId a String that identifies the queue.
+ * @param filename name of the WAL
+ */
+ void removeLog(String queueId, String filename);
+
+ /**
+ * Set the current position for a specific WAL in a given queue.
+ * @param queueId a String that identifies the queue
+ * @param filename name of the WAL
+ * @param position the current position in the file
+ */
+ void setLogPosition(String queueId, String filename, long position);
+
+ /**
+ * Get the current position for a specific WAL in a given queue.
+ * @param queueId a String that identifies the queue
+ * @param filename name of the WAL
+ * @return the current position in the file
+ */
+ long getLogPosition(String queueId, String filename) throws ReplicationException;
+
+ /**
+ * Remove all replication queues for this region server.
+ */
+ void removeAllQueues();
+
+ /**
+ * Get a list of all WALs in the given queue.
+ * @param queueId a String that identifies the queue
+ * @return a list of WALs, null if no such queue exists for this server
+ */
+ List<String> getLogsInQueue(String queueId);
+
+ /**
+ * Get a list of all queues for this region server.
+ * @return a list of queueIds, an empty list if this region server is dead and has no outstanding queues
+ */
+ List<String> getAllQueues();
+
+ /**
+ * Get queueIds from a dead region server, whose queues has not been claimed by other region
+ * servers.
+ * @return empty if the queue exists but no children, null if the queue does not exist.
+ */
+ List<String> getUnClaimedQueueIds(String regionserver);
+
+ /**
+ * Take ownership for the queue identified by queueId and belongs to a dead region server.
+ * @param regionserver the id of the dead region server
+ * @param queueId the id of the queue
+ * @return the new PeerId and A SortedSet of WALs in its queue, and null if no unclaimed queue.
+ */
+ Pair<String, SortedSet<String>> claimQueue(String regionserver, String queueId);
+
+ /**
+ * Remove the znode of region server if the queue is empty.
+ * @param regionserver
+ */
+ void removeReplicatorIfQueueIsEmpty(String regionserver);
+
+ /**
+ * Get a list of all region servers that have outstanding replication queues. These servers could
+ * be alive, dead or from a previous run of the cluster.
+ * @return a list of server names
+ */
+ List<String> getListOfReplicators();
+
+ /**
+ * Checks if the provided znode is the same as this region server's
+ * @param regionserver the id of the region server
+ * @return if this is this rs's znode
+ */
+ boolean isThisOurRegionServer(String regionserver);
+
+ /**
+ * Add a peer to hfile reference queue if peer does not exist.
+ * @param peerId peer cluster id to be added
+ * @throws ReplicationException if fails to add a peer id to hfile reference queue
+ */
+ void addPeerToHFileRefs(String peerId) throws ReplicationException;
+
+ /**
+ * Remove a peer from hfile reference queue.
+ * @param peerId peer cluster id to be removed
+ */
+ void removePeerFromHFileRefs(String peerId);
+
+ /**
+ * Add new hfile references to the queue.
+ * @param peerId peer cluster id to which the hfiles need to be replicated
+ * @param pairs list of pairs of { HFile location in staging dir, HFile path in region dir which
+ * will be added in the queue }
+ * @throws ReplicationException if fails to add a hfile reference
+ */
+ void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs) throws ReplicationException;
+
+ /**
+ * Remove hfile references from the queue.
+ * @param peerId peer cluster id from which this hfile references needs to be removed
+ * @param files list of hfile references to be removed
+ */
+ void removeHFileRefs(String peerId, List<String> files);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
new file mode 100644
index 0000000..12fc6a1
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesArguments.java
@@ -0,0 +1,70 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+
+/**
+ * Wrapper around common arguments used to construct ReplicationQueues. Used to construct various
+ * ReplicationQueues Implementations with different constructor arguments by reflection.
+ */
+@InterfaceAudience.Private
+public class ReplicationQueuesArguments {
+
+ private ZooKeeperWatcher zk;
+ private Configuration conf;
+ private Abortable abort;
+
+ public ReplicationQueuesArguments(Configuration conf, Abortable abort) {
+ this.conf = conf;
+ this.abort = abort;
+ }
+
+ public ReplicationQueuesArguments(Configuration conf, Abortable abort, ZooKeeperWatcher zk) {
+ this(conf, abort);
+ setZk(zk);
+ }
+
+ public ZooKeeperWatcher getZk() {
+ return zk;
+ }
+
+ public void setZk(ZooKeeperWatcher zk) {
+ this.zk = zk;
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ public Abortable getAbortable() {
+ return abort;
+ }
+
+ public void setAbortable(Abortable abort) {
+ this.abort = abort;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java
new file mode 100644
index 0000000..6d8900e
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClient.java
@@ -0,0 +1,93 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * This provides an interface for clients of replication to view replication queues. These queues
+ * keep track of the sources(WALs/HFile references) that still need to be replicated to remote
+ * clusters.
+ */
+@InterfaceAudience.Private
+public interface ReplicationQueuesClient {
+
+ /**
+ * Initialize the replication queue client interface.
+ */
+ public void init() throws ReplicationException;
+
+ /**
+ * Get a list of all region servers that have outstanding replication queues. These servers could
+ * be alive, dead or from a previous run of the cluster.
+ * @return a list of server names
+ * @throws KeeperException zookeeper exception
+ */
+ List<String> getListOfReplicators() throws KeeperException;
+
+ /**
+ * Get a list of all WALs in the given queue on the given region server.
+ * @param serverName the server name of the region server that owns the queue
+ * @param queueId a String that identifies the queue
+ * @return a list of WALs, null if this region server is dead and has no outstanding queues
+ * @throws KeeperException zookeeper exception
+ */
+ List<String> getLogsInQueue(String serverName, String queueId) throws KeeperException;
+
+ /**
+ * Get a list of all queues for the specified region server.
+ * @param serverName the server name of the region server that owns the set of queues
+ * @return a list of queueIds, null if this region server is not a replicator.
+ */
+ List<String> getAllQueues(String serverName) throws KeeperException;
+
+ /**
+ * Load all wals in all replication queues from ZK. This method guarantees to return a
+ * snapshot which contains all WALs in the zookeeper at the start of this call even there
+ * is concurrent queue failover. However, some newly created WALs during the call may
+ * not be included.
+ */
+ Set<String> getAllWALs() throws KeeperException;
+
+ /**
+ * Get the change version number of replication hfile references node. This can be used as
+ * optimistic locking to get a consistent snapshot of the replication queues of hfile references.
+ * @return change version number of hfile references node
+ */
+ int getHFileRefsNodeChangeVersion() throws KeeperException;
+
+ /**
+ * Get list of all peers from hfile reference queue.
+ * @return a list of peer ids
+ * @throws KeeperException zookeeper exception
+ */
+ List<String> getAllPeersFromHFileRefsQueue() throws KeeperException;
+
+ /**
+ * Get a list of all hfile references in the given peer.
+ * @param peerId a String that identifies the peer
+ * @return a list of hfile references, null if not found any
+ * @throws KeeperException zookeeper exception
+ */
+ List<String> getReplicableHFiles(String peerId) throws KeeperException;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
new file mode 100644
index 0000000..834f831
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientArguments.java
@@ -0,0 +1,40 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+
+/**
+ * Wrapper around common arguments used to construct ReplicationQueuesClient. Used to construct
+ * various ReplicationQueuesClient Implementations with different constructor arguments by
+ * reflection.
+ */
+@InterfaceAudience.Private
+public class ReplicationQueuesClientArguments extends ReplicationQueuesArguments {
+ public ReplicationQueuesClientArguments(Configuration conf, Abortable abort,
+ ZooKeeperWatcher zk) {
+ super(conf, abort, zk);
+ }
+ public ReplicationQueuesClientArguments(Configuration conf, Abortable abort) {
+ super(conf, abort);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java
new file mode 100644
index 0000000..1981131
--- /dev/null
+++ b/hbase-replication/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java
@@ -0,0 +1,175 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.replication;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.Sets;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.zookeeper.ZKUtil;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+@InterfaceAudience.Private
+public class ReplicationQueuesClientZKImpl extends ReplicationStateZKBase implements
+ ReplicationQueuesClient {
+
+ Log LOG = LogFactory.getLog(ReplicationQueuesClientZKImpl.class);
+
+ public ReplicationQueuesClientZKImpl(ReplicationQueuesClientArguments args) {
+ this(args.getZk(), args.getConf(), args.getAbortable());
+ }
+
+ public ReplicationQueuesClientZKImpl(final ZooKeeperWatcher zk, Configuration conf,
+ Abortable abortable) {
+ super(zk, conf, abortable);
+ }
+
+ @Override
+ public void init() throws ReplicationException {
+ try {
+ if (ZKUtil.checkExists(this.zookeeper, this.queuesZNode) < 0) {
+ ZKUtil.createWithParents(this.zookeeper, this.queuesZNode);
+ }
+ } catch (KeeperException e) {
+ throw new ReplicationException("Internal error while initializing a queues client", e);
+ }
+ }
+
+ @Override
+ public List<String> getLogsInQueue(String serverName, String queueId) throws KeeperException {
+ String znode = ZKUtil.joinZNode(this.queuesZNode, serverName);
+ znode = ZKUtil.joinZNode(znode, queueId);
+ List<String> result = null;
+ try {
+ result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to get list of wals for queueId=" + queueId
+ + " and serverName=" + serverName, e);
+ throw e;
+ }
+ return result;
+ }
+
+ @Override
+ public List<String> getAllQueues(String serverName) throws KeeperException {
+ String znode = ZKUtil.joinZNode(this.queuesZNode, serverName);
+ List<String> result = null;
+ try {
+ result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to get list of queues for serverName=" + serverName, e);
+ throw e;
+ }
+ return result;
+ }
+
+ @Override
+ public Set<String> getAllWALs() throws KeeperException {
+ /**
+ * Load all wals in all replication queues from ZK. This method guarantees to return a
+ * snapshot which contains all WALs in the zookeeper at the start of this call even there
+ * is concurrent queue failover. However, some newly created WALs during the call may
+ * not be included.
+ */
+ for (int retry = 0; ; retry++) {
+ int v0 = getQueuesZNodeCversion();
+ List<String> rss = getListOfReplicators();
+ if (rss == null || rss.isEmpty()) {
+ LOG.debug("Didn't find any region server that replicates, won't prevent any deletions.");
+ return ImmutableSet.of();
+ }
+ Set<String> wals = Sets.newHashSet();
+ for (String rs : rss) {
+ List<String> listOfPeers = getAllQueues(rs);
+ // if rs just died, this will be null
+ if (listOfPeers == null) {
+ continue;
+ }
+ for (String id : listOfPeers) {
+ List<String> peersWals = getLogsInQueue(rs, id);
+ if (peersWals != null) {
+ wals.addAll(peersWals);
+ }
+ }
+ }
+ int v1 = getQueuesZNodeCversion();
+ if (v0 == v1) {
+ return wals;
+ }
+ LOG.info(String.format("Replication queue node cversion changed from %d to %d, retry = %d",
+ v0, v1, retry));
+ }
+ }
+
+ public int getQueuesZNodeCversion() throws KeeperException {
+ try {
+ Stat stat = new Stat();
+ ZKUtil.getDataNoWatch(this.zookeeper, this.queuesZNode, stat);
+ return stat.getCversion();
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to get stat of replication rs node", e);
+ throw e;
+ }
+ }
+
+ @Override
+ public int getHFileRefsNodeChangeVersion() throws KeeperException {
+ Stat stat = new Stat();
+ try {
+ ZKUtil.getDataNoWatch(this.zookeeper, this.hfileRefsZNode, stat);
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to get stat of replication hfile references node.", e);
+ throw e;
+ }
+ return stat.getCversion();
+ }
+
+ @Override
+ public List<String> getAllPeersFromHFileRefsQueue() throws KeeperException {
+ List<String> result = null;
+ try {
+ result = ZKUtil.listChildrenNoWatch(this.zookeeper, this.hfileRefsZNode);
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to get list of all peers in hfile references node.", e);
+ throw e;
+ }
+ return result;
+ }
+
+ @Override
+ public List<String> getReplicableHFiles(String peerId) throws KeeperException {
+ String znode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
+ List<String> result = null;
+ try {
+ result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
+ } catch (KeeperException e) {
+ this.abortable.abort("Failed to get list of hfile references for peerId=" + peerId, e);
+ throw e;
+ }
+ return result;
+ }
+}
[3/4] hbase git commit: HBASE-17442 Move most of the replication
related classes from hbase-client to new hbase-replication package. (Guanghao
Zhang).
Posted by ap...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java
deleted file mode 100644
index 1981131..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesClientZKImpl.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.ImmutableSet;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Sets;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.data.Stat;
-
-@InterfaceAudience.Private
-public class ReplicationQueuesClientZKImpl extends ReplicationStateZKBase implements
- ReplicationQueuesClient {
-
- Log LOG = LogFactory.getLog(ReplicationQueuesClientZKImpl.class);
-
- public ReplicationQueuesClientZKImpl(ReplicationQueuesClientArguments args) {
- this(args.getZk(), args.getConf(), args.getAbortable());
- }
-
- public ReplicationQueuesClientZKImpl(final ZooKeeperWatcher zk, Configuration conf,
- Abortable abortable) {
- super(zk, conf, abortable);
- }
-
- @Override
- public void init() throws ReplicationException {
- try {
- if (ZKUtil.checkExists(this.zookeeper, this.queuesZNode) < 0) {
- ZKUtil.createWithParents(this.zookeeper, this.queuesZNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Internal error while initializing a queues client", e);
- }
- }
-
- @Override
- public List<String> getLogsInQueue(String serverName, String queueId) throws KeeperException {
- String znode = ZKUtil.joinZNode(this.queuesZNode, serverName);
- znode = ZKUtil.joinZNode(znode, queueId);
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of wals for queueId=" + queueId
- + " and serverName=" + serverName, e);
- throw e;
- }
- return result;
- }
-
- @Override
- public List<String> getAllQueues(String serverName) throws KeeperException {
- String znode = ZKUtil.joinZNode(this.queuesZNode, serverName);
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of queues for serverName=" + serverName, e);
- throw e;
- }
- return result;
- }
-
- @Override
- public Set<String> getAllWALs() throws KeeperException {
- /**
- * Load all wals in all replication queues from ZK. This method guarantees to return a
- * snapshot which contains all WALs in the zookeeper at the start of this call even there
- * is concurrent queue failover. However, some newly created WALs during the call may
- * not be included.
- */
- for (int retry = 0; ; retry++) {
- int v0 = getQueuesZNodeCversion();
- List<String> rss = getListOfReplicators();
- if (rss == null || rss.isEmpty()) {
- LOG.debug("Didn't find any region server that replicates, won't prevent any deletions.");
- return ImmutableSet.of();
- }
- Set<String> wals = Sets.newHashSet();
- for (String rs : rss) {
- List<String> listOfPeers = getAllQueues(rs);
- // if rs just died, this will be null
- if (listOfPeers == null) {
- continue;
- }
- for (String id : listOfPeers) {
- List<String> peersWals = getLogsInQueue(rs, id);
- if (peersWals != null) {
- wals.addAll(peersWals);
- }
- }
- }
- int v1 = getQueuesZNodeCversion();
- if (v0 == v1) {
- return wals;
- }
- LOG.info(String.format("Replication queue node cversion changed from %d to %d, retry = %d",
- v0, v1, retry));
- }
- }
-
- public int getQueuesZNodeCversion() throws KeeperException {
- try {
- Stat stat = new Stat();
- ZKUtil.getDataNoWatch(this.zookeeper, this.queuesZNode, stat);
- return stat.getCversion();
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get stat of replication rs node", e);
- throw e;
- }
- }
-
- @Override
- public int getHFileRefsNodeChangeVersion() throws KeeperException {
- Stat stat = new Stat();
- try {
- ZKUtil.getDataNoWatch(this.zookeeper, this.hfileRefsZNode, stat);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get stat of replication hfile references node.", e);
- throw e;
- }
- return stat.getCversion();
- }
-
- @Override
- public List<String> getAllPeersFromHFileRefsQueue() throws KeeperException {
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, this.hfileRefsZNode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of all peers in hfile references node.", e);
- throw e;
- }
- return result;
- }
-
- @Override
- public List<String> getReplicableHFiles(String peerId) throws KeeperException {
- String znode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of hfile references for peerId=" + peerId, e);
- throw e;
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
deleted file mode 100644
index 4733706..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++ /dev/null
@@ -1,407 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * This class provides an implementation of the
- * interface using ZooKeeper. The
- * base znode that this class works at is the myQueuesZnode. The myQueuesZnode contains a list of
- * all outstanding WAL files on this region server that need to be replicated. The myQueuesZnode is
- * the regionserver name (a concatenation of the region server’s hostname, client port and start
- * code). For example:
- *
- * /hbase/replication/rs/hostname.example.org,6020,1234
- *
- * Within this znode, the region server maintains a set of WAL replication queues. These queues are
- * represented by child znodes named using there give queue id. For example:
- *
- * /hbase/replication/rs/hostname.example.org,6020,1234/1
- * /hbase/replication/rs/hostname.example.org,6020,1234/2
- *
- * Each queue has one child znode for every WAL that still needs to be replicated. The value of
- * these WAL child znodes is the latest position that has been replicated. This position is updated
- * every time a WAL entry is replicated. For example:
- *
- * /hbase/replication/rs/hostname.example.org,6020,1234/1/23522342.23422 [VALUE: 254]
- */
-@InterfaceAudience.Private
-public class ReplicationQueuesZKImpl extends ReplicationStateZKBase implements ReplicationQueues {
-
- /** Znode containing all replication queues for this region server. */
- private String myQueuesZnode;
-
- private static final Log LOG = LogFactory.getLog(ReplicationQueuesZKImpl.class);
-
- public ReplicationQueuesZKImpl(ReplicationQueuesArguments args) {
- this(args.getZk(), args.getConf(), args.getAbortable());
- }
-
- public ReplicationQueuesZKImpl(final ZooKeeperWatcher zk, Configuration conf,
- Abortable abortable) {
- super(zk, conf, abortable);
- }
-
- @Override
- public void init(String serverName) throws ReplicationException {
- this.myQueuesZnode = ZKUtil.joinZNode(this.queuesZNode, serverName);
- try {
- if (ZKUtil.checkExists(this.zookeeper, this.myQueuesZnode) < 0) {
- ZKUtil.createWithParents(this.zookeeper, this.myQueuesZnode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Could not initialize replication queues.", e);
- }
- if (conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
- HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT)) {
- try {
- if (ZKUtil.checkExists(this.zookeeper, this.hfileRefsZNode) < 0) {
- ZKUtil.createWithParents(this.zookeeper, this.hfileRefsZNode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Could not initialize hfile references replication queue.",
- e);
- }
- }
- }
-
- @Override
- public void removeQueue(String queueId) {
- try {
- ZKUtil.deleteNodeRecursively(this.zookeeper, ZKUtil.joinZNode(this.myQueuesZnode, queueId));
- } catch (KeeperException e) {
- this.abortable.abort("Failed to delete queue (queueId=" + queueId + ")", e);
- }
- }
-
- @Override
- public void addLog(String queueId, String filename) throws ReplicationException {
- String znode = ZKUtil.joinZNode(this.myQueuesZnode, queueId);
- znode = ZKUtil.joinZNode(znode, filename);
- try {
- ZKUtil.createWithParents(this.zookeeper, znode);
- } catch (KeeperException e) {
- throw new ReplicationException(
- "Could not add log because znode could not be created. queueId=" + queueId
- + ", filename=" + filename);
- }
- }
-
- @Override
- public void removeLog(String queueId, String filename) {
- try {
- String znode = ZKUtil.joinZNode(this.myQueuesZnode, queueId);
- znode = ZKUtil.joinZNode(znode, filename);
- ZKUtil.deleteNode(this.zookeeper, znode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to remove wal from queue (queueId=" + queueId + ", filename="
- + filename + ")", e);
- }
- }
-
- @Override
- public void setLogPosition(String queueId, String filename, long position) {
- try {
- String znode = ZKUtil.joinZNode(this.myQueuesZnode, queueId);
- znode = ZKUtil.joinZNode(znode, filename);
- // Why serialize String of Long and not Long as bytes?
- ZKUtil.setData(this.zookeeper, znode, ZKUtil.positionToByteArray(position));
- } catch (KeeperException e) {
- this.abortable.abort("Failed to write replication wal position (filename=" + filename
- + ", position=" + position + ")", e);
- }
- }
-
- @Override
- public long getLogPosition(String queueId, String filename) throws ReplicationException {
- String clusterZnode = ZKUtil.joinZNode(this.myQueuesZnode, queueId);
- String znode = ZKUtil.joinZNode(clusterZnode, filename);
- byte[] bytes = null;
- try {
- bytes = ZKUtil.getData(this.zookeeper, znode);
- } catch (KeeperException e) {
- throw new ReplicationException("Internal Error: could not get position in log for queueId="
- + queueId + ", filename=" + filename, e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return 0;
- }
- try {
- return ZKUtil.parseWALPositionFrom(bytes);
- } catch (DeserializationException de) {
- LOG.warn("Failed to parse WALPosition for queueId=" + queueId + " and wal=" + filename
- + " znode content, continuing.");
- }
- // if we can not parse the position, start at the beginning of the wal file
- // again
- return 0;
- }
-
- @Override
- public boolean isThisOurRegionServer(String regionserver) {
- return ZKUtil.joinZNode(this.queuesZNode, regionserver).equals(this.myQueuesZnode);
- }
-
- @Override
- public List<String> getUnClaimedQueueIds(String regionserver) {
- if (isThisOurRegionServer(regionserver)) {
- return null;
- }
- String rsZnodePath = ZKUtil.joinZNode(this.queuesZNode, regionserver);
- List<String> queues = null;
- try {
- queues = ZKUtil.listChildrenNoWatch(this.zookeeper, rsZnodePath);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to getUnClaimedQueueIds for RS" + regionserver, e);
- }
- return queues;
- }
-
- @Override
- public Pair<String, SortedSet<String>> claimQueue(String regionserver, String queueId) {
- LOG.info("Atomically moving " + regionserver + "/" + queueId + "'s WALs to my queue");
- return moveQueueUsingMulti(regionserver, queueId);
- }
-
- @Override
- public void removeReplicatorIfQueueIsEmpty(String regionserver) {
- String rsPath = ZKUtil.joinZNode(this.queuesZNode, regionserver);
- try {
- List<String> list = ZKUtil.listChildrenNoWatch(this.zookeeper, rsPath);
- if (list != null && list.isEmpty()){
- ZKUtil.deleteNode(this.zookeeper, rsPath);
- }
- } catch (KeeperException e) {
- LOG.warn("Got error while removing replicator", e);
- }
- }
-
- @Override
- public void removeAllQueues() {
- try {
- ZKUtil.deleteNodeRecursively(this.zookeeper, this.myQueuesZnode);
- } catch (KeeperException e) {
- // if the znode is already expired, don't bother going further
- if (e instanceof KeeperException.SessionExpiredException) {
- return;
- }
- this.abortable.abort("Failed to delete replication queues for region server: "
- + this.myQueuesZnode, e);
- }
- }
-
- @Override
- public List<String> getLogsInQueue(String queueId) {
- String znode = ZKUtil.joinZNode(this.myQueuesZnode, queueId);
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, znode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of wals for queueId=" + queueId, e);
- }
- return result;
- }
-
- @Override
- public List<String> getAllQueues() {
- List<String> listOfQueues = null;
- try {
- listOfQueues = ZKUtil.listChildrenNoWatch(this.zookeeper, this.myQueuesZnode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get a list of queues for region server: "
- + this.myQueuesZnode, e);
- }
- return listOfQueues == null ? new ArrayList<>() : listOfQueues;
- }
-
- /**
- * It "atomically" copies one peer's wals queue from another dead region server and returns them
- * all sorted. The new peer id is equal to the old peer id appended with the dead server's znode.
- * @param znode pertaining to the region server to copy the queues from
- * @peerId peerId pertaining to the queue need to be copied
- */
- private Pair<String, SortedSet<String>> moveQueueUsingMulti(String znode, String peerId) {
- try {
- // hbase/replication/rs/deadrs
- String deadRSZnodePath = ZKUtil.joinZNode(this.queuesZNode, znode);
- List<ZKUtilOp> listOfOps = new ArrayList<>();
- ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(peerId);
-
- String newPeerId = peerId + "-" + znode;
- String newPeerZnode = ZKUtil.joinZNode(this.myQueuesZnode, newPeerId);
- // check the logs queue for the old peer cluster
- String oldClusterZnode = ZKUtil.joinZNode(deadRSZnodePath, peerId);
- List<String> wals = ZKUtil.listChildrenNoWatch(this.zookeeper, oldClusterZnode);
-
- if (!peerExists(replicationQueueInfo.getPeerId())) {
- LOG.warn("Peer " + replicationQueueInfo.getPeerId() +
- " didn't exist, will move its queue to avoid the failure of multi op");
- for (String wal : wals) {
- String oldWalZnode = ZKUtil.joinZNode(oldClusterZnode, wal);
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldWalZnode));
- }
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode));
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false);
- return null;
- }
-
- SortedSet<String> logQueue = new TreeSet<>();
- if (wals == null || wals.isEmpty()) {
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode));
- } else {
- // create the new cluster znode
- ZKUtilOp op = ZKUtilOp.createAndFailSilent(newPeerZnode, HConstants.EMPTY_BYTE_ARRAY);
- listOfOps.add(op);
- // get the offset of the logs and set it to new znodes
- for (String wal : wals) {
- String oldWalZnode = ZKUtil.joinZNode(oldClusterZnode, wal);
- byte[] logOffset = ZKUtil.getData(this.zookeeper, oldWalZnode);
- LOG.debug("Creating " + wal + " with data " + Bytes.toString(logOffset));
- String newLogZnode = ZKUtil.joinZNode(newPeerZnode, wal);
- listOfOps.add(ZKUtilOp.createAndFailSilent(newLogZnode, logOffset));
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldWalZnode));
- logQueue.add(wal);
- }
- // add delete op for peer
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(oldClusterZnode));
-
- if (LOG.isTraceEnabled())
- LOG.trace(" The multi list size is: " + listOfOps.size());
- }
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false);
-
- LOG.info("Atomically moved " + znode + "/" + peerId + "'s WALs to my queue");
- return new Pair<>(newPeerId, logQueue);
- } catch (KeeperException e) {
- // Multi call failed; it looks like some other regionserver took away the logs.
- LOG.warn("Got exception in copyQueuesFromRSUsingMulti: ", e);
- } catch (InterruptedException e) {
- LOG.warn("Got exception in copyQueuesFromRSUsingMulti: ", e);
- Thread.currentThread().interrupt();
- }
- return null;
- }
-
- @Override
- public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs)
- throws ReplicationException {
- String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
- boolean debugEnabled = LOG.isDebugEnabled();
- if (debugEnabled) {
- LOG.debug("Adding hfile references " + pairs + " in queue " + peerZnode);
- }
-
- int size = pairs.size();
- List<ZKUtilOp> listOfOps = new ArrayList<>(size);
-
- for (int i = 0; i < size; i++) {
- listOfOps.add(ZKUtilOp.createAndFailSilent(
- ZKUtil.joinZNode(peerZnode, pairs.get(i).getSecond().getName()),
- HConstants.EMPTY_BYTE_ARRAY));
- }
- if (debugEnabled) {
- LOG.debug(" The multi list size for adding hfile references in zk for node " + peerZnode
- + " is " + listOfOps.size());
- }
- try {
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to create hfile reference znode=" + e.getPath(), e);
- }
- }
-
- @Override
- public void removeHFileRefs(String peerId, List<String> files) {
- String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
- boolean debugEnabled = LOG.isDebugEnabled();
- if (debugEnabled) {
- LOG.debug("Removing hfile references " + files + " from queue " + peerZnode);
- }
-
- int size = files.size();
- List<ZKUtilOp> listOfOps = new ArrayList<>(size);
-
- for (int i = 0; i < size; i++) {
- listOfOps.add(ZKUtilOp.deleteNodeFailSilent(ZKUtil.joinZNode(peerZnode, files.get(i))));
- }
- if (debugEnabled) {
- LOG.debug(" The multi list size for removing hfile references in zk for node " + peerZnode
- + " is " + listOfOps.size());
- }
- try {
- ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true);
- } catch (KeeperException e) {
- LOG.error("Failed to remove hfile reference znode=" + e.getPath(), e);
- }
- }
-
- @Override
- public void addPeerToHFileRefs(String peerId) throws ReplicationException {
- String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
- try {
- if (ZKUtil.checkExists(this.zookeeper, peerZnode) == -1) {
- LOG.info("Adding peer " + peerId + " to hfile reference queue.");
- ZKUtil.createWithParents(this.zookeeper, peerZnode);
- }
- } catch (KeeperException e) {
- throw new ReplicationException("Failed to add peer " + peerId + " to hfile reference queue.",
- e);
- }
- }
-
- @Override
- public void removePeerFromHFileRefs(String peerId) {
- final String peerZnode = ZKUtil.joinZNode(this.hfileRefsZNode, peerId);
- try {
- if (ZKUtil.checkExists(this.zookeeper, peerZnode) == -1) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Peer " + peerZnode + " not found in hfile reference queue.");
- }
- return;
- } else {
- LOG.info("Removing peer " + peerZnode + " from hfile reference queue.");
- ZKUtil.deleteNodeRecursively(this.zookeeper, peerZnode);
- }
- } catch (KeeperException e) {
- LOG.error("Ignoring the exception to remove peer " + peerId + " from hfile reference queue.",
- e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
deleted file mode 100644
index 47b780d..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationStateZKBase.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
-
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos;
-import org.apache.hadoop.hbase.zookeeper.ZKConfig;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
-
-
-/**
- * This is a base class for maintaining replication state in zookeeper.
- */
-@InterfaceAudience.Private
-public abstract class ReplicationStateZKBase {
-
- /**
- * The name of the znode that contains the replication status of a remote slave (i.e. peer)
- * cluster.
- */
- protected final String peerStateNodeName;
- /** The name of the base znode that contains all replication state. */
- protected final String replicationZNode;
- /** The name of the znode that contains a list of all remote slave (i.e. peer) clusters. */
- protected final String peersZNode;
- /** The name of the znode that contains all replication queues */
- protected final String queuesZNode;
- /** The name of the znode that contains queues of hfile references to be replicated */
- protected final String hfileRefsZNode;
- /** The cluster key of the local cluster */
- protected final String ourClusterKey;
- /** The name of the znode that contains tableCFs */
- protected final String tableCFsNodeName;
-
- protected final ZooKeeperWatcher zookeeper;
- protected final Configuration conf;
- protected final Abortable abortable;
-
- // Public for testing
- public static final byte[] ENABLED_ZNODE_BYTES =
- toByteArray(ReplicationProtos.ReplicationState.State.ENABLED);
- public static final byte[] DISABLED_ZNODE_BYTES =
- toByteArray(ReplicationProtos.ReplicationState.State.DISABLED);
- public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY =
- "zookeeper.znode.replication.hfile.refs";
- public static final String ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT = "hfile-refs";
-
- public ReplicationStateZKBase(ZooKeeperWatcher zookeeper, Configuration conf,
- Abortable abortable) {
- this.zookeeper = zookeeper;
- this.conf = conf;
- this.abortable = abortable;
-
- String replicationZNodeName = conf.get("zookeeper.znode.replication", "replication");
- String peersZNodeName = conf.get("zookeeper.znode.replication.peers", "peers");
- String queuesZNodeName = conf.get("zookeeper.znode.replication.rs", "rs");
- String hfileRefsZNodeName = conf.get(ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
- ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT);
- this.peerStateNodeName = conf.get("zookeeper.znode.replication.peers.state", "peer-state");
- this.tableCFsNodeName = conf.get("zookeeper.znode.replication.peers.tableCFs", "tableCFs");
- this.ourClusterKey = ZKConfig.getZooKeeperClusterKey(this.conf);
- this.replicationZNode = ZKUtil.joinZNode(this.zookeeper.znodePaths.baseZNode,
- replicationZNodeName);
- this.peersZNode = ZKUtil.joinZNode(replicationZNode, peersZNodeName);
- this.queuesZNode = ZKUtil.joinZNode(replicationZNode, queuesZNodeName);
- this.hfileRefsZNode = ZKUtil.joinZNode(replicationZNode, hfileRefsZNodeName);
- }
-
- public List<String> getListOfReplicators() {
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenNoWatch(this.zookeeper, this.queuesZNode);
- } catch (KeeperException e) {
- this.abortable.abort("Failed to get list of replicators", e);
- }
- return result;
- }
-
- /**
- * @param state
- * @return Serialized protobuf of <code>state</code> with pb magic prefix prepended suitable for
- * use as content of a peer-state znode under a peer cluster id as in
- * /hbase/replication/peers/PEER_ID/peer-state.
- */
- protected static byte[] toByteArray(final ReplicationProtos.ReplicationState.State state) {
- ReplicationProtos.ReplicationState msg =
- ReplicationProtos.ReplicationState.newBuilder().setState(state).build();
- // There is no toByteArray on this pb Message?
- // 32 bytes is default which seems fair enough here.
- try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
- CodedOutputStream cos = CodedOutputStream.newInstance(baos, 16);
- msg.writeTo(cos);
- cos.flush();
- baos.flush();
- return ProtobufUtil.prependPBMagic(baos.toByteArray());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- protected boolean peerExists(String id) throws KeeperException {
- return ZKUtil.checkExists(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id)) >= 0;
- }
-
- /**
- * Determine if a ZK path points to a peer node.
- * @param path path to be checked
- * @return true if the path points to a peer node, otherwise false
- */
- protected boolean isPeerPath(String path) {
- return path.split("/").length == peersZNode.split("/").length + 1;
- }
-
- @VisibleForTesting
- protected String getTableCFsNode(String id) {
- return ZKUtil.joinZNode(this.peersZNode, ZKUtil.joinZNode(id, this.tableCFsNodeName));
- }
-
- @VisibleForTesting
- protected String getPeerStateNode(String id) {
- return ZKUtil.joinZNode(this.peersZNode, ZKUtil.joinZNode(id, this.peerStateNodeName));
- }
- @VisibleForTesting
- protected String getPeerNode(String id) {
- return ZKUtil.joinZNode(this.peersZNode, id);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
deleted file mode 100644
index fa1d2d6..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTableBase.java
+++ /dev/null
@@ -1,441 +0,0 @@
-/*
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.hadoop.hbase.replication;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.NamespaceDescriptor;
-import org.apache.hadoop.hbase.TableExistsException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.RetryCounter;
-import org.apache.hadoop.hbase.util.RetryCounterFactory;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executor;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/*
- * Abstract class that provides an interface to the Replication Table. Which is currently
- * being used for WAL offset tracking.
- * The basic schema of this table will store each individual queue as a
- * seperate row. The row key will be a unique identifier of the creating server's name and the
- * queueId. Each queue must have the following two columns:
- * COL_QUEUE_OWNER: tracks which server is currently responsible for tracking the queue
- * COL_QUEUE_OWNER_HISTORY: a "|" delimited list of the previous server's that have owned this
- * queue. The most recent previous owner is the leftmost entry.
- * They will also have columns mapping [WAL filename : offset]
- * The most flexible method of interacting with the Replication Table is by calling
- * getOrBlockOnReplicationTable() which will return a new copy of the Replication Table. It is up
- * to the caller to close the returned table.
- */
-@InterfaceAudience.Private
-abstract class ReplicationTableBase {
-
- /** Name of the HBase Table used for tracking replication*/
- public static final TableName REPLICATION_TABLE_NAME =
- TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "replication");
-
- // Column family and column names for Queues in the Replication Table
- public static final byte[] CF_QUEUE = Bytes.toBytes("q");
- public static final byte[] COL_QUEUE_OWNER = Bytes.toBytes("o");
- public static final byte[] COL_QUEUE_OWNER_HISTORY = Bytes.toBytes("h");
-
- // Column Descriptor for the Replication Table
- private static final HColumnDescriptor REPLICATION_COL_DESCRIPTOR =
- new HColumnDescriptor(CF_QUEUE).setMaxVersions(1)
- .setInMemory(true)
- .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
- // TODO: Figure out which bloom filter to use
- .setBloomFilterType(BloomType.NONE);
-
- // The value used to delimit the queueId and server name inside of a queue's row key. Currently a
- // hyphen, because it is guaranteed that queueId (which is a cluster id) cannot contain hyphens.
- // See HBASE-11394.
- public static final String ROW_KEY_DELIMITER = "-";
-
- // The value used to delimit server names in the queue history list
- public static final String QUEUE_HISTORY_DELIMITER = "|";
-
- /*
- * Make sure that HBase table operations for replication have a high number of retries. This is
- * because the server is aborted if any HBase table operation fails. Each RPC will be attempted
- * 3600 times before exiting. This provides each operation with 2 hours of retries
- * before the server is aborted.
- */
- private static final int CLIENT_RETRIES = 3600;
- private static final int RPC_TIMEOUT = 2000;
- private static final int OPERATION_TIMEOUT = CLIENT_RETRIES * RPC_TIMEOUT;
-
- // We only need a single thread to initialize the Replication Table
- private static final int NUM_INITIALIZE_WORKERS = 1;
-
- protected final Configuration conf;
- protected final Abortable abortable;
- private final Connection connection;
- private final Executor executor;
- private volatile CountDownLatch replicationTableInitialized;
-
- public ReplicationTableBase(Configuration conf, Abortable abort) throws IOException {
- this.conf = new Configuration(conf);
- this.abortable = abort;
- decorateConf();
- this.connection = ConnectionFactory.createConnection(this.conf);
- this.executor = setUpExecutor();
- this.replicationTableInitialized = new CountDownLatch(1);
- createReplicationTableInBackground();
- }
-
- /**
- * Modify the connection's config so that operations run on the Replication Table have longer and
- * a larger number of retries
- */
- private void decorateConf() {
- this.conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, CLIENT_RETRIES);
- }
-
- /**
- * Sets up the thread pool executor used to build the Replication Table in the background
- * @return the configured executor
- */
- private Executor setUpExecutor() {
- ThreadPoolExecutor tempExecutor = new ThreadPoolExecutor(NUM_INITIALIZE_WORKERS,
- NUM_INITIALIZE_WORKERS, 100, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>());
- ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
- tfb.setNameFormat("ReplicationTableExecutor-%d");
- tfb.setDaemon(true);
- tempExecutor.setThreadFactory(tfb.build());
- return tempExecutor;
- }
-
- /**
- * Get whether the Replication Table has been successfully initialized yet
- * @return whether the Replication Table is initialized
- */
- public boolean getInitializationStatus() {
- return replicationTableInitialized.getCount() == 0;
- }
-
- /**
- * Increases the RPC and operations timeouts for the Replication Table
- */
- private Table setReplicationTableTimeOuts(Table replicationTable) {
- replicationTable.setRpcTimeout(RPC_TIMEOUT);
- replicationTable.setOperationTimeout(OPERATION_TIMEOUT);
- return replicationTable;
- }
-
- /**
- * Build the row key for the given queueId. This will uniquely identify it from all other queues
- * in the cluster.
- * @param serverName The owner of the queue
- * @param queueId String identifier of the queue
- * @return String representation of the queue's row key
- */
- protected String buildQueueRowKey(String serverName, String queueId) {
- return queueId + ROW_KEY_DELIMITER + serverName;
- }
-
- /**
- * Parse the original queueId from a row key
- * @param rowKey String representation of a queue's row key
- * @return the original queueId
- */
- protected String getRawQueueIdFromRowKey(String rowKey) {
- return rowKey.split(ROW_KEY_DELIMITER)[0];
- }
-
- /**
- * Returns a queue's row key given either its raw or reclaimed queueId
- *
- * @param queueId queueId of the queue
- * @return byte representation of the queue's row key
- */
- protected byte[] queueIdToRowKey(String serverName, String queueId) {
- // Cluster id's are guaranteed to have no hyphens, so if the passed in queueId has no hyphen
- // then this is not a reclaimed queue.
- if (!queueId.contains(ROW_KEY_DELIMITER)) {
- return Bytes.toBytes(buildQueueRowKey(serverName, queueId));
- // If the queueId contained some hyphen it was reclaimed. In this case, the queueId is the
- // queue's row key
- } else {
- return Bytes.toBytes(queueId);
- }
- }
-
- /**
- * Creates a "|" delimited record of the queue's past region server owners.
- *
- * @param originalHistory the queue's original owner history
- * @param oldServer the name of the server that used to own the queue
- * @return the queue's new owner history
- */
- protected String buildClaimedQueueHistory(String originalHistory, String oldServer) {
- return oldServer + QUEUE_HISTORY_DELIMITER + originalHistory;
- }
-
- /**
- * Get a list of all region servers that have outstanding replication queues. These servers could
- * be alive, dead or from a previous run of the cluster.
- * @return a list of server names
- */
- protected List<String> getListOfReplicators() {
- // scan all of the queues and return a list of all unique OWNER values
- Set<String> peerServers = new HashSet<>();
- ResultScanner allQueuesInCluster = null;
- try (Table replicationTable = getOrBlockOnReplicationTable()){
- Scan scan = new Scan();
- scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER);
- allQueuesInCluster = replicationTable.getScanner(scan);
- for (Result queue : allQueuesInCluster) {
- peerServers.add(Bytes.toString(queue.getValue(CF_QUEUE, COL_QUEUE_OWNER)));
- }
- } catch (IOException e) {
- String errMsg = "Failed getting list of replicators";
- abortable.abort(errMsg, e);
- } finally {
- if (allQueuesInCluster != null) {
- allQueuesInCluster.close();
- }
- }
- return new ArrayList<>(peerServers);
- }
-
- protected List<String> getAllQueues(String serverName) {
- List<String> allQueues = new ArrayList<>();
- ResultScanner queueScanner = null;
- try {
- queueScanner = getQueuesBelongingToServer(serverName);
- for (Result queue : queueScanner) {
- String rowKey = Bytes.toString(queue.getRow());
- // If the queue does not have a Owner History, then we must be its original owner. So we
- // want to return its queueId in raw form
- if (Bytes.toString(queue.getValue(CF_QUEUE, COL_QUEUE_OWNER_HISTORY)).length() == 0) {
- allQueues.add(getRawQueueIdFromRowKey(rowKey));
- } else {
- allQueues.add(rowKey);
- }
- }
- return allQueues;
- } catch (IOException e) {
- String errMsg = "Failed getting list of all replication queues for serverName=" + serverName;
- abortable.abort(errMsg, e);
- return null;
- } finally {
- if (queueScanner != null) {
- queueScanner.close();
- }
- }
- }
-
- protected List<String> getLogsInQueue(String serverName, String queueId) {
- String rowKey = queueId;
- if (!queueId.contains(ROW_KEY_DELIMITER)) {
- rowKey = buildQueueRowKey(serverName, queueId);
- }
- return getLogsInQueue(Bytes.toBytes(rowKey));
- }
-
- protected List<String> getLogsInQueue(byte[] rowKey) {
- String errMsg = "Failed getting logs in queue queueId=" + Bytes.toString(rowKey);
- try (Table replicationTable = getOrBlockOnReplicationTable()) {
- Get getQueue = new Get(rowKey);
- Result queue = replicationTable.get(getQueue);
- if (queue == null || queue.isEmpty()) {
- abortable.abort(errMsg, new ReplicationException(errMsg));
- return null;
- }
- return readWALsFromResult(queue);
- } catch (IOException e) {
- abortable.abort(errMsg, e);
- return null;
- }
- }
-
- /**
- * Read all of the WAL's from a queue into a list
- *
- * @param queue HBase query result containing the queue
- * @return a list of all the WAL filenames
- */
- protected List<String> readWALsFromResult(Result queue) {
- List<String> wals = new ArrayList<>();
- Map<byte[], byte[]> familyMap = queue.getFamilyMap(CF_QUEUE);
- for (byte[] cQualifier : familyMap.keySet()) {
- // Ignore the meta data fields of the queue
- if (Arrays.equals(cQualifier, COL_QUEUE_OWNER) || Arrays.equals(cQualifier,
- COL_QUEUE_OWNER_HISTORY)) {
- continue;
- }
- wals.add(Bytes.toString(cQualifier));
- }
- return wals;
- }
-
- /**
- * Get the queue id's and meta data (Owner and History) for the queues belonging to the named
- * server
- *
- * @param server name of the server
- * @return a ResultScanner over the QueueIds belonging to the server
- * @throws IOException
- */
- protected ResultScanner getQueuesBelongingToServer(String server) throws IOException {
- Scan scan = new Scan();
- SingleColumnValueFilter filterMyQueues = new SingleColumnValueFilter(CF_QUEUE, COL_QUEUE_OWNER,
- CompareFilter.CompareOp.EQUAL, Bytes.toBytes(server));
- scan.setFilter(filterMyQueues);
- scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER);
- scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER_HISTORY);
- try (Table replicationTable = getOrBlockOnReplicationTable()) {
- ResultScanner results = replicationTable.getScanner(scan);
- return results;
- }
- }
-
- /**
- * Attempts to acquire the Replication Table. This operation will block until it is assigned by
- * the CreateReplicationWorker thread. It is up to the caller of this method to close the
- * returned Table
- * @return the Replication Table when it is created
- * @throws IOException
- */
- protected Table getOrBlockOnReplicationTable() throws IOException {
- // Sleep until the Replication Table becomes available
- try {
- replicationTableInitialized.await();
- } catch (InterruptedException e) {
- String errMsg = "Unable to acquire the Replication Table due to InterruptedException: " +
- e.getMessage();
- throw new InterruptedIOException(errMsg);
- }
- return getAndSetUpReplicationTable();
- }
-
- /**
- * Creates a new copy of the Replication Table and sets up the proper Table time outs for it
- *
- * @return the Replication Table
- * @throws IOException
- */
- private Table getAndSetUpReplicationTable() throws IOException {
- Table replicationTable = connection.getTable(REPLICATION_TABLE_NAME);
- setReplicationTableTimeOuts(replicationTable);
- return replicationTable;
- }
-
- /**
- * Builds the Replication Table in a background thread. Any method accessing the Replication Table
- * should do so through getOrBlockOnReplicationTable()
- *
- * @return the Replication Table
- * @throws IOException if the Replication Table takes too long to build
- */
- private void createReplicationTableInBackground() throws IOException {
- executor.execute(new CreateReplicationTableWorker());
- }
-
- /**
- * Attempts to build the Replication Table. Will continue blocking until we have a valid
- * Table for the Replication Table.
- */
- private class CreateReplicationTableWorker implements Runnable {
-
- private Admin admin;
-
- @Override
- public void run() {
- try {
- admin = connection.getAdmin();
- if (!replicationTableExists()) {
- createReplicationTable();
- }
- int maxRetries = conf.getInt("hbase.replication.queues.createtable.retries.number",
- CLIENT_RETRIES);
- RetryCounterFactory counterFactory = new RetryCounterFactory(maxRetries, RPC_TIMEOUT);
- RetryCounter retryCounter = counterFactory.create();
- while (!replicationTableExists()) {
- retryCounter.sleepUntilNextRetry();
- if (!retryCounter.shouldRetry()) {
- throw new IOException("Unable to acquire the Replication Table");
- }
- }
- replicationTableInitialized.countDown();
- } catch (IOException | InterruptedException e) {
- abortable.abort("Failed building Replication Table", e);
- }
- }
-
- /**
- * Create the replication table with the provided HColumnDescriptor REPLICATION_COL_DESCRIPTOR
- * in TableBasedReplicationQueuesImpl
- *
- * @throws IOException
- */
- private void createReplicationTable() throws IOException {
- HTableDescriptor replicationTableDescriptor = new HTableDescriptor(REPLICATION_TABLE_NAME);
- replicationTableDescriptor.addFamily(REPLICATION_COL_DESCRIPTOR);
- try {
- admin.createTable(replicationTableDescriptor);
- } catch (TableExistsException e) {
- // In this case we can just continue as normal
- }
- }
-
- /**
- * Checks whether the Replication Table exists yet
- *
- * @return whether the Replication Table exists
- * @throws IOException
- */
- private boolean replicationTableExists() {
- try {
- return admin.tableExists(REPLICATION_TABLE_NAME);
- } catch (IOException e) {
- return false;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java
deleted file mode 100644
index 51d7473..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTracker.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.List;
-
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-/**
- * This is the interface for a Replication Tracker. A replication tracker provides the facility to
- * subscribe and track events that reflect a change in replication state. These events are used by
- * the ReplicationSourceManager to coordinate replication tasks such as addition/deletion of queues
- * and queue failover. These events are defined in the ReplicationListener interface. If a class
- * would like to listen to replication events it must implement the ReplicationListener interface
- * and register itself with a Replication Tracker.
- */
-@InterfaceAudience.Private
-public interface ReplicationTracker {
-
- /**
- * Register a replication listener to receive replication events.
- * @param listener
- */
- public void registerListener(ReplicationListener listener);
-
- public void removeListener(ReplicationListener listener);
-
- /**
- * Returns a list of other live region servers in the cluster.
- * @return List of region servers.
- */
- public List<String> getListOfRegionServers();
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
deleted file mode 100644
index 9865d83..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationTrackerZKImpl.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.Stoppable;
-import org.apache.hadoop.hbase.zookeeper.ZKUtil;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperListener;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
-
-/**
- * This class is a ZooKeeper implementation of the ReplicationTracker interface. This class is
- * responsible for handling replication events that are defined in the ReplicationListener
- * interface.
- */
-@InterfaceAudience.Private
-public class ReplicationTrackerZKImpl extends ReplicationStateZKBase implements ReplicationTracker {
-
- private static final Log LOG = LogFactory.getLog(ReplicationTrackerZKImpl.class);
- // All about stopping
- private final Stoppable stopper;
- // listeners to be notified
- private final List<ReplicationListener> listeners = new CopyOnWriteArrayList<>();
- // List of all the other region servers in this cluster
- private final ArrayList<String> otherRegionServers = new ArrayList<>();
- private final ReplicationPeers replicationPeers;
-
- public ReplicationTrackerZKImpl(ZooKeeperWatcher zookeeper,
- final ReplicationPeers replicationPeers, Configuration conf, Abortable abortable,
- Stoppable stopper) {
- super(zookeeper, conf, abortable);
- this.replicationPeers = replicationPeers;
- this.stopper = stopper;
- this.zookeeper.registerListener(new OtherRegionServerWatcher(this.zookeeper));
- this.zookeeper.registerListener(new PeersWatcher(this.zookeeper));
- }
-
- @Override
- public void registerListener(ReplicationListener listener) {
- listeners.add(listener);
- }
-
- @Override
- public void removeListener(ReplicationListener listener) {
- listeners.remove(listener);
- }
-
- /**
- * Return a snapshot of the current region servers.
- */
- @Override
- public List<String> getListOfRegionServers() {
- refreshOtherRegionServersList();
-
- List<String> list = null;
- synchronized (otherRegionServers) {
- list = new ArrayList<>(otherRegionServers);
- }
- return list;
- }
-
- /**
- * Watcher used to be notified of the other region server's death in the local cluster. It
- * initiates the process to transfer the queues if it is able to grab the lock.
- */
- public class OtherRegionServerWatcher extends ZooKeeperListener {
-
- /**
- * Construct a ZooKeeper event listener.
- */
- public OtherRegionServerWatcher(ZooKeeperWatcher watcher) {
- super(watcher);
- }
-
- /**
- * Called when a new node has been created.
- * @param path full path of the new node
- */
- public void nodeCreated(String path) {
- refreshListIfRightPath(path);
- }
-
- /**
- * Called when a node has been deleted
- * @param path full path of the deleted node
- */
- public void nodeDeleted(String path) {
- if (stopper.isStopped()) {
- return;
- }
- boolean cont = refreshListIfRightPath(path);
- if (!cont) {
- return;
- }
- LOG.info(path + " znode expired, triggering replicatorRemoved event");
- for (ReplicationListener rl : listeners) {
- rl.regionServerRemoved(getZNodeName(path));
- }
- }
-
- /**
- * Called when an existing node has a child node added or removed.
- * @param path full path of the node whose children have changed
- */
- public void nodeChildrenChanged(String path) {
- if (stopper.isStopped()) {
- return;
- }
- refreshListIfRightPath(path);
- }
-
- private boolean refreshListIfRightPath(String path) {
- if (!path.startsWith(this.watcher.znodePaths.rsZNode)) {
- return false;
- }
- return refreshOtherRegionServersList();
- }
- }
-
- /**
- * Watcher used to follow the creation and deletion of peer clusters.
- */
- public class PeersWatcher extends ZooKeeperListener {
-
- /**
- * Construct a ZooKeeper event listener.
- */
- public PeersWatcher(ZooKeeperWatcher watcher) {
- super(watcher);
- }
-
- /**
- * Called when a node has been deleted
- * @param path full path of the deleted node
- */
- public void nodeDeleted(String path) {
- List<String> peers = refreshPeersList(path);
- if (peers == null) {
- return;
- }
- if (isPeerPath(path)) {
- String id = getZNodeName(path);
- LOG.info(path + " znode expired, triggering peerRemoved event");
- for (ReplicationListener rl : listeners) {
- rl.peerRemoved(id);
- }
- }
- }
-
- /**
- * Called when an existing node has a child node added or removed.
- * @param path full path of the node whose children have changed
- */
- public void nodeChildrenChanged(String path) {
- List<String> peers = refreshPeersList(path);
- if (peers == null) {
- return;
- }
- LOG.info(path + " znode expired, triggering peerListChanged event");
- for (ReplicationListener rl : listeners) {
- rl.peerListChanged(peers);
- }
- }
- }
-
- /**
- * Verify if this event is meant for us, and if so then get the latest peers' list from ZK. Also
- * reset the watches.
- * @param path path to check against
- * @return A list of peers' identifiers if the event concerns this watcher, else null.
- */
- private List<String> refreshPeersList(String path) {
- if (!path.startsWith(getPeersZNode())) {
- return null;
- }
- return this.replicationPeers.getAllPeerIds();
- }
-
- private String getPeersZNode() {
- return this.peersZNode;
- }
-
- /**
- * Extracts the znode name of a peer cluster from a ZK path
- * @param fullPath Path to extract the id from
- * @return the id or an empty string if path is invalid
- */
- private String getZNodeName(String fullPath) {
- String[] parts = fullPath.split("/");
- return parts.length > 0 ? parts[parts.length - 1] : "";
- }
-
- /**
- * Reads the list of region servers from ZK and atomically clears our local view of it and
- * replaces it with the updated list.
- * @return true if the local list of the other region servers was updated with the ZK data (even
- * if it was empty), false if the data was missing in ZK
- */
- private boolean refreshOtherRegionServersList() {
- List<String> newRsList = getRegisteredRegionServers();
- if (newRsList == null) {
- return false;
- } else {
- synchronized (otherRegionServers) {
- otherRegionServers.clear();
- otherRegionServers.addAll(newRsList);
- }
- }
- return true;
- }
-
- /**
- * Get a list of all the other region servers in this cluster and set a watch
- * @return a list of server nanes
- */
- private List<String> getRegisteredRegionServers() {
- List<String> result = null;
- try {
- result = ZKUtil.listChildrenAndWatchThem(this.zookeeper, this.zookeeper.znodePaths.rsZNode);
- } catch (KeeperException e) {
- this.abortable.abort("Get list of registered region servers", e);
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesClientImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesClientImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesClientImpl.java
deleted file mode 100644
index 3507547..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesClientImpl.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.replication;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.zookeeper.KeeperException;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Implements the ReplicationQueuesClient interface on top of the Replication Table. It utilizes
- * the ReplicationTableBase to access the Replication Table.
- */
-@InterfaceAudience.Private
-public class TableBasedReplicationQueuesClientImpl extends ReplicationTableBase
- implements ReplicationQueuesClient {
-
- public TableBasedReplicationQueuesClientImpl(ReplicationQueuesClientArguments args)
- throws IOException {
- super(args.getConf(), args.getAbortable());
- }
- public TableBasedReplicationQueuesClientImpl(Configuration conf,
- Abortable abortable) throws IOException {
- super(conf, abortable);
- }
-
- @Override
- public void init() throws ReplicationException{
- // no-op
- }
-
- @Override
- public List<String> getListOfReplicators() {
- return super.getListOfReplicators();
- }
-
- @Override
- public List<String> getLogsInQueue(String serverName, String queueId) {
- return super.getLogsInQueue(serverName, queueId);
- }
-
- @Override
- public List<String> getAllQueues(String serverName) {
- return super.getAllQueues(serverName);
- }
-
- @Override
- public Set<String> getAllWALs() {
- Set<String> allWals = new HashSet<>();
- ResultScanner allQueues = null;
- try (Table replicationTable = getOrBlockOnReplicationTable()) {
- allQueues = replicationTable.getScanner(new Scan());
- for (Result queue : allQueues) {
- for (String wal : readWALsFromResult(queue)) {
- allWals.add(wal);
- }
- }
- } catch (IOException e) {
- String errMsg = "Failed getting all WAL's in Replication Table";
- abortable.abort(errMsg, e);
- } finally {
- if (allQueues != null) {
- allQueues.close();
- }
- }
- return allWals;
- }
-
- @Override
- public int getHFileRefsNodeChangeVersion() throws KeeperException {
- // TODO
- throw new NotImplementedException();
- }
-
- @Override
- public List<String> getAllPeersFromHFileRefsQueue() throws KeeperException {
- // TODO
- throw new NotImplementedException();
- }
-
- @Override
- public List<String> getReplicableHFiles(String peerId) throws KeeperException {
- // TODO
- throw new NotImplementedException();
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java
deleted file mode 100644
index bf55e8c..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/TableBasedReplicationQueuesImpl.java
+++ /dev/null
@@ -1,450 +0,0 @@
-/*
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.hadoop.hbase.replication;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.Abortable;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.RowMutations;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Pair;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.zookeeper.KeeperException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-/**
- * This class provides an implementation of the ReplicationQueues interface using an HBase table
- * "Replication Table". It utilizes the ReplicationTableBase to access the Replication Table.
- */
-@InterfaceAudience.Private
-public class TableBasedReplicationQueuesImpl extends ReplicationTableBase
- implements ReplicationQueues {
-
- private static final Log LOG = LogFactory.getLog(TableBasedReplicationQueuesImpl.class);
-
- // Common byte values used in replication offset tracking
- private static final byte[] INITIAL_OFFSET_BYTES = Bytes.toBytes(0L);
- private static final byte[] EMPTY_STRING_BYTES = Bytes.toBytes("");
-
- private String serverName = null;
- private byte[] serverNameBytes = null;
-
- // TODO: Only use this variable temporarily. Eventually we want to use HBase to store all
- // TODO: replication information
- private ReplicationStateZKBase replicationState;
-
- public TableBasedReplicationQueuesImpl(ReplicationQueuesArguments args) throws IOException {
- this(args.getConf(), args.getAbortable(), args.getZk());
- }
-
- public TableBasedReplicationQueuesImpl(Configuration conf, Abortable abort, ZooKeeperWatcher zkw)
- throws IOException {
- super(conf, abort);
- replicationState = new ReplicationStateZKBase(zkw, conf, abort) {};
- }
-
- @Override
- public void init(String serverName) throws ReplicationException {
- this.serverName = serverName;
- this.serverNameBytes = Bytes.toBytes(serverName);
- }
-
- @Override
- public List<String> getListOfReplicators() {
- return super.getListOfReplicators();
- }
-
- @Override
- public void removeQueue(String queueId) {
- try {
- byte[] rowKey = queueIdToRowKey(queueId);
- if (checkQueueExists(queueId)) {
- Delete deleteQueue = new Delete(rowKey);
- safeQueueUpdate(deleteQueue);
- } else {
- LOG.info("No logs were registered for queue id=" + queueId + " so no rows were removed " +
- "from the replication table while removing the queue");
- }
- } catch (IOException | ReplicationException e) {
- String errMsg = "Failed removing queue queueId=" + queueId;
- abortable.abort(errMsg, e);
- }
- }
-
- @Override
- public void addLog(String queueId, String filename) throws ReplicationException {
- try (Table replicationTable = getOrBlockOnReplicationTable()) {
- if (!checkQueueExists(queueId)) {
- // Each queue will have an Owner, OwnerHistory, and a collection of [WAL:offset] key values
- Put putNewQueue = new Put(Bytes.toBytes(buildQueueRowKey(queueId)));
- putNewQueue.addColumn(CF_QUEUE, COL_QUEUE_OWNER, serverNameBytes);
- putNewQueue.addColumn(CF_QUEUE, COL_QUEUE_OWNER_HISTORY, EMPTY_STRING_BYTES);
- putNewQueue.addColumn(CF_QUEUE, Bytes.toBytes(filename), INITIAL_OFFSET_BYTES);
- replicationTable.put(putNewQueue);
- } else {
- // Otherwise simply add the new log and offset as a new column
- Put putNewLog = new Put(queueIdToRowKey(queueId));
- putNewLog.addColumn(CF_QUEUE, Bytes.toBytes(filename), INITIAL_OFFSET_BYTES);
- safeQueueUpdate(putNewLog);
- }
- } catch (IOException | ReplicationException e) {
- String errMsg = "Failed adding log queueId=" + queueId + " filename=" + filename;
- abortable.abort(errMsg, e);
- }
- }
-
- @Override
- public void removeLog(String queueId, String filename) {
- try {
- byte[] rowKey = queueIdToRowKey(queueId);
- Delete delete = new Delete(rowKey);
- delete.addColumns(CF_QUEUE, Bytes.toBytes(filename));
- safeQueueUpdate(delete);
- } catch (IOException | ReplicationException e) {
- String errMsg = "Failed removing log queueId=" + queueId + " filename=" + filename;
- abortable.abort(errMsg, e);
- }
- }
-
- @Override
- public void setLogPosition(String queueId, String filename, long position) {
- try (Table replicationTable = getOrBlockOnReplicationTable()) {
- byte[] rowKey = queueIdToRowKey(queueId);
- // Check that the log exists. addLog() must have been called before setLogPosition().
- Get checkLogExists = new Get(rowKey);
- checkLogExists.addColumn(CF_QUEUE, Bytes.toBytes(filename));
- if (!replicationTable.exists(checkLogExists)) {
- String errMsg = "Could not set position of non-existent log from queueId=" + queueId +
- ", filename=" + filename;
- abortable.abort(errMsg, new ReplicationException(errMsg));
- return;
- }
- // Update the log offset if it exists
- Put walAndOffset = new Put(rowKey);
- walAndOffset.addColumn(CF_QUEUE, Bytes.toBytes(filename), Bytes.toBytes(position));
- safeQueueUpdate(walAndOffset);
- } catch (IOException | ReplicationException e) {
- String errMsg = "Failed writing log position queueId=" + queueId + "filename=" +
- filename + " position=" + position;
- abortable.abort(errMsg, e);
- }
- }
-
- @Override
- public long getLogPosition(String queueId, String filename) throws ReplicationException {
- try {
- byte[] rowKey = queueIdToRowKey(queueId);
- Get getOffset = new Get(rowKey);
- getOffset.addColumn(CF_QUEUE, Bytes.toBytes(filename));
- Result result = getResultIfOwner(getOffset);
- if (result == null || !result.containsColumn(CF_QUEUE, Bytes.toBytes(filename))) {
- throw new ReplicationException("Could not read empty result while getting log position " +
- "queueId=" + queueId + ", filename=" + filename);
- }
- return Bytes.toLong(result.getValue(CF_QUEUE, Bytes.toBytes(filename)));
- } catch (IOException e) {
- throw new ReplicationException("Could not get position in log for queueId=" + queueId +
- ", filename=" + filename);
- }
- }
-
- @Override
- public void removeAllQueues() {
- List<String> myQueueIds = getAllQueues();
- for (String queueId : myQueueIds) {
- removeQueue(queueId);
- }
- }
-
- @Override
- public List<String> getLogsInQueue(String queueId) {
- String errMsg = "Failed getting logs in queue queueId=" + queueId;
- byte[] rowKey = queueIdToRowKey(queueId);
- List<String> logs = new ArrayList<>();
- try {
- Get getQueue = new Get(rowKey);
- Result queue = getResultIfOwner(getQueue);
- if (queue == null || queue.isEmpty()) {
- String errMsgLostOwnership = "Failed getting logs for queue queueId=" +
- Bytes.toString(rowKey) + " because the queue was missing or we lost ownership";
- abortable.abort(errMsg, new ReplicationException(errMsgLostOwnership));
- return null;
- }
- Map<byte[], byte[]> familyMap = queue.getFamilyMap(CF_QUEUE);
- for(byte[] cQualifier : familyMap.keySet()) {
- if (Arrays.equals(cQualifier, COL_QUEUE_OWNER) || Arrays.equals(cQualifier,
- COL_QUEUE_OWNER_HISTORY)) {
- continue;
- }
- logs.add(Bytes.toString(cQualifier));
- }
- } catch (IOException e) {
- abortable.abort(errMsg, e);
- return null;
- }
- return logs;
- }
-
- @Override
- public List<String> getAllQueues() {
- return getAllQueues(serverName);
- }
-
- @Override public List<String> getUnClaimedQueueIds(String regionserver) {
- if (isThisOurRegionServer(regionserver)) {
- return null;
- }
- try (ResultScanner queuesToClaim = getQueuesBelongingToServer(regionserver)) {
- List<String> res = new ArrayList<>();
- for (Result queue : queuesToClaim) {
- String rowKey = Bytes.toString(queue.getRow());
- res.add(rowKey);
- }
- return res.isEmpty() ? null : res;
- } catch (IOException e) {
- String errMsg = "Failed getUnClaimedQueueIds";
- abortable.abort(errMsg, e);
- }
- return null;
- }
-
- @Override public void removeReplicatorIfQueueIsEmpty(String regionserver) {
- // Do nothing here
- }
-
- @Override
- public Pair<String, SortedSet<String>> claimQueue(String regionserver, String queueId) {
- if (isThisOurRegionServer(regionserver)) {
- return null;
- }
-
- try (ResultScanner queuesToClaim = getQueuesBelongingToServer(regionserver)){
- for (Result queue : queuesToClaim) {
- String rowKey = Bytes.toString(queue.getRow());
- if (!rowKey.equals(queueId)){
- continue;
- }
- if (attemptToClaimQueue(queue, regionserver)) {
- ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(rowKey);
- if (replicationState.peerExists(replicationQueueInfo.getPeerId())) {
- SortedSet<String> sortedLogs = new TreeSet<>();
- List<String> logs = getLogsInQueue(queue.getRow());
- for (String log : logs) {
- sortedLogs.add(log);
- }
- LOG.info(serverName + " has claimed queue " + rowKey + " from " + regionserver);
- return new Pair<>(rowKey, sortedLogs);
- } else {
- // Delete orphaned queues
- removeQueue(Bytes.toString(queue.getRow()));
- LOG.info(serverName + " has deleted abandoned queue " + queueId + " from " +
- regionserver);
- }
- }
- }
- } catch (IOException | KeeperException e) {
- String errMsg = "Failed claiming queues for regionserver=" + regionserver;
- abortable.abort(errMsg, e);
- }
- return null;
- }
-
- @Override
- public boolean isThisOurRegionServer(String regionserver) {
- return this.serverName.equals(regionserver);
- }
-
- @Override
- public void addPeerToHFileRefs(String peerId) throws ReplicationException {
- // TODO
- throw new NotImplementedException();
- }
-
- @Override
- public void removePeerFromHFileRefs(String peerId) {
- // TODO
- throw new NotImplementedException();
- }
-
- @Override
- public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs)
- throws ReplicationException {
- // TODO
- throw new NotImplementedException();
- }
-
- @Override
- public void removeHFileRefs(String peerId, List<String> files) {
- // TODO
- throw new NotImplementedException();
- }
-
- private String buildQueueRowKey(String queueId) {
- return buildQueueRowKey(serverName, queueId);
- }
-
- /**
- * Convenience method that gets the row key of the queue specified by queueId
- * @param queueId queueId of a queue in this server
- * @return the row key of the queue in the Replication Table
- */
- private byte[] queueIdToRowKey(String queueId) {
- return queueIdToRowKey(serverName, queueId);
- }
-
- /**
- * See safeQueueUpdate(RowMutations mutate)
- *
- * @param put Row mutation to perform on the queue
- */
- private void safeQueueUpdate(Put put) throws ReplicationException, IOException {
- RowMutations mutations = new RowMutations(put.getRow());
- mutations.add(put);
- safeQueueUpdate(mutations);
- }
-
- /**
- * See safeQueueUpdate(RowMutations mutate)
- *
- * @param delete Row mutation to perform on the queue
- */
- private void safeQueueUpdate(Delete delete) throws ReplicationException,
- IOException{
- RowMutations mutations = new RowMutations(delete.getRow());
- mutations.add(delete);
- safeQueueUpdate(mutations);
- }
-
- /**
- * Attempt to mutate a given queue in the Replication Table with a checkAndPut on the OWNER column
- * of the queue. Abort the server if this checkAndPut fails: which means we have somehow lost
- * ownership of the column or an IO Exception has occurred during the transaction.
- *
- * @param mutate Mutation to perform on a given queue
- */
- private void safeQueueUpdate(RowMutations mutate) throws ReplicationException, IOException{
- try (Table replicationTable = getOrBlockOnReplicationTable()) {
- boolean updateSuccess = replicationTable.checkAndMutate(mutate.getRow(),
- CF_QUEUE, COL_QUEUE_OWNER, CompareFilter.CompareOp.EQUAL, serverNameBytes, mutate);
- if (!updateSuccess) {
- throw new ReplicationException("Failed to update Replication Table because we lost queue " +
- " ownership");
- }
- }
- }
-
- /**
- * Check if the queue specified by queueId is stored in HBase
- *
- * @param queueId Either raw or reclaimed format of the queueId
- * @return Whether the queue is stored in HBase
- * @throws IOException
- */
- private boolean checkQueueExists(String queueId) throws IOException {
- try (Table replicationTable = getOrBlockOnReplicationTable()) {
- byte[] rowKey = queueIdToRowKey(queueId);
- return replicationTable.exists(new Get(rowKey));
- }
- }
-
- /**
- * Attempt to claim the given queue with a checkAndPut on the OWNER column. We check that the
- * recently killed server is still the OWNER before we claim it.
- *
- * @param queue The queue that we are trying to claim
- * @param originalServer The server that originally owned the queue
- * @return Whether we successfully claimed the queue
- * @throws IOException
- */
- private boolean attemptToClaimQueue (Result queue, String originalServer) throws IOException{
- Put putQueueNameAndHistory = new Put(queue.getRow());
- putQueueNameAndHistory.addColumn(CF_QUEUE, COL_QUEUE_OWNER, Bytes.toBytes(serverName));
- String newOwnerHistory = buildClaimedQueueHistory(Bytes.toString(queue.getValue(CF_QUEUE,
- COL_QUEUE_OWNER_HISTORY)), originalServer);
- putQueueNameAndHistory.addColumn(CF_QUEUE, COL_QUEUE_OWNER_HISTORY,
- Bytes.toBytes(newOwnerHistory));
- RowMutations claimAndRenameQueue = new RowMutations(queue.getRow());
- claimAndRenameQueue.add(putQueueNameAndHistory);
- // Attempt to claim ownership for this queue by checking if the current OWNER is the original
- // server. If it is not then another RS has already claimed it. If it is we set ourselves as the
- // new owner and update the queue's history
- try (Table replicationTable = getOrBlockOnReplicationTable()) {
- boolean success = replicationTable.checkAndMutate(queue.getRow(),
- CF_QUEUE, COL_QUEUE_OWNER, CompareFilter.CompareOp.EQUAL, Bytes.toBytes(originalServer),
- claimAndRenameQueue);
- return success;
- }
- }
-
- /**
- * Attempts to run a Get on some queue. Will only return a non-null result if we currently own
- * the queue.
- *
- * @param get The Get that we want to query
- * @return The result of the Get if this server is the owner of the queue. Else it returns null.
- * @throws IOException
- */
- private Result getResultIfOwner(Get get) throws IOException {
- Scan scan = new Scan(get);
- // Check if the Get currently contains all columns or only specific columns
- if (scan.getFamilyMap().size() > 0) {
- // Add the OWNER column if the scan is already only over specific columns
- scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER);
- }
- scan.setMaxResultSize(1);
- SingleColumnValueFilter checkOwner = new SingleColumnValueFilter(CF_QUEUE, COL_QUEUE_OWNER,
- CompareFilter.CompareOp.EQUAL, serverNameBytes);
- scan.setFilter(checkOwner);
- ResultScanner scanner = null;
- try (Table replicationTable = getOrBlockOnReplicationTable()) {
- scanner = replicationTable.getScanner(scan);
- Result result = scanner.next();
- return (result == null || result.isEmpty()) ? null : result;
- } finally {
- if (scanner != null) {
- scanner.close();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index dafe421..1de3652 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.replication.ReplicationStateZKBase;
import org.apache.hadoop.hbase.security.Superusers;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
@@ -1772,23 +1771,18 @@ public class ZKUtil {
*/
private static void getReplicationZnodesDump(ZooKeeperWatcher zkw, StringBuilder sb)
throws KeeperException {
- String replicationZNodeName = zkw.getConfiguration().get("zookeeper.znode.replication",
- "replication");
- String replicationZnode = joinZNode(zkw.znodePaths.baseZNode, replicationZNodeName);
+ String replicationZnode = zkw.znodePaths.replicationZNode;
if (ZKUtil.checkExists(zkw, replicationZnode) == -1) return;
// do a ls -r on this znode
sb.append("\n").append(replicationZnode).append(": ");
List<String> children = ZKUtil.listChildrenNoWatch(zkw, replicationZnode);
for (String child : children) {
String znode = joinZNode(replicationZnode, child);
- if (child.equals(zkw.getConfiguration().get("zookeeper.znode.replication.peers", "peers"))) {
+ if (znode.equals(zkw.znodePaths.peersZNode)) {
appendPeersZnodes(zkw, znode, sb);
- } else if (child.equals(zkw.getConfiguration().
- get("zookeeper.znode.replication.rs", "rs"))) {
+ } else if (znode.equals(zkw.znodePaths.queuesZNode)) {
appendRSZnodes(zkw, znode, sb);
- } else if (child.equals(zkw.getConfiguration().get(
- ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_KEY,
- ReplicationStateZKBase.ZOOKEEPER_ZNODE_REPLICATION_HFILE_REFS_DEFAULT))) {
+ } else if (znode.equals(zkw.znodePaths.hfileRefsZNode)) {
appendHFileRefsZnodes(zkw, znode, sb);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26e6c2ce/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
index 4c66b8f..5608eb5 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZNodePaths.java
@@ -79,6 +79,15 @@ public class ZNodePaths {
// znode of indicating master maintenance mode
public final String masterMaintZNode;
+ // znode containing all replication state.
+ public final String replicationZNode;
+ // znode containing a list of all remote slave (i.e. peer) clusters.
+ public final String peersZNode;
+ // znode containing all replication queues
+ public final String queuesZNode;
+ // znode containing queues of hfile references to be replicated
+ public final String hfileRefsZNode;
+
public ZNodePaths(Configuration conf) {
baseZNode = conf.get(ZOOKEEPER_ZNODE_PARENT, DEFAULT_ZOOKEEPER_ZNODE_PARENT);
ImmutableMap.Builder<Integer, String> builder = ImmutableMap.builder();
@@ -113,6 +122,15 @@ public class ZNodePaths {
conf.get("zookeeper.znode.namespace", "namespace"));
masterMaintZNode = ZKUtil.joinZNode(baseZNode,
conf.get("zookeeper.znode.masterMaintenance", "master-maintenance"));
+ replicationZNode =
+ ZKUtil.joinZNode(baseZNode, conf.get("zookeeper.znode.replication", "replication"));
+ peersZNode =
+ ZKUtil.joinZNode(replicationZNode, conf.get("zookeeper.znode.replication.peers", "peers"));
+ queuesZNode =
+ ZKUtil.joinZNode(replicationZNode, conf.get("zookeeper.znode.replication.rs", "rs"));
+ hfileRefsZNode =
+ ZKUtil.joinZNode(replicationZNode,
+ conf.get("zookeeper.znode.replication.hfile.refs", "hfile-refs"));
}
@Override
@@ -125,7 +143,9 @@ public class ZNodePaths {
+ ", balancerZNode=" + balancerZNode + ", regionNormalizerZNode=" + regionNormalizerZNode
+ ", switchZNode=" + switchZNode + ", tableLockZNode=" + tableLockZNode
+ ", recoveringRegionsZNode=" + recoveringRegionsZNode + ", namespaceZNode="
- + namespaceZNode + ", masterMaintZNode=" + masterMaintZNode + "]";
+ + namespaceZNode + ", masterMaintZNode=" + masterMaintZNode + ", replicationZNode="
+ + replicationZNode + ", peersZNode=" + peersZNode + ", queuesZNode=" + queuesZNode
+ + ", hfileRefsZNode=" + hfileRefsZNode + "]";
}
/**