You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2017/02/10 22:54:13 UTC
hbase git commit: HBASE-17460 enable_table_replication can not
perform cyclic replication of a table (NITIN VERMA)
Repository: hbase
Updated Branches:
refs/heads/master 054acec84 -> d51e28787
HBASE-17460 enable_table_replication can not perform cyclic replication of a table (NITIN VERMA)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d51e2878
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d51e2878
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d51e2878
Branch: refs/heads/master
Commit: d51e2878774ebb6107290476264a6612bc3b7dea
Parents: 054acec
Author: tedyu <yu...@gmail.com>
Authored: Fri Feb 10 14:54:05 2017 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Fri Feb 10 14:54:05 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/HColumnDescriptor.java | 2 +-
.../apache/hadoop/hbase/HTableDescriptor.java | 102 ++++++++++++++++++-
.../apache/hadoop/hbase/client/HBaseAdmin.java | 31 ++++--
.../client/replication/ReplicationAdmin.java | 8 +-
4 files changed, 128 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d51e2878/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
index 1597a06..6d1ae3f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
@@ -1076,7 +1076,7 @@ public class HColumnDescriptor implements Comparable<HColumnDescriptor> {
public int compareTo(HColumnDescriptor o) {
int result = Bytes.compareTo(this.name, o.getName());
if (result == 0) {
- // punt on comparison for ordering, just calculate difference
+ // punt on comparison for ordering, just calculate difference.
result = this.values.hashCode() - o.values.hashCode();
if (result < 0)
result = -1;
http://git-wip-us.apache.org/repos/asf/hbase/blob/d51e2878/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index 60b85fe..a81c82a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -40,9 +40,9 @@ import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.TableSchema;
-import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.util.Bytes;
/**
@@ -1042,6 +1042,106 @@ public class HTableDescriptor implements Comparable<HTableDescriptor> {
}
/**
+ * Detects whether replication has been already enabled on any of the column families of this
+ * table descriptor.
+ * @return true if any of the column families has replication enabled.
+ */
+ public boolean isReplicationEnabled() {
+ // Go through each Column-Family descriptor and check if the
+ // Replication has been enabled already.
+ // Return 'true' if replication has been enabled on any CF,
+ // otherwise return 'false'.
+ //
+ boolean result = false;
+ Iterator<HColumnDescriptor> it = this.families.values().iterator();
+
+ while (it.hasNext()) {
+ HColumnDescriptor tempHcd = it.next();
+ if (tempHcd.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) {
+ result = true;
+ break;
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * Compare the contents of the descriptor with another one passed as a parameter for replication
+ * purpose. The REPLICATION_SCOPE field is ignored during comparison.
+ * @param obj descriptor on source cluster which needs to be replicated.
+ * @return true if the contents of the two descriptors match (ignoring just REPLICATION_SCOPE).
+ * @see java.lang.Object#equals(java.lang.Object)
+ */
+ public boolean compareForReplication(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof HTableDescriptor)) {
+ return false;
+ }
+
+ boolean result = false;
+
+ // Create a copy of peer HTD as we need to change its replication
+ // scope to match with the local HTD.
+ HTableDescriptor peerHtdCopy = new HTableDescriptor(this);
+
+ // Copy the replication scope of local Htd to remote Htd.
+ HTableDescriptor localHtd = (HTableDescriptor) obj;
+
+ result = (peerHtdCopy.copyReplicationScope(localHtd) == 0);
+
+ // If copy was successful, compare the two tables now.
+ if (result == true) {
+ result = (peerHtdCopy.compareTo(localHtd) == 0);
+ }
+
+ return result;
+ }
+
+ /**
+ * Copies the REPLICATION_SCOPE of table descriptor passed as an argument. Before copy, the method
+ * ensures that the name of table and column-families should match.
+ * @param localHtd - The HTableDescriptor of table from source cluster.
+ * @return 0 If the name of table and column families match and REPLICATION_SCOPE copied
+ * successfully. 1 If there is any mismatch in the names.
+ */
+ public int copyReplicationScope(final HTableDescriptor localHtd)
+ {
+ // Copy the REPLICATION_SCOPE only when table names and the names of
+ // Column-Families are same.
+ int result = this.name.compareTo(localHtd.name);
+
+ if (result == 0) {
+ Iterator<HColumnDescriptor> remoteHCDIter = families.values().iterator();
+ Iterator<HColumnDescriptor> localHCDIter = localHtd.families.values().iterator();
+
+ while (remoteHCDIter.hasNext() && localHCDIter.hasNext()) {
+ HColumnDescriptor remoteHCD = remoteHCDIter.next();
+ HColumnDescriptor localHCD = localHCDIter.next();
+
+ String remoteHCDName = remoteHCD.getNameAsString();
+ String localHCDName = localHCD.getNameAsString();
+
+ if (remoteHCDName.equals(localHCDName))
+ {
+ remoteHCD.setScope(localHCD.getScope());
+ }
+ else {
+ result = -1;
+ break;
+ }
+ }
+ }
+
+ return result;
+ }
+
+ /**
* @see java.lang.Object#hashCode()
*/
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/d51e2878/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 65070b9..31b959f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -29,7 +29,6 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
@@ -4087,7 +4086,9 @@ public class HBaseAdmin implements Admin {
* Connect to peer and check the table descriptor on peer:
* <ol>
* <li>Create the same table on peer when not exist.</li>
- * <li>Throw exception if the table exists on peer cluster but descriptors are not same.</li>
+ * <li>Throw an exception if the table already has replication enabled on any of the column
+ * families.</li>
+ * <li>Throw an exception if the table exists on peer cluster but descriptors are not same.</li>
* </ol>
* @param tableName name of the table to sync to the peer
* @param splits table split keys
@@ -4105,20 +4106,32 @@ public class HBaseAdmin implements Admin {
Configuration peerConf = getPeerClusterConfiguration(peerDesc);
try (Connection conn = ConnectionFactory.createConnection(peerConf);
Admin repHBaseAdmin = conn.getAdmin()) {
- HTableDescriptor htd = getTableDescriptor(tableName);
+ HTableDescriptor localHtd = getTableDescriptor(tableName);
HTableDescriptor peerHtd = null;
if (!repHBaseAdmin.tableExists(tableName)) {
- repHBaseAdmin.createTable(htd, splits);
+ repHBaseAdmin.createTable(localHtd, splits);
} else {
peerHtd = repHBaseAdmin.getTableDescriptor(tableName);
if (peerHtd == null) {
throw new IllegalArgumentException("Failed to get table descriptor for table "
+ tableName.getNameAsString() + " from peer cluster " + peerDesc.getPeerId());
- } else if (!peerHtd.equals(htd)) {
- throw new IllegalArgumentException("Table " + tableName.getNameAsString()
- + " exists in peer cluster " + peerDesc.getPeerId()
- + ", but the table descriptors are not same when compared with source cluster."
- + " Thus can not enable the table's replication switch.");
+ } else {
+ // To support cyclic replication (HBASE-17460), we need to match the
+ // REPLICATION_SCOPE of table on both the clusters. We should do this
+ // only when the replication is not already enabled on local HTD (local
+ // table on this cluster).
+ //
+ if (localHtd.isReplicationEnabled()) {
+ throw new IllegalArgumentException("Table " + tableName.getNameAsString()
+ + " has replication already enabled for atleast one Column Family.");
+ } else {
+ if (!peerHtd.compareForReplication(localHtd)) {
+ throw new IllegalArgumentException("Table " + tableName.getNameAsString()
+ + " exists in peer cluster " + peerDesc.getPeerId()
+ + ", but the table descriptors are not same when compared with source cluster."
+ + " Thus can not enable the table's replication switch.");
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d51e2878/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
index 706f81e..4e74d87 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
@@ -18,9 +18,6 @@
*/
package org.apache.hadoop.hbase.client.replication;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
@@ -29,9 +26,9 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -62,6 +59,9 @@ import org.apache.hadoop.hbase.replication.ReplicationQueuesClientArguments;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+
/**
* <p>
* This class provides the administrative interface to HBase cluster