You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/01/11 05:51:52 UTC
[02/41] hbase git commit: HBASE-19734 Fix IntegrationTestReplication
and related impl changes
HBASE-19734 Fix IntegrationTestReplication and related impl changes
Adds (client-side) validation to ReplicationPeerConfigBuilder and javadoc
to builder methods in addition to the test fix.
Signed-off-by: Guanghao Zhang <zg...@apache.org>
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/efa911f5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/efa911f5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/efa911f5
Branch: refs/heads/HBASE-19397-branch-2
Commit: efa911f56f9cdc7ffa1e82edbba93aa201d04fa9
Parents: b31b386
Author: Josh Elser <el...@apache.org>
Authored: Mon Jan 8 16:47:18 2018 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Wed Jan 10 13:51:22 2018 -0500
----------------------------------------------------------------------
.../replication/ReplicationPeerConfig.java | 2 +
.../ReplicationPeerConfigBuilder.java | 71 ++++++++++++++++++++
.../replication/TestReplicationPeerConfig.java | 1 -
.../hbase/test/IntegrationTestReplication.java | 36 +++++-----
4 files changed, 92 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/efa911f5/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
index ab75dff..b80ee16 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java
@@ -315,6 +315,8 @@ public class ReplicationPeerConfig {
@Override
public ReplicationPeerConfig build() {
+ // It would be nice to validate the configuration, but we have to work with "old" data
+ // from ZK which makes it much more difficult.
return new ReplicationPeerConfig(this);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/efa911f5/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java
index bbf8dd8..0b2f2e2 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java
@@ -43,32 +43,103 @@ public interface ReplicationPeerConfigBuilder {
*/
ReplicationPeerConfigBuilder setReplicationEndpointImpl(String replicationEndpointImpl);
+ /**
+ * Sets a "raw" configuration property for this replication peer. For experts only.
+ * @param key Configuration property key
+ * @param value Configuration property value
+ * @return {@code this}
+ */
+ @InterfaceAudience.Private
ReplicationPeerConfigBuilder putConfiguration(String key, String value);
+ /**
+ * Adds all of the provided "raw" configuration entries to {@code this}.
+ * @param configuration A collection of raw configuration entries
+ * @return {@code this}
+ */
+ @InterfaceAudience.Private
default ReplicationPeerConfigBuilder putAllConfiguration(Map<String, String> configuration) {
configuration.forEach(this::putConfiguration);
return this;
}
+ /**
+ * Sets the serialized peer configuration data
+ * @return {@code this}
+ */
+ @InterfaceAudience.Private
ReplicationPeerConfigBuilder putPeerData(byte[] key, byte[] value);
+ /**
+ * Sets all of the provided serialized peer configuration data.
+ * @return {@code this}
+ */
+ @InterfaceAudience.Private
default ReplicationPeerConfigBuilder putAllPeerData(Map<byte[], byte[]> peerData) {
peerData.forEach(this::putPeerData);
return this;
}
+ /**
+ * Sets an explicit map of tables and column families in those tables that should be replicated
+ * to the given peer. Use {@link #setReplicateAllUserTables(boolean)} to replicate all tables
+ * to a peer.
+ *
+ * @param tableCFsMap A map from tableName to column family names. An empty collection can be
+ * passed to indicate replicating all column families.
+ * @return {@code this}
+ * @see #setReplicateAllUserTables(boolean)
+ */
ReplicationPeerConfigBuilder
setTableCFsMap(Map<TableName, List<String>> tableCFsMap);
+ /**
+ * Sets a unique collection of HBase namespaces that should be replicated to this peer.
+ * @param namespaces A set of namespaces to be replicated to this peer.
+ * @return {@code this}
+ */
ReplicationPeerConfigBuilder setNamespaces(Set<String> namespaces);
+ /**
+ * Sets the speed, in bytes per second, for any one RegionServer to replicate data to the peer.
+ * @param bandwidth Bytes per second
+ * @return {@code this}.
+ */
ReplicationPeerConfigBuilder setBandwidth(long bandwidth);
+ /**
+ * Configures HBase to replicate all user tables (not system tables) to the peer. Default is
+ * {@code true}.
+ * @param replicateAllUserTables True if all user tables should be replicated, else false.
+ * @return {@code this}
+ */
ReplicationPeerConfigBuilder setReplicateAllUserTables(boolean replicateAllUserTables);
+ /**
+ * Sets the mapping of table name to column families which should not be replicated. This
+ * method sets state which is mutually exclusive to {@link #setTableCFsMap(Map)}. Invoking this
+ * method is only relevant when all user tables are being replicated.
+ *
+ * @param tableCFsMap A mapping of table names to column families which should not be
+ * replicated. An empty list of column families implies all families for the table.
+ * @return {@code this}.
+ */
ReplicationPeerConfigBuilder setExcludeTableCFsMap(Map<TableName, List<String>> tableCFsMap);
+ /**
+ * Sets the collection of namespaces which should not be replicated when all user tables are
+ * configured to be replicated. This method sets state which is mutually exclusive to
+ * {@link #setNamespaces(Set)}. Invoking this method is only relevant when all user tables are
+ * being replicated.
+ *
+ * @param namespaces A set of namespaces whose tables should not be replicated.
+ * @return {@code this}
+ */
ReplicationPeerConfigBuilder setExcludeNamespaces(Set<String> namespaces);
+ /**
+ * Builds the configuration object from the current state of {@code this}.
+ * @return A {@link ReplicationPeerConfig} instance.
+ */
ReplicationPeerConfig build();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/efa911f5/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java
index a0b8a32..edda760 100644
--- a/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java
+++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java
@@ -43,5 +43,4 @@ public class TestReplicationPeerConfig {
BuilderStyleTest.assertClassesAreBuilderStyle(ReplicationPeerConfig.class);
}
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/efa911f5/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestReplication.java
----------------------------------------------------------------------
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestReplication.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestReplication.java
index cb9aa56..33f3570 100644
--- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestReplication.java
+++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestReplication.java
@@ -31,16 +31,17 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
+import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
@@ -222,24 +223,25 @@ public class IntegrationTestReplication extends IntegrationTestBigLinkedList {
// setup the replication on the source
if (!source.equals(sink)) {
- ReplicationAdmin replicationAdmin = new ReplicationAdmin(source.getConfiguration());
- // remove any old replication peers
- for (String oldPeer : replicationAdmin.listPeerConfigs().keySet()) {
- replicationAdmin.removePeer(oldPeer);
- }
-
- // set the sink to be the target
- ReplicationPeerConfig peerConfig = new ReplicationPeerConfig();
- peerConfig.setClusterKey(sink.toString());
+ try (final Admin admin = source.getConnection().getAdmin()) {
+ // remove any old replication peers
+ for (ReplicationPeerDescription peer : admin.listReplicationPeers()) {
+ admin.removeReplicationPeer(peer.getPeerId());
+ }
- // set the test table to be the table to replicate
- HashMap<TableName, ArrayList<String>> toReplicate = new HashMap<>();
- toReplicate.put(tableName, new ArrayList<>(0));
+ // set the test table to be the table to replicate
+ HashMap<TableName, List<String>> toReplicate = new HashMap<>();
+ toReplicate.put(tableName, Collections.emptyList());
- replicationAdmin.addPeer("TestPeer", peerConfig, toReplicate);
+ // set the sink to be the target
+ final ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder()
+ .setClusterKey(sink.toString())
+ .setReplicateAllUserTables(false)
+ .setTableCFsMap(toReplicate).build();
- replicationAdmin.enableTableRep(tableName);
- replicationAdmin.close();
+ admin.addReplicationPeer("TestPeer", peerConfig);
+ admin.enableTableReplication(tableName);
+ }
}
for (ClusterID cluster : clusters) {