You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2013/08/13 05:53:37 UTC
svn commit: r1513333 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/client/replication/ main/ruby/
main/ruby/hbase/ test/java/org/apache/hadoop/hbase/replication/
Author: larsh
Date: Tue Aug 13 03:53:36 2013
New Revision: 1513333
URL: http://svn.apache.org/r1513333
Log:
HBASE-8663 a HBase Shell command to list the tables replicated from current cluster (Demai Ni)
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
hbase/branches/0.94/src/main/ruby/hbase/replication_admin.rb
hbase/branches/0.94/src/main/ruby/shell.rb
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java?rev=1513333&r1=1513332&r2=1513333&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/replication/ReplicationAdmin.java Tue Aug 13 03:53:36 2013
@@ -22,6 +22,10 @@ package org.apache.hadoop.hbase.client.r
import java.io.Closeable;
import java.io.IOException;
import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.lang.Integer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
@@ -30,6 +34,8 @@ import org.apache.hadoop.hbase.client.HC
import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.zookeeper.KeeperException;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HColumnDescriptor;
/**
* <p>
@@ -66,6 +72,15 @@ import org.apache.zookeeper.KeeperExcept
*/
public class ReplicationAdmin implements Closeable {
+ public static final String TNAME = "tableName";
+ public static final String CFNAME = "columnFamlyName";
+
+ // only Global for now, can add other type
+ // such as, 1) no global replication, or 2) the table is replicated to this cluster, etc.
+ public static final String REPLICATIONTYPE = "replicationType";
+ public static final String REPLICATIONGLOBAL = Integer
+ .toString(HConstants.REPLICATION_SCOPE_GLOBAL);
+
private final ReplicationZookeeper replicationZk;
private final HConnection connection;
@@ -199,4 +214,37 @@ public class ReplicationAdmin implements
this.connection.close();
}
}
+
+ /**
+ * Find all column families that are replicated from this cluster
+ * @return the full list of the replicated column families of this cluster as:
+ * tableName, family name, replicationType
+ *
+ * Currently replicationType is Global. In the future, more replication
+ * types may be extended here. For example
+ * 1) the replication may only apply to selected peers instead of all peers
+ * 2) the replicationType may indicate the host Cluster servers as Slave
+ * for the table:columnFam.
+ */
+ public List<HashMap<String, String>> listReplicated() throws IOException {
+ List<HashMap<String, String>> replicationColFams = new ArrayList<HashMap<String, String>>();
+ HTableDescriptor[] tables = this.connection.listTables();
+
+ for (HTableDescriptor table : tables) {
+ HColumnDescriptor[] columns = table.getColumnFamilies();
+ String tableName = table.getNameAsString();
+ for (HColumnDescriptor column : columns) {
+ if (column.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) {
+ // At this moment, the columfam is replicated to all peers
+ HashMap<String, String> replicationEntry = new HashMap<String, String>();
+ replicationEntry.put(TNAME, tableName);
+ replicationEntry.put(CFNAME, column.getNameAsString());
+ replicationEntry.put(REPLICATIONTYPE, REPLICATIONGLOBAL);
+ replicationColFams.add(replicationEntry);
+ }
+ }
+ }
+
+ return replicationColFams;
+ }
}
Modified: hbase/branches/0.94/src/main/ruby/hbase/replication_admin.rb
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/ruby/hbase/replication_admin.rb?rev=1513333&r1=1513332&r2=1513333&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/ruby/hbase/replication_admin.rb (original)
+++ hbase/branches/0.94/src/main/ruby/hbase/replication_admin.rb Tue Aug 13 03:53:36 2013
@@ -43,6 +43,12 @@ module Hbase
@replication_admin.removePeer(id)
end
+ #---------------------------------------------------------------------------------------------
+ # Show replcated tables/column families, and their ReplicationType
+ def list_replicated_tables
+ @replication_admin.listReplicated()
+ end
+
#----------------------------------------------------------------------------------------------
# List all peer clusters
def list_peers
Modified: hbase/branches/0.94/src/main/ruby/shell.rb
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/ruby/shell.rb?rev=1513333&r1=1513332&r2=1513333&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/ruby/shell.rb (original)
+++ hbase/branches/0.94/src/main/ruby/shell.rb Tue Aug 13 03:53:36 2013
@@ -286,6 +286,7 @@ Shell.load_command_group(
disable_peer
start_replication
stop_replication
+ list_replicated_tables
]
)
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java?rev=1513333&r1=1513332&r2=1513333&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java Tue Aug 13 03:53:36 2013
@@ -17,12 +17,18 @@
*/
package org.apache.hadoop.hbase.replication;
+import java.util.HashMap;
+import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.LargeTests;
import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
import org.apache.hadoop.hbase.mapreduce.replication.VerifyReplication;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -35,6 +41,7 @@ import org.junit.experimental.categories
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
@Category(LargeTests.class)
public class TestReplicationSmallTests extends TestReplicationBase {
@@ -527,4 +534,58 @@ public class TestReplicationSmallTests e
findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue());
}
+
+ /**
+ * Test for HBASE-8663
+ * Create two new Tables with colfamilies enabled for replication then run
+ * ReplicationAdmin.listReplicated(). Finally verify the table:colfamilies. Note:
+ * TestReplicationAdmin is a better place for this testing but it would need mocks.
+ * @throws Exception
+ */
+ @Test(timeout = 300000)
+ public void testVerifyListReplicatedTable() throws Exception {
+ LOG.info("testVerifyListReplicatedTable");
+
+ final String tName = "VerifyListReplicated_";
+ final String colFam = "cf1";
+ final int numOfTables = 3;
+
+ HBaseAdmin hadmin = new HBaseAdmin(conf1);
+
+ // Create Tables
+ for (int i = 0; i < numOfTables; i++) {
+ HTableDescriptor ht = new HTableDescriptor(tName + i);
+ HColumnDescriptor cfd = new HColumnDescriptor(colFam);
+ cfd.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
+ ht.addFamily(cfd);
+ hadmin.createTable(ht);
+ }
+
+ // verify the result
+ List<HashMap<String, String>> replicationColFams = admin.listReplicated();
+ int[] match = new int[numOfTables]; // array of 3 with init value of zero
+
+ for (int i = 0; i < replicationColFams.size(); i++) {
+ HashMap<String, String> replicationEntry = replicationColFams.get(i);
+ String tn = replicationEntry.get(ReplicationAdmin.TNAME);
+ if ((tn.startsWith(tName)) && replicationEntry.get(ReplicationAdmin.CFNAME).equals(colFam)) {
+ int m = Integer.parseInt(tn.substring(tn.length() - 1)); // get the last digit
+ match[m]++; // should only increase once
+ }
+ }
+
+ // check the matching result
+ for (int i = 0; i < match.length; i++) {
+ assertTrue("listReplicated() does not match table " + i, (match[i] == 1));
+ }
+
+ // drop tables
+ for (int i = 0; i < numOfTables; i++) {
+ String ht = tName + i;
+ hadmin.disableTable(ht);
+ hadmin.deleteTable(ht);
+ }
+
+ hadmin.close();
+ }
}