You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by to...@apache.org on 2012/03/23 18:45:18 UTC
svn commit: r1304556 - in /incubator/hcatalog/branches/branch-0.4: ./
storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/
storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/
storage-handlers/hbase/src/test/org/apache/hcatalog/hba...
Author: toffer
Date: Fri Mar 23 18:45:18 2012
New Revision: 1304556
URL: http://svn.apache.org/viewvc?rev=1304556&view=rev
Log:
merged from trunk: HCATALOG-310 Turn current RM implementation into HBase Coprocessor (thw via toffer)
Added:
incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
- copied unchanged from r1304536, incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
- copied unchanged from r1304536, incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java
- copied unchanged from r1304536, incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerProtocol.java
incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/package-info.java
- copied unchanged from r1304536, incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/package-info.java
incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
- copied unchanged from r1304536, incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
Modified:
incubator/hcatalog/branches/branch-0.4/ (props changed)
incubator/hcatalog/branches/branch-0.4/CHANGES.txt
incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java
incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java
incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
Propchange: incubator/hcatalog/branches/branch-0.4/
------------------------------------------------------------------------------
Merged /incubator/hcatalog/trunk:r1304536
Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Fri Mar 23 18:45:18 2012
@@ -56,6 +56,8 @@ Release 0.4.0 - Unreleased
HCAT-240. Changes to HCatOutputFormat to make it use SerDes instead of StorageDriver (toffer)
NEW FEATURES
+ HCAT-310 Turn current RM implementation into HBase Coprocessor (thw via toffer)
+
HCAT-334 HCatalog should generate a POM file so it can be deployed to a maven repo (traviscrawford via gates)
HCAT-296 Hcatalog should be able talk to secure hbase server using hbase delegation tokens (rohini via toffer)
Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java Fri Mar 23 18:45:18 2012
@@ -24,13 +24,11 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hive.hbase.HBaseSerDe;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.common.HCatUtil;
@@ -40,7 +38,6 @@ import org.apache.hcatalog.hbase.snapsho
import org.apache.hcatalog.hbase.snapshot.RevisionManagerFactory;
import org.apache.hcatalog.hbase.snapshot.TableSnapshot;
import org.apache.hcatalog.hbase.snapshot.Transaction;
-import org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager;
import org.apache.hcatalog.mapreduce.HCatTableInfo;
import org.apache.hcatalog.mapreduce.InputJobInfo;
import org.apache.hcatalog.mapreduce.OutputJobInfo;
@@ -126,37 +123,7 @@ class HBaseRevisionManagerUtil {
* @throws IOException
*/
static RevisionManager getOpenedRevisionManager(Configuration jobConf) throws IOException {
-
- Properties properties = new Properties();
- String zkHostList = jobConf.get(HConstants.ZOOKEEPER_QUORUM);
- int port = jobConf.getInt("hbase.zookeeper.property.clientPort",
- HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
-
- if (zkHostList != null) {
- String[] splits = zkHostList.split(",");
- StringBuffer sb = new StringBuffer();
- for (String split : splits) {
- sb.append(split);
- sb.append(':');
- sb.append(port);
- sb.append(',');
- }
-
- sb.deleteCharAt(sb.length() - 1);
- properties.put(ZKBasedRevisionManager.HOSTLIST, sb.toString());
- }
- String dataDir = jobConf.get(ZKBasedRevisionManager.DATADIR);
- if (dataDir != null) {
- properties.put(ZKBasedRevisionManager.DATADIR, dataDir);
- }
- String rmClassName = jobConf.get(
- RevisionManager.REVISION_MGR_IMPL_CLASS,
- ZKBasedRevisionManager.class.getName());
- properties.put(RevisionManager.REVISION_MGR_IMPL_CLASS, rmClassName);
- RevisionManager revisionManger = RevisionManagerFactory
- .getRevisionManager(properties);
- revisionManger.open();
- return revisionManger;
+ return RevisionManagerFactory.getOpenedRevisionManager(jobConf);
}
static void closeRevisionManagerQuietly(RevisionManager rm) {
Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java Fri Mar 23 18:45:18 2012
@@ -25,8 +25,11 @@ import java.util.Properties;
* This interface provides APIs for implementing revision management.
*/
public interface RevisionManager {
-
- public static final String REVISION_MGR_IMPL_CLASS = "revision.manager.impl.class";
+ /**
+ * Version property required by HBase to use this interface
+ * for CoprocessorProtocol / RPC.
+ */
+ public static final long VERSION = 1L; // do not change
/**
* Initialize the revision manager.
Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java Fri Mar 23 18:45:18 2012
@@ -20,16 +20,26 @@ package org.apache.hcatalog.hbase.snapsh
import java.io.IOException;
import java.util.Properties;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+
+/**
+ * Utility to instantiate the revision manager (not a true factory actually).
+ * Depends on HBase configuration to resolve ZooKeeper connection (when ZK is used).
+ */
public class RevisionManagerFactory {
- /**
- * Gets an instance of revision manager.
- *
- * @param properties The properties required to created the revision manager.
- * @return the revision manager An instance of revision manager.
- * @throws IOException Signals that an I/O exception has occurred.
- */
- public static RevisionManager getRevisionManager(Properties properties) throws IOException{
+ public static final String REVISION_MGR_IMPL_CLASS = "revision.manager.impl.class";
+
+ /**
+ * Gets an instance of revision manager.
+ *
+ * @param properties The properties required to created the revision manager.
+ * @return the revision manager An instance of revision manager.
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ private static RevisionManager getRevisionManager(String className, Properties properties) throws IOException{
RevisionManager revisionMgr;
ClassLoader classLoader = Thread.currentThread()
@@ -37,14 +47,9 @@ public class RevisionManagerFactory {
if (classLoader == null) {
classLoader = RevisionManagerFactory.class.getClassLoader();
}
- String className = properties.getProperty(
- RevisionManager.REVISION_MGR_IMPL_CLASS,
- ZKBasedRevisionManager.class.getName());
try {
-
- @SuppressWarnings("unchecked")
- Class<? extends RevisionManager> revisionMgrClass = (Class<? extends RevisionManager>) Class
- .forName(className, true , classLoader);
+ Class<? extends RevisionManager> revisionMgrClass = Class
+ .forName(className, true , classLoader).asSubclass(RevisionManager.class);
revisionMgr = (RevisionManager) revisionMgrClass.newInstance();
revisionMgr.initialize(properties);
} catch (ClassNotFoundException e) {
@@ -67,4 +72,58 @@ public class RevisionManagerFactory {
return revisionMgr;
}
+ /**
+ * Internally used by endpoint implementation to instantiate from different configuration setting.
+ * @param className
+ * @param conf
+ * @return
+ * @throws IOException
+ */
+ static RevisionManager getOpenedRevisionManager(String className, Configuration conf) throws IOException {
+
+ Properties properties = new Properties();
+ String zkHostList = conf.get(HConstants.ZOOKEEPER_QUORUM);
+ int port = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT,
+ HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
+
+ if (zkHostList != null) {
+ String[] splits = zkHostList.split(",");
+ StringBuffer sb = new StringBuffer();
+ for (String split : splits) {
+ sb.append(split);
+ sb.append(':');
+ sb.append(port);
+ sb.append(',');
+ }
+
+ sb.deleteCharAt(sb.length() - 1);
+ properties.put(ZKBasedRevisionManager.HOSTLIST, sb.toString());
+ }
+ String dataDir = conf.get(ZKBasedRevisionManager.DATADIR);
+ if (dataDir != null) {
+ properties.put(ZKBasedRevisionManager.DATADIR, dataDir);
+ }
+ RevisionManager revisionMgr = RevisionManagerFactory
+ .getRevisionManager(className, properties);
+ if (revisionMgr instanceof Configurable) {
+ ((Configurable)revisionMgr).setConf(conf);
+ }
+ revisionMgr.open();
+ return revisionMgr;
+ }
+
+ /**
+ * Gets an instance of revision manager which is opened.
+ * The revision manager implementation can be specified as {@link #REVISION_MGR_IMPL_CLASS},
+ * default is {@link ZKBasedRevisionManager}.
+ * @param hbaseConf The HBase configuration.
+ * @return RevisionManager An instance of revision manager.
+ * @throws IOException
+ */
+ public static RevisionManager getOpenedRevisionManager(Configuration hbaseConf) throws IOException {
+ String className = hbaseConf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS,
+ ZKBasedRevisionManager.class.getName());
+ return getOpenedRevisionManager(className, hbaseConf);
+ }
+
}
Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java Fri Mar 23 18:45:18 2012
@@ -17,6 +17,7 @@
*/
package org.apache.hcatalog.hbase.snapshot;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -24,7 +25,7 @@ import java.util.Map;
/**
* The snapshot for a table and a list of column families.
*/
-public class TableSnapshot {
+public class TableSnapshot implements Serializable {
private String name;
@@ -35,6 +36,9 @@ public class TableSnapshot {
public TableSnapshot(String name, Map<String, Long> cfRevMap, long latestRevision) {
this.name = name;
+ if (cfRevMap == null) {
+ throw new IllegalArgumentException("revision map cannot be null");
+ }
this.cfRevisionMap = cfRevMap;
this.latestRevision = latestRevision;
}
Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java Fri Mar 23 18:45:18 2012
@@ -254,8 +254,8 @@ public class ManyMiniCluster {
hbaseConf.set("hbase.rootdir", hbaseRoot);
hbaseConf.set("hbase.master", "local");
- hbaseConf.setInt("hbase.zookeeper.property.clientPort", zookeeperPort);
- hbaseConf.set("hbase.zookeeper.quorum", "127.0.0.1");
+ hbaseConf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zookeeperPort);
+ hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1");
hbaseConf.setInt("hbase.master.port", findFreePort());
hbaseConf.setInt("hbase.master.info.port", -1);
hbaseConf.setInt("hbase.regionserver.port", findFreePort());
@@ -306,7 +306,7 @@ public class ManyMiniCluster {
private File workDir;
private int numTaskTrackers = 1;
private JobConf jobConf;
- private HBaseConfiguration hbaseConf;
+ private Configuration hbaseConf;
private HiveConf hiveConf;
private boolean miniMRClusterEnabled = true;
@@ -329,7 +329,7 @@ public class ManyMiniCluster {
return this;
}
- public Builder hbaseConf(HBaseConfiguration hbaseConf) {
+ public Builder hbaseConf(Configuration hbaseConf) {
this.hbaseConf = hbaseConf;
return this;
}
Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java?rev=1304556&r1=1304555&r2=1304556&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java Fri Mar 23 18:45:18 2012
@@ -18,9 +18,18 @@
package org.apache.hcatalog.hbase;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -28,14 +37,6 @@ import org.apache.hadoop.hive.conf.HiveC
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-
/**
* Base class for HBase Tests which need a mini cluster instance
*/
@@ -48,6 +49,12 @@ public abstract class SkeletonHBaseTest
protected static Map<String,Context> contextMap = new HashMap<String,Context>();
protected static Set<String> tableNames = new HashSet<String>();
+ /**
+ * Allow tests to alter the default MiniCluster configuration.
+ * (requires static initializer block as all setup here is static)
+ */
+ protected static Configuration testConf = null;
+
protected void createTable(String tableName, String[] families) {
try {
HBaseAdmin admin = new HBaseAdmin(getHbaseConf());
@@ -76,6 +83,7 @@ public abstract class SkeletonHBaseTest
return name;
}
+
/**
* startup an hbase cluster instance before a test suite runs
*/
@@ -173,9 +181,13 @@ public abstract class SkeletonHBaseTest
public void start() {
if(usageCount++ == 0) {
- cluster = ManyMiniCluster.create(new File(testDir)).build();
+ ManyMiniCluster.Builder b = ManyMiniCluster.create(new File(testDir));
+ if (testConf != null) {
+ b.hbaseConf(HBaseConfiguration.create(testConf));
+ }
+ cluster = b.build();
cluster.start();
- hbaseConf = cluster.getHBaseConf();
+ this.hbaseConf = cluster.getHBaseConf();
jobConf = cluster.getJobConf();
fileSystem = cluster.getFileSystem();
hiveConf = cluster.getHiveConf();