You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by to...@apache.org on 2012/05/18 16:10:04 UTC
svn commit: r1340149 - in /incubator/hcatalog/trunk: ./
storage-handlers/hbase/ storage-handlers/hbase/conf/
storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/
storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/
storage-handlers...
Author: toffer
Date: Fri May 18 16:10:03 2012
New Revision: 1340149
URL: http://svn.apache.org/viewvc?rev=1340149&view=rev
Log:
HCAT-404 add separate configuration for revision manager (toffer)
Added:
incubator/hcatalog/trunk/storage-handlers/hbase/conf/
incubator/hcatalog/trunk/storage-handlers/hbase/conf/revision-manager-site.xml
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/
incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/revision-manager-default.xml
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/storage-handlers/hbase/build.xml
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/log4j.xml
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Fri May 18 16:10:03 2012
@@ -98,6 +98,8 @@ Release 0.4.0 - Release May 16 2012
HCAT-2 Support nested schema conversion between Hive an Pig (julienledem via hashutosh)
IMPROVEMENTS
+ HCAT-404 add separate configuration for revision manager (toffer)
+
HCAT-373 ProgressReporter should work with both old and new MR API (traviscrawford via toffer)
HCAT-68 Logging from HCat (avandana via toffer)
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/build.xml?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/build.xml (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/build.xml Fri May 18 16:10:03 2012
@@ -43,6 +43,7 @@
<!-- build properties -->
<property name="lib.dir" value="${basedir}/lib/" />
<property name="src.dir" location="${basedir}/src/java"/>
+ <property name="resources.dir" location="${basedir}/src/resources"/>
<property name="docs.src" value="${basedir}/src/docs"/>
<property name="build.dir" value="${basedir}/build"/>
<property name="build.classes" value="${build.dir}/classes" />
@@ -195,7 +196,7 @@
-->
<!-- Compile src files -->
<target name="compile-src" depends="init">
- <javac encoding="${build.encoding}" srcdir="${src.dir}:${basedir}/src/gen-java" excludes="${excludes}"
+ <javac encoding="${build.encoding}" srcdir="${src.dir}:${basedir}/src/gen-java:${basedir}/src/resources" excludes="${excludes}"
includes="**/*.java" destdir="${build.classes}" debug="${javac.debug}"
optimize="${javac.optimize}" target="${javac.version}"
source="${javac.version}" deprecation="${javac.deprecation}"
@@ -207,6 +208,11 @@
<!-- Build the jar -->
<target name="jar" depends="compile-src">
+ <copy todir="${build.classes}" includeEmptyDirs="false">
+ <fileset dir="${resources.dir}">
+ <include name="*"/>
+ </fileset>
+ </copy>
<jar jarfile="${build.dir}/${ant.project.name}/${handler.jar}" basedir="${build.classes}"/>
</target>
@@ -306,6 +312,7 @@
-->
<target name="package" depends="jar, docs" description="Create an HCatalog release">
<mkdir dir="${dist.dir}" />
+ <mkdir dir="${dist.dir}/conf" />
<mkdir dir="${dist.dir}/lib" />
<mkdir dir="${dist.dir}/docs" />
@@ -320,6 +327,9 @@
<copy todir="${dist.dir}/docs">
<fileset dir="${build.docs}" />
</copy>
+ <copy todir="${dist.dir}/conf">
+ <fileset dir="${basedir}/conf" />
+ </copy>
<antcall target="package-to-handlers"/>
</target>
@@ -329,6 +339,7 @@
<copy todir="${dist.handlers.dir}/hbase" includeEmptyDirs="true">
<fileset dir="${dist.dir}">
<include name="**/lib/hbase-storage-handler-*.jar"/>
+ <include name="**/conf/**/*"/>
</fileset>
</copy>
</target>
Added: incubator/hcatalog/trunk/storage-handlers/hbase/conf/revision-manager-site.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/conf/revision-manager-site.xml?rev=1340149&view=auto
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/conf/revision-manager-site.xml (added)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/conf/revision-manager-site.xml Fri May 18 16:10:03 2012
@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+</configuration>
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java Fri May 18 16:10:03 2012
@@ -59,6 +59,7 @@ import org.apache.hcatalog.data.schema.H
import org.apache.hcatalog.hbase.HBaseBulkOutputFormat.HBaseBulkOutputCommitter;
import org.apache.hcatalog.hbase.HBaseDirectOutputFormat.HBaseDirectOutputCommitter;
import org.apache.hcatalog.hbase.snapshot.RevisionManager;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
import org.apache.hcatalog.hbase.snapshot.Transaction;
import org.apache.hcatalog.mapreduce.HCatOutputFormat;
import org.apache.hcatalog.mapreduce.HCatTableInfo;
@@ -98,7 +99,7 @@ public class HBaseHCatStorageHandler ext
jobProperties.put(TableInputFormat.INPUT_TABLE, qualifiedTableName);
Configuration jobConf = getJobConf();
- addHbaseResources(jobConf, jobProperties);
+ addResources(jobConf, jobProperties);
JobConf copyOfConf = new JobConf(jobConf);
HBaseConfiguration.addHbaseResources(copyOfConf);
//Getting hbase delegation token in getInputSplits does not work with PIG. So need to
@@ -114,8 +115,10 @@ public class HBaseHCatStorageHandler ext
String serSnapshot = (String) inputJobInfo.getProperties().get(
HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY);
if (serSnapshot == null) {
- HCatTableSnapshot snapshot = HBaseRevisionManagerUtil.createSnapshot(copyOfConf,
- qualifiedTableName, tableInfo);
+ HCatTableSnapshot snapshot =
+ HBaseRevisionManagerUtil.createSnapshot(
+ RevisionManagerConfiguration.create(copyOfConf),
+ qualifiedTableName, tableInfo);
jobProperties.put(HBaseConstants.PROPERTY_TABLE_SNAPSHOT_KEY,
HCatUtil.serialize(snapshot));
}
@@ -146,7 +149,7 @@ public class HBaseHCatStorageHandler ext
jobProperties.put(TableOutputFormat.OUTPUT_TABLE, qualifiedTableName);
Configuration jobConf = getJobConf();
- addHbaseResources(jobConf, jobProperties);
+ addResources(jobConf, jobProperties);
Configuration copyOfConf = new Configuration(jobConf);
HBaseConfiguration.addHbaseResources(copyOfConf);
@@ -155,7 +158,8 @@ public class HBaseHCatStorageHandler ext
HBaseConstants.PROPERTY_WRITE_TXN_KEY);
Transaction txn = null;
if (txnString == null) {
- txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName, tableInfo, copyOfConf);
+ txn = HBaseRevisionManagerUtil.beginWriteTransaction(qualifiedTableName, tableInfo,
+ RevisionManagerConfiguration.create(copyOfConf));
String serializedTxn = HCatUtil.serialize(txn);
outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
serializedTxn);
@@ -477,7 +481,7 @@ public class HBaseHCatStorageHandler ext
//configureOutputJobProperties, we take care of adding the default properties
//that are not already present. TODO: Change to a copy for jobs after HCAT-308 is fixed.
jobConf = conf;
- hbaseConf = HBaseConfiguration.create(conf);
+ hbaseConf = RevisionManagerConfiguration.create(HBaseConfiguration.create(conf));
}
private void checkDeleteTable(Table table) throws MetaException {
@@ -540,10 +544,11 @@ public class HBaseHCatStorageHandler ext
* @param jobConf Job configuration
* @param newJobProperties Map to which new properties should be added
*/
- private void addHbaseResources(Configuration jobConf,
- Map<String, String> newJobProperties) {
+ private void addResources(Configuration jobConf,
+ Map<String, String> newJobProperties) {
Configuration conf = new Configuration(false);
HBaseConfiguration.addHbaseResources(conf);
+ RevisionManagerConfiguration.addResources(conf);
for (Entry<String, String> entry : conf) {
if (jobConf.get(entry.getKey()) == null)
newJobProperties.put(entry.getKey(), entry.getValue());
Added: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java?rev=1340149&view=auto
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java (added)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RMConstants.java Fri May 18 16:10:03 2012
@@ -0,0 +1,11 @@
+package org.apache.hcatalog.hbase.snapshot;
+
+public class RMConstants {
+ public static final String REVISION_MGR_ENDPOINT_IMPL_CLASS = "revision.manager.endpoint.impl.class";
+
+ public static final String WRITE_TRANSACTION_TIMEOUT = "revision.manager.writeTxn.timeout";
+
+ public static final String ZOOKEEPER_HOSTLIST = "revision.manager.zk.hostList";
+
+ public static final String ZOOKEEPER_DATADIR = "revision.manager.zk.dataDir";
+}
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManager.java Fri May 18 16:10:03 2012
@@ -17,6 +17,8 @@
*/
package org.apache.hcatalog.hbase.snapshot;
+import org.apache.hadoop.conf.Configuration;
+
import java.io.IOException;
import java.util.List;
import java.util.Properties;
@@ -34,7 +36,7 @@ public interface RevisionManager {
/**
* Initialize the revision manager.
*/
- public void initialize(Properties properties);
+ public void initialize(Configuration conf);
/**
* Opens the revision manager.
Added: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java?rev=1340149&view=auto
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java (added)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerConfiguration.java Fri May 18 16:10:03 2012
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hcatalog.hbase.snapshot;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+
+public class RevisionManagerConfiguration {
+
+
+
+ public static Configuration addResources(Configuration conf) {
+ conf.addDefaultResource("revision-manager-default.xml");
+ conf.addResource("revision-manager-site.xml");
+ return conf;
+ }
+
+ /**
+ * Creates a Configuration with Revision Manager resources
+ * @return a Configuration with Revision Manager resources
+ */
+ public static Configuration create() {
+ Configuration conf = new Configuration();
+ return addResources(conf);
+ }
+
+ /**
+ * Creates a clone of passed configuration.
+ * @param that Configuration to clone.
+ * @return a Configuration created with the revision-manager-*.xml files plus
+ * the given configuration.
+ */
+ public static Configuration create(final Configuration that) {
+ Configuration conf = create();
+ //we need to merge things instead of doing new Configuration(that)
+ //because of a bug in Configuration wherein the config
+ //set on the MR fronted will get loaded on the backend as resouce called job.xml
+ //hence adding resources on the backed could potentially overwrite properties
+ //set on the frontend which we shouldn't be doing here
+ HBaseConfiguration.merge(conf, that);
+ return conf;
+ }
+}
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpoint.java Fri May 18 16:10:03 2012
@@ -38,7 +38,6 @@ public class RevisionManagerEndpoint ext
private static final Logger LOGGER =
LoggerFactory.getLogger(RevisionManagerEndpoint.class.getName());
- public static final String REVISION_MGR_ENDPOINT_IMPL_CLASS = "revision.manager.endpoint.impl.class";
private RevisionManager rmImpl = null;
@@ -46,9 +45,10 @@ public class RevisionManagerEndpoint ext
public void start(CoprocessorEnvironment env) {
super.start(env);
try {
- Configuration conf = env.getConfiguration();
- String className = conf.get(REVISION_MGR_ENDPOINT_IMPL_CLASS,
+ Configuration conf = RevisionManagerConfiguration.create(env.getConfiguration());
+ String className = conf.get(RMConstants.REVISION_MGR_ENDPOINT_IMPL_CLASS,
ZKBasedRevisionManager.class.getName());
+ LOGGER.debug("Using Revision Manager implementation: {}",className);
rmImpl = RevisionManagerFactory.getOpenedRevisionManager(className, conf);
} catch (IOException e) {
LOGGER.error("Failed to initialize revision manager", e);
@@ -68,7 +68,7 @@ public class RevisionManagerEndpoint ext
}
@Override
- public void initialize(Properties properties) {
+ public void initialize(Configuration conf) {
// do nothing, HBase controls life cycle
}
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerEndpointClient.java Fri May 18 16:10:03 2012
@@ -30,7 +30,7 @@ public class RevisionManagerEndpointClie
}
@Override
- public void initialize(Properties properties) {
+ public void initialize(Configuration conf) {
// do nothing
}
@@ -41,7 +41,7 @@ public class RevisionManagerEndpointClie
// conf.set("hbase.ipc.client.connect.max.retries", "0");
// conf.setInt(HConstants.HBASE_CLIENT_RPC_MAXATTEMPTS, 1);
clonedConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); // do not retry RPC
- HTable table = new HTable(clonedConf, HConstants.META_TABLE_NAME);
+ HTable table = new HTable(clonedConf, HConstants.ROOT_TABLE_NAME);
rmProxy = table.coprocessorProxy(RevisionManagerProtocol.class,
Bytes.toBytes("anyRow"));
rmProxy.open();
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/RevisionManagerFactory.java Fri May 18 16:10:03 2012
@@ -35,11 +35,11 @@ public class RevisionManagerFactory {
/**
* Gets an instance of revision manager.
*
- * @param properties The properties required to created the revision manager.
+ * @param conf The configuration required to created the revision manager.
* @return the revision manager An instance of revision manager.
* @throws IOException Signals that an I/O exception has occurred.
*/
- private static RevisionManager getRevisionManager(String className, Properties properties) throws IOException{
+ private static RevisionManager getRevisionManager(String className, Configuration conf) throws IOException{
RevisionManager revisionMgr;
ClassLoader classLoader = Thread.currentThread()
@@ -51,7 +51,7 @@ public class RevisionManagerFactory {
Class<? extends RevisionManager> revisionMgrClass = Class
.forName(className, true , classLoader).asSubclass(RevisionManager.class);
revisionMgr = (RevisionManager) revisionMgrClass.newInstance();
- revisionMgr.initialize(properties);
+ revisionMgr.initialize(conf);
} catch (ClassNotFoundException e) {
throw new IOException(
"The implementation class of revision manager not found.",
@@ -81,30 +81,7 @@ public class RevisionManagerFactory {
*/
static RevisionManager getOpenedRevisionManager(String className, Configuration conf) throws IOException {
- Properties properties = new Properties();
- String zkHostList = conf.get(HConstants.ZOOKEEPER_QUORUM);
- int port = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT,
- HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
-
- if (zkHostList != null) {
- String[] splits = zkHostList.split(",");
- StringBuffer sb = new StringBuffer();
- for (String split : splits) {
- sb.append(split);
- sb.append(':');
- sb.append(port);
- sb.append(',');
- }
-
- sb.deleteCharAt(sb.length() - 1);
- properties.put(ZKBasedRevisionManager.HOSTLIST, sb.toString());
- }
- String dataDir = conf.get(ZKBasedRevisionManager.DATADIR);
- if (dataDir != null) {
- properties.put(ZKBasedRevisionManager.DATADIR, dataDir);
- }
- RevisionManager revisionMgr = RevisionManagerFactory
- .getRevisionManager(className, properties);
+ RevisionManager revisionMgr = RevisionManagerFactory.getRevisionManager(className, conf);
if (revisionMgr instanceof Configurable) {
((Configurable)revisionMgr).setConf(conf);
}
@@ -116,14 +93,14 @@ public class RevisionManagerFactory {
* Gets an instance of revision manager which is opened.
* The revision manager implementation can be specified as {@link #REVISION_MGR_IMPL_CLASS},
* default is {@link ZKBasedRevisionManager}.
- * @param hbaseConf The HBase configuration.
+ * @param conf revision manager configuration
* @return RevisionManager An instance of revision manager.
* @throws IOException
*/
- public static RevisionManager getOpenedRevisionManager(Configuration hbaseConf) throws IOException {
- String className = hbaseConf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS,
+ public static RevisionManager getOpenedRevisionManager(Configuration conf) throws IOException {
+ String className = conf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS,
ZKBasedRevisionManager.class.getName());
- return getOpenedRevisionManager(className, hbaseConf);
+ return getOpenedRevisionManager(className, conf);
}
}
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java Fri May 18 16:10:03 2012
@@ -25,6 +25,8 @@ import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hcatalog.hbase.snapshot.lock.LockListener;
import org.apache.hcatalog.hbase.snapshot.lock.WriteLock;
import org.apache.zookeeper.CreateMode;
@@ -36,27 +38,37 @@ import org.apache.zookeeper.ZooDefs.Ids;
*/
public class ZKBasedRevisionManager implements RevisionManager{
- public static final String HOSTLIST = "revision.manager.zk.HostList";
- public static final String DATADIR = "revision.manager.zk.DataDir";
- public static final String DEFAULT_DATADIR = "/revision-management";
- public static final String DEFAULT_HOSTLIST = "localhost:2181";
- private static int DEFAULT_WRITE_TRANSACTION_TIMEOUT = 14400000;
private static final Log LOG = LogFactory.getLog(ZKBasedRevisionManager.class);
private String zkHostList;
private String baseDir;
private ZKUtil zkUtil;
+ private long writeTxnTimeout;
/*
* @see org.apache.hcatalog.hbase.snapshot.RevisionManager#initialize()
*/
@Override
- public void initialize(Properties properties) {
- this.zkHostList = properties.getProperty(
- ZKBasedRevisionManager.HOSTLIST,
- ZKBasedRevisionManager.DEFAULT_HOSTLIST);
- this.baseDir = properties.getProperty(ZKBasedRevisionManager.DATADIR,
- ZKBasedRevisionManager.DEFAULT_DATADIR);
+ public void initialize(Configuration conf) {
+ conf = new Configuration(conf);
+ if (conf.get(RMConstants.ZOOKEEPER_HOSTLIST) == null) {
+ String zkHostList = conf.get(HConstants.ZOOKEEPER_QUORUM);
+ int port = conf.getInt(HConstants.ZOOKEEPER_CLIENT_PORT,
+ HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT);
+ String[] splits = zkHostList.split(",");
+ StringBuffer sb = new StringBuffer();
+ for (String split : splits) {
+ sb.append(split);
+ sb.append(':');
+ sb.append(port);
+ sb.append(',');
+ }
+ sb.deleteCharAt(sb.length() - 1);
+ conf.set(RMConstants.ZOOKEEPER_HOSTLIST, sb.toString());
+ }
+ this.zkHostList = conf.get(RMConstants.ZOOKEEPER_HOSTLIST);
+ this.baseDir = conf.get(RMConstants.ZOOKEEPER_DATADIR);
+ this.writeTxnTimeout = Long.parseLong(conf.get(RMConstants.WRITE_TRANSACTION_TIMEOUT));
}
/**
@@ -118,7 +130,7 @@ public class ZKBasedRevisionManager impl
if (keepAlive != -1) {
transaction.setKeepAlive(keepAlive);
} else {
- transaction.setKeepAlive(DEFAULT_WRITE_TRANSACTION_TIMEOUT);
+ transaction.setKeepAlive(writeTxnTimeout);
}
refreshTransactionList(transaction.getTableName());
Added: incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/revision-manager-default.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/revision-manager-default.xml?rev=1340149&view=auto
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/revision-manager-default.xml (added)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/resources/revision-manager-default.xml Fri May 18 16:10:03 2012
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>revision.manager.impl.class</name>
+ <value>org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager</value>
+ <description>
+ Which revision manager implementation to use.
+ </description>
+ </property>
+ <property>
+ <name>revision.manager.endpoint.impl.class</name>
+ <value>org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager</value>
+ <description> </description>
+ </property>
+ <property>
+ <name>revision.manager.zk.dataDir</name>
+ <value>/revision-management</value>
+ <description>zookeeper base directory</description>
+ </property>
+ <property>
+ <name>revision.manager.writeTxn.timeout</name>
+ <value>14400000</value>
+ <description>timeout for write transactions in seconds</description>
+ </property>
+</configuration>
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/log4j.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/log4j.xml?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/log4j.xml (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/log4j.xml Fri May 18 16:10:03 2012
@@ -27,7 +27,7 @@
<root>
- <priority value ="INFO" />
+ <priority value ="DEBUG" />
<appender-ref ref="ConsoleAppender" />
</root>
<!--<logger name="org.apache.hadoop.mapred">-->
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java Fri May 18 16:10:03 2012
@@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
@@ -59,6 +60,7 @@ import org.apache.hcatalog.hbase.TestHBa
import org.apache.hcatalog.hbase.TestHBaseDirectOutputFormat.MapWriteAbortTransaction;
import org.apache.hcatalog.hbase.snapshot.FamilyRevision;
import org.apache.hcatalog.hbase.snapshot.RevisionManager;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
import org.apache.hcatalog.hbase.snapshot.TableSnapshot;
import org.apache.hcatalog.hbase.snapshot.Transaction;
import org.apache.hcatalog.mapreduce.HCatInputFormat;
@@ -100,6 +102,9 @@ public class TestHBaseBulkOutputFormat e
for (Map.Entry<String, String> el : getJobConf())
allConf.set(el.getKey(), el.getValue());
+ HBaseConfiguration.merge(
+ allConf,
+ RevisionManagerConfiguration.create());
SessionState.start(new CliSessionState(allConf));
hcatDriver = new HCatDriver();
}
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java Fri May 18 16:10:03 2012
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
@@ -55,6 +56,7 @@ import org.apache.hcatalog.data.HCatReco
import org.apache.hcatalog.data.schema.HCatSchema;
import org.apache.hcatalog.hbase.snapshot.FamilyRevision;
import org.apache.hcatalog.hbase.snapshot.RevisionManager;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
import org.apache.hcatalog.hbase.snapshot.TableSnapshot;
import org.apache.hcatalog.hbase.snapshot.Transaction;
import org.apache.hcatalog.mapreduce.HCatInputFormat;
@@ -67,9 +69,11 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertTrue;
/**
@@ -92,7 +96,9 @@ public class TestHBaseDirectOutputFormat
allConf.set(el.getKey(), el.getValue());
for (Map.Entry<String, String> el : getJobConf())
allConf.set(el.getKey(), el.getValue());
-
+ HBaseConfiguration.merge(
+ allConf,
+ RevisionManagerConfiguration.create());
SessionState.start(new CliSessionState(allConf));
hcatDriver = new HCatDriver();
}
@@ -330,8 +336,11 @@ public class TestHBaseDirectOutputFormat
Scan scan = new Scan();
scan.addFamily(familyNameBytes);
ResultScanner scanner = table.getScanner(scan);
- int index = 0;
+ int count = 0;
for (Result result : scanner) {
+ String key = Bytes.toString(result.getRow());
+ assertNotSame(MapWriteAbortTransaction.failedKey, key);
+ int index = Integer.parseInt(key)-1;
String vals[] = data[index].toString().split(",");
for (int i = 1; i < vals.length; i++) {
String pair[] = vals[i].split(":");
@@ -341,9 +350,9 @@ public class TestHBaseDirectOutputFormat
assertEquals(1l, result.getColumn(familyNameBytes, Bytes.toBytes(pair[0])).get(0)
.getTimestamp());
}
- index++;
+ count++;
}
- assertEquals(data.length - 1, index);
+ assertEquals(data.length - 1, count);
// verify that the inputformat returns empty results.
Path outputDir = new Path(getTestDir(),
@@ -441,6 +450,8 @@ public class TestHBaseDirectOutputFormat
}
static class MapWriteAbortTransaction extends Mapper<LongWritable, Text, BytesWritable, HCatRecord> {
+ public static String failedKey;
+ private static int count = 0;
@Override
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
@@ -449,14 +460,19 @@ public class TestHBaseDirectOutputFormat
HCatSchema schema = jobInfo.getOutputSchema();
String vals[] = value.toString().split(",");
record.setInteger("key", schema, Integer.parseInt(vals[0]));
- if (vals[0].equals("3")) {
- throw new IOException("Failing map to test abort");
- }
- for (int i = 1; i < vals.length; i++) {
- String pair[] = vals[i].split(":");
- record.set(pair[0], schema, pair[1]);
+ synchronized (MapWriteAbortTransaction.class) {
+ if (count == 2) {
+ failedKey = vals[0];
+ throw new IOException("Failing map to test abort");
+ }
+ for (int i = 1; i < vals.length; i++) {
+ String pair[] = vals[i].split(":");
+ record.set(pair[0], schema, pair[1]);
+ }
+ context.write(null, record);
+ count++;
}
- context.write(null, record);
+
}
}
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java Fri May 18 16:10:03 2012
@@ -24,6 +24,7 @@ import java.net.URI;
import java.util.Map;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hive.ql.session
import org.apache.hcatalog.cli.HCatDriver;
import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
import org.apache.hcatalog.hbase.snapshot.RevisionManager;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
import org.apache.zookeeper.KeeperException.NoNodeException;
import org.junit.Test;
@@ -58,12 +60,14 @@ public class TestHBaseHCatStorageHandler
hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString());
//Add hbase properties
-
for (Map.Entry<String, String> el : getHbaseConf()) {
if (el.getKey().startsWith("hbase.")) {
hcatConf.set(el.getKey(), el.getValue());
}
}
+ HBaseConfiguration.merge(
+ hcatConf,
+ RevisionManagerConfiguration.create());
SessionState.start(new CliSessionState(hcatConf));
hcatDriver = new HCatDriver();
@@ -87,7 +91,7 @@ public class TestHBaseHCatStorageHandler
assertTrue(doesTableExist);
- RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+ RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
rm.open();
//Should be able to successfully query revision manager
rm.getAbortedWriteTransactions("test_table", "cf1");
@@ -123,7 +127,7 @@ public class TestHBaseHCatStorageHandler
assertTrue(doesTableExist);
- RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+ RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
rm.open();
//Should be able to successfully query revision manager
rm.getAbortedWriteTransactions("test_table", "cf1");
@@ -159,7 +163,7 @@ public class TestHBaseHCatStorageHandler
assertTrue(doesTableExist);
- RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+ RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
rm.open();
//Should be able to successfully query revision manager
rm.getAbortedWriteTransactions("CaseSensitiveTable", "cf1");
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseInputFormat.java Fri May 18 16:10:03 2012
@@ -32,6 +32,7 @@ import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
@@ -66,6 +67,7 @@ import org.apache.hcatalog.data.HCatReco
import org.apache.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hcatalog.data.schema.HCatSchema;
import org.apache.hcatalog.hbase.snapshot.RevisionManager;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
import org.apache.hcatalog.hbase.snapshot.Transaction;
import org.apache.hcatalog.mapreduce.HCatInputFormat;
import org.apache.hcatalog.mapreduce.InputJobInfo;
@@ -80,13 +82,39 @@ public class TestHBaseInputFormat extend
private final byte[] QUALIFIER1 = Bytes.toBytes("testQualifier1");
private final byte[] QUALIFIER2 = Bytes.toBytes("testQualifier2");
+ public TestHBaseInputFormat() throws Exception {
+ hcatConf = getHiveConf();
+ hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname,
+ HCatSemanticAnalyzer.class.getName());
+ URI fsuri = getFileSystem().getUri();
+ Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(),
+ getTestDir());
+ hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString());
+ hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString());
+
+ //Add hbase properties
+
+ for (Map.Entry<String, String> el : getHbaseConf()) {
+ if (el.getKey().startsWith("hbase.")) {
+ hcatConf.set(el.getKey(), el.getValue());
+ }
+ }
+ HBaseConfiguration.merge(hcatConf,
+ RevisionManagerConfiguration.create());
+
+
+ SessionState.start(new CliSessionState(hcatConf));
+ hcatDriver = new HCatDriver();
+
+ }
+
private List<Put> generatePuts(int num, String tableName) throws IOException {
List<String> columnFamilies = Arrays.asList("testFamily");
RevisionManager rm = null;
List<Put> myPuts;
try {
- rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+ rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
rm.open();
myPuts = new ArrayList<Put>();
for (int i = 1; i <= num; i++) {
@@ -106,29 +134,6 @@ public class TestHBaseInputFormat extend
return myPuts;
}
- private void Initialize() throws Exception {
- hcatConf = getHiveConf();
- hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname,
- HCatSemanticAnalyzer.class.getName());
- URI fsuri = getFileSystem().getUri();
- Path whPath = new Path(fsuri.getScheme(), fsuri.getAuthority(),
- getTestDir());
- hcatConf.set(HiveConf.ConfVars.HADOOPFS.varname, fsuri.toString());
- hcatConf.set(ConfVars.METASTOREWAREHOUSE.varname, whPath.toString());
-
- //Add hbase properties
-
- for (Map.Entry<String, String> el : getHbaseConf()) {
- if (el.getKey().startsWith("hbase.")) {
- hcatConf.set(el.getKey(), el.getValue());
- }
- }
-
- SessionState.start(new CliSessionState(hcatConf));
- hcatDriver = new HCatDriver();
-
- }
-
private void populateHBaseTable(String tName, int revisions) throws IOException {
List<Put> myPuts = generatePuts(revisions, tName);
HTable table = new HTable(getHbaseConf(), Bytes.toBytes(tName));
@@ -142,7 +147,7 @@ public class TestHBaseInputFormat extend
List<Put> myPuts = new ArrayList<Put>();
long revision;
try {
- rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(getHbaseConf());
+ rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
rm.open();
Transaction tsx = rm.beginWriteTransaction(tName, columnFamilies);
@@ -171,7 +176,6 @@ public class TestHBaseInputFormat extend
@Test
public void TestHBaseTableReadMR() throws Exception {
- Initialize();
String tableName = newTableName("MyTable");
String databaseName = newTableName("MyDatabase");
//Table name will be lower case unless specified by hbase.table.name property
@@ -243,7 +247,6 @@ public class TestHBaseInputFormat extend
@Test
public void TestHBaseTableProjectionReadMR() throws Exception {
- Initialize();
String tableName = newTableName("MyTable");
//Table name as specified by hbase.table.name property
String hbaseTableName = "MyDB_" + tableName;
@@ -304,7 +307,6 @@ public class TestHBaseInputFormat extend
@Test
public void TestHBaseInputFormatProjectionReadMR() throws Exception {
- Initialize();
String tableName = newTableName("mytable");
String tableQuery = "CREATE TABLE " + tableName
+ "(key string, testqualifier1 string, testqualifier2 string) STORED BY " +
@@ -377,7 +379,6 @@ public class TestHBaseInputFormat extend
@Test
public void TestHBaseTableIgnoreAbortedTransactions() throws Exception {
- Initialize();
String tableName = newTableName("mytable");
String tableQuery = "CREATE TABLE " + tableName
+ "(key string, testqualifier1 string, testqualifier2 string) STORED BY " +
@@ -437,7 +438,6 @@ public class TestHBaseInputFormat extend
@Test
public void TestHBaseTableIgnoreAbortedAndRunningTransactions() throws Exception {
- Initialize();
String tableName = newTableName("mytable");
String tableQuery = "CREATE TABLE " + tableName
+ "(key string, testqualifier1 string, testqualifier2 string) STORED BY " +
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java Fri May 18 16:10:03 2012
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -36,6 +37,7 @@ import org.apache.hcatalog.cli.HCatDrive
import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.common.HCatUtil;
+import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration;
import org.apache.hcatalog.hbase.snapshot.TableSnapshot;
import org.apache.hcatalog.mapreduce.InitializeInput;
import org.apache.hcatalog.mapreduce.InputJobInfo;
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java Fri May 18 16:10:03 2012
@@ -23,8 +23,8 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
-import java.util.Properties;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hcatalog.hbase.SkeletonHBaseTest;
import org.apache.hcatalog.hbase.snapshot.transaction.thrift.*;
import org.apache.zookeeper.KeeperException;
@@ -84,11 +84,10 @@ public class TestRevisionManager extends
sb.append(port);
}
- Properties props = new Properties();
- props.put(ZKBasedRevisionManager.HOSTLIST, sb.toString());
- props.put(ZKBasedRevisionManager.DATADIR, "/rm_base");
+ Configuration conf = RevisionManagerConfiguration.create(getHbaseConf());
+ conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base");
ZKBasedRevisionManager manager = new ZKBasedRevisionManager();
- manager.initialize(props);
+ manager.initialize(conf);
manager.open();
ZKUtil zkutil = new ZKUtil(sb.toString(), "/rm_base");
@@ -132,11 +131,10 @@ public class TestRevisionManager extends
int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181);
String host = getHbaseConf().get("hbase.zookeeper.quorum");
- Properties props = new Properties();
- props.put(ZKBasedRevisionManager.HOSTLIST, host + ':' + port);
- props.put(ZKBasedRevisionManager.DATADIR, "/rm_base");
+ Configuration conf = RevisionManagerConfiguration.create(getHbaseConf());
+ conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base");
ZKBasedRevisionManager manager = new ZKBasedRevisionManager();
- manager.initialize(props);
+ manager.initialize(conf);
manager.open();
ZKUtil zkutil = new ZKUtil(host + ':' + port, "/rm_base");
@@ -196,11 +194,10 @@ public class TestRevisionManager extends
sb.append(port);
}
- Properties props = new Properties();
- props.put(ZKBasedRevisionManager.HOSTLIST, sb.toString());
- props.put(ZKBasedRevisionManager.DATADIR, "/rm_base");
+ Configuration conf = RevisionManagerConfiguration.create(getHbaseConf());
+ conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base");
ZKBasedRevisionManager manager = new ZKBasedRevisionManager();
- manager.initialize(props);
+ manager.initialize(conf);
manager.open();
String tableName = newTableName("testTable");
List<String> columnFamilies = Arrays.asList("cf1", "cf2");
@@ -221,11 +218,10 @@ public class TestRevisionManager extends
public void testCreateSnapshot() throws IOException{
int port = getHbaseConf().getInt("hbase.zookeeper.property.clientPort", 2181);
String host = getHbaseConf().get("hbase.zookeeper.quorum");
- Properties props = new Properties();
- props.put(ZKBasedRevisionManager.HOSTLIST, host + ':' + port);
- props.put(ZKBasedRevisionManager.DATADIR, "/rm_base");
+ Configuration conf = RevisionManagerConfiguration.create(getHbaseConf());
+ conf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base");
ZKBasedRevisionManager manager = new ZKBasedRevisionManager();
- manager.initialize(props);
+ manager.initialize(conf);
manager.open();
String tableName = newTableName("testTable");
List<String> cfOne = Arrays.asList("cf1", "cf2");
Added: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java?rev=1340149&view=auto
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java (added)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerConfiguration.java Fri May 18 16:10:03 2012
@@ -0,0 +1,16 @@
+package org.apache.hcatalog.hbase.snapshot;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestRevisionManagerConfiguration {
+
+ @Test
+ public void testDefault() {
+ Configuration conf = RevisionManagerConfiguration.create();
+ assertEquals("org.apache.hcatalog.hbase.snapshot.ZKBasedRevisionManager",
+ conf.get(RevisionManagerFactory.REVISION_MGR_IMPL_CLASS));
+ }
+}
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java Fri May 18 16:10:03 2012
@@ -41,7 +41,7 @@ public class TestRevisionManagerEndpoint
testConf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
"org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpoint",
"org.apache.hadoop.hbase.coprocessor.GenericEndpoint");
- testConf.set(RevisionManagerEndpoint.REVISION_MGR_ENDPOINT_IMPL_CLASS, MockRM.class.getName());
+ testConf.set(RMConstants.REVISION_MGR_ENDPOINT_IMPL_CLASS, MockRM.class.getName());
}
/**
@@ -106,8 +106,9 @@ public class TestRevisionManagerEndpoint
}
@Override
- public void initialize(Properties properties) {
- INSTANCES.put("default", this);
+ public void initialize(Configuration conf) {
+ if(!INSTANCES.containsKey(DEFAULT_INSTANCE))
+ INSTANCES.put(DEFAULT_INSTANCE, this);
}
@Override
Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java?rev=1340149&r1=1340148&r2=1340149&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java Fri May 18 16:10:03 2012
@@ -26,6 +26,7 @@ import java.net.URI;
import java.util.Map;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -64,7 +65,9 @@ public class TestZNodeSetUp extends Skel
hcatConf.set(el.getKey(), el.getValue());
}
}
- hcatConf.set("revision.manager.zk.DataDir", "/rm_base");
+ HBaseConfiguration.merge(hcatConf,
+ RevisionManagerConfiguration.create());
+ hcatConf.set(RMConstants.ZOOKEEPER_DATADIR, "/rm_base");
SessionState.start(new CliSessionState(hcatConf));
hcatDriver = new HCatDriver();