You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/03/15 07:40:09 UTC
svn commit: r1300818 - in /hive/trunk: hbase-handler/
hbase-handler/src/test/org/apache/hadoop/hive/hbase/ ivy/ shims/
shims/src/common-secure/java/org/apache/hadoop/hive/thrift/
shims/src/test/org/apache/hadoop/hive/thrift/
Author: hashutosh
Date: Thu Mar 15 06:40:08 2012
New Revision: 1300818
URL: http://svn.apache.org/viewvc?rev=1300818&view=rev
Log:
HIVE-2748: Upgrade Hbase and ZK dependcies (enis via hashutosh)
Modified:
hive/trunk/hbase-handler/ivy.xml
hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
hive/trunk/ivy/ivysettings.xml
hive/trunk/ivy/libraries.properties
hive/trunk/shims/ivy.xml
hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/DelegationTokenStore.java
hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/MemoryTokenStore.java
hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java
hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
Modified: hive/trunk/hbase-handler/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/ivy.xml?rev=1300818&r1=1300817&r2=1300818&view=diff
==============================================================================
--- hive/trunk/hbase-handler/ivy.xml (original)
+++ hive/trunk/hbase-handler/ivy.xml Thu Mar 15 06:40:08 2012
@@ -49,7 +49,11 @@
<artifact name="hbase" type="test-jar" ext="jar"
m:classifier="tests"/>
</dependency>
+ <dependency org="com.github.stephenc.high-scale-lib" name="high-scale-lib" rev="1.1.1"
+ transitive="false"/>
<dependency org="org.apache.zookeeper" name="zookeeper" rev="${zookeeper.version}"
transitive="false"/>
+ <dependency org="org.codehaus.jackson" name="jackson-jaxrs" rev="${jackson.version}"/>
+ <dependency org="org.codehaus.jackson" name="jackson-xc" rev="${jackson.version}"/>
</dependencies>
</ivy-module>
Modified: hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
URL: http://svn.apache.org/viewvc/hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestSetup.java?rev=1300818&r1=1300817&r2=1300818&view=diff
==============================================================================
--- hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestSetup.java (original)
+++ hive/trunk/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestSetup.java Thu Mar 15 06:40:08 2012
@@ -31,10 +31,10 @@ import org.apache.hadoop.hbase.HBaseConf
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
@@ -68,7 +68,7 @@ public class HBaseTestSetup extends Test
setUpFixtures(conf);
}
conf.set("hbase.rootdir", hbaseRoot);
- conf.set("hbase.master", hbaseCluster.getHMasterAddress().toString());
+ conf.set("hbase.master", hbaseCluster.getMaster().getServerName().getHostAndPort());
conf.set("hbase.zookeeper.property.clientPort", Integer.toString(zooKeeperPort));
String auxJars = conf.getAuxJars();
auxJars = ((auxJars == null) ? "" : (auxJars + ",")) + "file://"
@@ -94,7 +94,7 @@ public class HBaseTestSetup extends Test
hbaseConf.setInt("hbase.regionserver.port", findFreePort());
hbaseConf.setInt("hbase.regionserver.info.port", -1);
hbaseCluster = new MiniHBaseCluster(hbaseConf, NUM_REGIONSERVERS);
- conf.set("hbase.master", hbaseCluster.getHMasterAddress().toString());
+ conf.set("hbase.master", hbaseCluster.getMaster().getServerName().getHostAndPort());
// opening the META table ensures that cluster is running
new HTable(hbaseConf, HConstants.META_TABLE_NAME);
createHBaseTable(hbaseConf);
Modified: hive/trunk/ivy/ivysettings.xml
URL: http://svn.apache.org/viewvc/hive/trunk/ivy/ivysettings.xml?rev=1300818&r1=1300817&r2=1300818&view=diff
==============================================================================
--- hive/trunk/ivy/ivysettings.xml (original)
+++ hive/trunk/ivy/ivysettings.xml Thu Mar 15 06:40:08 2012
@@ -32,7 +32,7 @@
<property name="repo.maven.org" value="http://repo1.maven.org/maven2/" override="false"/>
<property name="snapshot.apache.org" value="https://repository.apache.org/content/repositories/snapshots/" override="false"/>
- <property name="maven2.pattern" value="[organisation]/[module]/[revision]/[module]-[revision]"/>
+ <property name="maven2.pattern" value="[organisation]/[module]/[revision]/[module]-[revision](-[classifier])"/>
<property name="repo.dir" value="${user.home}/.m2/repository"/>
<property name="maven2.pattern.ext" value="${maven2.pattern}.[ext]"/>
<property name="resolvers" value="default" override="false"/>
Modified: hive/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hive/trunk/ivy/libraries.properties?rev=1300818&r1=1300817&r2=1300818&view=diff
==============================================================================
--- hive/trunk/ivy/libraries.properties (original)
+++ hive/trunk/ivy/libraries.properties Thu Mar 15 06:40:08 2012
@@ -37,9 +37,9 @@ commons-logging.version=1.0.4
commons-logging-api.version=1.0.4
commons-pool.version=1.5.4
derby.version=10.4.2.0
-guava.version=r06
-hbase.version=0.89.0-SNAPSHOT
-hbase-test.version=0.89.0-SNAPSHOT
+guava.version=r09
+hbase.version=0.92.0
+jackson.version=1.8.8
javaewah.version=0.3.2
jdo-api.version=2.3-ec
jdom.version=1.1
@@ -54,5 +54,4 @@ mockito-all.version=1.8.2
slf4j-api.version=1.6.1
slf4j-log4j12.version=1.6.1
velocity.version=1.5
-zookeeper.version=3.3.1
-
+zookeeper.version=3.4.3
Modified: hive/trunk/shims/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/shims/ivy.xml?rev=1300818&r1=1300817&r2=1300818&view=diff
==============================================================================
--- hive/trunk/shims/ivy.xml (original)
+++ hive/trunk/shims/ivy.xml Thu Mar 15 06:40:08 2012
@@ -29,6 +29,7 @@
<dependencies>
<dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop.version.ant-internal}">
<artifact name="hadoop" type="source" ext="tar.gz"/>
+ <exclude org="org.codehaus.jackson" name="*"/>
</dependency>
<dependency org="org.apache.zookeeper" name="zookeeper"
rev="${zookeeper.version}" transitive="false">
@@ -41,7 +42,9 @@
<dependency org="commons-logging" name="commons-logging-api" rev="${commons-logging-api.version}"
transitive="false"/>
<dependency org="com.google.guava" name="guava" rev="${guava.version}"
- transitive="false"/>
+ transitive="false"/>
+ <dependency org="org.codehaus.jackson" name="jackson-core-asl" rev="${jackson.version}"/>
+ <dependency org="org.codehaus.jackson" name="jackson-mapper-asl" rev="${jackson.version}"/>
<conflict manager="all" />
</dependencies>
</ivy-module>
Modified: hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/DelegationTokenStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/DelegationTokenStore.java?rev=1300818&r1=1300817&r2=1300818&view=diff
==============================================================================
--- hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/DelegationTokenStore.java (original)
+++ hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/DelegationTokenStore.java Thu Mar 15 06:40:08 2012
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.thrift;
+import java.io.Closeable;
import java.util.List;
import org.apache.hadoop.conf.Configurable;
@@ -27,7 +28,7 @@ import org.apache.hadoop.security.token.
* storage for load balancing and high availability (for example using ZooKeeper).
* Internal, store specific errors are translated into {@link TokenStoreException}.
*/
-public interface DelegationTokenStore extends Configurable {
+public interface DelegationTokenStore extends Configurable, Closeable {
/**
* Exception for internal token store errors that typically cannot be handled by the caller.
Modified: hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/MemoryTokenStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/MemoryTokenStore.java?rev=1300818&r1=1300817&r2=1300818&view=diff
==============================================================================
--- hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/MemoryTokenStore.java (original)
+++ hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/MemoryTokenStore.java Thu Mar 15 06:40:08 2012
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.thrift;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -101,4 +102,9 @@ public class MemoryTokenStore implements
return result;
}
+ @Override
+ public void close() throws IOException {
+ //no-op
+ }
+
}
Modified: hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java?rev=1300818&r1=1300817&r2=1300818&view=diff
==============================================================================
--- hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java (original)
+++ hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/thrift/ZooKeeperTokenStore.java Thu Mar 15 06:40:08 2012
@@ -58,7 +58,6 @@ public class ZooKeeperTokenStore impleme
private final int zkSessionTimeout = 3000;
private List<ACL> newNodeAcl = Ids.OPEN_ACL_UNSAFE;
-
private class ZooKeeperWatcher implements Watcher {
public void process(org.apache.zookeeper.WatchedEvent event) {
LOGGER.info(event.toString());
@@ -199,8 +198,8 @@ public class ZooKeeperTokenStore impleme
LOGGER.warn("Failed to close existing session.", ex);
}
}
-
ZooKeeper zk = getSession();
+
try {
ensurePath(zk, rootNode + NODE_KEYS, newNodeAcl);
ensurePath(zk, rootNode + NODE_TOKENS, newNodeAcl);
@@ -400,4 +399,15 @@ public class ZooKeeperTokenStore impleme
return result;
}
+ @Override
+ public void close() throws IOException {
+ if (this.zkSession != null) {
+ try {
+ this.zkSession.close();
+ } catch (InterruptedException ex) {
+ LOGGER.warn("Failed to close existing session.", ex);
+ }
+ }
+ }
+
}
Modified: hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java?rev=1300818&r1=1300817&r2=1300818&view=diff
==============================================================================
--- hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java (original)
+++ hive/trunk/shims/src/test/org/apache/hadoop/hive/thrift/TestZooKeeperTokenStore.java Thu Mar 15 06:40:08 2012
@@ -39,6 +39,8 @@ public class TestZooKeeperTokenStore ext
private MiniZooKeeperCluster zkCluster = null;
private ZooKeeper zkClient = null;
+ private int zkPort = -1;
+ private ZooKeeperTokenStore ts;
@Override
protected void setUp() throws Exception {
@@ -47,14 +49,17 @@ public class TestZooKeeperTokenStore ext
throw new IOException("Cluster already running");
}
this.zkCluster = new MiniZooKeeperCluster();
- this.zkCluster.startup(zkDataDir);
+ this.zkPort = this.zkCluster.startup(zkDataDir);
this.zkClient = new ZooKeeper("localhost:"
- + this.zkCluster.getClientPort(), 300, null);
+ + zkPort, 300, null);
}
@Override
protected void tearDown() throws Exception {
this.zkClient.close();
+ if (ts != null) {
+ ts.close();
+ }
this.zkCluster.shutdown();
this.zkCluster = null;
}
@@ -63,7 +68,7 @@ public class TestZooKeeperTokenStore ext
Configuration conf = new Configuration();
conf.set(
HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_CONNECT_STR,
- "localhost:" + this.zkCluster.getClientPort());
+ "localhost:" + this.zkPort);
conf.set(
HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ZNODE,
zkPath);
@@ -72,7 +77,7 @@ public class TestZooKeeperTokenStore ext
public void testTokenStorage() throws Exception {
String ZK_PATH = "/zktokenstore-testTokenStorage";
- ZooKeeperTokenStore ts = new ZooKeeperTokenStore();
+ ts = new ZooKeeperTokenStore();
ts.setConf(createConf(ZK_PATH));
int keySeq = ts.addMasterKey("key1Data");
@@ -124,7 +129,7 @@ public class TestZooKeeperTokenStore ext
HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ACL,
"ip:127.0.0.1:r");
- ZooKeeperTokenStore ts = new ZooKeeperTokenStore();
+ ts = new ZooKeeperTokenStore();
try {
ts.setConf(conf);
fail("expected ACL exception");
@@ -145,7 +150,7 @@ public class TestZooKeeperTokenStore ext
List<ACL> aclList = ZooKeeperTokenStore.parseACLs(aclString);
assertEquals(1, aclList.size());
- ZooKeeperTokenStore ts = new ZooKeeperTokenStore();
+ ts = new ZooKeeperTokenStore();
try {
ts.setConf(conf);
fail("expected ACL exception");
@@ -161,7 +166,7 @@ public class TestZooKeeperTokenStore ext
conf.set(
HadoopThriftAuthBridge20S.Server.DELEGATION_TOKEN_STORE_ZK_ACL,
"world:anyone:cdrwa,ip:127.0.0.1:cdrwa");
- ZooKeeperTokenStore ts = new ZooKeeperTokenStore();
+ ts = new ZooKeeperTokenStore();
ts.setConf(conf);
List<ACL> acl = zkClient.getACL(ZK_PATH, new Stat());
assertEquals(2, acl.size());