You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by he...@apache.org on 2010/09/16 20:23:40 UTC
svn commit: r997871 - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/
ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/
ql/src/test/templates/
Author: heyongqiang
Date: Thu Sep 16 18:23:40 2010
New Revision: 997871
URL: http://svn.apache.org/viewvc?rev=997871&view=rev
Log:
HIVE-1645. ability to specify parent directory for zookeeper lock manager.(namit via He Yongqiang)
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/trunk/conf/hive-default.xml
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
hadoop/hive/trunk/ql/src/test/templates/TestParse.vm
hadoop/hive/trunk/ql/src/test/templates/TestParseNegative.vm
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=997871&r1=997870&r2=997871&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Sep 16 18:23:40 2010
@@ -126,6 +126,9 @@ Trunk - Unreleased
HIVE-1614. UDTF json_tuple should return null row when input is not a valid
JSON string (Ning Zhang via namit)
+ HIVE-1645. ability to specify parent directory for zookeeper lock manager
+ (namit via He Yongqiang)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=997871&r1=997870&r2=997871&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Sep 16 18:23:40 2010
@@ -275,6 +275,7 @@ public class HiveConf extends Configurat
HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", ""),
HIVE_ZOOKEEPER_CLIENT_PORT("hive.zookeeper.client.port", ""),
HIVE_ZOOKEEPER_SESSION_TIMEOUT("hive.zookeeper.session.timeout", 600*1000),
+ HIVE_ZOOKEEPER_NAMESPACE("hive.zookeeper.namespace", "hive_zookeeper_namespace"),
// For HBase storage handler
HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true),
Modified: hadoop/hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/conf/hive-default.xml?rev=997871&r1=997870&r2=997871&view=diff
==============================================================================
--- hadoop/hive/trunk/conf/hive-default.xml (original)
+++ hadoop/hive/trunk/conf/hive-default.xml Thu Sep 16 18:23:40 2010
@@ -649,6 +649,12 @@
</property>
<property>
+ <name>hive.zookeeper.namespace</name>
+ <value>hive_zookeeper_namespace</value>
+ <description>The parent node under which all zookeeper nodes are created.</description>
+</property>
+
+<property>
<name>fs.har.impl</name>
<value>org.apache.hadoop.hive.shims.HiveHarFileSystem</value>
<description>The implementation for accessing Hadoop Archives. Note that this won't be applicable to Hadoop vers less than 0.20</description>
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java?rev=997871&r1=997870&r2=997871&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java Thu Sep 16 18:23:40 2010
@@ -34,6 +34,7 @@ import java.util.LinkedHashSet;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.zookeeper.KeeperException;
import org.apache.hadoop.hive.ql.parse.ErrorMsg;
import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager;
@@ -58,6 +59,9 @@ public class ZooKeeperHiveLockManager im
private ZooKeeper zooKeeper;
+ // All the locks are created under this parent
+ private String parent;
+
public ZooKeeperHiveLockManager() {
}
@@ -88,6 +92,15 @@ public class ZooKeeperHiveLockManager im
}
zooKeeper = new ZooKeeper(quorumServers, sessionTimeout, new DummyWatcher());
+ parent = conf.getVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_NAMESPACE);
+
+ try {
+ String par = zooKeeper.create("/" + parent, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ } catch (KeeperException e) {
+ // ignore if the parent already exists
+ }
+
+
} catch (Exception e) {
LOG.error("Failed to create ZooKeeper object: " + e);
throw new LockException(ErrorMsg.ZOOKEEPER_CLIENT_COULD_NOT_BE_INITIALIZED.getMsg());
@@ -99,7 +112,9 @@ public class ZooKeeperHiveLockManager im
* replace "/" by a dummy name to ensure a single hierarchy.
**/
private String getObjectName(HiveLockObject key, HiveLockMode mode) {
- return "/" + key.getName().replaceAll("/", ctx.getConf().getVar(HiveConf.ConfVars.DEFAULT_ZOOKEEPER_PARTITION_NAME)) + "-" + mode + "-";
+ return "/" + parent + "/" +
+ key.getName().replaceAll("/", ctx.getConf().getVar(HiveConf.ConfVars.DEFAULT_ZOOKEEPER_PARTITION_NAME)) +
+ "-" + mode + "-";
}
/**
@@ -127,13 +142,13 @@ public class ZooKeeperHiveLockManager im
return null;
}
- List<String> children = zooKeeper.getChildren("/", false);
+ List<String> children = zooKeeper.getChildren("/" + parent, false);
String exLock = getObjectName(key, HiveLockMode.EXCLUSIVE);
String shLock = getObjectName(key, HiveLockMode.SHARED);
for (String child : children) {
- child = "/" + child;
+ child = "/" + parent + "/" + child;
// Is there a conflicting lock on the same object with a lower sequence number
int childSeq = seqNo;
@@ -181,7 +196,9 @@ public class ZooKeeperHiveLockManager im
int sessionTimeout = conf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT);
String quorumServers = getQuorumServers(conf);
ZooKeeper zkpClient = new ZooKeeper(quorumServers, sessionTimeout, new DummyWatcher());
- List<HiveLock> locks = getLocks(conf, zkpClient, null);
+ String parent = conf.getVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_NAMESPACE);
+ List<HiveLock> locks = getLocks(conf, zkpClient, null, parent);
+
if (locks != null) {
for (HiveLock lock : locks) {
unlock(conf, zkpClient, lock);
@@ -198,12 +215,12 @@ public class ZooKeeperHiveLockManager im
/* Get all locks */
public List<HiveLock> getLocks() throws LockException {
- return getLocks(ctx.getConf(), zooKeeper, null);
+ return getLocks(ctx.getConf(), zooKeeper, null, parent);
}
/* Get all locks for a particular object */
public List<HiveLock> getLocks(HiveLockObject key) throws LockException {
- return getLocks(ctx.getConf(), zooKeeper, key);
+ return getLocks(ctx.getConf(), zooKeeper, key, parent);
}
/**
@@ -211,19 +228,20 @@ public class ZooKeeperHiveLockManager im
* @param zkpClient The ZooKeeper client
* @param key The object to be compared against - if key is null, then get all locks
**/
- private static List<HiveLock> getLocks(HiveConf conf, ZooKeeper zkpClient, HiveLockObject key) throws LockException {
+ private static List<HiveLock> getLocks(HiveConf conf, ZooKeeper zkpClient,
+ HiveLockObject key, String parent) throws LockException {
List<HiveLock> locks = new ArrayList<HiveLock>();
List<String> children;
try {
- children = zkpClient.getChildren("/", false);
+ children = zkpClient.getChildren("/" + parent, false);
} catch (Exception e) {
LOG.error("Failed to get ZooKeeper children: " + e);
throw new LockException(e);
}
for (String child : children) {
- child = "/" + child;
+ child = "/" + parent + "/" + child;
HiveLockMode mode = getLockMode(conf, child);
if (mode == null) {
@@ -231,7 +249,8 @@ public class ZooKeeperHiveLockManager im
}
HiveLockObject obj = getLockObject(conf, child, mode);
- if ((key == null) || (obj.getName().equals(key.getName()))) {
+ if ((key == null) ||
+ (obj.getName().equals(key.getName()))) {
HiveLock lck = (HiveLock)(new ZooKeeperHiveLock(child, obj, mode));
locks.add(lck);
}
@@ -276,8 +295,8 @@ public class ZooKeeperHiveLockManager im
Hive db = Hive.get(conf);
int indx = path.lastIndexOf(mode.toString());
String objName = path.substring(1, indx-1);
+ String[] names = objName.split("/")[1].split("@");
- String[] names = objName.split("@");
Table tab = db.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME,
names[1], false); // do not throw exception if table does not exist
assert (tab != null);
@@ -303,7 +322,7 @@ public class ZooKeeperHiveLockManager im
if (partn == null) {
return new HiveLockObject(new DummyPartition(
- objName.replaceAll(conf.getVar(HiveConf.ConfVars.DEFAULT_ZOOKEEPER_PARTITION_NAME), "/")));
+ objName.split("/")[1].replaceAll(conf.getVar(HiveConf.ConfVars.DEFAULT_ZOOKEEPER_PARTITION_NAME), "/")));
}
return new HiveLockObject(partn);
Modified: hadoop/hive/trunk/ql/src/test/templates/TestParse.vm
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/templates/TestParse.vm?rev=997871&r1=997870&r2=997871&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/templates/TestParse.vm (original)
+++ hadoop/hive/trunk/ql/src/test/templates/TestParse.vm Thu Sep 16 18:23:40 2010
@@ -51,9 +51,10 @@ public class $className extends TestCase
@Override
protected void tearDown() {
try {
- qt.clearPostTestEffects();
if (getName().equals("testParse_shutdown"))
qt.shutdown();
+ else
+ qt.clearPostTestEffects();
}
catch (Exception e) {
System.out.println("Exception: " + e.getMessage());
Modified: hadoop/hive/trunk/ql/src/test/templates/TestParseNegative.vm
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/templates/TestParseNegative.vm?rev=997871&r1=997870&r2=997871&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/templates/TestParseNegative.vm (original)
+++ hadoop/hive/trunk/ql/src/test/templates/TestParseNegative.vm Thu Sep 16 18:23:40 2010
@@ -51,9 +51,11 @@ public class $className extends TestCase
@Override
protected void tearDown() {
try {
- qt.clearPostTestEffects();
+
if (getName().equals("testParseNegative_shutdown"))
qt.shutdown();
+ else
+ qt.clearPostTestEffects();
}
catch (Exception e) {
System.out.println("Exception: " + e.getMessage());