You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/07/06 22:20:27 UTC
hive git commit: HIVE-14163 : LLAP: use different
kerberized/unkerberized zk paths for registry (Sergey Shelukhin,
reviewed by Siddharth Seth)
Repository: hive
Updated Branches:
refs/heads/branch-2.1 b70624127 -> 916c081c5
HIVE-14163 : LLAP: use different kerberized/unkerberized zk paths for registry (Sergey Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/916c081c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/916c081c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/916c081c
Branch: refs/heads/branch-2.1
Commit: 916c081c50d07fd9e657a5fabaaf1b2d52d8bcce
Parents: b706241
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Jul 6 15:16:10 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Jul 6 15:16:22 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../impl/LlapZookeeperRegistryImpl.java | 72 ++++++++++----------
2 files changed, 40 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/916c081c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 19a3a3d..88abb3f 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2770,6 +2770,9 @@ public class HiveConf extends Configuration {
LLAP_ZK_REGISTRY_USER("hive.llap.zk.registry.user", "",
"In the LLAP ZooKeeper-based registry, specifies the username in the Zookeeper path.\n" +
"This should be the hive user or whichever user is running the LLAP daemon."),
+ LLAP_ZK_REGISTRY_NAMESPACE("hive.llap.zk.registry.namespace", null,
+ "In the LLAP ZooKeeper-based registry, overrides the ZK path namespace. Note that\n" +
+ "using this makes the path management (e.g. setting correct ACLs) your responsibility."),
// Note: do not rename to ..service.acl; Hadoop generates .hosts setting name from this,
// resulting in a collision with existing hive.llap.daemon.service.hosts and bizarre errors.
// These are read by Hadoop IPC, so you should check the usage and naming conventions (e.g.
http://git-wip-us.apache.org/repos/asf/hive/blob/916c081c/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
index 154081d..6b0a42e 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
@@ -90,7 +90,8 @@ public class LlapZookeeperRegistryImpl implements ServiceRegistry {
private static final String IPC_SHUFFLE = "shuffle";
private static final String IPC_LLAP = "llap";
private static final String IPC_OUTPUTFORMAT = "llapoutputformat";
- private final static String ROOT_NAMESPACE = "llap";
+ private final static String SASL_NAMESPACE = "llap-sasl";
+ private final static String UNSECURE_NAMESPACE = "llap-unsecure";
private final static String USER_SCOPE_PATH_PREFIX = "user-";
private static final String DISABLE_MESSAGE =
"Set " + ConfVars.LLAP_VALIDATE_ACLS.varname + " to false to disable ACL validation";
@@ -126,29 +127,6 @@ public class LlapZookeeperRegistryImpl implements ServiceRegistry {
hostname = localhost;
}
- /**
- * ACLProvider for providing appropriate ACLs to CuratorFrameworkFactory
- */
- private final ACLProvider zooKeeperAclProvider = new ACLProvider() {
-
- @Override
- public List<ACL> getDefaultAcl() {
- // We always return something from getAclForPath so this should not happen.
- LOG.warn("getDefaultAcl was called");
- return Lists.newArrayList(ZooDefs.Ids.OPEN_ACL_UNSAFE);
- }
-
- @Override
- public List<ACL> getAclForPath(String path) {
- if (!UserGroupInformation.isSecurityEnabled() || path == null
- || !path.contains(userPathPrefix)) {
- // No security or the path is below the user path - full access.
- return Lists.newArrayList(ZooDefs.Ids.OPEN_ACL_UNSAFE);
- }
- return createSecureAcls();
- }
- };
-
public LlapZookeeperRegistryImpl(String instanceName, Configuration conf) {
this.conf = new Configuration(conf);
this.conf.addResource(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
@@ -161,17 +139,7 @@ public class LlapZookeeperRegistryImpl implements ServiceRegistry {
TimeUnit.MILLISECONDS);
int maxRetries = HiveConf.getIntVar(conf, ConfVars.HIVE_ZOOKEEPER_CONNECTION_MAX_RETRIES);
- // Create a CuratorFramework instance to be used as the ZooKeeper client
- // Use the zooKeeperAclProvider to create appropriate ACLs
- this.zooKeeperClient = CuratorFrameworkFactory.builder()
- .connectString(zkEnsemble)
- .sessionTimeoutMs(sessionTimeout)
- .aclProvider(zooKeeperAclProvider)
- .namespace(ROOT_NAMESPACE)
- .retryPolicy(new ExponentialBackoffRetry(baseSleepTime, maxRetries))
- .build();
-
- // sample path: /llap/hiveuser/hostname/workers/worker-0000000
+ // sample path: /llap-sasl/hiveuser/hostname/workers/worker-0000000
// worker-0000000 is the sequence number which will be retained until session timeout. If a
// worker does not respond due to communication interruptions it will retain the same sequence
// number when it returns back. If session timeout expires, the node will be deleted and new
@@ -181,6 +149,40 @@ public class LlapZookeeperRegistryImpl implements ServiceRegistry {
this.instancesCache = null;
this.instances = null;
this.stateChangeListeners = new HashSet<>();
+
+ final boolean isSecure = UserGroupInformation.isSecurityEnabled();
+ ACLProvider zooKeeperAclProvider = new ACLProvider() {
+ @Override
+ public List<ACL> getDefaultAcl() {
+ // We always return something from getAclForPath so this should not happen.
+ LOG.warn("getDefaultAcl was called");
+ return Lists.newArrayList(ZooDefs.Ids.OPEN_ACL_UNSAFE);
+ }
+
+ @Override
+ public List<ACL> getAclForPath(String path) {
+ if (!isSecure || path == null || !path.contains(userPathPrefix)) {
+ // No security or the path is below the user path - full access.
+ return Lists.newArrayList(ZooDefs.Ids.OPEN_ACL_UNSAFE);
+ }
+ return createSecureAcls();
+ }
+ };
+ String rootNs = HiveConf.getVar(conf, ConfVars.LLAP_ZK_REGISTRY_NAMESPACE);
+ if (rootNs == null) {
+ rootNs = isSecure ? SASL_NAMESPACE : UNSECURE_NAMESPACE; // The normal path.
+ }
+
+ // Create a CuratorFramework instance to be used as the ZooKeeper client
+ // Use the zooKeeperAclProvider to create appropriate ACLs
+ this.zooKeeperClient = CuratorFrameworkFactory.builder()
+ .connectString(zkEnsemble)
+ .sessionTimeoutMs(sessionTimeout)
+ .aclProvider(zooKeeperAclProvider)
+ .namespace(rootNs)
+ .retryPolicy(new ExponentialBackoffRetry(baseSleepTime, maxRetries))
+ .build();
+
LOG.info("Llap Zookeeper Registry is enabled with registryid: " + instanceName);
}