You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/05/06 17:25:03 UTC
[13/39] hive git commit: HIVE-13389: LLAP external submission client
ends up attempting to find an LLAP instance based on the submitting user
instead of the hive user
HIVE-13389: LLAP external submission client ends up attempting to find an LLAP instance based on the submitting user instead of the hive user
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/28d1082b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/28d1082b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/28d1082b
Branch: refs/heads/master
Commit: 28d1082b4a2acfa46beddc52e4d3944e783b6f21
Parents: 134f2f7
Author: Jason Dere <jd...@hortonworks.com>
Authored: Wed Mar 30 14:35:23 2016 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Wed Mar 30 14:35:23 2016 -0700
----------------------------------------------------------------------
.../src/java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++
.../org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java | 4 +++-
.../llap/registry/impl/LlapZookeeperRegistryImpl.java | 9 ++++++++-
.../org/apache/hadoop/hive/llap/LlapInputFormat.java | 3 +++
ql/pom.xml | 6 ++++++
.../java/org/apache/hadoop/hive/llap/LlapInputSplit.java | 11 +++++++++--
.../hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java | 4 +++-
7 files changed, 35 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 88b4387..ef32b2b 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2591,6 +2591,9 @@ public class HiveConf extends Configuration {
"ZooKeeper for ZooKeeper SecretManager."),
LLAP_ZKSM_ZK_CONNECTION_STRING("hive.llap.zk.sm.connectionString", "",
"ZooKeeper connection string for ZooKeeper SecretManager."),
+ LLAP_ZK_REGISTRY_USER("hive.llap.zk.registry.user", "",
+ "In the LLAP ZooKeeper-based registry, specifies the username in the Zookeeper path.\n" +
+ "This should be the hive user or whichever user is running the LLAP daemon."),
// Note: do not rename to ..service.acl; Hadoop generates .hosts setting name from this,
// resulting in a collision with existing hive.llap.daemon.service.hosts and bizarre errors.
LLAP_SECURITY_ACL("hive.llap.daemon.acl", "*", "The ACL for LLAP daemon."),
http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java
index b8b69a7..338930e 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/jdbc/TestLlapInputSplit.java
@@ -46,7 +46,8 @@ public class TestLlapInputSplit {
planBytes,
fragmentBytes,
locations,
- schema);
+ schema,
+ "hive");
ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(byteOutStream);
split1.write(dataOut);
@@ -94,6 +95,7 @@ public class TestLlapInputSplit {
}
assertArrayEquals(split1.getLocations(), split2.getLocations());
assertEquals(split1.getSchema(), split2.getSchema());
+ assertEquals(split1.getLlapUser(), split2.getLlapUser());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
index c440e1e..0536589 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
@@ -176,7 +176,7 @@ public class LlapZookeeperRegistryImpl implements ServiceRegistry {
// worker does not respond due to communication interruptions it will retain the same sequence
// number when it returns back. If session timeout expires, the node will be deleted and new
// addition of the same node (restart) will get next sequence number
- this.pathPrefix = "/" + RegistryUtils.currentUser() + "/" + instanceName + "/workers/worker-";
+ this.pathPrefix = "/" + getZkPathUser(this.conf) + "/" + instanceName + "/workers/worker-";
this.instancesCache = null;
this.instances = null;
this.stateChangeListeners = new HashSet<>();
@@ -210,6 +210,13 @@ public class LlapZookeeperRegistryImpl implements ServiceRegistry {
return quorum.toString();
}
+ private String getZkPathUser(Configuration conf) {
+ // External LLAP clients would need to set LLAP_ZK_REGISTRY_USER to the LLAP daemon user (hive),
+ // rather than relying on RegistryUtils.currentUser().
+ String user = HiveConf.getVar(conf, ConfVars.LLAP_ZK_REGISTRY_USER, RegistryUtils.currentUser());
+ return user;
+ }
+
public Endpoint getRpcEndpoint() {
final int rpcPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_RPC_PORT);
return RegistryTypeUtils.ipcEndpoint(IPC_LLAP, new InetSocketAddress(hostname, rpcPort));
http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapInputFormat.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/LlapInputFormat.java b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapInputFormat.java
index 7f11e1d..aaca7d6 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/LlapInputFormat.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapInputFormat.java
@@ -88,6 +88,9 @@ public class LlapInputFormat<V extends WritableComparable> implements InputForma
Reporter reporter) throws IOException {
LlapInputSplit llapSplit = (LlapInputSplit) split;
+
+ // Set conf to use LLAP user rather than current user for LLAP Zk registry.
+ HiveConf.setVar(job, HiveConf.ConfVars.LLAP_ZK_REGISTRY_USER, llapSplit.getLlapUser());
SubmitWorkInfo submitWorkInfo = SubmitWorkInfo.fromBytes(llapSplit.getPlanBytes());
ServiceInstance serviceInstance = getServiceInstance(job, llapSplit);
http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/ql/pom.xml
----------------------------------------------------------------------
diff --git a/ql/pom.xml b/ql/pom.xml
index ebb9599..e0de1fb 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -286,6 +286,12 @@
<version>${hadoop.version}</version>
<optional>true</optional>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ <version>${hadoop.version}</version>
+ <optional>true</optional>
+ </dependency>
<dependency>
<groupId>org.apache.ivy</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/ql/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java b/ql/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
index 2ac0ccd..17a0d2d 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
@@ -33,16 +33,18 @@ public class LlapInputSplit implements InputSplitWithLocationInfo {
byte[] fragmentBytes;
SplitLocationInfo[] locations;
Schema schema;
+ String llapUser;
public LlapInputSplit() {
}
- public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes, SplitLocationInfo[] locations, Schema schema) {
+ public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes, SplitLocationInfo[] locations, Schema schema, String llapUser) {
this.planBytes = planBytes;
this.fragmentBytes = fragmentBytes;
this.locations = locations;
this.schema = schema;
this.splitNum = splitNum;
+ this.llapUser = llapUser;
}
public Schema getSchema() {
@@ -102,7 +104,7 @@ public class LlapInputSplit implements InputSplitWithLocationInfo {
throw new IOException(e);
}
-
+ out.writeUTF(llapUser);
}
@Override
@@ -134,10 +136,15 @@ public class LlapInputSplit implements InputSplitWithLocationInfo {
} catch (Exception e) {
throw new IOException(e);
}
+ llapUser = in.readUTF();
}
@Override
public SplitLocationInfo[] getLocationInfo() throws IOException {
return locations;
}
+
+ public String getLlapUser() {
+ return llapUser;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/28d1082b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
index ebb0ca5..6267324 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SplitLocationInfo;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -319,6 +320,7 @@ public class GenericUDTFGetSplits extends GenericUDTF {
ApplicationId fakeApplicationId
= ApplicationId.newInstance(Math.abs(new Random().nextInt()), 0);
+ String llapUser = RegistryUtils.currentUser();
LOG.info("Number of splits: " + (eventList.size() - 1));
for (int i = 0; i < eventList.size() - 1; i++) {
@@ -355,7 +357,7 @@ public class GenericUDTFGetSplits extends GenericUDTF {
byte[] submitWorkBytes = SubmitWorkInfo.toBytes(submitWorkInfo);
- result[i] = new LlapInputSplit(i, submitWorkBytes, dob.getData(), locations, schema);
+ result[i] = new LlapInputSplit(i, submitWorkBytes, dob.getData(), locations, schema, llapUser);
}
return result;
} catch (Exception e) {