You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/05/01 03:31:05 UTC
[11/11] hive git commit: Merge branch 'master' into llap
Merge branch 'master' into llap
Conflicts:
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9f999f25
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9f999f25
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9f999f25
Branch: refs/heads/llap
Commit: 9f999f252746a11c766624ca947e31f3fe59ec07
Parents: 390cb8c 9e1fa0c
Author: Jason Dere <jd...@hortonworks.com>
Authored: Sat Apr 30 18:30:25 2016 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Sat Apr 30 18:30:25 2016 -0700
----------------------------------------------------------------------
HIVE-13509.2.patch | 478 ++++++++++++
.../org/apache/hadoop/hive/conf/HiveConf.java | 10 +-
dev-support/jenkins-common.sh | 14 +
dev-support/jenkins-submit-build.sh | 4 +
.../hive/hcatalog/common/HCatConstants.java | 3 +
.../hcatalog/mapreduce/HCatBaseInputFormat.java | 29 +-
.../hive/hcatalog/pig/TestHCatLoader.java | 55 ++
.../service/cli/session/TestQueryDisplay.java | 4 +-
.../test/resources/testconfiguration.properties | 1 +
.../impl/LlapZookeeperRegistryImpl.java | 74 +-
.../hive/llap/tezplugins/ContainerFactory.java | 3 +-
.../tezplugins/LlapTaskSchedulerService.java | 377 ++++++++--
.../llap/tezplugins/helpers/MonotonicClock.java | 24 +
.../scheduler/LoggingFutureCallback.java | 44 ++
.../TestLlapTaskSchedulerService.java | 734 ++++++++++++++++++-
.../hadoop/hive/metastore/HiveMetaStore.java | 2 +-
.../metastore/TestHiveMetaStoreGetMetaConf.java | 151 ++++
.../java/org/apache/hadoop/hive/ql/Driver.java | 13 +-
.../org/apache/hadoop/hive/ql/QueryDisplay.java | 19 +-
.../org/apache/hadoop/hive/ql/QueryPlan.java | 16 +-
.../hadoop/hive/ql/exec/ConditionalTask.java | 1 +
.../apache/hadoop/hive/ql/exec/Registry.java | 323 +++++---
.../org/apache/hadoop/hive/ql/exec/Task.java | 14 +-
.../hadoop/hive/ql/exec/mr/ExecDriver.java | 2 +-
.../hive/ql/exec/mr/HadoopJobExecHelper.java | 6 +-
.../hadoop/hive/ql/exec/mr/MapredLocalTask.java | 16 +-
.../hadoop/hive/ql/exec/tez/DagUtils.java | 1 -
.../apache/hadoop/hive/ql/metadata/Hive.java | 39 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +-
.../hadoop/hive/ql/session/SessionState.java | 3 +-
.../clientpositive/auto_sortmerge_join_8.q.out | 2 -
.../results/clientpositive/groupby1_limit.q.out | 2 +-
.../clientpositive/llap/tez_join_hash.q.out | 4 -
.../clientpositive/tez/tez_join_hash.q.out | 4 -
.../apache/hive/service/cli/CLIServiceTest.java | 4 +-
.../apache/hadoop/hive/shims/ShimLoader.java | 10 +-
36 files changed, 2172 insertions(+), 316 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9f999f25/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --cc common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index d7afa4d,b13de92..7db492f
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@@ -2806,8 -2808,9 +2811,11 @@@ public class HiveConf extends Configura
false,
"Whether to setup split locations to match nodes on which llap daemons are running," +
" instead of using the locations provided by the split itself"),
+ LLAP_VALIDATE_ACLS("hive.llap.validate.acls", true,
+ "Whether LLAP should reject permissive ACLs in some cases (e.g. its own management\n" +
+ "protocol or ZK paths), similar to how ssh refuses a key with bad access permissions."),
+ LLAP_DAEMON_OUTPUT_SERVICE_PORT("hive.llap.daemon.output.service.port", 15003,
+ "LLAP daemon output service port"),
SPARK_CLIENT_FUTURE_TIMEOUT("hive.spark.client.future.timeout",
"60s", new TimeValidator(TimeUnit.SECONDS),
http://git-wip-us.apache.org/repos/asf/hive/blob/9f999f25/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
----------------------------------------------------------------------
diff --cc llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
index 0dc7599,6981061..fde70e7
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
@@@ -87,8 -89,8 +89,9 @@@ public class LlapZookeeperRegistryImpl
private static final String IPC_MNG = "llapmng";
private static final String IPC_SHUFFLE = "shuffle";
private static final String IPC_LLAP = "llap";
+ private static final String IPC_OUTPUTFORMAT = "llapoutputformat";
private final static String ROOT_NAMESPACE = "llap";
+ private final static String USER_SCOPE_PATH_PREFIX = "user-";
private final Configuration conf;
private final CuratorFramework zooKeeperClient;
@@@ -173,7 -177,8 +178,8 @@@
// worker does not respond due to communication interruptions it will retain the same sequence
// number when it returns back. If session timeout expires, the node will be deleted and new
// addition of the same node (restart) will get next sequence number
- this.pathPrefix = "/" + getZkPathUser(this.conf) + "/" + instanceName + "/workers/worker-";
- this.userPathPrefix = USER_SCOPE_PATH_PREFIX + RegistryUtils.currentUser();
++ this.userPathPrefix = USER_SCOPE_PATH_PREFIX + getZkPathUser(this.conf);
+ this.pathPrefix = "/" + userPathPrefix + "/" + instanceName + "/workers/worker-";
this.instancesCache = null;
this.instances = null;
this.stateChangeListeners = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/hive/blob/9f999f25/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java
----------------------------------------------------------------------