You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by vi...@apache.org on 2016/05/10 21:04:27 UTC
hive git commit: HIVE-13342: Improve logging in llap decider and
throw exception in case llap mode is all but we cannot run in llap. (Vikram
Dixit K, reviewed by Sergey Shelukhin and Gunther Hagleitner)
Repository: hive
Updated Branches:
refs/heads/master e3b6c9b1e -> e087ba845
HIVE-13342: Improve logging in llap decider and throw exception in case llap mode is all but we cannot run in llap. (Vikram Dixit K, reviewed by Sergey Shelukhin and Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e087ba84
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e087ba84
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e087ba84
Branch: refs/heads/master
Commit: e087ba84537555eb37b0047e7ace3f1b9dc06d08
Parents: e3b6c9b
Author: vikram <vi...@hortonworks.com>
Authored: Tue May 10 14:01:30 2016 -0700
Committer: vikram <vi...@hortonworks.com>
Committed: Tue May 10 14:03:29 2016 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/physical/LlapDecider.java | 37 +++++++++++++++-----
ql/src/test/queries/clientpositive/llap_udf.q | 5 +++
.../results/clientpositive/llap/llap_udf.q.out | 2 --
3 files changed, 34 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e087ba84/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
index c0b7a32..e6f8f75 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
@@ -132,6 +132,12 @@ public class LlapDecider implements PhysicalPlanResolver {
throws SemanticException {
if (evaluateWork(tezWork, work)) {
convertWork(tezWork, work);
+ } else {
+ if (mode == all) {
+ throw new SemanticException("Llap mode is set to all but cannot run work in llap mode." +
+ "Set " + HiveConf.ConfVars.LLAP_EXECUTION_MODE + " = auto or set " +
+ HiveConf.ConfVars.HIVE_EXECUTION_MODE + " = container");
+ }
}
}
@@ -228,6 +234,7 @@ public class LlapDecider implements PhysicalPlanResolver {
}
// couldn't convince you otherwise? well then let's llap.
+ LOG.info("Can run work " + work.getName() + " in llap mode.");
return true;
}
@@ -304,6 +311,7 @@ public class LlapDecider implements PhysicalPlanResolver {
@Override
public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
Object... os) {
+ LOG.debug("Cannot run operator [" + n + "] in llap mode.");
return new Boolean(false);
}
});
@@ -313,7 +321,11 @@ public class LlapDecider implements PhysicalPlanResolver {
public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
Object... os) {
ExprNodeDesc expr = ((FilterOperator)n).getConf().getPredicate();
- return new Boolean(checkExpression(expr));
+ Boolean retval = new Boolean(checkExpression(expr));
+ if (!retval) {
+ LOG.info("Cannot run filter operator [" + n + "] in llap mode");
+ }
+ return new Boolean(retval);
}
});
opRules.put(new RuleRegExp("No user code in gby", GroupByOperator.getOperatorName() + "%"),
@@ -321,9 +333,13 @@ public class LlapDecider implements PhysicalPlanResolver {
@Override
public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
Object... os) {
- @SuppressWarnings("unchecked")
- List<AggregationDesc> aggs = ((Operator<GroupByDesc>) n).getConf().getAggregators();
- return new Boolean(checkAggregators(aggs));
+ @SuppressWarnings("unchecked")
+ List<AggregationDesc> aggs = ((Operator<GroupByDesc>) n).getConf().getAggregators();
+ Boolean retval = new Boolean(checkAggregators(aggs));
+ if (!retval) {
+ LOG.info("Cannot run group by operator [" + n + "] in llap mode");
+ }
+ return new Boolean(retval);
}
});
opRules.put(new RuleRegExp("No user code in select", SelectOperator.getOperatorName() + "%"),
@@ -331,9 +347,13 @@ public class LlapDecider implements PhysicalPlanResolver {
@Override
public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
Object... os) {
- @SuppressWarnings({ "unchecked" })
- List<ExprNodeDesc> exprs = ((Operator<SelectDesc>) n).getConf().getColList();
- return new Boolean(checkExpressions(exprs));
+ @SuppressWarnings({ "unchecked" })
+ List<ExprNodeDesc> exprs = ((Operator<SelectDesc>) n).getConf().getColList();
+ Boolean retval = new Boolean(checkExpressions(exprs));
+ if (!retval) {
+ LOG.info("Cannot run select operator [" + n + "] in llap mode");
+ }
+ return new Boolean(retval);
}
});
@@ -355,6 +375,7 @@ public class LlapDecider implements PhysicalPlanResolver {
for (Node n : nodeOutput.keySet()) {
if (nodeOutput.get(n) != null) {
if (!((Boolean)nodeOutput.get(n))) {
+ LOG.info("Cannot run in LLAP mode.");
return false;
}
}
@@ -379,7 +400,7 @@ public class LlapDecider implements PhysicalPlanResolver {
Arrays.asList(pd.getInputFileFormatClass().getInterfaces());
if (!interfaceList.contains(VectorizedInputFormatInterface.class)) {
LOG.info("Input format: " + pd.getInputFileFormatClassName()
- + ", doesn't provide vectorized input");
+ + ", doesn't provide vectorized input");
return false;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e087ba84/ql/src/test/queries/clientpositive/llap_udf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/llap_udf.q b/ql/src/test/queries/clientpositive/llap_udf.q
index 2224bd5..8b1dcb7 100644
--- a/ql/src/test/queries/clientpositive/llap_udf.q
+++ b/ql/src/test/queries/clientpositive/llap_udf.q
@@ -12,8 +12,10 @@ create table src_orc stored as orc as select * from src;
CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE';
+set hive.llap.execution.mode=auto;
EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc;
+set hive.llap.execution.mode=all;
CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString';
CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString';
CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE';
@@ -21,18 +23,21 @@ CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEv
EXPLAIN
SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc;
+set hive.llap.execution.mode=auto;
-- Verification is based on classes, so 0 would work based on 4.
EXPLAIN
SELECT test_udf0(cast(key as string)) from src_orc;
DROP FUNCTION test_udf2;
+set hive.llap.execution.mode=all;
-- ...verify that 3 still works
EXPLAIN
SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc;
DROP FUNCTION test_udf4;
+set hive.llap.execution.mode=auto;
-- ...now 0 should stop working
EXPLAIN
SELECT test_udf0(cast(key as string)) from src_orc;
http://git-wip-us.apache.org/repos/asf/hive/blob/e087ba84/ql/src/test/results/clientpositive/llap/llap_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/llap_udf.q.out b/ql/src/test/results/clientpositive/llap/llap_udf.q.out
index 6197cf0..e517942 100644
--- a/ql/src/test/results/clientpositive/llap/llap_udf.q.out
+++ b/ql/src/test/results/clientpositive/llap/llap_udf.q.out
@@ -157,8 +157,6 @@ STAGE PLANS:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: llap
- LLAP IO: no inputs
Stage: Stage-0
Fetch Operator