You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/07/31 02:43:04 UTC
[04/43] hive git commit: HIVE-11311: Avoid dumping AST tree String in
Explain unless necessary (Jesus Camacho Rodriguez,
reviewed by Hari Sankar Sivarama Subramaniyan)
HIVE-11311: Avoid dumping AST tree String in Explain unless necessary (Jesus Camacho Rodriguez, reviewed by Hari Sankar Sivarama Subramaniyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dfdc6700
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dfdc6700
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dfdc6700
Branch: refs/heads/spark
Commit: dfdc6700c7a81320a3e0909305645db0248bf01a
Parents: 55853b4
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Tue Jul 21 17:38:39 2015 +0100
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Tue Jul 21 17:38:39 2015 +0100
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/Driver.java | 6 ++--
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 7 ++--
.../hive/ql/parse/ExplainSemanticAnalyzer.java | 2 +-
.../apache/hadoop/hive/ql/plan/ExplainWork.java | 19 +++++++----
.../parse/TestUpdateDeleteSemanticAnalyzer.java | 34 ++++++++++----------
5 files changed, 37 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index b74e5fa..6ad3f49 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -482,7 +482,7 @@ public class Driver implements CommandProcessor {
}
if (conf.getBoolVar(ConfVars.HIVE_LOG_EXPLAIN_OUTPUT)) {
- String explainOutput = getExplainOutput(sem, plan, tree.dump());
+ String explainOutput = getExplainOutput(sem, plan, tree);
if (explainOutput != null) {
LOG.info("EXPLAIN output for queryid " + queryId + " : "
+ explainOutput);
@@ -533,7 +533,7 @@ public class Driver implements CommandProcessor {
* @throws java.io.IOException
*/
private String getExplainOutput(BaseSemanticAnalyzer sem, QueryPlan plan,
- String astStringTree) throws IOException {
+ ASTNode astTree) throws IOException {
String ret = null;
ExplainTask task = new ExplainTask();
task.initialize(conf, plan, null);
@@ -541,7 +541,7 @@ public class Driver implements CommandProcessor {
PrintStream ps = new PrintStream(baos);
try {
List<Task<?>> rootTasks = sem.getRootTasks();
- task.getJSONPlan(ps, astStringTree, rootTasks, sem.getFetchTask(), false, true, true);
+ task.getJSONPlan(ps, astTree, rootTasks, sem.getFetchTask(), false, true, true);
ret = baos.toString();
} catch (Exception e) {
LOG.warn("Exception generating explain output: " + e, e);
http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index c6b49bf..21de3cf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
import org.apache.hadoop.hive.ql.plan.Explain;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
@@ -171,11 +172,11 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
public JSONObject getJSONPlan(PrintStream out, ExplainWork work)
throws Exception {
- return getJSONPlan(out, work.getAstStringTree(), work.getRootTasks(), work.getFetchTask(),
+ return getJSONPlan(out, work.getAstTree(), work.getRootTasks(), work.getFetchTask(),
work.isFormatted(), work.getExtended(), work.isAppendTaskType());
}
- public JSONObject getJSONPlan(PrintStream out, String ast, List<Task<?>> tasks, Task<?> fetchTask,
+ public JSONObject getJSONPlan(PrintStream out, ASTNode ast, List<Task<?>> tasks, Task<?> fetchTask,
boolean jsonOutput, boolean isExtended, boolean appendTaskType) throws Exception {
// If the user asked for a formatted output, dump the json output
@@ -188,7 +189,7 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
// Print out the parse AST
if (ast != null && isExtended) {
- String jsonAST = outputAST(ast, out, jsonOutput, 0);
+ String jsonAST = outputAST(ast.dump(), out, jsonOutput, 0);
if (out != null) {
out.println();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
index 66d1546..2d365a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
@@ -102,7 +102,7 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer {
pCtx,
tasks,
fetchTask,
- input.dump(),
+ input,
sem,
extended,
formatted,
http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
index aa208a5..7a2f883 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
@@ -26,8 +26,9 @@ import java.util.List;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
-import org.apache.hadoop.hive.ql.parse.ParseContext;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
/**
* ExplainWork.
@@ -39,6 +40,7 @@ public class ExplainWork implements Serializable {
private Path resFile;
private ArrayList<Task<? extends Serializable>> rootTasks;
private Task<? extends Serializable> fetchTask;
+ private ASTNode astTree;
private String astStringTree;
private HashSet<ReadEntity> inputs;
private ParseContext pCtx;
@@ -63,7 +65,7 @@ public class ExplainWork implements Serializable {
ParseContext pCtx,
List<Task<? extends Serializable>> rootTasks,
Task<? extends Serializable> fetchTask,
- String astStringTree,
+ ASTNode astTree,
BaseSemanticAnalyzer analyzer,
boolean extended,
boolean formatted,
@@ -75,7 +77,7 @@ public class ExplainWork implements Serializable {
this.resFile = resFile;
this.rootTasks = new ArrayList<Task<? extends Serializable>>(rootTasks);
this.fetchTask = fetchTask;
- this.astStringTree = astStringTree;
+ this.astTree = astTree;
this.analyzer = analyzer;
this.inputs = analyzer.getInputs();
this.extended = extended;
@@ -112,12 +114,15 @@ public class ExplainWork implements Serializable {
this.fetchTask = fetchTask;
}
- public String getAstStringTree() {
- return astStringTree;
+ public ASTNode getAstTree() {
+ return astTree;
}
- public void setAstStringTree(String astStringTree) {
- this.astStringTree = astStringTree;
+ public String getAstStringTree() {
+ if (astStringTree == null) {
+ astStringTree = astTree.dump();
+ }
+ return astStringTree;
}
public HashSet<ReadEntity> getInputs() {
http://git-wip-us.apache.org/repos/asf/hive/blob/dfdc6700/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
index f0435cb..f78f226 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
@@ -59,7 +59,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
try {
ReturnInfo rc = parseAndAnalyze("insert into table T select a, b from U", "testInsertSelect");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
@@ -70,7 +70,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
public void testDeleteAllNonPartitioned() throws Exception {
try {
ReturnInfo rc = parseAndAnalyze("delete from T", "testDeleteAllNonPartitioned");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -80,7 +80,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
public void testDeleteWhereNoPartition() throws Exception {
try {
ReturnInfo rc = parseAndAnalyze("delete from T where a > 5", "testDeleteWhereNoPartition");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -90,7 +90,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
public void testDeleteAllPartitioned() throws Exception {
try {
ReturnInfo rc = parseAndAnalyze("delete from U", "testDeleteAllPartitioned");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -100,7 +100,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
public void testDeleteAllWherePartitioned() throws Exception {
try {
ReturnInfo rc = parseAndAnalyze("delete from U where a > 5", "testDeleteAllWherePartitioned");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -111,7 +111,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
try {
ReturnInfo rc = parseAndAnalyze("delete from U where ds = 'today'",
"testDeleteFromPartitionOnly");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -122,7 +122,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
try {
ReturnInfo rc = parseAndAnalyze("delete from U where ds = 'today' and a > 5",
"testDeletePartitionWhere");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -132,7 +132,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
public void testUpdateAllNonPartitioned() throws Exception {
try {
ReturnInfo rc = parseAndAnalyze("update T set b = 5", "testUpdateAllNonPartitioned");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -143,7 +143,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
try {
ReturnInfo rc = parseAndAnalyze("update T set b = 5 where b > 5",
"testUpdateAllNonPartitionedWhere");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -153,7 +153,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
public void testUpdateAllPartitioned() throws Exception {
try {
ReturnInfo rc = parseAndAnalyze("update U set b = 5", "testUpdateAllPartitioned");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -164,7 +164,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
try {
ReturnInfo rc = parseAndAnalyze("update U set b = 5 where b > 5",
"testUpdateAllPartitionedWhere");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -175,7 +175,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
try {
ReturnInfo rc = parseAndAnalyze("update U set b = 5 where ds = 'today'",
"testUpdateOnePartition");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -186,7 +186,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
try {
ReturnInfo rc = parseAndAnalyze("update U set b = 5 where ds = 'today' and b > 5",
"testUpdateOnePartitionWhere");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
}
@@ -198,7 +198,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
ReturnInfo rc = parseAndAnalyze("insert into table T values ('abc', 3), ('ghi', null)",
"testInsertValues");
- LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
@@ -212,7 +212,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
"('abc', 3, 'today'), ('ghi', 5, 'tomorrow')",
"testInsertValuesPartitioned");
- LOG.info(explain((SemanticAnalyzer) rc.sem, rc.plan, rc.ast.dump()));
+ LOG.info(explain((SemanticAnalyzer) rc.sem, rc.plan, rc.ast));
} finally {
cleanupTables();
@@ -285,7 +285,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
return new ReturnInfo(tree, sem, plan);
}
- private String explain(SemanticAnalyzer sem, QueryPlan plan, String astStringTree) throws
+ private String explain(SemanticAnalyzer sem, QueryPlan plan, ASTNode astTree) throws
IOException {
FileSystem fs = FileSystem.get(conf);
File f = File.createTempFile("TestSemanticAnalyzer", "explain");
@@ -293,7 +293,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
fs.create(tmp);
fs.deleteOnExit(tmp);
ExplainWork work = new ExplainWork(tmp, sem.getParseContext(), sem.getRootTasks(),
- sem.getFetchTask(), astStringTree, sem, true, false, false, false, false, false, null);
+ sem.getFetchTask(), astTree, sem, true, false, false, false, false, false, null);
ExplainTask task = new ExplainTask();
task.setWork(work);
task.initialize(conf, plan, null);