You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2014/10/27 23:59:12 UTC
svn commit: r1634725 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/optimizer/index/
java/org/apache/hadoop/hive/ql/parse/ test/queries/clientpositive/
test/results/clientpositive/
Author: sershe
Date: Mon Oct 27 22:59:12 2014
New Revision: 1634725
URL: http://svn.apache.org/r1634725
Log:
HIVE-8021 : CBO: support CTAS and insert ... select (Sergey Shelukhin, reviewed by Laljo John Pullokkaran)
Added:
hive/trunk/ql/src/test/queries/clientpositive/insert0.q
hive/trunk/ql/src/test/results/clientpositive/insert0.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/test/queries/clientpositive/cbo_correctness.q
hive/trunk/ql/src/test/queries/clientpositive/ctas_colname.q
hive/trunk/ql/src/test/queries/clientpositive/decimal_serde.q
hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out
hive/trunk/ql/src/test/results/clientpositive/decimal_serde.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java Mon Oct 27 22:59:12 2014
@@ -112,7 +112,7 @@ public final class RewriteParseContextGe
((SemanticAnalyzer) sem).initParseCtx(subPCtx);
LOG.info("Starting Sub-query Semantic Analysis");
- sem.doPhase1(child, qb, sem.initPhase1Ctx());
+ sem.doPhase1(child, qb, sem.initPhase1Ctx(), null);
LOG.info("Completed phase 1 of Sub-query Semantic Analysis");
sem.getMetaData(qb);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Mon Oct 27 22:59:12 2014
@@ -79,7 +79,7 @@ import com.google.common.annotations.Vis
*
*/
public abstract class BaseSemanticAnalyzer {
- private static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName());
+ protected static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName());
protected final Hive db;
protected final HiveConf conf;
protected List<Task<? extends Serializable>> rootTasks;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Mon Oct 27 22:59:12 2014
@@ -33,6 +33,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -368,7 +369,7 @@ public class SemanticAnalyzer extends Ba
//flag for partial scan during analyze ... compute statistics
protected boolean partialscan;
- private volatile boolean runCBO = true;
+ private volatile boolean runCBO = true; // TODO: why is this volatile?
private volatile boolean disableJoinMerge = false;
/*
@@ -380,6 +381,9 @@ public class SemanticAnalyzer extends Ba
*/
private ArrayList<String> ctesExpanded;
+ /** Not thread-safe. */
+ private ASTSearcher astSearcher = new ASTSearcher();
+
private static class Phase1Ctx {
String dest;
int nextNum;
@@ -512,7 +516,7 @@ public class SemanticAnalyzer extends Ba
case HiveParser.TOK_QUERY: {
QB qb = new QB(id, alias, true);
Phase1Ctx ctx_1 = initPhase1Ctx();
- doPhase1(ast, qb, ctx_1);
+ doPhase1(ast, qb, ctx_1, null);
qbexpr.setOpcode(QBExpr.Opcode.NULLOP);
qbexpr.setQB(qb);
@@ -1177,6 +1181,30 @@ public class SemanticAnalyzer extends Ba
return alias;
}
+ /** The context that doPhase1 uses to populate information pertaining
+ * to CBO (currently, this is used for CTAS and insert-as-select). */
+ private static class PreCboCtx {
+ enum Type {
+ NONE,
+ INSERT,
+ CTAS,
+
+ UNEXPECTED
+ }
+ public ASTNode nodeOfInterest;
+ public Type type = Type.NONE;
+ public void set(Type type, ASTNode ast) {
+ if (this.type != Type.NONE) {
+ STATIC_LOG.warn("Setting " + type + " when already " + this.type
+ + "; node " + ast.dump() + " vs old node " + nodeOfInterest.dump());
+ this.type = Type.UNEXPECTED;
+ return;
+ }
+ this.type = type;
+ this.nodeOfInterest = ast;
+ }
+ }
+
/**
* Phase 1: (including, but not limited to):
*
@@ -1194,7 +1222,7 @@ public class SemanticAnalyzer extends Ba
* @throws SemanticException
*/
@SuppressWarnings({"fallthrough", "nls"})
- public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1)
+ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PreCboCtx cboCtx)
throws SemanticException {
boolean phase1Result = true;
@@ -1238,20 +1266,26 @@ public class SemanticAnalyzer extends Ba
String currentDatabase = SessionState.get().getCurrentDatabase();
String tab_name = getUnescapedName((ASTNode) ast.getChild(0).getChild(0), currentDatabase);
qbp.addInsertIntoTable(tab_name);
- // TODO: is this supposed to fall thru?
case HiveParser.TOK_DESTINATION:
ctx_1.dest = "insclause-" + ctx_1.nextNum;
ctx_1.nextNum++;
+ boolean isTmpFileDest = false;
+ if (ast.getChildCount() > 0 && ast.getChild(0) instanceof ASTNode) {
+ ASTNode ch = (ASTNode)ast.getChild(0);
+ if (ch.getToken().getType() == HiveParser.TOK_DIR
+ && ch.getChildCount() > 0 && ch.getChild(0) instanceof ASTNode) {
+ ch = (ASTNode)ch.getChild(0);
+ isTmpFileDest = ch.getToken().getType() == HiveParser.TOK_TMP_FILE;
+ }
+ }
// is there a insert in the subquery
- if (qbp.getIsSubQ()) {
- ASTNode ch = (ASTNode) ast.getChild(0);
- if ((ch.getToken().getType() != HiveParser.TOK_DIR)
- || (((ASTNode) ch.getChild(0)).getToken().getType() != HiveParser.TOK_TMP_FILE)) {
- throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY
- .getMsg(ast));
- }
+ if (qbp.getIsSubQ() && !isTmpFileDest) {
+ throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY.getMsg(ast));
+ }
+ if (cboCtx != null && !isTmpFileDest) {
+ cboCtx.set(PreCboCtx.Type.INSERT, ast);
}
qbp.setDestForClause(ctx_1.dest, (ASTNode) ast.getChild(0));
@@ -1480,12 +1514,18 @@ public class SemanticAnalyzer extends Ba
int child_count = ast.getChildCount();
for (int child_pos = 0; child_pos < child_count && phase1Result; ++child_pos) {
// Recurse
- phase1Result = phase1Result && doPhase1((ASTNode) ast.getChild(child_pos), qb, ctx_1);
+ phase1Result = phase1Result && doPhase1(
+ (ASTNode)ast.getChild(child_pos), qb, ctx_1, cboCtx);
}
}
return phase1Result;
}
+ private void traceLogAst(ASTNode ast, String what) {
+ if (!LOG.isTraceEnabled()) return;
+ LOG.trace(what + ast.dump());
+ }
+
private void getMetaData(QBExpr qbexpr, ReadEntity parentInput)
throws SemanticException {
if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) {
@@ -1760,6 +1800,7 @@ public class SemanticAnalyzer extends Ba
qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts);
}
} else {
+ // This is the only place where isQuery is set to true; it defaults to false.
qb.setIsQuery(true);
fname = ctx.getMRTmpPath().toString();
ctx.setResDir(new Path(fname));
@@ -2464,7 +2505,7 @@ public class SemanticAnalyzer extends Ba
ISubQueryJoinInfo subQueryPredicate) throws SemanticException {
qbSQ.setSubQueryDef(subQueryPredicate.getSubQuery());
Phase1Ctx ctx_1 = initPhase1Ctx();
- doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1);
+ doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1, null);
getMetaData(qbSQ);
Operator op = genPlan(qbSQ);
return op;
@@ -6197,8 +6238,7 @@ public class SemanticAnalyzer extends Ba
ArrayList<ColumnInfo> colInfos = inputRR.getColumnInfos();
// CTAS case: the file output format and serde are defined by the create
- // table command
- // rather than taking the default value
+ // table command rather than taking the default value
List<FieldSchema> field_schemas = null;
CreateTableDesc tblDesc = qb.getTableDesc();
if (tblDesc != null) {
@@ -6219,7 +6259,8 @@ public class SemanticAnalyzer extends Ba
if (!("".equals(nm[0])) && nm[1] != null) {
colName = unescapeIdentifier(colInfo.getAlias()).toLowerCase(); // remove ``
}
- col.setName(colName);;
+ String ctasColName = fixCtasColumnName(colName, colInfo, inputRR);
+ col.setName(ctasColName);
col.setType(colInfo.getType().getTypeName());
field_schemas.add(col);
}
@@ -6397,6 +6438,14 @@ public class SemanticAnalyzer extends Ba
return output;
}
+ private static String fixCtasColumnName(String colName, ColumnInfo colInfo, RowResolver rr) {
+ int lastDot = colName.lastIndexOf('.');
+ if (lastDot < 0) return colName; // alias is not fully qualified
+ String nqColumnName = colName.substring(lastDot + 1);
+ STATIC_LOG.debug("Replacing " + colName + " (produced by CBO) by " + nqColumnName);
+ return nqColumnName;
+ }
+
// Check constraints on acid tables. This includes
// * no insert overwrites
// * no use of vectorization
@@ -9875,11 +9924,14 @@ public class SemanticAnalyzer extends Ba
// analyze and process the position alias
processPositionAlias(ast);
+ // Check configuration for CBO first.
+ runCBO = runCBO && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED);
// analyze create table command
+ PreCboCtx cboCtx = runCBO ? new PreCboCtx() : null;
if (ast.getToken().getType() == HiveParser.TOK_CREATETABLE) {
// if it is not CTAS, we don't need to go further and just return
- if ((child = analyzeCreateTable(ast, qb)) == null) {
+ if ((child = analyzeCreateTable(ast, qb, cboCtx)) == null) {
return;
}
} else {
@@ -9888,7 +9940,8 @@ public class SemanticAnalyzer extends Ba
// analyze create view command
if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW ||
- (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) {
+ (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW
+ && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) {
child = analyzeCreateView(ast, qb);
SessionState.get().setCommandType(HiveOperation.CREATEVIEW);
if (child == null) {
@@ -9901,7 +9954,7 @@ public class SemanticAnalyzer extends Ba
// continue analyzing from the child ASTNode.
Phase1Ctx ctx_1 = initPhase1Ctx();
- if (!doPhase1(child, qb, ctx_1)) {
+ if (!doPhase1(child, qb, ctx_1, cboCtx)) {
// if phase1Result false return
return;
}
@@ -9911,19 +9964,16 @@ public class SemanticAnalyzer extends Ba
getMetaData(qb);
LOG.info("Completed getting MetaData in Semantic Analysis");
-
+ // Note: for now, we don't actually pass the queryForCbo to CBO, because it accepts qb, not
+ // AST, and can also access all the private stuff in SA. We rely on the fact that CBO
+ // ignores the unknown tokens (create table, destination), so if the query is otherwise ok,
+ // it is as if we did remove those and gave CBO the proper AST. That is kinda hacky.
if (runCBO) {
- boolean tokenTypeIsQuery = ast.getToken().getType() == HiveParser.TOK_QUERY
- || ast.getToken().getType() == HiveParser.TOK_EXPLAIN;
- if (!tokenTypeIsQuery || createVwDesc != null
- || !HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED)
- || !canHandleQuery(qb, true) || !HiveOptiqUtil.validateASTForCBO(ast)) {
- runCBO = false;
- }
-
- if (runCBO) {
- disableJoinMerge = true;
+ ASTNode queryForCbo = ast;
+ if (cboCtx.type == PreCboCtx.Type.CTAS) {
+ queryForCbo = cboCtx.nodeOfInterest; // nodeOfInterest is the query
}
+ runCBO = canHandleAstForCbo(queryForCbo, qb, cboCtx);
}
// Save the result schema derived from the sink operator produced
@@ -9933,6 +9983,7 @@ public class SemanticAnalyzer extends Ba
Operator sinkOp = null;
if (runCBO) {
+ disableJoinMerge = true;
OptiqBasedPlanner optiqPlanner = new OptiqBasedPlanner();
boolean reAnalyzeAST = false;
@@ -9940,10 +9991,17 @@ public class SemanticAnalyzer extends Ba
// 1. Gen Optimized AST
ASTNode newAST = optiqPlanner.getOptimizedAST(prunedPartitions);
+ // 1.1. Fix up the query for insert/ctas
+ newAST = fixUpCtasAndInsertAfterCbo(ast, newAST, cboCtx);
+
// 2. Regen OP plan from optimized AST
init(false);
+ if (cboCtx.type == PreCboCtx.Type.CTAS) {
+ // Redo create-table analysis, because it's not part of doPhase1.
+ newAST = reAnalyzeCtasAfterCbo(newAST);
+ }
ctx_1 = initPhase1Ctx();
- if (!doPhase1(newAST, qb, ctx_1)) {
+ if (!doPhase1(newAST, qb, ctx_1, null)) {
throw new RuntimeException(
"Couldn't do phase1 on CBO optimized query plan");
}
@@ -10089,6 +10147,118 @@ public class SemanticAnalyzer extends Ba
return;
}
+ private ASTNode fixUpCtasAndInsertAfterCbo(
+ ASTNode originalAst, ASTNode newAst, PreCboCtx cboCtx) throws SemanticException {
+ switch (cboCtx.type) {
+ case NONE: return newAst; // nothing to do
+ case CTAS: {
+ // Patch the optimized query back into original CTAS AST, replacing the original query.
+ replaceASTChild(cboCtx.nodeOfInterest, newAst);
+ return originalAst;
+ }
+ case INSERT: {
+ // We need to patch the dest back to original into new query.
+ // This makes assumptions about the structure of the AST.
+ ASTNode newDest = astSearcher.simpleBreadthFirstSearch(
+ newAst, HiveParser.TOK_QUERY, HiveParser.TOK_INSERT, HiveParser.TOK_DESTINATION);
+ if (newDest == null) {
+ LOG.error("Cannot find destination after CBO; new ast is "+ newAst.dump());
+ throw new SemanticException("Cannot find destination after CBO");
+ }
+ replaceASTChild(newDest, cboCtx.nodeOfInterest);
+ return newAst;
+ }
+ default: throw new AssertionError("Unexpected type " + cboCtx.type);
+ }
+ }
+
+ private ASTNode reAnalyzeCtasAfterCbo(ASTNode newAst) throws SemanticException {
+ // analyzeCreateTable uses this.ast, but doPhase1 doesn't, so only reset it here.
+ this.ast = newAst;
+ newAst = analyzeCreateTable(newAst, qb, null);
+ if (newAst == null) {
+ LOG.error("analyzeCreateTable failed to initialize CTAS after CBO;"
+ + " new ast is " + this.ast.dump());
+ throw new SemanticException("analyzeCreateTable failed to initialize CTAS after CBO");
+ }
+ return newAst;
+ }
+
+ private boolean canHandleAstForCbo(ASTNode ast, QB qb, PreCboCtx cboCtx) {
+ int root = ast.getToken().getType();
+ boolean needToLogMessage = LOG.isInfoEnabled();
+ boolean isSupportedRoot =
+ root == HiveParser.TOK_QUERY || root == HiveParser.TOK_EXPLAIN || qb.isCTAS();
+ // Check AST.
+ // Assumption: If top level QB is query then everything below it must also be Query
+ // Can there be an insert or CTAS that wouldn't
+ // be supported and would require additional checks similar to IsQuery?
+ boolean isSupportedType =
+ qb.getIsQuery() || qb.isCTAS() || cboCtx.type == PreCboCtx.Type.INSERT;
+ boolean result = isSupportedRoot && isSupportedType && createVwDesc == null;
+ if (!result) {
+ if (needToLogMessage) {
+ String msg = "";
+ if (!isSupportedRoot) msg += "doesn't have QUERY or EXPLAIN as root and not a CTAS; ";
+ if (!isSupportedType) msg += "is not a query, CTAS, or insert; ";
+ if (createVwDesc != null) msg += "has create view; ";
+
+ if (msg.isEmpty()) msg += "has some unspecified limitations; ";
+ LOG.info("Not invoking CBO because the statement " + msg.substring(0, msg.length() - 2));
+ }
+ return false;
+ }
+ // Now check QB in more detail. canHandleQbForCbo returns null if query can be handled.
+ String msg = canHandleQbForCbo(qb, true, needToLogMessage);
+ if (msg == null) {
+ return true;
+ }
+ if (needToLogMessage) {
+ LOG.info("Not invoking CBO because the statement " + msg.substring(0, msg.length() - 2));
+ }
+ return false;
+ }
+
+ private class ASTSearcher {
+ private final LinkedList<ASTNode> searchQueue = new LinkedList<ASTNode>();
+ /**
+ * Performs breadth-first search of the AST for a nested set of tokens. Tokens don't have to be
+ * each others' direct children, they can be separated by layers of other tokens. For each token
+ * in the list, the first one found is matched and there's no backtracking; thus, if AST has
+ * multiple instances of some token, of which only one matches, it is not guaranteed to be found.
+ * We use this for simple things.
+ * Not thread-safe - reuses searchQueue.
+ */
+ public ASTNode simpleBreadthFirstSearch(ASTNode ast, int... tokens) {
+ searchQueue.clear();
+ searchQueue.add(ast);
+ for (int i = 0; i < tokens.length; ++i) {
+ boolean found = false;
+ int token = tokens[i];
+ while (!searchQueue.isEmpty() && !found) {
+ ASTNode next = searchQueue.poll();
+ found = next.getType() == token;
+ if (found) {
+ if (i == tokens.length - 1) return next;
+ searchQueue.clear();
+ }
+ for (int j = 0; j < next.getChildCount(); ++j) {
+ searchQueue.add((ASTNode)next.getChild(j));
+ }
+ }
+ if (!found) return null;
+ }
+ return null;
+ }
+ }
+
+ private void replaceASTChild(ASTNode child, ASTNode newChild) {
+ ASTNode parent = (ASTNode)child.parent;
+ int childIndex = child.childIndex;
+ parent.deleteChild(childIndex);
+ parent.insertChild(childIndex, newChild);
+ }
+
private void putAccessedColumnsToReadEntity(HashSet<ReadEntity> inputs, ColumnAccessInfo columnAccessInfo) {
Map<String, List<String>> tableToColumnAccessMap = columnAccessInfo.getTableToColumnAccessMap();
if (tableToColumnAccessMap != null && !tableToColumnAccessMap.isEmpty()) {
@@ -10585,8 +10755,8 @@ public class SemanticAnalyzer extends Ba
* the semantic analyzer need to deal with the select statement with respect
* to the SerDe and Storage Format.
*/
- private ASTNode analyzeCreateTable(ASTNode ast, QB qb)
- throws SemanticException {
+ private ASTNode analyzeCreateTable(
+ ASTNode ast, QB qb, PreCboCtx cboCtx) throws SemanticException {
String[] qualifiedTabName = getQualifiedTableName((ASTNode) ast.getChild(0));
String dbDotTab = getDotName(qualifiedTabName);
@@ -10676,6 +10846,9 @@ public class SemanticAnalyzer extends Ba
throw new SemanticException(ErrorMsg.CTAS_EXTTBL_COEXISTENCE.getMsg());
}
command_type = CTAS;
+ if (cboCtx != null) {
+ cboCtx.set(PreCboCtx.Type.CTAS, child);
+ }
selectStmt = child;
break;
case HiveParser.TOK_TABCOLLIST:
@@ -12247,27 +12420,43 @@ public class SemanticAnalyzer extends Ba
/**** Temporary Place Holder For Optiq plan Gen, Optimizer ****/
- /*
- * Entry point to Optimizations using Optiq.
+ /**
+ * Entry point to Optimizations using Optiq. Checks whether Optiq can handle the query.
+ * @param qbToChk Query block to check.
+ * @param verbose Whether return value should be verbose in case of failure.
+ * @return null if the query can be handled; non-null reason string if it cannot be.
*/
- private boolean canHandleQuery(QB qbToChk, boolean topLevelQB) {
- boolean runOptiqPlanner = false;
+ private String canHandleQbForCbo(QB qbToChk, boolean topLevelQB, boolean verbose) {
// Assumption:
// 1. If top level QB is query then everything below it must also be Query
// 2. Nested Subquery will return false for qbToChk.getIsQuery()
- if ((!topLevelQB || qbToChk.getIsQuery())
- && (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict"))
- && (!topLevelQB || (queryProperties.getJoinCount() > 1) || conf.getBoolVar(ConfVars.HIVE_IN_TEST))
- && !queryProperties.hasClusterBy() && !queryProperties.hasDistributeBy()
- && !queryProperties.hasSortBy() && !queryProperties.hasPTF()
- && !queryProperties.usesScript() && !queryProperties.hasMultiDestQuery()
- && !queryProperties.hasLateralViews()) {
- runOptiqPlanner = true;
- } else {
- LOG.info("Can not invoke CBO; query contains operators not supported for CBO.");
- }
+ boolean isInTest = conf.getBoolVar(ConfVars.HIVE_IN_TEST);
+ boolean isStrictTest = isInTest
+ && !conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict");
+ boolean hasEnoughJoins = !topLevelQB || (queryProperties.getJoinCount() > 1) || isInTest;
+ if (!isStrictTest && hasEnoughJoins && !queryProperties.hasClusterBy()
+ && !queryProperties.hasDistributeBy() && !queryProperties.hasSortBy()
+ && !queryProperties.hasPTF() && !queryProperties.usesScript()
+ && !queryProperties.hasMultiDestQuery() && !queryProperties.hasLateralViews()) {
+ return null; // Ok to run CBO.
+ }
+
+ // Not ok to run CBO, build error message.
+ String msg = "";
+ if (verbose) {
+ if (isStrictTest) msg += "is in test running in mode other than nonstrict; ";
+ if (!hasEnoughJoins) msg += "has too few joins; ";
+ if (queryProperties.hasClusterBy()) msg += "has cluster by; ";
+ if (queryProperties.hasDistributeBy()) msg += "has distribute by; ";
+ if (queryProperties.hasSortBy()) msg += "has sort by; ";
+ if (queryProperties.hasPTF()) msg += "has PTF; ";
+ if (queryProperties.usesScript()) msg += "uses scripts; ";
+ if (queryProperties.hasMultiDestQuery()) msg += "is a multi-destination query; ";
+ if (queryProperties.hasLateralViews()) msg += "has lateral views; ";
- return runOptiqPlanner;
+ if (msg.isEmpty()) msg += "has some unspecified limitations; ";
+ }
+ return msg;
}
private class OptiqBasedPlanner implements Frameworks.PlannerAction<RelNode> {
@@ -12984,7 +13173,7 @@ public class SemanticAnalyzer extends Ba
QB qbSQ = new QB(subQuery.getOuterQueryId(), subQuery.getAlias(), true);
qbSQ.setSubQueryDef(subQuery.getSubQuery());
Phase1Ctx ctx_1 = initPhase1Ctx();
- doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1);
+ doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1, null);
getMetaData(qbSQ);
RelNode subQueryRelNode = genLogicalPlan(qbSQ, false);
aliasToRel.put(subQuery.getAlias(), subQueryRelNode);
@@ -13012,7 +13201,7 @@ public class SemanticAnalyzer extends Ba
QB qbSQ_nic = new QB(subQuery.getOuterQueryId(), notInCheck.getAlias(), true);
qbSQ_nic.setSubQueryDef(notInCheck.getSubQuery());
ctx_1 = initPhase1Ctx();
- doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1);
+ doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1, null);
getMetaData(qbSQ_nic);
RelNode subQueryNICRelNode = genLogicalPlan(qbSQ_nic, false);
aliasToRel.put(notInCheck.getAlias(), subQueryNICRelNode);
@@ -14069,11 +14258,14 @@ public class SemanticAnalyzer extends Ba
// First generate all the opInfos for the elements in the from clause
Map<String, RelNode> aliasToRel = new HashMap<String, RelNode>();
- // 0. Check if we can handle the query
- // This check is needed here because of SubQuery
- if (!canHandleQuery(qb, false)) {
- String msg = String.format("CBO Can not handle Sub Query");
- LOG.debug(msg);
+ // 0. Check if we can handle the SubQuery;
+ // canHandleQbForCbo returns null if the query can be handled.
+ String reason = canHandleQbForCbo(qb, false, LOG.isDebugEnabled());
+ if (reason != null) {
+ String msg = "CBO can not handle Sub Query";
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(msg + " because it: " + reason);
+ }
throw new OptiqSemanticException(msg);
}
Modified: hive/trunk/ql/src/test/queries/clientpositive/cbo_correctness.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/cbo_correctness.q?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/cbo_correctness.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/cbo_correctness.q Mon Oct 27 22:59:12 2014
@@ -485,4 +485,4 @@ select unionsrc.key, count(1) FROM (sele
-- Windowing
select *, rank() over(partition by key order by value) as rr from src1;
-select *, rank() over(partition by key order by value) from src1;
\ No newline at end of file
+select *, rank() over(partition by key order by value) from src1;
Modified: hive/trunk/ql/src/test/queries/clientpositive/ctas_colname.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/ctas_colname.q?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/ctas_colname.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/ctas_colname.q Mon Oct 27 22:59:12 2014
@@ -3,9 +3,11 @@
-- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table
-- group by
+
+
explain
-create table summary as select *, sum(key), count(value) from src;
-create table summary as select *, sum(key), count(value) from src;
+create table summary as select *, key + 1, concat(value, value) from src limit 20;
+create table summary as select *, key + 1, concat(value, value) from src limit 20;
describe formatted summary;
select * from summary;
Modified: hive/trunk/ql/src/test/queries/clientpositive/decimal_serde.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/decimal_serde.q?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/decimal_serde.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/decimal_serde.q Mon Oct 27 22:59:12 2014
@@ -15,12 +15,15 @@ SELECT * FROM DECIMAL_TEXT ORDER BY key,
CREATE TABLE DECIMAL_RC
STORED AS RCFile AS
SELECT * FROM DECIMAL_TEXT;
+describe formatted DECIMAL_RC;
CREATE TABLE DECIMAL_LAZY_COL
ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
STORED AS RCFile AS
SELECT * FROM DECIMAL_RC;
+describe formatted DECIMAL_LAZY_COL;
+
CREATE TABLE DECIMAL_SEQUENCE
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\001'
Added: hive/trunk/ql/src/test/queries/clientpositive/insert0.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/insert0.q?rev=1634725&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/insert0.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/insert0.q Mon Oct 27 22:59:12 2014
@@ -0,0 +1,38 @@
+set hive.cbo.enable=true;
+
+DROP TABLE insert_into1;
+DROP TABLE ctas_table;
+DROP TABLE ctas_part;
+
+CREATE TABLE insert_into1 (key int, value string);
+
+INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10;
+
+select * from insert_into1 order by key;
+
+INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10;
+
+select * from insert_into1 order by key;
+
+create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10;
+
+describe extended ctas_table;
+
+select * from ctas_table order by key;
+
+
+set hive.exec.dynamic.partition=true;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+create table ctas_part (key int, value string) partitioned by (modkey bigint);
+
+insert overwrite table ctas_part partition (modkey)
+select key, value, ceil(key / 100) from src where key is not null limit 10;
+
+select * from ctas_part order by key;
+
+
+
+DROP TABLE insert_into1;
+DROP TABLE ctas_table;
+DROP TABLE ctas_part;
\ No newline at end of file
Modified: hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out Mon Oct 27 22:59:12 2014
@@ -3,16 +3,20 @@ PREHOOK: query: -- SORT_QUERY_RESULTS
-- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table
-- group by
+
+
explain
-create table summary as select *, sum(key), count(value) from src
+create table summary as select *, key + 1, concat(value, value) from src limit 20
PREHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: query: -- SORT_QUERY_RESULTS
-- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table
-- group by
+
+
explain
-create table summary as select *, sum(key), count(value) from src
+create table summary as select *, key + 1, concat(value, value) from src limit 20
POSTHOOK: type: CREATETABLE_AS_SELECT
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -28,31 +32,27 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ expressions: key (type: string), value (type: string), (key + 1) (type: double), concat(value, value) (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(key), count(value)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 20
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: double), _col1 (type: bigint)
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _col3 (type: string)
Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0), count(VALUE._col1)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: double), _col1 (type: bigint), _col0 (type: double), _col1 (type: bigint)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 20
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -68,7 +68,7 @@ STAGE PLANS:
Stage: Stage-3
Create Table Operator:
Create Table
- columns: _col0 double, _col1 bigint, _c1 double, _c2 bigint
+ columns: key string, value string, _c1 double, _c2 string
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
@@ -77,12 +77,12 @@ STAGE PLANS:
Stage: Stage-2
Stats-Aggr Operator
-PREHOOK: query: create table summary as select *, sum(key), count(value) from src
+PREHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@src
PREHOOK: Output: database:default
PREHOOK: Output: default@summary
-POSTHOOK: query: create table summary as select *, sum(key), count(value) from src
+POSTHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
@@ -95,10 +95,10 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@summary
# col_name data_type comment
-_col0 double
-_col1 bigint
+key string
+value string
_c1 double
-_c2 bigint
+_c2 string
# Detailed Table Information
Database: default
@@ -110,9 +110,9 @@ Table Type: MANAGED_TABLE
Table Parameters:
COLUMN_STATS_ACCURATE true
numFiles 1
- numRows 1
- rawDataSize 25
- totalSize 26
+ numRows 20
+ rawDataSize 620
+ totalSize 640
#### A masked pattern was here ####
# Storage Information
@@ -133,7 +133,26 @@ POSTHOOK: query: select * from summary
POSTHOOK: type: QUERY
POSTHOOK: Input: default@summary
#### A masked pattern was here ####
-130091.0 500 130091.0 500
+128 val_128 129.0 val_128val_128
+150 val_150 151.0 val_150val_150
+165 val_165 166.0 val_165val_165
+193 val_193 194.0 val_193val_193
+213 val_213 214.0 val_213val_213
+224 val_224 225.0 val_224val_224
+238 val_238 239.0 val_238val_238
+255 val_255 256.0 val_255val_255
+265 val_265 266.0 val_265val_265
+27 val_27 28.0 val_27val_27
+273 val_273 274.0 val_273val_273
+278 val_278 279.0 val_278val_278
+311 val_311 312.0 val_311val_311
+369 val_369 370.0 val_369val_369
+401 val_401 402.0 val_401val_401
+409 val_409 410.0 val_409val_409
+484 val_484 485.0 val_484val_484
+66 val_66 67.0 val_66val_66
+86 val_86 87.0 val_86val_86
+98 val_98 99.0 val_98val_98
PREHOOK: query: -- window functions
explain
create table x4 as select *, rank() over(partition by key order by value) as rr from src1
Modified: hive/trunk/ql/src/test/results/clientpositive/decimal_serde.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/decimal_serde.q.out?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/decimal_serde.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/decimal_serde.q.out Mon Oct 27 22:59:12 2014
@@ -96,6 +96,42 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@decimal_text
POSTHOOK: Output: database:default
POSTHOOK: Output: default@DECIMAL_RC
+PREHOOK: query: describe formatted DECIMAL_RC
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@decimal_rc
+POSTHOOK: query: describe formatted DECIMAL_RC
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@decimal_rc
+# col_name data_type comment
+
+key decimal(10,0)
+value int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE true
+ numFiles 1
+ numRows 38
+ rawDataSize 157
+ totalSize 278
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
PREHOOK: query: CREATE TABLE DECIMAL_LAZY_COL
ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
STORED AS RCFile AS
@@ -112,6 +148,42 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@decimal_rc
POSTHOOK: Output: database:default
POSTHOOK: Output: default@DECIMAL_LAZY_COL
+PREHOOK: query: describe formatted DECIMAL_LAZY_COL
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@decimal_lazy_col
+POSTHOOK: query: describe formatted DECIMAL_LAZY_COL
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@decimal_lazy_col
+# col_name data_type comment
+
+key decimal(10,0)
+value int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE true
+ numFiles 1
+ numRows 38
+ rawDataSize 157
+ totalSize 278
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
PREHOOK: query: CREATE TABLE DECIMAL_SEQUENCE
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\001'
Added: hive/trunk/ql/src/test/results/clientpositive/insert0.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/insert0.q.out?rev=1634725&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/insert0.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/insert0.q.out Mon Oct 27 22:59:12 2014
@@ -0,0 +1,208 @@
+PREHOOK: query: DROP TABLE insert_into1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE insert_into1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE ctas_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE ctas_table
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE ctas_part
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE ctas_part
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE insert_into1 (key int, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: CREATE TABLE insert_into1 (key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@insert_into1
+PREHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@insert_into1
+POSTHOOK: Lineage: insert_into1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: insert_into1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from insert_into1 order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from insert_into1 order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
+100 val_100
+103 val_103
+103 val_103
+104 val_104
+104 val_104
+PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@insert_into1
+POSTHOOK: Lineage: insert_into1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: insert_into1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from insert_into1 order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from insert_into1 order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+90 val_90
+90 val_90
+92 val_92
+95 val_95
+95 val_95
+96 val_96
+97 val_97
+97 val_97
+98 val_98
+98 val_98
+100 val_100
+100 val_100
+103 val_103
+103 val_103
+104 val_104
+104 val_104
+PREHOOK: query: create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ctas_table
+POSTHOOK: query: create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ctas_table
+PREHOOK: query: describe extended ctas_table
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@ctas_table
+POSTHOOK: query: describe extended ctas_table
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@ctas_table
+key string
+foo bigint
+
+#### A masked pattern was here ####
+PREHOOK: query: select * from ctas_table order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ctas_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ctas_table order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ctas_table
+#### A masked pattern was here ####
+0 3
+10 1
+100 2
+103 2
+104 2
+105 1
+11 1
+111 1
+113 2
+114 1
+PREHOOK: query: create table ctas_part (key int, value string) partitioned by (modkey bigint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ctas_part
+POSTHOOK: query: create table ctas_part (key int, value string) partitioned by (modkey bigint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ctas_part
+PREHOOK: query: insert overwrite table ctas_part partition (modkey)
+select key, value, ceil(key / 100) from src where key is not null limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ctas_part
+POSTHOOK: query: insert overwrite table ctas_part partition (modkey)
+select key, value, ceil(key / 100) from src where key is not null limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ctas_part@modkey=1
+POSTHOOK: Output: default@ctas_part@modkey=2
+POSTHOOK: Output: default@ctas_part@modkey=3
+POSTHOOK: Output: default@ctas_part@modkey=4
+POSTHOOK: Output: default@ctas_part@modkey=5
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from ctas_part order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ctas_part
+PREHOOK: Input: default@ctas_part@modkey=1
+PREHOOK: Input: default@ctas_part@modkey=2
+PREHOOK: Input: default@ctas_part@modkey=3
+PREHOOK: Input: default@ctas_part@modkey=4
+PREHOOK: Input: default@ctas_part@modkey=5
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ctas_part order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ctas_part
+POSTHOOK: Input: default@ctas_part@modkey=1
+POSTHOOK: Input: default@ctas_part@modkey=2
+POSTHOOK: Input: default@ctas_part@modkey=3
+POSTHOOK: Input: default@ctas_part@modkey=4
+POSTHOOK: Input: default@ctas_part@modkey=5
+#### A masked pattern was here ####
+27 val_27 1
+86 val_86 1
+98 val_98 1
+165 val_165 2
+238 val_238 3
+255 val_255 3
+278 val_278 3
+311 val_311 4
+409 val_409 5
+484 val_484 5
+PREHOOK: query: DROP TABLE insert_into1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@insert_into1
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: DROP TABLE insert_into1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@insert_into1
+POSTHOOK: Output: default@insert_into1
+PREHOOK: query: DROP TABLE ctas_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ctas_table
+PREHOOK: Output: default@ctas_table
+POSTHOOK: query: DROP TABLE ctas_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ctas_table
+POSTHOOK: Output: default@ctas_table
+PREHOOK: query: DROP TABLE ctas_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ctas_part
+PREHOOK: Output: default@ctas_part
+POSTHOOK: query: DROP TABLE ctas_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ctas_part
+POSTHOOK: Output: default@ctas_part