You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/03/02 23:38:08 UTC
svn commit: r1451954 [7/27] - in /hive/branches/ptf-windowing: ./
cli/src/java/org/apache/hadoop/hive/cli/ common/src/java/conf/
common/src/java/org/apache/hadoop/hive/conf/ conf/
contrib/src/test/results/clientpositive/ data/conf/ data/files/ hbase-ha...
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java Sat Mar 2 22:37:59 2013
@@ -19,15 +19,8 @@
package org.apache.hadoop.hive.ql.parse;
import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-
import org.antlr.runtime.ANTLRStringStream;
-import org.antlr.runtime.BitSet;
import org.antlr.runtime.CharStream;
-import org.antlr.runtime.FailedPredicateException;
-import org.antlr.runtime.IntStream;
-import org.antlr.runtime.MismatchedTokenException;
import org.antlr.runtime.NoViableAltException;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.Token;
@@ -48,170 +41,6 @@ public class ParseDriver {
private static final Log LOG = LogFactory.getLog("hive.ql.parse.ParseDriver");
- private static HashMap<String, String> xlateMap;
- static {
- xlateMap = new HashMap<String, String>();
-
- // Keywords
- xlateMap.put("KW_TRUE", "TRUE");
- xlateMap.put("KW_FALSE", "FALSE");
- xlateMap.put("KW_ALL", "ALL");
- xlateMap.put("KW_AND", "AND");
- xlateMap.put("KW_OR", "OR");
- xlateMap.put("KW_NOT", "NOT");
- xlateMap.put("KW_LIKE", "LIKE");
-
- xlateMap.put("KW_ASC", "ASC");
- xlateMap.put("KW_DESC", "DESC");
- xlateMap.put("KW_ORDER", "ORDER");
- xlateMap.put("KW_BY", "BY");
- xlateMap.put("KW_GROUP", "GROUP");
- xlateMap.put("KW_WHERE", "WHERE");
- xlateMap.put("KW_FROM", "FROM");
- xlateMap.put("KW_AS", "AS");
- xlateMap.put("KW_SELECT", "SELECT");
- xlateMap.put("KW_DISTINCT", "DISTINCT");
- xlateMap.put("KW_INSERT", "INSERT");
- xlateMap.put("KW_OVERWRITE", "OVERWRITE");
- xlateMap.put("KW_OUTER", "OUTER");
- xlateMap.put("KW_JOIN", "JOIN");
- xlateMap.put("KW_LEFT", "LEFT");
- xlateMap.put("KW_RIGHT", "RIGHT");
- xlateMap.put("KW_FULL", "FULL");
- xlateMap.put("KW_ON", "ON");
- xlateMap.put("KW_PARTITION", "PARTITION");
- xlateMap.put("KW_PARTITIONS", "PARTITIONS");
- xlateMap.put("KW_TABLE", "TABLE");
- xlateMap.put("KW_TABLES", "TABLES");
- xlateMap.put("KW_TBLPROPERTIES", "TBLPROPERTIES");
- xlateMap.put("KW_SHOW", "SHOW");
- xlateMap.put("KW_MSCK", "MSCK");
- xlateMap.put("KW_DIRECTORY", "DIRECTORY");
- xlateMap.put("KW_LOCAL", "LOCAL");
- xlateMap.put("KW_TRANSFORM", "TRANSFORM");
- xlateMap.put("KW_USING", "USING");
- xlateMap.put("KW_CLUSTER", "CLUSTER");
- xlateMap.put("KW_DISTRIBUTE", "DISTRIBUTE");
- xlateMap.put("KW_SORT", "SORT");
- xlateMap.put("KW_UNION", "UNION");
- xlateMap.put("KW_LOAD", "LOAD");
- xlateMap.put("KW_DATA", "DATA");
- xlateMap.put("KW_INPATH", "INPATH");
- xlateMap.put("KW_IS", "IS");
- xlateMap.put("KW_NULL", "NULL");
- xlateMap.put("KW_CREATE", "CREATE");
- xlateMap.put("KW_EXTERNAL", "EXTERNAL");
- xlateMap.put("KW_ALTER", "ALTER");
- xlateMap.put("KW_DESCRIBE", "DESCRIBE");
- xlateMap.put("KW_DROP", "DROP");
- xlateMap.put("KW_REANME", "REANME");
- xlateMap.put("KW_TO", "TO");
- xlateMap.put("KW_COMMENT", "COMMENT");
- xlateMap.put("KW_BOOLEAN", "BOOLEAN");
- xlateMap.put("KW_TINYINT", "TINYINT");
- xlateMap.put("KW_SMALLINT", "SMALLINT");
- xlateMap.put("KW_INT", "INT");
- xlateMap.put("KW_BIGINT", "BIGINT");
- xlateMap.put("KW_FLOAT", "FLOAT");
- xlateMap.put("KW_DOUBLE", "DOUBLE");
- xlateMap.put("KW_DATE", "DATE");
- xlateMap.put("KW_DATETIME", "DATETIME");
- xlateMap.put("KW_TIMESTAMP", "TIMESTAMP");
- xlateMap.put("KW_STRING", "STRING");
- xlateMap.put("KW_BINARY", "BINARY");
- xlateMap.put("KW_ARRAY", "ARRAY");
- xlateMap.put("KW_MAP", "MAP");
- xlateMap.put("KW_REDUCE", "REDUCE");
- xlateMap.put("KW_PARTITIONED", "PARTITIONED");
- xlateMap.put("KW_CLUSTERED", "CLUSTERED");
- xlateMap.put("KW_SORTED", "SORTED");
- xlateMap.put("KW_INTO", "INTO");
- xlateMap.put("KW_BUCKETS", "BUCKETS");
- xlateMap.put("KW_ROW", "ROW");
- xlateMap.put("KW_FORMAT", "FORMAT");
- xlateMap.put("KW_DELIMITED", "DELIMITED");
- xlateMap.put("KW_FIELDS", "FIELDS");
- xlateMap.put("KW_TERMINATED", "TERMINATED");
- xlateMap.put("KW_COLLECTION", "COLLECTION");
- xlateMap.put("KW_ITEMS", "ITEMS");
- xlateMap.put("KW_KEYS", "KEYS");
- xlateMap.put("KW_KEY_TYPE", "$KEY$");
- xlateMap.put("KW_LINES", "LINES");
- xlateMap.put("KW_STORED", "STORED");
- xlateMap.put("KW_SEQUENCEFILE", "SEQUENCEFILE");
- xlateMap.put("KW_TEXTFILE", "TEXTFILE");
- xlateMap.put("KW_INPUTFORMAT", "INPUTFORMAT");
- xlateMap.put("KW_OUTPUTFORMAT", "OUTPUTFORMAT");
- xlateMap.put("KW_LOCATION", "LOCATION");
- xlateMap.put("KW_TABLESAMPLE", "TABLESAMPLE");
- xlateMap.put("KW_BUCKET", "BUCKET");
- xlateMap.put("KW_OUT", "OUT");
- xlateMap.put("KW_OF", "OF");
- xlateMap.put("KW_CAST", "CAST");
- xlateMap.put("KW_ADD", "ADD");
- xlateMap.put("KW_REPLACE", "REPLACE");
- xlateMap.put("KW_COLUMNS", "COLUMNS");
- xlateMap.put("KW_RLIKE", "RLIKE");
- xlateMap.put("KW_REGEXP", "REGEXP");
- xlateMap.put("KW_TEMPORARY", "TEMPORARY");
- xlateMap.put("KW_FUNCTION", "FUNCTION");
- xlateMap.put("KW_EXPLAIN", "EXPLAIN");
- xlateMap.put("KW_EXTENDED", "EXTENDED");
- xlateMap.put("KW_SERDE", "SERDE");
- xlateMap.put("KW_WITH", "WITH");
- xlateMap.put("KW_SERDEPROPERTIES", "SERDEPROPERTIES");
- xlateMap.put("KW_LIMIT", "LIMIT");
- xlateMap.put("KW_SET", "SET");
- xlateMap.put("KW_PROPERTIES", "TBLPROPERTIES");
- xlateMap.put("KW_VALUE_TYPE", "$VALUE$");
- xlateMap.put("KW_ELEM_TYPE", "$ELEM$");
-
- // Operators
- xlateMap.put("DOT", ".");
- xlateMap.put("COLON", ":");
- xlateMap.put("COMMA", ",");
- xlateMap.put("SEMICOLON", ");");
-
- xlateMap.put("LPAREN", "(");
- xlateMap.put("RPAREN", ")");
- xlateMap.put("LSQUARE", "[");
- xlateMap.put("RSQUARE", "]");
-
- xlateMap.put("EQUAL", "=");
- xlateMap.put("NOTEQUAL", "<>");
- xlateMap.put("EQUAL_NS", "<=>");
- xlateMap.put("LESSTHANOREQUALTO", "<=");
- xlateMap.put("LESSTHAN", "<");
- xlateMap.put("GREATERTHANOREQUALTO", ">=");
- xlateMap.put("GREATERTHAN", ">");
-
- xlateMap.put("DIVIDE", "/");
- xlateMap.put("PLUS", "+");
- xlateMap.put("MINUS", "-");
- xlateMap.put("STAR", "*");
- xlateMap.put("MOD", "%");
-
- xlateMap.put("AMPERSAND", "&");
- xlateMap.put("TILDE", "~");
- xlateMap.put("BITWISEOR", "|");
- xlateMap.put("BITWISEXOR", "^");
- xlateMap.put("CharSetLiteral", "\\'");
- }
-
- public static Collection<String> getKeywords() {
- return xlateMap.values();
- }
-
- private static String xlate(String name) {
-
- String ret = xlateMap.get(name);
- if (ret == null) {
- ret = name;
- }
-
- return ret;
- }
-
/**
* ANTLRNoCaseStringStream.
*
@@ -298,89 +127,6 @@ public class ParseDriver {
}
/**
- * HiveParserX.
- *
- */
- public class HiveParserX extends HiveParser {
-
- private final ArrayList<ParseError> errors;
-
- public HiveParserX(TokenStream input) {
- super(input);
- errors = new ArrayList<ParseError>();
- }
-
- @Override
- public Object recoverFromMismatchedSet(IntStream input,
- RecognitionException re, BitSet follow) throws RecognitionException {
- throw re;
- }
-
- @Override
- public void displayRecognitionError(String[] tokenNames,
- RecognitionException e) {
-
- errors.add(new ParseError(this, e, tokenNames));
- }
-
- @Override
- public String getErrorHeader(RecognitionException e) {
- String header = null;
- if (e.charPositionInLine < 0 && input.LT(-1) != null) {
- Token t = input.LT(-1);
- header = "line " + t.getLine() + ":" + t.getCharPositionInLine();
- } else {
- header = super.getErrorHeader(e);
- }
-
- return header;
- }
-
-
- @Override
- public String getErrorMessage(RecognitionException e, String[] tokenNames) {
- String msg = null;
-
- // Translate the token names to something that the user can understand
- String[] xlateNames = new String[tokenNames.length];
- for (int i = 0; i < tokenNames.length; ++i) {
- xlateNames[i] = ParseDriver.xlate(tokenNames[i]);
- }
-
- if (e instanceof NoViableAltException) {
- @SuppressWarnings("unused")
- NoViableAltException nvae = (NoViableAltException) e;
- // for development, can add
- // "decision=<<"+nvae.grammarDecisionDescription+">>"
- // and "(decision="+nvae.decisionNumber+") and
- // "state "+nvae.stateNumber
- msg = "cannot recognize input near "
- + getTokenErrorDisplay(e.token)
- + (input.LT(2) != null ? " " + getTokenErrorDisplay(input.LT(2)) : "")
- + (input.LT(3) != null ? " " + getTokenErrorDisplay(input.LT(3)) : "");
- } else if (e instanceof MismatchedTokenException) {
- MismatchedTokenException mte = (MismatchedTokenException) e;
- msg = super.getErrorMessage(e, xlateNames) + (input.LT(-1) == null ? "":" near '" + input.LT(-1).getText()) + "'";
- } else if (e instanceof FailedPredicateException) {
- FailedPredicateException fpe = (FailedPredicateException) e;
- msg = "Failed to recognize predicate '" + fpe.token.getText() + "'. Failed rule: '" + fpe.ruleName + "'";
- } else {
- msg = super.getErrorMessage(e, xlateNames);
- }
-
- if (msgs.size() > 0) {
- msg = msg + " in " + msgs.peek();
- }
- return msg;
- }
-
- public ArrayList<ParseError> getErrors() {
- return errors;
- }
-
- }
-
- /**
* Tree adaptor for making antlr return ASTNodes instead of CommonTree nodes
* so that the graph walking algorithms and the rules framework defined in
* ql.lib can be used with the AST Nodes.
@@ -437,22 +183,22 @@ public class ParseDriver {
if (ctx != null) {
ctx.setTokenRewriteStream(tokens);
}
- HiveParserX parser = new HiveParserX(tokens);
+ HiveParser parser = new HiveParser(tokens);
parser.setTreeAdaptor(adaptor);
HiveParser.statement_return r = null;
try {
r = parser.statement();
} catch (RecognitionException e) {
e.printStackTrace();
- throw new ParseException(parser.getErrors());
+ throw new ParseException(parser.errors);
}
- if (lexer.getErrors().size() == 0 && parser.getErrors().size() == 0) {
+ if (lexer.getErrors().size() == 0 && parser.errors.size() == 0) {
LOG.info("Parse Completed");
} else if (lexer.getErrors().size() != 0) {
throw new ParseException(lexer.getErrors());
} else {
- throw new ParseException(parser.getErrors());
+ throw new ParseException(parser.errors);
}
return (ASTNode) r.getTree();
@@ -477,22 +223,22 @@ public class ParseDriver {
if (ctx != null) {
ctx.setTokenRewriteStream(tokens);
}
- HiveParserX parser = new HiveParserX(tokens);
+ HiveParser parser = new HiveParser(tokens);
parser.setTreeAdaptor(adaptor);
- HiveParser.selectClause_return r = null;
+ HiveParser_SelectClauseParser.selectClause_return r = null;
try {
r = parser.selectClause();
} catch (RecognitionException e) {
e.printStackTrace();
- throw new ParseException(parser.getErrors());
+ throw new ParseException(parser.errors);
}
- if (lexer.getErrors().size() == 0 && parser.getErrors().size() == 0) {
+ if (lexer.getErrors().size() == 0 && parser.errors.size() == 0) {
LOG.info("Parse Completed");
} else if (lexer.getErrors().size() != 0) {
throw new ParseException(lexer.getErrors());
} else {
- throw new ParseException(parser.getErrors());
+ throw new ParseException(parser.errors);
}
return (ASTNode) r.getTree();
Copied: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g (from r1451443, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g)
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g?p2=hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g&p1=hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g&r1=1451443&r2=1451954&rev=1451954&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g Sat Mar 2 22:37:59 2013
@@ -125,7 +125,10 @@ selectItem
@init { gParent.msgs.push("selection target"); }
@after { gParent.msgs.pop(); }
:
- ( selectExpression ((KW_AS? identifier) | (KW_AS LPAREN identifier (COMMA identifier)* RPAREN))?) -> ^(TOK_SELEXPR selectExpression identifier*)
+ ( selectExpression
+ ((KW_AS? identifier) | (KW_AS LPAREN identifier (COMMA identifier)* RPAREN))?
+ (KW_OVER ws=window_specification )?
+ ) -> ^(TOK_SELEXPR selectExpression identifier* $ws?)
;
trfmClause
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Mar 2 22:37:59 2013
@@ -78,6 +78,7 @@ import org.apache.hadoop.hive.ql.exec.Re
import org.apache.hadoop.hive.ql.exec.RecordWriter;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.RowSchema;
+import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.StatsTask;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -224,6 +225,7 @@ public class SemanticAnalyzer extends Ba
private List<LoadTableDesc> loadTableWork;
private List<LoadFileDesc> loadFileWork;
private Map<JoinOperator, QBJoinTree> joinContext;
+ private Map<SMBMapJoinOperator, QBJoinTree> smbMapJoinContext;
private final HashMap<TableScanOperator, Table> topToTable;
private QB qb;
private ASTNode ast;
@@ -281,6 +283,7 @@ public class SemanticAnalyzer extends Ba
loadFileWork = new ArrayList<LoadFileDesc>();
opParseCtx = new LinkedHashMap<Operator<? extends OperatorDesc>, OpParseContext>();
joinContext = new HashMap<JoinOperator, QBJoinTree>();
+ smbMapJoinContext = new HashMap<SMBMapJoinOperator, QBJoinTree>();
topToTable = new HashMap<TableScanOperator, Table>();
destTableId = 1;
uCtx = null;
@@ -309,6 +312,7 @@ public class SemanticAnalyzer extends Ba
ast = null;
uCtx = null;
joinContext.clear();
+ smbMapJoinContext.clear();
opParseCtx.clear();
groupOpToInputTables.clear();
prunedPartitions.clear();
@@ -324,6 +328,7 @@ public class SemanticAnalyzer extends Ba
loadTableWork = pctx.getLoadTableWork();
loadFileWork = pctx.getLoadFileWork();
joinContext = pctx.getJoinContext();
+ smbMapJoinContext = pctx.getSmbMapJoinContext();
ctx = pctx.getContext();
destTableId = pctx.getDestTableId();
idToTableNameMap = pctx.getIdToTableNameMap();
@@ -338,7 +343,7 @@ public class SemanticAnalyzer extends Ba
public ParseContext getParseContext() {
return new ParseContext(conf, qb, ast, opToPartPruner, opToPartList, topOps,
- topSelOps, opParseCtx, joinContext, topToTable, loadTableWork,
+ topSelOps, opParseCtx, joinContext, smbMapJoinContext, topToTable, loadTableWork,
loadFileWork, ctx, idToTableNameMap, destTableId, uCtx,
listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions,
opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks,
@@ -3695,21 +3700,30 @@ public class SemanticAnalyzer extends Ba
// Need to pass all of the columns used in the where clauses as reduce values
ASTNode whereClause = parseInfo.getWhrForClause(destination);
if (whereClause != null) {
- List<ASTNode> columnExprs =
- getColumnExprsFromASTNode(whereClause, reduceSinkInputRowResolver);
- for (int i = 0; i < columnExprs.size(); i++) {
- ASTNode parameter = columnExprs.get(i);
- if (reduceSinkOutputRowResolver.getExpression(parameter) == null) {
- reduceValues.add(genExprNodeDesc(parameter,
- reduceSinkInputRowResolver));
- outputValueColumnNames
- .add(getColumnInternalName(reduceValues.size() - 1));
- String field = Utilities.ReduceField.VALUE.toString() + "."
- + getColumnInternalName(reduceValues.size() - 1);
- reduceSinkOutputRowResolver.putExpression(parameter, new ColumnInfo(field,
- reduceValues.get(reduceValues.size() - 1).getTypeInfo(), null,
- false));
+ assert whereClause.getChildCount() == 1;
+ ASTNode predicates = (ASTNode) whereClause.getChild(0);
+
+ Map<ASTNode, ExprNodeDesc> nodeOutputs =
+ genAllExprNodeDesc(predicates, reduceSinkInputRowResolver);
+ removeMappingForKeys(predicates, nodeOutputs, reduceKeys);
+
+ // extract columns missing in current RS key/value
+ for (Map.Entry<ASTNode, ExprNodeDesc> entry : nodeOutputs.entrySet()) {
+ ASTNode parameter = (ASTNode) entry.getKey();
+ ExprNodeDesc expression = (ExprNodeDesc) entry.getValue();
+ if (!(expression instanceof ExprNodeColumnDesc)) {
+ continue;
}
+ if (ExprNodeDescUtils.indexOf(expression, reduceValues) >= 0) {
+ continue;
+ }
+ String internalName = getColumnInternalName(reduceValues.size());
+ String field = Utilities.ReduceField.VALUE.toString() + "." + internalName;
+
+ reduceValues.add(expression);
+ outputValueColumnNames.add(internalName);
+ reduceSinkOutputRowResolver.putExpression(parameter,
+ new ColumnInfo(field, expression.getTypeInfo(), null, false));
}
}
}
@@ -3724,32 +3738,27 @@ public class SemanticAnalyzer extends Ba
return rsOp;
}
- /**
- * Given an ASTNode, it returns all of the descendant ASTNodes which represent column expressions
- *
- * @param node
- * @param inputRR
- * @return
- * @throws SemanticException
- */
- private List<ASTNode> getColumnExprsFromASTNode(ASTNode node, RowResolver inputRR)
- throws SemanticException {
-
- List<ASTNode> nodes = new ArrayList<ASTNode>();
- if (node.getChildCount() == 0) {
- return nodes;
- }
- for (int i = 0; i < node.getChildCount(); i++) {
- ASTNode child = (ASTNode) node.getChild(i);
- if (child.getType() == HiveParser.TOK_TABLE_OR_COL && child.getChild(0) != null &&
- inputRR.get(null,
- BaseSemanticAnalyzer.unescapeIdentifier(child.getChild(0).getText())) != null) {
- nodes.add(child);
- } else {
- nodes.addAll(getColumnExprsFromASTNode(child, inputRR));
+ // Remove expression node descriptor and children of it for a given predicate
+ // from mapping if it's already on RS keys.
+ // Remaining column expressions would be a candidate for an RS value
+ private void removeMappingForKeys(ASTNode predicate, Map<ASTNode, ExprNodeDesc> mapping,
+ List<ExprNodeDesc> keys) {
+ ExprNodeDesc expr = (ExprNodeDesc) mapping.get(predicate);
+ if (expr != null && ExprNodeDescUtils.indexOf(expr, keys) >= 0) {
+ removeRecursively(predicate, mapping);
+ } else {
+ for (int i = 0; i < predicate.getChildCount(); i++) {
+ removeMappingForKeys((ASTNode) predicate.getChild(i), mapping, keys);
}
}
- return nodes;
+ }
+
+ // Remove expression node desc and all children of it from mapping
+ private void removeRecursively(ASTNode current, Map<ASTNode, ExprNodeDesc> mapping) {
+ mapping.remove(current);
+ for (int i = 0; i < current.getChildCount(); i++) {
+ removeRecursively((ASTNode) current.getChild(i), mapping);
+ }
}
/**
@@ -5263,6 +5272,10 @@ public class SemanticAnalyzer extends Ba
}
fileSinkDesc.setLbCtx(lbCtx);
+ // set it in plan instead of runtime in FileSinkOperator
+ fileSinkDesc.setStatsCollectRawDataSize(HiveConf.getBoolVar(conf,
+ HiveConf.ConfVars.HIVE_STATS_COLLECT_RAWDATASIZE));
+
// set the stats publishing/aggregating key prefix
// the same as directory name. The directory name
// can be changed in the optimizer but the key should not be changed
@@ -6055,7 +6068,7 @@ public class SemanticAnalyzer extends Ba
genJoinOperatorTypeCheck(joinSrcOp, srcOps);
JoinOperator joinOp = (JoinOperator) genJoinOperatorChildren(joinTree,
- joinSrcOp, srcOps, omitOpts);
+ joinSrcOp, srcOps, omitOpts);
joinContext.put(joinOp, joinTree);
return joinOp;
}
@@ -8672,7 +8685,8 @@ public class SemanticAnalyzer extends Ba
}
ParseContext pCtx = new ParseContext(conf, qb, child, opToPartPruner,
- opToPartList, topOps, topSelOps, opParseCtx, joinContext, topToTable,
+ opToPartList, topOps, topSelOps, opParseCtx, joinContext, smbMapJoinContext,
+ topToTable,
loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx,
listMapJoinOpsNoReducer, groupOpToInputTables, prunedPartitions,
opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks,
@@ -8834,16 +8848,7 @@ public class SemanticAnalyzer extends Ba
}
/**
- * Generates an expression node descriptor for the expression passed in the
- * arguments. This function uses the row resolver and the metadata information
- * that are passed as arguments to resolve the column names to internal names.
- *
- * @param expr
- * The expression
- * @param input
- * The row resolver
- * @return exprNodeDesc
- * @throws SemanticException
+ * Generates an expression node descriptor for the expression with TypeCheckCtx.
*/
public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input)
throws SemanticException {
@@ -8854,20 +8859,19 @@ public class SemanticAnalyzer extends Ba
}
/**
- * Generates an expression node descriptor for the expression passed in the
- * arguments. This function uses the row resolver and the metadata information
- * that are passed as arguments to resolve the column names to internal names.
- *
- * @param expr
- * The expression
- * @param input
- * The row resolver
- * @param tcCtx
- * Customized type-checking context
- * @return exprNodeDesc
- * @throws SemanticException
+ * Generates an expression node descriptors for the expression and children of it
+ * with default TypeCheckCtx.
+ */
+ public Map<ASTNode, ExprNodeDesc> genAllExprNodeDesc(ASTNode expr, RowResolver input)
+ throws SemanticException {
+ TypeCheckCtx tcCtx = new TypeCheckCtx(input);
+ return genAllExprNodeDesc(expr, input, tcCtx);
+ }
+
+ /**
+ * Returns expression node descriptor for the expression.
+ * If it's evaluated already in previous operator, it can be retrieved from cache.
*/
- @SuppressWarnings("nls")
public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input,
TypeCheckCtx tcCtx) throws SemanticException {
// We recursively create the exprNodeDesc. Base cases: when we encounter
@@ -8878,6 +8882,19 @@ public class SemanticAnalyzer extends Ba
// build the exprNodeFuncDesc with recursively built children.
// If the current subExpression is pre-calculated, as in Group-By etc.
+ ExprNodeDesc cached = getExprNodeDescCached(expr, input);
+ if (cached == null) {
+ Map<ASTNode, ExprNodeDesc> allExprs = genAllExprNodeDesc(expr, input, tcCtx);
+ return allExprs.get(expr);
+ }
+ return cached;
+ }
+
+ /**
+ * Find ExprNodeDesc for the expression cached in the RowResolver. Returns null if not exists.
+ */
+ private ExprNodeDesc getExprNodeDescCached(ASTNode expr, RowResolver input)
+ throws SemanticException {
ColumnInfo colInfo = input.getExpression(expr);
if (colInfo != null) {
ASTNode source = input.getExpressionSource(expr);
@@ -8888,11 +8905,30 @@ public class SemanticAnalyzer extends Ba
.getInternalName(), colInfo.getTabAlias(), colInfo
.getIsVirtualCol(), colInfo.isSkewedCol());
}
+ return null;
+ }
- // Create the walker and the rules dispatcher.
+ /**
+ * Generates all of the expression node descriptors for the expression and children of it
+ * passed in the arguments. This function uses the row resolver and the metadata information
+ * that are passed as arguments to resolve the column names to internal names.
+ *
+ * @param expr
+ * The expression
+ * @param input
+ * The row resolver
+ * @param tcCtx
+ * Customized type-checking context
+ * @return expression to exprNodeDesc mapping
+ * @throws SemanticException Failed to evaluate expression
+ */
+ @SuppressWarnings("nls")
+ public Map<ASTNode, ExprNodeDesc> genAllExprNodeDesc(ASTNode expr, RowResolver input,
+ TypeCheckCtx tcCtx) throws SemanticException {
+ // Create the walker and the rules dispatcher.
tcCtx.setUnparseTranslator(unparseTranslator);
- HashMap<Node, Object> nodeOutputs =
+ Map<ASTNode, ExprNodeDesc> nodeOutputs =
TypeCheckProcFactory.genExprNode(expr, tcCtx);
ExprNodeDesc desc = (ExprNodeDesc) nodeOutputs.get(expr);
if (desc == null) {
@@ -8905,17 +8941,14 @@ public class SemanticAnalyzer extends Ba
if (!unparseTranslator.isEnabled()) {
// Not creating a view, so no need to track view expansions.
- return desc;
+ return nodeOutputs;
}
- for (Map.Entry<Node, Object> entry : nodeOutputs.entrySet()) {
- if (!(entry.getKey() instanceof ASTNode)) {
- continue;
- }
+ for (Map.Entry<ASTNode, ExprNodeDesc> entry : nodeOutputs.entrySet()) {
if (!(entry.getValue() instanceof ExprNodeColumnDesc)) {
continue;
}
- ASTNode node = (ASTNode) entry.getKey();
+ ASTNode node = entry.getKey();
ExprNodeColumnDesc columnDesc = (ExprNodeColumnDesc) entry.getValue();
if ((columnDesc.getTabAlias() == null)
|| (columnDesc.getTabAlias().length() == 0)) {
@@ -8931,7 +8964,7 @@ public class SemanticAnalyzer extends Ba
unparseTranslator.addTranslation(node, replacementText.toString());
}
- return desc;
+ return nodeOutputs;
}
@Override
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java Sat Mar 2 22:37:59 2013
@@ -188,6 +188,7 @@ public final class SemanticAnalyzerFacto
case HiveParser.TOK_ALTERTABLE_TOUCH:
case HiveParser.TOK_ALTERTABLE_ARCHIVE:
case HiveParser.TOK_ALTERTABLE_UNARCHIVE:
+ case HiveParser.TOK_ALTERTABLE_ALTERPARTS:
case HiveParser.TOK_LOCKTABLE:
case HiveParser.TOK_UNLOCKTABLE:
case HiveParser.TOK_CREATEROLE:
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Sat Mar 2 22:37:59 2013
@@ -124,7 +124,7 @@ public final class TypeCheckProcFactory
return desc;
}
- public static HashMap<Node, Object> genExprNode(ASTNode expr,
+ public static Map<ASTNode, ExprNodeDesc> genExprNode(ASTNode expr,
TypeCheckCtx tcCtx) throws SemanticException {
// Create the walker, the rules dispatcher and the context.
// create a walker which walks the tree in a DFS manner while maintaining
@@ -162,10 +162,23 @@ public final class TypeCheckProcFactory
// Create a list of topop nodes
ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.add(expr);
- HashMap<Node, Object> nodeOutputs = new HashMap<Node, Object>();
+ HashMap<Node, Object> nodeOutputs = new LinkedHashMap<Node, Object>();
ogw.startWalking(topNodes, nodeOutputs);
- return nodeOutputs;
+ return convert(nodeOutputs);
+ }
+
+ // temporary type-safe casting
+ private static Map<ASTNode, ExprNodeDesc> convert(Map<Node, Object> outputs) {
+ Map<ASTNode, ExprNodeDesc> converted = new LinkedHashMap<ASTNode, ExprNodeDesc>();
+ for (Map.Entry<Node, Object> entry : outputs.entrySet()) {
+ if (entry.getKey() instanceof ASTNode && entry.getValue() instanceof ExprNodeDesc) {
+ converted.put((ASTNode)entry.getKey(), (ExprNodeDesc)entry.getValue());
+ } else {
+ LOG.warn("Invalid type entry " + entry);
+ }
+ }
+ return converted;
}
/**
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java Sat Mar 2 22:37:59 2013
@@ -48,7 +48,7 @@ public class AlterTableDesc extends DDLD
ADDFILEFORMAT, ADDCLUSTERSORTCOLUMN, RENAMECOLUMN, ADDPARTITION,
TOUCH, ARCHIVE, UNARCHIVE, ALTERPROTECTMODE, ALTERPARTITIONPROTECTMODE,
ALTERLOCATION, DROPPARTITION, RENAMEPARTITION, ADDSKEWEDBY, ALTERSKEWEDLOCATION,
- ALTERBUCKETNUM
+ ALTERBUCKETNUM, ALTERPARTITION
}
public static enum ProtectModeType {
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java Sat Mar 2 22:37:59 2013
@@ -62,6 +62,7 @@ public class DDLWork implements Serializ
private ShowIndexesDesc showIndexesDesc;
private DescDatabaseDesc descDbDesc;
private AlterDatabaseDesc alterDbDesc;
+ private AlterTableAlterPartDesc alterTableAlterPartDesc;
private TruncateTableDesc truncateTblDesc;
private RoleDDLDesc roleDDLDesc;
@@ -442,7 +443,13 @@ public class DDLWork implements Serializ
this.mergeFilesDesc = mergeDesc;
}
- /**
+ public DDLWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
+ AlterTableAlterPartDesc alterPartDesc) {
+ this(inputs, outputs);
+ this.alterTableAlterPartDesc = alterPartDesc;
+ }
+
+ /**
* @return Create Database descriptor
*/
public CreateDatabaseDesc getCreateDatabaseDesc() {
@@ -995,6 +1002,21 @@ public class DDLWork implements Serializ
this.needLock = needLock;
}
+ /**
+ * @return information about the partitions we want to change.
+ */
+ public AlterTableAlterPartDesc getAlterTableAlterPartDesc() {
+ return alterTableAlterPartDesc;
+ }
+
+ /**
+ * @param alterPartitionDesc
+ * information about the partitions we want to change.
+ */
+ public void setAlterTableAlterPartDesc(AlterTableAlterPartDesc alterPartitionDesc) {
+ this.alterTableAlterPartDesc = alterPartitionDesc;
+ }
+
@Explain(displayName = "Truncate Table Operator")
public TruncateTableDesc getTruncateTblDesc() {
return truncateTblDesc;
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java Sat Mar 2 22:37:59 2013
@@ -93,7 +93,7 @@ public class ExprNodeDescUtils {
/**
* bind two predicates by AND op
- */
+ */
public static ExprNodeDesc mergePredicates(ExprNodeDesc prev, ExprNodeDesc next) {
List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>(2);
children.add(prev);
@@ -103,6 +103,44 @@ public class ExprNodeDescUtils {
}
/**
+ * bind n predicates by AND op
+ */
+ public static ExprNodeDesc mergePredicates(List<ExprNodeDesc> exprs) {
+ ExprNodeDesc prev = null;
+ for (ExprNodeDesc expr : exprs) {
+ if (prev == null) {
+ prev = expr;
+ continue;
+ }
+ prev = mergePredicates(prev, expr);
+ }
+ return prev;
+ }
+
+ /**
+ * split predicates by AND op
+ */
+ public static List<ExprNodeDesc> split(ExprNodeDesc current) {
+ return split(current, new ArrayList<ExprNodeDesc>());
+ }
+
+ /**
+ * split predicates by AND op
+ */
+ public static List<ExprNodeDesc> split(ExprNodeDesc current, List<ExprNodeDesc> splitted) {
+ if (FunctionRegistry.isOpAnd(current)) {
+ for (ExprNodeDesc child : current.getChildren()) {
+ split(child, splitted);
+ }
+ return splitted;
+ }
+ if (indexOf(current, splitted) < 0) {
+ splitted.add(current);
+ }
+ return splitted;
+ }
+
+ /**
* Recommend name for the expression
*/
public static String recommendInputName(ExprNodeDesc desc) {
@@ -118,6 +156,25 @@ public class ExprNodeDescUtils {
}
/**
+ * Return false if the expression has any non determinitic function
+ */
+ public static boolean isDeterministic(ExprNodeDesc desc) {
+ if (desc instanceof ExprNodeGenericFuncDesc) {
+ if (!FunctionRegistry.isDeterministic(((ExprNodeGenericFuncDesc)desc).getGenericUDF())) {
+ return false;
+ }
+ }
+ if (desc.getChildren() != null) {
+ for (ExprNodeDesc child : desc.getChildren()) {
+ if (!isDeterministic(child)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
* Convert expressions in current operator to those in terminal operator, which
* is an ancestor of current or null (back to top operator).
*/
@@ -170,7 +227,7 @@ public class ExprNodeDescUtils {
return backtrack(mapped, parent, terminal); // forward with resolved expr
}
- private static Operator<?> getSingleParent(Operator<?> current, Operator<?> terminal)
+ public static Operator<?> getSingleParent(Operator<?> current, Operator<?> terminal)
throws SemanticException {
List<Operator<?>> parents = current.getParentOperators();
if (parents == null || parents.isEmpty()) {
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java Sat Mar 2 22:37:59 2013
@@ -66,6 +66,8 @@ public class FileSinkDesc extends Abstra
private ListBucketingCtx lbCtx;
private int maxStatsKeyPrefixLength = -1;
+ private boolean statsCollectRawDataSize;
+
public FileSinkDesc() {
}
@@ -115,6 +117,7 @@ public class FileSinkDesc extends Abstra
ret.setLinkedFileSinkDesc(linkedFileSinkDesc);
ret.setStatsReliable(statsReliable);
ret.setMaxStatsKeyPrefixLength(maxStatsKeyPrefixLength);
+ ret.setStatsCollectRawDataSize(statsCollectRawDataSize);
return (Object) ret;
}
@@ -353,4 +356,12 @@ public class FileSinkDesc extends Abstra
this.maxStatsKeyPrefixLength = maxStatsKeyPrefixLength;
}
+ public boolean isStatsCollectRawDataSize() {
+ return statsCollectRawDataSize;
+ }
+
+ public void setStatsCollectRawDataSize(boolean statsCollectRawDataSize) {
+ this.statsCollectRawDataSize = statsCollectRawDataSize;
+ }
+
}
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java Sat Mar 2 22:37:59 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.plan;
+import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -507,4 +508,21 @@ public class JoinDesc extends AbstractOp
}
return null;
}
+
+ public int getTagLength() {
+ int tagLength = -1;
+ for (byte tag : getExprs().keySet()) {
+ tagLength = Math.max(tagLength, tag + 1);
+ }
+ return tagLength;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T> T[] convertToArray(Map<Byte, T> source, Class<T> compType) {
+ T[] result = (T[]) Array.newInstance(compType, getTagLength());
+ for (Map.Entry<Byte, T> entry : source.entrySet()) {
+ result[entry.getKey()] = entry.getValue();
+ }
+ return result;
+ }
}
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/SMBJoinDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/SMBJoinDesc.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/SMBJoinDesc.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/plan/SMBJoinDesc.java Sat Mar 2 22:37:59 2013
@@ -31,7 +31,7 @@ public class SMBJoinDesc extends MapJoin
private MapredLocalWork localWork;
- //keep a mapping from tag to the fetch operator alias
+ // keep a mapping from tag to the fetch operator alias
private HashMap<Byte, String> tagToAlias;
private Map<String, DummyStoreOperator> aliasToSink;
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerProcFactory.java Sat Mar 2 22:37:59 2013
@@ -93,7 +93,7 @@ public final class ExprWalkerProcFactory
ctx.addAlias(exp, colAlias[0]);
} else {
if (colAlias == null) {
- assert false;
+ return false;
}
ctx.addAlias(colref, colAlias[0]);
}
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java Sat Mar 2 22:37:59 2013
@@ -672,30 +672,21 @@ public final class OpProcFactory {
RowResolver inputRR = owi.getRowResolver(op);
// combine all predicates into a single expression
- List<ExprNodeDesc> preds = null;
- ExprNodeDesc condn = null;
+ List<ExprNodeDesc> preds = new ArrayList<ExprNodeDesc>();
Iterator<List<ExprNodeDesc>> iterator = pushDownPreds.getFinalCandidates()
.values().iterator();
while (iterator.hasNext()) {
- preds = iterator.next();
- int i = 0;
- if (condn == null) {
- condn = preds.get(0);
- i++;
- }
-
- for (; i < preds.size(); i++) {
- ExprNodeDesc next = preds.get(i);
- if (!ExprNodeDescUtils.containsPredicate(condn, next)) {
- condn = ExprNodeDescUtils.mergePredicates(condn, next);
- }
+ for (ExprNodeDesc pred : iterator.next()) {
+ preds = ExprNodeDescUtils.split(pred, preds);
}
}
- if (condn == null) {
+ if (preds.isEmpty()) {
return null;
}
+ ExprNodeDesc condn = ExprNodeDescUtils.mergePredicates(preds);
+
if (op instanceof TableScanOperator) {
boolean pushFilterToStorage;
HiveConf hiveConf = owi.getParseContext().getConf();
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java Sat Mar 2 22:37:59 2013
@@ -63,18 +63,15 @@ public class PredicateTransitivePropagat
private ParseContext pGraphContext;
+ @Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
pGraphContext = pctx;
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
- opRules.put(new RuleRegExp("R1",
- "(" + FilterOperator.getOperatorName() + "%" +
+ opRules.put(new RuleRegExp("R1", "(" +
+ FilterOperator.getOperatorName() + "%" +
ReduceSinkOperator.getOperatorName() + "%" +
- JoinOperator.getOperatorName() + "%)|" +
- "(" + FilterOperator.getOperatorName() + "%" +
- ReduceSinkOperator.getOperatorName() + "%" +
- MapJoinOperator.getOperatorName() + "%)")
- , new JoinTransitive());
+ JoinOperator.getOperatorName() + "%)"), new JoinTransitive());
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
@@ -87,21 +84,22 @@ public class PredicateTransitivePropagat
topNodes.addAll(pGraphContext.getTopOps().values());
ogw.startWalking(topNodes, null);
- Map<ReduceSinkOperator, ExprNodeDesc> newFilters = context.getNewfilters();
+ Map<ReduceSinkOperator, List<ExprNodeDesc>> newFilters = context.getNewfilters();
// insert new filter between RS and parent of RS
- for (Map.Entry<ReduceSinkOperator, ExprNodeDesc> entry : newFilters.entrySet()) {
+ for (Map.Entry<ReduceSinkOperator, List<ExprNodeDesc>> entry : newFilters.entrySet()) {
ReduceSinkOperator reducer = entry.getKey();
Operator<?> parent = reducer.getParentOperators().get(0);
- ExprNodeDesc expr = entry.getValue();
+ List<ExprNodeDesc> exprs = entry.getValue();
if (parent instanceof FilterOperator) {
- ExprNodeDesc prev = ((FilterOperator)parent).getConf().getPredicate();
- ExprNodeDesc merged = ExprNodeDescUtils.mergePredicates(prev, expr);
+ exprs = ExprNodeDescUtils.split(((FilterOperator)parent).getConf().getPredicate(), exprs);
+ ExprNodeDesc merged = ExprNodeDescUtils.mergePredicates(exprs);
((FilterOperator)parent).getConf().setPredicate(merged);
} else {
+ ExprNodeDesc merged = ExprNodeDescUtils.mergePredicates(exprs);
RowResolver parentRR = pGraphContext.getOpParseCtx().get(parent).getRowResolver();
- Operator<FilterDesc> newFilter = createFilter(reducer, parent, parentRR, expr);
+ Operator<FilterDesc> newFilter = createFilter(reducer, parent, parentRR, merged);
pGraphContext.getOpParseCtx().put(newFilter, new OpParseContext(parentRR));
}
}
@@ -126,24 +124,24 @@ public class PredicateTransitivePropagat
private static class TransitiveContext implements NodeProcessorCtx {
private final Map<CommonJoinOperator, int[][]> filterPropagates;
- private final Map<ReduceSinkOperator, ExprNodeDesc> newFilters;
+ private final Map<ReduceSinkOperator, List<ExprNodeDesc>> newFilters;
public TransitiveContext() {
filterPropagates = new HashMap<CommonJoinOperator, int[][]>();
- newFilters = new HashMap<ReduceSinkOperator, ExprNodeDesc>();
+ newFilters = new HashMap<ReduceSinkOperator, List<ExprNodeDesc>>();
}
- public Map<CommonJoinOperator, int[][]> getFilterPropates() {
+ public Map<CommonJoinOperator, int[][]> getFilterPropagates() {
return filterPropagates;
}
- public Map<ReduceSinkOperator, ExprNodeDesc> getNewfilters() {
+ public Map<ReduceSinkOperator, List<ExprNodeDesc>> getNewfilters() {
return newFilters;
}
}
private static class JoinTransitive implements NodeProcessor {
-
+ @Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
@SuppressWarnings("unchecked")
@@ -153,8 +151,8 @@ public class PredicateTransitivePropagat
int srcPos = join.getParentOperators().indexOf(source);
TransitiveContext context = (TransitiveContext) procCtx;
- Map<CommonJoinOperator, int[][]> filterPropagates = context.getFilterPropates();
- Map<ReduceSinkOperator, ExprNodeDesc> newFilters = context.getNewfilters();
+ Map<CommonJoinOperator, int[][]> filterPropagates = context.getFilterPropagates();
+ Map<ReduceSinkOperator, List<ExprNodeDesc>> newFilters = context.getNewfilters();
int[][] targets = filterPropagates.get(join);
if (targets == null) {
@@ -170,11 +168,11 @@ public class PredicateTransitivePropagat
ExprNodeDesc predicate = filter.getConf().getPredicate();
ExprNodeDesc replaced = ExprNodeDescUtils.replace(predicate, sourceKeys, targetKeys);
if (replaced != null && !filterExists(target, replaced)) {
- ExprNodeDesc prev = newFilters.get(target);
+ List<ExprNodeDesc> prev = newFilters.get(target);
if (prev == null) {
- newFilters.put(target, replaced);
+ newFilters.put(target, ExprNodeDescUtils.split(replaced));
} else {
- newFilters.put(target, ExprNodeDescUtils.mergePredicates(prev, replaced));
+ ExprNodeDescUtils.split(replaced, prev);
}
}
}
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFEvaluator.java Sat Mar 2 22:37:59 2013
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.MapredContext;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.UDFType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -35,7 +39,7 @@ import org.apache.hadoop.hive.serde2.obj
* array<int>, array<array<int>> and so on (arbitrary levels of nesting).
*/
@UDFType(deterministic = true)
-public abstract class GenericUDAFEvaluator {
+public abstract class GenericUDAFEvaluator implements Closeable {
/**
* Mode.
@@ -73,6 +77,15 @@ public abstract class GenericUDAFEvaluat
}
/**
+ * Additionally setup GenericUDAFEvaluator with MapredContext before initializing.
+ * This is only called in runtime of MapRedTask.
+ *
+ * @param context context
+ */
+ public void configure(MapredContext mapredContext) {
+ }
+
+ /**
* Initialize the evaluator.
*
* @param m
@@ -126,6 +139,13 @@ public abstract class GenericUDAFEvaluat
public abstract void reset(AggregationBuffer agg) throws HiveException;
/**
+ * Close GenericUDFEvaluator.
+ * This is only called in runtime of MapRedTask.
+ */
+ public void close() throws IOException {
+ }
+
+ /**
* This function will be called by GroupByOperator when it sees a new input
* row.
*
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java Sat Mar 2 22:37:59 2013
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.hadoop.hive.ql.exec.MapredContext;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -39,7 +43,7 @@ import org.apache.hadoop.hive.serde2.obj
* can do short-circuit evaluations using DeferedObject.
*/
@UDFType(deterministic = true)
-public abstract class GenericUDF {
+public abstract class GenericUDF implements Closeable {
/**
* A Defered Object allows us to do lazy-evaluation and short-circuiting.
@@ -86,6 +90,15 @@ public abstract class GenericUDF {
throws UDFArgumentException;
/**
+ * Additionally setup GenericUDF with MapredContext before initializing.
+ * This is only called in runtime of MapRedTask.
+ *
+ * @param context context
+ */
+ public void configure(MapredContext context) {
+ }
+
+ /**
* Initialize this GenericUDF. Additionally, if the arguments are constant
* and the function is eligible to be folded, then the constant value
* returned by this UDF will be computed and stored in the
@@ -163,4 +176,10 @@ public abstract class GenericUDF {
*/
public abstract String getDisplayString(String[] children);
+ /**
+ * Close GenericUDF.
+ * This is only called in runtime of MapRedTask.
+ */
+ public void close() throws IOException {
+ }
}
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFReflect.java Sat Mar 2 22:37:59 2013
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hive.ql.udf.generic;
-import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Arrays;
@@ -29,12 +28,9 @@ import org.apache.hadoop.hive.ql.udf.UDF
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
import org.apache.hadoop.util.ReflectionUtils;
/**
@@ -44,16 +40,14 @@ import org.apache.hadoop.util.Reflection
value = "_FUNC_(class,method[,arg1[,arg2..]]) calls method with reflection",
extended = "Use this UDF to call Java methods by matching the argument signature\n")
@UDFType(deterministic = false)
-public class GenericUDFReflect extends GenericUDF {
+public class GenericUDFReflect extends AbstractGenericUDFReflect {
+
+ StringObjectInspector inputClassNameOI;
+ StringObjectInspector inputMethodNameOI;
- PrimitiveObjectInspector[] argumentOIs;
StringObjectInspector classNameOI;
StringObjectInspector methodNameOI;
-
- PrimitiveTypeEntry[] parameterTypes;
- Class[] parameterClasses;
- Object[] parameterJavaValues;
-
+
@Override
public ObjectInspector initialize(ObjectInspector[] arguments)
throws UDFArgumentException {
@@ -71,35 +65,16 @@ public class GenericUDFReflect extends G
+ " should be string.");
}
}
-
+ inputClassNameOI = (StringObjectInspector) arguments[0];
+ inputMethodNameOI = (StringObjectInspector) arguments[1];
+
classNameOI = (StringObjectInspector)
ObjectInspectorUtils.getStandardObjectInspector(arguments[0]);
methodNameOI = (StringObjectInspector)
ObjectInspectorUtils.getStandardObjectInspector(arguments[1]);
-
- parameterTypes = new PrimitiveTypeEntry[arguments.length - 2];
- parameterClasses = new Class[arguments.length - 2];
- for (int i = 2; i < arguments.length; i++) {
- if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) {
- throw new UDFArgumentTypeException(i,
- "The parameters of GenericUDFReflect(class,method[,arg1[,arg2]...])"
- + " must be primitive (int, double, string, etc).");
- }
- PrimitiveCategory category =
- ((PrimitiveObjectInspector)arguments[i]).getPrimitiveCategory();
- parameterTypes[i - 2] =
- PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveCategory(category);
- parameterClasses[i - 2] = parameterTypes[i - 2].primitiveJavaType == null ?
- parameterTypes[i - 2].primitiveJavaClass : parameterTypes[i - 2].primitiveJavaType;
- }
-
- parameterJavaValues = new Object[arguments.length - 2];
- argumentOIs = new PrimitiveObjectInspector[arguments.length];
- for (int i = 0; i < arguments.length; i++) {
- argumentOIs[i] = (PrimitiveObjectInspector)arguments[i];
- }
-
+ setupParameterOIs(arguments, 2);
+
return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveCategory.STRING);
}
@@ -109,8 +84,7 @@ public class GenericUDFReflect extends G
Method m;
Object className;
Object methodName;
- String result;
-
+
@Override
public Object evaluate(DeferredObject[] arguments) throws HiveException {
@@ -118,15 +92,14 @@ public class GenericUDFReflect extends G
// Skip class loading if the class name didn't change
boolean classNameChanged = false;
- ObjectInspector newClassNameOI = argumentOIs[0];
Object newClassName = arguments[0].get();
// We compare class name/method name using ObjectInspectorUtils.compare(...), to avoid
// any object conversion (which may cause object creation) in most cases, when the class
// name/method name is constant Java String, or constant Text (StringWritable).
if (className == null || ObjectInspectorUtils.compare(className, classNameOI, newClassName,
- newClassNameOI) != 0) {
- className = ObjectInspectorUtils.copyToStandardObject(newClassName, newClassNameOI);
+ inputClassNameOI) != 0) {
+ className = ObjectInspectorUtils.copyToStandardObject(newClassName, inputClassNameOI);
String classNameString = classNameOI.getPrimitiveJavaObject(className);
try {
c = Class.forName(classNameString);
@@ -144,80 +117,32 @@ public class GenericUDFReflect extends G
// Try to find the method
// Skip method finding if the method name didn't change, and class name didn't change.
- ObjectInspector newMethodNameOI = argumentOIs[1];
Object newMethodName = arguments[1].get();
-
+
if (methodName == null || ObjectInspectorUtils.compare(methodName, methodNameOI, newMethodName,
- newMethodNameOI) != 0 || classNameChanged) {
- methodName = ObjectInspectorUtils.copyToStandardObject(newMethodName, newMethodNameOI);
+ inputMethodNameOI) != 0 || classNameChanged) {
+ methodName = ObjectInspectorUtils.copyToStandardObject(newMethodName, inputMethodNameOI);
String methodNameString = methodNameOI.getPrimitiveJavaObject(methodName);
try {
- m = findMethod(c, methodNameString, parameterTypes, parameterClasses);
+ m = findMethod(c, methodNameString, String.class, false);
} catch (Exception e) {
throw new HiveException("UDFReflect getMethod ", e);
}
}
- // Get the parameter values
- for (int i = 2; i < arguments.length; i++) {
- parameterJavaValues[i - 2] = argumentOIs[i].getPrimitiveJavaObject(arguments[i].get());
- }
+ Object[] parameterJavaValues = setupParameters(arguments, 2);
try {
- result = String.valueOf(m.invoke(o, parameterJavaValues));
- return result;
- } catch (IllegalArgumentException e1) {
- System.err.println("UDFReflect evaluate "+ e1 + " method = " + m + " args = " +
- Arrays.asList(parameterJavaValues));
- } catch (IllegalAccessException e1) {
- System.err.println("UDFReflect evaluate "+ e1 + " method = " + m + " args = " +
- Arrays.asList(parameterJavaValues));
- } catch (InvocationTargetException e1) {
- System.err.println("UDFReflect evaluate "+ e1 + " method = " + m + " args = " +
+ return String.valueOf(m.invoke(o, parameterJavaValues));
+ } catch (Exception e1) {
+ System.err.println("UDFReflect evaluate " + e1 + " method = " + m + " args = " +
Arrays.asList(parameterJavaValues));
}
return null;
}
@Override
- public String getDisplayString(String[] children) {
- StringBuilder sb = new StringBuilder();
- sb.append("reflect(");
- for (int i = 0; i < children.length; i++) {
- if (i > 0) {
- sb.append(',');
- }
- sb.append(children[i]);
- }
- sb.append(')');
- return sb.toString();
- }
-
- // a(string,int,int) can be matched with methods like
- // a(string,int,int), a(string,int,Integer), a(string,Integer,int) and a(string,Integer,Integer)
- // and accepts the first one clazz.getMethods() returns
- private Method findMethod(Class clazz, String name, PrimitiveTypeEntry[] parameterTypes,
- Class[] parameterClasses) throws Exception {
- for (Method method : clazz.getMethods()) {
- if (!method.getName().equals(name) || method.getReturnType() != String.class ||
- method.getParameterTypes().length != parameterTypes.length) {
- continue;
- }
- // returns first one matches all of the params
- boolean match = true;
- Class<?>[] types = method.getParameterTypes();
- for (int i = 0; i < parameterTypes.length; i++) {
- if (types[i] != parameterTypes[i].primitiveJavaType &&
- types[i] != parameterTypes[i].primitiveJavaClass) {
- match = false;
- break;
- }
- }
- if (match) {
- return method;
- }
- }
- // tried all, back to original code (for error message)
- return clazz.getMethod(name, parameterClasses);
+ protected String functionName() {
+ return "reflect";
}
}
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTF.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTF.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTF.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTF.java Sat Mar 2 22:37:59 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.udf.generic;
+import org.apache.hadoop.hive.ql.exec.MapredContext;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -34,6 +35,15 @@ public abstract class GenericUDTF {
Collector collector = null;
/**
+ * Additionally setup GenericUDTF with MapredContext before initializing.
+ * This is only called in runtime of MapRedTask.
+ *
+ * @param context context
+ */
+ public void configure(MapredContext mapredContext) {
+ }
+
+ /**
* Initialize this GenericUDTF. This will be called only once per instance.
*
* @param argOIs
Modified: hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java (original)
+++ hive/branches/ptf-windowing/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java Sat Mar 2 22:37:59 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.udf.pt
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Map;
import java.util.List;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
@@ -818,9 +819,9 @@ public class NPath extends TableFunction
{
// todo: use SemanticAnalyzer::genExprNodeDesc
// currently SA not available to PTFTranslator.
- HashMap<Node, Object> map = TypeCheckProcFactory
+ Map<ASTNode, ExprNodeDesc> map = TypeCheckProcFactory
.genExprNode(expr, typeCheckCtx);
- ExprNodeDesc desc = (ExprNodeDesc) map.get(expr);
+ ExprNodeDesc desc = map.get(expr);
if (desc == null) {
String errMsg = typeCheckCtx.getError();
if ( errMsg == null) {
Modified: hive/branches/ptf-windowing/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/ptf-windowing/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java Sat Mar 2 22:37:59 2013
@@ -35,6 +35,7 @@ import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
@@ -100,6 +101,8 @@ public class QTestUtil {
protected final String logDir;
private final TreeMap<String, String> qMap;
private final Set<String> qSkipSet;
+ private final Set<String> qSortSet;
+ private static final String SORT_SUFFIX = ".sorted";
public static final HashSet<String> srcTables = new HashSet<String>
(Arrays.asList(new String [] {
"src", "src1", "srcbucket", "srcbucket2", "src_json", "src_thrift",
@@ -286,6 +289,7 @@ public class QTestUtil {
this.hadoopVer = getHadoopMainVersion(hadoopVer);
qMap = new TreeMap<String, String>();
qSkipSet = new HashSet<String>();
+ qSortSet = new HashSet<String>();
if (miniMr) {
dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);
@@ -347,15 +351,29 @@ public class QTestUtil {
while ((line = br.readLine()) != null) {
qsb.append(line + "\n");
}
- String query = qsb.toString();
+ br.close();
+ String query = qsb.toString();
qMap.put(qf.getName(), query);
if(checkHadoopVersionExclude(qf.getName(), query)
|| checkOSExclude(qf.getName(), query)) {
qSkipSet.add(qf.getName());
}
- br.close();
+
+ if (checkNeedsSort(qf.getName(), query)) {
+ qSortSet.add(qf.getName());
+ }
+ }
+
+ private boolean checkNeedsSort(String fileName, String query) {
+ Pattern pattern = Pattern.compile("-- SORT_BEFORE_DIFF");
+ Matcher matcher = pattern.matcher(query);
+
+ if (matcher.find()) {
+ return true;
+ }
+ return false;
}
private boolean checkHadoopVersionExclude(String fileName, String query){
@@ -394,14 +412,14 @@ public class QTestUtil {
String versions = matcher.group(2);
for (String s : versions.split("\\,")) {
s = s.trim();
- versionSet.add(s);
+ versionSet.add(s);
}
}
if (matcher.find()) {
//2nd match is not supposed to be there
String message = "QTestUtil: qfile " + fileName
- + " contains more than one reference to (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS";
+ + " contains more than one reference to (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS";
throw new UnsupportedOperationException(message);
}
@@ -427,20 +445,20 @@ public class QTestUtil {
if (matcher.find()) {
String prefix = matcher.group(1);
if ("EX".equals(prefix)) {
- //windows is to be exluded
- if(Shell.WINDOWS){
- System.out.println("Due to the OS being windows " +
- "adding the query " + fileName +
- " to the set of tests to skip");
- return true;
- }
+ //windows is to be exluded
+ if(Shell.WINDOWS){
+ System.out.println("Due to the OS being windows " +
+ "adding the query " + fileName +
+ " to the set of tests to skip");
+ return true;
+ }
}
else if(!Shell.WINDOWS){
- //non windows to be exluded
- System.out.println("Due to the OS not being windows " +
- "adding the query " + fileName +
- " to the set of tests to skip");
- return true;
+ //non windows to be exluded
+ System.out.println("Due to the OS not being windows " +
+ "adding the query " + fileName +
+ " to the set of tests to skip");
+ return true;
}
}
return false;
@@ -817,7 +835,8 @@ public class QTestUtil {
outfd.write(e.getMessage());
outfd.close();
- int exitVal = executeDiffCommand(outf.getPath(), expf, false);
+ int exitVal = executeDiffCommand(outf.getPath(), expf, false,
+ qSortSet.contains(qf.getName()));
if (exitVal != 0 && overWrite) {
exitVal = overwriteResults(outf.getPath(), expf);
}
@@ -839,7 +858,7 @@ public class QTestUtil {
outfd.write(tree.toStringTree());
outfd.close();
- int exitVal = executeDiffCommand(outf.getPath(), expf, false);
+ int exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
if (exitVal != 0 && overWrite) {
exitVal = overwriteResults(outf.getPath(), expf);
@@ -876,7 +895,7 @@ public class QTestUtil {
};
maskPatterns(patterns, outf.getPath());
- int exitVal = executeDiffCommand(outf.getPath(), planFile, true);
+ int exitVal = executeDiffCommand(outf.getPath(), planFile, true, false);
if (exitVal != 0 && overWrite) {
exitVal = overwriteResults(outf.getPath(), planFile);
@@ -1009,7 +1028,8 @@ public class QTestUtil {
maskPatterns(patterns, f.getPath());
int exitVal = executeDiffCommand(f.getPath(),
- outFileName, false);
+ outFileName, false,
+ qSortSet.contains(tname));
if (exitVal != 0 && overWrite) {
exitVal = overwriteResults(f.getPath(), outFileName);
@@ -1022,18 +1042,42 @@ public class QTestUtil {
// This method can be replaced with Files.copy(source, target, REPLACE_EXISTING)
// once Hive uses JAVA 7.
System.out.println("Overwriting results");
- String[] cmdArray = new String[] {
+ return executeCmd(new String[] {
"cp",
- Shell.WINDOWS ? getQuotedString(inFileName) : inFileName,
- Shell.WINDOWS ? getQuotedString(outFileName) : outFileName
- };
- Process executor = Runtime.getRuntime().exec(cmdArray);
- return executor.waitFor();
+ getQuotedString(inFileName),
+ getQuotedString(outFileName)
+ });
}
private static int executeDiffCommand(String inFileName,
String outFileName,
- boolean ignoreWhiteSpace) throws Exception {
+ boolean ignoreWhiteSpace,
+ boolean sortResults
+ ) throws Exception {
+
+ int result = 0;
+
+ if (sortResults) {
+ // sort will try to open the output file in write mode on windows. We need to
+ // close it first.
+ SessionState ss = SessionState.get();
+ if (ss != null && ss.out != null && ss.out != System.out) {
+ ss.out.close();
+ }
+
+ String inSorted = inFileName + SORT_SUFFIX;
+ String outSorted = outFileName + SORT_SUFFIX;
+
+ result = sortFiles(inFileName, inSorted);
+ result |= sortFiles(outFileName, outSorted);
+ if (result != 0) {
+ System.err.println("ERROR: Could not sort files before comparing");
+ return result;
+ }
+ inFileName = inSorted;
+ outFileName = outSorted;
+ }
+
ArrayList<String> diffCommandArgs = new ArrayList<String>();
diffCommandArgs.add("diff");
@@ -1055,26 +1099,75 @@ public class QTestUtil {
diffCommandArgs.add("-B"); // Ignore changes whose lines are all blank
}
// Add files to compare to the arguments list
- diffCommandArgs.add(Shell.WINDOWS ? getQuotedString(inFileName) : inFileName);
- diffCommandArgs.add(Shell.WINDOWS ? getQuotedString(outFileName) : outFileName);
- String[] cmdArray =(String [])diffCommandArgs.toArray(new String [diffCommandArgs.size ()]);
- System.out.println(org.apache.commons.lang.StringUtils.join(cmdArray, ' '));
-
- Process executor = Runtime.getRuntime().exec(cmdArray);
-
- StreamPrinter outPrinter = new StreamPrinter(
- executor.getInputStream(), null, SessionState.getConsole().getChildOutStream());
- StreamPrinter errPrinter = new StreamPrinter(
- executor.getErrorStream(), null, SessionState.getConsole().getChildErrStream());
+ diffCommandArgs.add(getQuotedString(inFileName));
+ diffCommandArgs.add(getQuotedString(outFileName));
+
+ result = executeCmd(diffCommandArgs);
+
+ if (sortResults) {
+ new File(inFileName).delete();
+ new File(outFileName).delete();
+ }
+
+ return result;
+ }
+
+ private static int sortFiles(String in, String out) throws Exception {
+ return executeCmd(new String[] {
+ "sort",
+ getQuotedString(in),
+ }, out, null);
+ }
+
+ private static int executeCmd(Collection<String> args) throws Exception {
+ return executeCmd(args, null, null);
+ }
+
+ private static int executeCmd(String[] args) throws Exception {
+ return executeCmd(args, null, null);
+ }
+
+ private static int executeCmd(Collection<String> args, String outFile, String errFile) throws Exception {
+ String[] cmdArray = (String[]) args.toArray(new String[args.size()]);
+ return executeCmd(cmdArray, outFile, errFile);
+ }
+
+ private static int executeCmd(String[] args, String outFile, String errFile) throws Exception {
+ System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' '));
+
+ PrintStream out = outFile == null ?
+ SessionState.getConsole().getChildOutStream() :
+ new PrintStream(new FileOutputStream(outFile), true);
+ PrintStream err = errFile == null ?
+ SessionState.getConsole().getChildErrStream() :
+ new PrintStream(new FileOutputStream(errFile), true);
+ Process executor = Runtime.getRuntime().exec(args);
+
+ StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err);
+ StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out);
+
outPrinter.start();
errPrinter.start();
- return executor.waitFor();
+ int result = executor.waitFor();
+
+ outPrinter.join();
+ errPrinter.join();
+
+ if (outFile != null) {
+ out.close();
+ }
+
+ if (errFile != null) {
+ err.close();
+ }
+
+ return result;
}
private static String getQuotedString(String str){
- return String.format("\"%s\"", str);
+ return Shell.WINDOWS ? String.format("\"%s\"", str) : str;
}
public ASTNode parseQuery(String tname) throws Exception {
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/add_partition_with_whitelist.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/add_partition_with_whitelist.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/add_partition_with_whitelist.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/add_partition_with_whitelist.q Sat Mar 2 22:37:59 2013
@@ -1,4 +1,3 @@
-SET hive.metastore.pre.event.listeners=org.apache.hadoop.hive.metastore.PartitionNameWhitelistPreEventListener;
SET hive.metastore.partition.name.whitelist.pattern=[\\x20-\\x7E&&[^,]]* ;
-- This pattern matches all printable ASCII characters (disallow unicode) and disallows commas
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_3.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_3.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_3.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_3.q Sat Mar 2 22:37:59 2013
@@ -1,3 +1,5 @@
+-- SORT_BEFORE_DIFF
+
create table authorization_fail_3 (key int, value string) partitioned by (ds string);
set hive.security.authorization.enabled=true;
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_4.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_4.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_4.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_4.q Sat Mar 2 22:37:59 2013
@@ -1,3 +1,5 @@
+-- SORT_BEFORE_DIFF
+
create table authorization_fail_4 (key int, value string) partitioned by (ds string);
set hive.security.authorization.enabled=true;
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_5.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_5.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_5.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_5.q Sat Mar 2 22:37:59 2013
@@ -1,3 +1,5 @@
+-- SORT_BEFORE_DIFF
+
create table authorization_fail (key int, value string) partitioned by (ds string);
set hive.security.authorization.enabled=true;
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_6.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_6.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_6.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_6.q Sat Mar 2 22:37:59 2013
@@ -1,3 +1,5 @@
+-- SORT_BEFORE_DIFF
+
create table authorization_part_fail (key int, value string) partitioned by (ds string);
set hive.security.authorization.enabled=true;
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_7.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_7.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_7.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_fail_7.q Sat Mar 2 22:37:59 2013
@@ -1,3 +1,5 @@
+-- SORT_BEFORE_DIFF
+
create table authorization_fail (key int, value string);
set hive.security.authorization.enabled=true;
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_part.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_part.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_part.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/authorization_part.q Sat Mar 2 22:37:59 2013
@@ -1,3 +1,5 @@
+-- SORT_BEFORE_DIFF
+
create table authorization_part_fail (key int, value string) partitioned by (ds string);
ALTER TABLE authorization_part_fail SET TBLPROPERTIES ("PARTITION_LEVEL_PRIVILEGE"="TRUE");
create table src_auth as select * from src;
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/dynamic_partitions_with_whitelist.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/dynamic_partitions_with_whitelist.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/dynamic_partitions_with_whitelist.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/dynamic_partitions_with_whitelist.q Sat Mar 2 22:37:59 2013
@@ -1,5 +1,5 @@
-SET hive.metastore.pre.event.listeners=org.apache.hadoop.hive.metastore.PartitionNameWhitelistPreEventListener;
-SET hive.metastore.partition.name.whitelist.pattern=[A-Za-z]*;
+SET hive.metastore.partition.name.whitelist.pattern=[^9]*;
+set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.VerifyTableDirectoryIsEmptyHook;
set hive.exec.dynamic.partition=true;
set hive.exec.dynamic.partition.mode=nonstrict;
@@ -10,5 +10,8 @@ create table dest_table like srcpart;
load data local inpath '../data/files/srcbucket20.txt' INTO TABLE source_table partition(ds='2008-04-08', hr=11);
-insert overwrite table dest_table partition (ds, hr) select key, value, hr from source_table where ds='2008-04-08';
+-- Tests creating dynamic partitions with characters not in the whitelist (i.e. 9)
+-- If the directory is not empty the hook will throw an error, instead the error should come from the metastore
+-- This shows that no dynamic partitions were created and left behind or had directories created
+insert overwrite table dest_table partition (ds, hr) select key, hr, ds, value from source_table where ds='2008-04-08' order by value asc;
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/show_tables_bad1.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/show_tables_bad1.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/show_tables_bad1.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/show_tables_bad1.q Sat Mar 2 22:37:59 2013
@@ -1 +1 @@
-SHOW TABLES LIKE;
+SHOW TABLES JOIN;
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/show_tables_bad2.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/show_tables_bad2.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/show_tables_bad2.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/show_tables_bad2.q Sat Mar 2 22:37:59 2013
@@ -1 +1 @@
-SHOW TABLES FROM default LIKE;
+SHOW TABLES FROM default LIKE a b;
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/smb_mapjoin_14.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/smb_mapjoin_14.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/smb_mapjoin_14.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientnegative/smb_mapjoin_14.q Sat Mar 2 22:37:59 2013
@@ -18,7 +18,7 @@ set hive.input.format = org.apache.hadoo
-- A join is being performed across different sub-queries, where a mapjoin is being performed in each of them.
-- Each sub-query should be converted to a sort-merge join.
-- A join followed by mapjoin is not allowed, so this query should fail.
--- Once HIVE-3433 is in, this should be automatically converted to a sort-merge join without the hint
+-- Once HIVE-3403 is in, this should be automatically converted to a sort-merge join without the hint
explain
select src1.key, src1.cnt1, src2.cnt1 from
(
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/add_partition_no_whitelist.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/add_partition_no_whitelist.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/add_partition_no_whitelist.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/add_partition_no_whitelist.q Sat Mar 2 22:37:59 2013
@@ -1,5 +1,5 @@
-SET hive.metastore.pre.event.listeners= ;
--- Test with PartitionNameWhitelistPreEventListener NOT registered
+SET hive.metastore.partition.name.whitelist.pattern=;
+-- Test with no partition name whitelist pattern
CREATE TABLE part_nowhitelist_test (key STRING, value STRING) PARTITIONED BY (ds STRING);
SHOW PARTITIONS part_nowhitelist_test;
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/add_partition_with_whitelist.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/add_partition_with_whitelist.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/add_partition_with_whitelist.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/add_partition_with_whitelist.q Sat Mar 2 22:37:59 2013
@@ -1,4 +1,3 @@
-SET hive.metastore.pre.event.listeners=org.apache.hadoop.hive.metastore.PartitionNameWhitelistPreEventListener;
SET hive.metastore.partition.name.whitelist.pattern=[A-Za-z]*;
-- This pattern matches only letters.
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/alter_rename_partition_authorization.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/alter_rename_partition_authorization.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/alter_rename_partition_authorization.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/alter_rename_partition_authorization.q Sat Mar 2 22:37:59 2013
@@ -1,3 +1,5 @@
+-- SORT_BEFORE_DIFF
+
create table src_auth_tmp as select * from src;
create table authorization_part (key int, value string) partitioned by (ds string);
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_1.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_1.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_1.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_1.q Sat Mar 2 22:37:59 2013
@@ -1,3 +1,5 @@
+-- SORT_BEFORE_DIFF
+
create table src_autho_test as select * from src;
set hive.security.authorization.enabled=true;
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_2.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_2.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_2.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_2.q Sat Mar 2 22:37:59 2013
@@ -1,3 +1,5 @@
+-- SORT_BEFORE_DIFF
+
create table authorization_part (key int, value string) partitioned by (ds string);
create table src_auth_tmp as select * from src;
ALTER TABLE authorization_part SET TBLPROPERTIES ("PARTITION_LEVEL_PRIVILEGE"="TRUE");
Modified: hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_3.q
URL: http://svn.apache.org/viewvc/hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_3.q?rev=1451954&r1=1451953&r2=1451954&view=diff
==============================================================================
--- hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_3.q (original)
+++ hive/branches/ptf-windowing/ql/src/test/queries/clientpositive/authorization_3.q Sat Mar 2 22:37:59 2013
@@ -1,3 +1,5 @@
+-- SORT_BEFORE_DIFF
+
create table src_autho_test as select * from src;
grant drop on table src_autho_test to user hive_test_user;