You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dh...@apache.org on 2008/10/21 20:30:34 UTC
svn commit: r706708 [4/13] - in /hadoop/core/branches/branch-0.19: ./
src/contrib/hive/ src/contrib/hive/bin/
src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/
src/contrib/hive/common/src/java/org/apache/hadoop/hive/conf/
src/contrib/hive/conf/...
Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=706708&r1=706707&r2=706708&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Oct 21 11:29:18 2008
@@ -24,7 +24,10 @@
import java.lang.reflect.Method;
import org.antlr.runtime.tree.*;
+import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
@@ -35,8 +38,10 @@
import org.apache.hadoop.hive.ql.typeinfo.TypeInfo;
import org.apache.hadoop.hive.ql.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.ql.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.ql.udf.UDFOPPositive;
import org.apache.hadoop.hive.ql.exec.*;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.fs.Path;
import org.apache.commons.lang.StringUtils;
@@ -342,7 +347,12 @@
skipRecursion = true;
}
break;
-
+
+ case HiveParser.TOK_LIMIT:
+ {
+ qbp.setDestLimit(ctx_1.dest, new Integer(ast.getChild(0).getText()));
+ }
+ break;
default:
skipRecursion = false;
break;
@@ -384,8 +394,18 @@
// Pass each where clause to the pruner
QBParseInfo qbp = qb.getParseInfo();
for(String clause: qbp.getClauseNames()) {
- if (qbp.getWhrForClause(clause) != null) {
- pruner.addExpression((CommonTree)qbp.getWhrForClause(clause).getChild(0));
+
+ CommonTree whexp = (CommonTree)qbp.getWhrForClause(clause);
+
+ if (pruner.getTable().isPartitioned() &&
+ conf.getVar(HiveConf.ConfVars.HIVEPARTITIONPRUNER).equalsIgnoreCase("strict") &&
+ (whexp == null || !pruner.hasPartitionPredicate((CommonTree)whexp.getChild(0)))) {
+ throw new SemanticException(ErrorMsg.NO_PARTITION_PREDICATE.getMsg(whexp != null ? whexp : qbp.getSelForClause(clause),
+ " for Alias " + alias + " Table " + pruner.getTable().getName()));
+ }
+
+ if (whexp != null) {
+ pruner.addExpression((CommonTree)whexp.getChild(0));
}
}
@@ -466,7 +486,7 @@
CommonTree ast = qbp.getDestForClause(name);
switch (ast.getToken().getType()) {
case HiveParser.TOK_TAB: {
- tableSpec ts = new tableSpec(this.db, ast);
+ tableSpec ts = new tableSpec(this.db, ast, true);
if(ts.partSpec == null) {
// This is a table
@@ -487,6 +507,7 @@
{
fname = getTmpFileName();
ctx.setResDir(new Path(fname));
+ qb.setIsQuery(true);
}
qb.getMetaData().setDestForAlias(name, fname,
(ast.getToken().getType() == HiveParser.TOK_DIR));
@@ -643,138 +664,50 @@
}
}
- /**
- * Returns the expression for the SerDe field.
- * @return null if the tree cannot be represented by a SerDe field.
- */
- public static String getSerDeFieldExpression(CommonTree node) {
- if (node.getToken().getType() == HiveParser.TOK_COLREF){
- // String tabAlias = node.getChild(0).getText();
- String colName = node.getChild(1).getText();
- return colName;
- }
- if (node.getChildCount() != 2) {
- return null;
- }
- String left = getSerDeFieldExpression((CommonTree)node.getChild(0));
- if (left == null) return null;
-
- if (node.getToken().getType() == HiveParser.DOT) {
- return left + '.' + node.getChild(1).getText();
- } else if (node.getToken().getType() == HiveParser.LSQUARE){
- return left + '[' + node.getChild(1).getText() + ']';
- }
- return null;
- }
-
- /**
- * Returns the table name for the SerDe field.
- * @return null if the tree cannot be represented by a SerDe field.
- */
- public static String getTableName(CommonTree node) {
- while (node.getToken().getType() != HiveParser.TOK_COLREF) {
- if (node.getChildCount() != 2) return null;
- node = (CommonTree) node.getChild(0);
- }
- return node.getChild(0).getText();
- }
@SuppressWarnings("nls")
- private OperatorInfoList genFilterPlan(String dest, QB qb,
- OperatorInfoList input) throws SemanticException {
-
- // We can assert here that the input list is of size one
- if (input.size() != 1) {
- throw new SemanticException("Filter has more than one inputs");
- }
+ private OperatorInfo genFilterPlan(String dest, QB qb,
+ OperatorInfo input) throws SemanticException {
CommonTree whereExpr = qb.getParseInfo().getWhrForClause(dest);
- OperatorInfoList output = (OperatorInfoList)input.clone();
- output.get(0).setOp(
+ OperatorInfo output = (OperatorInfo)input.clone();
+ output.setOp(
OperatorFactory.getAndMakeChild(
new filterDesc(genExprNodeDesc((CommonTree)whereExpr.getChild(0),
qb.getParseInfo().getAlias(),
- input.get(0).getRowResolver())),
- new RowSchema(output.get(0).getRowResolver().getColumnInfos()),
- input.get(0).getOp()
+ input.getRowResolver())),
+ new RowSchema(output.getRowResolver().getColumnInfos()),
+ input.getOp()
)
);
- LOG.debug("Created Filter Plan for " + qb.getId() + ":" + dest + " row schema: " + output.get(0).getRowResolver().toString());
+ LOG.debug("Created Filter Plan for " + qb.getId() + ":" + dest + " row schema: " + output.getRowResolver().toString());
return output;
}
@SuppressWarnings("nls")
private void genColList(String alias, CommonTree sel,
ArrayList<exprNodeDesc> col_list, RowResolver input, Integer pos,
- RowResolver output, String colAlias) throws SemanticException {
+ RowResolver output) throws SemanticException {
// TODO: Have to put in the support for AS clause
- String tabName = ((CommonTree) sel.getChild(0)).getToken().getText();
- ArrayList<TypeInfo> fieldTypeList = new ArrayList<TypeInfo>();
- ArrayList<String> fieldList = new ArrayList<String>();
- if (sel.getToken().getType() == HiveParser.TOK_ALLCOLREF) {
- // This is the tab.* case
- // In this case add all the columns to the fieldList
- // from the input schema
- for(ColumnInfo colInfo: input.getColumnInfos()) {
- String name = colInfo.getInternalName();
- String [] tmp = input.reverseLookup(name);
- fieldList.add(name);
- fieldTypeList.add(colInfo.getType());
- output.put(alias, tmp[1], new ColumnInfo(pos.toString(), colInfo.getType(),
- colInfo.getIsVirtual()));
- pos = Integer.valueOf(pos.intValue() + 1);
- }
- } else {
- // For now only allow columns of the form tab.col
- if (sel.getChildCount() == 1) {
- throw new SemanticException(ErrorMsg.NO_TABLE_ALIAS.getMsg(sel.getChild(0)));
- }
-
- // Lookup the name from the input
- ColumnInfo colInfo = input.get(tabName, sel.getChild(1).getText());
-
- // TODO: Hack it up for now: Later we have to pass the QB in order to check for the
- // table alias instead of relying on input.hasTableAlias
- if (colInfo == null && input.getIsExprResolver()) {
- throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(sel));
- } else if (!input.hasTableAlias(tabName)) {
- throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(sel.getChild(0)));
- } else if (colInfo == null) {
- throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(sel.getChild(1)));
- }
-
- // Add to the field list
- fieldList.add(colInfo.getInternalName());
- fieldTypeList.add(colInfo.getType());
- // Add to the output
- if (!StringUtils.isEmpty(alias) &&
- (output.get(alias, colAlias) != null)) {
- throw new SemanticException(ErrorMsg.AMBIGOUS_COLUMN.getMsg(sel.getChild(1)));
- }
- output.put(alias, colAlias,
- new ColumnInfo(pos.toString(), colInfo.getType(), colInfo.getIsVirtual()));
- pos = Integer.valueOf(pos.intValue() + 1);
- }
-
- // Generate the corresponding expressions
- for (int i=0; i<fieldList.size(); i++) {
- // Get the first expression
- exprNodeColumnDesc expr = new exprNodeColumnDesc(fieldTypeList.get(i), fieldList.get(i));
+ // This is the tab.* case
+ // In this case add all the columns to the fieldList
+ // from the input schema
+ for(ColumnInfo colInfo: input.getColumnInfos()) {
+ String name = colInfo.getInternalName();
+ String [] tmp = input.reverseLookup(name);
+ exprNodeColumnDesc expr = new exprNodeColumnDesc(colInfo.getType(), name);
col_list.add(expr);
+ output.put(alias, tmp[1], new ColumnInfo(pos.toString(), colInfo.getType()));
+ pos = Integer.valueOf(pos.intValue() + 1);
}
}
@SuppressWarnings("nls")
- private OperatorInfoList genScriptPlan(CommonTree trfm, QB qb,
- OperatorInfoList input) throws SemanticException {
+ private OperatorInfo genScriptPlan(CommonTree trfm, QB qb,
+ OperatorInfo input) throws SemanticException {
- // We can assert here that the input list is of size one
- if (input.size() != 1) {
- throw new SemanticException("Select has more than one inputs");
- }
-
- OperatorInfoList output = (OperatorInfoList) input.clone();
+ OperatorInfo output = (OperatorInfo)input.clone();
// Change the rws in this case
CommonTree collist = (CommonTree) trfm.getChild(1);
@@ -791,11 +724,11 @@
qb.getParseInfo().getAlias(),
((CommonTree)collist.getChild(i)).getText(),
new ColumnInfo(((CommonTree)collist.getChild(i)).getText(),
- String.class, false) // Everything is a string right now
+ String.class) // Everything is a string right now
);
}
- output.get(0)
+ output
.setOp(OperatorFactory
.getAndMakeChild(
new scriptDesc(
@@ -803,9 +736,9 @@
PlanUtils.getDefaultTableDesc(Integer.toString(Utilities.tabCode), sb.toString()),
PlanUtils.getDefaultTableDesc(Integer.toString(Utilities.tabCode), "")),
new RowSchema(
- out_rwsch.getColumnInfos()), input.get(0).getOp()));
+ out_rwsch.getColumnInfos()), input.getOp()));
- output.get(0).setRowResolver(out_rwsch);
+ output.setRowResolver(out_rwsch);
return output;
}
@@ -861,19 +794,15 @@
}
@SuppressWarnings("nls")
- private OperatorInfoList genSelectPlan(String dest, QB qb,
- OperatorInfoList input) throws SemanticException {
-
- // We can assert here that the input list is of size one
- if (input.size() != 1) {
- throw new SemanticException("Select has more than one inputs");
- }
+ private OperatorInfo genSelectPlan(String dest, QB qb,
+ OperatorInfo input) throws SemanticException {
CommonTree selExprList = qb.getParseInfo().getSelForClause(dest);
ArrayList<exprNodeDesc> col_list = new ArrayList<exprNodeDesc>();
RowResolver out_rwsch = new RowResolver();
CommonTree trfm = null;
+ String alias = qb.getParseInfo().getAlias();
Integer pos = Integer.valueOf(0);
// Iterate over the selects
@@ -884,10 +813,9 @@
String colAlias = getColAlias(selExpr, "_C" + i);
CommonTree sel = (CommonTree)selExpr.getChild(0);
- if (sel.getToken().getType() == HiveParser.TOK_COLREF ||
- sel.getToken().getType() == HiveParser.TOK_ALLCOLREF) {
+ if (sel.getToken().getType() == HiveParser.TOK_ALLCOLREF) {
genColList(qb.getParseInfo().getAlias(), sel, col_list,
- input.get(0).getRowResolver(), pos, out_rwsch, colAlias);
+ input.getRowResolver(), pos, out_rwsch);
} else if (sel.getToken().getType() == HiveParser.TOK_TRANSFORM) {
if (i > 0) {
throw new SemanticException(ErrorMsg.INVALID_TRANSFORM.getMsg(sel));
@@ -896,31 +824,37 @@
CommonTree cols = (CommonTree) trfm.getChild(0);
for (int j = 0; j < cols.getChildCount(); ++j) {
CommonTree expr = (CommonTree) cols.getChild(j);
- if (expr.getToken().getType() == HiveParser.TOK_COLREF ||
- expr.getToken().getType() == HiveParser.TOK_ALLCOLREF) {
- genColList(qb.getParseInfo().getAlias(), expr,
- col_list, input.get(0).getRowResolver(),
- pos, out_rwsch,
- expr.getChild(1).getText());
+ if (expr.getToken().getType() == HiveParser.TOK_ALLCOLREF) {
+ genColList(alias, expr,
+ col_list, input.getRowResolver(),
+ pos, out_rwsch);
} else {
- exprNodeDesc exp = genExprNodeDesc(expr, qb.getParseInfo()
- .getAlias(), input.get(0).getRowResolver());
+ exprNodeDesc exp = genExprNodeDesc(expr, alias, input.getRowResolver());
col_list.add(exp);
- out_rwsch.put(qb.getParseInfo().getAlias(), expr.getText(),
+ if (!StringUtils.isEmpty(alias) &&
+ (out_rwsch.get(alias, colAlias) != null)) {
+ throw new SemanticException(ErrorMsg.AMBIGOUS_COLUMN.getMsg(expr.getChild(1)));
+ }
+
+ out_rwsch.put(alias, expr.getText(),
new ColumnInfo((Integer.valueOf(pos)).toString(),
- String.class, false)); // Everything is a string right now
+ exp.getTypeInfo())); // Everything is a string right now
}
}
} else {
// Case when this is an expression
exprNodeDesc exp = genExprNodeDesc(sel, qb.getParseInfo()
- .getAlias(), input.get(0).getRowResolver());
+ .getAlias(), input.getRowResolver());
col_list.add(exp);
+ if (!StringUtils.isEmpty(alias) &&
+ (out_rwsch.get(alias, colAlias) != null)) {
+ throw new SemanticException(ErrorMsg.AMBIGOUS_COLUMN.getMsg(sel.getChild(1)));
+ }
// Since the as clause is lacking we just use the text representation
// of the expression as the column name
- out_rwsch.put(qb.getParseInfo().getAlias(), colAlias,
+ out_rwsch.put(alias, colAlias,
new ColumnInfo((Integer.valueOf(pos)).toString(),
- String.class, false)); // Everything is a string right now
+ exp.getTypeInfo())); // Everything is a string right now
}
pos = Integer.valueOf(pos.intValue() + 1);
}
@@ -931,51 +865,23 @@
}
}
- OperatorInfoList output = (OperatorInfoList) input.clone();
- output.get(0).setOp(OperatorFactory.getAndMakeChild(
+ OperatorInfo output = (OperatorInfo) input.clone();
+ output.setOp(OperatorFactory.getAndMakeChild(
new selectDesc(col_list), new RowSchema(out_rwsch.getColumnInfos()),
- input.get(0).getOp()));
+ input.getOp()));
- output.get(0).setRowResolver(out_rwsch);
+ output.setRowResolver(out_rwsch);
if (trfm != null) {
output = genScriptPlan(trfm, qb, output);
}
LOG.debug("Created Select Plan for clause: " + dest + " row schema: "
- + output.get(0).getRowResolver().toString());
+ + output.getRowResolver().toString());
return output;
}
- private OperatorInfo genGroupByPlanSelectOperator(
- QBParseInfo parseInfo, String dest, OperatorInfo groupByOperatorInfo)
- throws SemanticException {
-
- RowResolver groupByOutputRowResolver = groupByOperatorInfo.getRowResolver();
- RowResolver selectOutputRowResolver = new RowResolver();
- ArrayList<exprNodeDesc> selectCols = new ArrayList<exprNodeDesc>();
- CommonTree selectExpr = parseInfo.getSelForClause(dest);
- for (int i = 0; i < selectExpr.getChildCount(); ++i) {
- CommonTree sel = (CommonTree) selectExpr.getChild(i).getChild(0);
-
- // We need to recurse into the expression until we hit a UDAF or keys,
- // which are both in groupByOutputToColumns.
- exprNodeDesc exp = genExprNodeDesc(sel, parseInfo.getAlias(),
- groupByOutputRowResolver);
-
- selectCols.add(exp);
-
- selectOutputRowResolver.put(parseInfo.getAlias(), sel.getText(),
- new ColumnInfo((Integer.valueOf(i)).toString(),
- String.class, false)); // Everything is a class right now
- }
-
- return new OperatorInfo(OperatorFactory.getAndMakeChild(new selectDesc(
- selectCols), new RowSchema(selectOutputRowResolver.getColumnInfos()),
- groupByOperatorInfo.getOp()), selectOutputRowResolver);
- }
-
@SuppressWarnings("nls")
private OperatorInfo genGroupByPlanGroupByOperator(
QBParseInfo parseInfo, String dest, OperatorInfo reduceSinkOperatorInfo,
@@ -999,8 +905,7 @@
groupByKeys.add(new exprNodeColumnDesc(exprInfo.getType(), exprInfo.getInternalName()));
String field = (Integer.valueOf(i)).toString();
groupByOutputRowResolver.put("",grpbyExpr.toStringTree(),
- new ColumnInfo(field, exprInfo.getType(),
- exprInfo.getIsVirtual()));
+ new ColumnInfo(field, exprInfo.getType()));
}
// For each aggregation
HashMap<String, CommonTree> aggregationTrees = parseInfo
@@ -1037,7 +942,7 @@
value.getToken().getType() == HiveParser.TOK_FUNCTIONDI));
groupByOutputRowResolver.put("",value.toStringTree(),
new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(),
- String.class, false)); // Everything is a string right now
+ String.class)); // Everything is a string right now
}
return new OperatorInfo(
@@ -1071,7 +976,7 @@
groupByKeys.add(new exprNodeColumnDesc(exprInfo.getType(), exprInfo.getInternalName()));
String field = (Integer.valueOf(i)).toString();
outputRS.put("", text,
- new ColumnInfo(field, exprInfo.getType(), exprInfo.getIsVirtual()));
+ new ColumnInfo(field, exprInfo.getType()));
}
// For each aggregation
@@ -1109,7 +1014,7 @@
value.getToken().getType() == HiveParser.TOK_FUNCTIONDI));
outputRS.put("",value.toStringTree(),
new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(),
- String.class, false)); // Everything is a string right now
+ String.class)); // Everything is a string right now
}
return new OperatorInfo(
@@ -1139,7 +1044,7 @@
if (reduceSinkOutputRowResolver.get("", text) == null) {
reduceSinkOutputRowResolver.put("", text,
new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(),
- String.class, false)); // Everything is a string right now
+ String.class)); // Everything is a string right now
} else {
throw new SemanticException(ErrorMsg.DUPLICATE_GROUPBY_KEY.getMsg(grpbyExpr));
}
@@ -1156,7 +1061,7 @@
reduceKeys.add(genExprNodeDesc(parameter, parseInfo.getAlias(), reduceSinkInputRowResolver));
reduceSinkOutputRowResolver.put("", text,
new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(),
- String.class, false)); // Everything is a string right now
+ String.class)); // Everything is a string right now
}
}
}
@@ -1175,13 +1080,14 @@
reduceValues.add(genExprNodeDesc(parameter, parseInfo.getAlias(), reduceSinkInputRowResolver));
reduceSinkOutputRowResolver.put("", text,
new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(),
- String.class, false)); // Everything is a string right now
+ String.class)); // Everything is a string right now
}
}
}
return new OperatorInfo(
- OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, numPartitionFields),
+ OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, -1, numPartitionFields,
+ -1, false),
new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()),
inputOperatorInfo.getOp()),
reduceSinkOutputRowResolver
@@ -1205,7 +1111,7 @@
assert (outputRS.get("", text) == null);
outputRS.put("", text,
new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(),
- String.class, false));
+ String.class));
}
else {
// dummy key
@@ -1228,8 +1134,8 @@
if (outputRS.get(key, field) == null)
{
reduceValues.add(new exprNodeColumnDesc(valueInfo.getType(), valueInfo.getInternalName()));
- outputRS.put(key, field, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(), valueInfo.getType(),
- valueInfo.getIsVirtual()));
+ outputRS.put(key, field, new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(),
+ valueInfo.getType()));
}
}
}
@@ -1246,7 +1152,7 @@
reduceValues.add(grpbyExprNode);
outputRS.put("", text,
new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(),
- grpbyExprNode.getTypeInfo(), false));
+ grpbyExprNode.getTypeInfo()));
}
}
@@ -1263,14 +1169,15 @@
reduceValues.add(pNode);
outputRS.put("", text,
new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(),
- pNode.getTypeInfo(), false));
+ pNode.getTypeInfo()));
}
}
}
}
return new OperatorInfo(
- OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, distinctText == null ? -1 : 1),
+ OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues,
+ -1, distinctText == null ? -1 : 1, -1, false),
new RowSchema(outputRS.getColumnInfos()), input.getOp()),
outputRS);
}
@@ -1305,7 +1212,7 @@
reduceKeys.add(new exprNodeColumnDesc(TypeInfoFactory.getPrimitiveTypeInfo(String.class), field));
reduceSinkOutputRowResolver2.put("", grpbyExpr.toStringTree(),
new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + field,
- String.class, false)); // Everything is a string right now
+ String.class)); // Everything is a string right now
}
// Get partial aggregation results and store in reduceValues
ArrayList<exprNodeDesc> reduceValues = new ArrayList<exprNodeDesc>();
@@ -1314,15 +1221,16 @@
.getAggregationExprsForClause(dest);
for (Map.Entry<String, CommonTree> entry : aggregationTrees.entrySet()) {
reduceValues.add(new exprNodeColumnDesc(TypeInfoFactory.getPrimitiveTypeInfo(String.class),
- (Integer.valueOf(inputField)).toString()));
+ (Integer.valueOf(inputField)).toString()));
inputField++;
reduceSinkOutputRowResolver2.put("", ((CommonTree)entry.getValue()).toStringTree(),
new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + (Integer.valueOf(reduceValues.size()-1)).toString(),
- String.class, false)); // Everything is a string right now
+ String.class)); // Everything is a string right now
}
return new OperatorInfo(
- OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, numPartitionFields),
+ OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, -1,
+ numPartitionFields, -1, true),
new RowSchema(reduceSinkOutputRowResolver2.getColumnInfos()),
groupByOperatorInfo.getOp()),
reduceSinkOutputRowResolver2
@@ -1351,8 +1259,7 @@
groupByKeys.add(new exprNodeColumnDesc(exprInfo.getType(), expression));
String field = (Integer.valueOf(i)).toString();
groupByOutputRowResolver2.put("",grpbyExpr.toStringTree(),
- new ColumnInfo(field, exprInfo.getType(),
- exprInfo.getIsVirtual()));
+ new ColumnInfo(field, exprInfo.getType()));
}
HashMap<String, CommonTree> aggregationTrees = parseInfo
.getAggregationExprsForClause(dest);
@@ -1374,8 +1281,7 @@
aggregations.add(new aggregationDesc(aggClass, aggParameters, false));
groupByOutputRowResolver2.put("", value.toStringTree(),
new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() - 1).toString(),
- paraExprInfo.getType(),
- paraExprInfo.getIsVirtual())); // Everything is a string right now
+ paraExprInfo.getType())); // Everything is a string right now
}
return new OperatorInfo(
@@ -1402,14 +1308,10 @@
* @throws SemanticException
*/
@SuppressWarnings({ "unused", "nls" })
- private OperatorInfoList genGroupByPlan1MR(String dest, QB qb,
- OperatorInfoList inputList) throws SemanticException {
+ private OperatorInfo genGroupByPlan1MR(String dest, QB qb,
+ OperatorInfo input) throws SemanticException {
- // We can assert here that the input list is of size one
- if (inputList.size() != 1) {
- throw new SemanticException("Select has more than one inputs");
- }
- OperatorInfo inputOperatorInfo = inputList.get(0);
+ OperatorInfo inputOperatorInfo = input;
QBParseInfo parseInfo = qb.getParseInfo();
// ////// 1. Generate ReduceSinkOperator
@@ -1422,14 +1324,7 @@
OperatorInfo groupByOperatorInfo = genGroupByPlanGroupByOperator(parseInfo,
dest, reduceSinkOperatorInfo, groupByDesc.Mode.COMPLETE);
- // ////// 3. Generate SelectOperator
- OperatorInfo selectOperatorInfo = genGroupByPlanSelectOperator(parseInfo,
- dest, groupByOperatorInfo);
-
- // ////// 4. Create output
- OperatorInfoList output = new OperatorInfoList();
- output.add(selectOperatorInfo);
- return output;
+ return groupByOperatorInfo;
}
/**
@@ -1450,14 +1345,10 @@
* @throws SemanticException
*/
@SuppressWarnings("nls")
- private OperatorInfoList genGroupByPlan2MR(String dest, QB qb,
- OperatorInfoList inputList) throws SemanticException {
+ private OperatorInfo genGroupByPlan2MR(String dest, QB qb,
+ OperatorInfo input) throws SemanticException {
- // We can assert here that the input list is of size one
- if (inputList.size() != 1) {
- throw new SemanticException("Select has more than one inputs");
- }
- OperatorInfo inputOperatorInfo = inputList.get(0);
+ OperatorInfo inputOperatorInfo = input;
QBParseInfo parseInfo = qb.getParseInfo();
// ////// 1. Generate ReduceSinkOperator
@@ -1483,14 +1374,7 @@
OperatorInfo groupByOperatorInfo2 = genGroupByPlanGroupByOperator2MR(
parseInfo, dest, reduceSinkOperatorInfo2);
- // ////// 5. Generate SelectOperator
- OperatorInfo selectOperatorInfo = genGroupByPlanSelectOperator(parseInfo,
- dest, groupByOperatorInfo2);
-
- // ////// 6. Create output
- OperatorInfoList output = new OperatorInfoList();
- output.add(selectOperatorInfo);
- return output;
+ return groupByOperatorInfo2;
}
/**
@@ -1500,14 +1384,10 @@
* shared by all groupbys.
*/
@SuppressWarnings("nls")
- private OperatorInfoList genGroupByPlan3MR(String dest, QB qb,
- OperatorInfoList inputList) throws SemanticException {
+ private OperatorInfo genGroupByPlan3MR(String dest, QB qb,
+ OperatorInfo input) throws SemanticException {
- // We can assert here that the input list is of size one
- if (inputList.size() != 1) {
- throw new SemanticException("Select has more than one inputs");
- }
- OperatorInfo inputOperatorInfo = inputList.get(0);
+ OperatorInfo inputOperatorInfo = input;
QBParseInfo parseInfo = qb.getParseInfo();
// ////// Generate GroupbyOperator
@@ -1523,19 +1403,12 @@
OperatorInfo groupByOperatorInfo2 = genGroupByPlanGroupByOperator2MR(
parseInfo, dest, reduceSinkOperatorInfo2);
- // ////// Generate SelectOperator
- OperatorInfo selectOperatorInfo = genGroupByPlanSelectOperator(parseInfo,
- dest, groupByOperatorInfo2);
-
- // ////// Create output
- OperatorInfoList output = new OperatorInfoList();
- output.add(selectOperatorInfo);
- return output;
+ return groupByOperatorInfo2;
}
@SuppressWarnings("nls")
- private OperatorInfoList genConversionOps(String dest, QB qb,
- OperatorInfoList input) throws SemanticException {
+ private OperatorInfo genConversionOps(String dest, QB qb,
+ OperatorInfo input) throws SemanticException {
Integer dest_type = qb.getMetaData().getDestTypeForAlias(dest);
Table dest_tab = null;
@@ -1556,50 +1429,12 @@
}
}
- /*
- // We have the table object here - go over the row resolver
- // and check all the types are the same
- // Vector<ColumnInfo> srcOpns = input.get(0).getRowResolver().getColumnInfos();
-
- Vector<ColumnInfo> insOpns = new Vector<ColumnInfo>();
- try {
- StructObjectInspector rowObjectInspector = (StructObjectInspector)dest_tab.getDeserializer().getObjectInspector();
- List<? extends StructField> fields = rowObjectInspector.getAllStructFieldRefs();
- for (int i=0; i<fields.size(); i++) {
- insOpns.add(new ColumnInfo(
- fields.get(i).getFieldName(),
- TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i).getFieldObjectInspector()),
- false));
- }
- } catch (SerDeException e) {
- throw new RuntimeException(e);
- }
- if (insOpns.size() != srcOpns.size()) {
- // TODO: change this to error message later
- throw new SemanticException("Number of columns in the select expression does not " +
- "match the number of columns in table " + dest_tab.getName() +
- " expected " + insOpns.size() + " but got " + srcOpns.size());
- }
-
- boolean needSelOperator = false;
- for(int i=0; i < insOpns.size(); ++i) {
- if (!insOpns.get(i).getType().equals(srcOpns.get(i).getType())) {
- needSelOperator = true;
- throw new SemanticException("insert and select column type do not match for table " + dest_tab.getName());
- }
- }
- */
return input;
}
@SuppressWarnings("nls")
- private OperatorInfoList genFileSinkPlan(String dest, QB qb,
- OperatorInfoList input) throws SemanticException {
-
- // We can assert here that the input list is of size one
- if (input.size() != 1) {
- throw new SemanticException("Select has more than one inputs");
- }
+ private OperatorInfo genFileSinkPlan(String dest, QB qb,
+ OperatorInfo input) throws SemanticException {
// Generate the destination file
String queryTmpdir = this.scratchDir + File.separator + this.randomid + '.' + this.pathid + '.' + dest ;
@@ -1638,96 +1473,145 @@
case QBMetaData.DEST_DFS_FILE: {
table_desc = Utilities.defaultTd;
dest_path = qb.getMetaData().getDestFileForAlias(dest);
+ String cols = new String();
+ RowResolver inputRR = input.getRowResolver();
+ Vector<ColumnInfo> colInfos = inputRR.getColumnInfos();
+
+ boolean first = true;
+ for (ColumnInfo colInfo:colInfos) {
+ String[] nm = inputRR.reverseLookup(colInfo.getInternalName());
+ if (!first)
+ cols = cols.concat(",");
+
+ first = false;
+ if (nm[0] == null)
+ cols = cols.concat(nm[1]);
+ else
+ cols = cols.concat(nm[0] + "." + nm[1]);
+ }
+
this.loadFileWork.add(new loadFileDesc(queryTmpdir, dest_path,
- (dest_type.intValue() == QBMetaData.DEST_DFS_FILE)));
- break;
+ (dest_type.intValue() == QBMetaData.DEST_DFS_FILE), cols));
+ break;
}
default:
throw new SemanticException("Unknown destination type: " + dest_type);
}
- OperatorInfoList output = (OperatorInfoList)input.clone();
- output.get(0).setOp(
+ OperatorInfo output = (OperatorInfo)input.clone();
+ output.setOp(
OperatorFactory.getAndMakeChild(
new fileSinkDesc(queryTmpdir, table_desc),
- new RowSchema(output.get(0).getRowResolver().getColumnInfos()),
- input.get(0).getOp()
+ new RowSchema(output.getRowResolver().getColumnInfos()), input.getOp()
)
);
LOG.debug("Created FileSink Plan for clause: " + dest + "dest_path: "
+ dest_path + " row schema: "
- + output.get(0).getRowResolver().toString());
+ + output.getRowResolver().toString());
return output;
}
@SuppressWarnings("nls")
- private OperatorInfoList genReduceSinkPlan(String dest, QB qb,
- OperatorInfoList input) throws SemanticException {
+ private OperatorInfo genLimitPlan(String dest, QB qb, OperatorInfo input, int limit) throws SemanticException {
+ // A map-only job can be optimized - instead of converting it to a map-reduce job, we can have another map
+ // job to do the same to avoid the cost of sorting in the map-reduce phase. A better approach would be to
+ // write into a local file and then have a map-only job.
+ // Add the limit operator to get the value fields
- // We can assert here that the input list is of size one
- if (input.size() != 1) {
- throw new SemanticException("Select has more than one inputs");
- }
+ OperatorInfo limitMap = (OperatorInfo)input.clone();
+ limitMap.setOp(
+ OperatorFactory.getAndMakeChild(
+ new limitDesc(limit), new RowSchema(limitMap.getRowResolver().getColumnInfos()),
+ input.getOp()
+ )
+ );
+
+ LOG.debug("Created LimitOperator Plan for clause: " + dest + " row schema: "
+ + limitMap.getRowResolver().toString());
+
+ return limitMap;
+ }
+
+ @SuppressWarnings("nls")
+ private OperatorInfo genLimitMapRedPlan(String dest, QB qb, OperatorInfo input, int limit, boolean isOuterQuery) throws SemanticException {
+ // A map-only job can be optimized - instead of converting it to a map-reduce job, we can have another map
+ // job to do the same to avoid the cost of sorting in the map-reduce phase. A better approach would be to
+ // write into a local file and then have a map-only job.
+ // Add the limit operator to get the value fields
+ OperatorInfo curr = genLimitPlan(dest, qb, input, limit);
+
+ if (isOuterQuery)
+ return curr;
+
+ // Create a reduceSink operator followed by another limit
+ curr = genReduceSinkPlan(dest, qb, curr, 1);
+ return genLimitPlan(dest, qb, curr, limit);
+ }
+
+ @SuppressWarnings("nls")
+ private OperatorInfo genReduceSinkPlan(String dest, QB qb,
+ OperatorInfo input, int numReducers) throws SemanticException {
// First generate the expression for the key
// The cluster by clause has the aliases for the keys
- CommonTree clby = qb.getParseInfo().getClusterByForClause(dest);
ArrayList<exprNodeDesc> keyCols = new ArrayList<exprNodeDesc>();
- int ccount = clby.getChildCount();
- for(int i=0; i<ccount; ++i) {
- CommonTree cl = (CommonTree)clby.getChild(i);
- ColumnInfo colInfo = input.get(0).getRowResolver().get(qb.getParseInfo().getAlias(),
- cl.getText());
- if (colInfo == null) {
- throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(cl));
- }
- keyCols.add(new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName()));
+ CommonTree clby = qb.getParseInfo().getClusterByForClause(dest);
+ if (clby != null) {
+ int ccount = clby.getChildCount();
+ for(int i=0; i<ccount; ++i) {
+ CommonTree cl = (CommonTree)clby.getChild(i);
+ ColumnInfo colInfo = input.getRowResolver().get(qb.getParseInfo().getAlias(),
+ cl.getText());
+ if (colInfo == null) {
+ throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(cl));
+ }
+
+ keyCols.add(new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName()));
+ }
}
ArrayList<exprNodeDesc> valueCols = new ArrayList<exprNodeDesc>();
// For the generation of the values expression just get the inputs
// signature and generate field expressions for those
- for(ColumnInfo colInfo: input.get(0).getRowResolver().getColumnInfos()) {
+ for(ColumnInfo colInfo: input.getRowResolver().getColumnInfos()) {
valueCols.add(new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName()));
}
- OperatorInfoList interim = (OperatorInfoList)input.clone();
- interim.get(0).setOp(
+ OperatorInfo interim = (OperatorInfo)input.clone();
+ interim.setOp(
OperatorFactory.getAndMakeChild(
- PlanUtils.getReduceSinkDesc(keyCols, valueCols, keyCols.size()),
- new RowSchema(interim.get(0).getRowResolver().getColumnInfos()),
- input.get(0).getOp()
+ PlanUtils.getReduceSinkDesc(keyCols, valueCols, -1, keyCols.size(), numReducers, false),
+ new RowSchema(interim.getRowResolver().getColumnInfos()),
+ input.getOp()
)
);
// Add the extract operator to get the value fields
RowResolver out_rwsch = new RowResolver();
- RowResolver interim_rwsch = interim.get(0).getRowResolver();
+ RowResolver interim_rwsch = interim.getRowResolver();
Integer pos = Integer.valueOf(0);
for(ColumnInfo colInfo: interim_rwsch.getColumnInfos()) {
String [] info = interim_rwsch.reverseLookup(colInfo.getInternalName());
out_rwsch.put(info[0], info[1],
- new ColumnInfo(pos.toString(), colInfo.getType(), colInfo.getIsVirtual()));
+ new ColumnInfo(pos.toString(), colInfo.getType()));
pos = Integer.valueOf(pos.intValue() + 1);
}
- OperatorInfoList output = (OperatorInfoList)interim.clone();
- output.get(0).setOp(
+ OperatorInfo output = (OperatorInfo)interim.clone();
+ output.setOp(
OperatorFactory.getAndMakeChild(
- new extractDesc(
- new exprNodeColumnDesc(String.class, Utilities.ReduceField.VALUE.toString())
- ),
+ new extractDesc(new exprNodeColumnDesc(String.class, Utilities.ReduceField.VALUE.toString())),
new RowSchema(out_rwsch.getColumnInfos()),
- interim.get(0).getOp()
+ interim.getOp()
)
);
- output.get(0).setRowResolver(out_rwsch);
+ output.setRowResolver(out_rwsch);
LOG.debug("Created ReduceSink Plan for clause: " + dest + " row schema: "
- + output.get(0).getRowResolver().toString());
+ + output.getRowResolver().toString());
return output;
}
@@ -1760,8 +1644,8 @@
ColumnInfo valueInfo = inputRS.get(key, field);
keyDesc.add(new exprNodeColumnDesc(valueInfo.getType(), valueInfo.getInternalName()));
if (outputRS.get(key, field) == null)
- outputRS.put(key, field, new ColumnInfo((Integer.valueOf(outputPos++)).toString(), valueInfo.getType(),
- valueInfo.getIsVirtual()));
+ outputRS.put(key, field, new ColumnInfo((Integer.valueOf(outputPos++)).toString(),
+ valueInfo.getType()));
}
}
@@ -1808,17 +1692,19 @@
outputRS.put(src, field,
new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." +
Integer.valueOf(reduceValues.size() - 1).toString(),
- valueInfo.getType(), valueInfo.getIsVirtual()));
+ valueInfo.getType()));
}
}
- return new OperatorInfo(OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(
- reduceKeys, reduceValues, joinTree.getNextTag(), reduceKeys.size()), new RowSchema(outputRS.getColumnInfos()),
+ return new OperatorInfo(
+ OperatorFactory.getAndMakeChild(
+ PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, joinTree.getNextTag(), reduceKeys.size(), -1, false),
+ new RowSchema(outputRS.getColumnInfos()),
child.getOp()), outputRS);
}
private OperatorInfo genJoinOperator(QB qb, QBJoinTree joinTree,
- HashMap<String, OperatorInfoList> map) throws SemanticException {
+ HashMap<String, OperatorInfo> map) throws SemanticException {
QBJoinTree leftChild = joinTree.getJoinSrc();
OperatorInfo joinSrcOp = null;
if (leftChild != null)
@@ -1831,7 +1717,7 @@
int pos = 0;
for (String src : joinTree.getBaseSrc()) {
if (src != null) {
- OperatorInfo srcOp = map.get(src).get(0);
+ OperatorInfo srcOp = map.get(src);
srcOps[pos] = genJoinReduceSinkChild(qb, joinTree, srcOp, src, pos);
pos++;
} else {
@@ -1881,13 +1767,11 @@
}
}
- private OperatorInfoList genJoinPlan(QB qb, HashMap<String, OperatorInfoList> map)
+ private OperatorInfo genJoinPlan(QB qb, HashMap<String, OperatorInfo> map)
throws SemanticException {
QBJoinTree joinTree = qb.getQbJoinTree();
OperatorInfo joinOp = genJoinOperator(qb, joinTree, map);
- OperatorInfoList output = new OperatorInfoList();
- output.add(joinOp);
- return output;
+ return joinOp;
}
private QBJoinTree genJoinTree(CommonTree joinParseTree)
@@ -2104,19 +1988,19 @@
}
@SuppressWarnings("nls")
- private OperatorInfoList genBodyPlan(QB qb, OperatorInfoList input)
+ private OperatorInfo genBodyPlan(QB qb, OperatorInfo input)
throws SemanticException {
QBParseInfo qbp = qb.getParseInfo();
- OperatorInfoList output = new OperatorInfoList();
TreeSet<String> ks = new TreeSet<String>();
ks.addAll(qbp.getClauseNames());
String distinctText = null;
CommonTree distn = null;
- OperatorInfoList opList = null;
+ OperatorInfo op = null;
boolean grpBy = false;
+ int numGrpBy = 0;
// In case of a multiple group bys, all of them should have the same distinct key
for (String dest : ks) {
@@ -2124,6 +2008,7 @@
if ((qbp.getAggregationExprsForClause(dest).size() != 0)
|| (getGroupByForClause(qbp, dest).size() > 0)) {
grpBy = true;
+ numGrpBy++;
// If there is a distinctFuncExp, add all parameters to the reduceKeys.
if (qbp.getDistinctFuncExprForClause(dest) != null) {
@@ -2142,63 +2027,71 @@
// In the first stage, copy the input and all the group by expressions
// and aggregate paramaters. This can be optimized in the future to only
- // evaluate expressions that occur frequently
- if (grpBy) {
+ // evaluate expressions that occur frequently. For a single groupby, no need to do so
+ if (grpBy && (numGrpBy > 1)) {
OperatorInfo reduceSinkOperatorInfo =
- genGroupByPlanReduceSinkOperator(qbp, input.get(0), distn, ks);
+ genGroupByPlanReduceSinkOperator(qbp, input, distn, ks);
// ////// 2. Generate GroupbyOperator
OperatorInfo forwardOperatorInfo = genGroupByPlanForwardOperator(qbp, reduceSinkOperatorInfo);
- opList = new OperatorInfoList();
- opList.add(forwardOperatorInfo);
+ op = forwardOperatorInfo;
}
// Go over all the destination tables
+ OperatorInfo curr = null;
for (String dest : ks) {
boolean groupByExpr = false;
if (qbp.getAggregationExprsForClause(dest).size() != 0
|| getGroupByForClause(qbp, dest).size() > 0)
groupByExpr = true;
- OperatorInfoList curr = input;
- if (groupByExpr)
- curr = opList;
+ curr = input;
+ if (groupByExpr && (numGrpBy > 1))
+ curr = op;
if (qbp.getWhrForClause(dest) != null) {
curr = genFilterPlan(dest, qb, curr);
}
if (qbp.getAggregationExprsForClause(dest).size() != 0
- || getGroupByForClause(qbp, dest).size() > 0)
- curr = genGroupByPlan3MR(dest, qb, curr);
- else
- curr = genSelectPlan(dest, qb, curr);
-
- if (qbp.getClusterByForClause(dest) != null) {
- curr = genReduceSinkPlan(dest, qb, curr);
- } else if (!qbp.getIsSubQ()) {
+ || getGroupByForClause(qbp, dest).size() > 0) {
+ if (numGrpBy > 1)
+ curr = genGroupByPlan3MR(dest, qb, curr);
+ else
+ curr = genGroupByPlan2MR(dest, qb, curr);
+ }
+
+ curr = genSelectPlan(dest, qb, curr);
+ Integer limit = qbp.getDestLimit(dest);
+
+ if (qbp.getIsSubQ()) {
+ if (qbp.getClusterByForClause(dest) != null)
+ curr = genReduceSinkPlan(dest, qb, curr, -1);
+ if (limit != null)
+ curr = genLimitMapRedPlan(dest, qb, curr, limit.intValue(), false);
+ }
+ else
+ {
curr = genConversionOps(dest, qb, curr);
+ // exact limit can be taken care of by the fetch operator
+ if (limit != null) {
+ curr = genLimitMapRedPlan(dest, qb, curr, limit.intValue(), true);
+ qb.getParseInfo().setOuterQueryLimit(limit.intValue());
+ }
curr = genFileSinkPlan(dest, qb, curr);
}
-
- output.addAll(curr);
}
LOG.debug("Created Body Plan for Query Block " + qb.getId());
- return output;
+ return curr;
}
@SuppressWarnings("nls")
- private OperatorInfoList genUnionPlan(String unionalias, String leftalias,
- OperatorInfoList left, String rightalias, OperatorInfoList right)
+ private OperatorInfo genUnionPlan(String unionalias, String leftalias,
+ OperatorInfo leftOp, String rightalias, OperatorInfo rightOp)
throws SemanticException {
- if (left.size() != 1) {
- throw new SemanticException("Select has more than one inputs");
- }
- OperatorInfo leftOp = left.get(0);
RowResolver leftRR = leftOp.getRowResolver();
- OperatorInfo rightOp = right.get(0);
RowResolver rightRR = rightOp.getRowResolver();
HashMap<String, ColumnInfo> leftmap = leftRR.getFieldMap(leftalias);
HashMap<String, ColumnInfo> rightmap = rightRR.getFieldMap(rightalias);
@@ -2236,8 +2129,7 @@
rightOp.getOp().setChildOperators(child);
leftOp.getOp().setChildOperators(child);
// create operator info list to return
- OperatorInfoList unionout = new OperatorInfoList();
- unionout.add(new OperatorInfo(unionforward, unionoutRR));
+ OperatorInfo unionout = new OperatorInfo(unionforward, unionoutRR);
return unionout;
}
@@ -2245,7 +2137,7 @@
// ((default_sample_hashfn(cols) & Integer.MAX_VALUE) % denominator) == numerator
exprNodeDesc numeratorExpr = new exprNodeConstantDesc(
TypeInfoFactory.getPrimitiveTypeInfo(Integer.class),
- Integer.valueOf(ts.getNumerator()));
+ Integer.valueOf(ts.getNumerator() - 1));
exprNodeDesc denominatorExpr = new exprNodeConstantDesc(
TypeInfoFactory.getPrimitiveTypeInfo(Integer.class),
@@ -2277,7 +2169,7 @@
}
@SuppressWarnings("nls")
- private OperatorInfoList genTablePlan(String alias, QB qb)
+ private OperatorInfo genTablePlan(String alias, QB qb)
throws SemanticException {
Table tab = qb.getMetaData().getSrcForAlias(alias);
@@ -2289,7 +2181,7 @@
for (int i=0; i<fields.size(); i++) {
rwsch.put(alias, fields.get(i).getFieldName(),
new ColumnInfo(fields.get(i).getFieldName(),
- TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i).getFieldObjectInspector()), false));
+ TypeInfoUtils.getTypeInfoFromObjectInspector(fields.get(i).getFieldObjectInspector())));
}
} catch (SerDeException e) {
throw new RuntimeException(e);
@@ -2299,7 +2191,7 @@
for(FieldSchema part_col: tab.getPartCols()) {
LOG.trace("Adding partition col: " + part_col);
// TODO: use the right type by calling part_col.getType() instead of String.class
- rwsch.put(alias, part_col.getName(), new ColumnInfo(part_col.getName(), String.class, true));
+ rwsch.put(alias, part_col.getName(), new ColumnInfo(part_col.getName(), String.class));
}
// Create the root of the operator tree
@@ -2340,9 +2232,10 @@
// check if the sample columns are the same as the table bucket columns
// and if they are, create a new array of column names which is in the
- // same order as tabBucketCols
+ // same order as tabBucketCols.
+ // if sample cols is not specified then default is bucket cols
boolean colsEqual = true;
- if (sampleCols.size() != tabBucketCols.size()) {
+ if ( (sampleCols.size() != tabBucketCols.size()) && (sampleCols.size() != 0) ) {
colsEqual = false;
}
for (int i = 0; i < sampleCols.size() && colsEqual; i++) {
@@ -2363,35 +2256,33 @@
// check if input pruning is enough
if ((sampleCols == null || sampleCols.size() == 0 || colsEqual)
&& (num == den || den <= numBuckets && numBuckets % den == 0)) {
- // input pruning is enough; no need for filter
- LOG.info("No need for sample filter");
- }
+ // input pruning is enough; no need for filter
+ LOG.info("No need for sample filter");
+ }
else {
- // need to add filter
- // create tableOp to be filterDesc and set as child to 'top'
- LOG.info("Need sample filter");
+ // need to add filter
+ // create tableOp to be filterDesc and set as child to 'top'
+ LOG.info("Need sample filter");
exprNodeDesc samplePredicate = genSamplePredicate(ts);
tableOp = OperatorFactory.getAndMakeChild(
new filterDesc(samplePredicate),
top);
}
}
- OperatorInfoList output = new OperatorInfoList();
- output.add(new OperatorInfo(tableOp, rwsch));
+ OperatorInfo output = new OperatorInfo(tableOp, rwsch);
LOG.debug("Created Table Plan for " + alias + " " + tableOp.toString());
return output;
}
- private OperatorInfoList genPlan(QBExpr qbexpr) throws SemanticException {
+ private OperatorInfo genPlan(QBExpr qbexpr) throws SemanticException {
if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) {
- OperatorInfoList oplist = genPlan(qbexpr.getQB());
- return oplist;
+ return genPlan(qbexpr.getQB());
}
if (qbexpr.getOpcode() == QBExpr.Opcode.UNION) {
- OperatorInfoList qbexpr1Ops = genPlan(qbexpr.getQBExpr1());
- OperatorInfoList qbexpr2Ops = genPlan(qbexpr.getQBExpr2());
+ OperatorInfo qbexpr1Ops = genPlan(qbexpr.getQBExpr1());
+ OperatorInfo qbexpr2Ops = genPlan(qbexpr.getQBExpr2());
return genUnionPlan(qbexpr.getAlias(), qbexpr.getQBExpr1().getAlias(),
qbexpr1Ops, qbexpr.getQBExpr2().getAlias(), qbexpr2Ops);
@@ -2400,10 +2291,10 @@
}
@SuppressWarnings("nls")
- private OperatorInfoList genPlan(QB qb) throws SemanticException {
+ private OperatorInfo genPlan(QB qb) throws SemanticException {
// First generate all the opInfos for the elements in the from clause
- HashMap<String, OperatorInfoList> aliasToOpInfo = new HashMap<String, OperatorInfoList>();
+ HashMap<String, OperatorInfo> aliasToOpInfo = new HashMap<String, OperatorInfo>();
// Recurse over the subqueries to fill the subquery part of the plan
for (String alias : qb.getSubqAliases()) {
@@ -2417,7 +2308,7 @@
aliasToOpInfo.put(alias, genTablePlan(alias, qb));
}
- OperatorInfoList srcOpInfoList = null;
+ OperatorInfo srcOpInfo = null;
// process join
if (qb.getParseInfo().getJoinExpr() != null) {
@@ -2425,26 +2316,29 @@
QBJoinTree joinTree = genJoinTree(joinExpr);
qb.setQbJoinTree(joinTree);
mergeJoinTree(qb);
- srcOpInfoList = genJoinPlan(qb, aliasToOpInfo);
+ srcOpInfo = genJoinPlan(qb, aliasToOpInfo);
}
else
// Now if there are more than 1 sources then we have a join case
// later we can extend this to the union all case as well
- srcOpInfoList = aliasToOpInfo.values()
- .iterator().next();
+ srcOpInfo = aliasToOpInfo.values().iterator().next();
- OperatorInfoList bodyOpInfoList = genBodyPlan(qb, srcOpInfoList);
+ OperatorInfo bodyOpInfo = genBodyPlan(qb, srcOpInfo);
LOG.debug("Created Plan for Query Block " + qb.getId());
- return bodyOpInfoList;
+ // is it a top level QB, and can it be optimized ? For eg: select * from T does not need a map-reduce job
+ QBParseInfo qbp = qb.getParseInfo();
+ qbp.setCanOptTopQ(qb.isSelectStarQuery());
+
+ return bodyOpInfo;
}
- private Operator<? extends Serializable> getReducer(Operator<? extends Serializable> top) {
+ private Operator<? extends Serializable> getReduceSink(Operator<? extends Serializable> top) {
if (top.getClass() == ReduceSinkOperator.class) {
// Get the operator following the reduce sink
assert (top.getChildOperators().size() == 1);
- return top.getChildOperators().get(0);
+ return top;
}
List<Operator<? extends Serializable>> childOps = top.getChildOperators();
@@ -2453,7 +2347,7 @@
}
for (int i = 0; i < childOps.size(); ++i) {
- Operator<? extends Serializable> reducer = getReducer(childOps.get(i));
+ Operator<? extends Serializable> reducer = getReduceSink(childOps.get(i));
if (reducer != null) {
return reducer;
}
@@ -2463,13 +2357,41 @@
}
@SuppressWarnings("nls")
- private void genMapRedTasks() throws SemanticException {
+ private void genMapRedTasks(QB qb) throws SemanticException {
+ fetchWork fetch = null;
+ moveWork mv = null;
+ Task<? extends Serializable> mvTask = null;
+ Task<? extends Serializable> fetchTask = null;
+
+ if (qb.getParseInfo().getCanOptTopQ()) {
+ Iterator<Map.Entry<String, Table>> iter = qb.getMetaData().getAliasToTable().entrySet().iterator();
+ Table tab = ((Map.Entry<String, Table>)iter.next()).getValue();
+ fetch = new fetchWork(tab.getPath(), tab.getDeserializer().getClass(),
+ tab.getInputFormatClass(), tab.getSchema(), qb.getParseInfo().getOuterQueryLimit());
+
+ fetchTask = TaskFactory.get(fetch, this.conf);
+ setFetchTask(fetchTask);
+ return;
+ }
- // First we generate the move work as this needs to be made dependent on all
- // the tasks
- // that have a file sink operation
- moveWork mv = new moveWork(loadTableWork, loadFileWork);
- Task<? extends Serializable> mvTask = TaskFactory.get(mv, this.conf);
+ // In case of a select, use a fetch task instead of a move task
+ if (qb.getIsQuery()) {
+ if ((!loadTableWork.isEmpty()) || (loadFileWork.size() != 1))
+ throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg());
+ String cols = loadFileWork.get(0).getColumns();
+ fetch = new fetchWork(new Path(loadFileWork.get(0).getSourceDir()),
+ MetadataTypedColumnsetSerDe.class, TextInputFormat.class,
+ Utilities.makeProperties("columns", cols), qb.getParseInfo().getOuterQueryLimit());
+
+ fetchTask = TaskFactory.get(fetch, this.conf);
+ setFetchTask(fetchTask);
+ }
+ else {
+ // First we generate the move work as this needs to be made dependent on all
+ // the tasks that have a file sink operation
+ mv = new moveWork(loadTableWork, loadFileWork);
+ mvTask = TaskFactory.get(mv, this.conf);
+ }
// Maintain a map from the top level left most reducer in each of these
// trees
@@ -2479,12 +2401,24 @@
new HashMap<Operator<? extends Serializable>, Task<? extends Serializable>>();
for (String alias_id : this.topOps.keySet()) {
Operator<? extends Serializable> topOp = this.topOps.get(alias_id);
- Operator<? extends Serializable> reducer = getReducer(topOp);
+ Operator<? extends Serializable> reduceSink = getReduceSink(topOp);
+ Operator<? extends Serializable> reducer = null;
+ if (reduceSink != null)
+ reducer = reduceSink.getChildOperators().get(0);
Task<? extends Serializable> rootTask = opTaskMap.get(reducer);
if (rootTask == null) {
rootTask = TaskFactory.get(getMapRedWork(), this.conf);
opTaskMap.put(reducer, rootTask);
((mapredWork) rootTask.getWork()).setReducer(reducer);
+ reduceSinkDesc desc = (reduceSink == null) ? null : (reduceSinkDesc)reduceSink.getConf();
+
+ // The number of reducers may be specified in the plan in some cases, or may need to be inferred
+ if (desc != null) {
+ if (desc.getNumReducers() != -1)
+ ((mapredWork) rootTask.getWork()).setNumReduceTasks(new Integer(desc.getNumReducers()));
+ else if (desc.getInferNumReducers() == true)
+ ((mapredWork) rootTask.getWork()).setInferNumReducers(true);
+ }
this.rootTasks.add(rootTask);
}
genTaskPlan(topOp, rootTask, opTaskMap, mvTask);
@@ -2533,7 +2467,7 @@
HashMap<Operator<? extends Serializable>, Task<? extends Serializable>> redTaskMap,
Task<? extends Serializable> mvTask) {
// Check if this is a file sink operator
- if (op.getClass() == FileSinkOperator.class) {
+ if ((op.getClass() == FileSinkOperator.class) && (mvTask != null)) {
// If this is a file sink operator then set the move task to be dependent
// on the current task
currTask.addDependentTask(mvTask);
@@ -2577,6 +2511,10 @@
ctask = TaskFactory.get(cplan, this.conf);
// Add the reducer
cplan.setReducer(reducer);
+ if (((reduceSinkDesc)child.getConf()).getNumReducers() != -1)
+ cplan.setNumReduceTasks(new Integer(((reduceSinkDesc)child.getConf()).getNumReducers()));
+ else
+ cplan.setInferNumReducers(((reduceSinkDesc)child.getConf()).getInferNumReducers());
redTaskMap.put(reducer, ctask);
// Recurse on the reducer
@@ -2668,7 +2606,7 @@
@Override
@SuppressWarnings("nls")
- public void analyze(CommonTree ast, Context ctx) throws SemanticException {
+ public void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException {
this.ctx = ctx;
reset();
@@ -2696,7 +2634,7 @@
// At this point we have the complete operator tree
// from which we want to find the reduce operator
- genMapRedTasks();
+ genMapRedTasks(qb);
LOG.info("Completed plan generation");
@@ -2716,6 +2654,7 @@
/**
* This function create an ExprNodeDesc for a UDF function given the children (arguments).
* It will insert implicit type conversion functions if necessary.
+ * @throws SemanticException
*/
public static exprNodeDesc getFuncExprNodeDesc(String udfName, List<exprNodeDesc> children) {
// Find the corresponding method
@@ -2725,7 +2664,33 @@
assert(child != null);
TypeInfo childTypeInfo = child.getTypeInfo();
assert(childTypeInfo != null);
- argumentClasses.add(childTypeInfo.getPrimitiveClass());
+
+ // Note: we don't pass the element types of MAP/LIST to UDF.
+ // That will work for null test and size but not other more complex functionalities like list slice etc.
+ // For those more complex functionalities, we plan to have a ComplexUDF interface which has an evaluate
+ // method that accepts a list of objects and a list of objectinspectors.
+ switch (childTypeInfo.getCategory()) {
+ case PRIMITIVE: {
+ argumentClasses.add(childTypeInfo.getPrimitiveClass());
+ break;
+ }
+ case MAP: {
+ argumentClasses.add(Map.class);
+ break;
+ }
+ case LIST: {
+ argumentClasses.add(List.class);
+ break;
+ }
+ case STRUCT: {
+ argumentClasses.add(Object.class);
+ break;
+ }
+ default: {
+ // should never happen
+ assert(false);
+ }
+ }
}
Method udfMethod = FunctionRegistry.getUDFMethod(udfName, false, argumentClasses);
if (udfMethod == null) return null;
@@ -2740,7 +2705,7 @@
if (desc instanceof exprNodeNullDesc) {
exprNodeConstantDesc newCh = new exprNodeConstantDesc(TypeInfoFactory.getPrimitiveTypeInfo(pType), null);
ch.add(newCh);
- } else if (pType.isAssignableFrom(desc.getTypeInfo().getPrimitiveClass())) {
+ } else if (pType.isAssignableFrom(argumentClasses.get(i))) {
// no type conversion needed
ch.add(desc);
} else {
@@ -2783,8 +2748,7 @@
// If the current subExpression is pre-calculated, as in Group-By etc.
ColumnInfo colInfo = input.get("", expr.toStringTree());
if (colInfo != null) {
- desc = new exprNodeColumnDesc(
- colInfo.getType(), colInfo.getInternalName());
+ desc = new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName());
return desc;
}
@@ -2797,29 +2761,29 @@
int tokType = expr.getType();
switch (tokType) {
case HiveParser.TOK_COLREF: {
+
+ // For now only allow columns of the form tab.col
+ if (expr.getChildCount() == 1) {
+ throw new SemanticException(ErrorMsg.NO_TABLE_ALIAS.getMsg(expr.getChild(0)));
+ }
- assert(expr.getChildCount() == 2);
- String tabAlias = SemanticAnalyzer.getTableName(expr);
- String colName = SemanticAnalyzer.getSerDeFieldExpression(expr);
+ String tabAlias = expr.getChild(0).getText();
+ String colName = expr.getChild(1).getText();
if (tabAlias == null || colName == null) {
throw new SemanticException(ErrorMsg.INVALID_XPATH.getMsg(expr));
}
colInfo = input.get(tabAlias, colName);
- if (colInfo == null) {
- LOG.info("input: " + input.toString() + " expr: " + expr.toStringTree() + " e: " + input.getIsExprResolver());
- }
-
- // TODO: Hack it up for now: Later we have to pass the QB in order to check for the
- // table alias instead of relying on input.hasTableAlias
+
if (colInfo == null && input.getIsExprResolver()) {
throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(expr));
- } else if (!input.hasTableAlias(expr.getChild(0).getText())) {
- throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg((CommonTree)expr.getChild(0)));
+ }
+ else if (!input.hasTableAlias(tabAlias)) {
+ throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(expr.getChild(0)));
} else if (colInfo == null) {
- throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg((CommonTree)expr.getChild(1)));
+ throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)));
}
- desc = new exprNodeColumnDesc(
- colInfo.getType(), colInfo.getInternalName());
+
+ desc = new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName());
break;
}
@@ -2842,9 +2806,13 @@
return desc;
}
+ static HashMap<Integer, String> specialUnaryOperatorTextHashMap;
static HashMap<Integer, String> specialFunctionTextHashMap;
static HashMap<Integer, String> conversionFunctionTextHashMap;
static {
+ specialUnaryOperatorTextHashMap = new HashMap<Integer, String>();
+ specialUnaryOperatorTextHashMap.put(HiveParser.PLUS, "positive");
+ specialUnaryOperatorTextHashMap.put(HiveParser.MINUS, "negative");
specialFunctionTextHashMap = new HashMap<Integer, String>();
specialFunctionTextHashMap.put(HiveParser.TOK_ISNULL, "isnull");
specialFunctionTextHashMap.put(HiveParser.TOK_ISNOTNULL, "isnotnull");
@@ -2873,16 +2841,23 @@
}
public static String getFunctionText(CommonTree expr, boolean isFunction) {
- String funcText;
+ String funcText = null;
if (!isFunction) {
- // For operator, the function name is the operator text
- funcText = expr.getText();
+ // For operator, the function name is the operator text, unless it's in our special dictionary
+ if (expr.getChildCount() == 1) {
+ funcText = specialUnaryOperatorTextHashMap.get(expr.getType());
+ }
+ if (funcText == null) {
+ funcText = expr.getText();
+ }
} else {
- // For TOK_FUNCTION, the function name is stored in the first child.
+ // For TOK_FUNCTION, the function name is stored in the first child, unless it's in our
+ // special dictionary.
assert(expr.getChildCount() >= 1);
- funcText = specialFunctionTextHashMap.get(((CommonTree)expr.getChild(0)).getType());
+ int funcType = ((CommonTree)expr.getChild(0)).getType();
+ funcText = specialFunctionTextHashMap.get(funcType);
if (funcText == null) {
- funcText = conversionFunctionTextHashMap.get(((CommonTree)expr.getChild(0)).getType());
+ funcText = conversionFunctionTextHashMap.get(funcType);
}
if (funcText == null) {
funcText = ((CommonTree)expr.getChild(0)).getText();
@@ -2911,24 +2886,60 @@
exprNodeConstantDesc fieldName = (exprNodeConstantDesc)children.get(1);
assert(fieldName.getValue() instanceof String);
- // Calculate TypeInfo
+ // Calculate result TypeInfo
String fieldNameString = (String)fieldName.getValue();
- TypeInfo t = object.getTypeInfo().getStructFieldTypeInfo(fieldNameString);
+ TypeInfo objectTypeInfo = object.getTypeInfo();
- desc = new exprNodeFieldDesc(t, children.get(0), fieldNameString);
+ // Allow accessing a field of list element structs directly from a list
+ boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST);
+ if (isList) {
+ objectTypeInfo = objectTypeInfo.getListElementTypeInfo();
+ }
+ if (objectTypeInfo.getCategory() != Category.STRUCT) {
+ throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr));
+ }
+ TypeInfo t = objectTypeInfo.getStructFieldTypeInfo(fieldNameString);
+ if (isList) {
+ t = TypeInfoFactory.getListTypeInfo(t);
+ }
+
+ desc = new exprNodeFieldDesc(t, children.get(0), fieldNameString, isList);
} else if (funcText.equals("[")){
// "[]" : LSQUARE/INDEX Expression
assert(children.size() == 2);
- // Only allow constant integer index for now
- if (!(children.get(1) instanceof exprNodeConstantDesc)
- || !(((exprNodeConstantDesc)children.get(1)).getValue() instanceof Integer)) {
- throw new SemanticException(ErrorMsg.INVALID_ARRAYINDEX_CONSTANT.getMsg(expr));
- }
- // Calculate TypeInfo
- TypeInfo t = children.get(0).getTypeInfo().getListElementTypeInfo();
- desc = new exprNodeIndexDesc(t, children.get(0), children.get(1));
+ // Check whether this is a list or a map
+ TypeInfo myt = children.get(0).getTypeInfo();
+
+ if (myt.getCategory() == Category.LIST) {
+ // Only allow constant integer index for now
+ if (!(children.get(1) instanceof exprNodeConstantDesc)
+ || !(((exprNodeConstantDesc)children.get(1)).getValue() instanceof Integer)) {
+ throw new SemanticException(ErrorMsg.INVALID_ARRAYINDEX_CONSTANT.getMsg(expr));
+ }
+
+ // Calculate TypeInfo
+ TypeInfo t = myt.getListElementTypeInfo();
+ desc = new exprNodeIndexDesc(t, children.get(0), children.get(1));
+ }
+ else if (myt.getCategory() == Category.MAP) {
+ // Only allow only constant indexes for now
+ if (!(children.get(1) instanceof exprNodeConstantDesc)) {
+ throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_CONSTANT.getMsg(expr));
+ }
+ if (!(((exprNodeConstantDesc)children.get(1)).getValue().getClass() ==
+ myt.getMapKeyTypeInfo().getPrimitiveClass())) {
+ throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE.getMsg(expr));
+ }
+ // Calculate TypeInfo
+ TypeInfo t = myt.getMapValueTypeInfo();
+
+ desc = new exprNodeIndexDesc(t, children.get(0), children.get(1));
+ }
+ else {
+ throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr));
+ }
} else {
// other operators or functions
Class<? extends UDF> udf = FunctionRegistry.getUDFClass(funcText);
@@ -2955,6 +2966,16 @@
}
}
}
+ // UDFOPPositive is a no-op.
+ // However, we still create it, and then remove it here, to make sure we only allow
+ // "+" for numeric types.
+ if (desc instanceof exprNodeFuncDesc) {
+ exprNodeFuncDesc funcDesc = (exprNodeFuncDesc)desc;
+ if (funcDesc.getUDFClass().equals(UDFOPPositive.class)) {
+ assert(funcDesc.getChildren().size() == 1);
+ desc = funcDesc.getChildren().get(0);
+ }
+ }
assert(desc != null);
return desc;
}
Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java?rev=706708&r1=706707&r2=706708&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (original)
+++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java Tue Oct 21 11:29:18 2008
@@ -36,8 +36,11 @@
case HiveParser.TOK_DROPTABLE:
case HiveParser.TOK_DESCTABLE:
case HiveParser.TOK_ALTERTABLE_ADDCOLS:
+ case HiveParser.TOK_ALTERTABLE_REPLACECOLS:
case HiveParser.TOK_ALTERTABLE_RENAME:
- case HiveParser.TOK_SHOWTABLES:
+ case HiveParser.TOK_ALTERTABLE_DROPPARTS:
+ case HiveParser.TOK_SHOWTABLES:
+ case HiveParser.TOK_SHOWPARTITIONS:
return new DDLSemanticAnalyzer(conf);
case HiveParser.TOK_CREATEFUNCTION:
return new FunctionSemanticAnalyzer(conf);
Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java?rev=706708&r1=706707&r2=706708&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java (original)
+++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java Tue Oct 21 11:29:18 2008
@@ -22,11 +22,12 @@
public class DDLWork implements Serializable {
private static final long serialVersionUID = 1L;
- private createTableDesc createTblDesc;
- private dropTableDesc dropTblDesc;
- private alterTableDesc alterTblDesc;
- private showTablesDesc showTblsDesc;
- private descTableDesc descTblDesc;
+ private createTableDesc createTblDesc;
+ private dropTableDesc dropTblDesc;
+ private alterTableDesc alterTblDesc;
+ private showTablesDesc showTblsDesc;
+ private showPartitionsDesc showPartsDesc;
+ private descTableDesc descTblDesc;
public DDLWork() { }
@@ -66,6 +67,13 @@
}
/**
+ * @param showPartsDesc
+ */
+ public DDLWork(showPartitionsDesc showPartsDesc) {
+ this.showPartsDesc = showPartsDesc;
+ }
+
+ /**
* @return the createTblDesc
*/
@explain(displayName="Create Table Operator")
@@ -125,6 +133,22 @@
this.showTblsDesc = showTblsDesc;
}
+
+ /**
+ * @return the showPartsDesc
+ */
+ @explain(displayName="Show Partitions Operator")
+ public showPartitionsDesc getShowPartsDesc() {
+ return showPartsDesc;
+ }
+
+ /**
+ * @param showPartsDesc the showPartsDesc to set
+ */
+ public void setShowPartsDesc(showPartitionsDesc showPartsDesc) {
+ this.showPartsDesc = showPartsDesc;
+ }
+
/**
* @return the descTblDesc
*/
Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=706708&r1=706707&r2=706708&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Tue Oct 21 11:29:18 2008
@@ -63,21 +63,14 @@
// We will make reduce key and reduce value TableDesc with configurable SerDes
- public static reduceSinkDesc getReduceSinkDesc(ArrayList<exprNodeDesc> keyCols,
- ArrayList<exprNodeDesc> valueCols, int numPartitionFields) {
+ public static reduceSinkDesc getReduceSinkDesc(ArrayList<exprNodeDesc> keyCols,
+ ArrayList<exprNodeDesc> valueCols,
+ int tag, int numPartitionFields,
+ int numReducers, boolean inferNumReducers) {
- return new reduceSinkDesc(keyCols, valueCols, numPartitionFields,
- getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(keyCols.size())),
- getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(valueCols.size())));
- }
-
- // We will make reduce key and reduce value TableDesc with configurable SerDes
- public static reduceSinkDesc getReduceSinkDesc(ArrayList<exprNodeDesc> keyCols,
- ArrayList<exprNodeDesc> valueCols, int tag, int numPartitionFields) {
-
- return new reduceSinkDesc(keyCols, valueCols, tag, numPartitionFields,
- getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(keyCols.size())),
- getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(valueCols.size())));
+ return new reduceSinkDesc(keyCols, valueCols, tag, numPartitionFields, numReducers, inferNumReducers,
+ getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(keyCols.size())),
+ getDefaultTableDesc("" + Utilities.ctrlaCode, ObjectInspectorUtils.getIntegerCSV(valueCols.size())));
}
// We should read the TableDesc from gWork when it is available.
Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java?rev=706708&r1=706707&r2=706708&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java (original)
+++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/alterTableDesc.java Tue Oct 21 11:29:18 2008
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
-import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -29,7 +28,7 @@
public class alterTableDesc extends ddlDesc implements Serializable
{
private static final long serialVersionUID = 1L;
- public static enum alterTableTypes {RENAME, ADDCOLS};
+ public static enum alterTableTypes {RENAME, ADDCOLS, REPLACECOLS};
alterTableTypes op;
String oldName;
@@ -50,8 +49,8 @@
* @param name name of the table
* @param newCols new columns to be added
*/
- public alterTableDesc(String name, List<FieldSchema> newCols) {
- op = alterTableTypes.ADDCOLS;
+ public alterTableDesc(String name, List<FieldSchema> newCols, alterTableTypes alterType) {
+ this.op = alterType;
this.oldName = name;
this.newCols = newCols;
}
@@ -100,6 +99,8 @@
return "rename";
case ADDCOLS:
return "add columns";
+ case REPLACECOLS:
+ return "replace columns";
}
return "unknown";
@@ -128,4 +129,5 @@
public void setNewCols(List<FieldSchema> newCols) {
this.newCols = newCols;
}
+
}
Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java?rev=706708&r1=706707&r2=706708&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java (original)
+++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java Tue Oct 21 11:29:18 2008
@@ -20,6 +20,7 @@
import java.io.Serializable;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
@@ -43,6 +44,8 @@
String comment;
boolean isSequenceFile;
String location;
+ String serName;
+ Map<String, String> mapProp;
public createTableDesc(String tableName, boolean isExternal,
List<FieldSchema> cols, List<FieldSchema> partCols,
@@ -51,7 +54,7 @@
String collItemDelim,
String mapKeyDelim, String lineDelim,
String comment, boolean isSequenceFile,
- String location) {
+ String location, String serName, Map<String, String> mapProp) {
this.tableName = tableName;
this.isExternal = isExternal;
this.bucketCols = bucketCols;
@@ -66,6 +69,8 @@
this.mapKeyDelim = mapKeyDelim;
this.numBuckets = numBuckets;
this.partCols = partCols;
+ this.serName = serName;
+ this.mapProp = mapProp;
}
@explain(displayName="name")
@@ -166,7 +171,7 @@
this.comment = comment;
}
- @explain(displayName="isSequenceFile")
+ @explain(displayName="isSequenceFile")
public boolean isSequenceFile() {
return isSequenceFile;
}
@@ -207,4 +212,35 @@
public void setSortCols(List<Order> sortCols) {
this.sortCols = sortCols;
}
+
+ /**
+ * @return the serDeName
+ */
+ @explain(displayName="serde name")
+ public String getSerName() {
+ return serName;
+ }
+
+ /**
+ * @param serName the serName to set
+ */
+ public void setSerName(String serName) {
+ this.serName = serName;
+ }
+
+ /**
+ * @return the serDe properties
+ */
+ @explain(displayName="serde properties")
+ public Map<String,String> getMapProp() {
+ return mapProp;
+ }
+
+ /**
+ * @param mapProp the map properties to set
+ */
+ public void setMapProp(Map<String,String> mapProp) {
+ this.mapProp = mapProp;
+ }
+
}
Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/descTableDesc.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/descTableDesc.java?rev=706708&r1=706707&r2=706708&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/descTableDesc.java (original)
+++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/descTableDesc.java Tue Oct 21 11:29:18 2008
@@ -19,6 +19,8 @@
package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
+import java.util.HashMap;
+
import org.apache.hadoop.fs.Path;
@explain(displayName="Describe Table")
@@ -26,28 +28,69 @@
{
private static final long serialVersionUID = 1L;
- String tableName;
+ String tableName;
+ HashMap<String, String> partSpec;
Path resFile;
+ boolean isExt;
/**
+ * @param isExt
+ * @param partSpec
* @param resFile
* @param tableName
*/
- public descTableDesc(Path resFile, String tableName) {
+ public descTableDesc(Path resFile, String tableName, HashMap<String, String> partSpec, boolean isExt) {
+ this.isExt = isExt;
+ this.partSpec = partSpec;
this.resFile = resFile;
this.tableName = tableName;
}
+ /**
+ * @return the isExt
+ */
+ public boolean isExt() {
+ return isExt;
+ }
+
+ /**
+ * @param isExt the isExt to set
+ */
+ public void setExt(boolean isExt) {
+ this.isExt = isExt;
+ }
+
+ /**
+ * @return the tableName
+ */
@explain(displayName="table")
public String getTableName() {
return tableName;
}
+ /**
+ * @param tableName the tableName to set
+ */
public void setTableName(String tableName) {
this.tableName = tableName;
}
/**
+ * @return the partSpec
+ */
+ @explain(displayName="partition")
+ public HashMap<String, String> getPartSpec() {
+ return partSpec;
+ }
+
+ /**
+ * @param partSpec the partSpec to set
+ */
+ public void setPartSpecs(HashMap<String, String> partSpec) {
+ this.partSpec = partSpec;
+ }
+
+ /**
* @return the resFile
*/
public Path getResFile() {
Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/dropTableDesc.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/dropTableDesc.java?rev=706708&r1=706707&r2=706708&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/dropTableDesc.java (original)
+++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/dropTableDesc.java Tue Oct 21 11:29:18 2008
@@ -19,6 +19,8 @@
package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
@explain(displayName="Drop Table")
public class dropTableDesc extends ddlDesc implements Serializable
@@ -26,12 +28,19 @@
private static final long serialVersionUID = 1L;
String tableName;
+ List<HashMap<String, String>> partSpecs;
/**
* @param tableName
*/
public dropTableDesc(String tableName) {
this.tableName = tableName;
+ this.partSpecs = null;
+ }
+
+ public dropTableDesc(String tableName, List<HashMap<String, String>> partSpecs) {
+ this.tableName = tableName;
+ this.partSpecs = partSpecs;
}
/**
@@ -48,4 +57,18 @@
public void setTableName(String tableName) {
this.tableName = tableName;
}
+
+ /**
+ * @return the partSpecs
+ */
+ public List<HashMap<String, String>> getPartSpecs() {
+ return partSpecs;
+ }
+
+ /**
+ * @param partSpecs the partSpecs to set
+ */
+ public void setPartSpecs(List<HashMap<String, String>> partSpecs) {
+ this.partSpecs = partSpecs;
+ }
}
Modified: hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java?rev=706708&r1=706707&r2=706708&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java (original)
+++ hadoop/core/branches/branch-0.19/src/contrib/hive/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java Tue Oct 21 11:29:18 2008
@@ -26,11 +26,13 @@
public class exprNodeColumnDesc extends exprNodeDesc implements Serializable {
private static final long serialVersionUID = 1L;
private String column;
+ private boolean isVirtual;
public exprNodeColumnDesc() {}
public exprNodeColumnDesc(TypeInfo typeInfo, String column) {
super(typeInfo);
this.column = column;
+ this.isVirtual = isVirtual;
}
public exprNodeColumnDesc(Class<?> c, String column) {
super(TypeInfoFactory.getPrimitiveTypeInfo(c));
@@ -42,6 +44,7 @@
public void setColumn(String column) {
this.column = column;
}
+
public String toString() {
return "Column[" + column + "]";
}