You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by at...@apache.org on 2009/01/24 02:58:06 UTC
svn commit: r737291 [2/8] - in /hadoop/hive/trunk: ./ eclipse-templates/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/lib/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/
ql/src/java/org/apache/hadoop/hive/ql/parse...
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java Sat Jan 24 01:58:01 2009
@@ -43,7 +43,7 @@
* @param nd the reduce sink operator encountered
* @param opProcCtx context
*/
- public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException {
+ public Object process(Node nd, NodeProcessorCtx opProcCtx, Object... nodeOutputs) throws SemanticException {
ReduceSinkOperator op = (ReduceSinkOperator)nd;
GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
@@ -69,6 +69,7 @@
}
mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId()));
+ return null;
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java Sat Jan 24 01:58:01 2009
@@ -44,7 +44,7 @@
* @param nd the table sink operator encountered
* @param opProcCtx context
*/
- public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException {
+ public Object process(Node nd, NodeProcessorCtx opProcCtx, Object... nodeOutputs) throws SemanticException {
TableScanOperator op = (TableScanOperator)nd;
GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
ParseContext parseCtx = ctx.getParseCtx();
@@ -62,10 +62,11 @@
String currAliasId = alias;
ctx.setCurrAliasId(currAliasId);
mapCurrCtx.put(op, new GenMapRedCtx(currTask, currTopOp, currAliasId));
- return;
+ return null;
}
}
assert false;
+ return null;
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java Sat Jan 24 01:58:01 2009
@@ -154,7 +154,7 @@
}
// Create function desc
- desc = SemanticAnalyzer.getXpathOrFuncExprNodeDesc(expr, isFunction, children);
+ desc = TypeCheckProcFactory.DefaultExprProcessor.getXpathOrFuncExprNodeDesc(expr, isFunction, children);
if (desc instanceof exprNodeFuncDesc && (
((exprNodeFuncDesc)desc).getUDFMethod().getDeclaringClass().equals(UDFOPAnd.class)
@@ -245,7 +245,7 @@
if (this.prunerExpr == null)
this.prunerExpr = desc;
else
- this.prunerExpr = SemanticAnalyzer.getFuncExprNodeDesc("OR", this.prunerExpr, desc);
+ this.prunerExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("OR", this.prunerExpr, desc);
}
}
@@ -264,7 +264,7 @@
if (this.prunerExpr == null)
this.prunerExpr = desc;
else
- this.prunerExpr = SemanticAnalyzer.getFuncExprNodeDesc("AND", this.prunerExpr, desc);
+ this.prunerExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("AND", this.prunerExpr, desc);
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java Sat Jan 24 01:58:01 2009
@@ -70,14 +70,15 @@
return ret.toString();
}
- public void process(Node nd, NodeProcessorCtx ctx) throws SemanticException {
+ public Object process(Node nd, NodeProcessorCtx ctx, Object... nodeOutputs) throws SemanticException {
Operator<? extends Serializable> op = (Operator<? extends Serializable>)nd;
if (opMap.get(op) == null) {
opMap.put(op, curNum++);
}
out.println("[" + opMap.get(op) + "] " + op.getClass().getName() + " =p=> " + getParents(op) + " =c=> " + getChildren(op));
if(op.getConf() == null) {
- return;
+ return null;
}
+ return null;
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Jan 24 01:58:01 2009
@@ -23,7 +23,6 @@
import java.io.Serializable;
import java.lang.reflect.Method;
-import org.antlr.runtime.tree.*;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
@@ -35,6 +34,7 @@
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
+import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
import org.apache.hadoop.hive.ql.lib.GraphWalker;
@@ -901,7 +901,7 @@
Operator output = putOpInsertMap(
OperatorFactory.getAndMakeChild(
- new filterDesc(genExprNodeDesc(qb.getMetaData(), (ASTNode)whereExpr.getChild(0), inputRR)),
+ new filterDesc(genExprNodeDesc((ASTNode)whereExpr.getChild(0), inputRR)),
new RowSchema(inputRR.getColumnInfos()), input), inputRR);
LOG.debug("Created Filter Plan for " + qb.getId() + ":" + dest + " row schema: " + inputRR.toString());
@@ -921,7 +921,7 @@
RowResolver inputRR = inputCtx.getRR();
Operator output = putOpInsertMap(
OperatorFactory.getAndMakeChild(
- new filterDesc(genExprNodeDesc(qb.getMetaData(), condn, inputRR)),
+ new filterDesc(genExprNodeDesc(condn, inputRR)),
new RowSchema(inputRR.getColumnInfos()), input), inputRR);
LOG.debug("Created Filter Plan for " + qb.getId() + " row schema: " + inputRR.toString());
@@ -1150,7 +1150,7 @@
genColList(tabAlias, alias, expr, col_list, inputRR, pos, out_rwsch);
selectStar = true;
} else {
- exprNodeDesc exp = genExprNodeDesc(qb.getMetaData(), expr, inputRR);
+ exprNodeDesc exp = genExprNodeDesc(expr, inputRR);
col_list.add(exp);
if (!StringUtils.isEmpty(alias) &&
(out_rwsch.get(null, colAlias) != null)) {
@@ -1164,7 +1164,7 @@
}
} else {
// Case when this is an expression
- exprNodeDesc exp = genExprNodeDesc(qb.getMetaData(), sel, inputRR);
+ exprNodeDesc exp = genExprNodeDesc(sel, inputRR);
col_list.add(exp);
if (!StringUtils.isEmpty(alias) &&
(out_rwsch.get(null, colAlias) != null)) {
@@ -1203,36 +1203,58 @@
*/
static class UDAFInfo {
ArrayList<exprNodeDesc> convertedParameters;
- Method aggregateMethod;
- Method evaluateMethod;
+ Class<?> retType;
+ Class<? extends UDAFEvaluator> evalClass;
}
/**
* Returns the UDAFInfo struct for the aggregation
* @param aggName The name of the UDAF.
* @param mode The mode of the aggregation. This affects the evaluate method.
- * @param aggClasses The classes of the parameters to the UDAF.
* @param aggParameters The actual exprNodeDesc of the parameters.
* @param aggTree The ASTNode node of the UDAF in the query.
* @return UDAFInfo
* @throws SemanticException when the UDAF is not found or has problems.
*/
- UDAFInfo getUDAFInfo(String aggName, groupByDesc.Mode mode, ArrayList<Class<?>> aggClasses,
+ UDAFInfo getUDAFInfo(String aggName, groupByDesc.Mode mode,
ArrayList<exprNodeDesc> aggParameters, ASTNode aggTree) throws SemanticException {
UDAFInfo r = new UDAFInfo();
- r.aggregateMethod = FunctionRegistry.getUDAFMethod(aggName, aggClasses);
- if (null == r.aggregateMethod) {
- String reason = "Looking for UDAF \"" + aggName + "\" with parameters " + aggClasses;
+ ArrayList<Class<?>> aggClasses = new ArrayList<Class<?>>();
+ for(exprNodeDesc expr: aggParameters) {
+ aggClasses.add(expr.getTypeInfo().getPrimitiveClass());
+ }
+ r.evalClass = FunctionRegistry.getUDAFEvaluator(aggName, aggClasses);
+ if (null == r.evalClass) {
+ String reason = "Looking for UDAF Evaluator\"" + aggName + "\" with parameters " + aggClasses;
+ throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)aggTree.getChild(0), reason));
+ }
+
+ Method aggregateMethod = null;
+ for(Method m: r.evalClass.getMethods()) {
+ if (m.getName().equalsIgnoreCase("iterate")) {
+ aggregateMethod = m;
+ }
+ }
+
+ if (null == aggregateMethod) {
+ String reason = "Looking for UDAF Evaluator Iterator\"" + aggName + "\" with parameters " + aggClasses;
throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)aggTree.getChild(0), reason));
}
- r.convertedParameters = convertParameters(r.aggregateMethod, aggParameters);
+ r.convertedParameters = convertParameters(aggregateMethod, aggParameters);
- r.evaluateMethod = FunctionRegistry.getUDAFEvaluateMethod(aggName, mode);
- if (r.evaluateMethod == null) {
- String reason = "UDAF \"" + aggName + "\" does not have evaluate()/evaluatePartial() methods.";
+ Method evaluateMethod = FunctionRegistry.getUDAFEvaluateMethod(aggName, mode);
+ String funcName = (mode == groupByDesc.Mode.COMPLETE || mode == groupByDesc.Mode.FINAL) ? "terminate" : "terminatePartial";
+ for(Method m: r.evalClass.getMethods()) {
+ if (m.getName().equalsIgnoreCase(funcName)) {
+ evaluateMethod = m;
+ }
+ }
+ if (evaluateMethod == null) {
+ String reason = "UDAF \"" + aggName + "\" does not have terminate()/terminatePartial() methods.";
throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)aggTree.getChild(0), reason));
}
+ r.retType = evaluateMethod.getReturnType();
return r;
}
@@ -1279,7 +1301,6 @@
Class<? extends UDAF> aggClass = FunctionRegistry.getUDAF(aggName);
assert (aggClass != null);
ArrayList<exprNodeDesc> aggParameters = new ArrayList<exprNodeDesc>();
- ArrayList<Class<?>> aggClasses = new ArrayList<Class<?>>();
// 0 is the function name
for (int i = 1; i < value.getChildCount(); i++) {
String text = value.getChild(i).toStringTree();
@@ -1292,16 +1313,15 @@
String paraExpression = paraExprInfo.getInternalName();
assert(paraExpression != null);
aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExprInfo.getInternalName()));
- aggClasses.add(paraExprInfo.getType().getPrimitiveClass());
}
- UDAFInfo udaf = getUDAFInfo(aggName, mode, aggClasses, aggParameters, value);
+ UDAFInfo udaf = getUDAFInfo(aggName, mode, aggParameters, value);
- aggregations.add(new aggregationDesc(aggClass, udaf.convertedParameters,
+ aggregations.add(new aggregationDesc(udaf.evalClass, udaf.convertedParameters,
value.getToken().getType() == HiveParser.TOK_FUNCTIONDI));
groupByOutputRowResolver.put("",value.toStringTree(),
new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(),
- udaf.evaluateMethod.getReturnType()));
+ udaf.retType));
}
return
@@ -1352,12 +1372,9 @@
String aggName = value.getChild(0).getText();
Class<? extends UDAF> aggClass = FunctionRegistry.getUDAF(aggName);
assert (aggClass != null);
- Method aggEvaluateMethod = null;
- ArrayList<exprNodeDesc> aggParameters = null;
+ ArrayList<exprNodeDesc> aggParameters = new ArrayList<exprNodeDesc>();
if (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI) {
- ArrayList<Class<?>> aggClasses = new ArrayList<Class<?>>();
- ArrayList<exprNodeDesc> params = new ArrayList<exprNodeDesc>();
// 0 is the function name
for (int i = 1; i < value.getChildCount(); i++) {
String text = value.getChild(i).toStringTree();
@@ -1369,17 +1386,11 @@
String paraExpression = paraExprInfo.getInternalName();
assert(paraExpression != null);
- params.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExprInfo.getInternalName()));
- aggClasses.add(paraExprInfo.getType().getPrimitiveClass());
+ aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExprInfo.getInternalName()));
}
- UDAFInfo udaf = getUDAFInfo(aggName, mode, aggClasses, params, value);
- aggParameters = udaf.convertedParameters;
- aggEvaluateMethod = udaf.evaluateMethod;
}
else {
- aggParameters = new ArrayList<exprNodeDesc>();
- aggEvaluateMethod = FunctionRegistry.getUDAFEvaluateMethod(aggName, mode);
String text = entry.getKey();
ColumnInfo paraExprInfo = groupByInputRowResolver.get("",text);
if (paraExprInfo == null) {
@@ -1390,10 +1401,12 @@
aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExpression));
}
- aggregations.add(new aggregationDesc(aggClass, aggParameters, ((mode == groupByDesc.Mode.FINAL) ? false : (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI))));
+ UDAFInfo udaf = getUDAFInfo(aggName, mode, aggParameters, value);
+ aggregations.add(new aggregationDesc(udaf.evalClass, udaf.convertedParameters,
+ ((mode == groupByDesc.Mode.FINAL) ? false : (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI))));
groupByOutputRowResolver.put("", value.toStringTree(),
new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() - 1).toString(),
- aggEvaluateMethod.getReturnType()));
+ udaf.retType));
}
return putOpInsertMap(
@@ -1423,7 +1436,7 @@
List<ASTNode> grpByExprs = getGroupByForClause(parseInfo, dest);
for (int i = 0; i < grpByExprs.size(); ++i) {
ASTNode grpbyExpr = grpByExprs.get(i);
- exprNodeDesc grpByExprNode = genExprNodeDesc(qb.getMetaData(), grpbyExpr, groupByInputRowResolver);
+ exprNodeDesc grpByExprNode = genExprNodeDesc(grpbyExpr, groupByInputRowResolver);
groupByKeys.add(grpByExprNode);
String field = (Integer.valueOf(i)).toString();
@@ -1440,7 +1453,7 @@
ASTNode parameter = (ASTNode) value.getChild(i);
String text = parameter.toStringTree();
if (groupByOutputRowResolver.get("",text) == null) {
- exprNodeDesc distExprNode = genExprNodeDesc(qb.getMetaData(), parameter, groupByInputRowResolver);
+ exprNodeDesc distExprNode = genExprNodeDesc(parameter, groupByInputRowResolver);
groupByKeys.add(distExprNode);
numDistn++;
String field = (Integer.valueOf(grpByExprs.size() + numDistn -1)).toString();
@@ -1464,19 +1477,18 @@
// 0 is the function name
for (int i = 1; i < value.getChildCount(); i++) {
ASTNode paraExpr = (ASTNode)value.getChild(i);
- exprNodeDesc paraExprNode = genExprNodeDesc(qb.getMetaData(), paraExpr, groupByInputRowResolver);
+ exprNodeDesc paraExprNode = genExprNodeDesc(paraExpr, groupByInputRowResolver);
aggParameters.add(paraExprNode);
- aggClasses.add(paraExprNode.getTypeInfo().getPrimitiveClass());
}
- UDAFInfo udaf = getUDAFInfo(aggName, mode, aggClasses, aggParameters, value);
+ UDAFInfo udaf = getUDAFInfo(aggName, mode, aggParameters, value);
- aggregations.add(new aggregationDesc(aggClass, udaf.convertedParameters,
+ aggregations.add(new aggregationDesc(udaf.evalClass, udaf.convertedParameters,
value.getToken().getType() == HiveParser.TOK_FUNCTIONDI));
groupByOutputRowResolver.put("",value.toStringTree(),
new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() -1).toString(),
- udaf.evaluateMethod.getReturnType()));
+ udaf.retType));
}
return putOpInsertMap(
@@ -1506,7 +1518,7 @@
Class<?> from = desc.getTypeInfo().getPrimitiveClass();
Class<?> to = pType;
assert(FunctionRegistry.implicitConvertable(from, to));
- Method conv = FunctionRegistry.getUDFMethod(to.getName(), true, from);
+ Method conv = FunctionRegistry.getUDFMethod(to.getName(), from);
assert(conv != null);
Class<? extends UDF> c = FunctionRegistry.getUDFClass(to.getName());
assert(c != null);
@@ -1635,7 +1647,7 @@
List<ASTNode> grpByExprs = getGroupByForClause(parseInfo, dest);
for (int i = 0; i < grpByExprs.size(); ++i) {
ASTNode grpbyExpr = grpByExprs.get(i);
- reduceKeys.add(genExprNodeDesc(qb.getMetaData(), grpbyExpr, reduceSinkInputRowResolver));
+ reduceKeys.add(genExprNodeDesc(grpbyExpr, reduceSinkInputRowResolver));
String text = grpbyExpr.toStringTree();
if (reduceSinkOutputRowResolver.get("", text) == null) {
reduceSinkOutputRowResolver.put("", text,
@@ -1654,7 +1666,7 @@
ASTNode parameter = (ASTNode) value.getChild(i);
String text = parameter.toStringTree();
if (reduceSinkOutputRowResolver.get("",text) == null) {
- reduceKeys.add(genExprNodeDesc(qb.getMetaData(), parameter, reduceSinkInputRowResolver));
+ reduceKeys.add(genExprNodeDesc(parameter, reduceSinkInputRowResolver));
reduceSinkOutputRowResolver.put("", text,
new ColumnInfo(Utilities.ReduceField.KEY.toString() + "." + Integer.valueOf(reduceKeys.size() - 1).toString(),
reduceKeys.get(reduceKeys.size()-1).getTypeInfo()));
@@ -1673,7 +1685,7 @@
ASTNode parameter = (ASTNode) value.getChild(i);
String text = parameter.toStringTree();
if (reduceSinkOutputRowResolver.get("",text) == null) {
- reduceValues.add(genExprNodeDesc(qb.getMetaData(), parameter, reduceSinkInputRowResolver));
+ reduceValues.add(genExprNodeDesc(parameter, reduceSinkInputRowResolver));
reduceSinkOutputRowResolver.put("", text,
new ColumnInfo(Utilities.ReduceField.VALUE.toString() + "." + Integer.valueOf(reduceValues.size() - 1).toString(),
reduceValues.get(reduceValues.size()-1).getTypeInfo()));
@@ -1783,12 +1795,9 @@
HashMap<String, ASTNode> aggregationTrees = parseInfo
.getAggregationExprsForClause(dest);
for (Map.Entry<String, ASTNode> entry : aggregationTrees.entrySet()) {
- ASTNode value = entry.getValue();
- String aggName = value.getChild(0).getText();
- Class<? extends UDAF> aggClass = FunctionRegistry.getUDAF(aggName);
- Method aggEvaluateMethod = FunctionRegistry.getUDAFEvaluateMethod(aggName, mode);
- assert (aggClass != null);
ArrayList<exprNodeDesc> aggParameters = new ArrayList<exprNodeDesc>();
+ ArrayList<Class<?>> aggParamTypes = new ArrayList<Class<?>>();
+ ASTNode value = entry.getValue();
String text = entry.getKey();
ColumnInfo paraExprInfo = groupByInputRowResolver2.get("",text);
if (paraExprInfo == null) {
@@ -1797,10 +1806,18 @@
String paraExpression = paraExprInfo.getInternalName();
assert(paraExpression != null);
aggParameters.add(new exprNodeColumnDesc(paraExprInfo.getType(), paraExpression));
- aggregations.add(new aggregationDesc(aggClass, aggParameters, ((mode == groupByDesc.Mode.FINAL) ? false : (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI))));
+ aggParamTypes.add(paraExprInfo.getType().getPrimitiveClass());
+
+ String aggName = value.getChild(0).getText();
+ Class<? extends UDAF> aggClass = FunctionRegistry.getUDAF(aggName);
+ assert (aggClass != null);
+
+ UDAFInfo udaf = getUDAFInfo(aggName, mode, aggParameters, value);
+ aggregations.add(new aggregationDesc(udaf.evalClass, udaf.convertedParameters,
+ ((mode == groupByDesc.Mode.FINAL) ? false : (value.getToken().getType() == HiveParser.TOK_FUNCTIONDI))));
groupByOutputRowResolver2.put("", value.toStringTree(),
new ColumnInfo(Integer.valueOf(groupByKeys.size() + aggregations.size() - 1).toString(),
- aggEvaluateMethod.getReturnType()));
+ udaf.retType));
}
return putOpInsertMap(
@@ -2099,7 +2116,7 @@
// cannot convert to complex types
column = null;
} else {
- column = getFuncExprNodeDesc(tableFieldTypeInfo.getPrimitiveClass().getName(), column);
+ column = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc(tableFieldTypeInfo.getPrimitiveClass().getName(), column);
}
if (column == null) {
String reason = "Cannot convert column " + i + " from " + rowFieldTypeInfo + " to "
@@ -2183,7 +2200,7 @@
int ccount = partitionExprs.getChildCount();
for(int i=0; i<ccount; ++i) {
ASTNode cl = (ASTNode)partitionExprs.getChild(i);
- partitionCols.add(genExprNodeDescFromColRef(cl, inputRR));
+ partitionCols.add(genExprNodeDesc(cl, inputRR));
}
}
@@ -2212,7 +2229,7 @@
order.append("+");
}
- sortCols.add(genExprNodeDescFromColRef(cl, inputRR));
+ sortCols.add(genExprNodeDesc(cl, inputRR));
}
}
@@ -2312,7 +2329,7 @@
Vector<ASTNode> exprs = joinTree.getExpressions().get(pos);
for (int i = 0; i < exprs.size(); i++) {
ASTNode expr = exprs.get(i);
- reduceKeys.add(genExprNodeDesc(qb.getMetaData(), expr, inputRS));
+ reduceKeys.add(genExprNodeDesc(expr, inputRS));
}
// Walk over the input row resolver and copy in the output
@@ -2403,7 +2420,7 @@
// Add implicit type conversion if necessary
for(int i=0; i<right.length; i++) {
if (!commonClass.isAssignableFrom(keys.get(i).get(k).getTypeInfo().getPrimitiveClass())) {
- keys.get(i).set(k, getFuncExprNodeDesc(commonClass.getName(), keys.get(i).get(k)));
+ keys.get(i).set(k, TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc(commonClass.getName(), keys.get(i).get(k)));
}
}
}
@@ -2842,21 +2859,21 @@
}
else {
for(ASTNode expr: ts.getExprs()) {
- args.add(genExprNodeDesc(qbm, expr, rwsch));
+ args.add(genExprNodeDesc(expr, rwsch));
}
}
- exprNodeDesc hashfnExpr = getFuncExprNodeDesc("default_sample_hashfn", args);
+ exprNodeDesc hashfnExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("default_sample_hashfn", args);
assert(hashfnExpr != null);
LOG.info("hashfnExpr = " + hashfnExpr);
- exprNodeDesc andExpr = getFuncExprNodeDesc("&", hashfnExpr, intMaxExpr);
+ exprNodeDesc andExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("&", hashfnExpr, intMaxExpr);
assert(andExpr != null);
LOG.info("andExpr = " + andExpr);
- exprNodeDesc modExpr = getFuncExprNodeDesc("%", andExpr, denominatorExpr);
+ exprNodeDesc modExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("%", andExpr, denominatorExpr);
assert(modExpr != null);
LOG.info("modExpr = " + modExpr);
LOG.info("numeratorExpr = " + numeratorExpr);
- exprNodeDesc equalsExpr = getFuncExprNodeDesc("==", modExpr, numeratorExpr);
+ exprNodeDesc equalsExpr = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("==", modExpr, numeratorExpr);
LOG.info("equalsExpr = " + equalsExpr);
assert(equalsExpr != null);
return equalsExpr;
@@ -3159,7 +3176,7 @@
GraphWalker ogw = new GenMapRedWalker(disp);
ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.addAll(this.topOps.values());
- ogw.startWalking(topNodes);
+ ogw.startWalking(topNodes, null);
// reduce sink does not have any kids
breakOperatorTree(procCtx.getRootOps());
@@ -3240,379 +3257,65 @@
return;
}
-
- /**
- * Get the exprNodeDesc
- * @param name
- * @param children
- * @return
- */
- public static exprNodeDesc getFuncExprNodeDesc(String name, exprNodeDesc... children) {
- return getFuncExprNodeDesc(name, Arrays.asList(children));
- }
-
- /**
- * This function create an ExprNodeDesc for a UDF function given the children (arguments).
- * It will insert implicit type conversion functions if necessary.
- * @throws SemanticException
- */
- public static exprNodeDesc getFuncExprNodeDesc(String udfName, List<exprNodeDesc> children) {
- // Find the corresponding method
- ArrayList<Class<?>> argumentClasses = new ArrayList<Class<?>>(children.size());
- for(int i=0; i<children.size(); i++) {
- exprNodeDesc child = children.get(i);
- assert(child != null);
- TypeInfo childTypeInfo = child.getTypeInfo();
- assert(childTypeInfo != null);
-
- // Note: we don't pass the element types of MAP/LIST to UDF.
- // That will work for null test and size but not other more complex functionalities like list slice etc.
- // For those more complex functionalities, we plan to have a ComplexUDF interface which has an evaluate
- // method that accepts a list of objects and a list of objectinspectors.
- switch (childTypeInfo.getCategory()) {
- case PRIMITIVE: {
- argumentClasses.add(childTypeInfo.getPrimitiveClass());
- break;
- }
- case MAP: {
- argumentClasses.add(Map.class);
- break;
- }
- case LIST: {
- argumentClasses.add(List.class);
- break;
- }
- case STRUCT: {
- argumentClasses.add(Object.class);
- break;
- }
- default: {
- // should never happen
- assert(false);
- }
- }
- }
- Method udfMethod = FunctionRegistry.getUDFMethod(udfName, false, argumentClasses);
- if (udfMethod == null) return null;
-
- ArrayList<exprNodeDesc> ch = new ArrayList<exprNodeDesc>();
- Class<?>[] pTypes = udfMethod.getParameterTypes();
-
- for (int i = 0; i < children.size(); i++)
- {
- exprNodeDesc desc = children.get(i);
- Class<?> pType = ObjectInspectorUtils.generalizePrimitive(pTypes[i]);
- if (desc instanceof exprNodeNullDesc) {
- exprNodeConstantDesc newCh = new exprNodeConstantDesc(TypeInfoFactory.getPrimitiveTypeInfo(pType), null);
- ch.add(newCh);
- } else if (pType.isAssignableFrom(argumentClasses.get(i))) {
- // no type conversion needed
- ch.add(desc);
- } else {
- // must be implicit type conversion
- Class<?> from = argumentClasses.get(i);
- Class<?> to = pType;
- assert(FunctionRegistry.implicitConvertable(from, to));
- Method m = FunctionRegistry.getUDFMethod(to.getName(), true, from);
- assert(m != null);
- Class<? extends UDF> c = FunctionRegistry.getUDFClass(to.getName());
- assert(c != null);
-
- // get the conversion method
- ArrayList<exprNodeDesc> conversionArg = new ArrayList<exprNodeDesc>(1);
- conversionArg.add(desc);
- ch.add(new exprNodeFuncDesc(
- TypeInfoFactory.getPrimitiveTypeInfo(pType),
- c, m, conversionArg));
- }
- }
-
- exprNodeFuncDesc desc = new exprNodeFuncDesc(
- TypeInfoFactory.getPrimitiveTypeInfo(udfMethod.getReturnType()),
- FunctionRegistry.getUDFClass(udfName),
- udfMethod, ch);
- return desc;
- }
-
/**
* Generates and expression node descriptor for the expression passed in the arguments. This
* function uses the row resolver and the metadata informatinon that are passed as arguments
* to resolve the column names to internal names.
- * @param qbm The metadata infromation for the query block
* @param expr The expression
* @param input The row resolver
* @return exprNodeDesc
* @throws SemanticException
*/
@SuppressWarnings("nls")
- private exprNodeDesc genExprNodeDesc(QBMetaData qbm, ASTNode expr, RowResolver input)
+ private exprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input)
throws SemanticException {
// We recursively create the exprNodeDesc. Base cases: when we encounter
// a column ref, we convert that into an exprNodeColumnDesc; when we encounter
// a constant, we convert that into an exprNodeConstantDesc. For others we just
// build the exprNodeFuncDesc with recursively built children.
- exprNodeDesc desc = null;
-
// If the current subExpression is pre-calculated, as in Group-By etc.
ColumnInfo colInfo = input.get("", expr.toStringTree());
if (colInfo != null) {
- desc = new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName());
- return desc;
+ return new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName());
}
- // Is this a simple expr node (not a TOK_COLREF or a TOK_FUNCTION or an operator)?
- desc = genSimpleExprNodeDesc(expr);
- if (desc != null) {
- return desc;
- }
+ // Create the walker, the rules dispatcher and the context.
+ TypeCheckCtx tcCtx = new TypeCheckCtx(input);
- int tokType = expr.getType();
- switch (tokType) {
- case HiveParser.TOK_COLREF: {
- desc = genExprNodeDescFromColRef(expr, input);
- break;
- }
-
- default: {
- boolean isFunction = (expr.getType() == HiveParser.TOK_FUNCTION);
-
- // Create all children
- int childrenBegin = (isFunction ? 1 : 0);
- ArrayList<exprNodeDesc> children = new ArrayList<exprNodeDesc>(expr.getChildCount() - childrenBegin);
- for (int ci=childrenBegin; ci<expr.getChildCount(); ci++) {
- children.add(genExprNodeDesc(qbm, (ASTNode)expr.getChild(ci), input));
- }
-
- // Create function desc
- desc = getXpathOrFuncExprNodeDesc(expr, isFunction, children);
- break;
- }
- }
- assert(desc != null);
- return desc;
- }
-
- /**
- * Generates expression node from a TOK_COLREF AST Node
- * @param expr Antrl node
- * @param input row resolver for this col reference
- * @return exprNodeDesc or null if ASTNode is not a TOK_COLREF
- * @throws SemanticException
- */
- private exprNodeDesc genExprNodeDescFromColRef(ASTNode expr, RowResolver input)
- throws SemanticException {
- if(expr.getType() != HiveParser.TOK_COLREF) {
- throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(expr));
- }
- exprNodeDesc desc;
- ColumnInfo colInfo;
- String tabAlias = null;
- String colName = null;
- if (expr.getChildCount() != 1) {
- tabAlias = unescapeIdentifier(expr.getChild(0).getText());
- colName = unescapeIdentifier(expr.getChild(1).getText());
- }
- else {
- colName = unescapeIdentifier(expr.getChild(0).getText());
- }
-
- if (colName == null) {
- throw new SemanticException(ErrorMsg.INVALID_XPATH.getMsg(expr));
- }
-
- colInfo = input.get(tabAlias, colName);
+ // create a walker which walks the tree in a DFS manner while maintaining the operator stack. The dispatcher
+ // generates the plan from the operator tree
+ Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
+ StringBuilder sb = new StringBuilder();
+ Formatter fm = new Formatter(sb);
+ opRules.put(new RuleRegExp("R1", HiveParser.TOK_NULL + "%"), TypeCheckProcFactory.getNullExprProcessor());
+ opRules.put(new RuleRegExp("R2", HiveParser.Number + "%"), TypeCheckProcFactory.getNumExprProcessor());
+ opRules.put(new RuleRegExp("R3", HiveParser.Identifier + "%|" +
+ HiveParser.StringLiteral + "%|" +
+ HiveParser.TOK_CHARSETLITERAL + "%"),
+ TypeCheckProcFactory.getStrExprProcessor());
+ opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|" + HiveParser.KW_FALSE + "%"),
+ TypeCheckProcFactory.getBoolExprProcessor());
+ opRules.put(new RuleRegExp("R4", HiveParser.TOK_COLREF + "%"), TypeCheckProcFactory.getColumnExprProcessor());
- if (colInfo == null && input.getIsExprResolver()) {
- throw new SemanticException(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(expr));
- }
- else if (tabAlias != null && !input.hasTableAlias(tabAlias)) {
- throw new SemanticException(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(expr.getChild(0)));
- } else if (colInfo == null) {
- throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(tabAlias == null? expr.getChild(0) : expr.getChild(1)));
+ // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
+ Dispatcher disp = new DefaultRuleDispatcher(TypeCheckProcFactory.getDefaultExprProcessor(), opRules, tcCtx);
+ GraphWalker ogw = new DefaultGraphWalker(disp);
+
+ // Create a list of topop nodes
+ ArrayList<Node> topNodes = new ArrayList<Node>();
+ topNodes.add(expr);
+ HashMap<Node, Object> nodeOutputs = new HashMap<Node, Object>();
+ ogw.startWalking(topNodes, nodeOutputs);
+ exprNodeDesc desc = (exprNodeDesc)nodeOutputs.get(expr);
+ if (desc == null) {
+ throw new SemanticException(tcCtx.getError());
}
- desc = new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName());
return desc;
}
-
- static HashMap<Integer, String> specialUnaryOperatorTextHashMap;
- static HashMap<Integer, String> specialFunctionTextHashMap;
- static HashMap<Integer, String> conversionFunctionTextHashMap;
- static {
- specialUnaryOperatorTextHashMap = new HashMap<Integer, String>();
- specialUnaryOperatorTextHashMap.put(HiveParser.PLUS, "positive");
- specialUnaryOperatorTextHashMap.put(HiveParser.MINUS, "negative");
- specialFunctionTextHashMap = new HashMap<Integer, String>();
- specialFunctionTextHashMap.put(HiveParser.TOK_ISNULL, "isnull");
- specialFunctionTextHashMap.put(HiveParser.TOK_ISNOTNULL, "isnotnull");
- conversionFunctionTextHashMap = new HashMap<Integer, String>();
- conversionFunctionTextHashMap.put(HiveParser.TOK_BOOLEAN, Boolean.class.getName());
- conversionFunctionTextHashMap.put(HiveParser.TOK_TINYINT, Byte.class.getName());
- conversionFunctionTextHashMap.put(HiveParser.TOK_SMALLINT, Short.class.getName());
- conversionFunctionTextHashMap.put(HiveParser.TOK_INT, Integer.class.getName());
- conversionFunctionTextHashMap.put(HiveParser.TOK_BIGINT, Long.class.getName());
- conversionFunctionTextHashMap.put(HiveParser.TOK_FLOAT, Float.class.getName());
- conversionFunctionTextHashMap.put(HiveParser.TOK_DOUBLE, Double.class.getName());
- conversionFunctionTextHashMap.put(HiveParser.TOK_STRING, String.class.getName());
- conversionFunctionTextHashMap.put(HiveParser.TOK_DATE, java.sql.Date.class.getName());
- }
-
- public static boolean isRedundantConversionFunction(ASTNode expr, boolean isFunction, ArrayList<exprNodeDesc> children) {
- if (!isFunction) return false;
- // children is always one less than the expr.getChildCount(), since the latter contains function name.
- assert(children.size() == expr.getChildCount() - 1);
- // conversion functions take a single parameter
- if (children.size() != 1) return false;
- String funcText = conversionFunctionTextHashMap.get(((ASTNode)expr.getChild(0)).getType());
- // not a conversion function
- if (funcText == null) return false;
- // return true when the child type and the conversion target type is the same
- return children.get(0).getTypeInfo().getPrimitiveClass().getName().equals(funcText);
- }
-
- public static String getFunctionText(ASTNode expr, boolean isFunction) {
- String funcText = null;
- if (!isFunction) {
- // For operator, the function name is the operator text, unless it's in our special dictionary
- if (expr.getChildCount() == 1) {
- funcText = specialUnaryOperatorTextHashMap.get(expr.getType());
- }
- if (funcText == null) {
- funcText = expr.getText();
- }
- } else {
- // For TOK_FUNCTION, the function name is stored in the first child, unless it's in our
- // special dictionary.
- assert(expr.getChildCount() >= 1);
- int funcType = ((ASTNode)expr.getChild(0)).getType();
- funcText = specialFunctionTextHashMap.get(funcType);
- if (funcText == null) {
- funcText = conversionFunctionTextHashMap.get(funcType);
- }
- if (funcText == null) {
- funcText = ((ASTNode)expr.getChild(0)).getText();
- }
- }
- return funcText;
- }
-
- static exprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, boolean isFunction,
- ArrayList<exprNodeDesc> children)
- throws SemanticException {
- // return the child directly if the conversion is redundant.
- if (isRedundantConversionFunction(expr, isFunction, children)) {
- assert(children.size() == 1);
- assert(children.get(0) != null);
- return children.get(0);
- }
- String funcText = getFunctionText(expr, isFunction);
- exprNodeDesc desc;
- if (funcText.equals(".")) {
- // "." : FIELD Expression
- assert(children.size() == 2);
- // Only allow constant field name for now
- assert(children.get(1) instanceof exprNodeConstantDesc);
- exprNodeDesc object = children.get(0);
- exprNodeConstantDesc fieldName = (exprNodeConstantDesc)children.get(1);
- assert(fieldName.getValue() instanceof String);
-
- // Calculate result TypeInfo
- String fieldNameString = (String)fieldName.getValue();
- TypeInfo objectTypeInfo = object.getTypeInfo();
-
- // Allow accessing a field of list element structs directly from a list
- boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST);
- if (isList) {
- objectTypeInfo = objectTypeInfo.getListElementTypeInfo();
- }
- if (objectTypeInfo.getCategory() != Category.STRUCT) {
- throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr));
- }
- TypeInfo t = objectTypeInfo.getStructFieldTypeInfo(fieldNameString);
- if (isList) {
- t = TypeInfoFactory.getListTypeInfo(t);
- }
-
- desc = new exprNodeFieldDesc(t, children.get(0), fieldNameString, isList);
-
- } else if (funcText.equals("[")){
- // "[]" : LSQUARE/INDEX Expression
- assert(children.size() == 2);
-
- // Check whether this is a list or a map
- TypeInfo myt = children.get(0).getTypeInfo();
-
- if (myt.getCategory() == Category.LIST) {
- // Only allow constant integer index for now
- if (!(children.get(1) instanceof exprNodeConstantDesc)
- || !(((exprNodeConstantDesc)children.get(1)).getValue() instanceof Integer)) {
- throw new SemanticException(ErrorMsg.INVALID_ARRAYINDEX_CONSTANT.getMsg(expr));
- }
-
- // Calculate TypeInfo
- TypeInfo t = myt.getListElementTypeInfo();
- desc = new exprNodeIndexDesc(t, children.get(0), children.get(1));
- }
- else if (myt.getCategory() == Category.MAP) {
- // Only allow only constant indexes for now
- if (!(children.get(1) instanceof exprNodeConstantDesc)) {
- throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_CONSTANT.getMsg(expr));
- }
- if (!(((exprNodeConstantDesc)children.get(1)).getValue().getClass() ==
- myt.getMapKeyTypeInfo().getPrimitiveClass())) {
- throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE.getMsg(expr));
- }
- // Calculate TypeInfo
- TypeInfo t = myt.getMapValueTypeInfo();
-
- desc = new exprNodeIndexDesc(t, children.get(0), children.get(1));
- }
- else {
- throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr,
- myt.getTypeName()));
- }
- } else {
- // other operators or functions
- Class<? extends UDF> udf = FunctionRegistry.getUDFClass(funcText);
- if (udf == null) {
- if (isFunction)
- throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)expr.getChild(0)));
- else
- throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)expr));
- }
-
- desc = getFuncExprNodeDesc(funcText, children);
- if (desc == null) {
- ArrayList<Class<?>> argumentClasses = new ArrayList<Class<?>>(children.size());
- for(int i=0; i<children.size(); i++) {
- argumentClasses.add(children.get(i).getTypeInfo().getPrimitiveClass());
- }
- if (isFunction) {
- String reason = "Looking for UDF \"" + expr.getChild(0).getText() + "\" with parameters " + argumentClasses;
- throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)expr.getChild(0), reason));
- } else {
- String reason = "Looking for Operator \"" + expr.getText() + "\" with parameters " + argumentClasses;
- throw new SemanticException(ErrorMsg.INVALID_OPERATOR_SIGNATURE.getMsg(expr, reason));
- }
- }
- }
- // UDFOPPositive is a no-op.
- // However, we still create it, and then remove it here, to make sure we only allow
- // "+" for numeric types.
- if (desc instanceof exprNodeFuncDesc) {
- exprNodeFuncDesc funcDesc = (exprNodeFuncDesc)desc;
- if (funcDesc.getUDFClass().equals(UDFOPPositive.class)) {
- assert(funcDesc.getChildren().size() == 1);
- desc = funcDesc.getChildren().get(0);
- }
- }
- assert(desc != null);
- return desc;
- }
-
static exprNodeDesc genSimpleExprNodeDesc(ASTNode expr) throws SemanticException {
exprNodeDesc desc = null;
switch(expr.getType()) {
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java?rev=737291&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckCtx.java Sat Jan 24 01:58:01 2009
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+
+/**
+ * This class implements the context information that is used for typechecking phase
+ * in query compilation.
+ */
+public class TypeCheckCtx implements NodeProcessorCtx {
+
+ /**
+ * The row resolver of the previous operator. This field is used to generate expression
+ * descriptors from the expression ASTs.
+ */
+ private RowResolver inputRR;
+
+ /**
+ * Potential typecheck error reason.
+ */
+ private String error;
+
+ /**
+ * Constructor.
+ *
+ * @param inputRR The input row resolver of the previous operator.
+ */
+ public TypeCheckCtx(RowResolver inputRR) {
+ this.setInputRR(inputRR);
+ this.error = null;
+ }
+
+ /**
+ * @param inputRR the inputRR to set
+ */
+ public void setInputRR(RowResolver inputRR) {
+ this.inputRR = inputRR;
+ }
+
+ /**
+ * @return the inputRR
+ */
+ public RowResolver getInputRR() {
+ return inputRR;
+ }
+
+ /**
+ * @param error the error to set
+ */
+ public void setError(String error) {
+ this.error = error;
+ }
+
+ /**
+ * @return the error
+ */
+ public String getError() {
+ return error;
+ }
+
+}
Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=737291&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Sat Jan 24 01:58:01 2009
@@ -0,0 +1,628 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeConstantDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeFieldDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeFuncDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeIndexDesc;
+import org.apache.hadoop.hive.ql.plan.exprNodeNullDesc;
+import org.apache.hadoop.hive.ql.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.ql.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.ql.udf.UDFOPPositive;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+/**
+ * The Factory for creating typecheck processors. The typecheck processors are used to
+ * processes the syntax trees for expressions and convert them into expression Node
+ * Descriptor trees. They also introduce the correct conversion functions to do proper
+ * implicit conversion.
+ */
+public class TypeCheckProcFactory {
+
+ /**
+ * Function to do groupby subexpression elimination. This is called by all the processors initially.
+ * As an example, consider the query
+ * select a+b, count(1) from T group by a+b;
+ * Then a+b is already precomputed in the group by operators key, so we substitute a+b in the select
+ * list with the internal column name of the a+b expression that appears in the in input row resolver.
+ *
+ * @param nd The node that is being inspected.
+ * @param procCtx The processor context.
+ *
+ * @return exprNodeColumnDesc.
+ */
+ public static exprNodeDesc processGByExpr(Node nd, Object procCtx)
+ throws SemanticException {
+ // We recursively create the exprNodeDesc. Base cases: when we encounter
+ // a column ref, we convert that into an exprNodeColumnDesc; when we encounter
+ // a constant, we convert that into an exprNodeConstantDesc. For others we just
+ // build the exprNodeFuncDesc with recursively built children.
+ ASTNode expr = (ASTNode)nd;
+ TypeCheckCtx ctx = (TypeCheckCtx) procCtx;
+ RowResolver input = ctx.getInputRR();
+ exprNodeDesc desc = null;
+
+ // If the current subExpression is pre-calculated, as in Group-By etc.
+ ColumnInfo colInfo = input.get("", expr.toStringTree());
+ if (colInfo != null) {
+ desc = new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName());
+ return desc;
+ }
+ return desc;
+ }
+
+ /**
+ * Processor for processing NULL expression.
+ */
+ public static class NullExprProcessor implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+
+ exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+ if (desc != null) {
+ return desc;
+ }
+
+ return new exprNodeNullDesc();
+ }
+
+ }
+
+ /**
+ * Factory method to get NullExprProcessor.
+ * @return NullExprProcessor.
+ */
+ public static NullExprProcessor getNullExprProcessor() {
+ return new NullExprProcessor();
+ }
+
+ /**
+ * Processor for processing numeric constants.
+ */
+ public static class NumExprProcessor implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+
+ exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+ if (desc != null) {
+ return desc;
+ }
+
+ Number v = null;
+ ASTNode expr = (ASTNode)nd;
+ // The expression can be any one of Double, Long and Integer. We
+ // try to parse the expression in that order to ensure that the
+ // most specific type is used for conversion.
+ try {
+ v = Double.valueOf(expr.getText());
+ v = Long.valueOf(expr.getText());
+ v = Integer.valueOf(expr.getText());
+ } catch (NumberFormatException e) {
+ // do nothing here, we will throw an exception in the following block
+ }
+ if (v == null) {
+ throw new SemanticException(ErrorMsg.INVALID_NUMERICAL_CONSTANT.getMsg(expr));
+ }
+ return new exprNodeConstantDesc(v);
+ }
+
+ }
+
+ /**
+ * Factory method to get NumExprProcessor.
+ * @return NumExprProcessor.
+ */
+ public static NumExprProcessor getNumExprProcessor() {
+ return new NumExprProcessor();
+ }
+
+ /**
+ * Processor for processing string constants.
+ */
+ public static class StrExprProcessor implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+
+ exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+ if (desc != null) {
+ return desc;
+ }
+
+ ASTNode expr = (ASTNode)nd;
+ String str = null;
+
+ switch (expr.getToken().getType()) {
+ case HiveParser.Identifier:
+ str = BaseSemanticAnalyzer.unescapeIdentifier(expr.getText());
+ break;
+ case HiveParser.StringLiteral:
+ str = BaseSemanticAnalyzer.unescapeSQLString(expr.getText());
+ break;
+ case HiveParser.TOK_CHARSETLITERAL:
+ str = BaseSemanticAnalyzer.charSetString(expr.getChild(0).getText(), expr.getChild(1).getText());
+ break;
+ default:
+ assert false;
+ }
+ return new exprNodeConstantDesc(String.class, str);
+ }
+
+ }
+
+ /**
+ * Factory method to get StrExprProcessor.
+ * @return StrExprProcessor.
+ */
+ public static StrExprProcessor getStrExprProcessor() {
+ return new StrExprProcessor();
+ }
+
+ /**
+ * Processor for boolean constants.
+ */
+ public static class BoolExprProcessor implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+
+ exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+ if (desc != null) {
+ return desc;
+ }
+
+ ASTNode expr = (ASTNode)nd;
+ Boolean bool = null;
+
+ switch (expr.getToken().getType()) {
+ case HiveParser.KW_TRUE:
+ bool = Boolean.TRUE;
+ break;
+ case HiveParser.KW_FALSE:
+ bool = Boolean.FALSE;
+ break;
+ default:
+ assert false;
+ }
+ return new exprNodeConstantDesc(Boolean.class, bool);
+ }
+
+ }
+
+ /**
+ * Factory method to get BoolExprProcessor.
+ * @return BoolExprProcessor.
+ */
+ public static BoolExprProcessor getBoolExprProcessor() {
+ return new BoolExprProcessor();
+ }
+
+ /**
+ * Processor for table columns
+ */
+ public static class ColumnExprProcessor implements NodeProcessor {
+
+ @Override
+ public Object process(Node nd, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+
+ exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+ if (desc != null) {
+ return desc;
+ }
+
+ ASTNode expr = (ASTNode)nd;
+ TypeCheckCtx ctx = (TypeCheckCtx)procCtx;
+ RowResolver input = ctx.getInputRR();
+
+ if(expr.getType() != HiveParser.TOK_COLREF) {
+ ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr));
+ return null;
+ }
+
+ String tabAlias = null;
+ String colName = null;
+
+ if (expr.getChildCount() != 1) {
+ tabAlias = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText());
+ colName = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(1).getText());
+ }
+ else {
+ colName = BaseSemanticAnalyzer.unescapeIdentifier(expr.getChild(0).getText());
+ }
+
+ if (colName == null) {
+ ctx.setError(ErrorMsg.INVALID_XPATH.getMsg(expr));
+ return null;
+ }
+
+ ColumnInfo colInfo = input.get(tabAlias, colName);
+
+ if (colInfo == null && input.getIsExprResolver()) {
+ ctx.setError(ErrorMsg.NON_KEY_EXPR_IN_GROUPBY.getMsg(expr));
+ return null;
+ }
+ else if (tabAlias != null && !input.hasTableAlias(tabAlias)) {
+ ctx.setError(ErrorMsg.INVALID_TABLE_ALIAS.getMsg(expr.getChild(0)));
+ return null;
+ } else if (colInfo == null) {
+ ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(tabAlias == null? expr.getChild(0) : expr.getChild(1)));
+ return null;
+ }
+
+ return new exprNodeColumnDesc(colInfo.getType(), colInfo.getInternalName());
+ }
+
+ }
+
+ /**
+ * Factory method to get ColumnExprProcessor.
+ * @return ColumnExprProcessor.
+ */
+ public static ColumnExprProcessor getColumnExprProcessor() {
+ return new ColumnExprProcessor();
+ }
+
+ /**
+ * The default processor for typechecking.
+ */
+ public static class DefaultExprProcessor implements NodeProcessor {
+
+ static HashMap<Integer, String> specialUnaryOperatorTextHashMap;
+ static HashMap<Integer, String> specialFunctionTextHashMap;
+ static HashMap<Integer, String> conversionFunctionTextHashMap;
+ static {
+ specialUnaryOperatorTextHashMap = new HashMap<Integer, String>();
+ specialUnaryOperatorTextHashMap.put(HiveParser.PLUS, "positive");
+ specialUnaryOperatorTextHashMap.put(HiveParser.MINUS, "negative");
+ specialFunctionTextHashMap = new HashMap<Integer, String>();
+ specialFunctionTextHashMap.put(HiveParser.TOK_ISNULL, "isnull");
+ specialFunctionTextHashMap.put(HiveParser.TOK_ISNOTNULL, "isnotnull");
+ conversionFunctionTextHashMap = new HashMap<Integer, String>();
+ conversionFunctionTextHashMap.put(HiveParser.TOK_BOOLEAN, Boolean.class.getName());
+ conversionFunctionTextHashMap.put(HiveParser.TOK_TINYINT, Byte.class.getName());
+ conversionFunctionTextHashMap.put(HiveParser.TOK_SMALLINT, Short.class.getName());
+ conversionFunctionTextHashMap.put(HiveParser.TOK_INT, Integer.class.getName());
+ conversionFunctionTextHashMap.put(HiveParser.TOK_BIGINT, Long.class.getName());
+ conversionFunctionTextHashMap.put(HiveParser.TOK_FLOAT, Float.class.getName());
+ conversionFunctionTextHashMap.put(HiveParser.TOK_DOUBLE, Double.class.getName());
+ conversionFunctionTextHashMap.put(HiveParser.TOK_STRING, String.class.getName());
+ conversionFunctionTextHashMap.put(HiveParser.TOK_DATE, java.sql.Date.class.getName());
+ }
+
+ public static boolean isRedundantConversionFunction(ASTNode expr, boolean isFunction, ArrayList<exprNodeDesc> children) {
+ if (!isFunction) return false;
+ // children is always one less than the expr.getChildCount(), since the latter contains function name.
+ assert(children.size() == expr.getChildCount() - 1);
+ // conversion functions take a single parameter
+ if (children.size() != 1) return false;
+ String funcText = conversionFunctionTextHashMap.get(((ASTNode)expr.getChild(0)).getType());
+ // not a conversion function
+ if (funcText == null) return false;
+ // return true when the child type and the conversion target type is the same
+ return children.get(0).getTypeInfo().getPrimitiveClass().getName().equals(funcText);
+ }
+
+ public static String getFunctionText(ASTNode expr, boolean isFunction) {
+ String funcText = null;
+ if (!isFunction) {
+ // For operator, the function name is the operator text, unless it's in our special dictionary
+ if (expr.getChildCount() == 1) {
+ funcText = specialUnaryOperatorTextHashMap.get(expr.getType());
+ }
+ if (funcText == null) {
+ funcText = expr.getText();
+ }
+ } else {
+ // For TOK_FUNCTION, the function name is stored in the first child, unless it's in our
+ // special dictionary.
+ assert(expr.getChildCount() >= 1);
+ int funcType = ((ASTNode)expr.getChild(0)).getType();
+ funcText = specialFunctionTextHashMap.get(funcType);
+ if (funcText == null) {
+ funcText = conversionFunctionTextHashMap.get(funcType);
+ }
+ if (funcText == null) {
+ funcText = ((ASTNode)expr.getChild(0)).getText();
+ }
+ }
+ return funcText;
+ }
+
+
+ /**
+ * Get the exprNodeDesc
+ * @param name
+ * @param children
+ * @return
+ */
+ public static exprNodeDesc getFuncExprNodeDesc(String name, exprNodeDesc... children) {
+ return getFuncExprNodeDesc(name, Arrays.asList(children));
+ }
+
+ /**
+ * This function create an ExprNodeDesc for a UDF function given the children (arguments).
+ * It will insert implicit type conversion functions if necessary.
+ * @throws SemanticException
+ */
+ public static exprNodeDesc getFuncExprNodeDesc(String udfName, List<exprNodeDesc> children) {
+ // Find the corresponding method
+ ArrayList<Class<?>> argumentClasses = new ArrayList<Class<?>>(children.size());
+ for(int i=0; i<children.size(); i++) {
+ exprNodeDesc child = children.get(i);
+ assert(child != null);
+ TypeInfo childTypeInfo = child.getTypeInfo();
+ assert(childTypeInfo != null);
+
+ // Note: we don't pass the element types of MAP/LIST to UDF.
+ // That will work for null test and size but not other more complex functionalities like list slice etc.
+ // For those more complex functionalities, we plan to have a ComplexUDF interface which has an evaluate
+ // method that accepts a list of objects and a list of objectinspectors.
+ switch (childTypeInfo.getCategory()) {
+ case PRIMITIVE: {
+ argumentClasses.add(childTypeInfo.getPrimitiveClass());
+ break;
+ }
+ case MAP: {
+ argumentClasses.add(Map.class);
+ break;
+ }
+ case LIST: {
+ argumentClasses.add(List.class);
+ break;
+ }
+ case STRUCT: {
+ argumentClasses.add(Object.class);
+ break;
+ }
+ default: {
+ // should never happen
+ assert(false);
+ }
+ }
+ }
+ Method udfMethod = FunctionRegistry.getUDFMethod(udfName, argumentClasses);
+ if (udfMethod == null) return null;
+
+ ArrayList<exprNodeDesc> ch = new ArrayList<exprNodeDesc>();
+ Class<?>[] pTypes = udfMethod.getParameterTypes();
+
+ for (int i = 0; i < children.size(); i++)
+ {
+ exprNodeDesc desc = children.get(i);
+ Class<?> pType = ObjectInspectorUtils.generalizePrimitive(pTypes[i]);
+ if (desc instanceof exprNodeNullDesc) {
+ exprNodeConstantDesc newCh = new exprNodeConstantDesc(TypeInfoFactory.getPrimitiveTypeInfo(pType), null);
+ ch.add(newCh);
+ } else if (pType.isAssignableFrom(argumentClasses.get(i))) {
+ // no type conversion needed
+ ch.add(desc);
+ } else {
+ // must be implicit type conversion
+ Class<?> from = argumentClasses.get(i);
+ Class<?> to = pType;
+ assert(FunctionRegistry.implicitConvertable(from, to));
+ Method m = FunctionRegistry.getUDFMethod(to.getName(), from);
+ assert(m != null);
+ Class<? extends UDF> c = FunctionRegistry.getUDFClass(to.getName());
+ assert(c != null);
+
+ // get the conversion method
+ ArrayList<exprNodeDesc> conversionArg = new ArrayList<exprNodeDesc>(1);
+ conversionArg.add(desc);
+ ch.add(new exprNodeFuncDesc(
+ TypeInfoFactory.getPrimitiveTypeInfo(pType),
+ c, m, conversionArg));
+ }
+ }
+
+ exprNodeFuncDesc desc = new exprNodeFuncDesc(
+ TypeInfoFactory.getPrimitiveTypeInfo(udfMethod.getReturnType()),
+ FunctionRegistry.getUDFClass(udfName),
+ udfMethod, ch);
+ return desc;
+ }
+
+ static exprNodeDesc getXpathOrFuncExprNodeDesc(ASTNode expr, boolean isFunction,
+ ArrayList<exprNodeDesc> children)
+ throws SemanticException {
+ // return the child directly if the conversion is redundant.
+ if (isRedundantConversionFunction(expr, isFunction, children)) {
+ assert(children.size() == 1);
+ assert(children.get(0) != null);
+ return children.get(0);
+ }
+ String funcText = getFunctionText(expr, isFunction);
+ exprNodeDesc desc;
+ if (funcText.equals(".")) {
+ // "." : FIELD Expression
+ assert(children.size() == 2);
+ // Only allow constant field name for now
+ assert(children.get(1) instanceof exprNodeConstantDesc);
+ exprNodeDesc object = children.get(0);
+ exprNodeConstantDesc fieldName = (exprNodeConstantDesc)children.get(1);
+ assert(fieldName.getValue() instanceof String);
+
+ // Calculate result TypeInfo
+ String fieldNameString = (String)fieldName.getValue();
+ TypeInfo objectTypeInfo = object.getTypeInfo();
+
+ // Allow accessing a field of list element structs directly from a list
+ boolean isList = (object.getTypeInfo().getCategory() == ObjectInspector.Category.LIST);
+ if (isList) {
+ objectTypeInfo = objectTypeInfo.getListElementTypeInfo();
+ }
+ if (objectTypeInfo.getCategory() != Category.STRUCT) {
+ throw new SemanticException(ErrorMsg.INVALID_DOT.getMsg(expr));
+ }
+ TypeInfo t = objectTypeInfo.getStructFieldTypeInfo(fieldNameString);
+ if (isList) {
+ t = TypeInfoFactory.getListTypeInfo(t);
+ }
+
+ desc = new exprNodeFieldDesc(t, children.get(0), fieldNameString, isList);
+
+ } else if (funcText.equals("[")){
+ // "[]" : LSQUARE/INDEX Expression
+ assert(children.size() == 2);
+
+ // Check whether this is a list or a map
+ TypeInfo myt = children.get(0).getTypeInfo();
+
+ if (myt.getCategory() == Category.LIST) {
+ // Only allow constant integer index for now
+ if (!(children.get(1) instanceof exprNodeConstantDesc)
+ || !(((exprNodeConstantDesc)children.get(1)).getValue() instanceof Integer)) {
+ throw new SemanticException(ErrorMsg.INVALID_ARRAYINDEX_CONSTANT.getMsg(expr));
+ }
+
+ // Calculate TypeInfo
+ TypeInfo t = myt.getListElementTypeInfo();
+ desc = new exprNodeIndexDesc(t, children.get(0), children.get(1));
+ }
+ else if (myt.getCategory() == Category.MAP) {
+ // Only allow only constant indexes for now
+ if (!(children.get(1) instanceof exprNodeConstantDesc)) {
+ throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_CONSTANT.getMsg(expr));
+ }
+ if (!(((exprNodeConstantDesc)children.get(1)).getValue().getClass() ==
+ myt.getMapKeyTypeInfo().getPrimitiveClass())) {
+ throw new SemanticException(ErrorMsg.INVALID_MAPINDEX_TYPE.getMsg(expr));
+ }
+ // Calculate TypeInfo
+ TypeInfo t = myt.getMapValueTypeInfo();
+
+ desc = new exprNodeIndexDesc(t, children.get(0), children.get(1));
+ }
+ else {
+ throw new SemanticException(ErrorMsg.NON_COLLECTION_TYPE.getMsg(expr,
+ myt.getTypeName()));
+ }
+ } else {
+ // other operators or functions
+ Class<? extends UDF> udf = FunctionRegistry.getUDFClass(funcText);
+ if (udf == null) {
+ if (isFunction)
+ throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)expr.getChild(0)));
+ else
+ throw new SemanticException(ErrorMsg.INVALID_FUNCTION.getMsg((ASTNode)expr));
+ }
+
+ desc = getFuncExprNodeDesc(funcText, children);
+ if (desc == null) {
+ ArrayList<Class<?>> argumentClasses = new ArrayList<Class<?>>(children.size());
+ for(int i=0; i<children.size(); i++) {
+ argumentClasses.add(children.get(i).getTypeInfo().getPrimitiveClass());
+ }
+
+ if (isFunction) {
+ String reason = "Looking for UDF \"" + expr.getChild(0).getText() + "\" with parameters " + argumentClasses;
+ throw new SemanticException(ErrorMsg.INVALID_FUNCTION_SIGNATURE.getMsg((ASTNode)expr.getChild(0), reason));
+ } else {
+ String reason = "Looking for Operator \"" + expr.getText() + "\" with parameters " + argumentClasses;
+ throw new SemanticException(ErrorMsg.INVALID_OPERATOR_SIGNATURE.getMsg(expr, reason));
+ }
+ }
+ }
+ // UDFOPPositive is a no-op.
+ // However, we still create it, and then remove it here, to make sure we only allow
+ // "+" for numeric types.
+ if (desc instanceof exprNodeFuncDesc) {
+ exprNodeFuncDesc funcDesc = (exprNodeFuncDesc)desc;
+ if (funcDesc.getUDFClass().equals(UDFOPPositive.class)) {
+ assert(funcDesc.getChildren().size() == 1);
+ desc = funcDesc.getChildren().get(0);
+ }
+ }
+ assert(desc != null);
+ return desc;
+ }
+
+ @Override
+ public Object process(Node nd, NodeProcessorCtx procCtx,
+ Object... nodeOutputs) throws SemanticException {
+
+ ASTNode expr = (ASTNode)nd;
+
+ // Return nulls for conversion operators
+ if (conversionFunctionTextHashMap.keySet().contains(expr.getType()) ||
+ specialFunctionTextHashMap.keySet().contains(expr.getType()) ||
+ expr.getToken().getType() == HiveParser.CharSetName ||
+ expr.getToken().getType() == HiveParser.CharSetLiteral) {
+ return null;
+ }
+
+ exprNodeDesc desc = TypeCheckProcFactory.processGByExpr(nd, procCtx);
+ if (desc != null) {
+ return desc;
+ }
+
+ boolean isFunction = (expr.getType() == HiveParser.TOK_FUNCTION);
+
+ // Create all children
+ int childrenBegin = (isFunction ? 1 : 0);
+ ArrayList<exprNodeDesc> children = new ArrayList<exprNodeDesc>(expr.getChildCount() - childrenBegin);
+ for (int ci=childrenBegin; ci<expr.getChildCount(); ci++) {
+ children.add((exprNodeDesc)nodeOutputs[ci]);
+ }
+
+ // If any of the children contains null, then return a null
+ // this is a hack for now to handle the group by case
+ if (children.contains(null)) {
+ return null;
+ }
+
+ // Create function desc
+ return getXpathOrFuncExprNodeDesc(expr, isFunction, children);
+ }
+
+ }
+
+ /**
+ * Factory method to get DefaultExprProcessor.
+ * @return DefaultExprProcessor.
+ */
+ public static DefaultExprProcessor getDefaultExprProcessor() {
+ return new DefaultExprProcessor();
+ }
+}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Sat Jan 24 01:58:01 2009
@@ -29,6 +29,7 @@
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
import org.apache.hadoop.hive.ql.typeinfo.TypeInfoUtils;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
@@ -222,7 +223,7 @@
} else {
// numPartitionFields = -1 means random partitioning
partitionCols = new ArrayList<exprNodeDesc>(1);
- partitionCols.add(SemanticAnalyzer.getFuncExprNodeDesc("rand"));
+ partitionCols.add(TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("rand"));
}
StringBuilder order = new StringBuilder();
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/aggregationDesc.java Sat Jan 24 01:58:01 2009
@@ -21,25 +21,26 @@
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.UDAF;
+import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
public class aggregationDesc implements java.io.Serializable {
private static final long serialVersionUID = 1L;
- private Class<? extends UDAF> aggregationClass;
+ private Class<? extends UDAFEvaluator> aggregationClass;
private java.util.ArrayList<exprNodeDesc> parameters;
private boolean distinct;
public aggregationDesc() {}
public aggregationDesc(
- final Class<? extends UDAF> aggregationClass,
+ final Class<? extends UDAFEvaluator> aggregationClass,
final java.util.ArrayList<exprNodeDesc> parameters,
final boolean distinct) {
this.aggregationClass = aggregationClass;
this.parameters = parameters;
this.distinct = distinct;
}
- public Class<? extends UDAF> getAggregationClass() {
+ public Class<? extends UDAFEvaluator> getAggregationClass() {
return this.aggregationClass;
}
- public void setAggregationClass(final Class<? extends UDAF> aggregationClass) {
+ public void setAggregationClass(final Class<? extends UDAFEvaluator> aggregationClass) {
this.aggregationClass = aggregationClass;
}
public java.util.ArrayList<exprNodeDesc> getParameters() {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeColumnDesc.java Sat Jan 24 01:58:01 2009
@@ -24,8 +24,6 @@
import org.apache.hadoop.hive.ql.typeinfo.TypeInfo;
import org.apache.hadoop.hive.ql.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.hive.ql.parse.RowResolver;
-import org.apache.hadoop.hive.ql.exec.ColumnInfo;
public class exprNodeColumnDesc extends exprNodeDesc implements Serializable {
private static final long serialVersionUID = 1L;
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeDesc.java Sat Jan 24 01:58:01 2009
@@ -22,8 +22,6 @@
import java.util.List;
import org.apache.hadoop.hive.ql.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
-import org.apache.hadoop.hive.ql.parse.RowResolver;
public class exprNodeDesc implements Serializable {
private static final long serialVersionUID = 1L;
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/exprNodeFuncDesc.java Sat Jan 24 01:58:01 2009
@@ -23,13 +23,11 @@
import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.typeinfo.TypeInfo;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.parse.RowResolver;
/**
* The reason that we have to store UDFClass as well as UDFMethod is because
@@ -39,12 +37,13 @@
public class exprNodeFuncDesc extends exprNodeDesc implements Serializable {
private static final long serialVersionUID = 1L;
- private Class UDFClass;
+ private Class<? extends UDF> UDFClass;
private Method UDFMethod;
private ArrayList<exprNodeDesc> children;
public exprNodeFuncDesc() {}
- public exprNodeFuncDesc(TypeInfo typeInfo, Class UDFClass, Method UDFMethod, ArrayList<exprNodeDesc> children) {
+ public exprNodeFuncDesc(TypeInfo typeInfo, Class<? extends UDF> UDFClass,
+ Method UDFMethod, ArrayList<exprNodeDesc> children) {
super(typeInfo);
assert(UDFClass != null);
this.UDFClass = UDFClass;
@@ -53,10 +52,11 @@
this.children = children;
}
- public Class getUDFClass() {
+ public Class<? extends UDF> getUDFClass() {
return UDFClass;
}
- public void setUDFClass(Class UDFClass) {
+
+ public void setUDFClass(Class<? extends UDF> UDFClass) {
this.UDFClass = UDFClass;
}
public Method getUDFMethod() {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java Sat Jan 24 01:58:01 2009
@@ -30,8 +30,8 @@
import org.apache.hadoop.hive.ql.lib.Dispatcher;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
-import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.lib.GraphWalker;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
@@ -76,7 +76,7 @@
/**
* Implements the process method for the NodeProcessor interface.
*/
- public void process(Node nd, NodeProcessorCtx procCtx)
+ public Object process(Node nd, NodeProcessorCtx procCtx, Object... nodeOutputs)
throws SemanticException {
ASTNode pt = (ASTNode)nd;
@@ -91,7 +91,7 @@
inputTableList.add(table_name);
break;
}
-
+ return null;
}
/**
@@ -128,7 +128,7 @@
// Create a list of topop nodes
ArrayList<Node> topNodes = new ArrayList<Node>();
topNodes.add(tree);
- ogw.startWalking(topNodes);
+ ogw.startWalking(topNodes, null);
}
public static void main(String[] args) throws IOException, ParseException,
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFAvg.java Sat Jan 24 01:58:01 2009
@@ -18,11 +18,12 @@
package org.apache.hadoop.hive.ql.udf;
-import org.apache.hadoop.hive.ql.exec.UDAF;
+import org.apache.hadoop.hive.ql.exec.NumericUDAF;
+import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
-public class UDAFAvg extends UDAF {
+public class UDAFAvg extends NumericUDAF implements UDAFEvaluator {
private long mCount;
private double mSum;
@@ -37,7 +38,7 @@
mCount = 0;
}
- public boolean aggregate(Double o) {
+ public boolean iterate(Double o) {
if (o != null) {
mSum += o;
mCount ++;
@@ -45,12 +46,12 @@
return true;
}
- public String evaluatePartial() {
+ public String terminatePartial() {
// This is SQL standard - average of zero items should be null.
return mCount == 0 ? null : String.valueOf(mSum) + '/' + String.valueOf(mCount);
}
- public boolean aggregatePartial(String o) {
+ public boolean merge(String o) {
if (o != null && !o.isEmpty()) {
int pos = o.indexOf('/');
assert(pos != -1);
@@ -60,7 +61,7 @@
return true;
}
- public Double evaluate() {
+ public Double terminate() {
// This is SQL standard - average of zero items should be null.
return mCount == 0 ? null : Double.valueOf(mSum / mCount);
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java?rev=737291&r1=737290&r2=737291&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDAFCount.java Sat Jan 24 01:58:01 2009
@@ -19,9 +19,10 @@
package org.apache.hadoop.hive.ql.udf;
import org.apache.hadoop.hive.ql.exec.UDAF;
+import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
-public class UDAFCount extends UDAF {
+public class UDAFCount extends UDAF implements UDAFEvaluator {
private long mCount;
@@ -34,7 +35,7 @@
mCount = 0;
}
- public boolean aggregate(Object o) {
+ public boolean iterate(Object o) {
// Our SerDe between map/reduce boundary may convert MetadataTypedSerDe to
if (o != null && !o.equals("")) {
mCount ++;
@@ -42,18 +43,17 @@
return true;
}
- public Long evaluatePartial() {
+ public Long terminatePartial() {
return Long.valueOf(mCount);
}
- public boolean aggregatePartial(Long count) {
+ public boolean merge(Long count) {
mCount += count;
return true;
}
- public Long evaluate() {
+ public Long terminate() {
return Long.valueOf(mCount);
}
-
}