You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/05/30 18:42:00 UTC
svn commit: r661785 - in /incubator/pig/branches/types:
src/org/apache/pig/data/ src/org/apache/pig/impl/logicalLayer/parser/
src/org/apache/pig/impl/logicalLayer/validators/ test/org/apache/pig/test/
Author: gates
Date: Fri May 30 09:41:59 2008
New Revision: 661785
URL: http://svn.apache.org/viewvc?rev=661785&view=rev
Log:
PIG-158 Santhosh's udf_fix patch.
Modified:
incubator/pig/branches/types/src/org/apache/pig/data/DataType.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
incubator/pig/branches/types/test/org/apache/pig/test/TestSchema.java
Modified: incubator/pig/branches/types/src/org/apache/pig/data/DataType.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/data/DataType.java?rev=661785&r1=661784&r2=661785&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/data/DataType.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/data/DataType.java Fri May 30 09:41:59 2008
@@ -103,7 +103,17 @@
// implements
if (t instanceof Class) {
Class c = (Class)t;
- Class[] interfaces = c.getInterfaces();
+ Class[] ioeInterfaces = c.getInterfaces();
+ Class[] interfaces = null;
+ if(c.isInterface()){
+ interfaces = new Class[ioeInterfaces.length+1];
+ interfaces[0] = c;
+ for (int i = 1; i < interfaces.length; i++) {
+ interfaces[i] = ioeInterfaces[i-1];
+ }
+ } else {
+ interfaces = ioeInterfaces;
+ }
for (int i = 0; i < interfaces.length; i++) {
if (interfaces[i].getName().equals("org.apache.pig.data.Tuple")) {
return TUPLE;
@@ -851,15 +861,11 @@
}
// One is bytearray and the other is (number or chararray)
- if ( (type1 == DataType.BYTEARRAY) &&
- ( (type2 == DataType.CHARARRAY) || (DataType.isNumberType(type2)) )
- ) {
+ if (type1 == DataType.BYTEARRAY) {
return type2 ;
}
- if ( (type2 == DataType.BYTEARRAY) &&
- ( (type1 == DataType.CHARARRAY) || (DataType.isNumberType(type1)) )
- ) {
+ if (type2 == DataType.BYTEARRAY) {
return type1 ;
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=661785&r1=661784&r2=661785&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Fri May 30 09:41:59 2008
@@ -817,7 +817,7 @@
Token t1;
String funcName;
List<ExpressionOperator> args;
- byte type = DataType.BOOLEAN;
+ EvalFunc evalFunc = null;
log.trace("Entering PUnaryCond");
}
{
@@ -917,9 +917,14 @@
log.debug("PUnaryCond: Connected operator " + cond.getClass().getName() + " " + cond + " to " + lhs + " logical plan " + lp);
}
)
-| LOOKAHEAD(EvalFunction(type) "(")
- (funcName=EvalFunction(type) "(" args=EvalArgs(over,specs,lp,input) ")"
+| LOOKAHEAD(EvalFunction() "(")
+ (evalFunc=EvalFunction() "(" args=EvalArgs(over,specs,lp,input) ")"
{
+ funcName = evalFunc.getClass().getName();
+ Type javaType = evalFunc.getReturnType();
+ byte type = DataType.findType(javaType);
+
+ log.debug("Return type of UDF: " + DataType.findTypeName(type));
cond = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcName, args, type);
lp.add(cond);
log.debug("PUnaryCond: Added operator " + cond.getClass().getName() + " " + cond + " to logical plan " + lp);
@@ -1778,14 +1783,19 @@
ExpressionOperator FuncEvalSpec(Schema over, Map<String, LogicalOperator> specs, LogicalPlan lp, LogicalOperator input) :
{
- String funcName;
+ String funcName = null;
List<ExpressionOperator> args;
ExpressionOperator userFunc;
- byte type = DataType.BYTEARRAY;
+ LOUserFunc userAliasFunc = null;
+ EvalFunc evalFunc = null;
log.trace("Entering FuncEvalSpec");
}
{
- funcName=EvalFunction(type) "(" args=EvalArgs(over,specs,lp,input) ")"
+ (
+ LOOKAHEAD({getDefineOp(getToken(1).image) != null}) funcName=QualifiedFunction()
+| evalFunc=EvalFunction()
+ )
+ "(" args=EvalArgs(over,specs,lp,input) ")"
{
//check if the function name is an alias
//if the user has defined an alias then
@@ -1793,13 +1803,15 @@
//and create a new LOUserFunc with the information
//from the lookup table
- LOUserFunc userAliasFunc = (LOUserFunc)getDefineOp(funcName);
-
- if(null == userAliasFunc) {
-
+ if(null != evalFunc) {
+ funcName = evalFunc.getClass().getName();
+ Type javaType = evalFunc.getReturnType();
+ byte type = DataType.findType(javaType);
+ log.debug("Return type of UDF: " + DataType.findTypeName(type));
userFunc = new LOUserFunc(lp, new OperatorKey(scope, getNextId()), funcName, args, type);
} else {
//we have an alias
+ userAliasFunc = (LOUserFunc)getDefineOp(funcName);
int expectedNumArgs = userAliasFunc.getArguments().size();
int numArgs = args.size();
if(expectedNumArgs != numArgs) {
@@ -1864,12 +1876,16 @@
List<ExpressionOperator> args;
byte userSpecifiedType = DataType.BYTEARRAY;
boolean userSpecified = false;
- byte type = DataType.BYTEARRAY;
+ EvalFunc evalFunc = null;
log.trace("Entering FuncDeclareSpec");
}
{
- [userSpecifiedType = Type() {userSpecified = true;}] funcName=EvalFunction(type) "(" args=FuncDeclareArgs(lp) ")"
+ [userSpecifiedType = Type() {userSpecified = true;}] evalFunc=EvalFunction() "(" args=FuncDeclareArgs(lp) ")"
{
+ funcName = evalFunc.getClass().getName();
+ Type javaType = evalFunc.getReturnType();
+ byte type = DataType.findType(javaType);
+ log.debug("Return type of UDF: " + DataType.findTypeName(type));
if(userSpecified && (userSpecifiedType != type)) {
throw new ParseException("User specified return type " + DataType.findTypeName(userSpecifiedType) + " and deduced return type " + DataType.findTypeName(type) + " are not the same");
}
@@ -2096,30 +2112,26 @@
// These the simple non-terminals that are shared across many
-String EvalFunction(byte type) :
+EvalFunc EvalFunction() :
{
String funcName;
+ EvalFunc ef;
log.trace("Entering EvalFunction");
}
{
funcName = QualifiedFunction()
{
- ExpressionOperator userFunc = getDefineOp(funcName);
- if(null == userFunc) {
- //TODO
- //Commented out the code for instaniateFunc as it's failing
- try{
- EvalFunc ef = (EvalFunc) pigContext.instantiateFuncFromAlias(funcName);
- Type javaType = ((EvalFunc)ef).getReturnType();
- log.debug("Type: " + javaType);
- type = DataType.findType(javaType);
- }catch (Exception e){
- throw new ParseException(e.getMessage());
- }
+ try{
+ ef = (EvalFunc) pigContext.instantiateFuncFromAlias(funcName);
+ Type javaType = ef.getReturnType();
+ log.debug("Type: " + javaType);
+ log.debug("funcName: " + funcName + " class name: " + ef.getClass().getName() + " return type: " + DataType.findType(javaType));
+ }catch (Exception e){
+ throw new ParseException(e.getMessage());
}
log.trace("Exiting EvalFunction");
- return funcName;
+ return ef;
}
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=661785&r1=661784&r2=661785&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java Fri May 30 09:41:59 2008
@@ -1975,10 +1975,16 @@
// flattening, we unfold field by field
if (flattens.get(i)) {
Schema innerSchema = leaf.getSchema() ;
- for (int j=0;j < innerSchema.size(); j++) {
- FieldSchema fs = innerSchema.getField(j) ;
- schema.getField(outputSchemaIdx).type = fs.type ;
- schema.getField(outputSchemaIdx).schema = fs.schema ;
+ if(null != innerSchema) {
+ for (int j=0;j < innerSchema.size(); j++) {
+ FieldSchema fs = innerSchema.getField(j) ;
+ schema.getField(outputSchemaIdx).type = fs.type ;
+ schema.getField(outputSchemaIdx).schema = fs.schema ;
+ outputSchemaIdx++ ;
+ }
+ } else {
+ schema.getField(outputSchemaIdx).type = leaf.getType() ;
+ schema.getField(outputSchemaIdx).schema = leaf.getSchema() ;
outputSchemaIdx++ ;
}
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestSchema.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestSchema.java?rev=661785&r1=661784&r2=661785&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestSchema.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestSchema.java Fri May 30 09:41:59 2008
@@ -458,7 +458,7 @@
// Generate expected schema
List<FieldSchema> expectedList = new ArrayList<FieldSchema>() ;
expectedList.add(new FieldSchema("2a", DataType.BYTEARRAY)) ;
- expectedList.add(new FieldSchema("2b", DataType.BYTEARRAY)) ;
+ expectedList.add(new FieldSchema("2b", DataType.TUPLE)) ;
expectedList.add(new FieldSchema("2c", DataType.BYTEARRAY)) ;
Schema expected = new Schema(expectedList) ;