You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by xu...@apache.org on 2011/05/05 21:25:09 UTC
svn commit: r1099930 [1/2] - in /pig/branches/branch-0.9:
src/org/apache/pig/impl/logicalLayer/schema/ src/org/apache/pig/impl/util/
src/org/apache/pig/newplan/logical/
src/org/apache/pig/newplan/logical/relational/ src/org/apache/pig/parser/
src/org/a...
Author: xuefu
Date: Thu May 5 19:25:08 2011
New Revision: 1099930
URL: http://svn.apache.org/viewvc?rev=1099930&view=rev
Log:
PIG-1775: Removal of old logical plan
Modified:
pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
pig/branches/branch-0.9/src/org/apache/pig/impl/util/Utils.java
pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/Util.java
pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java
pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g
pig/branches/branch-0.9/src/org/apache/pig/parser/AstPrinter.g
pig/branches/branch-0.9/src/org/apache/pig/parser/AstValidator.g
pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanGenerator.g
pig/branches/branch-0.9/src/org/apache/pig/parser/QueryLexer.g
pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g
pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserDriver.java
pig/branches/branch-0.9/src/org/apache/pig/scripting/js/JsFunction.java
pig/branches/branch-0.9/src/org/apache/pig/scripting/jython/JythonFunction.java
pig/branches/branch-0.9/src/org/apache/pig/scripting/jython/JythonScriptEngine.java
pig/branches/branch-0.9/test/org/apache/pig/parser/TestLogicalPlanGenerator.java
pig/branches/branch-0.9/test/org/apache/pig/parser/TestQueryLexer.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipelineLocal.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestLogicalPlanBuilder.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestMultiQuery.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestPartitionFilterPushDown.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestPigServer.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectRange.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectStarExpander.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestSchema.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptUDF.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestStore.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestUnionOnSchema.java
pig/branches/branch-0.9/test/org/apache/pig/test/Util.java
Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/schema/Schema.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/schema/Schema.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/logicalLayer/schema/Schema.java Thu May 5 19:25:08 2011
@@ -1074,10 +1074,7 @@ public class Schema implements Serializa
}
// TODO: Map Support
- if (schema == null) {
- sb.append("null") ;
- }
- else {
+ if (schema != null) {
boolean isFirst = true ;
for (int i=0; i< schema.size() ;i++) {
@@ -1091,7 +1088,6 @@ public class Schema implements Serializa
FieldSchema fs = schema.getField(i) ;
if(fs == null) {
- sb.append("null");
continue;
}
Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/util/Utils.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/util/Utils.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/util/Utils.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/util/Utils.java Thu May 5 19:25:08 2011
@@ -20,6 +20,7 @@ package org.apache.pig.impl.util;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -38,9 +39,11 @@ import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.InterStorage;
import org.apache.pig.impl.io.ReadToEndLoader;
import org.apache.pig.impl.io.TFileStorage;
-import org.apache.pig.impl.logicalLayer.parser.ParseException;
-import org.apache.pig.impl.logicalLayer.parser.QueryParser;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.newplan.logical.Util;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.parser.ParserException;
+import org.apache.pig.parser.QueryParserDriver;
import com.google.common.collect.Lists;
@@ -154,18 +157,26 @@ public class Utils {
return new ResourceSchema(s);
}
- public static Schema getSchemaFromString(String schemaString) throws ParseException {
- return Utils.getSchemaFromString(schemaString, DataType.BYTEARRAY);
- }
-
- public static Schema getSchemaFromString(String schemaString, byte defaultType) throws ParseException {
- ByteArrayInputStream stream = new ByteArrayInputStream(schemaString.getBytes()) ;
- QueryParser queryParser = new QueryParser(stream) ;
- Schema schema = queryParser.TupleSchema() ;
- Schema.setSchemaDefaultType(schema, defaultType);
- return schema;
+ /**
+ * @deprecated Use parseSchema() instead.
+ * @param schemaString
+ * @return Schema instance
+ * @throws ParserException
+ */
+ public static Schema getSchemaFromString(String schemaString) throws ParserException {
+ LogicalSchema schema = parseSchema(schemaString);
+ Schema result = org.apache.pig.newplan.logical.Util.translateSchema(schema);
+ Schema.setSchemaDefaultType(result, DataType.BYTEARRAY);
+ return result;
}
+ public static LogicalSchema parseSchema(String schemaString) throws ParserException {
+ QueryParserDriver queryParser = new QueryParserDriver( new PigContext(),
+ "util", new HashMap<String, String>() ) ;
+ LogicalSchema schema = queryParser.parseSchema(schemaString);
+ return schema;
+ }
+
public static String getTmpFileCompressorName(PigContext pigContext) {
if (pigContext == null)
return InterStorage.class.getName();
Modified: pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/Util.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/Util.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/Util.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/Util.java Thu May 5 19:25:08 2011
@@ -21,11 +21,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Set;
-import org.apache.pig.data.DataType;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
-import org.apache.pig.impl.util.Pair;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
Modified: pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java Thu May 5 19:25:08 2011
@@ -103,7 +103,7 @@ public class LogicalSchema {
return ( aliasToPrint + uidString + ":bag{" + schema.toString(verbose) + "}" );
} else if( type == DataType.TUPLE ) {
if( schema == null ) {
- return ( aliasToPrint + uidString + ":tuple{}" );
+ return ( aliasToPrint + uidString + ":tuple()" );
}
return ( aliasToPrint + uidString + ":tuple(" + schema.toString(verbose) + ")" );
} else if (type == DataType.MAP) {
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g Thu May 5 19:25:08 2011
@@ -325,11 +325,7 @@ col_alias_or_index : col_alias | col_ind
col_alias
: GROUP
- | scoped_col_alias
-;
-
-scoped_col_alias
- : ^( SCOPED_ALIAS IDENTIFIER+ )
+ | IDENTIFIER
;
col_index
@@ -488,15 +484,19 @@ col_ref : alias_col_ref | dollar_col_ref
alias_col_ref
: GROUP
- | scoped_alias_col_ref
-;
-
-scoped_alias_col_ref
- : ^( SCOPED_ALIAS (name=IDENTIFIER {
- if (aliasSeen.contains($name.text)) {
- $name.getToken().setText(getMask($name.text));
- }
- } )+ )
+ | IDENTIFIER
+ {
+ String alias = $IDENTIFIER.text;
+ String[] names = alias.split( "::" );
+ StringBuilder sb = new StringBuilder();
+ for( int i = 0; i < names.length; i++ ) {
+ String name = names[i];
+ sb.append( aliasSeen.contains( name ) ? getMask( name ) : name );
+ if( i < names.length - 1 )
+ sb.append( "::" );
+ }
+ $IDENTIFIER.token.setText( sb.toString() );
+ }
;
dollar_col_ref
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/AstPrinter.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/AstPrinter.g?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/AstPrinter.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/AstPrinter.g Thu May 5 19:25:08 2011
@@ -295,14 +295,7 @@ col_alias_or_index : col_alias | col_ind
col_alias
: GROUP { sb.append($GROUP.text); }
- | scoped_col_alias
-;
-
-scoped_col_alias
- : ^( SCOPED_ALIAS a=IDENTIFIER {
- sb.append($a.text);
- }
- (b=IDENTIFIER { sb.append("::").append($b.text); })* )
+ | IDENTIFIER { sb.append($IDENTIFIER.text); }
;
col_index
@@ -471,15 +464,7 @@ col_ref : alias_col_ref | dollar_col_ref
alias_col_ref
: GROUP { sb.append($GROUP.text); }
- | scoped_alias_col_ref
-;
-
-scoped_alias_col_ref
- : ^( SCOPED_ALIAS name=IDENTIFIER {
- sb.append($name.text);
- }
- (name1=IDENTIFIER { sb.append("::").append($name1.text); }
- )* )
+ | IDENTIFIER { sb.append($IDENTIFIER.text); }
;
dollar_col_ref
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/AstValidator.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/AstValidator.g?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/AstValidator.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/AstValidator.g Thu May 5 19:25:08 2011
@@ -315,10 +315,7 @@ dot_proj : ^( PERIOD col_alias_or_index+
col_alias_or_index : col_alias | col_index
;
-col_alias : GROUP | scoped_col_alias
-;
-
-scoped_col_alias : ^( SCOPED_ALIAS IDENTIFIER+ )
+col_alias : GROUP | IDENTIFIER
;
col_index : DOLLARVAR
@@ -479,10 +476,7 @@ split_branch
col_ref : alias_col_ref | dollar_col_ref
;
-alias_col_ref : GROUP | scoped_alias_col_ref
-;
-
-scoped_alias_col_ref : ^( SCOPED_ALIAS IDENTIFIER+ )
+alias_col_ref : GROUP | IDENTIFIER
;
dollar_col_ref : DOLLARVAR
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanGenerator.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanGenerator.g?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanGenerator.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanGenerator.g Thu May 5 19:25:08 2011
@@ -359,9 +359,6 @@ simple_type returns[byte datatype]
;
tuple_type returns[LogicalSchema logicalSchema]
-@init {
- $logicalSchema = new LogicalSchema();
-}
: ^( TUPLE_TYPE
( field_def_list
{
@@ -850,20 +847,7 @@ col_alias_or_index returns[Object col]
col_alias returns[Object col]
: GROUP { $col = $GROUP.text; }
- | scoped_col_alias { $col = $scoped_col_alias.col; }
-;
-
-scoped_col_alias returns[Object col]
-@init {
- StringBuilder sb = new StringBuilder();
-}
-@after {
- $col = sb.toString();
-}
- : ^( SCOPED_ALIAS
- id1 = IDENTIFIER { sb.append( $id1.text ); }
- ( id2 = IDENTIFIER { sb.append( "::" ); sb.append( $id2.text ); } )*
- )
+ | IDENTIFIER { $col = $IDENTIFIER.text; }
;
col_index returns[Integer col]
@@ -1320,10 +1304,10 @@ alias_col_ref[LogicalExpressionPlan plan
$expr = builder.buildProjectExpr( new SourceLocation( (PigParserNode)$GROUP ), $plan, $GScope::currentOp,
$statement::inputIndex, $GROUP.text, 0 );
}
- | scoped_alias_col_ref
+ | IDENTIFIER
{
- SourceLocation loc = new SourceLocation( (PigParserNode)$scoped_alias_col_ref.start );
- String alias = $scoped_alias_col_ref.alias;
+ SourceLocation loc = new SourceLocation( (PigParserNode)$IDENTIFIER );
+ String alias = $IDENTIFIER.text;
Operator inOp = builder.lookupOperator( $statement::inputAlias );
LogicalSchema schema;
try {
@@ -1349,19 +1333,6 @@ alias_col_ref[LogicalExpressionPlan plan
}
;
-scoped_alias_col_ref returns[String alias]
-@init {
- StringBuilder sb = new StringBuilder();
-}
-@after {
- $alias = sb.toString();
-}
- : ^( SCOPED_ALIAS
- id1 = IDENTIFIER { sb.append( $id1.text ); }
- ( id2 = IDENTIFIER { sb.append( "::" ); sb.append( $id2.text ); } )*
- )
-;
-
dollar_col_ref[LogicalExpressionPlan plan] returns[LogicalExpression expr]
: DOLLARVAR
{
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/QueryLexer.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/QueryLexer.g?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/QueryLexer.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/QueryLexer.g Thu May 5 19:25:08 2011
@@ -285,7 +285,14 @@ fragment LETTER : 'A'..'Z'
fragment SPECIALCHAR : '_'
;
-IDENTIFIER: LETTER ( DIGIT | LETTER | SPECIALCHAR )*
+fragment ID: LETTER ( DIGIT | LETTER | SPECIALCHAR )*
+;
+
+DCOLON : '::'
+;
+
+IDENTIFIER : ( ID DCOLON ) => ( ID DCOLON IDENTIFIER )
+ | ID
;
fragment FLOATINGPOINT : INTEGER ( PERIOD INTEGER )? | PERIOD INTEGER
@@ -332,9 +339,6 @@ EXECCOMMAND : '`' ( ~( '`' ) )* '`'
STAR : '*'
;
-DCOLON : '::'
-;
-
COLON : ':'
;
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g Thu May 5 19:25:08 2011
@@ -58,7 +58,6 @@ tokens {
NEG;
EXPR_IN_PAREN;
JOIN_ITEM;
- SCOPED_ALIAS;
TUPLE_TYPE_CAST;
BAG_TYPE_CAST;
PARAMS;
@@ -443,11 +442,7 @@ dot_proj : PERIOD ( col_alias_or_index
col_alias_or_index : col_alias | col_index
;
-col_alias : GROUP | scoped_col_alias
-;
-
-scoped_col_alias : IDENTIFIER ( DCOLON IDENTIFIER )*
- -> ^( SCOPED_ALIAS IDENTIFIER+ )
+col_alias : GROUP | IDENTIFIER
;
col_index : DOLLARVAR
@@ -617,11 +612,7 @@ split_branch : alias IF cond
col_ref : alias_col_ref | dollar_col_ref
;
-alias_col_ref : GROUP | scoped_alias_col_ref
-;
-
-scoped_alias_col_ref : IDENTIFIER ( DCOLON IDENTIFIER )*
- -> ^( SCOPED_ALIAS IDENTIFIER+ )
+alias_col_ref : GROUP | IDENTIFIER
;
dollar_col_ref : DOLLARVAR
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserDriver.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserDriver.java Thu May 5 19:25:08 2011
@@ -19,10 +19,7 @@
package org.apache.pig.parser;
import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -43,7 +40,9 @@ import org.apache.commons.logging.LogFac
import org.apache.pig.impl.PigContext;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
-import org.apache.pig.tools.pigscript.parser.ParseException;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.parser.QueryParser.field_def_list_return;
+import org.apache.pig.parser.QueryParser.literal_return;
import org.apache.pig.tools.pigstats.ScriptState;
public class QueryParserDriver {
@@ -69,6 +68,94 @@ public class QueryParserDriver {
importSeen = new HashSet<String>();
macroSeen = new HashSet<String>();
}
+
+ private static Tree parseSchema(CommonTokenStream tokens) throws ParserException {
+ QueryParser parser = QueryParserUtils.createParser(tokens);
+
+ field_def_list_return result = null;
+ try {
+ result = parser.field_def_list();
+ } catch (RecognitionException e) {
+ String msg = parser.getErrorHeader(e) + " "
+ + parser.getErrorMessage(e, parser.getTokenNames());
+ throw new ParserException(msg);
+ } catch(RuntimeException ex) {
+ throw new ParserException( ex.getMessage() );
+ }
+
+ Tree ast = (Tree)result.getTree();
+ checkError( parser );
+
+ return ast;
+ }
+
+ public LogicalSchema parseSchema(String input) throws ParserException {
+ CommonTokenStream tokenStream = tokenize( input, null );
+ LogicalSchema schema = null;
+ Tree ast = parseSchema( tokenStream );
+
+ try{
+ CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
+ AstValidator walker = new AstValidator( nodes );
+ ast = (Tree)walker.field_def_list().getTree();
+ checkError( walker );
+
+ LogicalPlanGenerator planGenerator =
+ new LogicalPlanGenerator( new CommonTreeNodeStream( ast ), pigContext, scope, fileNameMap );
+ schema = planGenerator.field_def_list().schema;
+ checkError( planGenerator );
+ } catch(RecognitionException ex) {
+ throw new ParserException( ex );
+ } catch(Exception ex) {
+ throw new ParserException( ex.getMessage(), ex );
+ }
+
+ return schema;
+ }
+
+ private static Tree parseConstant(CommonTokenStream tokens) throws ParserException {
+ QueryParser parser = QueryParserUtils.createParser(tokens);
+
+ literal_return result = null;
+ try {
+ result = parser.literal();
+ } catch (RecognitionException e) {
+ String msg = parser.getErrorHeader(e) + " "
+ + parser.getErrorMessage(e, parser.getTokenNames());
+ throw new ParserException(msg);
+ } catch(RuntimeException ex) {
+ throw new ParserException( ex.getMessage() );
+ }
+
+ Tree ast = (Tree)result.getTree();
+ checkError( parser );
+
+ return ast;
+ }
+
+ public Object parseConstant(String input) throws ParserException {
+ CommonTokenStream tokenStream = tokenize( input, null );
+ Object value = null;
+ Tree ast = parseConstant( tokenStream );
+
+ try{
+ CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
+ AstValidator walker = new AstValidator( nodes );
+ ast = (Tree)walker.literal().getTree();
+ checkError( walker );
+
+ LogicalPlanGenerator planGenerator =
+ new LogicalPlanGenerator( new CommonTreeNodeStream( ast ), pigContext, scope, fileNameMap );
+ value = planGenerator.literal().value;
+ checkError( planGenerator );
+ } catch(RecognitionException ex) {
+ throw new ParserException( ex );
+ } catch(Exception ex) {
+ throw new ParserException( ex.getMessage(), ex );
+ }
+
+ return value;
+ }
public LogicalPlan parse(String query) throws ParserException {
LogicalPlan plan = null;
@@ -76,15 +163,8 @@ public class QueryParserDriver {
ScriptState ss = ScriptState.get();
CommonTokenStream tokenStream = tokenize(query, ss.getFileName());
- Tree ast = null;
-
- try {
- ast = parse( tokenStream );
- } catch(RuntimeException ex) {
- throw new ParserException( ex.getMessage() );
- }
-
- ast = expandMacro(ast);
+ Tree ast = parse( tokenStream );
+ ast = expandMacro( ast );
try{
ast = validateAst( ast );
@@ -142,7 +222,9 @@ public class QueryParserDriver {
String msg = parser.getErrorHeader(e) + " "
+ parser.getErrorMessage(e, parser.getTokenNames());
throw new ParserException(msg);
- }
+ } catch(RuntimeException ex) {
+ throw new ParserException( ex.getMessage() );
+ }
Tree ast = (Tree) result.getTree();
checkError(parser);
Modified: pig/branches/branch-0.9/src/org/apache/pig/scripting/js/JsFunction.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/scripting/js/JsFunction.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/scripting/js/JsFunction.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/scripting/js/JsFunction.java Thu May 5 19:25:08 2011
@@ -36,11 +36,11 @@ import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.impl.util.Utils;
+import org.apache.pig.parser.ParserException;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.NativeJavaObject;
@@ -181,7 +181,7 @@ public class JsFunction extends EvalFunc
outputSchemaDef = jsScriptEngine.jsEval(this.getClass().getName()+"(String)", functionName+".outputSchema").toString();
try {
this.outputSchema = Utils.getSchemaFromString(outputSchemaDef);
- } catch (ParseException e) {
+ } catch (ParserException e) {
throw new IllegalArgumentException(functionName+".outputSchema is not a valid schema: "+e.getMessage(), e);
}
Modified: pig/branches/branch-0.9/src/org/apache/pig/scripting/jython/JythonFunction.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/scripting/jython/JythonFunction.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/scripting/jython/JythonFunction.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/scripting/jython/JythonFunction.java Thu May 5 19:25:08 2011
@@ -24,9 +24,9 @@ import org.apache.pig.EvalFunc;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.util.Utils;
+import org.apache.pig.parser.ParserException;
import org.python.core.Py;
import org.python.core.PyBaseCode;
import org.python.core.PyException;
@@ -79,7 +79,7 @@ public class JythonFunction extends Eval
throw new ExecException(
"unregistered " + functionName);
}
- } catch (ParseException pe) {
+ } catch (ParserException pe) {
throw new ExecException("Could not parse schema for script function " + pe, pe);
} catch (IOException e) {
throw new IllegalStateException("Could not initialize: " + filename, e);
Modified: pig/branches/branch-0.9/src/org/apache/pig/scripting/jython/JythonScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/scripting/jython/JythonScriptEngine.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/scripting/jython/JythonScriptEngine.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/scripting/jython/JythonScriptEngine.java Thu May 5 19:25:08 2011
@@ -36,8 +36,8 @@ import org.apache.pig.FuncSpec;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.util.Utils;
+import org.apache.pig.parser.ParserException;
import org.apache.pig.scripting.ScriptEngine;
import org.apache.pig.tools.pigstats.PigStats;
import org.python.core.Py;
@@ -151,7 +151,7 @@ public class JythonScriptEngine extends
LOG.info("Register scripting UDF: " + namespace + key);
}
}
- } catch (ParseException pe) {
+ } catch (ParserException pe) {
throw new IOException(
"Error parsing schema for script function from the decorator",
pe);
Modified: pig/branches/branch-0.9/test/org/apache/pig/parser/TestLogicalPlanGenerator.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/parser/TestLogicalPlanGenerator.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/parser/TestLogicalPlanGenerator.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/parser/TestLogicalPlanGenerator.java Thu May 5 19:25:08 2011
@@ -18,7 +18,6 @@
package org.apache.pig.parser;
-
import java.io.File;
import java.io.IOException;
Modified: pig/branches/branch-0.9/test/org/apache/pig/parser/TestQueryLexer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/parser/TestQueryLexer.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/parser/TestQueryLexer.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/parser/TestQueryLexer.java Thu May 5 19:25:08 2011
@@ -48,7 +48,7 @@ public class TestQueryLexer {
// While we can check more conditions, such as type of each token, for now I think the following
// is enough. If the token type is wrong, it will be most likely caught by the parser.
- Assert.assertEquals( 402, tokenCount );
+ Assert.assertEquals( 400, tokenCount );
Assert.assertEquals( 0, lexer.getNumberOfSyntaxErrors() );
}
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipelineLocal.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipelineLocal.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipelineLocal.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipelineLocal.java Thu May 5 19:25:08 2011
@@ -1053,6 +1053,12 @@ public class TestEvalPipelineLocal {
// Filter out the random number generated on hash
realPlan = realPlan.replaceAll("\\d{3,}", "");
+ System.out.println("-----------golden");
+ System.out.println(goldenPlan);
+ System.out.println("-----------");
+ System.out.println(realPlan);
+
+
Assert.assertEquals(realPlan, goldenPlan);
}
}
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestLogicalPlanBuilder.java Thu May 5 19:25:08 2011
@@ -18,7 +18,9 @@
package org.apache.pig.test;
import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import junit.framework.Assert;
@@ -37,21 +39,28 @@ import org.apache.pig.FuncSpec;
import org.apache.pig.PigServer;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.PigContext;
import org.apache.pig.ExecType;
import org.apache.pig.impl.builtin.GFAny;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.util.Utils;
import org.apache.pig.data.DataType;
import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.logical.expression.ConstantExpression;
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
import org.apache.pig.newplan.logical.relational.LOCogroup;
import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOGenerate;
+import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LOSort;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.test.utils.Identity;
public class TestLogicalPlanBuilder {
PigContext pigContext = new PigContext(ExecType.LOCAL, new Properties());
@@ -153,8 +162,6 @@ public class TestLogicalPlanBuilder {
buildPlan(query);
}
-
-
@Test
public void testQueryFail1() throws Exception {
String query = " foreach (group (A = load 'a') by $1) generate A.'1' ;";
@@ -176,7 +183,6 @@ public class TestLogicalPlanBuilder {
}
Assert.fail("Test case should fail" );
}
-
@Test
public void testQueryFail3() throws Exception {
@@ -211,8 +217,6 @@ public class TestLogicalPlanBuilder {
}
Assert.fail("Test case should fail" );
}
-
-
/**
* User generate functions must be in default package Bug 831620 - fixed
@@ -225,7 +229,6 @@ public class TestLogicalPlanBuilder {
buildPlan(query);
}
-
static public class TestApplyFunc extends org.apache.pig.EvalFunc<Tuple> {
@Override
public Tuple exec(Tuple input) throws IOException {
@@ -234,27 +237,23 @@ public class TestLogicalPlanBuilder {
}
}
-
/**
* Validate that parallel is parsed correctly Bug 831714 - fixed
*/
-
-// @Test Waiting for PIG-1996
-// public void testQuery18() throws Exception {
-// String query = "FOREACH (group (load 'a') ALL PARALLEL 16) generate group;";
-// LogicalPlan lp = buildPlan(query);
-// Operator root = lp.getSources().get(0);
-//
-// List<Operator> listOp = lp.getSuccessors(root);
-//
-// Operator lo = listOp.get(0);
-//
-// if (lo instanceof LOCogroup) {
-// Assert.assertTrue(((LOCogroup) lo).getRequestedParallelisam() == 16);
-// } else {
-// Assert.fail("Error: Unexpected Parse Tree output");
-// }
-// }
+ @Test //PIG-1996
+ public void testQuery18() throws Exception {
+ String query = "store (FOREACH (group (load 'a') ALL PARALLEL 16) generate group ) into 'y';";
+ LogicalPlan lp = buildPlan(query);
+ Operator root = lp.getSources().get(0);
+ List<Operator> listOp = lp.getSuccessors(root);
+ Operator lo = listOp.get(0);
+
+ if (lo instanceof LOCogroup) {
+ Assert.assertEquals( 1, ((LOCogroup) lo).getRequestedParallelisam() );//Local mode, paraallel = 1
+ } else {
+ Assert.fail("Error: Unexpected Parse Tree output");
+ }
+ }
@Test
public void testQuery19() throws Exception {
@@ -263,7 +262,6 @@ public class TestLogicalPlanBuilder {
buildPlan( query );
}
-
@Test
public void testQuery20() throws Exception {
String query = "foreach (load 'a') generate ($1 == '3'? $2 : $3) ;";
@@ -367,7 +365,6 @@ public class TestLogicalPlanBuilder {
}
Assert.assertTrue(exceptionThrown);
}
-
@Test
public void testQuery24() throws Exception {
@@ -384,7 +381,6 @@ public class TestLogicalPlanBuilder {
buildPlan(query);
}
-
@Test
public void testQuery26() throws Exception {
String query = "foreach (load 'a') generate ((NOT (($1 == $2) OR ('a' < 'b'))) ? 'a' : $2), 'x' ;";
@@ -413,14 +409,12 @@ public class TestLogicalPlanBuilder {
buildPlan(query);
}
-
@Test
public void testQuery30() throws Exception {
String query = "load 'myfile' using " + TestStorageFunc.class.getName() + "() as (col1, col2);";
buildPlan(query);
}
-
public static class TestStorageFunc extends LoadFunc{
public Tuple getNext() throws IOException {
@@ -455,7 +449,6 @@ public class TestLogicalPlanBuilder {
}
}
-
@Test
public void testQuery31() throws Exception {
String query = "load 'myfile' as (col1, col2);";
@@ -477,7 +470,6 @@ public class TestLogicalPlanBuilder {
buildPlan(query);
}
-
@Test
//TODO: Nested schemas don't work now. Probably a bug in the new parser.
public void testQuery34() throws Exception {
@@ -488,8 +480,6 @@ public class TestLogicalPlanBuilder {
buildPlan(query);
}
-
-
@Test
public void testQuery35() throws Exception {
String query = "foreach (load 'a' as (col1, col2)) generate col1, col2 ;";
@@ -519,7 +509,6 @@ public class TestLogicalPlanBuilder {
buildPlan(query);
}
-
// TODO FIX Query39 and Query40
@Test
public void testQuery39() throws Exception{
@@ -599,7 +588,6 @@ public class TestLogicalPlanBuilder {
Assert.fail( "Query should fail." );
}
-
@Test
public void testQuery44() throws Exception {
String q = "a = load 'a' as (url, pagerank);" +
@@ -609,8 +597,6 @@ public class TestLogicalPlanBuilder {
buildPlan( q );
}
-//TODO
-//Commented out testQueryFail44 as I am not able to include org.apache.pig.PigServer;
@Test
public void testQueryFail44() throws Throwable {
PigServer pig = null;
@@ -635,7 +621,6 @@ public class TestLogicalPlanBuilder {
buildPlan(query);
}
-
@Test
public void testQuery58() throws Exception {
String query = "a = load 'a' as (name, age, gpa);" +
@@ -713,7 +698,6 @@ public class TestLogicalPlanBuilder {
"d = foreach b generate group.age;" +
"e = foreach a generate name, details;";
buildPlan(query);
-
}
@Test
@@ -1068,10 +1052,10 @@ public class TestLogicalPlanBuilder {
Operator store = lp.getSinks().get(0);
LOForEach foreach = (LOForEach) lp.getPredecessors(store).get(0);
LogicalPlan nestedPlan = foreach.getInnerPlan();
-// LOProject sortInput = (LOProject)nestedPlan.getSources().get(0);
-// LOSort nestedSort = (LOSort)nestedPlan.getSuccessors(sortInput).get(0);
-// LogicalPlan sortPlan = nestedSort.getSortColPlans().get(0);
-// Assert.assertTrue(sortPlan.getLeaves().size() == 1);
+ LOGenerate gen = (LOGenerate) nestedPlan.getSinks().get(0);
+ LOSort nestedSort = (LOSort)nestedPlan.getPredecessors(gen).get(0);
+ LogicalExpressionPlan sortPlan = nestedSort.getSortColPlans().get(0);
+ Assert.assertTrue(sortPlan.getSinks().size() == 1);
}
@Test
@@ -1120,7 +1104,7 @@ public class TestLogicalPlanBuilder {
"store c into 'output';");
Operator store = lp.getSinks().get(0);
foreach = (LOForEach)lp.getPredecessors(store).get(0);
- Assert.assertTrue(foreach.getSchema().toString( false ).equals("myname:chararray,myage:int,mycount:long"));
+ Assert.assertTrue(foreach.getSchema().isEqual(Utils.parseSchema("myname: chararray, age: int, mycount: long")));
//the schema of group is unchanged
lp = buildPlan( query +
@@ -1304,13 +1288,10 @@ public class TestLogicalPlanBuilder {
Operator store = lp.getSinks().get(0);
LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
LOCogroup cogroup = (LOCogroup) lp.getPredecessors(foreach).get(0);
- Schema.FieldSchema bagFs = new Schema.FieldSchema("a", Util.getSchemaFromString("name: bytearray, age: bytearray, gpa: bytearray"), DataType.BAG);
- Schema.FieldSchema groupFs = new Schema.FieldSchema("group", DataType.BYTEARRAY);
- Schema cogroupExpectedSchema = new Schema();
- cogroupExpectedSchema.add(groupFs);
- cogroupExpectedSchema.add(bagFs);
-// Assert.assertTrue(LogicalSchema.equals(cogroup.getSchema(), cogroupExpectedSchema, false, false));
-// Assert.assertTrue(Schema.equals(foreach.getSchema(), Util.getSchemaFromString("name: bytearray, age: bytearray, gpa: bytearray, max_age: double"), false, true));
+ String s = cogroup.getSchema().toString(false);
+ Assert.assertTrue( s.equals("group:bytearray,a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}"));
+ s = foreach.getSchema().toString(false);
+ Assert.assertTrue( s.equals("d::name:bytearray,d::age:bytearray,d::gpa:bytearray,max_age:double"));
}
@Test
@@ -1479,377 +1460,313 @@ public class TestLogicalPlanBuilder {
query = "foreach (load 'data') generate FUNC($0);";
buildPlan(query);
}
-
+*/
@Test
public void testQuery102() throws Exception {
// test basic store
- "a = load 'a';");
- buildPlan("store a into 'out';");
+ buildPlan( "a = load 'a';" + "store a into 'out';" );
}
@Test
public void testQuery103() throws Exception {
// test store with store function
- "a = load 'a';");
- buildPlan("store a into 'out' using PigStorage();");
+ buildPlan("a = load 'a';" + "store a into 'out' using PigStorage();");
}
- @Test
- public void testQuery104() throws Exception {
- // check that a field alias can be referenced
- // by unambiguous free form alias, fully qualified alias
- // and partially qualified unambiguous alias
- "a = load 'st10k' as (name, age, gpa);" );
- "b = group a by name;" );
- "c = foreach b generate flatten(a);" );
- "d = filter c by name != 'fred';" );
- "e = group d by name;" );
- "f = foreach e generate flatten(d);" );
- "g = foreach f generate name, d::a::name, a::name;");
-
- }
+// @Test // Commented out due to PIG-2037
+// public void testQuery104() throws Exception {
+// // check that a field alias can be referenced
+// // by unambiguous free form alias, fully qualified alias
+// // and partially qualified unambiguous alias
+// String query = "a = load 'st10k' as (name, age, gpa);\n" +
+// "b = group a by name;\n" +
+// "c = foreach b generate flatten(a);\n" +
+// "d = filter c by name != 'fred';\n" +
+// "e = group d by name;\n" +
+// "f = foreach e generate flatten(d);\n" +
+// "g = foreach f generate name, d::a::name, a::name;\n" +
+// "store g into 'output';";
+// buildPlan( query );
+// }
@Test
public void testQuery105() throws Exception {
// test that the alias "group" can be used
// after a flatten(group)
- "a = load 'st10k' as (name, age, gpa);" );
- "b = group a by name;" );
- "c = foreach b generate flatten(group), COUNT(a) as cnt;" );
- "d = foreach c generate group;");
-
+ String query = "a = load 'st10k' as (name, age, gpa);" +
+ "b = group a by name;" +
+ "c = foreach b generate flatten(group), COUNT(a) as cnt;" +
+ "d = foreach c generate group;" +
+ "store d into 'output';";
+ buildPlan( query );
}
@Test
- public void testQuery106() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOForEach foreach;
-
- "a = load 'one' as (name, age, gpa);");
-
- lp = buildPlan("b = foreach a generate *;");
- foreach = (LOForEach) lp.getLeaves().get(0);
- Assert.assertTrue(Schema.equals(foreach.getSchema(), Util.getSchemaFromString("name: bytearray, age: bytearray, gpa: bytearray"), false, true));
+ public void testQuery106() throws Exception {
+ String query = "a = load 'one' as (name, age, gpa);" +
+ "b = foreach a generate *;" + "store b into 'output';";
+ LogicalPlan lp = buildPlan(query);
+ Operator store = lp.getSinks().get(0);
+ LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
+ String s = foreach.getSchema().toString(false);
+ Assert.assertTrue( s.equals("name:bytearray,age:bytearray,gpa:bytearray"));
}
@Test
- public void testQuery107() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOForEach foreach;
-
- "a = load 'one';");
-
- lp = buildPlan("b = foreach a generate *;");
- foreach = (LOForEach) lp.getLeaves().get(0);
- LogicalPlan foreachPlan = foreach.getForEachPlans().get(0);
-
+ public void testQuery107() throws Exception {
+ String query = "a = load 'one';" + "b = foreach a generate *;" + "store b into 'output';";
+ LogicalPlan lp = buildPlan( query );
+ Operator store = lp.getSinks().get(0);
+ LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
+ LOGenerate gen = (LOGenerate) foreach.getInnerPlan().getSinks().get(0);
+ LogicalExpressionPlan foreachPlan = gen.getOutputPlans().get(0);
Assert.assertTrue(checkPlanForProjectStar(foreachPlan));
-
}
@Test
- public void testQuery108() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOCogroup cogroup;
-
- buildPlan("a = load 'one' as (name, age, gpa);");
-
- lp = buildPlan("b = group a by *;");
- cogroup = (LOCogroup) lp.getLeaves().get(0);
- Schema groupSchema = Util.getSchemaFromString("name: bytearray, age: bytearray, gpa: bytearray");
- Schema bagASchema = Util.getSchemaFromString("name: bytearray, age: bytearray, gpa: bytearray");
- Schema.FieldSchema groupFs = new Schema.FieldSchema("group", groupSchema, DataType.TUPLE);
- Schema.FieldSchema bagAFs = new Schema.FieldSchema("a", bagASchema, DataType.BAG);
- Schema expectedSchema = new Schema(groupFs);
- expectedSchema.add(bagAFs);
- Assert.assertTrue(Schema.equals(cogroup.getSchema(), expectedSchema, false, true));
-
+ public void testQuery108() throws Exception {
+ String query = "a = load 'one' as (name, age, gpa);" +
+ "b = group a by *;" +
+ "store b into 'output';";
+ LogicalPlan lp = buildPlan(query);
+ Operator store = lp.getSinks().get(0);
+ LOCogroup cogroup = (LOCogroup)lp.getPredecessors(store).get(0);
+ String s = cogroup.getSchema().toString(false);
+ Assert.assertTrue(s.equals("group:tuple(name:bytearray,age:bytearray,gpa:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)}"));
}
@Test
- public void testQuery109() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOCogroup cogroup;
-
- "a = load 'one' as (name, age, gpa);");
- "b = load 'two' as (first_name, enrol_age, high_school_gpa);");
-
- lp = buildPlan("c = group a by *, b by *;");
- cogroup = (LOCogroup) lp.getLeaves().get(0);
- Schema groupSchema = Util.getSchemaFromString("name: bytearray, age: bytearray, gpa: bytearray");
- Schema bagASchema = Util.getSchemaFromString("name: bytearray, age: bytearray, gpa: bytearray");
- Schema bagBSchema = Util.getSchemaFromString("first_name: bytearray, enrol_age: bytearray, high_school_gpa: bytearray");
- Schema.FieldSchema groupFs = new Schema.FieldSchema("group", groupSchema, DataType.TUPLE);
- Schema.FieldSchema bagAFs = new Schema.FieldSchema("a", bagASchema, DataType.BAG);
- Schema.FieldSchema bagBFs = new Schema.FieldSchema("b", bagBSchema, DataType.BAG);
- Schema expectedSchema = new Schema(groupFs);
- expectedSchema.add(bagAFs);
- expectedSchema.add(bagBFs);
- Assert.assertTrue(Schema.equals(cogroup.getSchema(), expectedSchema, false, true));
-
+ public void testQuery109() throws Exception {
+ String query = "a = load 'one' as (name, age, gpa);" +
+ "b = load 'two' as (first_name, enrol_age, high_school_gpa);" +
+ "c = group a by *, b by *;" +
+ "store c into 'output';";
+ LogicalPlan lp = buildPlan(query);
+ Operator store = lp.getSinks().get(0);
+ LOCogroup cogroup = (LOCogroup)lp.getPredecessors(store).get(0);
+ String s = cogroup.getSchema().toString(false);
+ Assert.assertTrue(s.equals("group:tuple(name:bytearray,age:bytearray,gpa:bytearray),a:bag{:tuple(name:bytearray,age:bytearray,gpa:bytearray)},b:bag{:tuple(first_name:bytearray,enrol_age:bytearray,high_school_gpa:bytearray)}"));
}
@Test
- public void testQuery110Fail() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOLoad load;
- LOCogroup cogroup;
+ public void testQuery110Fail() throws Exception {
+ String query = "a = load 'one' as (name, age, gpa);" +
+ "b = load 'two';" + "c = cogroup a by $0, b by *;";
- buildPlan("a = load 'one' as (name, age, gpa);");
- lp = "b = load 'two';");
-
- load = (LOLoad) lp.getLeaves().get(0);
- boolean exceptionThrown = false;
- try{
- lp = buildPlan("c = cogroup a by $0, b by *;");
+ try {
+ buildPlan( query );
} catch(AssertionFailedError e) {
- Assert.assertTrue(e.getMessage().contains("Cogroup/Group by * is only allowed if " +
- "the input has a schema"));
- exceptionThrown = true;
+ Assert.assertTrue(e.getMessage().contains("Cogroup/Group by '*' or 'x..' (range of columns to the end) is only allowed " +
+ "if the input has a schema" ) );
+ return;
}
- Assert.assertTrue(exceptionThrown);
-
-
+ Assert.fail( "Test case should fail." );
}
@Test
- public void testQuery111() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOSort sort;
-
- buildPlan("a = load 'one' as (name, age, gpa);");
+ public void testQuery111() throws Exception {
+ String query = "a = load 'one' as (name, age, gpa);" +
+ "b = order a by *;" + "store b into 'y';";
- lp = "b = order a by *;");
- sort = (LOSort) lp.getLeaves().get(0);
+ LogicalPlan lp = buildPlan(query);
+ Operator store = lp.getSinks().get(0);
+ LOSort sort = (LOSort)lp.getPredecessors(store).get(0);
- for(LogicalPlan sortPlan: sort.getSortColPlans()) {
+ for(LogicalExpressionPlan sortPlan: sort.getSortColPlans() ) {
Assert.assertTrue(checkPlanForProjectStar(sortPlan) == false);
}
-
}
@Test
- public void testQuery112() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOForEach foreach;
- LOSort sort;
-
- buildPlan("a = load 'one' as (name, age, gpa);");
-
- "b = group a by *;");
- lp = buildPlan("c = foreach b {a1 = order a by *; generate a1;};");
- foreach = (LOForEach) lp.getLeaves().get(0);
-
- for(LogicalPlan foreachPlan: foreach.getForEachPlans()) {
+ public void testQuery112() throws Exception {
+ String query = "a = load 'one' as (name, age, gpa);" +
+ "b = group a by *;" +
+ "c = foreach b {a1 = order a by *; generate a1;};" +
+ "store c into 'y';";
+ LogicalPlan lp = buildPlan(query);
+ Operator store = lp.getSinks().get(0);
+ LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
+ LOGenerate gen = (LOGenerate) foreach.getInnerPlan().getSinks().get(0);
+ for(LogicalExpressionPlan foreachPlan: gen.getOutputPlans()) {
Assert.assertTrue(checkPlanForProjectStar(foreachPlan) == true);
}
- LogicalPlan foreachPlan = foreach.getForEachPlans().get(0);
-
- sort = (LOSort)foreachPlan.getPredecessors(foreachPlan.getLeaves().get(0)).get(0);
+ LogicalPlan foreachPlan = foreach.getInnerPlan();
+ LOSort sort = (LOSort)foreachPlan.getPredecessors(gen).get(0);
// project (*) operator here is translated to a list of projection
// operators
- for(LogicalPlan sortPlan: sort.getSortColPlans()) {
+ for(LogicalExpressionPlan sortPlan: sort.getSortColPlans()) {
Assert.assertTrue(checkPlanForProjectStar(sortPlan) == false);
}
-
}
@Test
- public void testQuery113() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOForEach foreach;
- LOSort sort;
-
- "a = load 'one' as (name, age, gpa);");
-
- lp = buildPlan("b = foreach a {exp1 = age + gpa; exp2 = exp1 + age; generate exp1, exp2;};");
- foreach = (LOForEach) lp.getLeaves().get(0);
-
- for(LogicalPlan foreachPlan: foreach.getForEachPlans()) {
- printPlan(foreachPlan);
- Assert.assertTrue(checkPlanForProjectStar(foreachPlan) == false);
- }
-
- }
-
- @Test
- public void testQuery114() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOForEach foreach;
- LOSort sort;
-
- "a = load 'one' as (name, age, gpa);");
-
- lp = buildPlan("b = foreach a generate " + Identity.class.getName() + "(name, age);");
- foreach = (LOForEach) lp.getLeaves().get(0);
-
- Schema s = new Schema();
- s.add(new Schema.FieldSchema("name", DataType.BYTEARRAY));
- s.add(new Schema.FieldSchema("age", DataType.BYTEARRAY));
- Schema.FieldSchema tupleFs = new Schema.FieldSchema(null, s, DataType.TUPLE);
- Schema expectedSchema = new Schema(tupleFs);
- Assert.assertTrue(Schema.equals(foreach.getSchema(), expectedSchema, false, true));
+ public void testQuery114() throws Exception {
+ String query = "a = load 'one' as (name, age, gpa);" +
+ "b = foreach a generate " + Identity.class.getName() + "(name, age);" +
+ "store b into 'y';";
+ LogicalPlan lp = buildPlan(query);
+ Operator store = lp.getSinks().get(0);
+ LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
+ String s = foreach.getSchema().toString(false);
+ Assert.assertTrue(s.equals(":tuple(name:bytearray,age:bytearray)"));
}
@Test
- public void testQuery115() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOForEach foreach;
- LOSort sort;
-
- "a = load 'one' as (name, age, gpa);");
-
- lp = buildPlan("b = foreach a generate " + Identity.class.getName() + "(*);");
- foreach = (LOForEach) lp.getLeaves().get(0);
-
- Schema s = new Schema();
- s.add(new Schema.FieldSchema("name", DataType.BYTEARRAY));
- s.add(new Schema.FieldSchema("age", DataType.BYTEARRAY));
- s.add(new Schema.FieldSchema("gpa", DataType.BYTEARRAY));
- Schema.FieldSchema tupleFs = new Schema.FieldSchema(null, s, DataType.TUPLE);
- Schema expectedSchema = new Schema(tupleFs);
- Assert.assertTrue(Schema.equals(foreach.getSchema(), expectedSchema, false, true));
+ public void testQuery115() throws Exception {
+ String query = "a = load 'one' as (name, age, gpa);" +
+ "b = foreach a generate " + Identity.class.getName() + "(*);" +
+ "store b into 'y';";
+ LogicalPlan lp = buildPlan(query);
+ Operator store = lp.getSinks().get(0);
+ LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
+ String s = foreach.getSchema().toString(false);
+ Assert.assertTrue(s.equals(":tuple(name:bytearray,age:bytearray,gpa:bytearray)"));
}
@Test
- public void testQuery116() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOForEach foreach;
- LOSort sort;
-
- "a = load 'one';");
-
- lp = buildPlan("b = foreach a generate " + Identity.class.getName() + "($0, $1);");
- foreach = (LOForEach) lp.getLeaves().get(0);
-
- Schema s = new Schema();
- s.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));
- s.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));
- Schema.FieldSchema tupleFs = new Schema.FieldSchema(null, s, DataType.TUPLE);
- Schema expectedSchema = new Schema(tupleFs);
- Assert.assertTrue(Schema.equals(foreach.getSchema(), expectedSchema, false, true));
+ public void testQuery116() throws Exception {
+ String query = "a = load 'one';" +
+ "b = foreach a generate " + Identity.class.getName() + "($0, $1);" +
+ "store b into 'y';";
+ LogicalPlan lp = buildPlan(query);
+ Operator store = lp.getSinks().get(0);
+ LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
+ String s = foreach.getSchema().toString(false);
+ Assert.assertTrue(s.equals(":tuple(:bytearray,:bytearray)"));
}
@Test
- public void testQuery117() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOForEach foreach;
- LOSort sort;
-
- "a = load 'one';");
-
- lp = buildPlan("b = foreach a generate " + Identity.class.getName() + "(*);");
- foreach = (LOForEach) lp.getLeaves().get(0);
-
- Schema.FieldSchema tupleFs = new Schema.FieldSchema(null, null, DataType.TUPLE);
- Schema expectedSchema = new Schema(tupleFs);
- Assert.assertTrue(Schema.equals(foreach.getSchema(), expectedSchema, false, true));
-
+ public void testQuery117() throws Exception {
+ String query = "a = load 'one';" +
+ "b = foreach a generate " + Identity.class.getName() + "(*);" +
+ "store b into 'y';";
+ LogicalPlan lp = buildPlan(query);
+ Operator store = lp.getSinks().get(0);
+ LOForEach foreach = (LOForEach)lp.getPredecessors(store).get(0);
+ String s = foreach.getSchema().toString(false);
+ Assert.assertTrue(s.equals(":tuple()"));
}
@Test
public void testNullConsArithExprs() throws Exception {
- "a = load 'a' as (x:int, y:double);" );
+ String query = "a = load 'a' as (x:int, y:double);" +
"b = foreach a generate x + null, x * null, x / null, x - null, null % x, " +
- "y + null, y * null, y / null, y - null;"
- );
+ "y + null, y * null, y / null, y - null;" +
+ "store b into 'output';";
+ buildPlan( query );
}
@Test
public void testNullConsBincond1() throws Exception {
- "a = load 'a' as (x:int, y:double);" );
+ String query = "a = load 'a' as (x:int, y:double);" +
"b = foreach a generate (2 > 1? null : 1), ( 2 < 1 ? null : 1), " +
"(2 > 1 ? 1 : null), ( 2 < 1 ? 1 : null);"
- );
+ + "store b into 'output';";
+ buildPlan( query );
}
@Test
public void testNullConsBincond2() throws Exception {
- "a = load 'a' as (x:int, y:double);" );
- "b = foreach a generate (null is null ? 1 : 2), ( null is not null ? 2 : 1);");
+ String query = "a = load 'a' as (x:int, y:double);" +
+ "b = foreach a generate (null is null ? 1 : 2), ( null is not null ? 2 : 1);" +
+ "store b into 'output';";
+ buildPlan( query );
}
@Test
public void testNullConsForEachGenerate() throws Exception {
- "a = load 'a' as (x:int, y:double);" );
- "b = foreach a generate x, null, y, null;");
-
+ String query = "a = load 'a' as (x:int, y:double);" +
+ "b = foreach a generate x, null, y, null;" +
+ "store b into 'output';";
+ buildPlan( query );
}
@Test
public void testNullConsOuterJoin() throws Exception {
- "a = load 'a' as (x:int, y:chararray);" );
- "b = load 'b' as (u:int, v:chararray);" );
- "c = cogroup a by x, b by u;" );
+ String query = "a = load 'a' as (x:int, y:chararray);" +
+ "b = load 'b' as (u:int, v:chararray);" +
+ "c = cogroup a by x, b by u;" +
"d = foreach c generate flatten((SIZE(a) == 0 ? null : a)), " +
- "flatten((SIZE(b) == 0 ? null : b));"
- );
+ "flatten((SIZE(b) == 0 ? null : b));" +
+ "store d into 'output';";
+ buildPlan(query);
}
@Test
public void testNullConsConcatSize() throws Exception {
- "a = load 'a' as (x:int, y:double, str:chararray);" );
+ String query = "a = load 'a' as (x:int, y:double, str:chararray);" +
"b = foreach a generate SIZE(null), CONCAT(str, null), " +
- "CONCAT(null, str);"
- );
+ "CONCAT(null, str);" +
+ "store b into 'output';";
+ buildPlan(query);
}
@Test
public void testFilterUdfDefine() throws Exception {
- "define isempty IsEmpty();");
- "a = load 'a' as (x:int, y:double, str:chararray);");
- "b = filter a by isempty(*);");
+ String query = "define isempty IsEmpty();" +
+ "a = load 'a' as (x:int, y:double, str:chararray);" +
+ "b = filter a by isempty(*);" + "store b into 'output';";
+ buildPlan(query);
}
@Test
public void testLoadUdfDefine() throws Exception {
- "define PS PigStorage();");
- "a = load 'a' using PS as (x:int, y:double, str:chararray);" );
- "b = filter a by IsEmpty(*);");
+ String query = "define PS PigStorage();" +
+ "a = load 'a' using PS as (x:int, y:double, str:chararray);" +
+ "b = filter a by IsEmpty(*);" +
+ " store b into 'x' using PS;";
+ buildPlan(query);
}
@Test
public void testLoadUdfConstructorArgDefine() throws Exception {
- "define PS PigStorage(':');");
- "a = load 'a' using PS as (x:int, y:double, str:chararray);" );
- "b = filter a by IsEmpty(*);");
+ String query = "define PS PigStorage(':');" +
+ "a = load 'a' using PS as (x:int, y:double, str:chararray);" +
+ "b = filter a by IsEmpty(*);" +
+ " store b into 'x' using PS;";
+ buildPlan(query);
}
@Test
public void testStoreUdfDefine() throws Exception {
- "define PS PigStorage();");
- "a = load 'a' using PS as (x:int, y:double, str:chararray);" );
- "b = filter a by IsEmpty(*);" );
- "store b into 'x' using PS;");
+ String query = "define PS PigStorage();" +
+ "a = load 'a' using PS as (x:int, y:double, str:chararray);" +
+ "b = filter a by IsEmpty(*);" +
+ "store b into 'x' using PS;";
+ buildPlan(query);
}
@Test
public void testStoreUdfConstructorArgDefine() throws Exception {
- "define PS PigStorage(':');");
- " a = load 'a' using PS as (x:int, y:double, str:chararray);" );
- " b = filter a by IsEmpty(*);" );
- " store b into 'x' using PS;") ;
-
+ String query = "define PS PigStorage(':');" +
+ " a = load 'a' using PS as (x:int, y:double, str:chararray);" +
+ " b = filter a by IsEmpty(*);" +
+ " store b into 'x' using PS;";
+ buildPlan(query);
}
@Test
public void testCastAlias() throws Exception {
- "a = load 'one.txt' as (x,y); ");
- "b = foreach a generate (int)x, (double)y;");
- "c = group b by x;");
+ String query = "a = load 'one.txt' as (x,y); " +
+ "b = foreach a generate (int)x, (double)y;" +
+ "c = group b by x;" +
+ "store c into 'output';";
+ buildPlan(query);
}
@Test
public void testCast() throws Exception {
- "a = load 'one.txt' as (x,y); " );
- "b = foreach a generate (int)$0, (double)$1;" );
- "c = group b by $0;");
+ String query = "a = load 'one.txt' as (x,y); " +
+ "b = foreach a generate (int)$0, (double)$1;" +
+ "c = group b by $0;"+
+ "store c into 'output';";
+ buildPlan(query);
}
@Test
@@ -1926,45 +1843,36 @@ public class TestLogicalPlanBuilder {
for(String keyword: keywords) {
String query = "define FUNC org.apache."+keyword+"();";
- LogicalOperator lo = buildPlan(query).getRoots().get(0);
- Assert.assertTrue(lo instanceof LODefine);
+ LogicalPlan lp = buildPlan( query );
}
}
-
@Test
- public void testTokenizeSchema() throws FrontendException, ParseException {
- LogicalPlan lp;
- LOForEach foreach;
+ public void testTokenizeSchema() throws Exception {
+ String query = "a = load 'one' as (f1: chararray);" +
+ "b = foreach a generate TOKENIZE(f1);" +
+ "store b into 'output';";
+ LogicalPlan lp = buildPlan(query);
+ Operator store = lp.getSinks().get(0);
+ LOForEach foreach = (LOForEach) lp.getPredecessors(store).get(0);
+ String s = foreach.getSchema().toString(false);
+ Assert.assertTrue( s.equals("bag_of_tokenTuples:bag{tuple_of_tokens:tuple(token:chararray)}"));
+ }
- "a = load 'one' as (f1: chararray);");
- lp = buildPlan("b = foreach a generate TOKENIZE(f1);");
- foreach = (LOForEach) lp.getLeaves().get(0);
+/** The following test cases are disabled due to PIG-2038
- Schema.FieldSchema tokenFs = new Schema.FieldSchema("token",
- DataType.CHARARRAY);
- Schema tupleSchema = new Schema(tokenFs);
-
- Schema.FieldSchema tupleFs;
- tupleFs = new Schema.FieldSchema("tuple_of_tokens", tupleSchema,
- DataType.TUPLE);
-
- Schema bagSchema = new Schema(tupleFs);
- bagSchema.setTwoLevelAccessRequired(true);
- Schema.FieldSchema bagFs = new Schema.FieldSchema(
- "bag_of_tokenTuples",bagSchema, DataType.BAG);
-
- Assert.assertTrue(Schema.equals(foreach.getSchema(), new Schema(bagFs), false, true));
- }
@Test
- public void testEmptyTupleConst() throws FrontendException{
-
- LogicalPlan lp = buildPlan("a = foreach (load 'b') generate ();");
- LOForEach foreach = (LOForEach) lp.getLeaves().get(0);
- LogicalOperator logOp = foreach.getForEachPlans().get(0).getLeaves().get(0);
- Assert.assertTrue( logOp instanceof LOConst);
+ public void testEmptyTupleConst() throws Exception{
+ String query = "a = foreach (load 'b') generate ();" + "store a into 'output';";
+ LogicalPlan lp = buildPlan( query );
+ Operator store = lp.getSinks().get(0);
+ LOForEach foreach = (LOForEach) lp.getPredecessors(store).get(0);
+ LOGenerate gen = (LOGenerate)foreach.getInnerPlan().getSinks().get(0);
+ LogicalExpressionPlan exprPlan = gen.getOutputPlans().get(0);
+ Operator logOp = exprPlan.getSources().get(0);
+ Assert.assertTrue( logOp instanceof ConstantExpression);
- LOConst loConst = (LOConst)logOp;
+ ConstantExpression loConst = (ConstantExpression)logOp;
Assert.assertTrue(loConst.getType() == DataType.TUPLE);
Assert.assertTrue(loConst.getValue() instanceof Tuple);
Assert.assertTrue(loConst.getValue().equals(TupleFactory.getInstance().newTuple()));
@@ -1972,133 +1880,117 @@ public class TestLogicalPlanBuilder {
Schema.FieldSchema tupleFs = new Schema.FieldSchema(null, null, DataType.TUPLE);
Schema expectedSchema = new Schema(tupleFs);
- Assert.assertTrue(Schema.equals(foreach.getSchema(), expectedSchema, false, true));
+ Assert.assertTrue( foreach.getSchema().toString(false).equals(""));
}
@Test
- public void testEmptyMapConst() throws FrontendException{
-
- LogicalPlan lp = buildPlan("a = foreach (load 'b') generate [];");
- LOForEach foreach = (LOForEach) lp.getLeaves().get(0);
- LogicalOperator logOp = foreach.getForEachPlans().get(0).getLeaves().get(0);
- Assert.assertTrue( logOp instanceof LOConst);
+ public void testEmptyMapConst() throws Exception{
+ String query = "a = foreach (load 'b') generate [];" + "store a into 'output';";
+ LogicalPlan lp = buildPlan(query);
+ LOForEach foreach = (LOForEach) lp.getSinks().get(0);
+ LOGenerate gen = (LOGenerate)foreach.getInnerPlan().getSinks().get(0);
+ LogicalExpressionPlan exprPlan = gen.getOutputPlans().get(0);
+ Operator logOp = exprPlan.getSources().get(0);
+ Assert.assertTrue( logOp instanceof ConstantExpression);
- LOConst loConst = (LOConst)logOp;
+ ConstantExpression loConst = (ConstantExpression)logOp;
Assert.assertTrue(loConst.getType() == DataType.MAP);
Assert.assertTrue(loConst.getValue() instanceof Map);
Assert.assertTrue(loConst.getValue().equals(new HashMap<String,Object>()));
- Schema.FieldSchema mapFs = new Schema.FieldSchema(null, null, DataType.MAP);
- Schema expectedSchema = new Schema(mapFs);
-
- Assert.assertTrue(Schema.equals(foreach.getSchema(), expectedSchema, false, true));
+ Assert.assertTrue( foreach.getSchema().toString(false).equals(""));
}
@Test
- public void testEmptyBagConst() throws FrontendException{
-
- LogicalPlan lp = buildPlan("a = foreach (load 'b') generate {};");
- LOForEach foreach = (LOForEach) lp.getLeaves().get(0);
- LogicalOperator logOp = foreach.getForEachPlans().get(0).getLeaves().get(0);
- Assert.assertTrue( logOp instanceof LOConst);
+ public void testEmptyBagConst() throws Exception{
+ String query = "a = foreach (load 'b') generate {};" +
+ "store a into 'output';";
+ LogicalPlan lp = buildPlan(query);
+ LOForEach foreach = (LOForEach) lp.getSinks().get(0);
+ LOGenerate gen = (LOGenerate)foreach.getInnerPlan().getSinks().get(0);
+ LogicalExpressionPlan exprPlan = gen.getOutputPlans().get(0);
+ Operator logOp = exprPlan.getSources().get(0);
+ Assert.assertTrue( logOp instanceof ConstantExpression);
- LOConst loConst = (LOConst)logOp;
+ ConstantExpression loConst = (ConstantExpression)logOp;
Assert.assertTrue(loConst.getType() == DataType.BAG);
Assert.assertTrue(loConst.getValue() instanceof DataBag);
Assert.assertTrue(loConst.getValue().equals(BagFactory.getInstance().newDefaultBag()));
- Schema.FieldSchema bagFs = new Schema.FieldSchema(null, null, DataType.BAG);
- Schema expectedSchema = new Schema(bagFs);
-
- Assert.assertTrue(Schema.equals(foreach.getSchema(), expectedSchema, false, true));
+ Assert.assertTrue( foreach.getSchema().toString(false).equals(""));
}
-
+
@Test
- public void testEmptyTupConstRecursive1() throws FrontendException{
-
- LogicalPlan lp = buildPlan("a = foreach (load 'b') generate (());");
- LOForEach foreach = (LOForEach) lp.getLeaves().get(0);
+ public void testEmptyTupConstRecursive1() throws Exception{
+ String query = "a = foreach (load 'b') generate (());" +
+ "store a into 'output';";
+ LogicalPlan lp = buildPlan(query);
+ LOForEach foreach = (LOForEach) lp.getSinks().get(0);
- Schema.FieldSchema tupleFs = new Schema.FieldSchema(null, null, DataType.TUPLE);
- Schema tupleSchema = new Schema(tupleFs);
- Schema.FieldSchema tupleFs2 = new Schema.FieldSchema(null, tupleSchema, DataType.TUPLE);
- Schema expectedSchema = new Schema(tupleFs2);
-
- Assert.assertTrue(Schema.equals(foreach.getSchema(), expectedSchema, false, true));
+ Assert.assertTrue( foreach.getSchema().toString(false).equals(""));
}
@Test
- public void testEmptyTupConstRecursive2() throws FrontendException{
-
- LogicalPlan lp = buildPlan("a = foreach (load 'b') generate ([]);");
- LOForEach foreach = (LOForEach) lp.getLeaves().get(0);
-
- Schema.FieldSchema mapFs = new Schema.FieldSchema(null, null, DataType.MAP);
- Schema tupleSchema = new Schema(mapFs);
- Schema.FieldSchema tupleFs = new Schema.FieldSchema(null, tupleSchema, DataType.TUPLE);
- Schema expectedSchema = new Schema(tupleFs);
+ public void testEmptyTupConstRecursive2() throws Exception{
+ String query = "a = foreach (load 'b') generate ([]);" +
+ "store a into 'output';";
+ LogicalPlan lp = buildPlan( query );
+ LOForEach foreach = (LOForEach) lp.getSinks().get(0);
- Assert.assertTrue(Schema.equals(foreach.getSchema(), expectedSchema, false, true));
+ Assert.assertTrue( foreach.getSchema().toString(false).equals(""));
}
@Test
- public void testEmptyTupConstRecursive3() throws FrontendException{
+ public void testEmptyTupConstRecursive3() throws Exception{
String query = "a = foreach (load 'b') generate ({});" +
"store a into 'output';";
LogicalPlan lp = buildPlan(query);
Operator op = lp.getSinks().get(0);
LOForEach foreach = (LOForEach)lp.getPredecessors(op).get(0);
- Schema.FieldSchema bagFs = new Schema.FieldSchema(null, null, DataType.BAG);
- Schema innerSchema = new Schema(bagFs);
- Schema.FieldSchema outerTupleFs = new Schema.FieldSchema(null,innerSchema,DataType.TUPLE);
- Schema expectedSchema = new Schema(outerTupleFs);
-
-// Assert.assertTrue(Schema.equals(foreach.getSchema(), expectedSchema, false, true));
+ Assert.assertTrue( foreach.getSchema().toString(false).equals(""));
}
@Test
- public void testEmptyBagConstRecursive() throws FrontendException{
+ public void testEmptyBagConstRecursive() throws Exception{
String query = "a = foreach (load 'b') generate {()};" +
- "store a into 'output';";
+ "store a into 'output';";
LogicalPlan lp = buildPlan(query);
Operator op = lp.getSinks().get(0);
LOForEach foreach = (LOForEach)lp.getPredecessors(op).get(0);
- Schema.FieldSchema bagFs = new Schema.FieldSchema(null,null,DataType.TUPLE);
- Schema bagSchema = new Schema(bagFs);
- bagSchema.setTwoLevelAccessRequired(true);
-
- Schema.FieldSchema outerBagFs = new Schema.FieldSchema(null,bagSchema,DataType.BAG);
- Schema expectedSchema = new Schema(outerBagFs);
-
-// Assert.assertTrue(Schema.equals(foreach.getSchema(), expectedSchema, false, true));
+ Assert.assertTrue( foreach.getSchema().toString(false).equals(""));
}
@Test
- public void testRandomEmptyConst(){
+ public void testRandomEmptyConst() throws Exception{
// Various random scripts to test recursive nature of parser with empty constants.
- buildPlan("a = foreach (load 'b') generate {({})};");
- buildPlan("a = foreach (load 'b') generate ({()});");
- buildPlan("a = foreach (load 'b') generate {(),()};");
- buildPlan("a = foreach (load 'b') generate ({},{});");
- buildPlan("a = foreach (load 'b') generate ((),());");
- buildPlan("a = foreach (load 'b') generate ([],[]);");
- buildPlan("a = foreach (load 'b') generate {({},{})};");
- buildPlan("a = foreach (load 'b') generate {([],[])};");
- buildPlan("a = foreach (load 'b') generate (({},{}));");
- buildPlan("a = foreach (load 'b') generate (([],[]));");
+ buildPlan("a = foreach (load 'b') generate {({})}; store a into 'output';");
+ buildPlan("a = foreach (load 'b') generate ({()}); store a into 'output';");
+ buildPlan("a = foreach (load 'b') generate {(),()}; store a into 'output';");
+ buildPlan("a = foreach (load 'b') generate ({},{}); store a into 'output';");
+ buildPlan("a = foreach (load 'b') generate ((),()); store a into 'output';");
+ buildPlan("a = foreach (load 'b') generate ([],[]); store a into 'output';");
+ buildPlan("a = foreach (load 'b') generate {({},{})}; store a into 'output';");
+ buildPlan("a = foreach (load 'b') generate {([],[])}; store a into 'output';");
+ buildPlan("a = foreach (load 'b') generate (({},{})); store a into 'output';");
+ buildPlan("a = foreach (load 'b') generate (([],[])); store a into 'output';");
}
-
+*/
+
+/** The following test is disabled due to PIG-2039
@Test
// See PIG-1024, shall not throw exception
public void testLimitMultipleOutput() throws Exception {
String query = " a = load '1.txt' as (a0:int, a1:int, a2:int);" +
" b = group a by a0;" +
- buildPlan( query + " c = foreach b { c1 = limit a 10;c2 = (c1.a0/c1.a1);c3 = (c1.a0/c1.a2);generate c2, c3;};");
+ " c = foreach b { c1 = limit a 10; c2 = distinct c1.a1; c3 = distinct c1.a2; generate c2, c3;};" +
+ " store c into 'output';";
+ buildPlan( query );
}
-
+*/
+
@Test
public void testCogroupByStarFailure1() throws Exception {
boolean exceptionThrown = false;
@@ -2109,11 +2001,11 @@ public class TestLogicalPlanBuilder {
"store c into 'output';";
buildPlan(query);
} catch (AssertionFailedError e) {
- Assert.assertTrue(e.getMessage().contains("Cogroup/Group by * is only allowed if " +
- "the input has a schema"));
+ Assert.assertTrue(e.getMessage().contains("Cogroup/Group by '*' or 'x..' (range of columns to the end) is only" +
+ " allowed if the input has a schema"));
exceptionThrown = true;
}
- assertEquals("An exception was expected but did " +
+ Assert.assertEquals("An exception was expected but did " +
"not occur", true, exceptionThrown);
}
@@ -2127,11 +2019,10 @@ public class TestLogicalPlanBuilder {
"store c into 'output';";
buildPlan( query );
} catch (AssertionFailedError e) {
- Assert.assertTrue(e.getMessage().contains("Cogroup/Group by * is only allowed if " +
- "the input has a schema"));
+ Assert.assertTrue(e.getMessage().contains("Cogroup/Group by '*' or 'x..' (range of columns to the end) is only allowed if the input has a schema"));
exceptionThrown = true;
}
- assertEquals("An exception was expected but did " +
+ Assert.assertEquals("An exception was expected but did " +
"not occur", true, exceptionThrown);
}
@@ -2142,14 +2033,13 @@ public class TestLogicalPlanBuilder {
String query = " a = load '1.txt' as (a0:int, a1:int);" +
" b = load '2.txt' as (a0:int, a1:chararray); " +
"c = cogroup a by (a0,a1), b by (a0,a1);" +
- "store c into 'output'";
+ "store c into 'output';";
buildPlan( query );
} catch (AssertionFailedError e) {
- Assert.assertTrue(e.getMessage().contains("Cogroup column"));
- Assert.assertTrue(e.getMessage().contains("has incompatible types"));
+ Assert.assertTrue(e.getMessage().contains("Cannot resolve COGroup output schema"));
exceptionThrown = true;
}
- assertEquals("An exception was expected but did " +
+ Assert.assertEquals("An exception was expected but did " +
"not occur", true, exceptionThrown);
}
@@ -2169,8 +2059,8 @@ public class TestLogicalPlanBuilder {
load = (LOLoad)plan.getPredecessors(op).get(0);
Assert.assertTrue(((PigStorageWithSchema)(load).getLoadFunc()).getUDFContextSignature().equals("b"));
}
- */
- private void printPlan(LogicalPlan lp) {
+
+ private void printPlan(LogicalExpressionPlan lp) {
System.err.println( lp.toString() );
}
@@ -2184,7 +2074,6 @@ public class TestLogicalPlanBuilder {
}
}
}
-
return false;
}
@@ -2193,7 +2082,9 @@ public class TestLogicalPlanBuilder {
try {
return Util.buildLp(pigServer, query);
} catch(Throwable t) {
- throw new AssertionFailedError(t.getMessage());
+ Throwable cause = t.getCause();
+ String msg = cause != null ? cause.toString() : t.toString();
+ throw new AssertionFailedError( msg );
}
}
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestMultiQuery.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestMultiQuery.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestMultiQuery.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestMultiQuery.java Thu May 5 19:25:08 2011
@@ -424,7 +424,8 @@ public class TestMultiQuery {
assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
}
}
-
+
+/** Test case is disabled due to PIG-2038
@Test
public void testMultiQueryJiraPig1113() throws Exception {
@@ -710,7 +711,7 @@ public class TestMultiQuery {
assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
}
}
-
+*/
@Test
public void testMultiQueryJiraPig976_6() throws Exception {
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestPartitionFilterPushDown.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestPartitionFilterPushDown.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestPartitionFilterPushDown.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestPartitionFilterPushDown.java Thu May 5 19:25:08 2011
@@ -55,10 +55,11 @@ import org.apache.pig.newplan.PColFilter
import org.apache.pig.newplan.optimizer.PlanOptimizer;
import org.apache.pig.newplan.optimizer.Rule;
import org.apache.pig.newplan.optimizer.Transformer;
+import org.apache.pig.parser.ParserException;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.util.LogUtils;
+import org.apache.pig.impl.util.Utils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -634,8 +635,8 @@ public class TestPartitionFilterPushDown
static Expression partFilter = null;
public TestLoader(String schemaString, String commaSepPartitionCols)
- throws ParseException {
- schema = Util.getSchemaFromString(schemaString);
+ throws ParserException {
+ schema = Utils.getSchemaFromString(schemaString);
partCols = commaSepPartitionCols.split(",");
}
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestPigServer.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestPigServer.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestPigServer.java Thu May 5 19:25:08 2011
@@ -51,6 +51,7 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.util.PropertiesUtil;
+import org.apache.pig.impl.util.Utils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -624,8 +625,8 @@ public class TestPigServer {
// the schema object from the schema string
s = s.replaceAll("^.*\\{", "");
s = s.replaceAll("\\}$", "");
- Schema actual = Util.getSchemaFromString( s);
- Schema expected = Util.getSchemaFromString(
+ Schema actual = Utils.getSchemaFromString( s);
+ Schema expected = Utils.getSchemaFromString(
"site: chararray,count: int," +
"itemCounts::type: chararray,itemCounts::typeCount: int," +
"itemCounts::f: float,itemCounts::m: map[ ]");
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectRange.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectRange.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectRange.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectRange.java Thu May 5 19:25:08 2011
@@ -41,6 +41,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
import org.apache.pig.impl.plan.CompilationMessageCollector;
+import org.apache.pig.impl.util.Utils;
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
import org.apache.pig.newplan.logical.relational.LOCogroup;
import org.apache.pig.newplan.logical.relational.LOJoin;
@@ -191,7 +192,7 @@ public class TestProjectRange {
Schema expectedSch = null;
if(expectedSchStr != null)
- expectedSch = Util.getSchemaFromString(expectedSchStr);
+ expectedSch = Utils.getSchemaFromString(expectedSchStr);
compileAndCompareSchema(expectedSch, query, alias);
@@ -999,7 +1000,7 @@ public class TestProjectRange {
* @throws FrontendException
*/
private Schema getCleanedGroupSchema(String expectedSchStr) throws ParseException, FrontendException {
- Schema expectedSch = Util.getSchemaFromString(expectedSchStr);
+ Schema expectedSch = Utils.getSchemaFromString(expectedSchStr);
expectedSch.getField(0).alias = "group";
if(expectedSch.size() > 1)
expectedSch.getField(1).schema.getField(0).alias = null;
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectStarExpander.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectStarExpander.java?rev=1099930&r1=1099929&r2=1099930&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectStarExpander.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestProjectStarExpander.java Thu May 5 19:25:08 2011
@@ -32,6 +32,7 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.util.Utils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -88,7 +89,7 @@ public class TestProjectStarExpander {
Util.registerMultiLineQuery(pig, query);
- Schema expectedSch = Util.getSchemaFromString("aa : int, bb : int, cc : int, d : int, e : int");
+ Schema expectedSch = Utils.getSchemaFromString("aa : int, bb : int, cc : int, d : int, e : int");
Schema sch = pig.dumpSchema("f");
assertEquals("Checking expected schema", expectedSch, sch);
@@ -99,7 +100,7 @@ public class TestProjectStarExpander {
;
Util.registerMultiLineQuery(pig, query);
- expectedSch = Util.getSchemaFromString("aa : int, bb : int, cc : int, dd : int, ee : int");
+ expectedSch = Utils.getSchemaFromString("aa : int, bb : int, cc : int, dd : int, ee : int");
sch = pig.dumpSchema("f");
assertEquals("Checking expected schema", expectedSch, sch);
Iterator<Tuple> it = pig.openIterator("f");
@@ -129,7 +130,7 @@ public class TestProjectStarExpander {
Util.registerMultiLineQuery(pig, query);
- Schema expectedSch = Util.getSchemaFromString(
+ Schema expectedSch = Utils.getSchemaFromString(
"aa : int, bb : int, cc : int, a : int, b : int, c : int");
Schema sch = pig.dumpSchema("f");
assertEquals("Checking expected schema", expectedSch, sch);