You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by xu...@apache.org on 2011/04/26 19:13:30 UTC
svn commit: r1096810 - in /pig/branches/branch-0.9: ./
src/org/apache/pig/parser/ test/org/apache/pig/test/
Author: xuefu
Date: Tue Apr 26 17:13:30 2011
New Revision: 1096810
URL: http://svn.apache.org/viewvc?rev=1096810&view=rev
Log:
PIG-1622: DEFINE streaming options are ill defined and not properly documented
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g
pig/branches/branch-0.9/src/org/apache/pig/parser/AstPrinter.g
pig/branches/branch-0.9/src/org/apache/pig/parser/AstValidator.g
pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanGenerator.g
pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g
pig/branches/branch-0.9/test/org/apache/pig/test/TestPigRunner.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestStreaming.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1096810&r1=1096809&r2=1096810&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Tue Apr 26 17:13:30 2011
@@ -32,6 +32,8 @@ PIG-1876: Typed map for Pig (daijy)
IMPROVEMENTS
+PIG-1622: DEFINE streaming options are ill defined and not properly documented (xuefu)
+
PIG-2003: Using keyward as alias doesn't either emit an error or produce a logical plan (xuefu)
PIG-1981: LoadPushDown.pushProjection should pass alias in addition to position (daijy)
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g?rev=1096810&r1=1096809&r2=1096810&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g Tue Apr 26 17:13:30 2011
@@ -121,7 +121,7 @@ define_clause
cmd
: ^( EXECCOMMAND
- ( ship_clause | cache_caluse | input_clause | output_clause | error_clause )* )
+ ( ship_clause | cache_clause | input_clause | output_clause | error_clause )* )
;
ship_clause
@@ -132,7 +132,7 @@ path_list
: QUOTEDSTRING+
;
-cache_caluse
+cache_clause
: ^( CACHE path_list )
;
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/AstPrinter.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/AstPrinter.g?rev=1096810&r1=1096809&r2=1096810&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/AstPrinter.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/AstPrinter.g Tue Apr 26 17:13:30 2011
@@ -99,7 +99,7 @@ define_clause
cmd
: ^( EXECCOMMAND { sb.append($EXECCOMMAND.text); }
- ( ship_clause | cache_caluse | input_clause | output_clause | error_clause )* )
+ ( ship_clause | cache_clause | input_clause | output_clause | error_clause )* )
;
ship_clause
@@ -111,7 +111,7 @@ path_list
(b=QUOTEDSTRING { sb.append(", ").append($b.text); } )*
;
-cache_caluse
+cache_clause
: ^( CACHE { sb.append(" ").append($CACHE.text).append(" ("); } path_list { sb.append(")"); } )
;
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/AstValidator.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/AstValidator.g?rev=1096810&r1=1096809&r2=1096810&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/AstValidator.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/AstValidator.g Tue Apr 26 17:13:30 2011
@@ -75,6 +75,13 @@ throws UndefinedAliasException {
}
}
+private void checkDuplication(int count, CommonTree node) throws ParserValidationException {
+ if( count > 1 ) {
+ throw new ParserValidationException( input, new SourceLocation( (PigParserNode)node ),
+ "Duplicated command option" );
+ }
+}
+
private Set<String> aliases = new HashSet<String>();
} // End of @members
@@ -130,7 +137,21 @@ op_clause : define_clause
define_clause : ^( DEFINE alias ( cmd | func_clause ) )
;
-cmd : ^( EXECCOMMAND ( ship_clause | cache_caluse | input_clause | output_clause | error_clause )* )
+cmd
+@init {
+ int ship = 0;
+ int cache = 0;
+ int in = 0;
+ int out = 0;
+ int error = 0;
+}
+ : ^( EXECCOMMAND ( ship_clause { checkDuplication( ++ship, $ship_clause.start ); }
+ | cache_clause { checkDuplication( ++cache, $cache_clause.start ); }
+ | input_clause { checkDuplication( ++in, $input_clause.start ); }
+ | output_clause { checkDuplication( ++out, $output_clause.start ); }
+ | error_clause { checkDuplication( ++error, $error_clause.start ); }
+ )*
+ )
;
ship_clause : ^( SHIP path_list? )
@@ -139,7 +160,7 @@ ship_clause : ^( SHIP path_list? )
path_list : QUOTEDSTRING+
;
-cache_caluse : ^( CACHE path_list )
+cache_clause : ^( CACHE path_list )
;
input_clause : ^( INPUT stream_cmd+ )
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanGenerator.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanGenerator.g?rev=1096810&r1=1096809&r2=1096810&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanGenerator.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/LogicalPlanGenerator.g Tue Apr 26 17:13:30 2011
@@ -216,7 +216,7 @@ cmd[String alias] returns[StreamingComma
List<String> cachePaths = new ArrayList<String>();
SourceLocation loc = new SourceLocation( (PigParserNode)$cmd.start );
}
- : ^( EXECCOMMAND ( ship_clause[shipPaths] | cache_caluse[cachePaths] | input_clause | output_clause | error_clause )* )
+ : ^( EXECCOMMAND ( ship_clause[shipPaths] | cache_clause[cachePaths] | input_clause | output_clause | error_clause )* )
{
$command = builder.buildCommand( loc, builder.unquote( $EXECCOMMAND.text ), shipPaths,
cachePaths, $input_clause.inputHandleSpecs, $output_clause.outputHandleSpecs,
@@ -232,7 +232,7 @@ path_list[List<String> paths]
: ( QUOTEDSTRING { $paths.add( builder.unquote( $QUOTEDSTRING.text ) ); } )+
;
-cache_caluse[List<String> paths]
+cache_clause[List<String> paths]
: ^( CACHE path_list[$paths] )
;
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g?rev=1096810&r1=1096809&r2=1096810&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g Tue Apr 26 17:13:30 2011
@@ -245,7 +245,7 @@ import_clause : IMPORT^ QUOTEDSTRING
define_clause : DEFINE^ alias ( cmd | func_clause | macro_clause)
;
-cmd : EXECCOMMAND^ ( ship_clause | cache_caluse | input_clause | output_clause | error_clause )*
+cmd : EXECCOMMAND^ ( ship_clause | cache_clause | input_clause | output_clause | error_clause )*
;
ship_clause : SHIP^ LEFT_PAREN! path_list? RIGHT_PAREN!
@@ -255,7 +255,7 @@ path_list : QUOTEDSTRING ( COMMA QUOTEDS
-> QUOTEDSTRING+
;
-cache_caluse : CACHE^ LEFT_PAREN! path_list RIGHT_PAREN!
+cache_clause : CACHE^ LEFT_PAREN! path_list RIGHT_PAREN!
;
input_clause : INPUT^ LEFT_PAREN! stream_cmd_list RIGHT_PAREN!
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestPigRunner.java?rev=1096810&r1=1096809&r2=1096810&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestPigRunner.java Tue Apr 26 17:13:30 2011
@@ -603,17 +603,21 @@ public class TestPigRunner {
@Test // PIG-2006
public void testEmptyFile() throws IOException {
- File f1 = new File("myscript.pig");
-
+ File f1 = new File( PIG_FILE );
+
FileWriter fw1 = new FileWriter(f1);
fw1.close();
- String[] args = { "-x", "local", "-c", "myscript.pig" };
- PigStats stats = PigRunner.run(args, null);
+ try {
+ String[] args = { "-x", "local", "-c", PIG_FILE };
+ PigStats stats = PigRunner.run(args, null);
- Assert.assertTrue(stats.isSuccessful());
- Assert.assertEquals( 0, stats.getReturnCode() );
- Assert.assertEquals( null, stats.getErrorMessage() );
+ Assert.assertTrue(stats.isSuccessful());
+ Assert.assertEquals( 0, stats.getReturnCode() );
+ } finally {
+ new File(PIG_FILE).delete();
+ Util.deleteFile(cluster, OUTPUT_FILE);
+ }
}
@Test
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestStreaming.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestStreaming.java?rev=1096810&r1=1096809&r2=1096810&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestStreaming.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestStreaming.java Tue Apr 26 17:13:30 2011
@@ -33,6 +33,7 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.util.TupleFormat;
+import org.apache.pig.parser.ParserException;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -750,6 +751,40 @@ public class TestStreaming {
}
}
+ @Test
+ public void testNegativeMutipleInput() throws IOException {
+ // Perl script
+ String[] script =
+ new String[] {
+ "#!/usr/bin/perl",
+ "open(INFILE, $ARGV[0]) or die \"Can't open \".$ARGV[0].\"!: $!\";",
+ "while (<INFILE>) {",
+ " chomp $_;",
+ " print STDOUT \"$_\n\";",
+ " print STDERR \"STDERR: $_\n\";",
+ "}",
+ };
+ File command1 = Util.createInputFile("script", "pl", script);
+ String query =
+ "define CMD1 `" + command1.getName() + " foo` " +
+ "ship ('" + Util.encodeEscape(command1.toString()) + "') " +
+ "input('foo' using " + PigStreaming.class.getName() + "(',')) " +
+ "output(stdout using " + PigStreaming.class.getName() + "(',')) " +
+ "input('foo' using " + PigStreaming.class.getName() + "(',')) " +
+ "stderr();";
+
+ try {
+ pigServer.registerQuery( query );
+ } catch(ParserException ex) {
+ String expectedMsg = "pig script failed to validate: Duplicated command option";
+ System.out.println( ex.getMessage() );
+ Assert.assertTrue( ex.getMessage().contains( expectedMsg ) );
+ return;
+ }
+
+ Assert.fail( "Testcase is supposed to fail." );
+ }
+
public static class PigStreamDump implements PigToStream {
public static final String recordDelimiter = "\n";