You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2011/05/25 21:50:27 UTC
svn commit: r1127655 - in /pig/branches/branch-0.9: ./
src/org/apache/pig/parser/ test/org/apache/pig/parser/
test/org/apache/pig/test/
Author: thejas
Date: Wed May 25 19:50:27 2011
New Revision: 1127655
URL: http://svn.apache.org/viewvc?rev=1127655&view=rev
Log:
PIG-2083: bincond ERROR 1025: Invalid field projection when null is used
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/parser/QueryLexer.g
pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g
pig/branches/branch-0.9/test/org/apache/pig/parser/TestAST.pig
pig/branches/branch-0.9/test/org/apache/pig/parser/TestQueryParser.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestNullConstant.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1127655&r1=1127654&r2=1127655&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Wed May 25 19:50:27 2011
@@ -184,6 +184,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-2083: bincond ERROR 1025: Invalid field projection when null is used (thejas)
+
PIG-2089: Javadoc for ResourceFieldSchema.getSchema() is wrong (daijy)
PIG-2084: pig is running validation for a statement at a time batch mode,
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/QueryLexer.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/QueryLexer.g?rev=1127655&r1=1127654&r2=1127655&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/QueryLexer.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/QueryLexer.g Wed May 25 19:50:27 2011
@@ -186,9 +186,6 @@ MAP : 'MAP'
IS : 'IS'
;
-NULL : 'NULL'
-;
-
STREAM : 'STREAM'
;
@@ -291,7 +288,7 @@ fragment ID: LETTER ( DIGIT | LETTER | S
DCOLON : '::'
;
-IDENTIFIER : ( ID DCOLON ) => ( ID DCOLON IDENTIFIER )
+IDENTIFIER_L : ( ID DCOLON ) => ( ID DCOLON IDENTIFIER_L )
| ID
;
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g?rev=1127655&r1=1127654&r2=1127655&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g Wed May 25 19:50:27 2011
@@ -65,6 +65,8 @@ tokens {
MACRO_DEF;
MACRO_BODY;
MACRO_INLINE;
+ NULL;
+ IDENTIFIER;
}
@header {
@@ -179,11 +181,11 @@ foreach_simple_statement : ( alias EQUAL
-> ^( STATEMENT alias? foreach_clause_simple parallel_clause? )
;
-alias : IDENTIFIER
+alias : identifier
;
parameter
- : IDENTIFIER
+ : identifier
| INTEGER
| DOUBLENUMBER
| QUOTEDSTRING
@@ -283,8 +285,8 @@ filename : QUOTEDSTRING
as_clause: AS^ ( field_def | ( LEFT_PAREN! field_def_list RIGHT_PAREN! ) )
;
-field_def : IDENTIFIER ( COLON type )?
- -> ^( FIELD_DEF IDENTIFIER type? )
+field_def : identifier ( COLON type )?
+ -> ^( FIELD_DEF identifier type? )
;
field_def_list : field_def ( COMMA field_def )*
@@ -301,7 +303,7 @@ tuple_type : TUPLE? LEFT_PAREN field_def
-> ^( TUPLE_TYPE field_def_list? )
;
-bag_type : BAG? LEFT_CURLY ( ( IDENTIFIER COLON )? tuple_type )? RIGHT_CURLY
+bag_type : BAG? LEFT_CURLY ( ( identifier COLON )? tuple_type )? RIGHT_CURLY
-> ^( BAG_TYPE tuple_type? )
;
@@ -387,7 +389,7 @@ real_arg_list : real_arg ( COMMA real_ar
real_arg : expr | STAR | col_range
;
-null_check_cond : expr IS! NOT? NULL^
+null_check_cond : expr IS! NOT? null_keyword^
;
expr : add_expr
@@ -438,7 +440,7 @@ dot_proj : PERIOD ( col_alias_or_index
col_alias_or_index : col_alias | col_index
;
-col_alias : GROUP | IDENTIFIER
+col_alias : GROUP | identifier
;
col_index : DOLLARVAR
@@ -451,7 +453,7 @@ col_range : c1 = col_ref DOUBLE_PERIOD c
;
-pound_proj : POUND^ ( QUOTEDSTRING | NULL )
+pound_proj : POUND^ ( QUOTEDSTRING | null_keyword )
;
bin_expr : LEFT_PAREN cond QMARK exp1 = expr COLON exp2 = expr RIGHT_PAREN
@@ -554,12 +556,12 @@ nested_command_list : ( nested_command S
|
;
-nested_command : ( IDENTIFIER EQUAL col_ref PERIOD col_ref_list { input.LA( 1 ) == SEMI_COLON }? ) => ( IDENTIFIER EQUAL nested_proj )
- -> ^( NESTED_CMD IDENTIFIER nested_proj )
- | IDENTIFIER EQUAL expr
- -> ^( NESTED_CMD_ASSI IDENTIFIER expr )
- | IDENTIFIER EQUAL nested_op
- -> ^( NESTED_CMD IDENTIFIER nested_op )
+nested_command : ( identifier EQUAL col_ref PERIOD col_ref_list { input.LA( 1 ) == SEMI_COLON }? ) => ( identifier EQUAL nested_proj )
+ -> ^( NESTED_CMD identifier nested_proj )
+ | identifier EQUAL expr
+ -> ^( NESTED_CMD_ASSI identifier expr )
+ | identifier EQUAL nested_op
+ -> ^( NESTED_CMD identifier nested_op )
;
nested_op : nested_filter
@@ -608,7 +610,7 @@ split_branch : alias IF cond
col_ref : alias_col_ref | dollar_col_ref
;
-alias_col_ref : GROUP | IDENTIFIER
+alias_col_ref : GROUP | identifier
;
dollar_col_ref : DOLLARVAR
@@ -621,7 +623,7 @@ literal : scalar | map | bag | tuple
;
-scalar : num_scalar | QUOTEDSTRING | NULL
+scalar : num_scalar | QUOTEDSTRING | null_keyword
;
num_scalar : MINUS? ( INTEGER | LONGINTEGER | FLOATNUMBER | DOUBLENUMBER )
@@ -697,7 +699,6 @@ eid : rel_str_op
| TUPLE
| MAP
| IS
- | NULL
| STREAM
| THROUGH
| STORE
@@ -714,7 +715,8 @@ eid : rel_str_op
| LEFT
| RIGHT
| FULL
- | IDENTIFIER
+ | identifier
+ | null_keyword
;
// relational operator
@@ -754,3 +756,10 @@ rel_str_op : STR_OP_EQ
| STR_OP_MATCHES
;
+null_keyword : {input.LT(1).getText().equalsIgnoreCase("NULL")}? IDENTIFIER_L
+ -> NULL[$IDENTIFIER_L]
+;
+
+identifier : {!input.LT(1).getText().equalsIgnoreCase("NULL")}? IDENTIFIER_L
+ -> IDENTIFIER[$IDENTIFIER_L]
+;
Modified: pig/branches/branch-0.9/test/org/apache/pig/parser/TestAST.pig
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/parser/TestAST.pig?rev=1127655&r1=1127654&r2=1127655&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/parser/TestAST.pig (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/parser/TestAST.pig Wed May 25 19:50:27 2011
@@ -16,7 +16,7 @@
* limitations under the License.
*/
- A = LOAD 'source.txt' USING org.apache.pig.TextLoader() AS (a:int, b:long, c:chararray);
+ A = LOAD 'source.txt' USING org.apache.pig.builtin.TextLoader() AS (a:int, b:long, c:chararray);
B = FOREACH A generate $0 * 5, b, $2;
C = FILTER B by a > 100;
D = LIMIT C 400;
Modified: pig/branches/branch-0.9/test/org/apache/pig/parser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/parser/TestQueryParser.java?rev=1127655&r1=1127654&r2=1127655&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/parser/TestQueryParser.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/parser/TestQueryParser.java Wed May 25 19:50:27 2011
@@ -29,7 +29,11 @@ import org.antlr.runtime.CommonTokenStre
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.Tree;
+import org.apache.pig.ExecType;
import org.apache.pig.PigRunner;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.test.Util;
import org.apache.pig.tools.pigstats.PigStats;
import org.junit.Test;
@@ -201,6 +205,15 @@ public class TestQueryParser {
int errorCount = parse( query );
Assert.assertTrue( errorCount == 0 );
}
+
+ @Test //PIG-2083
+ public void testNullInBinCondNoSpace() throws IOException{
+ String query = "a = load '1.txt' as (a0, a1);" +
+ "b = foreach a generate (a0==0?null:2);"; //no space around the null keyword
+ PigServer pig = new PigServer(ExecType.LOCAL);
+ Util.registerMultiLineQuery(pig, query);
+ pig.explain("b", System.out);
+ }
@Test
public void testAST() throws IOException, RecognitionException {
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java?rev=1127655&r1=1127654&r2=1127655&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java Wed May 25 19:50:27 2011
@@ -44,8 +44,6 @@ import org.junit.Test;
public class TestMacroExpansion {
- private static final MiniCluster cluster = MiniCluster.buildCluster();
-
static File command;
@BeforeClass
@@ -66,7 +64,6 @@ public class TestMacroExpansion {
@AfterClass
public static void tearDownAfterClass() throws Exception {
- cluster.shutDown();
}
@Before
@@ -1231,7 +1228,7 @@ public class TestMacroExpansion {
String expectedErr =
- "<file myscript.pig, line 9, column 17> mismatched input '$0' expecting set null";
+ "<file myscript.pig, line 9, column 17> Syntax error, unexpected symbol at or near '$0'";
validateDryrunFailure(sb.toString(), expectedErr, "<file");
}
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestNullConstant.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestNullConstant.java?rev=1127655&r1=1127654&r2=1127655&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestNullConstant.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestNullConstant.java Wed May 25 19:50:27 2011
@@ -124,7 +124,7 @@ public class TestNullConstant extends Te
pigServer.registerQuery("a = load '" + inputFileName1 + "' as (x:int, y:chararray);");
pigServer.registerQuery("b = load '" + inputFileName2 + "' as (u:int, v:chararray);");
pigServer.registerQuery("c = cogroup a by x, b by u;");
- pigServer.registerQuery("d = foreach c generate flatten((SIZE(a) == 0 ? null : a)), flatten((SIZE(b) == 0 ? null : b));");
+ pigServer.registerQuery("d = foreach c generate flatten((SIZE(a) == 0 ? null: a)), flatten((SIZE(b) == 0 ? null : b));");
Iterator<Tuple> it = pigServer.openIterator("d");
Object[][] results = new Object[][]{{10, "will_join", 10, "will_join"}, {11, "will_not_join", null}, {null, 12, "will_not_join"}};
int i = 0;