You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2011/05/05 19:04:57 UTC
svn commit: r1099873 - in /pig/branches/branch-0.9: CHANGES.txt
src/org/apache/pig/parser/AliasMasker.g
test/org/apache/pig/test/TestMacroExpansion.java
Author: rding
Date: Thu May 5 17:04:57 2011
New Revision: 1099873
URL: http://svn.apache.org/viewvc?rev=1099873&view=rev
Log:
PIG-1999: Macro alias masker should consider schema context
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g
pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1099873&r1=1099872&r2=1099873&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Thu May 5 17:04:57 2011
@@ -174,6 +174,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1999: Macro alias masker should consider schema context (rding)
+
PIG-1821: UDFContext.getUDFProperties does not handle collisions
in hashcode of udf classname (+ arg hashcodes) (thejas)
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g?rev=1099873&r1=1099872&r2=1099873&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/AliasMasker.g Thu May 5 17:04:57 2011
@@ -41,6 +41,14 @@ import java.util.Set;
@members {
+@Override
+public String getErrorMessage(RecognitionException e, String[] tokenNames) {
+ if (e instanceof ParserValidationException) {
+ return e.toString();
+ }
+ return super.getErrorMessage(e, tokenNames);
+}
+
public void setParams(Set ps, String macro, long idx) {
params = ps;
macroName = macro;
@@ -61,6 +69,8 @@ private String macroName = "";
private long index = 0;
+private boolean inAsOrGenClause = false;
+
} // End of @members
@rulecatch {
@@ -163,11 +173,24 @@ filename
;
as_clause
+@init {
+ inAsOrGenClause = true;
+}
+@after {
+ inAsOrGenClause = false;
+}
: ^( AS field_def_list )
;
field_def
- : ^( FIELD_DEF IDENTIFIER type? )
+ : ^( FIELD_DEF IDENTIFIER type? ) {
+ if (inAsOrGenClause) {
+ if (aliasSeen.contains($IDENTIFIER.text)) {
+ throw new ParserValidationException(input, new SourceLocation((PigParserNode)$field_def.start),
+ "Macro doesn't support user defined schema that contains name that conflicts with alias name: " + $IDENTIFIER.text);
+ }
+ }
+}
;
field_def_list
@@ -220,7 +243,13 @@ rel
: alias | ( op_clause parallel_clause? )
;
-flatten_generated_item
+flatten_generated_item
+@init {
+ inAsOrGenClause = true;
+}
+@after {
+ inAsOrGenClause = false;
+}
: ( flatten_clause | col_range | expr | STAR ) field_def_list?
;
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java?rev=1099873&r1=1099872&r2=1099873&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java Thu May 5 17:04:57 2011
@@ -973,6 +973,8 @@ public class TestMacroExpansion {
validateFailure(macro + script, expectedErr);
}
+
+
// macro doesn't contain return alias
@Test
public void negativeTest5() throws Throwable {
@@ -1047,6 +1049,46 @@ public class TestMacroExpansion {
validateFailure(macro + script, expectedErr);
}
+ // PIG-1999: macro contains schema name that conflicts with alias
+ @Test
+ public void negativeTest7() throws Throwable {
+ String macro = "define toBytearray(in, intermediate) returns e {\n"
+ + "b = load '$in' as (name:chararray, age:long, gpa: float);\n"
+ + "d = load '$intermediate' using BinStorage() as (b:bag{t:tuple(x,y,z)}, t2:tuple(a,b,c));\n"
+ + "$e = foreach d generate COUNT(b), t2.a, t2.b, t2.c;};\n";
+
+ String script = "f = toBytearray ('data', 'output1');\n";
+
+ String expectedErr = "pig script failed to validate: Macro doesn't support user defined schema "
+ + "that contains name that conflicts with alias name: b\n"
+ + "macro content: \n"
+ + "b = load 'data' as (name:chararray, age:long, gpa: float);\n"
+ + "d = load 'output1' using BinStorage() as (b:bag{t:tuple(x,y,z)}, t2:tuple(a,b,c));\n"
+ + "f = foreach d generate COUNT(b), t2.a, t2.b, t2.c;\n";
+
+ validateFailure(macro + script, expectedErr, "pig script failed to validate:");
+ }
+
+ // PIG-1999: macro contains schema name that conflicts with alias
+ @Test
+ public void negativeTest8() throws Throwable {
+ String macro = "define toBytearray(in, intermediate) returns e {\n"
+ + "b = load '$in' as (name:chararray, age:long, gpa: float);\n"
+ + "d = foreach b generate $0 as b:bag{t:tuple(x,y,z)};\n"
+ + "$e = foreach d generate COUNT(b), t2.a, t2.b, t2.c;};\n";
+
+ String script = "f = toBytearray ('data', 'output1');\n";
+
+ String expectedErr = "pig script failed to validate: Macro doesn't support user defined schema "
+ + "that contains name that conflicts with alias name: b\n"
+ + "macro content: \n"
+ + "b = load 'data' as (name:chararray, age:long, gpa: float);\n"
+ + "d = foreach b generate $0 as b:bag{t:tuple(x,y,z)};\n"
+ + "f = foreach d generate COUNT(b), t2.a, t2.b, t2.c;\n";
+
+ validateFailure(macro + script, expectedErr, "pig script failed to validate:");
+ }
+
@Test
public void recursiveMacrosTest() throws Exception {
String macro1 = "define group_and_partition (A, group_key, reducers) returns B {\n" +
@@ -1972,7 +2014,7 @@ public class TestMacroExpansion {
}
}
- private void validateFailure(String piglatin, String expectedErr) throws Throwable {
+ private void validateFailure(String piglatin, String expectedErr, String keyword) throws Throwable {
String scriptFile = "mymacrotest.pig";
try {
@@ -1988,10 +2030,14 @@ public class TestMacroExpansion {
Assert.fail("Expected exception isn't thrown");
} catch (FrontendException e) {
String msg = e.getMessage();
- int pos = msg.indexOf("Reason:");
+ int pos = msg.indexOf(keyword);
Assert.assertEquals(expectedErr, msg.substring(pos));
} finally {
new File(scriptFile).delete();
}
}
+
+ private void validateFailure(String piglatin, String expectedErr) throws Throwable {
+ validateFailure(piglatin, expectedErr, "Reason:");
+ }
}