You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2011/04/01 18:28:58 UTC
svn commit: r1087789 - in /pig/trunk: src/org/apache/pig/parser/
test/org/apache/pig/parser/
Author: rding
Date: Fri Apr 1 16:28:57 2011
New Revision: 1087789
URL: http://svn.apache.org/viewvc?rev=1087789&view=rev
Log:
PIG-1931: Integrate Macro Expansion with New Parser
Modified:
pig/trunk/src/org/apache/pig/parser/PigMacro.java
pig/trunk/src/org/apache/pig/parser/PigParserNode.java
pig/trunk/src/org/apache/pig/parser/PigParserNodeAdaptor.java
pig/trunk/src/org/apache/pig/parser/QueryLexer.g
pig/trunk/src/org/apache/pig/parser/QueryParser.g
pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
pig/trunk/src/org/apache/pig/parser/QueryParserStringStream.java
pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java
pig/trunk/test/org/apache/pig/parser/ParserTestingUtils.java
pig/trunk/test/org/apache/pig/parser/TestQueryLexer.java
pig/trunk/test/org/apache/pig/parser/TestQueryParser.java
Modified: pig/trunk/src/org/apache/pig/parser/PigMacro.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/PigMacro.java?rev=1087789&r1=1087788&r2=1087789&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/PigMacro.java (original)
+++ pig/trunk/src/org/apache/pig/parser/PigMacro.java Fri Apr 1 16:28:57 2011
@@ -149,7 +149,7 @@ public class PigMacro {
CharStream input = null;
try {
// parse macro body into ast
- input = new QueryParserStringStream(in);
+ input = new QueryParserStringStream(in, file);
} catch (IOException e) {
String msg = getErrorMessage(file, line, "Failed to inline macro '"
+ name + "'", e.getMessage() + "\nmacro content: " + in);
@@ -291,7 +291,7 @@ public class PigMacro {
StringBuilder sb = new StringBuilder();
sb.append("<");
if (file != null) {
- sb.append("at ").append(file).append(", ");
+ sb.append("file ").append(file).append(", ");
}
sb.append("line ").append(line).append("> ").append(header);
if (reason != null) {
Modified: pig/trunk/src/org/apache/pig/parser/PigParserNode.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/PigParserNode.java?rev=1087789&r1=1087788&r2=1087789&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/PigParserNode.java (original)
+++ pig/trunk/src/org/apache/pig/parser/PigParserNode.java Fri Apr 1 16:28:57 2011
@@ -26,13 +26,14 @@ public class PigParserNode extends Commo
// the script file this node belongs to
private String fileName = null;
- public PigParserNode(Token t) {
+ public PigParserNode(Token t, String fileName) {
super(t);
+ this.fileName = fileName;
}
public PigParserNode(PigParserNode node) {
super(node);
- this.setFileName(node.getFileName());
+ this.fileName = node.getFileName();
}
@@ -40,13 +41,8 @@ public class PigParserNode extends Commo
return new PigParserNode(this);
}
- public void setFileName(String fileName) {
- this.fileName = fileName;
- }
-
public String getFileName() {
return fileName;
}
-
}
Modified: pig/trunk/src/org/apache/pig/parser/PigParserNodeAdaptor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/PigParserNodeAdaptor.java?rev=1087789&r1=1087788&r2=1087789&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/PigParserNodeAdaptor.java (original)
+++ pig/trunk/src/org/apache/pig/parser/PigParserNodeAdaptor.java Fri Apr 1 16:28:57 2011
@@ -22,9 +22,15 @@ import org.antlr.runtime.tree.CommonTree
public class PigParserNodeAdaptor extends CommonTreeAdaptor {
+ private String source;
+
+ PigParserNodeAdaptor(String source) {
+ this.source = source;
+ }
+
@Override
public Object create(Token t) {
- return new PigParserNode(t);
+ return new PigParserNode(t, source);
}
}
Modified: pig/trunk/src/org/apache/pig/parser/QueryLexer.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryLexer.g?rev=1087789&r1=1087788&r2=1087789&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryLexer.g (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryLexer.g Fri Apr 1 16:28:57 2011
@@ -52,7 +52,7 @@ public String getErrorMessage(Recognitio
@Override
public String getErrorHeader(RecognitionException ex) {
- return QueryParserUtils.generateErrorHeader( ex );
+ return QueryParserUtils.generateErrorHeader( ex, this.getSourceName() );
}
} // End of members.
Modified: pig/trunk/src/org/apache/pig/parser/QueryParser.g
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParser.g?rev=1087789&r1=1087788&r2=1087789&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParser.g (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParser.g Fri Apr 1 16:28:57 2011
@@ -123,7 +123,7 @@ public String getTokenErrorDisplay(Token
@Override
public String getErrorHeader(RecognitionException ex) {
- return QueryParserUtils.generateErrorHeader( ex );
+ return QueryParserUtils.generateErrorHeader( ex, this.getSourceName() );
}
} // End of @members
@@ -210,7 +210,7 @@ macro_return_clause : RETURNS alias (COM
;
macro_body_clause : content
- -> ^(MACRO_BODY { new PigParserNode(new CommonToken(1, $content.text)) } )
+ -> ^(MACRO_BODY { new PigParserNode(new CommonToken(1, $content.text), this.getSourceName()) } )
;
macro_clause : macro_param_clause macro_return_clause macro_body_clause
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1087789&r1=1087788&r2=1087789&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Fri Apr 1 16:28:57 2011
@@ -72,7 +72,9 @@ public class QueryParserDriver {
public LogicalPlan parse(String query) throws ParserException {
LogicalPlan plan = null;
- CommonTokenStream tokenStream = tokenize( query );
+ ScriptState ss = ScriptState.get();
+ CommonTokenStream tokenStream = tokenize(query, ss.getFileName());
+
Tree ast = null;
try {
@@ -111,7 +113,7 @@ public class QueryParserDriver {
line = rd.readLine();
}
- CommonTokenStream tokenStream = tokenize(sb.toString());
+ CommonTokenStream tokenStream = tokenize(sb.toString(), scriptFile);
Tree ast = null;
try {
@@ -144,10 +146,11 @@ public class QueryParserDriver {
return operators;
}
- private static CommonTokenStream tokenize(String query) throws ParserException {
+ private static CommonTokenStream tokenize(String query, String source)
+ throws ParserException {
CharStream input;
try {
- input = new QueryParserStringStream(query);
+ input = new QueryParserStringStream(query, source);
} catch (IOException ex) {
throw new ParserException("Unexpected IOException: "
+ ex.getMessage());
@@ -207,12 +210,6 @@ public class QueryParserDriver {
private Tree expandMacro(Tree ast) throws ParserException {
LOG.debug("Original macro AST:\n" + ast.toStringTree() + "\n");
- ScriptState ss = ScriptState.get();
- if (ss != null) {
- QueryParserUtils.recursiveSetFileName((PigParserNode) ast,
- ss.getFileName());
- }
-
// first insert the import files
while (expandImport(ast))
;
@@ -400,7 +397,7 @@ public class QueryParserDriver {
}
// parse
- CommonTokenStream tokenStream = tokenize(sb.toString());
+ CommonTokenStream tokenStream = tokenize(sb.toString(), fname);
Tree ast = null;
try {
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserStringStream.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserStringStream.java?rev=1087789&r1=1087788&r2=1087789&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserStringStream.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserStringStream.java Fri Apr 1 16:28:57 2011
@@ -27,14 +27,14 @@ import org.antlr.runtime.ANTLRStringStre
* while preserving case.
*/
public class QueryParserStringStream extends ANTLRStringStream {
- public QueryParserStringStream(String input) throws IOException {
+ public QueryParserStringStream(String input, String source) throws IOException {
super( input );
+ this.name = source;
}
@Override
public int LA(int i) {
return QueryParserStreamUtil.LA( i, n, p, data );
}
-
}
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java?rev=1087789&r1=1087788&r2=1087789&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java Fri Apr 1 16:28:57 2011
@@ -176,7 +176,7 @@ public class QueryParserUtils {
static String generateErrorHeader(RecognitionException ex, String fname) {
StringBuilder sb = new StringBuilder();
- sb.append("<At ").append(fname).append(", line ")
+ sb.append("<file ").append(fname).append(", line ")
.append(ex.line).append(", column ")
.append(ex.charPositionInLine).append(">");
return sb.toString();
@@ -192,12 +192,6 @@ public class QueryParserUtils {
List childList = new ArrayList(parent.getChildren());
List macroList = newTree.getChildren();
-
- // set file name
- for (Object obj : macroList) {
- PigParserNode node = (PigParserNode)obj;
- recursiveSetFileName(node, fileName);
- }
while (parent.getChildCount() > 0) {
parent.deleteChild(0);
@@ -212,15 +206,6 @@ public class QueryParserUtils {
}
}
- static void recursiveSetFileName(PigParserNode node, String fname) {
- int n = node.getChildCount();
- for (int i = 0; i < n; i++) {
- Tree t = node.getChild(i);
- recursiveSetFileName((PigParserNode)t, fname);
- }
- node.setFileName(fname);
- }
-
static BufferedReader getImportScriptAsReader(String scriptPath)
throws FileNotFoundException {
File f = new File(scriptPath);
@@ -250,7 +235,8 @@ public class QueryParserUtils {
static QueryParser createParser(CommonTokenStream tokens) {
QueryParser parser = new QueryParser(tokens);
- PigParserNodeAdaptor adaptor = new PigParserNodeAdaptor();
+ PigParserNodeAdaptor adaptor = new PigParserNodeAdaptor(
+ tokens.getSourceName());
parser.setTreeAdaptor(adaptor);
return parser;
}
Modified: pig/trunk/test/org/apache/pig/parser/ParserTestingUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/parser/ParserTestingUtils.java?rev=1087789&r1=1087788&r2=1087789&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/parser/ParserTestingUtils.java (original)
+++ pig/trunk/test/org/apache/pig/parser/ParserTestingUtils.java Fri Apr 1 16:28:57 2011
@@ -30,7 +30,7 @@ import org.apache.pig.newplan.logical.re
public class ParserTestingUtils {
public static CommonTokenStream tokenize(String query) throws IOException, ParsingFailureException {
- CharStream input = new QueryParserStringStream( query );
+ CharStream input = new QueryParserStringStream( query, null );
QueryLexer lexer = new QueryLexer( input );
CommonTokenStream tokens = new CommonTokenStream( lexer );
if( 0 < lexer.getNumberOfSyntaxErrors() )
Modified: pig/trunk/test/org/apache/pig/parser/TestQueryLexer.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/parser/TestQueryLexer.java?rev=1087789&r1=1087788&r2=1087789&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/parser/TestQueryLexer.java (original)
+++ pig/trunk/test/org/apache/pig/parser/TestQueryLexer.java Fri Apr 1 16:28:57 2011
@@ -56,7 +56,7 @@ public class TestQueryLexer {
public void test2() throws IOException {
String query = "A = load 'input' using PigStorage(';');" +
"B = foreach ^ A generate string.concatsep( ';', $1, $2 );";
- CharStream input = new QueryParserStringStream( query );
+ CharStream input = new QueryParserStringStream( query, null );
QueryLexer lexer = new QueryLexer( input );
Token token;
try {
Modified: pig/trunk/test/org/apache/pig/parser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/parser/TestQueryParser.java?rev=1087789&r1=1087788&r2=1087789&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/parser/TestQueryParser.java (original)
+++ pig/trunk/test/org/apache/pig/parser/TestQueryParser.java Fri Apr 1 16:28:57 2011
@@ -18,6 +18,8 @@
package org.apache.pig.parser;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import junit.framework.Assert;
@@ -27,6 +29,8 @@ import org.antlr.runtime.CommonTokenStre
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.Tree;
+import org.apache.pig.PigRunner;
+import org.apache.pig.tools.pigstats.PigStats;
import org.junit.Test;
public class TestQueryParser {
@@ -78,6 +82,27 @@ public class TestQueryParser {
shouldFail("A = load '/Users/gates/test/data/studenttab10'; B = foreach A generate $0, 3.0e10.1;");
}
+ @Test // test error message with file name
+ public void testNagative7() throws IOException {
+ File f1 = new File("myscript.pig");
+ f1.deleteOnExit();
+
+ FileWriter fw1 = new FileWriter(f1);
+ fw1.append("A = loadd '1.txt';");
+ fw1.close();
+
+ String[] args = { "-x", "local", "-c", "myscript.pig" };
+ PigStats stats = PigRunner.run(args, null);
+
+ Assert.assertFalse(stats.isSuccessful());
+
+ String expected = "Error during parsing. <file myscript.pig, line 1, column 10>";
+ String msg = stats.getErrorMessage();
+
+ Assert.assertFalse(msg == null);
+ Assert.assertTrue(msg.startsWith(expected));
+ }
+
@Test
public void test2() throws IOException, RecognitionException {
shouldPass("A = load '/Users/gates/test/data/studenttab10'; B = foreach A generate ( $0 == 0 ? 1 : 0 );");
@@ -265,7 +290,7 @@ public class TestQueryParser {
}
private int parse(String query) throws IOException, RecognitionException {
- CharStream input = new QueryParserStringStream( query );
+ CharStream input = new QueryParserStringStream( query, null );
QueryLexer lexer = new QueryLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);