You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ju...@apache.org on 2012/08/31 01:03:38 UTC
svn commit: r1379210 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/parser/QueryParserDriver.java
Author: julien
Date: Thu Aug 30 23:03:37 2012
New Revision: 1379210
URL: http://svn.apache.org/viewvc?rev=1379210&view=rev
Log:
PIG-2896 Pig does not fail anymore if two macros are declared with the same name
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1379210&r1=1379209&r2=1379210&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Aug 30 23:03:37 2012
@@ -254,6 +254,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-2896: Pig does not fail anymore if two macros are declared with the same name (julien)
+
PIG-2848: TestBuiltInBagToTupleOrString fails now that mock.Storage enforces not overwriting output (julien)
PIG-2884: JobControlCompiler mis-logs after reducer estimation (billgraham)
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1379210&r1=1379209&r2=1379210&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Thu Aug 30 23:03:37 2012
@@ -58,28 +58,28 @@ import org.apache.pig.tools.pigstats.Scr
public class QueryParserDriver {
private static final Log LOG = LogFactory.getLog(QueryParserDriver.class);
-
+
private static final String MACRO_DEF = "MACRO_DEF";
private static final String MACRO_INLINE = "MACRO_INLINE";
private static final String IMPORT_DEF = "import";
-
+
private PigContext pigContext;
private String scope;
private Map<String, String>fileNameMap;
private Map<String, Operator> operators;
private Set<String> importSeen;
private Set<String> macroSeen;
-
+
private static Map<String, FetchFileRet> fnameMap = new HashMap<String, FetchFileRet>();
-
+
public QueryParserDriver(PigContext pigContext, String scope, Map<String, String> fileNameMap) {
this.pigContext = pigContext;
this.scope = scope;
this.fileNameMap = fileNameMap;
importSeen = new HashSet<String>();
- macroSeen = new HashSet<String>();
+ macroSeen = new HashSet<String>();
}
-
+
private static Tree parseSchema(CommonTokenStream tokens) throws ParserException {
QueryParser parser = QueryParserUtils.createParser(tokens);
@@ -104,14 +104,14 @@ public class QueryParserDriver {
CommonTokenStream tokenStream = tokenize( input, null );
LogicalSchema schema = null;
Tree ast = parseSchema( tokenStream );
-
- try{
+
+ try{
CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
AstValidator walker = new AstValidator( nodes );
ast = (Tree)walker.field_def_list().getTree();
checkError( walker );
-
- LogicalPlanGenerator planGenerator =
+
+ LogicalPlanGenerator planGenerator =
new LogicalPlanGenerator( new CommonTreeNodeStream( ast ), pigContext, scope, fileNameMap );
schema = planGenerator.field_def_list().schema;
checkError( planGenerator );
@@ -120,7 +120,7 @@ public class QueryParserDriver {
} catch(Exception ex) {
throw new ParserException( ex.getMessage(), ex );
}
-
+
return schema;
}
@@ -148,14 +148,14 @@ public class QueryParserDriver {
CommonTokenStream tokenStream = tokenize( input, null );
Object value = null;
Tree ast = parseConstant( tokenStream );
-
- try{
+
+ try{
CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
AstValidator walker = new AstValidator( nodes );
ast = (Tree)walker.literal().getTree();
checkError( walker );
- LogicalPlanGenerator planGenerator =
+ LogicalPlanGenerator planGenerator =
new LogicalPlanGenerator( new CommonTreeNodeStream( ast ), pigContext, scope, fileNameMap );
value = planGenerator.literal().value;
checkError( planGenerator );
@@ -164,27 +164,27 @@ public class QueryParserDriver {
} catch(Exception ex) {
throw new ParserException( ex.getMessage(), ex );
}
-
+
return value;
}
public LogicalPlan parse(String query) throws ParserException {
LogicalPlan plan = null;
-
+
ScriptState ss = ScriptState.get();
CommonTokenStream tokenStream = tokenize(query, ss.getFileName());
-
+
Tree ast = parse( tokenStream );
ast = expandMacro( ast );
- try{
+ try{
ast = validateAst( ast );
- LogicalPlanGenerator planGenerator =
+ LogicalPlanGenerator planGenerator =
new LogicalPlanGenerator( new CommonTreeNodeStream( ast ), pigContext, scope, fileNameMap );
planGenerator.query();
-
+
checkError( planGenerator );
-
+
plan = planGenerator.getLogicalPlan();
operators = planGenerator.getOperators();
} catch(RecognitionException ex) {
@@ -192,10 +192,10 @@ public class QueryParserDriver {
} catch(Exception ex) {
throw new ParserException( ex.getMessage(), ex );
}
-
+
return plan;
}
-
+
public Map<String, Operator> getOperators() {
return operators;
}
@@ -214,7 +214,7 @@ public class QueryParserDriver {
checkError(lexer);
return tokens;
}
-
+
private static void checkError(BaseRecognizer recognizer)
throws ParserException {
int errorCount = recognizer.getNumberOfSyntaxErrors();
@@ -235,25 +235,25 @@ public class QueryParserDriver {
throw new ParserException(msg);
} catch(RuntimeException ex) {
throw new ParserException( ex.getMessage() );
- }
+ }
Tree ast = (Tree) result.getTree();
checkError(parser);
return ast;
}
-
+
private static Tree validateAst(Tree ast) throws RecognitionException, ParserException {
CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
AstValidator walker = new AstValidator( nodes );
AstValidator.query_return newResult = walker.query();
Tree newAst = (Tree)newResult.getTree();
-
+
checkError( walker );
-
+
return newAst;
}
-
+
Tree expandMacro(Tree ast) throws ParserException {
LOG.debug("Original macro AST:\n" + ast.toStringTree() + "\n");
@@ -284,16 +284,16 @@ public class QueryParserDriver {
return ast;
}
-
+
private void inlineMacro(List<CommonTree> inlineNodes,
List<PigMacro> macroDefs) throws ParserException {
for (CommonTree t : inlineNodes) {
Set<String> macroStack = new HashSet<String>();
CommonTree newTree = PigMacro.macroInline(t, macroDefs, macroStack);
-
+
List<CommonTree> nodes = new ArrayList<CommonTree>();
traverseInline(newTree, nodes);
-
+
if (nodes.isEmpty()) {
QueryParserUtils.replaceNodeWithNodeList(t, newTree, null);
} else {
@@ -301,7 +301,7 @@ public class QueryParserDriver {
}
}
}
-
+
private void traverseInline(Tree t, List<CommonTree> nodes) {
if (t.getText().equals(MACRO_INLINE)) {
nodes.add((CommonTree)t);
@@ -310,21 +310,21 @@ public class QueryParserDriver {
for (int i = 0; i < n; i++) {
Tree t0 = t.getChild(i);
traverseInline(t0, nodes);
- }
+ }
}
-
+
private boolean expandImport(Tree ast) throws ParserException {
List<CommonTree> nodes = new ArrayList<CommonTree>();
traverseImport(ast, nodes);
if (nodes.isEmpty()) return false;
-
+
for (CommonTree t : nodes) {
macroImport(t);
}
-
+
return true;
}
-
+
static void traverseImport(Tree t, List<CommonTree> nodes) {
if (t.getText().equalsIgnoreCase(IMPORT_DEF)) {
nodes.add((CommonTree)t);
@@ -335,11 +335,11 @@ public class QueryParserDriver {
traverseImport(t0, nodes);
}
}
-
+
static void traverse(Tree t, List<CommonTree> macroNodes,
List<CommonTree> inlineNodes) {
if (t.getText().equals(MACRO_DEF)) {
- macroNodes.add((CommonTree) t.getParent());
+ macroNodes.add((CommonTree) t.getParent());
} else if (t.getText().equals(MACRO_INLINE)) {
inlineNodes.add((CommonTree) t);
}
@@ -349,7 +349,7 @@ public class QueryParserDriver {
traverse(t0, macroNodes, inlineNodes);
}
}
-
+
private FetchFileRet getMacroFile(String fname) {
FetchFileRet localFileRet = null;
try {
@@ -365,11 +365,11 @@ public class QueryParserDriver {
// ignore this since we'll attempt to load as a resource before failing
LOG.debug(String.format("Macro file %s was not found", fname));
}
-
+
// try loading the macro file as a resource in case it is packaged in a registered jar
if (localFileRet == null) {
LOG.debug(String.format("Attempting to load macro file %s as a resource", fname));
-
+
try
{
localFileRet = FileLocalizer.fetchResource(fname);
@@ -382,7 +382,7 @@ public class QueryParserDriver {
throw new ExecException("file '" + fname + "' does not exist.", 101, PigException.INPUT);
}
}
-
+
fnameMap.put(fname, localFileRet);
}
} catch (IOException e) {
@@ -390,7 +390,7 @@ public class QueryParserDriver {
}
return localFileRet;
}
-
+
/*
* MacroDef node has two child nodes:
* 1. name
@@ -399,13 +399,11 @@ public class QueryParserDriver {
private PigMacro makeMacroDef(CommonTree t, Map<String, PigMacro> seen)
throws ParserException {
String mn = t.getChild(0).getText();
-
+
if (!macroSeen.add(mn)) {
String msg = getErrorMessage(null, t, null,
"Duplicated macro name '" + mn + "'");
- LOG.warn(msg);
- // Disabling this exception, see https://issues.apache.org/jira/browse/PIG-2279
- // throw new ParserException(msg);
+ throw new ParserException(msg);
}
if (seen != null) {
@@ -413,13 +411,13 @@ public class QueryParserDriver {
macroSeen.add(s);
}
}
-
+
String fname = ((PigParserNode)t).getFileName();
-
+
Tree defNode = t.getChild(1);
// get parameter markers
- ArrayList<String> params = new ArrayList<String>();
+ ArrayList<String> params = new ArrayList<String>();
Tree paramNode = defNode.getChild(0);
int n = paramNode.getChildCount();
for (int i = 0; i < n; i++) {
@@ -427,7 +425,7 @@ public class QueryParserDriver {
}
// get return alias markers
- ArrayList<String> returns = new ArrayList<String>();
+ ArrayList<String> returns = new ArrayList<String>();
Tree retNode = defNode.getChild(1);
int m = retNode.getChildCount();
for (int i = 0; i < m; i++) {
@@ -448,7 +446,7 @@ public class QueryParserDriver {
}
PigMacro pm = new PigMacro(mn, fname, params, returns, body, seen);
-
+
try {
pm.validate();
} catch (IOException e) {
@@ -456,11 +454,11 @@ public class QueryParserDriver {
"Invalid macro definition: ", e.getMessage());
throw new ParserException(msg);
}
-
- // set the starting line number of the macro
+
+ // set the starting line number of the macro
PigParserNode pnode = (PigParserNode)bodyNode.getChild(0);
pm.setStartLine(pnode.getStartLine());
-
+
seen.put(mn, pm);
// delete this node
@@ -470,7 +468,7 @@ public class QueryParserDriver {
return pm;
}
-
+
private void macroImport(CommonTree t) throws ParserException {
// remove quote
String fname = t.getChild(0).getText();
@@ -482,9 +480,9 @@ public class QueryParserDriver {
// Disabling this exception, see https://issues.apache.org/jira/browse/PIG-2279
// throw new ParserException(msg);
}
-
+
FetchFileRet localFileRet = getMacroFile(fname);
-
+
BufferedReader in = null;
try {
in = new BufferedReader(new FileReader(localFileRet.file));
@@ -493,7 +491,7 @@ public class QueryParserDriver {
"Failed to import file '" + fname + "'", e.getMessage());
throw new ParserException(msg);
}
-
+
StringBuilder sb = new StringBuilder();
String line = null;
try {
@@ -507,20 +505,20 @@ public class QueryParserDriver {
"Failed to read file '" + fname + "'", e.getMessage());
throw new ParserException(msg);
}
-
+
// parse
CommonTokenStream tokenStream = tokenize(sb.toString(), fname);
-
+
Tree ast = null;
try {
ast = parse( tokenStream );
} catch(RuntimeException ex) {
throw new ParserException( ex.getMessage() );
- }
-
+ }
+
QueryParserUtils.replaceNodeWithNodeList(t, (CommonTree)ast, fname);
}
-
+
private String getErrorMessage(String importFile,
CommonTree t, String header, String reason) {
StringBuilder sb = new StringBuilder();
@@ -539,5 +537,5 @@ public class QueryParserDriver {
sb.append(". Reason: ").append(reason);
}
return sb.toString();
- }
+ }
}