You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2011/04/20 00:22:42 UTC

svn commit: r1095228 - in /pig/trunk: CHANGES.txt src/org/apache/pig/Main.java src/org/apache/pig/parser/DryRunGruntParser.java src/org/apache/pig/parser/QueryParserDriver.java test/org/apache/pig/test/TestMacroExpansion.java

Author: rding
Date: Tue Apr 19 22:22:42 2011
New Revision: 1095228

URL: http://svn.apache.org/viewvc?rev=1095228&view=rev
Log:
PIG-1987: -dryrun does not work with set

Added:
    pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java
Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/Main.java
    pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
    pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1095228&r1=1095227&r2=1095228&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Apr 19 22:22:42 2011
@@ -158,6 +158,8 @@ PIG-1696: Performance: Use System.arrayc
 
 BUG FIXES
 
+PIG-1987: -dryrun does not work with set (rding)
+
 PIG-1871: Dont throw exception if partition filters cannot be pushed up. (rding)
 
 PIG-1870: HBaseStorage doesn't project correctly (dvryaboy)

Modified: pig/trunk/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1095228&r1=1095227&r2=1095228&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Tue Apr 19 22:22:42 2011
@@ -35,7 +35,6 @@ import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -48,6 +47,7 @@ import jline.ConsoleReader;
 import jline.ConsoleReaderInputStream;
 import jline.History;
 
+import org.antlr.runtime.RecognitionException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -66,6 +66,7 @@ import org.apache.pig.impl.util.LogUtils
 import org.apache.pig.impl.util.ObjectSerializer;
 import org.apache.pig.impl.util.PropertiesUtil;
 import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.parser.DryRunGruntParser;
 import org.apache.pig.parser.QueryParserDriver;
 import org.apache.pig.scripting.ScriptEngine;
 import org.apache.pig.scripting.ScriptEngine.SupportedScriptLang;
@@ -389,9 +390,7 @@ static int run(String args[], PigProgres
                 pin = runParamPreprocessor(properties, in, params, paramFiles,
                         substFile, debug || dryrun || checkScriptOnly);
             if (dryrun) {
-                QueryParserDriver driver = new QueryParserDriver(
-                        pigContext, "0", new HashMap<String, String>());
-                if (driver.dryrun(substFile)) {
+                if (dryrun(substFile, pigContext)) {
                     log.info("Dry run completed. Substituted pig script is at "
                             + substFile
                             + ". Expanded pig script is at "
@@ -518,9 +517,7 @@ static int run(String args[], PigProgres
             substFile = remainders[0] + ".substituted";
             pin = runParamPreprocessor(properties, in, params, paramFiles, substFile, debug || dryrun || checkScriptOnly);
             if (dryrun) {
-                QueryParserDriver driver = new QueryParserDriver(
-                        pigContext, "0", new HashMap<String, String>());
-                if (driver.dryrun(substFile)) {
+                if (dryrun(substFile, pigContext)) {
                     log.info("Dry run completed. Substituted pig script is at "
                             + substFile
                             + ". Expanded pig script is at "
@@ -605,6 +602,25 @@ private static int getReturnCodeForStats
                         : ReturnCode.PARTIAL_FAILURE;   // some jobs have failed
 }
 
+public static boolean dryrun(String scriptFile, PigContext pigContext)
+        throws RecognitionException, IOException {
+    BufferedReader rd = new BufferedReader(new FileReader(scriptFile));
+
+    DryRunGruntParser dryrun = new DryRunGruntParser(rd, scriptFile,
+            pigContext);
+
+    boolean hasMacro = dryrun.parseStopOnError();
+
+    if (hasMacro) {
+        String expandedFile = scriptFile.replace(".substituted",
+                ".expanded");
+        BufferedWriter fw = new BufferedWriter(new FileWriter(expandedFile));
+        fw.append(dryrun.getResult());
+        fw.close();
+    }
+    return hasMacro;
+}
+
 //TODO jz: log4j.properties should be used instead
 private static void configureLog4J(Properties properties, PigContext pigContext) {
     // TODO Add a file appender for the logs

Added: pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java?rev=1095228&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java (added)
+++ pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java Tue Apr 19 22:22:42 2011
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.parser;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.tree.CommonTree;
+import org.antlr.runtime.tree.CommonTreeNodeStream;
+import org.antlr.runtime.tree.Tree;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.tools.pigscript.parser.ParseException;
+import org.apache.pig.tools.pigscript.parser.PigScriptParser;
+
+public class DryRunGruntParser extends PigScriptParser {
+    
+    private StringBuilder sb = new StringBuilder();
+    
+    private PigContext context;
+    
+    private Tree parserTree;
+    
+    private String source;
+    
+    private int toSkip = 0;
+    
+    private boolean done = false;
+    
+    private boolean hasMacro = false;
+    
+    public DryRunGruntParser(Reader stream, String source, PigContext context) {
+        super(stream);
+        this.source = source;
+        this.context = context;
+    }
+
+    public String getResult() { return sb.toString(); }
+    
+    public boolean parseStopOnError() throws IOException {
+        done = false;
+        while (!done) {
+            try {
+                parse();
+            } catch (ParseException e) {
+               throw new ParserException("Dry run parsing failed", e);
+            }
+        }
+        return hasMacro;
+    }
+    
+    @Override
+    public void prompt() {
+
+    }
+
+    @Override
+    protected void quit() {
+        done = true;
+    }
+
+    @Override
+    protected void printAliases() throws IOException {
+
+    }
+
+    @Override
+    protected void processFsCommand(String[] cmdTokens) throws IOException {
+        String cmds = LoadFunc.join((AbstractList<String>)Arrays.asList(cmdTokens), " ");
+        sb.append("fs ").append(cmds).append("\n");
+    }
+
+    @Override
+    protected void processShCommand(String[] cmdTokens) throws IOException {
+        String cmds = LoadFunc.join((AbstractList<String>)Arrays.asList(cmdTokens), " ");
+        sb.append("sh ").append(cmds).append("\n");
+    }
+
+    @Override
+    protected void processDescribe(String alias) throws IOException {
+        sb.append("describe ").append(alias).append(";\n");
+    }
+
+    @Override
+    protected void processExplain(String alias, String script,
+            boolean isVerbose, String format, String target,
+            List<String> params, List<String> files) throws IOException,
+            ParseException {
+        sb.append("explain ");
+        if (script != null) {
+            sb.append("-script ").append(script).append(" ");
+        }
+        if (target != null) {
+            sb.append("-out ").append(target).append(" ");
+        }
+        if (isVerbose) sb.append("-brief ");
+        if (format != null && format.equals("dot")) {
+            sb.append("-dot ");
+        }
+        if (params != null) {
+            for (String param : params) {
+                sb.append("-param ").append(param).append(" ");
+            }
+        }
+        if (files != null) {
+            for (String file : files) {
+                sb.append("-param_file ").append(file).append(" ");
+            }
+        }
+        if (alias != null) {
+            sb.append(alias);
+        }
+        sb.append("\n");
+    }
+
+    @Override
+    protected void processRegister(String jar) throws IOException {
+        sb.append("register ").append(jar).append(";\n");
+    }
+
+    @Override
+    protected void processRegister(String path, String scriptingEngine,
+            String namespace) throws IOException, ParseException {
+        sb.append("register '").append(path).append("'");
+        if (scriptingEngine != null) {
+            sb.append(" using ").append(scriptingEngine);
+        }
+        if (namespace != null) {
+            sb.append(" as ").append(namespace);
+        }
+        sb.append(";\n");
+
+    }
+
+    @Override
+    protected void processSet(String key, String value) throws IOException,
+            ParseException {
+        sb.append("set ").append(key).append(" ").append(value).append("\n");
+    }
+
+    @Override
+    protected void processCat(String path) throws IOException {
+        sb.append("cat ").append(path).append("\n");
+    }
+
+    @Override
+    protected void processCD(String path) throws IOException {
+        sb.append("cd ").append(path).append("\n");
+    }
+
+    @Override
+    protected void processDump(String alias) throws IOException {
+        sb.append("dump ").append(alias).append("\n");
+    }
+
+    @Override
+    protected void processKill(String jobid) throws IOException {
+        sb.append("kill ").append(jobid).append("\n");
+    }
+
+    @Override
+    protected void processLS(String path) throws IOException {
+        sb.append("ls");
+        if (path != null) sb.append(" ").append(path);
+        sb.append("\n");
+    }
+
+    @Override
+    protected void processPWD() throws IOException {
+        sb.append("pwd\n");
+    }
+
+    @Override
+    protected void printHelp() {
+
+    }
+
+    @Override
+    protected void processMove(String src, String dst) throws IOException {
+        sb.append("mv ").append(src).append(" ").append(dst).append("\n");
+    }
+
+    @Override
+    protected void processCopy(String src, String dst) throws IOException {
+        sb.append("cp ").append(src).append(" ").append(dst).append("\n");
+    }
+
+    @Override
+    protected void processCopyToLocal(String src, String dst)
+            throws IOException {
+        sb.append("CopyToLocal ").append(src).append(" ").append(dst).append("\n");
+    }
+
+    @Override
+    protected void processCopyFromLocal(String src, String dst)
+            throws IOException {
+        sb.append("CopyFrom,Local ").append(src).append(" ").append(dst).append("\n");
+    }
+
+    @Override
+    protected void processMkdir(String dir) throws IOException {
+        sb.append("mkdir ").append(dir).append("\n");
+    }
+
+    @Override
+    protected void processPig(String cmd) throws IOException {
+        CommonTokenStream tokenStream = QueryParserDriver.tokenize(cmd, source);
+        Tree ast = null;
+
+        try {
+            ast = QueryParserDriver.parse( tokenStream );
+        } catch(RuntimeException ex) {
+            throw new ParserException( ex.getMessage() );
+        }    
+
+        if (!hasMacro) {
+            List<CommonTree> importNodes = new ArrayList<CommonTree>();
+            List<CommonTree> macroNodes = new ArrayList<CommonTree>();
+            List<CommonTree> inlineNodes = new ArrayList<CommonTree>();
+            
+            QueryParserDriver.traverseImport(ast, importNodes);
+            QueryParserDriver.traverse(ast, macroNodes, inlineNodes);
+            
+            if (!importNodes.isEmpty() || !macroNodes.isEmpty()
+                            || !inlineNodes.isEmpty()) {
+                hasMacro = true;
+            }
+        }
+
+        if (parserTree == null) {
+            parserTree = ast;
+        } else {
+            int n = ast.getChildCount();
+            for (int i = 0; i < n; i++) {
+                parserTree.addChild(ast.getChild(i));
+            }
+        }
+ 
+        CommonTree dup = (CommonTree)parserTree.dupNode();
+        dup.addChildren(((CommonTree)parserTree).getChildren());
+
+        QueryParserDriver driver = new QueryParserDriver(context, "0",
+                new HashMap<String, String>());
+        Tree newAst = driver.expandMacro(dup);
+        
+        CommonTreeNodeStream nodes = new CommonTreeNodeStream( newAst );
+        AstPrinter walker = new AstPrinter( nodes );
+        try {
+            walker.query();
+        } catch (RecognitionException e) {
+            throw new ParserException("Failed to print AST for command " + cmd,
+                    e);
+        }
+
+        String result = walker.getResult().trim();
+       
+        if (!result.isEmpty()) {
+            String[] lines = result.split("\n");
+            for (int i = toSkip; i < lines.length; i++) {
+                sb.append(lines[i]).append("\n");
+                toSkip++;
+            }   
+        }
+    }
+
+    @Override
+    protected void processRemove(String path, String opt) throws IOException {
+        if (opt == null || !opt.equalsIgnoreCase("force")) {
+            sb.append("rm ");
+        } else {
+            sb.append("rmf ");
+        }
+        sb.append(path).append("\n");
+    }
+
+    @Override
+    protected void processIllustrate(String alias, String script,
+            String target, List<String> params, List<String> files)
+            throws IOException, ParseException {
+        sb.append("illustrate ");
+        if (script != null) {
+            sb.append("-script ").append(script).append(" ");
+        }
+        if (target != null) {
+            sb.append("-out ").append(target).append(" ");
+        }
+        if (params != null) {
+            for (String param : params) {
+                sb.append("-param ").append(param).append(" ");
+            }
+        }
+        if (files != null) {
+            for (String file : files) {
+                sb.append("-param_file ").append(file).append(" ");
+            }
+        }
+        if (alias != null) {
+            sb.append(alias);
+        }
+        sb.append("\n");
+    }
+
+    @Override
+    protected void processScript(String script, boolean batch,
+            List<String> params, List<String> files) throws IOException,
+            ParseException {
+        if (batch) sb.append("exec ");
+        else sb.append("run ");
+        if (params != null) {
+            for (String param : params) {
+                sb.append("-param ").append(param).append(" ");
+            }
+        }
+        if (files != null) {
+            for (String file : files) {
+                sb.append("-param_file ").append(file).append(" ");
+            }
+        }
+        sb.append(script).append("\n");
+    }
+    
+}

Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1095228&r1=1095227&r2=1095228&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Tue Apr 19 22:22:42 2011
@@ -43,6 +43,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.newplan.Operator;
 import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.tools.pigscript.parser.ParseException;
 import org.apache.pig.tools.pigstats.ScriptState;
 
 public class QueryParserDriver {
@@ -103,50 +104,12 @@ public class QueryParserDriver {
         
         return plan;
     }
-    
-    public boolean dryrun(String scriptFile) throws ParserException, IOException, RecognitionException {
-        BufferedReader rd = new BufferedReader(new FileReader(scriptFile));
-        StringBuilder sb = new StringBuilder();
-        String line = rd.readLine();
-        while (line != null) {
-            sb.append(line).append("\n");
-            line = rd.readLine();
-        }
-        
-        CommonTokenStream tokenStream = tokenize(sb.toString(),  scriptFile);
-        Tree ast = null;
-            
-        try {
-            ast = parse( tokenStream );
-        } catch(RuntimeException ex) {
-            throw new ParserException( ex.getMessage() );
-        }          
-        
-        List<CommonTree> importNodes = new ArrayList<CommonTree>();
-        List<CommonTree> macroNodes = new ArrayList<CommonTree>();
-        List<CommonTree> inlineNodes = new ArrayList<CommonTree>();
         
-        traverseImport(ast, importNodes);
-        traverse(ast, macroNodes, inlineNodes);
-        
-        if (importNodes.isEmpty() && macroNodes.isEmpty()
-                && inlineNodes.isEmpty()) {
-            return false;
-        }
-
-        ast = expandMacro(ast);
-
-        String expandedFile = scriptFile.replace(".substituted", ".expanded");
-        dryrun(ast, expandedFile);
-        
-        return true;
-    }
-    
     public Map<String, Operator> getOperators() {
         return operators;
     }
 
-    private static CommonTokenStream tokenize(String query, String source)
+    static CommonTokenStream tokenize(String query, String source)
             throws ParserException {
         CharStream input;
         try {
@@ -169,7 +132,7 @@ public class QueryParserDriver {
                     + " parsing errors in the query");
     }
 
-    private static Tree parse(CommonTokenStream tokens) throws ParserException {
+    static Tree parse(CommonTokenStream tokens) throws ParserException {
         QueryParser parser = QueryParserUtils.createParser(tokens);
 
         QueryParser.query_return result = null;
@@ -187,15 +150,6 @@ public class QueryParserDriver {
         return ast;
     }
     
-    private static void dryrun(Tree ast, String scriptFile) throws RecognitionException, IOException {
-        CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
-        AstPrinter walker = new AstPrinter( nodes );
-        walker.query();
-        BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
-        fw.append(walker.getResult());
-        fw.close();
-    }
-    
     private static Tree validateAst(Tree ast) throws RecognitionException, ParserException {
         CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
         AstValidator walker = new AstValidator( nodes );
@@ -207,7 +161,7 @@ public class QueryParserDriver {
         return newAst;
     }
     
-    private Tree expandMacro(Tree ast) throws ParserException {
+    Tree expandMacro(Tree ast) throws ParserException {
         LOG.debug("Original macro AST:\n" + ast.toStringTree() + "\n");
 
         // first insert the import files
@@ -277,7 +231,7 @@ public class QueryParserDriver {
         return true;
     }
     
-    private void traverseImport(Tree t, List<CommonTree> nodes) {
+    static void traverseImport(Tree t, List<CommonTree> nodes) {
         if (t.getText().equalsIgnoreCase(IMPORT_DEF)) {
             nodes.add((CommonTree)t);
         }
@@ -288,7 +242,7 @@ public class QueryParserDriver {
         }
     }
     
-    private void traverse(Tree t, List<CommonTree> macroNodes,
+    static void traverse(Tree t, List<CommonTree> macroNodes,
             List<CommonTree> inlineNodes) {
         if (t.getText().equals(MACRO_DEF)) {
             macroNodes.add((CommonTree) t.getParent());  

Modified: pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java?rev=1095228&r1=1095227&r2=1095228&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java Tue Apr 19 22:22:42 2011
@@ -1688,6 +1688,42 @@ public class TestMacroExpansion {
         verify(query, expected);
     }
     
+    // PIG-1987
+    @Test
+    public void test37() throws Exception {
+        String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
+        "    $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
+        "};\n";
+    
+        File f = new File("mytest.pig");
+        f.deleteOnExit();
+        
+        FileWriter fw = new FileWriter(f);
+        fw.append(macro);
+        fw.close();
+        
+        String script =
+            "set default_parallel 10\n" +
+            "import 'mytest.pig';\n" +
+            "alpha = load 'users' as (user, age, zip);\n" +
+            "ls\n" +
+            "gamma = group_and_count (alpha, user, 23);\n" +
+            "fs -copyFromLocal test test2\n" +
+            "store gamma into 'byuser';\n";
+        
+        String expected =
+            "set default_parallel 10\n" +
+            "alpha = load 'users' as (user, age, zip);\n" +
+            "ls\n" +
+            "gamma = distinct alpha partition BY org.apache.pig.test.utils.SimpleCustomPartitioner parallel 23;\n" +
+            "fs -copyFromLocal test test2\n" +
+            "store gamma INTO 'byuser';\n";
+            
+        verify(script, expected);
+    }
+    
+
+    
     @Test
     public void testCommentInMacro() throws Exception {
         String query = "a = load 'testComplexCast' as (m);\n" +