You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2011/04/20 00:22:42 UTC
svn commit: r1095228 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/Main.java src/org/apache/pig/parser/DryRunGruntParser.java
src/org/apache/pig/parser/QueryParserDriver.java
test/org/apache/pig/test/TestMacroExpansion.java
Author: rding
Date: Tue Apr 19 22:22:42 2011
New Revision: 1095228
URL: http://svn.apache.org/viewvc?rev=1095228&view=rev
Log:
PIG-1987: -dryrun does not work with set
Added:
pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/Main.java
pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1095228&r1=1095227&r2=1095228&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Apr 19 22:22:42 2011
@@ -158,6 +158,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1987: -dryrun does not work with set (rding)
+
PIG-1871: Dont throw exception if partition filters cannot be pushed up. (rding)
PIG-1870: HBaseStorage doesn't project correctly (dvryaboy)
Modified: pig/trunk/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1095228&r1=1095227&r2=1095228&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Tue Apr 19 22:22:42 2011
@@ -35,7 +35,6 @@ import java.util.AbstractList;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -48,6 +47,7 @@ import jline.ConsoleReader;
import jline.ConsoleReaderInputStream;
import jline.History;
+import org.antlr.runtime.RecognitionException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -66,6 +66,7 @@ import org.apache.pig.impl.util.LogUtils
import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.impl.util.PropertiesUtil;
import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.parser.DryRunGruntParser;
import org.apache.pig.parser.QueryParserDriver;
import org.apache.pig.scripting.ScriptEngine;
import org.apache.pig.scripting.ScriptEngine.SupportedScriptLang;
@@ -389,9 +390,7 @@ static int run(String args[], PigProgres
pin = runParamPreprocessor(properties, in, params, paramFiles,
substFile, debug || dryrun || checkScriptOnly);
if (dryrun) {
- QueryParserDriver driver = new QueryParserDriver(
- pigContext, "0", new HashMap<String, String>());
- if (driver.dryrun(substFile)) {
+ if (dryrun(substFile, pigContext)) {
log.info("Dry run completed. Substituted pig script is at "
+ substFile
+ ". Expanded pig script is at "
@@ -518,9 +517,7 @@ static int run(String args[], PigProgres
substFile = remainders[0] + ".substituted";
pin = runParamPreprocessor(properties, in, params, paramFiles, substFile, debug || dryrun || checkScriptOnly);
if (dryrun) {
- QueryParserDriver driver = new QueryParserDriver(
- pigContext, "0", new HashMap<String, String>());
- if (driver.dryrun(substFile)) {
+ if (dryrun(substFile, pigContext)) {
log.info("Dry run completed. Substituted pig script is at "
+ substFile
+ ". Expanded pig script is at "
@@ -605,6 +602,25 @@ private static int getReturnCodeForStats
: ReturnCode.PARTIAL_FAILURE; // some jobs have failed
}
+public static boolean dryrun(String scriptFile, PigContext pigContext)
+ throws RecognitionException, IOException {
+ BufferedReader rd = new BufferedReader(new FileReader(scriptFile));
+
+ DryRunGruntParser dryrun = new DryRunGruntParser(rd, scriptFile,
+ pigContext);
+
+ boolean hasMacro = dryrun.parseStopOnError();
+
+ if (hasMacro) {
+ String expandedFile = scriptFile.replace(".substituted",
+ ".expanded");
+ BufferedWriter fw = new BufferedWriter(new FileWriter(expandedFile));
+ fw.append(dryrun.getResult());
+ fw.close();
+ }
+ return hasMacro;
+}
+
//TODO jz: log4j.properties should be used instead
private static void configureLog4J(Properties properties, PigContext pigContext) {
// TODO Add a file appender for the logs
Added: pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java?rev=1095228&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java (added)
+++ pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java Tue Apr 19 22:22:42 2011
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.parser;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.tree.CommonTree;
+import org.antlr.runtime.tree.CommonTreeNodeStream;
+import org.antlr.runtime.tree.Tree;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.tools.pigscript.parser.ParseException;
+import org.apache.pig.tools.pigscript.parser.PigScriptParser;
+
+public class DryRunGruntParser extends PigScriptParser {
+
+ private StringBuilder sb = new StringBuilder();
+
+ private PigContext context;
+
+ private Tree parserTree;
+
+ private String source;
+
+ private int toSkip = 0;
+
+ private boolean done = false;
+
+ private boolean hasMacro = false;
+
+ public DryRunGruntParser(Reader stream, String source, PigContext context) {
+ super(stream);
+ this.source = source;
+ this.context = context;
+ }
+
+ public String getResult() { return sb.toString(); }
+
+ public boolean parseStopOnError() throws IOException {
+ done = false;
+ while (!done) {
+ try {
+ parse();
+ } catch (ParseException e) {
+ throw new ParserException("Dry run parsing failed", e);
+ }
+ }
+ return hasMacro;
+ }
+
+ @Override
+ public void prompt() {
+
+ }
+
+ @Override
+ protected void quit() {
+ done = true;
+ }
+
+ @Override
+ protected void printAliases() throws IOException {
+
+ }
+
+ @Override
+ protected void processFsCommand(String[] cmdTokens) throws IOException {
+ String cmds = LoadFunc.join((AbstractList<String>)Arrays.asList(cmdTokens), " ");
+ sb.append("fs ").append(cmds).append("\n");
+ }
+
+ @Override
+ protected void processShCommand(String[] cmdTokens) throws IOException {
+ String cmds = LoadFunc.join((AbstractList<String>)Arrays.asList(cmdTokens), " ");
+ sb.append("sh ").append(cmds).append("\n");
+ }
+
+ @Override
+ protected void processDescribe(String alias) throws IOException {
+ sb.append("describe ").append(alias).append(";\n");
+ }
+
+ @Override
+ protected void processExplain(String alias, String script,
+ boolean isVerbose, String format, String target,
+ List<String> params, List<String> files) throws IOException,
+ ParseException {
+ sb.append("explain ");
+ if (script != null) {
+ sb.append("-script ").append(script).append(" ");
+ }
+ if (target != null) {
+ sb.append("-out ").append(target).append(" ");
+ }
+ if (isVerbose) sb.append("-brief ");
+ if (format != null && format.equals("dot")) {
+ sb.append("-dot ");
+ }
+ if (params != null) {
+ for (String param : params) {
+ sb.append("-param ").append(param).append(" ");
+ }
+ }
+ if (files != null) {
+ for (String file : files) {
+ sb.append("-param_file ").append(file).append(" ");
+ }
+ }
+ if (alias != null) {
+ sb.append(alias);
+ }
+ sb.append("\n");
+ }
+
+ @Override
+ protected void processRegister(String jar) throws IOException {
+ sb.append("register ").append(jar).append(";\n");
+ }
+
+ @Override
+ protected void processRegister(String path, String scriptingEngine,
+ String namespace) throws IOException, ParseException {
+ sb.append("register '").append(path).append("'");
+ if (scriptingEngine != null) {
+ sb.append(" using ").append(scriptingEngine);
+ }
+ if (namespace != null) {
+ sb.append(" as ").append(namespace);
+ }
+ sb.append(";\n");
+
+ }
+
+ @Override
+ protected void processSet(String key, String value) throws IOException,
+ ParseException {
+ sb.append("set ").append(key).append(" ").append(value).append("\n");
+ }
+
+ @Override
+ protected void processCat(String path) throws IOException {
+ sb.append("cat ").append(path).append("\n");
+ }
+
+ @Override
+ protected void processCD(String path) throws IOException {
+ sb.append("cd ").append(path).append("\n");
+ }
+
+ @Override
+ protected void processDump(String alias) throws IOException {
+ sb.append("dump ").append(alias).append("\n");
+ }
+
+ @Override
+ protected void processKill(String jobid) throws IOException {
+ sb.append("kill ").append(jobid).append("\n");
+ }
+
+ @Override
+ protected void processLS(String path) throws IOException {
+ sb.append("ls");
+ if (path != null) sb.append(" ").append(path);
+ sb.append("\n");
+ }
+
+ @Override
+ protected void processPWD() throws IOException {
+ sb.append("pwd\n");
+ }
+
+ @Override
+ protected void printHelp() {
+
+ }
+
+ @Override
+ protected void processMove(String src, String dst) throws IOException {
+ sb.append("mv ").append(src).append(" ").append(dst).append("\n");
+ }
+
+ @Override
+ protected void processCopy(String src, String dst) throws IOException {
+ sb.append("cp ").append(src).append(" ").append(dst).append("\n");
+ }
+
+ @Override
+ protected void processCopyToLocal(String src, String dst)
+ throws IOException {
+ sb.append("CopyToLocal ").append(src).append(" ").append(dst).append("\n");
+ }
+
+ @Override
+ protected void processCopyFromLocal(String src, String dst)
+ throws IOException {
+ sb.append("CopyFrom,Local ").append(src).append(" ").append(dst).append("\n");
+ }
+
+ @Override
+ protected void processMkdir(String dir) throws IOException {
+ sb.append("mkdir ").append(dir).append("\n");
+ }
+
+ @Override
+ protected void processPig(String cmd) throws IOException {
+ CommonTokenStream tokenStream = QueryParserDriver.tokenize(cmd, source);
+ Tree ast = null;
+
+ try {
+ ast = QueryParserDriver.parse( tokenStream );
+ } catch(RuntimeException ex) {
+ throw new ParserException( ex.getMessage() );
+ }
+
+ if (!hasMacro) {
+ List<CommonTree> importNodes = new ArrayList<CommonTree>();
+ List<CommonTree> macroNodes = new ArrayList<CommonTree>();
+ List<CommonTree> inlineNodes = new ArrayList<CommonTree>();
+
+ QueryParserDriver.traverseImport(ast, importNodes);
+ QueryParserDriver.traverse(ast, macroNodes, inlineNodes);
+
+ if (!importNodes.isEmpty() || !macroNodes.isEmpty()
+ || !inlineNodes.isEmpty()) {
+ hasMacro = true;
+ }
+ }
+
+ if (parserTree == null) {
+ parserTree = ast;
+ } else {
+ int n = ast.getChildCount();
+ for (int i = 0; i < n; i++) {
+ parserTree.addChild(ast.getChild(i));
+ }
+ }
+
+ CommonTree dup = (CommonTree)parserTree.dupNode();
+ dup.addChildren(((CommonTree)parserTree).getChildren());
+
+ QueryParserDriver driver = new QueryParserDriver(context, "0",
+ new HashMap<String, String>());
+ Tree newAst = driver.expandMacro(dup);
+
+ CommonTreeNodeStream nodes = new CommonTreeNodeStream( newAst );
+ AstPrinter walker = new AstPrinter( nodes );
+ try {
+ walker.query();
+ } catch (RecognitionException e) {
+ throw new ParserException("Failed to print AST for command " + cmd,
+ e);
+ }
+
+ String result = walker.getResult().trim();
+
+ if (!result.isEmpty()) {
+ String[] lines = result.split("\n");
+ for (int i = toSkip; i < lines.length; i++) {
+ sb.append(lines[i]).append("\n");
+ toSkip++;
+ }
+ }
+ }
+
+ @Override
+ protected void processRemove(String path, String opt) throws IOException {
+ if (opt == null || !opt.equalsIgnoreCase("force")) {
+ sb.append("rm ");
+ } else {
+ sb.append("rmf ");
+ }
+ sb.append(path).append("\n");
+ }
+
+ @Override
+ protected void processIllustrate(String alias, String script,
+ String target, List<String> params, List<String> files)
+ throws IOException, ParseException {
+ sb.append("illustrate ");
+ if (script != null) {
+ sb.append("-script ").append(script).append(" ");
+ }
+ if (target != null) {
+ sb.append("-out ").append(target).append(" ");
+ }
+ if (params != null) {
+ for (String param : params) {
+ sb.append("-param ").append(param).append(" ");
+ }
+ }
+ if (files != null) {
+ for (String file : files) {
+ sb.append("-param_file ").append(file).append(" ");
+ }
+ }
+ if (alias != null) {
+ sb.append(alias);
+ }
+ sb.append("\n");
+ }
+
+ @Override
+ protected void processScript(String script, boolean batch,
+ List<String> params, List<String> files) throws IOException,
+ ParseException {
+ if (batch) sb.append("exec ");
+ else sb.append("run ");
+ if (params != null) {
+ for (String param : params) {
+ sb.append("-param ").append(param).append(" ");
+ }
+ }
+ if (files != null) {
+ for (String file : files) {
+ sb.append("-param_file ").append(file).append(" ");
+ }
+ }
+ sb.append(script).append("\n");
+ }
+
+}
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1095228&r1=1095227&r2=1095228&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Tue Apr 19 22:22:42 2011
@@ -43,6 +43,7 @@ import org.apache.commons.logging.LogFac
import org.apache.pig.impl.PigContext;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.tools.pigscript.parser.ParseException;
import org.apache.pig.tools.pigstats.ScriptState;
public class QueryParserDriver {
@@ -103,50 +104,12 @@ public class QueryParserDriver {
return plan;
}
-
- public boolean dryrun(String scriptFile) throws ParserException, IOException, RecognitionException {
- BufferedReader rd = new BufferedReader(new FileReader(scriptFile));
- StringBuilder sb = new StringBuilder();
- String line = rd.readLine();
- while (line != null) {
- sb.append(line).append("\n");
- line = rd.readLine();
- }
-
- CommonTokenStream tokenStream = tokenize(sb.toString(), scriptFile);
- Tree ast = null;
-
- try {
- ast = parse( tokenStream );
- } catch(RuntimeException ex) {
- throw new ParserException( ex.getMessage() );
- }
-
- List<CommonTree> importNodes = new ArrayList<CommonTree>();
- List<CommonTree> macroNodes = new ArrayList<CommonTree>();
- List<CommonTree> inlineNodes = new ArrayList<CommonTree>();
- traverseImport(ast, importNodes);
- traverse(ast, macroNodes, inlineNodes);
-
- if (importNodes.isEmpty() && macroNodes.isEmpty()
- && inlineNodes.isEmpty()) {
- return false;
- }
-
- ast = expandMacro(ast);
-
- String expandedFile = scriptFile.replace(".substituted", ".expanded");
- dryrun(ast, expandedFile);
-
- return true;
- }
-
public Map<String, Operator> getOperators() {
return operators;
}
- private static CommonTokenStream tokenize(String query, String source)
+ static CommonTokenStream tokenize(String query, String source)
throws ParserException {
CharStream input;
try {
@@ -169,7 +132,7 @@ public class QueryParserDriver {
+ " parsing errors in the query");
}
- private static Tree parse(CommonTokenStream tokens) throws ParserException {
+ static Tree parse(CommonTokenStream tokens) throws ParserException {
QueryParser parser = QueryParserUtils.createParser(tokens);
QueryParser.query_return result = null;
@@ -187,15 +150,6 @@ public class QueryParserDriver {
return ast;
}
- private static void dryrun(Tree ast, String scriptFile) throws RecognitionException, IOException {
- CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
- AstPrinter walker = new AstPrinter( nodes );
- walker.query();
- BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
- fw.append(walker.getResult());
- fw.close();
- }
-
private static Tree validateAst(Tree ast) throws RecognitionException, ParserException {
CommonTreeNodeStream nodes = new CommonTreeNodeStream( ast );
AstValidator walker = new AstValidator( nodes );
@@ -207,7 +161,7 @@ public class QueryParserDriver {
return newAst;
}
- private Tree expandMacro(Tree ast) throws ParserException {
+ Tree expandMacro(Tree ast) throws ParserException {
LOG.debug("Original macro AST:\n" + ast.toStringTree() + "\n");
// first insert the import files
@@ -277,7 +231,7 @@ public class QueryParserDriver {
return true;
}
- private void traverseImport(Tree t, List<CommonTree> nodes) {
+ static void traverseImport(Tree t, List<CommonTree> nodes) {
if (t.getText().equalsIgnoreCase(IMPORT_DEF)) {
nodes.add((CommonTree)t);
}
@@ -288,7 +242,7 @@ public class QueryParserDriver {
}
}
- private void traverse(Tree t, List<CommonTree> macroNodes,
+ static void traverse(Tree t, List<CommonTree> macroNodes,
List<CommonTree> inlineNodes) {
if (t.getText().equals(MACRO_DEF)) {
macroNodes.add((CommonTree) t.getParent());
Modified: pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java?rev=1095228&r1=1095227&r2=1095228&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java Tue Apr 19 22:22:42 2011
@@ -1688,6 +1688,42 @@ public class TestMacroExpansion {
verify(query, expected);
}
+ // PIG-1987
+ @Test
+ public void test37() throws Exception {
+ String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
+ " $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
+ "};\n";
+
+ File f = new File("mytest.pig");
+ f.deleteOnExit();
+
+ FileWriter fw = new FileWriter(f);
+ fw.append(macro);
+ fw.close();
+
+ String script =
+ "set default_parallel 10\n" +
+ "import 'mytest.pig';\n" +
+ "alpha = load 'users' as (user, age, zip);\n" +
+ "ls\n" +
+ "gamma = group_and_count (alpha, user, 23);\n" +
+ "fs -copyFromLocal test test2\n" +
+ "store gamma into 'byuser';\n";
+
+ String expected =
+ "set default_parallel 10\n" +
+ "alpha = load 'users' as (user, age, zip);\n" +
+ "ls\n" +
+ "gamma = distinct alpha partition BY org.apache.pig.test.utils.SimpleCustomPartitioner parallel 23;\n" +
+ "fs -copyFromLocal test test2\n" +
+ "store gamma INTO 'byuser';\n";
+
+ verify(script, expected);
+ }
+
+
+
@Test
public void testCommentInMacro() throws Exception {
String query = "a = load 'testComplexCast' as (m);\n" +