You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2009/02/12 23:27:47 UTC
svn commit: r743908 - in /hadoop/pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/tools/grunt/ src/org/apache/pig/tools/pigscript/parser/
test/org/apache/pig/test/ test/org/apache/pig/test/data/
Author: olga
Date: Thu Feb 12 22:27:46 2009
New Revision: 743908
URL: http://svn.apache.org/viewvc?rev=743908&view=rev
Log:
PIG-574: allowing to run scripts from within grunt shell (hagleitn via olgan)
Added:
hadoop/pig/trunk/test/org/apache/pig/test/data/test.ppf
hadoop/pig/trunk/test/org/apache/pig/test/data/test_broken.ppf
hadoop/pig/trunk/test/org/apache/pig/test/data/testsub.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_exec.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_run.pig
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/PigServer.java
hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=743908&r1=743907&r2=743908&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Feb 12 22:27:46 2009
@@ -413,3 +413,6 @@
PIG-553: EvalFunc.finish() not getting called (shravanmn via sms)
PIG-654: Optimize build.xml (daijy)
+
+ PIG-574: allowing to run scripts from within grunt shell (hagleitn via
+ olgan)
Modified: hadoop/pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/PigServer.java?rev=743908&r1=743907&r2=743908&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/PigServer.java Thu Feb 12 22:27:46 2009
@@ -123,19 +123,21 @@
}
public PigServer(ExecType execType, Properties properties) throws ExecException {
- this.pigContext = new PigContext(execType, properties);
- if (this.pigContext.getProperties().getProperty(PigContext.JOB_NAME) == null) {
- setJobName("DefaultJobName") ;
- }
- pigContext.connect();
+ this(new PigContext(execType, properties), true);
}
-
+
public PigServer(PigContext context) throws ExecException {
+ this(context, true);
+ }
+
+ public PigServer(PigContext context, boolean connect) throws ExecException {
this.pigContext = context;
if (this.pigContext.getProperties().getProperty(PigContext.JOB_NAME) == null) {
setJobName("DefaultJobName") ;
}
- pigContext.connect();
+ if (connect) {
+ pigContext.connect();
+ }
}
public PigContext getPigContext(){
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=743908&r1=743907&r2=743908&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Thu Feb 12 22:27:46 2009
@@ -21,13 +21,26 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
+import java.io.FileReader;
+import java.io.FileInputStream;
+import java.io.OutputStreamWriter;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.InputStreamReader;
+import java.io.FileNotFoundException;
+import java.io.StringReader;
+import java.io.StringWriter;
import java.util.Iterator;
import java.util.Map;
+import java.util.ArrayList;
import java.util.Properties;
import java.io.ByteArrayOutputStream;
+import java.io.ByteArrayInputStream;
import java.io.PrintStream;
import jline.ConsoleReader;
+import jline.ConsoleReaderInputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -47,6 +60,7 @@
import org.apache.pig.tools.pigscript.parser.ParseException;
import org.apache.pig.tools.pigscript.parser.PigScriptParser;
import org.apache.pig.tools.pigscript.parser.PigScriptParserTokenManager;
+import org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor;
public class GruntParser extends PigScriptParser {
@@ -80,15 +94,16 @@
{
prompt();
mDone = false;
- while(!mDone)
+ while(!mDone) {
parse();
+ }
}
public void parseContOnError()
{
prompt();
mDone = false;
- while(!mDone)
+ while(!mDone) {
try
{
parse();
@@ -109,6 +124,7 @@
} catch (Error e) {
log.error(e);
}
+ }
}
public void setParams(PigServer pigServer)
@@ -161,6 +177,69 @@
mPigServer.registerJar(jar);
}
+ private String runPreprocessor(String script, ArrayList<String> params,
+ ArrayList<String> files)
+ throws IOException, ParseException {
+
+ ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
+ StringWriter writer = new StringWriter();
+
+ try{
+ psp.genSubstitutedFile(new BufferedReader(new FileReader(script)),
+ writer,
+ params.size() > 0 ? params.toArray(new String[0]) : null,
+ files.size() > 0 ? files.toArray(new String[0]) : null);
+ } catch (org.apache.pig.tools.parameters.ParseException pex) {
+ throw new ParseException(pex.getMessage());
+ }
+
+ return writer.toString();
+ }
+
+ protected void processScript(String script, boolean batch,
+ ArrayList<String> params, ArrayList<String> files)
+ throws IOException, ParseException {
+
+ Reader inputReader;
+ ConsoleReader reader;
+ boolean interactive;
+
+ try {
+ String cmds = runPreprocessor(script, params, files);
+
+ if (mInteractive && !batch) { // Write prompt and echo commands
+ reader = new ConsoleReader(new ByteArrayInputStream(cmds.getBytes()),
+ new OutputStreamWriter(System.out));
+ reader.setHistory(mConsoleReader.getHistory());
+ InputStream in = new ConsoleReaderInputStream(reader);
+ inputReader = new BufferedReader(new InputStreamReader(in));
+ interactive = true;
+ } else { // Quietly parse the statements
+ inputReader = new StringReader(cmds);
+ reader = null;
+ interactive = false;
+ }
+ } catch (FileNotFoundException fnfe) {
+ throw new ParseException("File not found: " + script);
+ } catch (SecurityException se) {
+ throw new ParseException("Cannot access file: " + script);
+ }
+
+ // In batch mode: Use a new server to avoid side-effects (handles, etc)
+ PigServer pigServer = batch ?
+ new PigServer(mPigServer.getPigContext(), false) : mPigServer;
+
+ GruntParser parser = new GruntParser(inputReader);
+ parser.setParams(pigServer);
+ parser.setConsoleReader(reader);
+ parser.setInteractive(interactive);
+
+ parser.parseStopOnError();
+ if (interactive) {
+ System.out.println("");
+ }
+ }
+
protected void processSet(String key, String value) throws IOException, ParseException {
if (key.equals("debug"))
{
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=743908&r1=743907&r2=743908&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Thu Feb 12 22:27:46 2009
@@ -31,6 +31,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.Stack;
+import java.util.ArrayList;
import jline.ConsoleReader;
@@ -62,7 +63,7 @@
abstract protected void processDescribe(String alias) throws IOException;
- abstract protected void processExplain(String alias) throws IOException;
+ abstract protected void processExplain(String alias) throws IOException;
abstract protected void processRegister(String jar) throws IOException;
@@ -98,7 +99,9 @@
abstract protected void processIllustrate(String alias) throws IOException;
- static String unquote(String s)
+ abstract protected void processScript(String script, boolean batch, ArrayList<String> params, ArrayList<String> files) throws IOException, ParseException;
+
+ static String unquote(String s)
{
if (s.charAt(0) == '\'' && s.charAt(s.length()-1) == '\'')
return s.substring(1, s.length()-1);
@@ -142,6 +145,10 @@
TOKEN: {<REMOVEFORCE: "rmf">}
TOKEN: {<SET: "set">}
TOKEN: {<ILLUSTRATE: "illustrate">}
+TOKEN: {<RUN: "run">}
+TOKEN: {<EXEC: "exec">}
+TOKEN: {<PARAM: "-param">}
+TOKEN: {<PARAM_FILE: "-param_file">}
// internal use commands
TOKEN: {<SCRIPT_DONE: "scriptDone">}
@@ -320,8 +327,8 @@
| <#DIGIT : ["0"-"9"] >
| <#SPECIALCHAR : ["_"] >
| <#FSSPECIALCHAR: ["/"]>
-| <#FLOAT: <INTEGER> ( "." <INTEGER> )? | "." <INTEGER> >
-| <#INTEGER: ( <DIGIT> )+ >
+| <#FLOAT: <INTEGER> ( "." <INTEGER> )? | "." <INTEGER> >
+| <#INTEGER: ( <DIGIT> )+ >
| <#NUMBER: <INTEGER> | <FLOAT> | <FLOAT> ( ["e","E"] ([ "-","+"])? <FLOAT> )?>
}
@@ -433,6 +440,8 @@
t1 = GetPath()
{processRegister(t1.image);}
|
+ Script()
+ |
<REMOVE>
(
t1 = GetPath()
@@ -467,6 +476,40 @@
)
}
+void Script() throws IOException:
+{
+ Token t;
+ String script;
+ boolean batch = false;
+ ArrayList<String> params;
+ ArrayList<String> files;
+}
+{
+ (
+ <RUN>
+ {batch = false;}
+ |
+ <EXEC>
+ {batch = true;}
+ )
+ {
+ params = new ArrayList<String>();
+ files = new ArrayList<String>();
+ }
+ (
+ <PARAM>
+ t = GetPath()
+ {params.add(t.image);}
+ |
+ <PARAM_FILE>
+ t = GetPath()
+ {files.add(t.image);}
+ )*
+ t = GetPath()
+ {script = t.image;}
+ {processScript(script, batch, params, files);}
+}
+
Token GetPath() :
{
Token t;
@@ -475,7 +518,7 @@
(
t = <IDENTIFIER>
|
- t = <PATH>
+ t = <PATH>
|
t = GetReserved()
)
@@ -548,6 +591,14 @@
t = <SET>
|
t = <SCRIPT_DONE>
+ |
+ t = <RUN>
+ |
+ t = <EXEC>
+ |
+ t = <PARAM>
+ |
+ t = <PARAM_FILE>
)
{return t;}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=743908&r1=743907&r2=743908&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java Thu Feb 12 22:27:46 2009
@@ -36,8 +36,14 @@
public class TestGrunt extends TestCase {
MiniCluster cluster = MiniCluster.buildCluster();
+ private String basedir;
private final Log log = LogFactory.getLog(getClass());
+
+ public TestGrunt(String name) {
+ super(name);
+ basedir = "test/org/apache/pig/test/data";
+ }
/* @Test
public void testCopyFromLocal() throws Throwable {
@@ -305,4 +311,84 @@
grunt.exec();
}
+
+ @Test
+ public void testRunStatment() throws Throwable {
+ PigServer server = new PigServer("MAPREDUCE");
+ PigContext context = server.getPigContext();
+
+ String strCmd = "a = load 'foo' as (foo, fast, regenerate);" +
+ " run -param LIMIT=5 -param_file " + basedir +
+ "/test_broken.ppf " + basedir + "/testsub.pig; explain bar";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+
+ grunt.exec();
+ }
+
+ @Test
+ public void testExecStatment() throws Throwable {
+ PigServer server = new PigServer("MAPREDUCE");
+ PigContext context = server.getPigContext();
+ boolean caught = false;
+
+ String strCmd = "a = load 'foo' as (foo, fast, regenerate);" +
+ " exec -param LIMIT=5 -param FUNCTION=COUNT " +
+ "-param FILE=foo " + basedir + "/testsub.pig; explain bar";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+
+ try {
+ grunt.exec();
+ } catch (Exception e) {
+ caught = true;
+ assertTrue(e.getMessage().contains("alias bar"));
+ }
+ assertTrue(caught);
+ }
+
+ @Test
+ public void testRunStatmentNested() throws Throwable {
+ PigServer server = new PigServer("MAPREDUCE");
+ PigContext context = server.getPigContext();
+
+ String strCmd = "a = load 'foo' as (foo, fast, regenerate); run "
+ +basedir+"/testsubnested_run.pig; explain bar";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+
+ grunt.exec();
+ }
+
+ @Test
+ public void testExecStatmentNested() throws Throwable {
+ PigServer server = new PigServer("MAPREDUCE");
+ PigContext context = server.getPigContext();
+ boolean caught = false;
+
+ String strCmd = "a = load 'foo' as (foo, fast, regenerate); exec "
+ +basedir+"/testsubnested_exec.pig; explain bar";
+
+ ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmd);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+
+ try {
+ grunt.exec();
+ } catch (Exception e) {
+ caught = true;
+ assertTrue(e.getMessage().contains("alias bar"));
+ }
+ assertTrue(caught);
+ }
}
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/test.ppf
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/test.ppf?rev=743908&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/test.ppf (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/test.ppf Thu Feb 12 22:27:46 2009
@@ -0,0 +1,3 @@
+LIMIT = 5
+FILE = foo
+FUNCTION = COUNT
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/test_broken.ppf
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/test_broken.ppf?rev=743908&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/test_broken.ppf (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/test_broken.ppf Thu Feb 12 22:27:46 2009
@@ -0,0 +1,3 @@
+LIMIT = WONTWORK
+FILE = foo
+FUNCTION = COUNT
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/testsub.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/testsub.pig?rev=743908&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/testsub.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/testsub.pig Thu Feb 12 22:27:46 2009
@@ -0,0 +1,4 @@
+foo = load '$FILE' as (foo, fast, regenerate);
+bar = limit foo $LIMIT;
+baz = foreach bar generate $FUNCTION($0);
+explain baz;
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_exec.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_exec.pig?rev=743908&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_exec.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_exec.pig Thu Feb 12 22:27:46 2009
@@ -0,0 +1 @@
+exec -param_file test/org/apache/pig/test/data/test.ppf test/org/apache/pig/test/data/testsub.pig;
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_run.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_run.pig?rev=743908&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_run.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/testsubnested_run.pig Thu Feb 12 22:27:46 2009
@@ -0,0 +1 @@
+run -param LIMIT=5 -param FILE=foo -param FUNCTION=COUNT test/org/apache/pig/test/data/testsub.pig;