You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2013/01/31 19:18:39 UTC
svn commit: r1441098 - in /pig/trunk: CHANGES.txt
src/docs/src/documentation/content/xdocs/start.xml
src/org/apache/pig/tools/grunt/GruntParser.java
src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
test/org/apache/pig/test/TestShortcuts.java
Author: cheolsoo
Date: Thu Jan 31 18:18:39 2013
New Revision: 1441098
URL: http://svn.apache.org/viewvc?rev=1441098&view=rev
Log:
PIG-2994: Grunt shortcuts (prasanth_j via cheolsoo)
Added:
pig/trunk/test/org/apache/pig/test/TestShortcuts.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/docs/src/documentation/content/xdocs/start.xml
pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1441098&r1=1441097&r2=1441098&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Jan 31 18:18:39 2013
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2994: Grunt shortcuts (prasanth_j via cheolsoo)
+
PIG-3140: Document PigProgressNotificationListener configs (billgraham)
PIG-3139: Document reducer estimation (billgraham)
Modified: pig/trunk/src/docs/src/documentation/content/xdocs/start.xml
URL: http://svn.apache.org/viewvc/pig/trunk/src/docs/src/documentation/content/xdocs/start.xml?rev=1441098&r1=1441097&r2=1441098&view=diff
==============================================================================
--- pig/trunk/src/docs/src/documentation/content/xdocs/start.xml (original)
+++ pig/trunk/src/docs/src/documentation/content/xdocs/start.xml Thu Jan 31 18:18:39 2013
@@ -380,18 +380,37 @@ However, in a production environment you
<p>Pig Latin provides operators that can help you debug your Pig Latin statements:</p>
<ul>
<li>
- <p>Use the <a href="test.html#DUMP">DUMP</a> operator to display results to your terminal screen. </p>
+ <p>Use the <a href="test.html#dump">DUMP</a> operator to display results to your terminal screen. </p>
</li>
<li>
- <p>Use the <a href="test.html#DESCRIBE">DESCRIBE</a> operator to review the schema of a relation.</p>
+ <p>Use the <a href="test.html#describe">DESCRIBE</a> operator to review the schema of a relation.</p>
</li>
<li>
- <p>Use the <a href="test.html#EXPLAIN">EXPLAIN</a> operator to view the logical, physical, or map reduce execution plans to compute a relation.</p>
+ <p>Use the <a href="test.html#explain">EXPLAIN</a> operator to view the logical, physical, or map reduce execution plans to compute a relation.</p>
</li>
<li>
- <p>Use the <a href="test.html#ILLUSTRATE">ILLUSTRATE</a> operator to view the step-by-step execution of a series of statements.</p>
+ <p>Use the <a href="test.html#illustrate">ILLUSTRATE</a> operator to view the step-by-step execution of a series of statements.</p>
</li>
</ul>
+ <p><strong>Shortcuts for Debugging Operators</strong></p>
+ <p>Pig provides shortcuts for the frequently used debugging operators (DUMP, DESCRIBE, EXPLAIN, ILLUSTRATE). These shortcuts can be used in Grunt shell or within pig scripts. Following are the shortcuts supported by pig</p>
+ <ul>
+ <li>
+ <p> \d alias - shourtcut for <a href="test.html#dump">DUMP</a> operator. If alias is ignored last defined alias will be used.</p>
+ </li>
+ <li>
+ <p> \de alias - shourtcut for <a href="test.html#describe">DESCRIBE</a> operator. If alias is ignored last defined alias will be used.</p>
+ </li>
+ <li>
+ <p> \e alias - shourtcut for <a href="test.html#explain">EXPLAIN</a> operator. If alias is ignored last defined alias will be used.</p>
+ </li>
+ <li>
+ <p> \i alias - shourtcut for <a href="test.html#illustrate">ILLUSTRATE</a> operator. If alias is ignored last defined alias will be used.</p>
+ </li>
+ <li>
+ <p> \q - To quit grunt shell </p>
+ </li>
+ </ul>
</section>
</section>
Modified: pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=1441098&r1=1441097&r2=1441098&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Thu Jan 31 18:18:39 2013
@@ -272,6 +272,12 @@ public class GruntParser extends PigScri
if(mExplain == null) { // process only if not in "explain" mode
if(alias==null) {
alias = mPigServer.getPigContext().getLastAlias();
+ // if describe is used immediately after launching grunt shell then
+ // last defined alias will be null
+ if (alias == null) {
+ throw new IOException(
+ "No previously defined alias found. Please define an alias and use 'describe' operator.");
+ }
}
if(alias.contains("::")) {
nestedAlias = alias.substring(alias.indexOf("::") + 2);
@@ -291,6 +297,14 @@ public class GruntParser extends PigScri
String format, String target,
List<String> params, List<String> files)
throws IOException, ParseException {
+ if (alias == null) {
+ alias = mPigServer.getPigContext().getLastAlias();
+ // if explain is used immediately after launching grunt shell then
+ // last defined alias will be null
+ if (alias == null && script == null) {
+ throw new ParseException("'explain' statement must be on an alias or on a script.");
+ }
+ }
processExplain(alias, script, isVerbose, format, target, params, files,
false);
}
@@ -691,6 +705,16 @@ public class GruntParser extends PigScri
@Override
protected void processDump(String alias) throws IOException
{
+ if (alias == null) {
+ alias = mPigServer.getPigContext().getLastAlias();
+ // if dump is used immediately after launching grunt shell then
+ // last defined alias will be null
+ if (alias == null) {
+ throw new IOException(
+ "No previously defined alias found. Please define an alias and use 'dump' operator.");
+ }
+ }
+
if(mExplain == null) { // process only if not in "explain" mode
executeBatch();
Iterator<Tuple> result = mPigServer.openIterator(alias);
@@ -734,7 +758,12 @@ public class GruntParser extends PigScri
throw e;
}
} else if (alias == null) {
- throw new ParseException("'illustrate' statement must be on an alias or on a script.");
+ alias = mPigServer.getPigContext().getLastAlias();
+ // if illustrate is used immediately after launching grunt shell then
+ // last defined alias will be null
+ if (alias == null) {
+ throw new ParseException("'illustrate' statement must be on an alias or on a script.");
+ }
}
mPigServer.getExamples(alias);
} finally {
Modified: pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=1441098&r1=1441097&r2=1441098&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original)
+++ pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Thu Jan 31 18:18:39 2013
@@ -151,9 +151,12 @@ TOKEN: {<COPY: "cp">}
TOKEN: {<COPYFROMLOCAL: "copyFromLocal">}
TOKEN: {<COPYTOLOCAL: "copyToLocal">}
TOKEN: {<DUMP: "dump">}
+TOKEN: {<DUMP_SHORT: "\\d">}
TOKEN: {<DESCRIBE: "describe">}
+TOKEN: {<DESCRIBE_SHORT: "\\de">}
TOKEN: {<ALIASES: "aliases">}
TOKEN: {<EXPLAIN: "explain">}
+TOKEN: {<EXPLAIN_SHORT: "\\e">}
TOKEN: {<HELP: "help">}
TOKEN: {<HISTORY: "history">}
TOKEN: {<KILL: "kill">}
@@ -162,6 +165,7 @@ TOKEN: {<MOVE: "mv">}
TOKEN: {<MKDIR: "mkdir">}
TOKEN: {<PWD: "pwd">}
TOKEN: {<QUIT: "quit">}
+TOKEN: {<QUIT_SHORT: "\\q">}
TOKEN: {<REGISTER: "register">}
TOKEN: {<USING: "using">}
TOKEN: {<AS: "as"> }
@@ -169,6 +173,7 @@ TOKEN: {<REMOVE: "rm">}
TOKEN: {<REMOVEFORCE: "rmf">}
TOKEN: {<SET: "set">}
TOKEN: {<ILLUSTRATE: "illustrate">}
+TOKEN: {<ILLUSTRATE_SHORT: "\\i">}
TOKEN: {<RUN: "run">}
TOKEN: {<EXEC: "exec">}
TOKEN: {<PARAM: "-param">}
@@ -481,17 +486,21 @@ void parse() throws IOException:
t2 = GetPath()
{processCopyToLocal(t1.image, t2.image);}
|
- <DUMP>
- t1 = <IDENTIFIER>
- {processDump(t1.image);}
+ (<DUMP> | <DUMP_SHORT>)
+ (
+ t1 = <IDENTIFIER>
+ {processDump(t1.image);}
+ |
+ {processDump(null);}
+ )
|
Illustrate()
|
- <DESCRIBE>
+ (<DESCRIBE> | <DESCRIBE_SHORT>)
(
- t1 = <IDENTIFIER>
- {processDescribe(t1.image);}
- |
+ t1 = <IDENTIFIER>
+ {processDescribe(t1.image);}
+ |
{processDescribe(null);}
)
|
@@ -535,7 +544,7 @@ void parse() throws IOException:
<PWD>
{processPWD();}
|
- <QUIT>
+ (<QUIT> | <QUIT_SHORT>)
{quit();}
|
<REGISTER>
@@ -605,7 +614,7 @@ void Illustrate() throws IOException:
}
{
- <ILLUSTRATE>
+ (<ILLUSTRATE> | <ILLUSTRATE_SHORT>)
{
params = new ArrayList<String>();
files = new ArrayList<String>();
@@ -660,7 +669,7 @@ void Explain() throws IOException:
}
{
- <EXPLAIN>
+ (<EXPLAIN> | <EXPLAIN_SHORT>)
{
params = new ArrayList<String>();
files = new ArrayList<String>();
@@ -789,12 +798,22 @@ Token GetReserved () :
|
t = <DUMP>
|
+ t = <DUMP_SHORT>
+ |
t = <DESCRIBE>
|
+ t = <DESCRIBE_SHORT>
+ |
t = <ALIASES>
|
t = <EXPLAIN>
|
+ t = <EXPLAIN_SHORT>
+ |
+ t = <ILLUSTRATE>
+ |
+ t = <ILLUSTRATE_SHORT>
+ |
t = <HELP>
|
t = <HISTORY>
@@ -811,6 +830,8 @@ Token GetReserved () :
|
t = <QUIT>
|
+ t = <QUIT_SHORT>
+ |
t = <REGISTER>
|
t = <USING>
Added: pig/trunk/test/org/apache/pig/test/TestShortcuts.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestShortcuts.java?rev=1441098&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestShortcuts.java (added)
+++ pig/trunk/test/org/apache/pig/test/TestShortcuts.java Thu Jan 31 18:18:39 2013
@@ -0,0 +1,249 @@
+package org.apache.pig.test;
+
+import static org.apache.pig.builtin.mock.Storage.resetData;
+import static org.apache.pig.builtin.mock.Storage.tuple;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.builtin.mock.Storage.Data;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.tools.grunt.Grunt;
+import org.apache.pig.tools.pigscript.parser.ParseException;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestShortcuts {
+ private String basedir = "test/org/apache/pig/test/data";
+ private Data data;
+ private PigServer server;
+ private PigContext context;
+
+ @Before
+ public void setup() throws ExecException, IOException {
+ server = new PigServer("local");
+ context = server.getPigContext();
+ data = resetData(server);
+ data.set("input", tuple("dog", "miami", 12), tuple("cat", "miami", 18), tuple("turtle", "tampa", 4),
+ tuple("dog", "tampa", 14), tuple("cat", "naples", 9), tuple("dog", "naples", 5),
+ tuple("turtle", "naples", 1));
+ }
+
+ @Test
+ public void testExplainShortcut() throws Throwable {
+ String cmd = "a = load 'input' USING mock.Storage() as (x:chararray,y:chararray,z:long);"
+ + "b = foreach a generate $0 as foo;" + "\\e b;";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ /**
+ * When no alias is passed, last alias should be explained
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testExplainShortcutNoAlias() throws Throwable {
+ String cmd = "a = load 'input' USING mock.Storage() as (x:chararray,y:chararray,z:long);"
+ + "b = foreach a generate $0 as foo;" + "\\e;";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ /**
+ * When no alias is defined earlier, exception is expected
+ *
+ * @throws Throwable
+ */
+ @Test(expected = ParseException.class)
+ public void testExplainShortcutNoAliasDefined() throws Throwable {
+ String cmd = "\\e";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ @Test
+ public void testExplainShortcutScript() throws Throwable {
+ String cmd = "\\e -script " + basedir + "/explainScript.pig;";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ @Test
+ public void testDescribeShortcut() throws Throwable {
+ String cmd = "a = load 'input' USING mock.Storage() as (x:chararray,y:chararray,z:long);"
+ + "b = foreach a generate $0 as foo;" + "\\de b;";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ /**
+ * When no alias is passed, last alias should be described
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testDescribeShortcutNoAlias() throws Throwable {
+ String cmd = "a = load 'input' USING mock.Storage() as (x:chararray,y:chararray,z:long);"
+ + "b = foreach a generate $0 as foo;" + "\\de;";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ /**
+ * When no alias is define, then exception is expected
+ *
+ * @throws Throwable
+ */
+ @Test(expected = IOException.class)
+ public void testDescribeShortcutNoAliasDefined() throws Throwable {
+ String cmd = "\\de";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ @Test
+ public void testDumpShortcut() throws Throwable {
+ String cmd = "a = load 'input' USING mock.Storage() as (x:chararray,y:chararray,z:long);"
+ + "b = foreach a generate $0 as foo;" + "\\d b;";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ /**
+ * When no alias is passed, last alias should be dumped
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testDumpShortcutNoAlias() throws Throwable {
+ String cmd = "a = load 'input' USING mock.Storage() as (x:chararray,y:chararray,z:long);"
+ + "b = foreach a generate $0 as foo;" + "\\d;";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ /**
+ * When no alias is defined, exception is expected
+ *
+ * @throws Throwable
+ */
+ @Test(expected = IOException.class)
+ public void testDumpShortcutNoAliasDefined() throws Throwable {
+ String cmd = "\\d";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ @Test
+ public void testIllustrateShortcut() throws Throwable {
+ String cmd = "a = load 'input' USING mock.Storage() as (x:chararray,y:chararray,z:long);"
+ + "b = group a by $0;" + "\\i b;";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ /**
+ * When no alias is passed, last alias should be illustrated
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testIllustrateShortcutNoAlias() throws Throwable {
+ String cmd = "a = load 'input' USING mock.Storage() as (x:chararray,y:chararray,z:long);"
+ + "b = group a by $0;" + "\\i";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ /**
+ * When no alias is defined, exception is expected
+ *
+ * @throws Throwable
+ */
+ @Test(expected = ParseException.class)
+ public void testIllustrateShortcutNoAliasDefined() throws Throwable {
+ String cmd = "\\i";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ @Test
+ public void testIllustrateShortcutScript() throws Throwable {
+ String cmd = "\\i -script " + basedir + "/illustrate.pig;";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+
+ @Test
+ public void testQuit() throws Throwable {
+ String cmd = "a = load 'input' USING mock.Storage() as (x:chararray,y:chararray,z:long);"
+ + "b = group a by $0;" + "\\q";
+
+ ByteArrayInputStream cmdstream = new ByteArrayInputStream(cmd.getBytes());
+ InputStreamReader reader = new InputStreamReader(cmdstream);
+
+ Grunt grunt = new Grunt(new BufferedReader(reader), context);
+ grunt.exec();
+ }
+}