You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2011/01/19 19:28:40 UTC
svn commit: r1060894 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/scripting/Pig.java
src/org/apache/pig/scripting/ScriptEngine.java
src/org/apache/pig/scripting/jython/JythonScriptEngine.java
test/org/apache/pig/test/TestScriptLanguage.java
Author: rding
Date: Wed Jan 19 18:28:40 2011
New Revision: 1060894
URL: http://svn.apache.org/viewvc?rev=1060894&view=rev
Log:
PIG-1806: Modify embedded Pig API for usability
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/scripting/Pig.java
pig/trunk/src/org/apache/pig/scripting/ScriptEngine.java
pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java
pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1060894&r1=1060893&r2=1060894&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jan 19 18:28:40 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-1806: Modify embedded Pig API for usability (rding)
+
PIG-1799: Provide deployable maven artifacts for pigunit and pig smoke tests
(cos via gates)
Modified: pig/trunk/src/org/apache/pig/scripting/Pig.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/Pig.java?rev=1060894&r1=1060893&r2=1060894&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/Pig.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/Pig.java Wed Jan 19 18:28:40 2011
@@ -53,19 +53,21 @@ public class Pig {
* string.
* @throws IOException
*/
- public static void fs(String cmd) throws IOException {
+ public static int fs(String cmd) throws IOException {
ScriptPigContext ctx = getScriptContext();
FsShell shell = new FsShell(ConfigurationUtil.toConfiguration(ctx
.getPigContext().getProperties()));
+ int code = -1;
if (cmd != null) {
String[] cmdTokens = cmd.split("\\s+");
if (!cmdTokens[0].startsWith("-")) cmdTokens[0] = "-" + cmdTokens[0];
try {
- shell.run(cmdTokens);
+ code = shell.run(cmdTokens);
} catch (Exception e) {
throw new IOException("Run filesystem command failed", e);
}
}
+ return code;
}
/**
@@ -201,7 +203,7 @@ public class Pig {
* @throws IOException if there is not a key for each
* Pig Latin parameter or if they contain unsupported types.
*/
- public BoundScript bind(Map<String, String> vars) throws IOException {
+ public BoundScript bind(Map<String, Object> vars) throws IOException {
return new BoundScript(replaceParameters(script, vars), scriptContext, name);
}
@@ -217,9 +219,9 @@ public class Pig {
* @throws IOException if there is not a key for each
* Pig Latin parameter or if they contain unsupported types.
*/
- public BoundScript bind(List<Map<String, String>> vars) throws IOException {
+ public BoundScript bind(List<Map<String, Object>> vars) throws IOException {
List<String> lst = new ArrayList<String>();
- for (Map<String, String> var : vars) {
+ for (Map<String, Object> var : vars) {
lst.add(replaceParameters(script, var));
}
return new BoundScript(lst, scriptContext, name);
@@ -242,7 +244,7 @@ public class Pig {
*/
public BoundScript bind() throws IOException {
ScriptEngine engine = scriptContext.getScriptEngine();
- Map<String, String> vars = engine.getParamsFromVariables();
+ Map<String, Object> vars = engine.getParamsFromVariables();
return bind(vars);
}
@@ -266,11 +268,11 @@ public class Pig {
* @param vars parameters and their values
* @return the modified version
*/
- private String replaceParameters(String qstr, Map<String, String> vars)
+ private String replaceParameters(String qstr, Map<String, Object> vars)
throws IOException {
ArrayList<String> plist = new ArrayList<String>();
- for (Entry<String, String> entry : vars.entrySet()) {
- plist.add(entry.getKey() + "=" + entry.getValue());
+ for (Entry<String, Object> entry : vars.entrySet()) {
+ plist.add(entry.getKey() + "=" + entry.getValue().toString());
}
ParameterSubstitutionPreprocessor psp =
Modified: pig/trunk/src/org/apache/pig/scripting/ScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/ScriptEngine.java?rev=1060894&r1=1060893&r2=1060894&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/ScriptEngine.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/ScriptEngine.java Wed Jan 19 18:28:40 2011
@@ -89,7 +89,7 @@ public abstract class ScriptEngine {
* Returns a map from local variable names to their values
* @throws IOException
*/
- protected abstract Map<String, String> getParamsFromVariables()
+ protected abstract Map<String, Object> getParamsFromVariables()
throws IOException;
/**
Modified: pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java?rev=1060894&r1=1060893&r2=1060894&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java Wed Jan 19 18:28:40 2011
@@ -232,11 +232,11 @@ public class JythonScriptEngine extends
}
@Override
- protected Map<String, String> getParamsFromVariables() throws IOException {
+ protected Map<String, Object> getParamsFromVariables() throws IOException {
PyFrame frame = Py.getFrame();
@SuppressWarnings("unchecked")
List<PyTuple> locals = (List<PyTuple>) ((PyStringMap) frame.getLocals()).items();
- Map<String, String> vars = new HashMap<String, String>();
+ Map<String, Object> vars = new HashMap<String, Object>();
for (PyTuple item : locals) {
String key = (String) item.get(0);
Object obj = item.get(1);
Modified: pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java?rev=1060894&r1=1060893&r2=1060894&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java Wed Jan 19 18:28:40 2011
@@ -17,7 +17,6 @@
*/
package org.apache.pig.test;
-
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -339,6 +338,69 @@ public class TestScriptLanguage {
}
@Test
+ public void bindNonStringVariableTest() throws Exception {
+ String[] script = {
+ "#!/usr/bin/python",
+ "from org.apache.pig.scripting import *",
+ "Pig.fs(\"-rmr simple_out\")",
+ "input = 'simple_table'",
+ "output = 'simple_out'",
+ "max = 2",
+ "P = Pig.compile(\"\"\"a = load '$in' as (a0:int, a1:int);" +
+ " b = filter a by a0 > $max;" +
+ " store b into '$out';\"\"\")",
+ "Q = P.bind({'in':input, 'out':output, 'max':max})",
+ "stats = Q.runSingle()",
+ "if stats.isSuccessful():",
+ "\tprint 'success!'",
+ "else:",
+ "\traise 'failed'"
+ };
+ String[] input = {
+ "1\t3",
+ "2\t4",
+ "3\t5"
+ };
+
+ Util.deleteFile(cluster, "simple_table");
+ Util.createInputFile(cluster, "simple_table", input);
+ Util.createLocalInputFile( "testScript.py", script);
+
+ ScriptEngine scriptEngine = ScriptEngine.getInstance("jython");
+ Map<String, List<PigStats>> statsMap = scriptEngine.run(pigServer.getPigContext(), "testScript.py");
+ assertEquals(1, statsMap.size());
+ Iterator<List<PigStats>> it = statsMap.values().iterator();
+ PigStats stats = it.next().get(0);
+ assertTrue(stats.isSuccessful());
+ assertEquals(1, stats.getNumberJobs());
+ String name = stats.getOutputNames().get(0);
+ assertEquals("simple_out", name);
+ assertEquals(4, stats.getBytesWritten());
+ assertEquals(1, stats.getRecordWritten());
+ }
+
+ @Test
+ public void fsTest() throws Exception {
+ String[] script = {
+ "#!/usr/bin/python",
+ "from org.apache.pig.scripting import *",
+ "ret = Pig.fs(\"-rmr simple_out\")",
+ "if ret == 0:",
+ "\tprint 'success!'",
+ "else:",
+ "\traise 'fs command failed'"
+ };
+
+ Util.createLocalInputFile( "testScript.py", script);
+
+ String[] args = { "-x", "local", "testScript.py"};
+ PigStats stats = PigRunner.run(args, null);
+ assertFalse(stats.isSuccessful());
+ //assertTrue(stats.getErrorCode() == 1121);
+ //assertTrue(stats.getReturnCode() == PigRunner.ReturnCode.PIG_EXCEPTION);
+ }
+
+ @Test
public void NegativeTest() throws Exception {
String[] script = {
"#!/usr/bin/python",