You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/04/02 09:07:08 UTC
svn commit: r1308251 - in /pig/branches/branch-0.10: CHANGES.txt
src/org/apache/pig/PigServer.java test/org/apache/pig/test/TestPigServer.java
Author: daijy
Date: Mon Apr 2 07:07:07 2012
New Revision: 1308251
URL: http://svn.apache.org/viewvc?rev=1308251&view=rev
Log:
PIG-2623: Support S3 paths for registering UDFs
Modified:
pig/branches/branch-0.10/CHANGES.txt
pig/branches/branch-0.10/src/org/apache/pig/PigServer.java
pig/branches/branch-0.10/test/org/apache/pig/test/TestPigServer.java
Modified: pig/branches/branch-0.10/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/CHANGES.txt?rev=1308251&r1=1308250&r2=1308251&view=diff
==============================================================================
--- pig/branches/branch-0.10/CHANGES.txt (original)
+++ pig/branches/branch-0.10/CHANGES.txt Mon Apr 2 07:07:07 2012
@@ -184,6 +184,8 @@ PIG-2228: support partial aggregation in
BUG FIXES
+PIG-2623: Support S3 paths for registering UDFs (nshkrob via daijy)
+
PIG-2540: [piggybank] AvroStorage can't read schema on amazon s3 in elastic mapreduce (rjurney via jcoveney)
PIG-2618: e2e local fails to build
Modified: pig/branches/branch-0.10/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/src/org/apache/pig/PigServer.java?rev=1308251&r1=1308250&r2=1308251&view=diff
==============================================================================
--- pig/branches/branch-0.10/src/org/apache/pig/PigServer.java (original)
+++ pig/branches/branch-0.10/src/org/apache/pig/PigServer.java Mon Apr 2 07:07:07 2012
@@ -512,8 +512,7 @@ public class PigServer {
*/
public void registerCode(String path, String scriptingLang, String namespace)
throws IOException {
- File f = new File(path);
-
+ File f = FileLocalizer.fetchFile(pigContext.getProperties(), path).file;
if (!f.canRead()) {
int errCode = 4002;
String msg = "Can't read file: " + path;
@@ -522,9 +521,9 @@ public class PigServer {
}
if(scriptingLang != null) {
ScriptEngine se = ScriptEngine.getInstance(scriptingLang);
- se.registerFunctions(path, namespace, pigContext);
+ se.registerFunctions(f.getPath(), namespace, pigContext);
}
- pigContext.addScriptFile(path);
+ pigContext.addScriptFile(f.getPath());
}
/**
Modified: pig/branches/branch-0.10/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/TestPigServer.java?rev=1308251&r1=1308250&r2=1308251&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/TestPigServer.java (original)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/TestPigServer.java Mon Apr 2 07:07:07 2012
@@ -423,6 +423,32 @@ public class TestPigServer {
Assert.assertTrue(((Long)iter.next().get(0))==2);
}
+
+ @Test
+ public void testRegisterRemoteScript() throws Throwable {
+ String scriptName = "script.py";
+ File scriptFile = File.createTempFile("tmp", "");
+ PrintWriter pw = new PrintWriter(new FileWriter(scriptFile));
+ pw.println("@outputSchema(\"word:chararray\")\ndef helloworld():\n return 'Hello, World'");
+ pw.close();
+
+ FileSystem fs = cluster.getFileSystem();
+ fs.copyFromLocalFile(new Path(scriptFile.getAbsolutePath()), new Path(scriptName));
+
+ // find the absolute path for the directory so that it does not
+ // depend on configuration
+ String absPath = fs.getFileStatus(new Path(scriptName)).getPath().toString();
+
+ Util.createInputFile(cluster, "testRegisterRemoteScript_input", new String[]{"1", "2"});
+ pig.registerCode(absPath, "jython", "pig");
+ pig.registerQuery("a = load 'testRegisterRemoteScript_input';");
+ pig.registerQuery("b = foreach a generate pig.helloworld($0);");
+ Iterator<Tuple> iter = pig.openIterator("b");
+
+ Assert.assertTrue(iter.next().get(0).equals("Hello, World"));
+ Assert.assertTrue(iter.next().get(0).equals("Hello, World"));
+ Assert.assertFalse(iter.hasNext());
+ }
@Test
public void testDescribeLoad() throws Throwable {