You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by dv...@apache.org on 2012/08/24 04:57:06 UTC
svn commit: r1376815 - in /pig/trunk: ./ src/org/apache/pig/impl/io/
src/org/apache/pig/parser/ test/org/apache/pig/test/
Author: dvryaboy
Date: Fri Aug 24 02:57:05 2012
New Revision: 1376815
URL: http://svn.apache.org/viewvc?rev=1376815&view=rev
Log:
PIG-2850: Pig should support loading macro files as resources stored in JAR files
Added:
pig/trunk/src/org/apache/pig/impl/io/ResourceNotFoundException.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1376815&r1=1376814&r2=1376815&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Aug 24 02:57:05 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2850: Pig should support loading macro files as resources stored in JAR files (matterhayes via dvryaboy)
+
PIG-1314: Add DateTime Support to Pig (zjshen via thejas)
PIG-2785: NoClassDefFoundError after upgrading to pig 0.10.0 from 0.9.0 (matterhayes via sms)
Modified: pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java?rev=1376815&r1=1376814&r2=1376815&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java (original)
+++ pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java Fri Aug 24 02:57:05 2012
@@ -17,6 +17,7 @@
*/
package org.apache.pig.impl.io;
+import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
@@ -818,4 +819,36 @@ public class FileLocalizer {
return fetchFiles;
}
+
+ /**
+ * Ensures that the passed resource is available from the local file system, fetching
+ * it to a temporary directory.
+ *
+ * @throws ResourceNotFoundException
+ */
+ public static FetchFileRet fetchResource(String name) throws IOException, ResourceNotFoundException {
+ FetchFileRet localFileRet = null;
+ InputStream resourceStream = PigContext.getClassLoader().getResourceAsStream(name);
+ if (resourceStream != null) {
+ File dest = new File(localTempDir, name);
+ dest.getParentFile().mkdirs();
+ dest.deleteOnExit();
+
+ OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(dest));
+ byte[] buffer = new byte[1024];
+ int len;
+ while ((len=resourceStream.read(buffer)) > 0) {
+ outputStream.write(buffer,0,len);
+ }
+ outputStream.close();
+
+ localFileRet = new FetchFileRet(dest,false);
+ }
+ else
+ {
+ throw new ResourceNotFoundException(name);
+ }
+
+ return localFileRet;
+ }
}
Added: pig/trunk/src/org/apache/pig/impl/io/ResourceNotFoundException.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/io/ResourceNotFoundException.java?rev=1376815&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/io/ResourceNotFoundException.java (added)
+++ pig/trunk/src/org/apache/pig/impl/io/ResourceNotFoundException.java Fri Aug 24 02:57:05 2012
@@ -0,0 +1,18 @@
+package org.apache.pig.impl.io;
+
+public class ResourceNotFoundException extends Exception
+{
+ private static final long serialVersionUID = 1L;
+
+ private String resourceName;
+
+ public ResourceNotFoundException(String resourceName)
+ {
+ this.resourceName = resourceName;
+ }
+
+ public String toString()
+ {
+ return "Resource '" + resourceName + "' does not exist";
+ }
+}
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1376815&r1=1376814&r2=1376815&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Fri Aug 24 02:57:05 2012
@@ -22,7 +22,10 @@ import java.io.BufferedReader;
import java.io.FileReader;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -39,9 +42,13 @@ import org.antlr.runtime.tree.CommonTree
import org.antlr.runtime.tree.Tree;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.pig.PigException;
+import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.io.FileLocalizer.FetchFileRet;
+import org.apache.pig.impl.io.ResourceNotFoundException;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
@@ -349,11 +356,34 @@ public class QueryParserDriver {
if (fnameMap.get(fname) != null) {
localFileRet = fnameMap.get(fname);
} else {
+ try {
File localFile = QueryParserUtils.getFileFromImportSearchPath(fname);
localFileRet = localFile == null ?
FileLocalizer.fetchFile(pigContext.getProperties(), fname)
: new FetchFileRet(localFile.getCanonicalFile(), false);
- fnameMap.put(fname, localFileRet);
+ } catch (FileNotFoundException e) {
+ // ignore this since we'll attempt to load as a resource before failing
+ LOG.debug(String.format("Macro file %s was not found", fname));
+ }
+
+ // try loading the macro file as a resource in case it is packaged in a registered jar
+ if (localFileRet == null) {
+ LOG.debug(String.format("Attempting to load macro file %s as a resource", fname));
+
+ try
+ {
+ localFileRet = FileLocalizer.fetchResource(fname);
+ LOG.debug(String.format("Found macro file %s as resource", fname));
+ }
+ catch (ResourceNotFoundException e)
+ {
+ LOG.debug(String.format("Macro file %s was not found as resource either", fname));
+ LOG.error(String.format("Failed to find macro file %s", fname));
+ throw new ExecException("file '" + fname + "' does not exist.", 101, PigException.INPUT);
+ }
+ }
+
+ fnameMap.put(fname, localFileRet);
}
} catch (IOException e) {
throw new RuntimeException("Unable to fetch macro file '" + fname + "'", e);
@@ -369,11 +399,13 @@ public class QueryParserDriver {
private PigMacro makeMacroDef(CommonTree t, Map<String, PigMacro> seen)
throws ParserException {
String mn = t.getChild(0).getText();
-
+
if (!macroSeen.add(mn)) {
String msg = getErrorMessage(null, t, null,
"Duplicated macro name '" + mn + "'");
- throw new ParserException(msg);
+ LOG.warn(msg);
+ // Disabling this exception, see https://issues.apache.org/jira/browse/PIG-2279
+ // throw new ParserException(msg);
}
if (seen != null) {
@@ -383,7 +415,7 @@ public class QueryParserDriver {
}
String fname = ((PigParserNode)t).getFileName();
-
+
Tree defNode = t.getChild(1);
// get parameter markers
@@ -446,7 +478,9 @@ public class QueryParserDriver {
if (!importSeen.add(fname)) {
String msg = getErrorMessage(fname, t,
": Duplicated import file '" + fname + "'", null);
- throw new ParserException(msg);
+ LOG.warn(msg);
+ // Disabling this exception, see https://issues.apache.org/jira/browse/PIG-2279
+ // throw new ParserException(msg);
}
FetchFileRet localFileRet = getMacroFile(fname);
Modified: pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java?rev=1376815&r1=1376814&r2=1376815&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigServerWithMacros.java Fri Aug 24 02:57:05 2012
@@ -30,16 +30,21 @@ import org.junit.Before;
import org.junit.Test;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.Iterator;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
import static org.apache.pig.builtin.mock.Storage.resetData;
import static org.apache.pig.builtin.mock.Storage.tuple;
public class TestPigServerWithMacros {
private PigServer pig = null;
-
+
@Before
public void setUp() throws Exception{
pig = new PigServer(ExecType.LOCAL);
@@ -120,4 +125,40 @@ public class TestPigServerWithMacros {
Assert.assertTrue(iter.next().get(0).equals("Hello, World"));
Assert.assertFalse(iter.hasNext());
}
+
+ @Test
+ public void testRegisterResourceMacro() throws Throwable {
+ String macrosFile = "test/pig/macros.pig";
+ File macrosJarFile = File.createTempFile("macros", ".jar");
+
+ System.out.println("Creating macros jar " + macrosJarFile);
+
+ Manifest manifest = new Manifest();
+ manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+
+ JarOutputStream jarStream = new JarOutputStream(new FileOutputStream(macrosJarFile), manifest);
+
+ JarEntry jarEntry = new JarEntry(macrosFile);
+ jarEntry.setTime(System.currentTimeMillis());
+ jarStream.putNextEntry(jarEntry);
+
+ PrintWriter pw = new PrintWriter(jarStream);
+ pw.println("DEFINE row_count_in_jar(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate COUNT($X); };");
+ pw.close();
+
+ jarStream.close();
+
+ Storage.Data data = resetData(pig);
+ data.set("some_path", "(l:int)", tuple(tuple("1")), tuple(tuple("2")), tuple(tuple("3")), tuple(tuple("10")), tuple(tuple("11")));
+
+ System.out.println("Registering macros jar " + macrosJarFile);
+ pig.registerJar(macrosJarFile.toString());
+
+ pig.registerQuery("import '" + macrosFile + "';");
+ pig.registerQuery("a = load 'some_path' USING mock.Storage();");
+ pig.registerQuery("b = row_count_in_jar(a);");
+ Iterator<Tuple> iter = pig.openIterator("b");
+
+ Assert.assertTrue(((Long)iter.next().get(0))==5);
+ }
}