You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ju...@apache.org on 2012/02/27 00:48:35 UTC
svn commit: r1293967 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/impl/util/ test/org/apache/pig/test/ test/resources/
test/resources/org/ test/resources/org/apache/ test/resources/org/apache/...
Author: julien
Date: Sun Feb 26 23:48:35 2012
New Revision: 1293967
URL: http://svn.apache.org/viewvc?rev=1293967&view=rev
Log:
PIG-2532: Registered classes fail deserialization in frontend (traviscrawford via julien)
Added:
pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java
pig/trunk/test/resources/
pig/trunk/test/resources/org/
pig/trunk/test/resources/org/apache/
pig/trunk/test/resources/org/apache/pig/
pig/trunk/test/resources/org/apache/pig/test/
pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java
pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/build.xml
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
pig/trunk/src/org/apache/pig/impl/util/UDFContext.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1293967&r1=1293966&r2=1293967&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sun Feb 26 23:48:35 2012
@@ -100,6 +100,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-2532: Registered classes fail deserialization in frontend (traviscrawford via julien)
+
PIG-2549: org.apache.pig.piggybank.storage.avro - Broken documentation link for AvroStorage (chrisas via daijy)
PIG-2322: varargs functions do not get passed the arguments in Python embedding (julien)
Modified: pig/trunk/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1293967&r1=1293966&r2=1293967&view=diff
==============================================================================
--- pig/trunk/build.xml (original)
+++ pig/trunk/build.xml Sun Feb 26 23:48:35 2012
@@ -266,7 +266,7 @@
<source path="${src.shims.dir}"/>
<source path="${src.shims.test.dir}"/>
<source path="tutorial/src"/>
- <source path="${test.src.dir}" excluding="e2e/pig/udfs/java/"/>
+ <source path="${test.src.dir}" excluding="e2e/pig/udfs/java/|resources/"/>
<output path="${build.dir.eclipse-main-classes}" />
<library pathref="eclipse.classpath" exported="true" />
<!--library pathref="classpath" exported="false"/-->
@@ -464,7 +464,7 @@
<param name="dist" value="${test.build.classes}" />
<param name="cp" value="test.classpath" />
<!-- don't compile PigTestLoader -->
- <param name="excludes" value="**/PigTestLoader.java" />
+ <param name="excludes" value="**/PigTestLoader.java **/resources/**" />
</antcall>
<antcall target="compile-sources-all-warnings" inheritRefs="true" inheritall="true">
@@ -472,7 +472,7 @@
<param name="dist" value="${test.build.classes}" />
<param name="cp" value="test.classpath" />
<!-- don't compile PigTestLoader -->
- <param name="excludes" value="**/PigTestLoader.java" />
+ <param name="excludes" value="**/PigTestLoader.java **/resources/**" />
</antcall>
<copy file="${basedir}/test/hbase-site.xml" tofile="${test.build.classes}/hbase-site.xml"/>
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=1293967&r1=1293966&r2=1293967&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Sun Feb 26 23:48:35 2012
@@ -64,6 +64,7 @@ import org.apache.pig.impl.plan.VisitorE
import org.apache.pig.impl.plan.CompilationMessageCollector.MessageType;
import org.apache.pig.impl.util.ConfigurationValidator;
import org.apache.pig.impl.util.LogUtils;
+import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.impl.util.Utils;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStatsUtil;
@@ -249,8 +250,17 @@ public class MapReduceLauncher extends L
}
completeFailedJobsInThisRun.clear();
-
- Thread jcThread = new Thread(jc);
+
+ // Set the thread UDFContext so registered classes are available.
+ final UDFContext udfContext = UDFContext.getUDFContext();
+ Thread jcThread = new Thread(jc) {
+ @Override
+ public void run() {
+ UDFContext.setUdfContext(udfContext);
+ super.run();
+ }
+ };
+
jcThread.setUncaughtExceptionHandler(jctExceptionHandler);
jcThread.setContextClassLoader(PigContext.getClassLoader());
Modified: pig/trunk/src/org/apache/pig/impl/util/UDFContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/UDFContext.java?rev=1293967&r1=1293966&r2=1293967&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/UDFContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/UDFContext.java Sun Feb 26 23:48:35 2012
@@ -54,6 +54,13 @@ public class UDFContext {
/*
* internal pig use only - should NOT be called from user code
*/
+ public static void setUdfContext(UDFContext udfContext) {
+ tss.set(udfContext);
+ }
+
+ /*
+ * internal pig use only - should NOT be called from user code
+ */
public void setClientSystemProps(Properties properties) {
clientSysProps = properties;
}
Added: pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java?rev=1293967&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java (added)
+++ pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java Sun Feb 26 23:48:35 2012
@@ -0,0 +1,163 @@
+package org.apache.pig.test;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.io.ByteStreams;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Ensure classes from a registered jar are available in the UDFContext.
+ * Please see PIG-2532 for additional details.
+ */
+public class TestRegisteredJarVisibility {
+ private static final Log LOG = LogFactory.getLog(TestRegisteredJarVisibility.class);
+ private static final String JAR_FILE_NAME = "test-foo-loader.jar";
+ private static final String PACKAGE_NAME = "org.apache.pig.test";
+ // Actual data is not important. Reusing an existing input file.
+ private static final File INPUT_FILE = new File("test/data/pigunit/top_queries_input_data.txt");
+
+ private MiniCluster cluster;
+ private File jarFile;
+
+ @Before()
+ public void setUp() throws IOException {
+
+ String testResourcesDir = "test/resources/" + PACKAGE_NAME.replace(".", "/");
+
+ String testBuildDataDir = "build/test/data";
+ // Create the test data directory if needed
+ File testDataDir = new File(testBuildDataDir,
+ TestRegisteredJarVisibility.class.getCanonicalName());
+ testDataDir.mkdirs();
+
+ jarFile = new File(testDataDir, JAR_FILE_NAME);
+
+ File[] javaFiles = new File[]{
+ new File(testResourcesDir, "RegisteredJarVisibilityLoader.java"),
+ new File(testResourcesDir, "RegisteredJarVisibilitySchema.java")};
+
+ List<File> classFiles = compile(javaFiles);
+
+ // Canonical class name to class file
+ Map<String, File> filesToJar = Maps.newHashMap();
+ for (File classFile : classFiles) {
+ filesToJar.put(
+ PACKAGE_NAME + "." + classFile.getName().replace(".class", ""),
+ classFile);
+ }
+
+ jar(filesToJar);
+
+ cluster = MiniCluster.buildCluster();
+ }
+
+ @After()
+ public void tearDown() {
+ cluster.shutDown();
+ }
+
+ @Test()
+ public void testRegisteredJarVisibilitySchemaNotOnClasspath() {
+ boolean exceptionThrown = false;
+ try {
+ Class.forName("org.apache.pig.test.FooSchema");
+ } catch (ClassNotFoundException e) {
+ exceptionThrown = true;
+ }
+ Assert.assertTrue(exceptionThrown);
+ }
+
+ @Test()
+ public void testRegisteredJarVisibility() throws IOException {
+ cluster.getFileSystem().copyFromLocalFile(
+ new Path("file://" + INPUT_FILE.getAbsolutePath()), new Path(INPUT_FILE.getName()));
+ PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+
+ String query = "register " + jarFile.getAbsolutePath() + ";\n"
+ + "a = load '" + INPUT_FILE.getName()
+ + "' using org.apache.pig.test.RegisteredJarVisibilityLoader();";
+ LOG.info("Running pig script:\n" + query);
+ pigServer.registerScript(new ByteArrayInputStream(query.getBytes()));
+
+ pigServer.openIterator("a");
+ pigServer.shutdown();
+ }
+
+ private List<File> compile(File[] javaFiles) {
+ LOG.info("Compiling: " + Arrays.asList(javaFiles));
+
+ JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+ StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null);
+ Iterable<? extends JavaFileObject> compilationUnits1 =
+ fileManager.getJavaFileObjects(javaFiles);
+ JavaCompiler.CompilationTask task =
+ compiler.getTask(null, fileManager, null, null, null, compilationUnits1);
+ task.call();
+
+ List<File> classFiles = Lists.newArrayList();
+ for (File javaFile : javaFiles) {
+ File classFile = new File(javaFile.getAbsolutePath().replace(".java", ".class"));
+ classFile.deleteOnExit();
+ Assert.assertTrue(classFile.exists());
+ classFiles.add(classFile);
+ LOG.info("Created " + classFile.getAbsolutePath());
+ }
+
+ return classFiles;
+ }
+
+ /**
+ * Create a jar file containing the generated classes.
+ *
+ * @param filesToJar map of canonical class name to class file
+ * @throws IOException on error
+ */
+ private void jar(Map<String, File> filesToJar) throws IOException {
+ LOG.info("Creating jar file containing: " + filesToJar);
+
+ JarOutputStream jos = new JarOutputStream(new FileOutputStream(jarFile.getAbsolutePath()));
+ try {
+ for (Map.Entry<String, File> entry : filesToJar.entrySet()) {
+ String zipEntryName = entry.getKey().replace(".", "/") + ".class";
+ LOG.info("Adding " + zipEntryName + " to " + jarFile.getAbsolutePath());
+ jos.putNextEntry(new ZipEntry(zipEntryName));
+ InputStream classInputStream = new FileInputStream(entry.getValue().getAbsolutePath());
+ try {
+ ByteStreams.copy(classInputStream, jos);
+ } finally {
+ classInputStream.close();
+ }
+ }
+ } finally {
+ jos.close();
+ }
+ Assert.assertTrue(jarFile.exists());
+ LOG.info("Created " + jarFile.getAbsolutePath());
+ }
+}
Added: pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java?rev=1293967&view=auto
==============================================================================
--- pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java (added)
+++ pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java Sun Feb 26 23:48:35 2012
@@ -0,0 +1,37 @@
+package org.apache.pig.test;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.impl.util.UDFContext;
+
+import java.io.IOException;
+import java.util.Properties;
+
+/**
+ * Please see {@link TestRegisteredJarVisibility} for information about this class.
+ */
+public class RegisteredJarVisibilityLoader extends PigStorage {
+ private static final Log LOG = LogFactory.getLog(RegisteredJarVisibilityLoader.class);
+ private static final String REGISTERED_JAR_VISIBILITY_SCHEMA = "registered.jar.visibility.schema";
+
+ @Override
+ public void setLocation(String location, Job job) throws IOException {
+ UDFContext udfContext = UDFContext.getUDFContext();
+ Properties properties = udfContext.getUDFProperties(RegisteredJarVisibilityLoader.class);
+
+ if (!properties.containsKey(REGISTERED_JAR_VISIBILITY_SCHEMA)) {
+ LOG.info("Storing " + RegisteredJarVisibilitySchema.class.getName() + " in UDFContext.");
+ properties.put(REGISTERED_JAR_VISIBILITY_SCHEMA, new RegisteredJarVisibilitySchema());
+ LOG.info("Stored " + RegisteredJarVisibilitySchema.class.getName() + " in UDFContext.");
+ } else {
+ LOG.info("Retrieving " + REGISTERED_JAR_VISIBILITY_SCHEMA + " from UDFContext.");
+ RegisteredJarVisibilitySchema registeredJarVisibilitySchema =
+ (RegisteredJarVisibilitySchema) properties.get(REGISTERED_JAR_VISIBILITY_SCHEMA);
+ LOG.info("Retrieved " + REGISTERED_JAR_VISIBILITY_SCHEMA + " from UDFContext.");
+ }
+
+ super.setLocation(location, job);
+ }
+}
Added: pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java?rev=1293967&view=auto
==============================================================================
--- pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java (added)
+++ pig/trunk/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java Sun Feb 26 23:48:35 2012
@@ -0,0 +1,9 @@
+package org.apache.pig.test;
+
+import java.io.Serializable;
+
+/**
+ * Please see {@link TestRegisteredJarVisibility} for information about this class.
+ */
+public class RegisteredJarVisibilitySchema implements Serializable {
+}