You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ju...@apache.org on 2012/02/27 02:29:15 UTC
svn commit: r1293990 - in /pig/branches/branch-0.9: ./ .eclipse.templates/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/impl/util/ test/org/apache/pig/test/ test/resources/
test/resources/org/ test/resources/org/...
Author: julien
Date: Mon Feb 27 01:29:15 2012
New Revision: 1293990
URL: http://svn.apache.org/viewvc?rev=1293990&view=rev
Log:
PIG-2532: Registered classes fail deserialization in frontend (traviscrawford via julien)
Added:
pig/branches/branch-0.9/test/org/apache/pig/test/TestRegisteredJarVisibility.java
pig/branches/branch-0.9/test/resources/
pig/branches/branch-0.9/test/resources/org/
pig/branches/branch-0.9/test/resources/org/apache/
pig/branches/branch-0.9/test/resources/org/apache/pig/
pig/branches/branch-0.9/test/resources/org/apache/pig/test/
pig/branches/branch-0.9/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java
pig/branches/branch-0.9/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java
Modified:
pig/branches/branch-0.9/.eclipse.templates/.classpath
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/build.xml
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
pig/branches/branch-0.9/src/org/apache/pig/impl/util/UDFContext.java
Modified: pig/branches/branch-0.9/.eclipse.templates/.classpath
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/.eclipse.templates/.classpath?rev=1293990&r1=1293989&r2=1293990&view=diff
==============================================================================
--- pig/branches/branch-0.9/.eclipse.templates/.classpath (original)
+++ pig/branches/branch-0.9/.eclipse.templates/.classpath Mon Feb 27 01:29:15 2012
@@ -6,7 +6,7 @@
<classpathentry kind="src" path="shims/src/hadoop20"/>
<classpathentry kind="src" path="src-gen"/>
<classpathentry kind="src" path="src"/>
- <classpathentry excluding="e2e/pig/udfs/java/" kind="src" path="test"/>
+ <classpathentry excluding="e2e/pig/udfs/java/|resources/" kind="src" path="test"/>
<classpathentry kind="src" path="test/e2e/pig/udfs/java"/>
<classpathentry kind="src" path="tutorial/src"/>
<classpathentry exported="true" kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1293990&r1=1293989&r2=1293990&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Mon Feb 27 01:29:15 2012
@@ -22,6 +22,8 @@ Release 0.9.3 - Unreleased
BUG FIXES
+PIG-2532: Registered classes fail deserialization in frontend (traviscrawford via julien)
+
PIG-2493: UNION causes casting issues (vivekp via daijy)
PIG-2497: Order of execution of fs, store and sh commands in Pig is not maintained (daijy)
Modified: pig/branches/branch-0.9/build.xml
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/build.xml?rev=1293990&r1=1293989&r2=1293990&view=diff
==============================================================================
--- pig/branches/branch-0.9/build.xml (original)
+++ pig/branches/branch-0.9/build.xml Mon Feb 27 01:29:15 2012
@@ -403,7 +403,7 @@
<param name="dist" value="${test.build.classes}" />
<param name="cp" value="test.classpath" />
<!-- don't compile PigTestLoader -->
- <param name="excludes" value="**/PigTestLoader.java" />
+ <param name="excludes" value="**/PigTestLoader.java **/resources/**" />
</antcall>
<antcall target="compile-sources-all-warnings" inheritRefs="true" inheritall="true">
@@ -411,7 +411,7 @@
<param name="dist" value="${test.build.classes}" />
<param name="cp" value="test.classpath" />
<!-- don't compile PigTestLoader -->
- <param name="excludes" value="**/PigTestLoader.java" />
+ <param name="excludes" value="**/PigTestLoader.java **/resources/**" />
</antcall>
<copy file="${basedir}/test/hbase-site.xml" tofile="${test.build.classes}/hbase-site.xml"/>
Modified: pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=1293990&r1=1293989&r2=1293990&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Mon Feb 27 01:29:15 2012
@@ -65,6 +65,7 @@ import org.apache.pig.impl.plan.VisitorE
import org.apache.pig.impl.plan.CompilationMessageCollector.MessageType;
import org.apache.pig.impl.util.ConfigurationValidator;
import org.apache.pig.impl.util.LogUtils;
+import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.impl.util.Utils;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStatsUtil;
@@ -220,8 +221,17 @@ public class MapReduceLauncher extends L
}
completeFailedJobsInThisRun.clear();
-
- Thread jcThread = new Thread(jc);
+
+ // Set the thread UDFContext so registered classes are available.
+ final UDFContext udfContext = UDFContext.getUDFContext();
+ Thread jcThread = new Thread(jc) {
+ @Override
+ public void run() {
+ UDFContext.setUdfContext(udfContext);
+ super.run();
+ }
+ };
+
jcThread.setUncaughtExceptionHandler(jctExceptionHandler);
jcThread.setContextClassLoader(PigContext.getClassLoader());
Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/util/UDFContext.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/util/UDFContext.java?rev=1293990&r1=1293989&r2=1293990&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/util/UDFContext.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/util/UDFContext.java Mon Feb 27 01:29:15 2012
@@ -50,6 +50,13 @@ public class UDFContext {
/*
* internal pig use only - should NOT be called from user code
*/
+ public static void setUdfContext(UDFContext udfContext) {
+ tss.set(udfContext);
+ }
+
+ /*
+ * internal pig use only - should NOT be called from user code
+ */
public void setClientSystemProps(Properties properties) {
clientSysProps = properties;
}
Added: pig/branches/branch-0.9/test/org/apache/pig/test/TestRegisteredJarVisibility.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestRegisteredJarVisibility.java?rev=1293990&view=auto
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestRegisteredJarVisibility.java (added)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestRegisteredJarVisibility.java Mon Feb 27 01:29:15 2012
@@ -0,0 +1,163 @@
+package org.apache.pig.test;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.io.ByteStreams;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Ensure classes from a registered jar are available in the UDFContext.
+ * Please see PIG-2532 for additional details.
+ */
+public class TestRegisteredJarVisibility {
+ private static final Log LOG = LogFactory.getLog(TestRegisteredJarVisibility.class);
+ private static final String JAR_FILE_NAME = "test-foo-loader.jar";
+ private static final String PACKAGE_NAME = "org.apache.pig.test";
+ // Actual data is not important. Reusing an existing input file.
+ private static final File INPUT_FILE = new File("test/data/pigunit/top_queries_input_data.txt");
+
+ private MiniCluster cluster;
+ private File jarFile;
+
+ @Before()
+ public void setUp() throws IOException {
+
+ String testResourcesDir = "test/resources/" + PACKAGE_NAME.replace(".", "/");
+
+ String testBuildDataDir = "build/test/data";
+ // Create the test data directory if needed
+ File testDataDir = new File(testBuildDataDir,
+ TestRegisteredJarVisibility.class.getCanonicalName());
+ testDataDir.mkdirs();
+
+ jarFile = new File(testDataDir, JAR_FILE_NAME);
+
+ File[] javaFiles = new File[]{
+ new File(testResourcesDir, "RegisteredJarVisibilityLoader.java"),
+ new File(testResourcesDir, "RegisteredJarVisibilitySchema.java")};
+
+ List<File> classFiles = compile(javaFiles);
+
+ // Canonical class name to class file
+ Map<String, File> filesToJar = Maps.newHashMap();
+ for (File classFile : classFiles) {
+ filesToJar.put(
+ PACKAGE_NAME + "." + classFile.getName().replace(".class", ""),
+ classFile);
+ }
+
+ jar(filesToJar);
+
+ cluster = MiniCluster.buildCluster();
+ }
+
+ @After()
+ public void tearDown() {
+ cluster.shutDown();
+ }
+
+ @Test()
+ public void testRegisteredJarVisibilitySchemaNotOnClasspath() {
+ boolean exceptionThrown = false;
+ try {
+ Class.forName("org.apache.pig.test.FooSchema");
+ } catch (ClassNotFoundException e) {
+ exceptionThrown = true;
+ }
+ Assert.assertTrue(exceptionThrown);
+ }
+
+ @Test()
+ public void testRegisteredJarVisibility() throws IOException {
+ cluster.getFileSystem().copyFromLocalFile(
+ new Path("file://" + INPUT_FILE.getAbsolutePath()), new Path(INPUT_FILE.getName()));
+ PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+
+ String query = "register " + jarFile.getAbsolutePath() + ";\n"
+ + "a = load '" + INPUT_FILE.getName()
+ + "' using org.apache.pig.test.RegisteredJarVisibilityLoader();";
+ LOG.info("Running pig script:\n" + query);
+ pigServer.registerScript(new ByteArrayInputStream(query.getBytes()));
+
+ pigServer.openIterator("a");
+ pigServer.shutdown();
+ }
+
+ private List<File> compile(File[] javaFiles) {
+ LOG.info("Compiling: " + Arrays.asList(javaFiles));
+
+ JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+ StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null);
+ Iterable<? extends JavaFileObject> compilationUnits1 =
+ fileManager.getJavaFileObjects(javaFiles);
+ JavaCompiler.CompilationTask task =
+ compiler.getTask(null, fileManager, null, null, null, compilationUnits1);
+ task.call();
+
+ List<File> classFiles = Lists.newArrayList();
+ for (File javaFile : javaFiles) {
+ File classFile = new File(javaFile.getAbsolutePath().replace(".java", ".class"));
+ classFile.deleteOnExit();
+ Assert.assertTrue(classFile.exists());
+ classFiles.add(classFile);
+ LOG.info("Created " + classFile.getAbsolutePath());
+ }
+
+ return classFiles;
+ }
+
+ /**
+ * Create a jar file containing the generated classes.
+ *
+ * @param filesToJar map of canonical class name to class file
+ * @throws IOException on error
+ */
+ private void jar(Map<String, File> filesToJar) throws IOException {
+ LOG.info("Creating jar file containing: " + filesToJar);
+
+ JarOutputStream jos = new JarOutputStream(new FileOutputStream(jarFile.getAbsolutePath()));
+ try {
+ for (Map.Entry<String, File> entry : filesToJar.entrySet()) {
+ String zipEntryName = entry.getKey().replace(".", "/") + ".class";
+ LOG.info("Adding " + zipEntryName + " to " + jarFile.getAbsolutePath());
+ jos.putNextEntry(new ZipEntry(zipEntryName));
+ InputStream classInputStream = new FileInputStream(entry.getValue().getAbsolutePath());
+ try {
+ ByteStreams.copy(classInputStream, jos);
+ } finally {
+ classInputStream.close();
+ }
+ }
+ } finally {
+ jos.close();
+ }
+ Assert.assertTrue(jarFile.exists());
+ LOG.info("Created " + jarFile.getAbsolutePath());
+ }
+}
Added: pig/branches/branch-0.9/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java?rev=1293990&view=auto
==============================================================================
--- pig/branches/branch-0.9/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java (added)
+++ pig/branches/branch-0.9/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java Mon Feb 27 01:29:15 2012
@@ -0,0 +1,37 @@
+package org.apache.pig.test;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.impl.util.UDFContext;
+
+import java.io.IOException;
+import java.util.Properties;
+
+/**
+ * Please see {@link TestRegisteredJarVisibility} for information about this class.
+ */
+public class RegisteredJarVisibilityLoader extends PigStorage {
+ private static final Log LOG = LogFactory.getLog(RegisteredJarVisibilityLoader.class);
+ private static final String REGISTERED_JAR_VISIBILITY_SCHEMA = "registered.jar.visibility.schema";
+
+ @Override
+ public void setLocation(String location, Job job) throws IOException {
+ UDFContext udfContext = UDFContext.getUDFContext();
+ Properties properties = udfContext.getUDFProperties(RegisteredJarVisibilityLoader.class);
+
+ if (!properties.containsKey(REGISTERED_JAR_VISIBILITY_SCHEMA)) {
+ LOG.info("Storing " + RegisteredJarVisibilitySchema.class.getName() + " in UDFContext.");
+ properties.put(REGISTERED_JAR_VISIBILITY_SCHEMA, new RegisteredJarVisibilitySchema());
+ LOG.info("Stored " + RegisteredJarVisibilitySchema.class.getName() + " in UDFContext.");
+ } else {
+ LOG.info("Retrieving " + REGISTERED_JAR_VISIBILITY_SCHEMA + " from UDFContext.");
+ RegisteredJarVisibilitySchema registeredJarVisibilitySchema =
+ (RegisteredJarVisibilitySchema) properties.get(REGISTERED_JAR_VISIBILITY_SCHEMA);
+ LOG.info("Retrieved " + REGISTERED_JAR_VISIBILITY_SCHEMA + " from UDFContext.");
+ }
+
+ super.setLocation(location, job);
+ }
+}
Added: pig/branches/branch-0.9/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java?rev=1293990&view=auto
==============================================================================
--- pig/branches/branch-0.9/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java (added)
+++ pig/branches/branch-0.9/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java Mon Feb 27 01:29:15 2012
@@ -0,0 +1,9 @@
+package org.apache.pig.test;
+
+import java.io.Serializable;
+
+/**
+ * Please see {@link TestRegisteredJarVisibility} for information about this class.
+ */
+public class RegisteredJarVisibilitySchema implements Serializable {
+}