You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/03/08 00:43:26 UTC
svn commit: r1298227 - in /pig/branches/branch-0.10/test:
org/apache/pig/test/ resources/ resources/org/ resources/org/apache/
resources/org/apache/pig/ resources/org/apache/pig/test/
Author: daijy
Date: Wed Mar 7 23:43:26 2012
New Revision: 1298227
URL: http://svn.apache.org/viewvc?rev=1298227&view=rev
Log:
PIG-2532: Registered classes fail deserialization in frontend
Added:
pig/branches/branch-0.10/test/org/apache/pig/test/TestRegisteredJarVisibility.java
pig/branches/branch-0.10/test/resources/
pig/branches/branch-0.10/test/resources/org/
pig/branches/branch-0.10/test/resources/org/apache/
pig/branches/branch-0.10/test/resources/org/apache/pig/
pig/branches/branch-0.10/test/resources/org/apache/pig/test/
pig/branches/branch-0.10/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java
pig/branches/branch-0.10/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java
Added: pig/branches/branch-0.10/test/org/apache/pig/test/TestRegisteredJarVisibility.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/org/apache/pig/test/TestRegisteredJarVisibility.java?rev=1298227&view=auto
==============================================================================
--- pig/branches/branch-0.10/test/org/apache/pig/test/TestRegisteredJarVisibility.java (added)
+++ pig/branches/branch-0.10/test/org/apache/pig/test/TestRegisteredJarVisibility.java Wed Mar 7 23:43:26 2012
@@ -0,0 +1,165 @@
+package org.apache.pig.test;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.io.ByteStreams;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Ensure classes from a registered jar are available in the UDFContext.
+ * Please see PIG-2532 for additional details.
+ */
+public class TestRegisteredJarVisibility {
+ private static final Log LOG = LogFactory.getLog(TestRegisteredJarVisibility.class);
+ private static final String JAR_FILE_NAME = "test-foo-loader.jar";
+ private static final String PACKAGE_NAME = "org.apache.pig.test";
+ // Actual data is not important. Reusing an existing input file.
+ private static final File INPUT_FILE = new File("test/data/pigunit/top_queries_input_data.txt");
+
+ private static MiniCluster cluster;
+ private static File jarFile;
+
+ @BeforeClass()
+ public static void setUp() throws IOException {
+
+ String testResourcesDir = "test/resources/" + PACKAGE_NAME.replace(".", "/");
+
+ String testBuildDataDir = "build/test/data";
+ // Create the test data directory if needed
+ File testDataDir = new File(testBuildDataDir,
+ TestRegisteredJarVisibility.class.getCanonicalName());
+ testDataDir.mkdirs();
+
+ jarFile = new File(testDataDir, JAR_FILE_NAME);
+
+ File[] javaFiles = new File[]{
+ new File(testResourcesDir, "RegisteredJarVisibilityLoader.java"),
+ new File(testResourcesDir, "RegisteredJarVisibilitySchema.java")};
+
+ List<File> classFiles = compile(javaFiles);
+
+ // Canonical class name to class file
+ Map<String, File> filesToJar = Maps.newHashMap();
+ for (File classFile : classFiles) {
+ filesToJar.put(
+ PACKAGE_NAME + "." + classFile.getName().replace(".class", ""),
+ classFile);
+ }
+
+ jar(filesToJar);
+
+ cluster = MiniCluster.buildCluster();
+ }
+
+ @AfterClass()
+ public static void tearDown() {
+ cluster.shutDown();
+ }
+
+ @Test()
+ public void testRegisteredJarVisibilitySchemaNotOnClasspath() {
+ boolean exceptionThrown = false;
+ try {
+ Class.forName("org.apache.pig.test.FooSchema");
+ } catch (ClassNotFoundException e) {
+ exceptionThrown = true;
+ }
+ Assert.assertTrue(exceptionThrown);
+ }
+
+ @Test()
+ public void testRegisteredJarVisibility() throws IOException {
+ cluster.getFileSystem().copyFromLocalFile(
+ new Path("file://" + INPUT_FILE.getAbsolutePath()), new Path(INPUT_FILE.getName()));
+ PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+
+ String query = "register " + jarFile.getAbsolutePath() + ";\n"
+ + "a = load '" + INPUT_FILE.getName()
+ + "' using org.apache.pig.test.RegisteredJarVisibilityLoader();";
+ LOG.info("Running pig script:\n" + query);
+ pigServer.registerScript(new ByteArrayInputStream(query.getBytes()));
+
+ pigServer.openIterator("a");
+ pigServer.shutdown();
+ }
+
+ private static List<File> compile(File[] javaFiles) {
+ LOG.info("Compiling: " + Arrays.asList(javaFiles));
+
+ JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+ StandardJavaFileManager fileManager = compiler.getStandardFileManager(null, null, null);
+ Iterable<? extends JavaFileObject> compilationUnits1 =
+ fileManager.getJavaFileObjects(javaFiles);
+ JavaCompiler.CompilationTask task =
+ compiler.getTask(null, fileManager, null, null, null, compilationUnits1);
+ task.call();
+
+ List<File> classFiles = Lists.newArrayList();
+ for (File javaFile : javaFiles) {
+ File classFile = new File(javaFile.getAbsolutePath().replace(".java", ".class"));
+ classFile.deleteOnExit();
+ Assert.assertTrue(classFile.exists());
+ classFiles.add(classFile);
+ LOG.info("Created " + classFile.getAbsolutePath());
+ }
+
+ return classFiles;
+ }
+
+ /**
+ * Create a jar file containing the generated classes.
+ *
+ * @param filesToJar map of canonical class name to class file
+ * @throws IOException on error
+ */
+ private static void jar(Map<String, File> filesToJar) throws IOException {
+ LOG.info("Creating jar file containing: " + filesToJar);
+
+ JarOutputStream jos = new JarOutputStream(new FileOutputStream(jarFile.getAbsolutePath()));
+ try {
+ for (Map.Entry<String, File> entry : filesToJar.entrySet()) {
+ String zipEntryName = entry.getKey().replace(".", "/") + ".class";
+ LOG.info("Adding " + zipEntryName + " to " + jarFile.getAbsolutePath());
+ jos.putNextEntry(new ZipEntry(zipEntryName));
+ InputStream classInputStream = new FileInputStream(entry.getValue().getAbsolutePath());
+ try {
+ ByteStreams.copy(classInputStream, jos);
+ } finally {
+ classInputStream.close();
+ }
+ }
+ } finally {
+ jos.close();
+ }
+ Assert.assertTrue(jarFile.exists());
+ LOG.info("Created " + jarFile.getAbsolutePath());
+ }
+}
Added: pig/branches/branch-0.10/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java?rev=1298227&view=auto
==============================================================================
--- pig/branches/branch-0.10/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java (added)
+++ pig/branches/branch-0.10/test/resources/org/apache/pig/test/RegisteredJarVisibilityLoader.java Wed Mar 7 23:43:26 2012
@@ -0,0 +1,37 @@
+package org.apache.pig.test;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.impl.util.UDFContext;
+
+import java.io.IOException;
+import java.util.Properties;
+
+/**
+ * Please see {@link TestRegisteredJarVisibility} for information about this class.
+ */
+public class RegisteredJarVisibilityLoader extends PigStorage {
+ private static final Log LOG = LogFactory.getLog(RegisteredJarVisibilityLoader.class);
+ private static final String REGISTERED_JAR_VISIBILITY_SCHEMA = "registered.jar.visibility.schema";
+
+ @Override
+ public void setLocation(String location, Job job) throws IOException {
+ UDFContext udfContext = UDFContext.getUDFContext();
+ Properties properties = udfContext.getUDFProperties(RegisteredJarVisibilityLoader.class);
+
+ if (!properties.containsKey(REGISTERED_JAR_VISIBILITY_SCHEMA)) {
+ LOG.info("Storing " + RegisteredJarVisibilitySchema.class.getName() + " in UDFContext.");
+ properties.put(REGISTERED_JAR_VISIBILITY_SCHEMA, new RegisteredJarVisibilitySchema());
+ LOG.info("Stored " + RegisteredJarVisibilitySchema.class.getName() + " in UDFContext.");
+ } else {
+ LOG.info("Retrieving " + REGISTERED_JAR_VISIBILITY_SCHEMA + " from UDFContext.");
+ RegisteredJarVisibilitySchema registeredJarVisibilitySchema =
+ (RegisteredJarVisibilitySchema) properties.get(REGISTERED_JAR_VISIBILITY_SCHEMA);
+ LOG.info("Retrieved " + REGISTERED_JAR_VISIBILITY_SCHEMA + " from UDFContext.");
+ }
+
+ super.setLocation(location, job);
+ }
+}
Added: pig/branches/branch-0.10/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java?rev=1298227&view=auto
==============================================================================
--- pig/branches/branch-0.10/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java (added)
+++ pig/branches/branch-0.10/test/resources/org/apache/pig/test/RegisteredJarVisibilitySchema.java Wed Mar 7 23:43:26 2012
@@ -0,0 +1,9 @@
+package org.apache.pig.test;
+
+import java.io.Serializable;
+
+/**
+ * Please see {@link TestRegisteredJarVisibility} for information about this class.
+ */
+public class RegisteredJarVisibilitySchema implements Serializable {
+}