You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2012/11/19 13:00:31 UTC
svn commit: r1411143 - in /pig/trunk: CHANGES.txt build.xml
ivy/libraries.properties src/org/apache/pig/impl/util/JarManager.java
test/org/apache/pig/test/TestRegisteredJarVisibility.java
Author: rohini
Date: Mon Nov 19 12:00:29 2012
New Revision: 1411143
URL: http://svn.apache.org/viewvc?rev=1411143&view=rev
Log:
PIG-3039: Not possible to use custom version of jackson jars (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/build.xml
pig/trunk/ivy/libraries.properties
pig/trunk/src/org/apache/pig/impl/util/JarManager.java
pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1411143&r1=1411142&r2=1411143&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Nov 19 12:00:29 2012
@@ -364,6 +364,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3039: Not possible to use custom version of jackson jars (rohini)
+
PIG-3045: Specifying sorting field(s) at nightly.conf - fix sortArgs (rohini via cheolsoo)
PIG-2979: Pig.jar doesn't work with hadoop-2.0.x (cheolsoo)
Modified: pig/trunk/build.xml
URL: http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1411143&r1=1411142&r2=1411143&view=diff
==============================================================================
--- pig/trunk/build.xml (original)
+++ pig/trunk/build.xml Mon Nov 19 12:00:29 2012
@@ -197,6 +197,12 @@
<property name="build.ivy.maven.pom" location="${build.ivy.maven.dir}/pig-${version}.pom" />
<property name="build.ivy.maven.jar" location="${build.ivy.maven.dir}/pig-${version}-core.jar" />
<property name="javacc.home" location="${ivy.lib.dir}" />
+ <property name="jackson_core.jar" location="${test.src.dir}/resources/jackson-core-asl-${jackson-pig-3039-test.version}.jar"/>
+ <property name="jackson_mapper.jar" location="${test.src.dir}/resources/jackson-mapper-asl-${jackson-pig-3039-test.version}.jar"/>
+ <property name="jackson_core_repo_url"
+ value="${mvnrepo}/org/codehaus/jackson/jackson-core-asl/${jackson-pig-3039-test.version}/jackson-core-asl-${jackson-pig-3039-test.version}.jar"/>
+ <property name="jackson_mapper_repo_url"
+ value="${mvnrepo}/org/codehaus/jackson/jackson-mapper-asl/${jackson-pig-3039-test.version}/jackson-mapper-asl-${jackson-pig-3039-test.version}.jar"/>
<!--this is the naming policy for artifacts we want pulled down-->
<property name="ivy.artifact.retrieve.pattern" value="${ant.project.name}/[artifact]-[revision](-[classifier]).[ext]"/>
@@ -356,7 +362,7 @@
<ant dir="${test.e2e.dir}" target="clean"/>
</target>
- <target name="very-clean" unless="offline" depends="ivy-clean-cache,clean"
+ <target name="very-clean" unless="offline" depends="ivy-clean-cache,jackson-pig-3039-test-clean,clean"
description="Clean build artifacts and flush Ivy cache" />
<target name="clean-piggybank" description="Cleanup piggybank">
@@ -826,7 +832,7 @@
<!-- ================================================================== -->
<!-- Run unit tests -->
<!-- ================================================================== -->
- <target name="test-core" depends="compile-test,jar-withouthadoop,debugger.check" description="Run full set of unit tests">
+ <target name="test-core" depends="compile-test,jar-withouthadoop,debugger.check,jackson-pig-3039-test-download" description="Run full set of unit tests">
<macro-test-runner test.file="${test.all.file}" />
</target>
@@ -887,6 +893,7 @@
<excludesfile name="${test.exclude.file.23}" if="isHadoop23" />
</patternset>
<exclude name="**/${exclude.testcase}.java" if="exclude.testcase" />
+ <exclude name="**/TestRegisteredJarVisibility.java" if="offline"/>
<exclude name="**/TestInvokerSpeed.java" if="clover.enabled"/>
</fileset>
</batchtest>
@@ -1698,6 +1705,16 @@
<ivy:cleancache />
</target>
+ <target name="jackson-pig-3039-test-download" description="To download jackson" unless="offline">
+ <get src="${jackson_core_repo_url}" dest="${jackson_core.jar}" usetimestamp="true"/>
+ <get src="${jackson_mapper_repo_url}" dest="${jackson_mapper.jar}" usetimestamp="true"/>
+ </target>
+
+ <target name="jackson-pig-3039-test-clean" description="Clean the jackson jar files">
+ <delete file="${jackson_core.jar}"/>
+ <delete file="${jackson_mapper.jar}"/>
+ </target>
+
<target name="assert-pig-jar-exists" depends="ivy-init">
<fail>
<condition >
Modified: pig/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/trunk/ivy/libraries.properties?rev=1411143&r1=1411142&r2=1411143&view=diff
==============================================================================
--- pig/trunk/ivy/libraries.properties (original)
+++ pig/trunk/ivy/libraries.properties Mon Nov 19 12:00:29 2012
@@ -45,6 +45,7 @@ hsqldb.version=1.8.0.10
hive.version=0.8.0
httpcomponents.version=4.1
jackson.version=1.8.8
+jackson-pig-3039-test.version=1.9.9
javacc.version=4.2
javax-inject.version=1
jaxb-api.version=2.2.2
Modified: pig/trunk/src/org/apache/pig/impl/util/JarManager.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/JarManager.java?rev=1411143&r1=1411142&r2=1411143&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/JarManager.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/JarManager.java Mon Nov 19 12:00:29 2012
@@ -39,11 +39,19 @@ import java.util.jar.JarInputStream;
import java.util.jar.JarOutputStream;
import java.util.zip.ZipEntry;
+import org.antlr.runtime.CommonTokenStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
import org.apache.pig.impl.PigContext;
+import org.apache.tools.bzip2r.BZip2Constants;
+import org.codehaus.jackson.annotate.JsonPropertyOrder;
+import org.codehaus.jackson.map.annotate.JacksonStdImpl;
+import org.joda.time.DateTime;
+import com.google.common.collect.Multimaps;
+
+import dk.brics.automaton.Automaton;
public class JarManager {
@@ -84,10 +92,34 @@ public class JarManager {
}
}
- final static String pigPackagesToSend[] = { "org/apache/pig","org/apache/tools/bzip2r",
- "dk/brics/automaton", "org/antlr/runtime", "com/google/common", "org/codehaus/jackson",
- "org/joda/time"};
-
+ private static enum DefaultPigPackages {
+
+ PIG("org/apache/pig", PigMapReduce.class),
+ BZIP2R("org/apache/tools/bzip2r", BZip2Constants.class),
+ AUTOMATON("dk/brics/automaton", Automaton.class),
+ ANTLR("org/antlr/runtime", CommonTokenStream.class),
+ GUAVA("com/google/common", Multimaps.class),
+ JACKSON_CORE("org/codehaus/jackson", JsonPropertyOrder.class),
+ JACKSON_MAPPER("org/codehaus/jackson", JacksonStdImpl.class),
+ JODATIME("org/joda/time", DateTime.class);
+
+ private final String pkgPrefix;
+ private final Class pkgClass;
+
+ DefaultPigPackages(String pkgPrefix, Class pkgClass) {
+ this.pkgPrefix = pkgPrefix;
+ this.pkgClass = pkgClass;
+ }
+
+ public String getPkgPrefix() {
+ return pkgPrefix;
+ }
+
+ public Class getPkgClass() {
+ return pkgClass;
+ }
+ }
+
/**
* Create a jarfile in a temporary path, that is a merge of all the jarfiles containing the
* functions and the core pig classes.
@@ -101,10 +133,10 @@ public class JarManager {
@SuppressWarnings("deprecation")
public static void createJar(OutputStream os, Set<String> funcs, PigContext pigContext) throws ClassNotFoundException, IOException {
Vector<JarListEntry> jarList = new Vector<JarListEntry>();
- for(String toSend: pigPackagesToSend) {
- addContainingJar(jarList, PigMapReduce.class, toSend, pigContext);
+ for (DefaultPigPackages pkgToSend : DefaultPigPackages.values()) {
+ addContainingJar(jarList, pkgToSend.getPkgClass(), pkgToSend.getPkgPrefix(), pigContext);
}
-
+
for (String func: funcs) {
Class clazz = pigContext.getClassForAlias(func);
if (clazz != null) {
@@ -283,10 +315,19 @@ public class JarManager {
* @throws IOException
*/
public static String findContainingJar(Class my_class) {
- ClassLoader loader = my_class.getClassLoader();
+ ClassLoader loader = PigContext.getClassLoader();
String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
try {
- for (Enumeration itr = loader.getResources(class_file); itr.hasMoreElements();) {
+ Enumeration<URL> itr = null;
+ //Try to find the class in registered jars
+ if (loader instanceof URLClassLoader) {
+ itr = ((URLClassLoader) loader).findResources(class_file);
+ }
+ //Try system classloader if not URLClassLoader or no resources found in URLClassLoader
+ if (itr == null || !itr.hasMoreElements()) {
+ itr = loader.getResources(class_file);
+ }
+ for (; itr.hasMoreElements();) {
URL url = (URL) itr.nextElement();
if ("jar".equals(url.getProtocol())) {
String toReturn = url.getPath();
Modified: pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java?rev=1411143&r1=1411142&r2=1411143&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java Mon Nov 19 12:00:29 2012
@@ -24,8 +24,11 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
+import java.util.jar.JarFile;
import java.util.jar.JarOutputStream;
import java.util.zip.ZipEntry;
@@ -43,6 +46,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.Path;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
+import org.apache.pig.impl.util.JarManager;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -134,6 +138,33 @@ public class TestRegisteredJarVisibility
pigServer.shutdown();
}
+ // See PIG-3039
+ @Test
+ public void testRegisterJarOverridePigJarPackages() throws IOException, ClassNotFoundException {
+ // When jackson jar is not registered, jackson-core from the first jar in
+ // classpath (pig.jar) should be picked up (version 1.8.8 in this case).
+ PigServer pigServer = new PigServer(ExecType.LOCAL, new Properties());
+ File jobJarFile = Util.createTempFileDelOnExit("Job", ".jar");
+ FileOutputStream fos = new FileOutputStream(jobJarFile);
+ JarManager.createJar(fos, new HashSet<String>(), pigServer.getPigContext());
+ JarFile jobJar = new JarFile(jobJarFile);
+ // JsonClass present in 1.8.8 but not in 1.9.9
+ Assert.assertNotNull(jobJar.getJarEntry("org/codehaus/jackson/annotate/JsonClass.class"));
+ // JsonUnwrapped present in 1.9.9 but not in 1.8.8
+ Assert.assertNull(jobJar.getJarEntry("org/codehaus/jackson/annotate/JsonUnwrapped.class"));
+
+ // When jackson jar is registered, the registered version should be picked up.
+ pigServer = new PigServer(ExecType.LOCAL, new Properties());
+ pigServer.registerJar("test/resources/jackson-core-asl-1.9.9.jar");
+ pigServer.registerJar("test/resources/jackson-mapper-asl-1.9.9.jar");
+ jobJarFile = Util.createTempFileDelOnExit("Job", ".jar");
+ fos = new FileOutputStream(jobJarFile);
+ JarManager.createJar(fos, new HashSet<String>(), pigServer.getPigContext());
+ jobJar = new JarFile(jobJarFile);
+ Assert.assertNull(jobJar.getJarEntry("org/codehaus/jackson/annotate/JsonClass.class"));
+ Assert.assertNotNull(jobJar.getJarEntry("org/codehaus/jackson/annotate/JsonUnwrapped.class"));
+ }
+
private static List<File> compile(File[] javaFiles) {
LOG.info("Compiling: " + Arrays.asList(javaFiles));