You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/12/16 20:50:06 UTC
sqoop git commit: SQOOP-2635: Sqoop2: Improve ConnectorClassLoader to
support loading classes from the dependencies inside the connector jar
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 5329c1b33 -> 857432246
SQOOP-2635: Sqoop2: Improve ConnectorClassLoader to support loading classes from the dependencies inside the connector jar
(Dian Fu via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/85743224
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/85743224
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/85743224
Branch: refs/heads/sqoop2
Commit: 857432246790c2791e7cd3a10ee21d174dfa12f9
Parents: 5329c1b
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Wed Dec 16 20:48:54 2015 +0100
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Wed Dec 16 20:48:54 2015 +0100
----------------------------------------------------------------------
.../sqoop/classloader/ConnectorClassLoader.java | 657 +++++++++++++++++++
.../sqoop/classloader/ConnectorURLFactory.java | 74 +++
.../apache/sqoop/classloader/URLFactory.java | 29 +
.../sqoop/utils/ConnectorClassLoader.java | 257 --------
...pache.sqoop.connector-classloader.properties | 2 +
.../classloader/TestConnectorClassLoader.java | 311 +++++++++
.../org/apache/sqoop/utils/TestClassUtils.java | 1 +
.../sqoop/utils/TestConnectorClassLoader.java | 149 -----
.../resources/TestConnectorClassLoader/A.java | 20 +
.../TestConnectorClassLoader/lib/B.java | 20 +
10 files changed, 1114 insertions(+), 406 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/main/java/org/apache/sqoop/classloader/ConnectorClassLoader.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/classloader/ConnectorClassLoader.java b/common/src/main/java/org/apache/sqoop/classloader/ConnectorClassLoader.java
new file mode 100644
index 0000000..370de2a
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/classloader/ConnectorClassLoader.java
@@ -0,0 +1,657 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.classloader;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.security.CodeSource;
+import java.security.ProtectionDomain;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.jar.Attributes;
+import java.util.jar.Attributes.Name;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarInputStream;
+import java.util.jar.Manifest;
+
+import org.apache.log4j.Logger;
+
+import sun.misc.CompoundEnumeration;
+
+/**
+ * A {@link URLClassLoader} for connector isolation. Classes from the
+ * connector JARs are loaded in preference to the parent loader.
+ */
+public class ConnectorClassLoader extends URLClassLoader {
+ /**
+ * Default value of the system classes if the user did not override them.
+ * JDK classes, sqoop classes and resources, and some select third-party
+ * classes are considered system classes, and are not loaded by the
+ * connector classloader.
+ */
+ public static final String SYSTEM_CLASSES_DEFAULT;
+
+ private static final String PROPERTIES_FILE =
+ "org.apache.sqoop.connector-classloader.properties";
+ private static final String SYSTEM_CLASSES_DEFAULT_KEY =
+ "system.classes.default";
+
+ private static final String MANIFEST = "META-INF/MANIFEST.MF";
+ private static final String CLASS = ".class";
+ private static final String LIB_PREFIX = "lib/";
+
+ private Map<String, ByteCode> byteCodeCache = new HashMap<String, ByteCode>();
+ private Set<String> jarNames = new LinkedHashSet<String>();
+ private Map<String, ProtectionDomain> pdCache = new HashMap<String, ProtectionDomain>();
+ private URLFactory urlFactory;
+
+ private static final Logger LOG = Logger.getLogger(ConnectorClassLoader.class);
+
+ private static final FilenameFilter JAR_FILENAME_FILTER =
+ new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.endsWith(".jar") || name.endsWith(".JAR");
+ }
+ };
+
+ private static class ByteCode {
+ public byte[] bytes;
+ public String codebase;
+ public Manifest manifest;
+
+ public ByteCode(byte[] bytes,
+ String codebase, Manifest manifest) {
+ this.bytes = bytes;
+ this.codebase = codebase;
+ this.manifest = manifest;
+ }
+ }
+
+ static {
+ try (InputStream is = ConnectorClassLoader.class.getClassLoader()
+ .getResourceAsStream(PROPERTIES_FILE);) {
+ if (is == null) {
+ throw new ExceptionInInitializerError("properties file " +
+ PROPERTIES_FILE + " is not found");
+ }
+ Properties props = new Properties();
+ props.load(is);
+ // get the system classes default
+ String systemClassesDefault =
+ props.getProperty(SYSTEM_CLASSES_DEFAULT_KEY);
+ if (systemClassesDefault == null) {
+ throw new ExceptionInInitializerError("property " +
+ SYSTEM_CLASSES_DEFAULT_KEY + " is not found");
+ }
+ SYSTEM_CLASSES_DEFAULT = systemClassesDefault;
+ } catch (IOException e) {
+ throw new ExceptionInInitializerError(e);
+ }
+ }
+
+ private final ClassLoader parent;
+ private final List<String> systemClasses;
+
+ public ConnectorClassLoader(URL[] urls, ClassLoader parent,
+ List<String> systemClasses, boolean overrideDefaultSystemClasses) throws IOException {
+ super(urls, parent);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("urls: " + Arrays.toString(urls));
+ LOG.debug("system classes: " + systemClasses);
+ }
+ this.parent = parent;
+ if (parent == null) {
+ throw new IllegalArgumentException("No parent classloader!");
+ }
+ // if the caller-specified system classes are null or empty, use the default
+ this.systemClasses = new ArrayList<String>();
+ if (systemClasses != null && !systemClasses.isEmpty()) {
+ this.systemClasses.addAll(systemClasses);
+ }
+ if (!overrideDefaultSystemClasses || this.systemClasses.isEmpty()) {
+ this.systemClasses.addAll(Arrays.asList(SYSTEM_CLASSES_DEFAULT.split("\\s*,\\s*")));
+ }
+ LOG.info("system classes: " + this.systemClasses);
+
+ urlFactory = new ConnectorURLFactory(this);
+ load(urls);
+ }
+
+ public ConnectorClassLoader(String classpath, ClassLoader parent,
+ List<String> systemClasses) throws IOException {
+ this(constructUrlsFromClasspath(classpath), parent, systemClasses, true);
+ }
+
+ public ConnectorClassLoader(String classpath, ClassLoader parent,
+ List<String> systemClasses, boolean overrideDefaultSystemClasses) throws IOException {
+ this(constructUrlsFromClasspath(classpath), parent, systemClasses, overrideDefaultSystemClasses);
+ }
+
+ static URL[] constructUrlsFromClasspath(String classpath)
+ throws MalformedURLException {
+ List<URL> urls = new ArrayList<URL>();
+ for (String element : classpath.split(File.pathSeparator)) {
+ if (element.endsWith("/*")) {
+ String dir = element.substring(0, element.length() - 1);
+ File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
+ if (files != null) {
+ for (File file : files) {
+ urls.add(file.toURI().toURL());
+ }
+ }
+ } else {
+ File file = new File(element);
+ if (file.exists()) {
+ urls.add(new File(element).toURI().toURL());
+ }
+ }
+ }
+ return urls.toArray(new URL[urls.size()]);
+ }
+
+ @Override
+ public URL getResource(String name) {
+ URL url = null;
+
+ if (!isSystemClass(name, systemClasses)) {
+ url= findResource(name);
+ if (url == null && name.startsWith("/")) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Remove leading / off " + name);
+ }
+ url= findResource(name.substring(1));
+ }
+ }
+
+ if (url == null) {
+ url= parent.getResource(name);
+ }
+
+ if (url != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getResource("+name+")=" + url);
+ }
+ }
+
+ return url;
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public Enumeration<URL> getResources(String name) throws IOException {
+ Enumeration[] tmp = new Enumeration[2];
+
+ if (!isSystemClass(name, systemClasses)) {
+ tmp[0]= findResources(name);
+ if (!tmp[0].hasMoreElements() && name.startsWith("/")) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Remove leading / off " + name);
+ }
+ tmp[0]= findResources(name.substring(1));
+ }
+ }
+
+ tmp[1]= parent.getResources(name);
+
+ return new CompoundEnumeration<>(tmp);
+ }
+
+ @Override
+ public Class<?> loadClass(String name) throws ClassNotFoundException {
+ return this.loadClass(name, false);
+ }
+
+ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve)
+ throws ClassNotFoundException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loading class: " + name);
+ }
+
+ Class<?> c = findLoadedClass(name);
+ ClassNotFoundException ex = null;
+
+ if (c == null && !isSystemClass(name, systemClasses)) {
+ // Try to load class from this classloader's URLs. Note that this is like
+ // the servlet spec, not the usual Java 2 behaviour where we ask the
+ // parent to attempt to load first.
+ try {
+ c = findClass(name);
+ if (LOG.isDebugEnabled() && c != null) {
+ LOG.debug("Loaded class: " + name + " ");
+ }
+ } catch (ClassNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(e);
+ }
+ ex = e;
+ }
+ }
+
+ if (c == null) { // try parent
+ c = parent.loadClass(name);
+ if (LOG.isDebugEnabled() && c != null) {
+ LOG.debug("Loaded class from parent: " + name + " ");
+ }
+ }
+
+ if (c == null) {
+ throw ex != null ? ex : new ClassNotFoundException(name);
+ }
+
+ if (resolve) {
+ resolveClass(c);
+ }
+
+ return c;
+ }
+
+ @Override
+ protected Class<?> findClass(String name) throws ClassNotFoundException {
+ // Make sure not to load duplicate classes.
+ Class<?> cls = findLoadedClass(name);
+ if (cls != null) {
+ return cls;
+ }
+
+ // Look up the class in the byte codes.
+ String cache = name.replace('.', '/') + CLASS;
+ cache = resolve(cache);
+ if (cache != null) {
+ ByteCode bytecode = byteCodeCache.get(cache);
+ // Use a protectionDomain to associate the codebase with the class.
+ ProtectionDomain pd = pdCache.get(bytecode.codebase);
+ if (pd == null) {
+ try {
+ URL url = urlFactory.getCodeBase(bytecode.codebase);
+ CodeSource source = new CodeSource(url, (java.security.cert.Certificate[]) null);
+ pd = new ProtectionDomain(source, null, this, null);
+ pdCache.put(bytecode.codebase, pd);
+ } catch (MalformedURLException mux) {
+ throw new ClassNotFoundException(name, mux);
+ }
+ }
+
+ byte bytes[] = bytecode.bytes;
+ int i = name.lastIndexOf('.');
+ if (i != -1) {
+ String pkgname = name.substring(0, i);
+ // Check if package already loaded.
+ Package pkg = getPackage(pkgname);
+ Manifest man = bytecode.manifest;
+ if (pkg != null) {
+ // Package found, so check package sealing.
+ if (pkg.isSealed()) {
+ // Verify that code source URL is the same.
+ if (!pkg.isSealed(pd.getCodeSource().getLocation())) {
+ throw new SecurityException("sealing violation: package " + pkgname + " is sealed");
+ }
+ } else {
+ // Make sure we are not attempting to seal the package at this code source URL.
+ if ((man != null) && isSealed(pkgname, man)) {
+ throw new SecurityException("sealing violation: can't seal package " + pkgname + ": already loaded");
+ }
+ }
+ } else {
+ if (man != null) {
+ definePackage(pkgname, man, pd.getCodeSource().getLocation());
+ } else {
+ definePackage(pkgname, null, null, null, null, null, null, null);
+ }
+ }
+ }
+
+ return defineClass(name, bytes, 0, bytes.length, pd);
+ }
+
+ return null;
+ }
+
+ @Override
+ public URL findResource(final String name) {
+ LOG.debug("Finding resource: " + name);
+
+ try {
+ // Do we have the named resource in our cache? If so, construct a 'sqoopconnector:' URL.
+ String resource = resolve(name);
+ if (resource != null) {
+ ByteCode entry = byteCodeCache.get(resource);
+ return urlFactory.getURL(entry.codebase, name);
+ }
+ } catch (MalformedURLException mux) {
+ LOG.debug("Unable to find resource: " + name + " due to " + mux);
+ }
+
+ return null;
+ }
+
+ @Override
+ @edu.umd.cs.findbugs.annotations.SuppressWarnings("SIC_INNER_SHOULD_BE_STATIC_ANON")
+ public Enumeration<URL> findResources(final String name) throws IOException {
+ LOG.debug("Finding resources: " + name);
+
+ final List<URL> resources = new ArrayList<URL>();
+ ByteCode entry = byteCodeCache.get(name);
+ if (entry != null) {
+ URL url = urlFactory.getURL(entry.codebase, name);
+ LOG.debug("Adding " + url + " to resources list for " + name);
+ resources.add(url);
+ }
+
+ Iterator<String> jarNameIter = jarNames.iterator();
+ while (jarNameIter.hasNext()) {
+ String resource = jarNameIter.next() + "/" + name;
+ entry = byteCodeCache.get(resource);
+ if (entry != null) {
+ URL url = urlFactory.getURL(entry.codebase, name);
+ LOG.debug("Adding " + url + " to resources list for " + name);
+ resources.add(url);
+ }
+ }
+
+ final Iterator<URL> resIter = resources.iterator();
+ return new Enumeration<URL>() {
+ public boolean hasMoreElements() {
+ return resIter.hasNext();
+ }
+ public URL nextElement() {
+ return resIter.next();
+ }
+ };
+ }
+
+ /**
+ * Return resources from the appropriate codebase.
+ */
+ public InputStream getByteStream(String resource) {
+ InputStream result = null;
+
+ if (!isSystemClass(resource, systemClasses)) {
+ // Make resource canonical (remove ., .., etc).
+ resource = canon(resource);
+
+ ByteCode bytecode = null;
+ String name = resolve(resource);
+ if (name != null) {
+ bytecode = byteCodeCache.get(name);
+ }
+
+ if (bytecode != null) {
+ result = new ByteArrayInputStream(bytecode.bytes);
+ }
+
+ if (result == null) {
+ if (jarNames.contains(resource)) {
+ // resource wanted is an actual jar
+ LOG.debug("Loading resource file directly: " + resource);
+ result = super.getResourceAsStream(resource);
+ }
+ }
+ }
+
+ if (result == null) {
+ // Delegate to parent.
+ result = parent.getResourceAsStream(resource);
+ }
+
+ return result;
+ }
+
+ private boolean isSealed(String name, Manifest man) {
+ String path = name.concat("/");
+ Attributes attr = man.getAttributes(path);
+ String sealed = null;
+ if (attr != null) {
+ sealed = attr.getValue(Name.SEALED);
+ }
+ if (sealed == null) {
+ if ((attr = man.getMainAttributes()) != null) {
+ sealed = attr.getValue(Name.SEALED);
+ }
+ }
+ return "true".equalsIgnoreCase(sealed);
+ }
+
+ /**
+ * Make a path canonical, removing . and ..
+ */
+ private String canon(String path) {
+ path = path.replaceAll("/\\./", "/");
+ String canon = path;
+ String next;
+ do {
+ next = canon;
+ canon = canon.replaceFirst("([^/]*/\\.\\./)", "");
+ } while (!next.equals(canon));
+ return canon;
+ }
+
+ /**
+ * Resolve a resource name.
+ */
+ private String resolve(String name) {
+ if (name.startsWith("/")) {
+ name = name.substring(1);
+ }
+
+ String resource = null;
+ if (byteCodeCache.containsKey(name)) {
+ resource = name;
+ }
+
+ if (resource == null) {
+ Iterator<String> jarNameIter = jarNames.iterator();
+ while (jarNameIter.hasNext()) {
+ String tmp = jarNameIter.next() + "/" + name;
+ if (byteCodeCache.containsKey(tmp)) {
+ resource = tmp;
+ break;
+ }
+ }
+ }
+ return resource;
+ }
+
+ private void load(URL[] urls) throws IOException {
+ for (URL url : urls) {
+ String jarName = url.getPath();
+ JarFile jarFile = null;
+ try {
+ jarFile = new JarFile(jarName);
+ Manifest manifest = jarFile.getManifest();
+
+ Enumeration<JarEntry> entryEnum = jarFile.entries();
+ while (entryEnum.hasMoreElements()) {
+ JarEntry entry = entryEnum.nextElement();
+ if (entry.isDirectory()) {
+ continue;
+ }
+
+ String entryName = entry.getName();
+ InputStream is = jarFile.getInputStream(entry);
+ if (is == null) {
+ throw new IOException("Unable to load resource " + entryName);
+ }
+ try {
+ if (entryName.startsWith(LIB_PREFIX)) {
+ LOG.debug("Caching " + entryName);
+ loadBytesFromJar(is, entryName);
+ } else if (entryName.endsWith(CLASS)) {
+ // A plain vanilla class file rooted at the top of the jar file.
+ loadBytes(entry, is, "/", manifest);
+ LOG.debug("Loaded class: " + jarFile.getName() + "!/" + entry.getName());
+ } else {
+ // A resource
+ loadBytes(entry, is, "/", manifest);
+ LOG.debug("Loaded resource: " + jarFile.getName() + "!/" + entry.getName());
+ }
+ } finally {
+ is.close();
+ }
+ }
+ } finally {
+ if (jarFile != null) {
+ try {
+ jarFile.close();
+ } catch (IOException e) {
+ LOG.debug("Exception closing jarFile: " + jarName, e);
+ }
+ }
+ }
+ }
+ }
+
+ private void loadBytesFromJar(InputStream is, String jar) throws IOException {
+ JarInputStream jis = new JarInputStream(is);
+ Manifest manifest = jis.getManifest();
+ JarEntry entry = null;
+ while ((entry = jis.getNextJarEntry()) != null) {
+ loadBytes(entry, jis, jar, manifest);
+ }
+
+ if (manifest != null) {
+ entry = new JarEntry(MANIFEST);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ manifest.write(baos);
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ loadBytes(entry, bais, jar, manifest);
+ }
+ }
+
+ private void loadBytes(JarEntry entry, InputStream is, String jar, Manifest man) throws IOException {
+ String entryName = entry.getName();
+ int index = entryName.lastIndexOf('.');
+
+ // Add package handling to avoid NullPointerException
+ // after calls to getPackage method of this ClassLoader
+ int index2 = entryName.lastIndexOf('/', index - 1);
+ if (entryName.endsWith(CLASS) && index2 > -1) {
+ String packageName = entryName.substring(0, index2).replace('/', '.');
+ if (getPackage(packageName) == null) {
+ if (man != null) {
+ definePackage(packageName, man, urlFactory.getCodeBase(jar));
+ } else {
+ definePackage(packageName, null, null, null, null, null, null, null);
+ }
+ }
+ }
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ copy(is, baos);
+
+ // If entry is a class, check to see that it hasn't been defined
+ // already. Class names must be unique within a classloader because
+ // they are cached inside the VM until the classloader is released.
+ if (entryName.endsWith(CLASS)) {
+ if (byteCodeCache.containsKey(entryName)) {
+ return;
+ }
+ byteCodeCache.put(entryName, new ByteCode(baos.toByteArray(), jar, man));
+ } else {
+ // Another kind of resource. Cache this by name and prefixed by the jar name.
+ String localname = jar + "/" + entryName;
+ byte[] bytes = baos.toByteArray();
+ byteCodeCache.put(localname, new ByteCode(bytes, jar, man));
+ // Keep a set of jar names.
+ jarNames.add(jar);
+ }
+ }
+
+ /**
+ * Copying InputStream to OutputStream. Both streams are left open after copy.
+ * @param in Source of bytes to copy.
+ * @param out Destination of bytes to copy.
+ * @throws IOException
+ */
+ private static void copy(InputStream in, OutputStream out) throws IOException {
+ byte[] buf = new byte[1024];
+ while (true) {
+ int len = in.read(buf);
+ if (len < 0) {
+ break;
+ }
+ out.write(buf, 0, len);
+ }
+ }
+
+ /**
+ * Checks if a class should be included as a system class.
+ *
+ * A class is a system class if and only if the longest match pattern is positive.
+ *
+ * @param name the class name to check
+ * @param systemClasses a list of system class configurations.
+ * @return true if the class is a system class
+ */
+ public static boolean isSystemClass(String name, List<String> systemClasses) {
+ boolean result = false;
+ if (systemClasses != null) {
+ String canonicalName = name.replace('/', '.');
+ while (canonicalName.startsWith(".")) {
+ canonicalName=canonicalName.substring(1);
+ }
+ int maxMatchLength = -1;
+ for (String c : systemClasses) {
+ boolean shouldInclude = true;
+ if (c.startsWith("-")) {
+ c = c.substring(1);
+ shouldInclude = false;
+ }
+ if (canonicalName.startsWith(c)) {
+ if ( c.endsWith(".") // package
+ || canonicalName.length() == c.length() // class
+ || canonicalName.length() > c.length() // nested
+ && canonicalName.charAt(c.length()) == '$' ) {
+ if (c.length() > maxMatchLength) {
+ maxMatchLength = c.length();
+
+ if (shouldInclude) {
+ result = true;
+ } else {
+ result = false;
+ }
+ } else if (c.length() == maxMatchLength) {
+ if (!shouldInclude) {
+ result = false;
+ }
+ }
+ }
+ }
+ }
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/main/java/org/apache/sqoop/classloader/ConnectorURLFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/classloader/ConnectorURLFactory.java b/common/src/main/java/org/apache/sqoop/classloader/ConnectorURLFactory.java
new file mode 100644
index 0000000..9ef58f8
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/classloader/ConnectorURLFactory.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.classloader;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.FileNameMap;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.net.URLStreamHandler;
+
+/**
+ * Generates URLs which are efficient, using the in-memory bytecode to access the resources.
+ */
+@edu.umd.cs.findbugs.annotations.SuppressWarnings("SIC_INNER_SHOULD_BE_STATIC_ANON")
+public class ConnectorURLFactory implements URLFactory {
+ private URLStreamHandler handler;
+
+ public ConnectorURLFactory(final ConnectorClassLoader loader) {
+ handler = new URLStreamHandler() {
+ @Override
+ protected URLConnection openConnection(final URL u) throws IOException {
+ final String resource = u.getPath();
+ return new URLConnection(u) {
+ public void connect() {
+ }
+
+ public String getContentType() {
+ FileNameMap fileNameMap = java.net.URLConnection.getFileNameMap();
+ String contentType = fileNameMap.getContentTypeFor(resource);
+ if (contentType == null) {
+ contentType = "text/plain";
+ }
+ return contentType;
+ }
+
+ public InputStream getInputStream() throws IOException {
+ InputStream is = loader.getByteStream(resource);
+ if (is == null) {
+ throw new IOException("loader.getByteStream() returned null for " + resource);
+ }
+ return is;
+ }
+ };
+ }
+ };
+ }
+
+ public URL getURL(String codebase, String resource) throws MalformedURLException {
+ String base = resource.endsWith(".class") ? "": codebase + "/";
+ URL url = new URL(null, PROTOCOL + ":/" + base + resource, handler);
+ return url;
+ }
+
+ public URL getCodeBase(String jar) throws MalformedURLException {
+ return new URL(null, PROTOCOL + ":" + jar, handler);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/main/java/org/apache/sqoop/classloader/URLFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/classloader/URLFactory.java b/common/src/main/java/org/apache/sqoop/classloader/URLFactory.java
new file mode 100644
index 0000000..b43df45
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/classloader/URLFactory.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.classloader;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public interface URLFactory {
+
+ public static final String PROTOCOL = "sqoopconnector";
+
+ public URL getURL(String codebase, String resource) throws MalformedURLException;
+ public URL getCodeBase(String jar) throws MalformedURLException;
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java b/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java
deleted file mode 100644
index 4c42a78..0000000
--- a/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.utils;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.log4j.Logger;
-
-/**
- * A {@link URLClassLoader} for connector isolation. Classes from the
- * connector JARs are loaded in preference to the parent loader.
- */
-public class ConnectorClassLoader extends URLClassLoader {
- /**
- * Default value of the system classes if the user did not override them.
- * JDK classes, sqoop classes and resources, and some select third-party
- * classes are considered system classes, and are not loaded by the
- * connector classloader.
- */
- public static final String SYSTEM_CLASSES_DEFAULT;
-
- private static final String PROPERTIES_FILE =
- "org.apache.sqoop.connector-classloader.properties";
- private static final String SYSTEM_CLASSES_DEFAULT_KEY =
- "system.classes.default";
-
- private static final Logger LOG = Logger.getLogger(ConnectorClassLoader.class);
-
- private static final FilenameFilter JAR_FILENAME_FILTER =
- new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.endsWith(".jar") || name.endsWith(".JAR");
- }
- };
-
- static {
- try (InputStream is = ConnectorClassLoader.class.getClassLoader()
- .getResourceAsStream(PROPERTIES_FILE);) {
- if (is == null) {
- throw new ExceptionInInitializerError("properties file " +
- PROPERTIES_FILE + " is not found");
- }
- Properties props = new Properties();
- props.load(is);
- // get the system classes default
- String systemClassesDefault =
- props.getProperty(SYSTEM_CLASSES_DEFAULT_KEY);
- if (systemClassesDefault == null) {
- throw new ExceptionInInitializerError("property " +
- SYSTEM_CLASSES_DEFAULT_KEY + " is not found");
- }
- SYSTEM_CLASSES_DEFAULT = systemClassesDefault;
- } catch (IOException e) {
- throw new ExceptionInInitializerError(e);
- }
- }
-
- private final ClassLoader parent;
- private final List<String> systemClasses;
-
- public ConnectorClassLoader(URL[] urls, ClassLoader parent,
- List<String> systemClasses, boolean overrideDefaultSystemClasses) {
- super(urls, parent);
- if (LOG.isDebugEnabled()) {
- LOG.debug("urls: " + Arrays.toString(urls));
- LOG.debug("system classes: " + systemClasses);
- }
- this.parent = parent;
- if (parent == null) {
- throw new IllegalArgumentException("No parent classloader!");
- }
- // if the caller-specified system classes are null or empty, use the default
- this.systemClasses = new ArrayList<String>();
- if (systemClasses != null && !systemClasses.isEmpty()) {
- this.systemClasses.addAll(systemClasses);
- }
- if (!overrideDefaultSystemClasses || this.systemClasses.isEmpty()) {
- this.systemClasses.addAll(Arrays.asList(SYSTEM_CLASSES_DEFAULT.split("\\s*,\\s*")));
- }
- LOG.info("system classes: " + this.systemClasses);
- }
-
- public ConnectorClassLoader(String classpath, ClassLoader parent,
- List<String> systemClasses) throws MalformedURLException {
- this(constructUrlsFromClasspath(classpath), parent, systemClasses, true);
- }
-
- public ConnectorClassLoader(String classpath, ClassLoader parent,
- List<String> systemClasses, boolean overrideDefaultSystemClasses) throws MalformedURLException {
- this(constructUrlsFromClasspath(classpath), parent, systemClasses, overrideDefaultSystemClasses);
- }
-
- static URL[] constructUrlsFromClasspath(String classpath)
- throws MalformedURLException {
- List<URL> urls = new ArrayList<URL>();
- for (String element : classpath.split(File.pathSeparator)) {
- if (element.endsWith("/*")) {
- String dir = element.substring(0, element.length() - 1);
- File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
- if (files != null) {
- for (File file : files) {
- urls.add(file.toURI().toURL());
- }
- }
- } else {
- File file = new File(element);
- if (file.exists()) {
- urls.add(new File(element).toURI().toURL());
- }
- }
- }
- return urls.toArray(new URL[urls.size()]);
- }
-
- @Override
- public URL getResource(String name) {
- URL url = null;
-
- if (!isSystemClass(name, systemClasses)) {
- url= findResource(name);
- if (url == null && name.startsWith("/")) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Remove leading / off " + name);
- }
- url= findResource(name.substring(1));
- }
- }
-
- if (url == null) {
- url= parent.getResource(name);
- }
-
- if (url != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("getResource("+name+")=" + url);
- }
- }
-
- return url;
- }
-
- @Override
- public Class<?> loadClass(String name) throws ClassNotFoundException {
- return this.loadClass(name, false);
- }
-
- @Override
- protected synchronized Class<?> loadClass(String name, boolean resolve)
- throws ClassNotFoundException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Loading class: " + name);
- }
-
- Class<?> c = findLoadedClass(name);
- ClassNotFoundException ex = null;
-
- if (c == null && !isSystemClass(name, systemClasses)) {
- // Try to load class from this classloader's URLs. Note that this is like
- // the servlet spec, not the usual Java 2 behaviour where we ask the
- // parent to attempt to load first.
- try {
- c = findClass(name);
- if (LOG.isDebugEnabled() && c != null) {
- LOG.debug("Loaded class: " + name + " ");
- }
- } catch (ClassNotFoundException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(e);
- }
- ex = e;
- }
- }
-
- if (c == null) { // try parent
- c = parent.loadClass(name);
- if (LOG.isDebugEnabled() && c != null) {
- LOG.debug("Loaded class from parent: " + name + " ");
- }
- }
-
- if (c == null) {
- throw ex != null ? ex : new ClassNotFoundException(name);
- }
-
- if (resolve) {
- resolveClass(c);
- }
-
- return c;
- }
-
- /**
- * Checks if a class should be included as a system class.
- *
- * A class is a system class if and only if it matches one of the positive
- * patterns and none of the negative ones.
- *
- * @param name the class name to check
- * @param systemClasses a list of system class configurations.
- * @return true if the class is a system class
- */
- public static boolean isSystemClass(String name, List<String> systemClasses) {
- boolean result = false;
- if (systemClasses != null) {
- String canonicalName = name.replace('/', '.');
- while (canonicalName.startsWith(".")) {
- canonicalName=canonicalName.substring(1);
- }
- for (String c : systemClasses) {
- boolean shouldInclude = true;
- if (c.startsWith("-")) {
- c = c.substring(1);
- shouldInclude = false;
- }
- if (canonicalName.startsWith(c)) {
- if ( c.endsWith(".") // package
- || canonicalName.length() == c.length() // class
- || canonicalName.length() > c.length() // nested
- && canonicalName.charAt(c.length()) == '$' ) {
- if (shouldInclude) {
- result = true;
- } else {
- return false;
- }
- }
- }
- }
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/main/resources/org.apache.sqoop.connector-classloader.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/org.apache.sqoop.connector-classloader.properties b/common/src/main/resources/org.apache.sqoop.connector-classloader.properties
index e2936a9..c0082cc 100644
--- a/common/src/main/resources/org.apache.sqoop.connector-classloader.properties
+++ b/common/src/main/resources/org.apache.sqoop.connector-classloader.properties
@@ -51,5 +51,7 @@ system.classes.default=java.,\
org.apache.commons.logging.,\
org.apache.log4j.,\
org.apache.sqoop.,\
+ -org.apache.sqoop.connector.,\
+ org.xerial.snappy.,\
sqoop.properties,\
sqoop_bootstrap.properties
http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/test/java/org/apache/sqoop/classloader/TestConnectorClassLoader.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/classloader/TestConnectorClassLoader.java b/common/src/test/java/org/apache/sqoop/classloader/TestConnectorClassLoader.java
new file mode 100644
index 0000000..87edf3d
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/classloader/TestConnectorClassLoader.java
@@ -0,0 +1,311 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.classloader;
+
+import static org.apache.sqoop.classloader.ConnectorClassLoader.constructUrlsFromClasspath;
+import static org.apache.sqoop.classloader.ConnectorClassLoader.isSystemClass;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.StandardLocation;
+import javax.tools.ToolProvider;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.sqoop.classloader.ConnectorClassLoader;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
+public class TestConnectorClassLoader {
+ private static File testDir = new File(System.getProperty("maven.build.directory",
+ System.getProperty("java.io.tmpdir")), "connectorclassloader");
+
+ @BeforeMethod(alwaysRun = true)
+ public void setUp() {
+ FileUtils.deleteQuietly(testDir);
+ testDir.mkdirs();
+ }
+
+ @Test
+ public void testConstructUrlsFromClasspath() throws Exception {
+ File file = new File(testDir, "file");
+ assertTrue(file.createNewFile(), "Create file");
+
+ File dir = new File(testDir, "dir");
+ assertTrue(dir.mkdir(), "Make dir");
+
+ File jarsDir = new File(testDir, "jarsdir");
+ assertTrue(jarsDir.mkdir(), "Make jarsDir");
+ File nonJarFile = new File(jarsDir, "nonjar");
+ assertTrue(nonJarFile.createNewFile(), "Create non-jar file");
+ File jarFile = new File(jarsDir, "a.jar");
+ assertTrue(jarFile.createNewFile(), "Create jar file");
+
+ File nofile = new File(testDir, "nofile");
+ // don't create nofile
+
+ StringBuilder cp = new StringBuilder();
+ cp.append(file.getAbsolutePath()).append(File.pathSeparator)
+ .append(dir.getAbsolutePath()).append(File.pathSeparator)
+ .append(jarsDir.getAbsolutePath() + "/*").append(File.pathSeparator)
+ .append(nofile.getAbsolutePath()).append(File.pathSeparator)
+ .append(nofile.getAbsolutePath() + "/*").append(File.pathSeparator);
+
+ URL[] urls = constructUrlsFromClasspath(cp.toString());
+
+ assertEquals(3, urls.length);
+ assertEquals(file.toURI().toURL(), urls[0]);
+ assertEquals(dir.toURI().toURL(), urls[1]);
+ assertEquals(jarFile.toURI().toURL(), urls[2]);
+ // nofile should be ignored
+ }
+
+ @Test
+ public void testIsSystemClass() {
+ testIsSystemClassInternal("");
+ }
+
+ @Test
+ public void testIsSystemNestedClass() {
+ testIsSystemClassInternal("$Klass");
+ }
+
+ private void testIsSystemClassInternal(String nestedClass) {
+ assertFalse(isSystemClass("org.example.Foo" + nestedClass, null));
+ assertTrue(isSystemClass("org.example.Foo" + nestedClass,
+ classes("org.example.Foo")));
+ assertTrue(isSystemClass("/org.example.Foo" + nestedClass,
+ classes("org.example.Foo")));
+ assertTrue(isSystemClass("org.example.Foo" + nestedClass,
+ classes("org.example.")));
+ assertTrue(isSystemClass("net.example.Foo" + nestedClass,
+ classes("org.example.,net.example.")));
+ assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+ classes("-org.example.Foo,org.example.")));
+ assertTrue(isSystemClass("org.example.Bar" + nestedClass,
+ classes("-org.example.Foo.,org.example.")));
+ assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+ classes("org.example.,-org.example.Foo")));
+ assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+ classes("org.example.Foo,-org.example.Foo")));
+ }
+
+ private List<String> classes(String classes) {
+ return Lists.newArrayList(Splitter.on(',').split(classes));
+ }
+
+ @Test
+ public void testGetResource() throws IOException {
+ URL testJar = makeTestJar().toURI().toURL();
+
+ ClassLoader currentClassLoader = getClass().getClassLoader();
+ ClassLoader connectorClassloader = new ConnectorClassLoader(
+ new URL[] { testJar }, currentClassLoader, null, false);
+
+ assertNull(currentClassLoader.getResourceAsStream("resource.txt"),
+ "Resource should be null for current classloader");
+ assertNull(currentClassLoader.getResourceAsStream("resource-dep.txt"),
+ "Resource should be null for current classloader");
+
+ InputStream in = connectorClassloader.getResourceAsStream("resource.txt");
+ assertNotNull(in, "Resource should not be null for connector classloader");
+
+ in = connectorClassloader.getResourceAsStream("resource-dep.txt");
+ assertNotNull(in, "Resource should not be null for connector classloader");
+ assertEquals(IOUtils.toString(in), "hello dep");
+ }
+
+ @Test
+ public void testGetResources() throws IOException {
+ URL testJar = makeTestJar().toURI().toURL();
+
+ ClassLoader currentClassLoader = getClass().getClassLoader();
+ ClassLoader connectorClassloader = new ConnectorClassLoader(
+ new URL[] { testJar }, currentClassLoader, null, false);
+
+ List<String> resourceContents = new ArrayList<String>();
+ resourceContents.add("hello A");
+ resourceContents.add("hello B");
+ Enumeration<URL> urlEnum = connectorClassloader.getResources("resource.txt");
+ assertTrue(urlEnum.hasMoreElements());
+ resourceContents.remove(IOUtils.toString(urlEnum.nextElement().openStream()));
+
+ assertTrue(urlEnum.hasMoreElements());
+ resourceContents.remove(IOUtils.toString(urlEnum.nextElement().openStream()));
+
+ assertEquals(resourceContents.size(), 0);
+ }
+
+ @Test
+ public void testLoadClass() throws Exception {
+ URL testJar = makeTestJar().toURI().toURL();
+
+ ClassLoader currentClassLoader = getClass().getClassLoader();
+ ClassLoader connectorClassloader = new ConnectorClassLoader(
+ new URL[] { testJar }, currentClassLoader, null, false);
+
+ try {
+ currentClassLoader.loadClass("A");
+ fail("Should throw ClassNotFoundException when loading class A");
+ } catch (ClassNotFoundException e) {
+ // Expected
+ }
+ try {
+ currentClassLoader.loadClass("B");
+ fail("Should throw ClassNotFoundException when loading class B");
+ } catch (ClassNotFoundException e) {
+ // Expected
+ }
+
+ assertNotNull(connectorClassloader.loadClass("A"));
+ assertNotNull(connectorClassloader.loadClass("B"));
+ }
+
+ private File makeTestJar() throws IOException{
+ File jarFile = new File(testDir, "test.jar");
+
+ Manifest manifest = new Manifest();
+ manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+ manifest.getMainAttributes().put(Attributes.Name.CLASS_PATH, ".");
+
+ // Create test.jar
+ JarOutputStream jos = new JarOutputStream(new FileOutputStream(jarFile), manifest);
+ List<String> classFiles = new ArrayList<String>();
+ classFiles.add("TestConnectorClassLoader/A.java");
+ addFilesToJar(classFiles, jos);
+ JarEntry entry = new JarEntry("resource.txt");
+ addEntry(new ByteArrayInputStream("hello A".getBytes()), jos, entry);
+
+ // Create lib/test-dep.jar
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ JarOutputStream depJos = new JarOutputStream(baos, manifest);
+ classFiles.clear();
+ classFiles.add("TestConnectorClassLoader/lib/B.java");
+ addFilesToJar(classFiles, depJos);
+ entry = new JarEntry("resource.txt");
+ addEntry(new ByteArrayInputStream("hello B".getBytes()), depJos, entry);
+ entry = new JarEntry("resource-dep.txt");
+ addEntry(new ByteArrayInputStream("hello dep".getBytes()), depJos, entry);
+ depJos.close();
+
+ // Add lib/test-dep.jar to test.jar
+ entry = new JarEntry("lib/test-dep.jar");
+ addEntry(new ByteArrayInputStream(baos.toByteArray()), jos, entry);
+
+ jos.close();
+ return jarFile;
+ }
+
+ private void addFilesToJar(List<String> classFiles, JarOutputStream jos) throws IOException {
+ ClassLoader classLoader = getClass().getClassLoader();
+ List<File> sourceFiles = new ArrayList<>();
+ for (String classFile : classFiles) {
+ File sourceFile = new File(classLoader.getResource(classFile).getFile());
+ sourceFiles.add(sourceFile);
+ }
+
+ JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+ if (compiler == null) {
+ throw new IllegalStateException(
+ "Cannot find the system Java compiler. "
+ + "Check that your class path includes tools.jar");
+ }
+ StandardJavaFileManager fileManager = compiler.getStandardFileManager
+ (null, null, null);
+
+ fileManager.setLocation(StandardLocation.CLASS_OUTPUT, Arrays.asList(testDir));
+
+ Iterable<? extends JavaFileObject> compilationUnits1 =
+ fileManager.getJavaFileObjectsFromFiles(sourceFiles);
+
+ boolean compiled = compiler.getTask(null, fileManager, null, null, null, compilationUnits1).call();
+ if (!compiled) {
+ throw new RuntimeException("failed to compile");
+ }
+
+ List<String> classesForJar = new ArrayList<String>();
+ // Split the file on dot to get the filename from FILENAME.java
+ for (File source : sourceFiles) {
+ String fileName = source.getName().split("\\.")[0];
+ classesForJar.add(fileName);
+ }
+
+ File[] directoryListing = testDir.listFiles();
+ for (File compiledClass : directoryListing) {
+ String classFileName = compiledClass.getName().split("\\$")[0].split("\\.")[0];
+ if (classesForJar.contains(classFileName)){
+ addFileToJar(compiledClass, jos);
+ }
+ }
+ }
+
+ private void addFileToJar(File source, JarOutputStream jos) throws IOException {
+ JarEntry entry = new JarEntry(source.getName());
+ entry.setTime(source.lastModified());
+
+ BufferedInputStream in = new BufferedInputStream(new FileInputStream(source));
+ addEntry(in, jos, entry);
+
+ if (in != null) {
+ in.close();
+ }
+ }
+
+ private void addEntry(InputStream in, JarOutputStream jos, JarEntry entry) throws IOException {
+ jos.putNextEntry(entry);
+
+ byte[] buffer = new byte[1024];
+ while (true) {
+ int count = in.read(buffer);
+ if (count == -1) {
+ break;
+ }
+ jos.write(buffer, 0, count);
+ }
+
+ jos.closeEntry();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
index 674ae6a..ec48f82 100644
--- a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
+++ b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
@@ -32,6 +32,7 @@ import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
+import org.apache.sqoop.classloader.ConnectorClassLoader;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java b/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java
deleted file mode 100644
index 1ec1bb4..0000000
--- a/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.utils;
-
-import static org.apache.sqoop.utils.ConnectorClassLoader.constructUrlsFromClasspath;
-import static org.apache.sqoop.utils.ConnectorClassLoader.isSystemClass;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.List;
-import java.util.jar.JarOutputStream;
-import java.util.zip.ZipEntry;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
-
-public class TestConnectorClassLoader {
- private static File testDir = new File(System.getProperty("maven.build.directory",
- System.getProperty("java.io.tmpdir")), "connectorclassloader");
-
- @BeforeMethod(alwaysRun = true)
- public void setUp() {
- FileUtils.deleteQuietly(testDir);
- testDir.mkdirs();
- }
-
- @Test
- public void testConstructUrlsFromClasspath() throws Exception {
- File file = new File(testDir, "file");
- assertTrue(file.createNewFile(), "Create file");
-
- File dir = new File(testDir, "dir");
- assertTrue(dir.mkdir(), "Make dir");
-
- File jarsDir = new File(testDir, "jarsdir");
- assertTrue(jarsDir.mkdir(), "Make jarsDir");
- File nonJarFile = new File(jarsDir, "nonjar");
- assertTrue(nonJarFile.createNewFile(), "Create non-jar file");
- File jarFile = new File(jarsDir, "a.jar");
- assertTrue(jarFile.createNewFile(), "Create jar file");
-
- File nofile = new File(testDir, "nofile");
- // don't create nofile
-
- StringBuilder cp = new StringBuilder();
- cp.append(file.getAbsolutePath()).append(File.pathSeparator)
- .append(dir.getAbsolutePath()).append(File.pathSeparator)
- .append(jarsDir.getAbsolutePath() + "/*").append(File.pathSeparator)
- .append(nofile.getAbsolutePath()).append(File.pathSeparator)
- .append(nofile.getAbsolutePath() + "/*").append(File.pathSeparator);
-
- URL[] urls = constructUrlsFromClasspath(cp.toString());
-
- assertEquals(3, urls.length);
- assertEquals(file.toURI().toURL(), urls[0]);
- assertEquals(dir.toURI().toURL(), urls[1]);
- assertEquals(jarFile.toURI().toURL(), urls[2]);
- // nofile should be ignored
- }
-
- @Test
- public void testIsSystemClass() {
- testIsSystemClassInternal("");
- }
-
- @Test
- public void testIsSystemNestedClass() {
- testIsSystemClassInternal("$Klass");
- }
-
- private void testIsSystemClassInternal(String nestedClass) {
- assertFalse(isSystemClass("org.example.Foo" + nestedClass, null));
- assertTrue(isSystemClass("org.example.Foo" + nestedClass,
- classes("org.example.Foo")));
- assertTrue(isSystemClass("/org.example.Foo" + nestedClass,
- classes("org.example.Foo")));
- assertTrue(isSystemClass("org.example.Foo" + nestedClass,
- classes("org.example.")));
- assertTrue(isSystemClass("net.example.Foo" + nestedClass,
- classes("org.example.,net.example.")));
- assertFalse(isSystemClass("org.example.Foo" + nestedClass,
- classes("-org.example.Foo,org.example.")));
- assertTrue(isSystemClass("org.example.Bar" + nestedClass,
- classes("-org.example.Foo.,org.example.")));
- assertFalse(isSystemClass("org.example.Foo" + nestedClass,
- classes("org.example.,-org.example.Foo")));
- assertFalse(isSystemClass("org.example.Foo" + nestedClass,
- classes("org.example.Foo,-org.example.Foo")));
- }
-
- private List<String> classes(String classes) {
- return Lists.newArrayList(Splitter.on(',').split(classes));
- }
-
- @Test
- public void testGetResource() throws IOException {
- URL testJar = makeTestJar().toURI().toURL();
-
- ClassLoader currentClassLoader = getClass().getClassLoader();
- ClassLoader connectorClassloader = new ConnectorClassLoader(
- new URL[] { testJar }, currentClassLoader, null, false);
-
- assertNull(currentClassLoader.getResourceAsStream("resource.txt"),
- "Resource should be null for current classloader");
-
- InputStream in = connectorClassloader.getResourceAsStream("resource.txt");
- assertNotNull(in, "Resource should not be null for connector classloader");
- assertEquals("hello", IOUtils.toString(in));
- }
-
- private File makeTestJar() throws IOException {
- File jarFile = new File(testDir, "test.jar");
- JarOutputStream out = new JarOutputStream(new FileOutputStream(jarFile));
- ZipEntry entry = new ZipEntry("resource.txt");
- out.putNextEntry(entry);
- out.write("hello".getBytes());
- out.closeEntry();
- out.close();
- return jarFile;
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/test/resources/TestConnectorClassLoader/A.java
----------------------------------------------------------------------
diff --git a/common/src/test/resources/TestConnectorClassLoader/A.java b/common/src/test/resources/TestConnectorClassLoader/A.java
new file mode 100644
index 0000000..15294be
--- /dev/null
+++ b/common/src/test/resources/TestConnectorClassLoader/A.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class A {
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/test/resources/TestConnectorClassLoader/lib/B.java
----------------------------------------------------------------------
diff --git a/common/src/test/resources/TestConnectorClassLoader/lib/B.java b/common/src/test/resources/TestConnectorClassLoader/lib/B.java
new file mode 100644
index 0000000..8828670
--- /dev/null
+++ b/common/src/test/resources/TestConnectorClassLoader/lib/B.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class B {
+
+}