You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/09/28 18:46:33 UTC
sqoop git commit: SQOOP-2578: Sqoop2: Port the ApplicationClassLoader
in Hadoop into Sqoop
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 7e5075b2a -> 52e9c11ad
SQOOP-2578: Sqoop2: Port the ApplicationClassLoader in Hadoop into Sqoop
(Dian Fu via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/52e9c11a
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/52e9c11a
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/52e9c11a
Branch: refs/heads/sqoop2
Commit: 52e9c11ad25c7063e9b0790d718d740be6537797
Parents: 7e5075b
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Mon Sep 28 09:45:55 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Mon Sep 28 09:45:55 2015 -0700
----------------------------------------------------------------------
common/pom.xml | 18 --
.../sqoop/utils/ConnectorClassLoader.java | 248 +++++++++++++++++++
...pache.sqoop.connector-classloader.properties | 55 ++++
.../sqoop/utils/TestConnectorClassLoader.java | 149 +++++++++++
4 files changed, 452 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/52e9c11a/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 748fa7a..078a785 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -39,24 +39,6 @@ limitations under the License.
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
<groupId>com.googlecode.json-simple</groupId>
<artifactId>json-simple</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/52e9c11a/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java b/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java
new file mode 100644
index 0000000..69e972d
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.utils;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.log4j.Logger;
+
+/**
+ * A {@link URLClassLoader} for connector isolation. Classes from the
+ * connector JARs are loaded in preference to the parent loader.
+ */
+public class ConnectorClassLoader extends URLClassLoader {
+ /**
+ * Default value of the system classes if the user did not override them.
+ * JDK classes, sqoop classes and resources, and some select third-party
+ * classes are considered system classes, and are not loaded by the
+ * connector classloader.
+ */
+ public static final String SYSTEM_CLASSES_DEFAULT;
+
+ private static final String PROPERTIES_FILE =
+ "org.apache.sqoop.connector-classloader.properties";
+ private static final String SYSTEM_CLASSES_DEFAULT_KEY =
+ "system.classes.default";
+
+ private static final Logger LOG = Logger.getLogger(ConnectorClassLoader.class);
+
+ private static final FilenameFilter JAR_FILENAME_FILTER =
+ new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.endsWith(".jar") || name.endsWith(".JAR");
+ }
+ };
+
+ static {
+ try (InputStream is = ConnectorClassLoader.class.getClassLoader()
+ .getResourceAsStream(PROPERTIES_FILE);) {
+ if (is == null) {
+ throw new ExceptionInInitializerError("properties file " +
+ PROPERTIES_FILE + " is not found");
+ }
+ Properties props = new Properties();
+ props.load(is);
+ // get the system classes default
+ String systemClassesDefault =
+ props.getProperty(SYSTEM_CLASSES_DEFAULT_KEY);
+ if (systemClassesDefault == null) {
+ throw new ExceptionInInitializerError("property " +
+ SYSTEM_CLASSES_DEFAULT_KEY + " is not found");
+ }
+ SYSTEM_CLASSES_DEFAULT = systemClassesDefault;
+ } catch (IOException e) {
+ throw new ExceptionInInitializerError(e);
+ }
+ }
+
+ private final ClassLoader parent;
+ private final List<String> systemClasses;
+
+ public ConnectorClassLoader(URL[] urls, ClassLoader parent,
+ List<String> systemClasses) {
+ super(urls, parent);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("urls: " + Arrays.toString(urls));
+ LOG.debug("system classes: " + systemClasses);
+ }
+ this.parent = parent;
+ if (parent == null) {
+ throw new IllegalArgumentException("No parent classloader!");
+ }
+ // if the caller-specified system classes are null or empty, use the default
+ this.systemClasses = (systemClasses == null || systemClasses.isEmpty()) ?
+ Arrays.asList(SYSTEM_CLASSES_DEFAULT.split("\\s*,\\s*")) :
+ systemClasses;
+ LOG.info("system classes: " + this.systemClasses);
+ }
+
+ public ConnectorClassLoader(String classpath, ClassLoader parent,
+ List<String> systemClasses) throws MalformedURLException {
+ this(constructUrlsFromClasspath(classpath), parent, systemClasses);
+ }
+
+ static URL[] constructUrlsFromClasspath(String classpath)
+ throws MalformedURLException {
+ List<URL> urls = new ArrayList<URL>();
+ for (String element : classpath.split(File.pathSeparator)) {
+ if (element.endsWith("/*")) {
+ String dir = element.substring(0, element.length() - 1);
+ File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
+ if (files != null) {
+ for (File file : files) {
+ urls.add(file.toURI().toURL());
+ }
+ }
+ } else {
+ File file = new File(element);
+ if (file.exists()) {
+ urls.add(new File(element).toURI().toURL());
+ }
+ }
+ }
+ return urls.toArray(new URL[urls.size()]);
+ }
+
+ @Override
+ public URL getResource(String name) {
+ URL url = null;
+
+ if (!isSystemClass(name, systemClasses)) {
+ url= findResource(name);
+ if (url == null && name.startsWith("/")) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Remove leading / off " + name);
+ }
+ url= findResource(name.substring(1));
+ }
+ }
+
+ if (url == null) {
+ url= parent.getResource(name);
+ }
+
+ if (url != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getResource("+name+")=" + url);
+ }
+ }
+
+ return url;
+ }
+
+ @Override
+ public Class<?> loadClass(String name) throws ClassNotFoundException {
+ return this.loadClass(name, false);
+ }
+
+ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve)
+ throws ClassNotFoundException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loading class: " + name);
+ }
+
+ Class<?> c = findLoadedClass(name);
+ ClassNotFoundException ex = null;
+
+ if (c == null && !isSystemClass(name, systemClasses)) {
+ // Try to load class from this classloader's URLs. Note that this is like
+ // the servlet spec, not the usual Java 2 behaviour where we ask the
+ // parent to attempt to load first.
+ try {
+ c = findClass(name);
+ if (LOG.isDebugEnabled() && c != null) {
+ LOG.debug("Loaded class: " + name + " ");
+ }
+ } catch (ClassNotFoundException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(e);
+ }
+ ex = e;
+ }
+ }
+
+ if (c == null) { // try parent
+ c = parent.loadClass(name);
+ if (LOG.isDebugEnabled() && c != null) {
+ LOG.debug("Loaded class from parent: " + name + " ");
+ }
+ }
+
+ if (c == null) {
+ throw ex != null ? ex : new ClassNotFoundException(name);
+ }
+
+ if (resolve) {
+ resolveClass(c);
+ }
+
+ return c;
+ }
+
+ /**
+ * Checks if a class should be included as a system class.
+ *
+ * A class is a system class if and only if it matches one of the positive
+ * patterns and none of the negative ones.
+ *
+ * @param name the class name to check
+ * @param systemClasses a list of system class configurations.
+ * @return true if the class is a system class
+ */
+ public static boolean isSystemClass(String name, List<String> systemClasses) {
+ boolean result = false;
+ if (systemClasses != null) {
+ String canonicalName = name.replace('/', '.');
+ while (canonicalName.startsWith(".")) {
+ canonicalName=canonicalName.substring(1);
+ }
+ for (String c : systemClasses) {
+ boolean shouldInclude = true;
+ if (c.startsWith("-")) {
+ c = c.substring(1);
+ shouldInclude = false;
+ }
+ if (canonicalName.startsWith(c)) {
+ if ( c.endsWith(".") // package
+ || canonicalName.length() == c.length() // class
+ || canonicalName.length() > c.length() // nested
+ && canonicalName.charAt(c.length()) == '$' ) {
+ if (shouldInclude) {
+ result = true;
+ } else {
+ return false;
+ }
+ }
+ }
+ }
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/52e9c11a/common/src/main/resources/org.apache.sqoop.connector-classloader.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/org.apache.sqoop.connector-classloader.properties b/common/src/main/resources/org.apache.sqoop.connector-classloader.properties
new file mode 100644
index 0000000..e2936a9
--- /dev/null
+++ b/common/src/main/resources/org.apache.sqoop.connector-classloader.properties
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# contains key properties for setting up the connector classloader
+system.classes.default=java.,\
+ javax.accessibility.,\
+ javax.activation.,\
+ javax.activity.,\
+ javax.annotation.,\
+ javax.annotation.processing.,\
+ javax.crypto.,\
+ javax.imageio.,\
+ javax.jws.,\
+ javax.lang.model.,\
+ -javax.management.j2ee.,\
+ javax.management.,\
+ javax.naming.,\
+ javax.net.,\
+ javax.print.,\
+ javax.rmi.,\
+ javax.script.,\
+ -javax.security.auth.message.,\
+ javax.security.auth.,\
+ javax.security.cert.,\
+ javax.security.sasl.,\
+ javax.sound.,\
+ javax.sql.,\
+ javax.swing.,\
+ javax.tools.,\
+ javax.transaction.,\
+ -javax.xml.registry.,\
+ -javax.xml.rpc.,\
+ javax.xml.,\
+ org.w3c.dom.,\
+ org.xml.sax.,\
+ org.apache.commons.logging.,\
+ org.apache.log4j.,\
+ org.apache.sqoop.,\
+ sqoop.properties,\
+ sqoop_bootstrap.properties
http://git-wip-us.apache.org/repos/asf/sqoop/blob/52e9c11a/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java b/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java
new file mode 100644
index 0000000..442be4d
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.utils;
+
+import static org.apache.sqoop.utils.ConnectorClassLoader.constructUrlsFromClasspath;
+import static org.apache.sqoop.utils.ConnectorClassLoader.isSystemClass;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.List;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
+public class TestConnectorClassLoader {
+ private static File testDir = new File(System.getProperty("maven.build.directory",
+ System.getProperty("java.io.tmpdir")), "connectorclassloader");
+
+ @BeforeMethod(alwaysRun = true)
+ public void setUp() {
+ FileUtils.deleteQuietly(testDir);
+ testDir.mkdirs();
+ }
+
+ @Test
+ public void testConstructUrlsFromClasspath() throws Exception {
+ File file = new File(testDir, "file");
+ assertTrue(file.createNewFile(), "Create file");
+
+ File dir = new File(testDir, "dir");
+ assertTrue(dir.mkdir(), "Make dir");
+
+ File jarsDir = new File(testDir, "jarsdir");
+ assertTrue(jarsDir.mkdir(), "Make jarsDir");
+ File nonJarFile = new File(jarsDir, "nonjar");
+ assertTrue(nonJarFile.createNewFile(), "Create non-jar file");
+ File jarFile = new File(jarsDir, "a.jar");
+ assertTrue(jarFile.createNewFile(), "Create jar file");
+
+ File nofile = new File(testDir, "nofile");
+ // don't create nofile
+
+ StringBuilder cp = new StringBuilder();
+ cp.append(file.getAbsolutePath()).append(File.pathSeparator)
+ .append(dir.getAbsolutePath()).append(File.pathSeparator)
+ .append(jarsDir.getAbsolutePath() + "/*").append(File.pathSeparator)
+ .append(nofile.getAbsolutePath()).append(File.pathSeparator)
+ .append(nofile.getAbsolutePath() + "/*").append(File.pathSeparator);
+
+ URL[] urls = constructUrlsFromClasspath(cp.toString());
+
+ assertEquals(3, urls.length);
+ assertEquals(file.toURI().toURL(), urls[0]);
+ assertEquals(dir.toURI().toURL(), urls[1]);
+ assertEquals(jarFile.toURI().toURL(), urls[2]);
+ // nofile should be ignored
+ }
+
+ @Test
+ public void testIsSystemClass() {
+ testIsSystemClassInternal("");
+ }
+
+ @Test
+ public void testIsSystemNestedClass() {
+ testIsSystemClassInternal("$Klass");
+ }
+
+ private void testIsSystemClassInternal(String nestedClass) {
+ assertFalse(isSystemClass("org.example.Foo" + nestedClass, null));
+ assertTrue(isSystemClass("org.example.Foo" + nestedClass,
+ classes("org.example.Foo")));
+ assertTrue(isSystemClass("/org.example.Foo" + nestedClass,
+ classes("org.example.Foo")));
+ assertTrue(isSystemClass("org.example.Foo" + nestedClass,
+ classes("org.example.")));
+ assertTrue(isSystemClass("net.example.Foo" + nestedClass,
+ classes("org.example.,net.example.")));
+ assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+ classes("-org.example.Foo,org.example.")));
+ assertTrue(isSystemClass("org.example.Bar" + nestedClass,
+ classes("-org.example.Foo.,org.example.")));
+ assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+ classes("org.example.,-org.example.Foo")));
+ assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+ classes("org.example.Foo,-org.example.Foo")));
+ }
+
+ private List<String> classes(String classes) {
+ return Lists.newArrayList(Splitter.on(',').split(classes));
+ }
+
+ @Test
+ public void testGetResource() throws IOException {
+ URL testJar = makeTestJar().toURI().toURL();
+
+ ClassLoader currentClassLoader = getClass().getClassLoader();
+ ClassLoader connectorClassloader = new ConnectorClassLoader(
+ new URL[] { testJar }, currentClassLoader, null);
+
+ assertNull(currentClassLoader.getResourceAsStream("resource.txt"),
+ "Resource should be null for current classloader");
+
+ InputStream in = connectorClassloader.getResourceAsStream("resource.txt");
+ assertNotNull(in, "Resource should not be null for connector classloader");
+ assertEquals("hello", IOUtils.toString(in));
+ }
+
+ private File makeTestJar() throws IOException {
+ File jarFile = new File(testDir, "test.jar");
+ JarOutputStream out = new JarOutputStream(new FileOutputStream(jarFile));
+ ZipEntry entry = new ZipEntry("resource.txt");
+ out.putNextEntry(entry);
+ out.write("hello".getBytes());
+ out.closeEntry();
+ out.close();
+ return jarFile;
+ }
+}