You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/04/30 23:26:22 UTC
sqoop git commit: SQOOP-2342: Sqoop2: Provide an external application
classloader
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 89e377a4d -> 5abd3f91f
SQOOP-2342: Sqoop2: Provide an external application classloader
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5abd3f91
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5abd3f91
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5abd3f91
Branch: refs/heads/sqoop2
Commit: 5abd3f91f0d7c4480aa8b0f4f7d53c8304c2c276
Parents: 89e377a
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Thu Apr 30 14:25:43 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Thu Apr 30 14:25:43 2015 -0700
----------------------------------------------------------------------
.../org/apache/sqoop/error/code/CoreError.java | 5 +-
.../sqoop/connector/ConnectorManager.java | 6 --
.../sqoop/connector/ConnectorManagerUtils.java | 64 --------------------
.../sqoop/core/ConfigurationConstants.java | 6 +-
.../apache/sqoop/core/SqoopConfiguration.java | 53 ++++++++++++++++
.../connector/TestConnectorManagerUtils.java | 28 ---------
.../sqoop/core/TestSqoopConfiguration.java | 3 +
core/src/test/resources/test_config.properties | 3 +
dist/src/main/server/conf/sqoop.properties | 5 ++
docs/src/site/sphinx/ConnectorDevelopment.rst | 9 ++-
.../test/minicluster/SqoopMiniCluster.java | 7 ---
11 files changed, 74 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/common/src/main/java/org/apache/sqoop/error/code/CoreError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/error/code/CoreError.java b/common/src/main/java/org/apache/sqoop/error/code/CoreError.java
index 24e73c3..bc3076c 100644
--- a/common/src/main/java/org/apache/sqoop/error/code/CoreError.java
+++ b/common/src/main/java/org/apache/sqoop/error/code/CoreError.java
@@ -52,7 +52,10 @@ public enum CoreError implements ErrorCode {
CORE_0007("System not initialized"),
/** The system has not been reconfigured */
- CORE_0008("System not reconfigured");
+ CORE_0008("System not reconfigured"),
+
+ /** Sqoop classpath error */
+ CORE_0009("Sqoop classpath error"),
;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
index 0517413..d1537bf 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
@@ -27,7 +27,6 @@ import java.util.Map;
import java.util.ResourceBundle;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.SqoopConnector;
@@ -162,11 +161,6 @@ public class ConnectorManager implements Reconfigurable {
LOG.trace("Begin connector manager initialization");
}
- // add external connectors into the class path
- // NOTE: class loading happens later in the ConnectorHandler
- ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(SqoopConfiguration.getInstance().getContext()
- .getString(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, StringUtils.EMPTY));
-
List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs();
LOG.info("Connector config urls: " + connectorConfigs);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
index 99736c6..7402c5a 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
@@ -17,21 +17,16 @@
*/
package org.apache.sqoop.connector;
-import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.core.ConfigurationConstants;
import org.apache.sqoop.error.code.ConnectorError;
import java.io.File;
import java.io.IOException;
-import java.net.MalformedURLException;
import java.net.URL;
-import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Enumeration;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
@@ -75,25 +70,6 @@ public class ConnectorManagerUtils {
return connectorConfigs;
}
- public static Set<File> getConnectorJars(String path) {
- if (StringUtils.isEmpty(path)) {
- return null;
- }
- Set<File> jarFiles = new HashSet<File>();
- File folder = new File(path);
- if (folder.exists()) {
- for (File file : folder.listFiles()) {
- if (file.isDirectory()) {
- jarFiles.addAll(getConnectorJars(file.getPath()));
- }
- if (file.getName().endsWith(".jar") && isConnectorJar(file)) {
- jarFiles.add(file);
- }
- }
- }
- return jarFiles;
- }
-
static boolean isConnectorJar(File file) {
try {
@SuppressWarnings("resource")
@@ -103,44 +79,4 @@ public class ConnectorManagerUtils {
throw new RuntimeException(e);
}
}
-
- public static void addExternalConnectorsJarsToClasspath(String path) {
- if (StringUtils.isEmpty(path)) {
- return;
- }
-
- ClassLoader currentThreadClassLoader = Thread.currentThread().getContextClassLoader();
- if (currentThreadClassLoader != null) {
-
- // Add the 'org.apache.sqoop.connector.external.loadpath' to the classpath
- // Chain the current thread classloader
- ExternalConnectorJarFileLoader connectorUrlClassLoader = new ExternalConnectorJarFileLoader(new URL[] {},
- currentThreadClassLoader);
- // the property always holds a path to the folder containing the jars
- Set<File> connectorJars = getConnectorJars(path);
- if (connectorJars != null && !connectorJars.isEmpty()) {
- for (File jar : connectorJars) {
- connectorUrlClassLoader.addJarFile(jar.getPath());
- }
- // Replace the thread classloader- assuming there is permission to do so
- Thread.currentThread().setContextClassLoader(connectorUrlClassLoader);
- }
- }
- }
-
- public static class ExternalConnectorJarFileLoader extends URLClassLoader {
- public ExternalConnectorJarFileLoader(URL[] urls, ClassLoader parent) {
- super(urls, parent);
- }
-
- public void addJarFile(String path) {
- String urlPath = "jar:file://" + path + "!/";
- try {
- addURL(new URL(urlPath));
- } catch (MalformedURLException e) {
- throw new RuntimeException(e);
- }
- }
-
- }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
index 49e5d6f..d7fe27b 100644
--- a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
+++ b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
@@ -77,11 +77,9 @@ public final class ConfigurationConstants {
"org.apache.sqoop.driver.autoupgrade";
/**
- # Support loading external connector jars only
- # The loader will look for sqoopconnector.properties file in the jar before loading
- # "/path/to/external/connectors/": Add all the connector JARs in the specified folder
+ * Add external jars to application classpath.
*/
- public static final String EXTERNAL_CONNECTOR_LOAD_PATH = "org.apache.sqoop.connector.external.loadpath";
+ public static final String CLASSPATH = "org.apache.sqoop.classpath.extra";
/**
* Enable Sqoop App to kill Tomcat in case that it will fail to load.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java b/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
index 15ee12f..58c4c34 100644
--- a/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
+++ b/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
@@ -21,9 +21,18 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
+import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import org.apache.sqoop.classification.InterfaceAudience;
@@ -175,6 +184,8 @@ public class SqoopConfiguration implements Reconfigurable {
provider.registerListener(new CoreConfigurationListener(SqoopConfiguration.getInstance()));
initialized = true;
+
+ configureClassLoader(ConfigurationConstants.CLASSPATH);
}
public synchronized MapContext getContext() {
@@ -212,6 +223,48 @@ public class SqoopConfiguration implements Reconfigurable {
initialized = false;
}
+ /**
+ * Load extra classpath from sqoop configuration.
+ * @param classpathProperty
+ */
+ private synchronized void configureClassLoader(String classpathProperty) {
+ LOG.info("Adding jars to current classloader from property: " + classpathProperty);
+
+ String classpath = getContext().getString(classpathProperty);
+
+ if (StringUtils.isEmpty(classpath)) {
+ LOG.debug("Property " + classpathProperty + " is null or empty. Not adding any extra jars.");
+ return;
+ }
+
+ ClassLoader currentThreadClassLoader = Thread.currentThread().getContextClassLoader();
+ if (currentThreadClassLoader == null) {
+ throw new SqoopException(CoreError.CORE_0009, "No thread context classloader to override.");
+ }
+
+ // CSV URL list separated by ":".
+ Set<String> paths = new HashSet(Arrays.asList(classpath.split(":")));
+ List<URL> urls = new LinkedList<URL>();
+
+ for (String path : paths) {
+ try {
+ LOG.debug("Found jar in path: " + path);
+ URL url = new File(path).toURI().toURL();
+ urls.add(url);
+ LOG.debug("Using URL: " + url.toString());
+ } catch (MalformedURLException e) {
+ throw new SqoopException(CoreError.CORE_0009, "Malformed URL found.", e);
+ }
+ }
+
+ // Chain the current thread classloader so that
+ // configured classpath adds to existing classloader.
+ // Existing classpath is not changed.
+ URLClassLoader classLoader = new URLClassLoader(urls.toArray(new URL[urls.size()]),
+ currentThreadClassLoader);
+ Thread.currentThread().setContextClassLoader(classLoader);
+ }
+
private synchronized void configureLogging() {
Properties props = new Properties();
for (String key : config.keySet()) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java
index f2e64cd..258b5f9 100644
--- a/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java
+++ b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java
@@ -20,16 +20,10 @@ package org.apache.sqoop.connector;
import static org.testng.AssertJUnit.assertFalse;
import static org.testng.AssertJUnit.assertTrue;
-import static org.testng.AssertJUnit.assertNull;
-import static org.testng.AssertJUnit.assertEquals;
-import org.apache.sqoop.utils.ClassUtils;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.io.File;
-import java.net.URL;
-import java.util.List;
-import java.util.Set;
public class TestConnectorManagerUtils {
@@ -41,19 +35,6 @@ public class TestConnectorManagerUtils {
}
@Test
- public void testGetConnectorJarsNullPath() {
- Set<File> files = ConnectorManagerUtils.getConnectorJars(null);
- assertNull(files);
- }
-
- @Test
- public void testGetConnectorJarsNonNullPath() {
- String path = workingDir + "/src/test/resources";
- Set<File> files = ConnectorManagerUtils.getConnectorJars(path);
- assertEquals(1, files.size());
- }
-
- @Test
public void testIsConnectorJar() {
String path = workingDir + "/src/test/resources/test-connector.jar";
File connectorJar = new File(path);
@@ -69,13 +50,4 @@ public class TestConnectorManagerUtils {
assertFalse(ConnectorManagerUtils.isConnectorJar(file));
}
- @Test
- public void testAddExternalConnectorJarToClasspath() {
- String path = workingDir + "/src/test/resources";
- ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(path);
- List<URL> urls = ConnectorManagerUtils.getConnectorConfigs();
- assertEquals(1, urls.size());
- ClassUtils.loadClass("org.apache.sqoop.connector.jdbc.GenericJdbcConnector");
- }
-
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java b/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java
index b11587c..499c560 100644
--- a/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java
+++ b/core/src/test/java/org/apache/sqoop/core/TestSqoopConfiguration.java
@@ -162,5 +162,8 @@ public class TestSqoopConfiguration {
public void testConfigurationInitSuccess() throws Exception {
TestUtils.setupTestConfigurationWithExtraConfig(null, null);
SqoopConfiguration.getInstance().initialize();
+
+ // Make sure "test" is in classpath.
+ Assert.assertNotNull(Thread.currentThread().getContextClassLoader().getResource("test_config.properties"));
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/core/src/test/resources/test_config.properties
----------------------------------------------------------------------
diff --git a/core/src/test/resources/test_config.properties b/core/src/test/resources/test_config.properties
index 4ad1267..50aeb3b 100644
--- a/core/src/test/resources/test_config.properties
+++ b/core/src/test/resources/test_config.properties
@@ -23,3 +23,6 @@ org.apache.sqoop.log4j.logger.org.apache.sqoop=debug,console
org.apache.sqoop.log4j.appender.console=org.apache.log4j.ConsoleAppender
org.apache.sqoop.log4j.appender.console.layout=org.apache.log4j.PatternLayout
org.apache.sqoop.log4j.appender.console.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+
+# Extra classpath
+org.apache.sqoop.classpath.extra=core/src/test/resources/test_config.properties
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/dist/src/main/server/conf/sqoop.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties
index 5226a19..ba6e09f 100755
--- a/dist/src/main/server/conf/sqoop.properties
+++ b/dist/src/main/server/conf/sqoop.properties
@@ -171,3 +171,8 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce
# "/path/to/external/connectors/": Add all the connector JARs in the specified folder
#
org.apache.sqoop.connector.external.loadpath=
+
+# Sqoop application classpath
+# ":" separated list of jars to be included in sqoop.
+#
+org.apache.sqoop.classpath.extra=
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/docs/src/site/sphinx/ConnectorDevelopment.rst
----------------------------------------------------------------------
diff --git a/docs/src/site/sphinx/ConnectorDevelopment.rst b/docs/src/site/sphinx/ConnectorDevelopment.rst
index 2d9e7d2..1ea1881 100644
--- a/docs/src/site/sphinx/ConnectorDevelopment.rst
+++ b/docs/src/site/sphinx/ConnectorDevelopment.rst
@@ -483,15 +483,14 @@ Loading new connector say sqoop-foo-connector to the sqoop2, here are the steps
1. Create a ``sqoop-foo-connector.jar``. Make sure the jar contains the ``sqoopconnector.properties`` for it to be picked up by Sqoop
-2. Add this jar to the a folder on your installation machine and update the path to this folder in the sqoop.properties located under the ``server/conf`` directory under the Sqoop2 for the key ``org.apache.sqoop.connector.external.loadpath``
+2. Add this jar to the ``org.apache.sqoop.classpath.extra`` property in the sqoop.properties located under the ``server/conf`` directory.
::
+ # Sqoop application classpath
+ # ":" separated list of jars to be included in sqoop.
#
- # External connectors load path
- # "/path/to/external/connectors/": Add all the connector JARs in the specified folder
- #
- org.apache.sqoop.connector.external.loadpath=/path/to/connector
+ org.apache.sqoop.classpath.extra=/path/to/connector.jar
3. Start the Sqoop 2 server and while initializing the server this jar should be loaded into the Sqoop 2's class path and registered into the Sqoop 2 repository
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5abd3f91/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
index ad45189..6ec6883 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
@@ -132,7 +132,6 @@ public abstract class SqoopMiniCluster {
mapToProperties(sqoopProperties, getSecurityConfiguration());
mapToProperties(sqoopProperties, getConnectorManagerConfiguration());
mapToProperties(sqoopProperties, getDriverManagerConfiguration());
- mapToProperties(sqoopProperties, getExternalConnectorLoadPathConfiguration());
FileUtils.writeLines(f, sqoopProperties);
@@ -222,10 +221,4 @@ public abstract class SqoopMiniCluster {
properties.put(ConfigurationConstants.DRIVER_AUTO_UPGRADE, "true");
return properties;
}
-
- protected Map<String, String> getExternalConnectorLoadPathConfiguration() {
- Map<String, String> properties = new HashMap<String, String>();
- properties.put(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, "");
- return properties;
- }
}