You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ab...@apache.org on 2015/01/20 21:50:44 UTC
sqoop git commit: SQOOP-1821: Sqoop2: External connector loading
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 49f104bb5 -> 913521e1a
SQOOP-1821: Sqoop2: External connector loading
(Veena Basavaraj via Abraham Elmahrek)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/913521e1
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/913521e1
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/913521e1
Branch: refs/heads/sqoop2
Commit: 913521e1a93b2571d2c83b10c51609b54744eee9
Parents: 49f104b
Author: Abraham Elmahrek <ab...@apache.org>
Authored: Tue Jan 20 12:50:15 2015 -0800
Committer: Abraham Elmahrek <ab...@apache.org>
Committed: Tue Jan 20 12:50:15 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/sqoop/utils/ClassUtils.java | 2 +
.../org/apache/sqoop/utils/TestClassUtils.java | 2 +-
.../sqoop/connector/ConnectorHandler.java | 5 +-
.../sqoop/connector/ConnectorManager.java | 6 ++
.../sqoop/connector/ConnectorManagerUtils.java | 87 +++++++++++++++++--
.../sqoop/core/ConfigurationConstants.java | 7 ++
.../connector/TestConnectorManagerUtils.java | 64 ++++++++++++++
core/src/test/resources/test-connector.jar | Bin 0 -> 55938 bytes
core/src/test/resources/test-non-connector.jar | Bin 0 -> 19827 bytes
dist/src/main/server/conf/sqoop.properties | 8 +-
.../test/minicluster/SqoopMiniCluster.java | 13 +--
11 files changed, 179 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
index 0be4d41..d6a8254 100644
--- a/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
+++ b/common/src/main/java/org/apache/sqoop/utils/ClassUtils.java
@@ -57,6 +57,8 @@ public final class ClassUtils {
klass = ctxLoader.loadClass(className);
} catch (ClassNotFoundException ex) {
LOG.debug("Exception while load class: " + className, ex);
+ // wrapping it in runtime, to avoid chainging the signature of methods currently invoking this method
+ throw new RuntimeException(ex);
}
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
index 161a1fa..58a27ba 100644
--- a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
+++ b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
@@ -29,7 +29,7 @@ import static org.testng.Assert.assertNull;
*/
public class TestClassUtils {
- @Test
+ @Test(expectedExceptions = Exception.class)
public void testLoadClass() {
assertNull(ClassUtils.loadClass("A"));
assertEquals(A.class, ClassUtils.loadClass(A.class.getName()));
http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
index 1919b4b..bfdb7b3 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorHandler.java
@@ -31,6 +31,7 @@ import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MToConfig;
+import org.apache.sqoop.utils.ClassUtils;
public final class ConnectorHandler {
@@ -71,8 +72,8 @@ public final class ConnectorHandler {
Class<?> connectorClass = null;
try {
- connectorClass = Class.forName(connectorClassName);
- } catch (ClassNotFoundException ex) {
+ connectorClass = ClassUtils.loadClass(connectorClassName);
+ } catch (Exception ex) {
throw new SqoopException(ConnectorError.CONN_0005,
connectorClassName, ex);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
index b9d4d60..0907efb 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManager.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.ResourceBundle;
import java.util.Set;
+import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.connector.spi.SqoopConnector;
@@ -158,6 +159,11 @@ public class ConnectorManager implements Reconfigurable {
LOG.trace("Begin connector manager initialization");
}
+ // add external connectors into the class path
+ // NOTE: class loading happens later in the ConnectorHandler
+ ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(SqoopConfiguration.getInstance().getContext()
+ .getString(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, StringUtils.EMPTY));
+
List<URL> connectorConfigs = ConnectorManagerUtils.getConnectorConfigs();
LOG.info("Connector config urls: " + connectorConfigs);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
index c7193ee..bd85e99 100644
--- a/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
+++ b/core/src/main/java/org/apache/sqoop/connector/ConnectorManagerUtils.java
@@ -17,14 +17,22 @@
*/
package org.apache.sqoop.connector;
+import org.apache.commons.lang.StringUtils;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.core.ConfigurationConstants;
+import java.io.File;
import java.io.IOException;
+import java.net.MalformedURLException;
import java.net.URL;
+import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Enumeration;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
/**
* Utilities for ConnectorManager.
@@ -33,7 +41,7 @@ public class ConnectorManagerUtils {
/**
* Get a list of URLs of connectors that are installed.
- * Check
+ *
* @return List of URLs.
*/
public static List<URL> getConnectorConfigs() {
@@ -41,9 +49,8 @@ public class ConnectorManagerUtils {
try {
// Check ConnectorManager classloader.
- Enumeration<URL> appPathConfigs =
- ConnectorManager.class.getClassLoader().getResources(
- ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
+ Enumeration<URL> appPathConfigs = ConnectorManager.class.getClassLoader().getResources(
+ ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
while (appPathConfigs.hasMoreElements()) {
connectorConfigs.add(appPathConfigs.nextElement());
}
@@ -51,8 +58,7 @@ public class ConnectorManagerUtils {
// Check thread context classloader.
ClassLoader ctxLoader = Thread.currentThread().getContextClassLoader();
if (ctxLoader != null) {
- Enumeration<URL> ctxPathConfigs = ctxLoader.getResources(
- ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
+ Enumeration<URL> ctxPathConfigs = ctxLoader.getResources(ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
while (ctxPathConfigs.hasMoreElements()) {
URL configUrl = ctxPathConfigs.nextElement();
@@ -67,4 +73,73 @@ public class ConnectorManagerUtils {
return connectorConfigs;
}
+
+ public static Set<File> getConnectorJars(String path) {
+ if (StringUtils.isEmpty(path)) {
+ return null;
+ }
+ Set<File> jarFiles = new HashSet<File>();
+ File folder = new File(path);
+ if (folder.exists()) {
+ for (File file : folder.listFiles()) {
+ if (file.isDirectory()) {
+ jarFiles.addAll(getConnectorJars(file.getPath()));
+ }
+ if (file.getName().endsWith(".jar") && isConnectorJar(file)) {
+ jarFiles.add(file);
+ }
+ }
+ }
+ return jarFiles;
+ }
+
+ static boolean isConnectorJar(File file) {
+ try {
+ @SuppressWarnings("resource")
+ JarEntry entry = new JarFile(file).getJarEntry(ConfigurationConstants.FILENAME_CONNECTOR_PROPERTIES);
+ return entry != null;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static void addExternalConnectorsJarsToClasspath(String path) {
+ if (StringUtils.isEmpty(path)) {
+ return;
+ }
+
+ ClassLoader currentThreadClassLoader = Thread.currentThread().getContextClassLoader();
+ if (currentThreadClassLoader != null) {
+
+ // Add the 'org.apache.sqoop.connector.external.loadpath' to the classpath
+ // Chain the current thread classloader
+ ExternalConnectorJarFileLoader connectorUrlClassLoader = new ExternalConnectorJarFileLoader(new URL[] {},
+ currentThreadClassLoader);
+ // the property always holds a path to the folder containing the jars
+ Set<File> connectorJars = getConnectorJars(path);
+ if (connectorJars != null && !connectorJars.isEmpty()) {
+ for (File jar : connectorJars) {
+ connectorUrlClassLoader.addJarFile(jar.getPath());
+ }
+ // Replace the thread classloader- assuming there is permission to do so
+ Thread.currentThread().setContextClassLoader(connectorUrlClassLoader);
+ }
+ }
+ }
+
+ public static class ExternalConnectorJarFileLoader extends URLClassLoader {
+ public ExternalConnectorJarFileLoader(URL[] urls, ClassLoader parent) {
+ super(urls, parent);
+ }
+
+ public void addJarFile(String path) {
+ String urlPath = "jar:file://" + path + "!/";
+ try {
+ addURL(new URL(urlPath));
+ } catch (MalformedURLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
index f341108..49e5d6f 100644
--- a/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
+++ b/core/src/main/java/org/apache/sqoop/core/ConfigurationConstants.java
@@ -77,6 +77,13 @@ public final class ConfigurationConstants {
"org.apache.sqoop.driver.autoupgrade";
/**
+ # Support loading external connector jars only
+ # The loader will look for sqoopconnector.properties file in the jar before loading
+ # "/path/to/external/connectors/": Add all the connector JARs in the specified folder
+ */
+ public static final String EXTERNAL_CONNECTOR_LOAD_PATH = "org.apache.sqoop.connector.external.loadpath";
+
+ /**
* Enable Sqoop App to kill Tomcat in case that it will fail to load.
*/
public static final String KILL_TOMCAT_ON_FAILURE = "sqoop.kill_tomcat_on_load_failure";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java
new file mode 100644
index 0000000..18f8edb
--- /dev/null
+++ b/core/src/test/java/org/apache/sqoop/connector/TestConnectorManagerUtils.java
@@ -0,0 +1,64 @@
+package org.apache.sqoop.connector;
+
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.AssertJUnit.assertNull;
+import static org.testng.AssertJUnit.assertEquals;
+import org.apache.sqoop.utils.ClassUtils;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+import org.testng.AssertJUnit;
+
+import java.io.File;
+import java.net.URL;
+import java.util.List;
+import java.util.Set;
+
+public class TestConnectorManagerUtils {
+
+ private String workingDir;
+
+ @BeforeMethod(alwaysRun = true)
+ public void setUp() {
+ workingDir = System.getProperty("user.dir");
+ }
+
+ @Test
+ public void testGetConnectorJarsNullPath() {
+ Set<File> files = ConnectorManagerUtils.getConnectorJars(null);
+ assertNull(files);
+ }
+
+ @Test
+ public void testGetConnectorJarsNonNullPath() {
+ String path = workingDir + "/src/test/resources";
+ Set<File> files = ConnectorManagerUtils.getConnectorJars(path);
+ assertEquals(1, files.size());
+ }
+
+ @Test
+ public void testIsConnectorJar() {
+ String path = workingDir + "/src/test/resources/test-connector.jar";
+ File connectorJar = new File(path);
+ assertTrue(connectorJar.exists());
+ assertTrue(ConnectorManagerUtils.isConnectorJar(connectorJar));
+ }
+
+ @Test
+ public void testIsNotConnectorJar() {
+ String path = workingDir + "/src/test/resources/test-non-connector.jar";
+ File file = new File(path);
+ assertTrue(file.exists());
+ assertFalse(ConnectorManagerUtils.isConnectorJar(file));
+ }
+
+ @Test
+ public void testAddExternalConnectorJarToClasspath() {
+ String path = workingDir + "/src/test/resources";
+ ConnectorManagerUtils.addExternalConnectorsJarsToClasspath(path);
+ List<URL> urls = ConnectorManagerUtils.getConnectorConfigs();
+ assertEquals(1, urls.size());
+ ClassUtils.loadClass("org.apache.sqoop.connector.jdbc.GenericJdbcConnector");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/test/resources/test-connector.jar
----------------------------------------------------------------------
diff --git a/core/src/test/resources/test-connector.jar b/core/src/test/resources/test-connector.jar
new file mode 100644
index 0000000..fd3b116
Binary files /dev/null and b/core/src/test/resources/test-connector.jar differ
http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/core/src/test/resources/test-non-connector.jar
----------------------------------------------------------------------
diff --git a/core/src/test/resources/test-non-connector.jar b/core/src/test/resources/test-non-connector.jar
new file mode 100644
index 0000000..01f82ca
Binary files /dev/null and b/core/src/test/resources/test-non-connector.jar differ
http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/dist/src/main/server/conf/sqoop.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties
index 824c5eb..fbcb1fa 100755
--- a/dist/src/main/server/conf/sqoop.properties
+++ b/dist/src/main/server/conf/sqoop.properties
@@ -163,4 +163,10 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce
#
#org.apache.sqoop.security.authorization.handler=org.apache.sqoop.security.Authorization.DefaultAuthorizationHandler
#org.apache.sqoop.security.authorization.access_controller=org.apache.sqoop.security.Authorization.DefaultAuthorizationAccessController
-#org.apache.sqoop.security.authorization.validator=org.apache.sqoop.security.Authorization.DefaultAuthorizationValidator
\ No newline at end of file
+#org.apache.sqoop.security.authorization.validator=org.apache.sqoop.security.Authorization.DefaultAuthorizationValidator
+
+
+# External connectors load path
+# "/path/to/external/connectors/": Add all the connector JARs in the specified folder
+#
+org.apache.sqoop.connector.external.loadpath=
http://git-wip-us.apache.org/repos/asf/sqoop/blob/913521e1/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
index e4eecbf..ad45189 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
@@ -132,6 +132,7 @@ public abstract class SqoopMiniCluster {
mapToProperties(sqoopProperties, getSecurityConfiguration());
mapToProperties(sqoopProperties, getConnectorManagerConfiguration());
mapToProperties(sqoopProperties, getDriverManagerConfiguration());
+ mapToProperties(sqoopProperties, getExternalConnectorLoadPathConfiguration());
FileUtils.writeLines(f, sqoopProperties);
@@ -212,17 +213,19 @@ public abstract class SqoopMiniCluster {
protected Map<String, String> getConnectorManagerConfiguration() {
Map<String, String> properties = new HashMap<String, String>();
-
- properties.put("org.apache.sqoop.connector.autoupgrade", "true");
-
+ properties.put(ConfigurationConstants.CONNECTOR_AUTO_UPGRADE, "true");
return properties;
}
protected Map<String, String> getDriverManagerConfiguration() {
Map<String, String> properties = new HashMap<String, String>();
+ properties.put(ConfigurationConstants.DRIVER_AUTO_UPGRADE, "true");
+ return properties;
+ }
- properties.put("org.apache.sqoop.driver.autoupgrade", "true");
-
+ protected Map<String, String> getExternalConnectorLoadPathConfiguration() {
+ Map<String, String> properties = new HashMap<String, String>();
+ properties.put(ConfigurationConstants.EXTERNAL_CONNECTOR_LOAD_PATH, "");
return properties;
}
}