You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2016/02/26 18:37:43 UTC
[2/3] sqoop git commit: SQOOP-2852: Sqoop2: Revert and re-commit
SQOOP-2843: Sqoop2: Enable SSL/TLS for API calls
SQOOP-2852: Sqoop2: Revert and re-commit SQOOP-2843: Sqoop2: Enable SSL/TLS for API calls
* Revert "SQOOP-2843: Sqoop2: Enable SSL/TLS for API calls"
* This reverts commit 96f3d9c195c78c1f315a61f7c84aa095a041f72b.
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5fd80fd5
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5fd80fd5
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5fd80fd5
Branch: refs/heads/sqoop2
Commit: 5fd80fd5b66dce6cd9dec66bb7070cd3ff67004c
Parents: 96f3d9c
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Fri Feb 26 09:36:13 2016 -0800
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Fri Feb 26 09:36:13 2016 -0800
----------------------------------------------------------------------
.../sqoop/security/SecurityConstants.java | 43 ---
dist/src/main/conf/sqoop.properties | 9 -
.../sqoop/docs/generator/DocPreprocessor.java | 2 -
.../plugins/GenerateConnectorPages.java | 65 -----
.../apache/sqoop/server/SqoopJettyServer.java | 49 +---
.../apache/sqoop/server/common/ServerError.java | 3 -
.../apache/sqoop/shell/ShellEnvironment.java | 13 -
.../sqoop/shell/ShowConnectorFunction.java | 3 +-
.../org/apache/sqoop/shell/ShowJobFunction.java | 6 +-
.../apache/sqoop/shell/ShowLinkFunction.java | 8 +-
.../sqoop/shell/ShowPrincipalFunction.java | 4 +-
.../sqoop/shell/ShowPrivilegeFunction.java | 4 +-
.../apache/sqoop/shell/ShowRoleFunction.java | 4 +-
.../sqoop/shell/ShowSubmissionFunction.java | 3 +-
.../sqoop/shell/utils/TableDisplayer.java | 41 +--
test/pom.xml | 6 +-
.../apache/sqoop/test/kdc/MiniKdcRunner.java | 247 ++++++++++++++++-
.../apache/sqoop/test/utils/SecurityUtils.java | 269 -------------------
.../org/apache/sqoop/test/utils/SqoopUtils.java | 25 +-
.../BlacklistedConnectorTest.java | 82 ++++++
.../connectorloading/ClasspathTest.java | 202 ++++++++++++++
.../ConnectorClasspathIsolationTest.java | 190 +++++++++++++
.../BlacklistedConnectorTest.java | 82 ------
.../serverproperties/ClasspathTest.java | 202 --------------
.../ConnectorClasspathIsolationTest.java | 190 -------------
.../integration/serverproperties/SslTest.java | 164 -----------
.../resources/connector-loading-tests-suite.xml | 34 +++
.../resources/server-properties-tests-suite.xml | 34 ---
28 files changed, 785 insertions(+), 1199 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java b/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java
index 91a1b8b..6f32e04 100644
--- a/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java
+++ b/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java
@@ -37,13 +37,6 @@ public final class SecurityConstants {
PREFIX_SECURITY_CONFIG + "authentication.";
/**
- * All tls related configuration is prefixed with this:
- * <tt>org.apache.sqoop.security.tls.</tt>
- */
- public static final String PREFIX_TLS_CONFIG =
- PREFIX_SECURITY_CONFIG + "tls.";
-
- /**
* The config specifies the sqoop authentication type (SIMPLE, KERBEROS).
* The default type is SIMPLE
* <tt>org.apache.sqoop.security.authentication.type</tt>.
@@ -165,42 +158,6 @@ public final class SecurityConstants {
PREFIX_AUTHORIZATION_CONFIG + "server_name";
/**
- * The config specifies the whether the http server should use TLS.
- * <tt>org.apache.sqoop.security.tls.enabled</tt>.
- */
- public static final String TLS_ENABLED =
- PREFIX_TLS_CONFIG + "enabled";
-
- /**
- * The config specifies the tls protocol version
- * <tt>org.apache.sqoop.security.tls.protocol</tt>.
- */
- public static final String TLS_PROTOCOL =
- PREFIX_TLS_CONFIG + "protocol";
-
- /**
- * The config specifies the location of the JKS formatted keystore
- * <tt>org.apache.sqoop.security.tls.keystore</tt>.
- */
- public static final String KEYSTORE_LOCATION =
- PREFIX_TLS_CONFIG + "keystore";
-
- /**
- * The config specifies the password of the JKS formatted keystore
- * <tt>org.apache.sqoop.security.tls.keystorepassword</tt>.
- */
- public static final String KEYSTORE_PASSWORD =
- PREFIX_TLS_CONFIG + "keystore_password";
-
- /**
- * The config specifies the password for the private key in the JKS formatted
- * keystore
- * <tt>org.apache.sqoop.security.tls.keymanagerpassword</tt>.
- */
- public static final String KEYMANAGER_PASSWORD =
- PREFIX_TLS_CONFIG + "keymanager_password";
-
- /**
* The config specifies the token kind in delegation token.
*/
public static final String TOKEN_KIND = "sqoop_token_kind";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/dist/src/main/conf/sqoop.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/conf/sqoop.properties b/dist/src/main/conf/sqoop.properties
index 767d3f2..620146d 100755
--- a/dist/src/main/conf/sqoop.properties
+++ b/dist/src/main/conf/sqoop.properties
@@ -180,15 +180,6 @@ org.apache.sqoop.execution.engine=org.apache.sqoop.execution.mapreduce.Mapreduce
#org.apache.sqoop.security.authorization.authentication_provider=org.apache.sqoop.security.authorization.DefaultAuthenticationProvider
#org.apache.sqoop.security.authorization.server_name=SqoopServer1
-#
-# SSL/TLS configuration
-#
-#org.apache.sqoop.security.tls.enabled=false
-#org.apache.sqoop.security.tls.protocol="TLSv1.2"
-#org.apache.sqoop.security.tls.keystore=
-#org.apache.sqoop.security.tls.keystore_password=
-
-
# External connectors load path
# "/path/to/external/connectors/": Add all the connector JARs in the specified folder
#
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java
----------------------------------------------------------------------
diff --git a/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java b/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java
index cb79654..5ea3aa0 100644
--- a/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java
+++ b/docs/src/main/java/org/apache/sqoop/docs/generator/DocPreprocessor.java
@@ -20,7 +20,6 @@ package org.apache.sqoop.docs.generator;
import org.apache.log4j.Logger;
import org.apache.sqoop.docs.generator.plugins.AbstractPlugin;
import org.apache.sqoop.docs.generator.plugins.CopySourceToDestination;
-import org.apache.sqoop.docs.generator.plugins.GenerateConnectorPages;
import org.apache.sqoop.utils.ClassUtils;
import java.util.LinkedList;
@@ -37,7 +36,6 @@ public class DocPreprocessor {
static {
plugins = new LinkedList<>();
plugins.add(CopySourceToDestination.class);
- plugins.add(GenerateConnectorPages.class);
}
public static void main(String []args) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/docs/src/main/java/org/apache/sqoop/docs/generator/plugins/GenerateConnectorPages.java
----------------------------------------------------------------------
diff --git a/docs/src/main/java/org/apache/sqoop/docs/generator/plugins/GenerateConnectorPages.java b/docs/src/main/java/org/apache/sqoop/docs/generator/plugins/GenerateConnectorPages.java
deleted file mode 100644
index 119b76d..0000000
--- a/docs/src/main/java/org/apache/sqoop/docs/generator/plugins/GenerateConnectorPages.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.docs.generator.plugins;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.Logger;
-import org.apache.sqoop.connector.ConnectorHandler;
-import org.apache.sqoop.connector.ConnectorManagerUtils;
-import org.apache.sqoop.connector.spi.SqoopConnector;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.net.URL;
-import java.util.Collections;
-import java.util.List;
-
-public class GenerateConnectorPages extends AbstractPlugin {
-
- private static final Logger LOG = Logger.getLogger(GenerateConnectorPages.class);
-
- @Override
- public void run() {
- try {
- runWithException();
- } catch (IOException e) {
- throw new RuntimeException("Can't generate connector documentation", e);
- }
- }
-
- public void runWithException() throws IOException {
- List<URL> connectorUrls = ConnectorManagerUtils.getConnectorConfigs(Collections.<String>emptySet());
- for (URL url : connectorUrls) {
- SqoopConnector connector = new ConnectorHandler(url).getSqoopConnector();
- LOG.info("Loaded " + connector.getConnectorName() + " from " + url.toString());
-
- String connectorName = connector.getConnectorName();
-
- File outputFile = FileUtils.getFile(getDestination(), "user", "connectors", connectorName + ".rst");
- PrintWriter writer = new PrintWriter(outputFile);
-
- // Writing headline
- writer.println(connectorName);
- writer.println(StringUtils.repeat("=", connectorName.length()));
-
- writer.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
index 60368af..2c4cb7a 100644
--- a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
+++ b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
@@ -19,21 +19,12 @@
package org.apache.sqoop.server;
import org.apache.log4j.Logger;
-import org.apache.sqoop.common.MapContext;
-import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.core.SqoopConfiguration;
import org.apache.sqoop.core.SqoopServer;
import org.apache.sqoop.filter.SqoopAuthenticationFilter;
-import org.apache.sqoop.security.SecurityConstants;
-import org.apache.sqoop.server.common.ServerError;
import org.apache.sqoop.server.v1.*;
-import org.eclipse.jetty.server.HttpConfiguration;
-import org.eclipse.jetty.server.HttpConnectionFactory;
-import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.util.thread.ExecutorThreadPool;
import org.eclipse.jetty.server.ServerConnector;
@@ -67,45 +58,7 @@ public class SqoopJettyServer {
webServer = new Server(threadPool);
// Connector configs
- ServerConnector connector;
-
- MapContext configurationContext = SqoopConfiguration.getInstance().getContext();
-
- if (configurationContext.getBoolean(SecurityConstants.TLS_ENABLED, false)) {
- String keyStorePath = configurationContext.getString(SecurityConstants.KEYSTORE_LOCATION);
- if (keyStorePath == null) {
- throw new SqoopException(ServerError.SERVER_0007);
- }
-
- SslContextFactory sslContextFactory = new SslContextFactory();
- sslContextFactory.setKeyStorePath(keyStorePath);
-
- String protocol = configurationContext.getString(SecurityConstants.TLS_PROTOCOL);
- if (protocol != null && protocol.length() > 0) {
- sslContextFactory.setProtocol(protocol.trim());
- }
-
- String keyStorePassword = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD);
- if (keyStorePassword != null && keyStorePassword.length() > 0) {
- sslContextFactory.setKeyStorePassword(keyStorePassword);
- }
-
- String keyManagerPassword = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD);
- if (keyManagerPassword != null && keyManagerPassword.length() > 0) {
- sslContextFactory.setKeyManagerPassword(keyManagerPassword);
- }
-
- HttpConfiguration https = new HttpConfiguration();
- https.addCustomizer(new SecureRequestCustomizer());
-
- connector = new ServerConnector(webServer,
- new SslConnectionFactory(sslContextFactory, "http/1.1"),
- new HttpConnectionFactory(https));
- } else {
- connector = new ServerConnector(webServer);
- }
-
-
+ ServerConnector connector = new ServerConnector(webServer);
connector.setPort(sqoopJettyContext.getPort());
webServer.addConnector(connector);
webServer.setHandler(createServletContextHandler());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/server/src/main/java/org/apache/sqoop/server/common/ServerError.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/common/ServerError.java b/server/src/main/java/org/apache/sqoop/server/common/ServerError.java
index a644e9f..1b021cf 100644
--- a/server/src/main/java/org/apache/sqoop/server/common/ServerError.java
+++ b/server/src/main/java/org/apache/sqoop/server/common/ServerError.java
@@ -38,9 +38,6 @@ public enum ServerError implements ErrorCode {
/** Entity requested doesn't exist*/
SERVER_0006("Entity requested doesn't exist"),
-
- /** TLS enabled but keystore location not set*/
- SERVER_0007("TLS enabled but keystore location not set"),
;
private final String message;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java b/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java
index b7f1129..36d0712 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShellEnvironment.java
@@ -21,14 +21,12 @@ import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.shell.core.ShellError;
import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.shell.utils.TableDisplayer;
import org.codehaus.groovy.tools.shell.IO;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.MessageFormat;
-import java.util.List;
import java.util.Locale;
import java.util.ResourceBundle;
@@ -59,13 +57,6 @@ public final class ShellEnvironment {
private static boolean interactive = false;
private static long pollTimeout = DEFAULT_POLL_TIMEOUT;
- private static TableDisplayer tableDisplayer = new TableDisplayer(new TableDisplayer.TableDisplayerWriter() {
- @Override
- public void append(String text) {
- print(text);
- }
- });
-
static ResourceBundle resource = ResourceBundle.getBundle(Constants.RESOURCE_NAME, Locale.getDefault());
static SqoopClient client = new SqoopClient(getServerUrl());
static IO io;
@@ -235,10 +226,6 @@ public final class ShellEnvironment {
io.out.printf(format, args);
}
- public static void displayTable(List<String> headers, List<String> ...columns) {
- tableDisplayer.display(headers, columns);
- }
-
// for tests only
public static void cleanup() {
serverHost = DEFAULT_SERVER_HOST;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
index c90fe94..d973499 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowConnectorFunction.java
@@ -27,6 +27,7 @@ import org.apache.sqoop.client.ClientError;
import org.apache.sqoop.common.SqoopException;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.shell.utils.TableDisplayer;
import org.apache.sqoop.validation.Status;
import static org.apache.sqoop.shell.ShellEnvironment.*;
@@ -82,7 +83,7 @@ public class ShowConnectorFunction extends SqoopFunction {
supportedDirections.add(connector.getSupportedDirections().toString());
}
- displayTable(header, uniqueNames, versions, classes, supportedDirections);
+ TableDisplayer.display(header, uniqueNames, versions, classes, supportedDirections);
}
private void showConnectors() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
index 139e756..ebbfe82 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowJobFunction.java
@@ -19,8 +19,12 @@ package org.apache.sqoop.shell;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.shell.utils.TableDisplayer;
import org.apache.sqoop.validation.Status;
import java.text.DateFormat;
@@ -93,7 +97,7 @@ public class ShowJobFunction extends SqoopFunction {
availabilities.add(String.valueOf(job.getEnabled()));
}
- displayTable(header, ids, names, fromConnectors, toConnectors, availabilities);
+ TableDisplayer.display(header, ids, names, fromConnectors, toConnectors, availabilities);
}
private void showJobs(String jArg) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
index 4976c98..25bd1db 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowLinkFunction.java
@@ -19,13 +19,19 @@ package org.apache.sqoop.shell;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.shell.utils.TableDisplayer;
import org.apache.sqoop.validation.Status;
import java.text.DateFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import static org.apache.sqoop.shell.ShellEnvironment.*;
import static org.apache.sqoop.shell.utils.ConfigDisplayer.*;
@@ -80,7 +86,7 @@ public class ShowLinkFunction extends SqoopFunction {
availabilities.add(String.valueOf(link.getEnabled()));
}
- displayTable(header, names, connectorNames, availabilities);
+ TableDisplayer.display(header, names, connectorNames, availabilities);
}
private void showLinks() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java
index 45abf14..a450aaf 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowPrincipalFunction.java
@@ -22,13 +22,13 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.model.MPrincipal;
import org.apache.sqoop.model.MRole;
import org.apache.sqoop.shell.core.Constants;
+import org.apache.sqoop.shell.utils.TableDisplayer;
import org.apache.sqoop.validation.Status;
import java.util.LinkedList;
import java.util.List;
import static org.apache.sqoop.shell.ShellEnvironment.client;
-import static org.apache.sqoop.shell.ShellEnvironment.displayTable;
import static org.apache.sqoop.shell.ShellEnvironment.resourceString;
@SuppressWarnings("serial")
@@ -70,6 +70,6 @@ public class ShowPrincipalFunction extends SqoopFunction {
types.add(principal.getType());
}
- displayTable(header, names, types);
+ TableDisplayer.display(header, names, types);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java
index efd4b19..2cf6972 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowPrivilegeFunction.java
@@ -25,13 +25,13 @@ import org.apache.sqoop.model.MPrivilege;
import org.apache.sqoop.model.MResource;
import org.apache.sqoop.shell.core.Constants;
import org.apache.sqoop.shell.core.ShellError;
+import org.apache.sqoop.shell.utils.TableDisplayer;
import org.apache.sqoop.validation.Status;
import java.util.LinkedList;
import java.util.List;
import static org.apache.sqoop.shell.ShellEnvironment.client;
-import static org.apache.sqoop.shell.ShellEnvironment.displayTable;
import static org.apache.sqoop.shell.ShellEnvironment.resourceString;
@SuppressWarnings("serial")
@@ -108,6 +108,6 @@ public class ShowPrivilegeFunction extends SqoopFunction {
withGrant.add(Boolean.toString(privilege.isWith_grant_option()));
}
- displayTable(header, actions, resourceNames, resourceTypes, withGrant);
+ TableDisplayer.display(header, actions, resourceNames, resourceTypes, withGrant);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java
index 91ceff3..6b61921 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowRoleFunction.java
@@ -24,13 +24,13 @@ import org.apache.sqoop.model.MPrincipal;
import org.apache.sqoop.model.MRole;
import org.apache.sqoop.shell.core.Constants;
import org.apache.sqoop.shell.core.ShellError;
+import org.apache.sqoop.shell.utils.TableDisplayer;
import org.apache.sqoop.validation.Status;
import java.util.LinkedList;
import java.util.List;
import static org.apache.sqoop.shell.ShellEnvironment.client;
-import static org.apache.sqoop.shell.ShellEnvironment.displayTable;
import static org.apache.sqoop.shell.ShellEnvironment.resourceString;
@SuppressWarnings("serial")
@@ -83,6 +83,6 @@ public class ShowRoleFunction extends SqoopFunction {
names.add(role.getName());
}
- displayTable(header, names);
+ TableDisplayer.display(header, names);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java b/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java
index c7f371f..8989913 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/ShowSubmissionFunction.java
@@ -25,6 +25,7 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.shell.core.Constants;
import org.apache.sqoop.shell.utils.SubmissionDisplayer;
+import org.apache.sqoop.shell.utils.TableDisplayer;
import org.apache.sqoop.validation.Status;
import static org.apache.sqoop.shell.ShellEnvironment.*;
@@ -90,7 +91,7 @@ public class ShowSubmissionFunction extends SqoopFunction {
dates.add(submission.getLastUpdateDate().toString());
}
- displayTable(header, jnames, eids, status, dates);
+ TableDisplayer.display(header, jnames, eids, status, dates);
}
private void showSubmissions(String jArg) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java b/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java
index ba3698e..51030d0 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/utils/TableDisplayer.java
@@ -22,43 +22,23 @@ import org.apache.commons.lang.StringUtils;
import java.util.LinkedList;
import java.util.List;
+import static org.apache.sqoop.shell.ShellEnvironment.*;
+
/**
* Display table based data
*/
public class TableDisplayer {
/**
- * Interface that this displayer will use to write out formatted text.
- */
- public interface TableDisplayerWriter {
- /**
- * Print out addition formatted text to the output
- *
- * @param text
- */
- void append(String text);
- }
-
- public TableDisplayer(TableDisplayerWriter writer) {
- this.writer = writer;
- }
-
- /**
- * Writer instance that we should use to write something out
- */
- private TableDisplayerWriter writer;
-
- /**
* Display given columns in nice table structure to given IO object.
*
* @param headers List of headers
* @param columns Array of columns
*/
- public void display(List<String> headers, List<String> ...columns) {
+ public static void display(List<String> headers, List<String> ...columns) {
assert headers != null;
assert columns != null;
assert headers.size() == columns.length;
- assert writer != null;
// Count of columns
int columnCount = headers.size();
@@ -105,7 +85,7 @@ public class TableDisplayer {
*
* @param widths List of widths of each column
*/
- private void drawLine(List<Integer> widths) {
+ private static void drawLine(List<Integer> widths) {
int last = widths.size() - 1;
print("+-");
for(int i = 0; i < widths.size(); i++) {
@@ -122,7 +102,7 @@ public class TableDisplayer {
* @param column All column values
* @return Maximal
*/
- private int getMaximalWidth(String header, List<String> column) {
+ private static int getMaximalWidth(String header, List<String> column) {
assert header != null;
assert column != null;
@@ -143,7 +123,7 @@ public class TableDisplayer {
* @param columns Array with all column values
* @return
*/
- private int getMaximalRows(List<String>... columns) {
+ private static int getMaximalRows(List<String>... columns) {
int max = 0;
for(List<String> column : columns) {
@@ -155,12 +135,7 @@ public class TableDisplayer {
return max;
}
- private void print(String text) {
- writer.append(text);
+ private TableDisplayer() {
+ // Instantiation is prohibited
}
-
- private void println() {
- writer.append("\n");
- }
-
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index 4bac683..1e88b34 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -274,17 +274,17 @@ limitations under the License.
</configuration>
</execution>
<execution>
- <id>server-properties-test</id>
+ <id>connector-loading-test</id>
<goals>
<goal>test</goal>
</goals>
<phase>integration-test</phase>
<configuration>
<suiteXmlFiles>
- <suiteXmlFile>src/test/resources/server-properties-tests-suite.xml</suiteXmlFile>
+ <suiteXmlFile>src/test/resources/connector-loading-tests-suite.xml</suiteXmlFile>
</suiteXmlFiles>
<properties>
- <suitename>server-properties-tests</suitename>
+ <suitename>connector-loading-tests</suitename>
</properties>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java b/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java
index bca4412..83f960b 100644
--- a/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java
+++ b/test/src/main/java/org/apache/sqoop/test/kdc/MiniKdcRunner.java
@@ -18,12 +18,31 @@
package org.apache.sqoop.test.kdc;
import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.lang.reflect.Constructor;
+import java.math.BigInteger;
import java.net.URL;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.Key;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.KeyStore;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
import java.security.Principal;
import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;
+import java.security.SecureRandom;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
import java.util.Collection;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -34,16 +53,19 @@ import java.util.concurrent.Callable;
import javax.security.auth.Subject;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.LoginContext;
+import javax.security.auth.x500.X500Principal;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL;
import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.model.MConnector;
import org.apache.sqoop.test.utils.HdfsUtils;
-import org.apache.sqoop.test.utils.SecurityUtils;
import org.apache.sqoop.test.utils.SqoopUtils;
+import org.bouncycastle.x509.X509V1CertificateGenerator;
/**
* Represents a Minikdc setup. Minikdc should be only used together with
@@ -85,17 +107,17 @@ public class MiniKdcRunner extends KdcRunner {
config.set("dfs.datanode.kerberos.principal", hadoopPrincipal);
config.set("dfs.datanode.keytab.file", hadoopKeytabFile);
String sslKeystoresDir = getTemporaryPath() + "/ssl-keystore";
- String sslConfDir = SqoopUtils.getClasspathDir(MiniKdcRunner.class);
+ String sslConfDir = getClasspathDir(MiniKdcRunner.class);
FileUtils.deleteDirectory(new File(sslKeystoresDir));
FileUtils.forceMkdir(new File(sslKeystoresDir));
- SecurityUtils.setupSSLConfig(sslKeystoresDir, sslConfDir, config, false, true);
- config.set("dfs.https.server.keystore.resource", SecurityUtils.getSSLConfigFileName("ssl-server"));
+ setupSSLConfig(sslKeystoresDir, sslConfDir, config, false, true);
+ config.set("dfs.https.server.keystore.resource", getSSLConfigFileName("ssl-server"));
// Configurations used by both NameNode and DataNode
config.set("dfs.block.access.token.enable", "true");
config.set("dfs.http.policy", "HTTPS_ONLY");
// Configurations used by DFSClient
config.set("dfs.data.transfer.protection", "privacy");
- config.set("dfs.client.https.keystore.resource", SecurityUtils.getSSLConfigFileName("ssl-client"));
+ config.set("dfs.client.https.keystore.resource", getSSLConfigFileName("ssl-client"));
// YARN related configurations
config.set("yarn.resourcemanager.principal", hadoopPrincipal);
@@ -309,4 +331,219 @@ public class MiniKdcRunner extends KdcRunner {
}
}
}
+
+ @SuppressWarnings("rawtypes")
+ private static String getClasspathDir(Class klass) throws Exception {
+ String file = klass.getName();
+ file = file.replace('.', '/') + ".class";
+ URL url = Thread.currentThread().getContextClassLoader().getResource(file);
+ String baseDir = url.toURI().getPath();
+ baseDir = baseDir.substring(0, baseDir.length() - file.length() - 1);
+ return baseDir;
+ }
+
+ /**
+ * Performs complete setup of SSL configuration. This includes keys, certs,
+ * keystores, truststores, the server SSL configuration file,
+ * the client SSL configuration file.
+ *
+ * @param keystoresDir String directory to save keystores
+ * @param sslConfDir String directory to save SSL configuration files
+ * @param conf Configuration
+ * @param useClientCert boolean true to make the client present a cert in the
+ * SSL handshake
+ * @param trustStore boolean true to create truststore, false not to create it
+ */
+ private void setupSSLConfig(String keystoresDir, String sslConfDir,
+ Configuration conf, boolean useClientCert, boolean trustStore)
+ throws Exception {
+ String clientKS = keystoresDir + "/clientKS.jks";
+ String clientPassword = "clientP";
+ String serverKS = keystoresDir + "/serverKS.jks";
+ String serverPassword = "serverP";
+ String trustKS = null;
+ String trustPassword = "trustP";
+
+ File sslClientConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-client"));
+ File sslServerConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-server"));
+
+ Map<String, X509Certificate> certs = new HashMap<String, X509Certificate>();
+
+ if (useClientCert) {
+ KeyPair cKP = generateKeyPair("RSA");
+ X509Certificate cCert = generateCertificate("CN=localhost, O=client", cKP, 30, "SHA1withRSA");
+ createKeyStore(clientKS, clientPassword, "client", cKP.getPrivate(), cCert);
+ certs.put("client", cCert);
+ }
+
+ KeyPair sKP = generateKeyPair("RSA");
+ X509Certificate sCert = generateCertificate("CN=localhost, O=server", sKP, 30, "SHA1withRSA");
+ createKeyStore(serverKS, serverPassword, "server", sKP.getPrivate(), sCert);
+ certs.put("server", sCert);
+
+ if (trustStore) {
+ trustKS = keystoresDir + "/trustKS.jks";
+ createTrustStore(trustKS, trustPassword, certs);
+ }
+
+ Configuration clientSSLConf = createSSLConfig(
+ SSLFactory.Mode.CLIENT, clientKS, clientPassword, clientPassword, trustKS);
+ Configuration serverSSLConf = createSSLConfig(
+ SSLFactory.Mode.SERVER, serverKS, serverPassword, serverPassword, trustKS);
+
+ saveConfig(sslClientConfFile, clientSSLConf);
+ saveConfig(sslServerConfFile, serverSSLConf);
+
+ conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL");
+ conf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile.getName());
+ conf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile.getName());
+ conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, useClientCert);
+ }
+
+ /**
+ * Returns an SSL configuration file name. Under parallel test
+ * execution, this file name is parameterized by a unique ID to ensure that
+ * concurrent tests don't collide on an SSL configuration file.
+ *
+ * @param base the base of the file name
+ * @return SSL configuration file name for base
+ */
+ private static String getSSLConfigFileName(String base) {
+ String testUniqueForkId = System.getProperty("test.unique.fork.id");
+ String fileSuffix = testUniqueForkId != null ? "-" + testUniqueForkId : "";
+ return base + fileSuffix + ".xml";
+ }
+
+ /**
+ * Creates SSL configuration.
+ *
+ * @param mode SSLFactory.Mode mode to configure
+ * @param keystore String keystore file
+ * @param password String store password, or null to avoid setting store
+ * password
+ * @param keyPassword String key password, or null to avoid setting key
+ * password
+ * @param trustKS String truststore file
+ * @return Configuration for SSL
+ */
+ private static Configuration createSSLConfig(SSLFactory.Mode mode,
+ String keystore, String password, String keyPassword, String trustKS) {
+ String trustPassword = "trustP";
+
+ Configuration sslConf = new Configuration(false);
+ if (keystore != null) {
+ sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+ FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), keystore);
+ }
+ if (password != null) {
+ sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+ FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), password);
+ }
+ if (keyPassword != null) {
+ sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+ FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY),
+ keyPassword);
+ }
+ if (trustKS != null) {
+ sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+ FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS);
+ }
+ sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+ FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY),
+ trustPassword);
+ sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
+ FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000");
+
+ return sslConf;
+ }
+
+ private static KeyPair generateKeyPair(String algorithm)
+ throws NoSuchAlgorithmException {
+ KeyPairGenerator keyGen = KeyPairGenerator.getInstance(algorithm);
+ keyGen.initialize(1024);
+ return keyGen.genKeyPair();
+ }
+
+ /**
+ * Create a self-signed X.509 Certificate.
+ *
+ * @param dn the X.509 Distinguished Name, eg "CN=Test, L=London, C=GB"
+ * @param pair the KeyPair
+ * @param days how many days from now the Certificate is valid for
+ * @param algorithm the signing algorithm, eg "SHA1withRSA"
+ * @return the self-signed certificate
+ */
+ private static X509Certificate generateCertificate(String dn, KeyPair pair, int days, String algorithm)
+ throws CertificateEncodingException,
+ InvalidKeyException,
+ IllegalStateException,
+ NoSuchProviderException, NoSuchAlgorithmException, SignatureException{
+ Date from = new Date();
+ Date to = new Date(from.getTime() + days * 86400000l);
+ BigInteger sn = new BigInteger(64, new SecureRandom());
+ KeyPair keyPair = pair;
+ X509V1CertificateGenerator certGen = new X509V1CertificateGenerator();
+ X500Principal dnName = new X500Principal(dn);
+
+ certGen.setSerialNumber(sn);
+ certGen.setIssuerDN(dnName);
+ certGen.setNotBefore(from);
+ certGen.setNotAfter(to);
+ certGen.setSubjectDN(dnName);
+ certGen.setPublicKey(keyPair.getPublic());
+ certGen.setSignatureAlgorithm(algorithm);
+
+ X509Certificate cert = certGen.generate(pair.getPrivate());
+ return cert;
+ }
+
+ private static void createKeyStore(String filename,
+ String password, String alias,
+ Key privateKey, Certificate cert)
+ throws GeneralSecurityException, IOException {
+ KeyStore ks = KeyStore.getInstance("JKS");
+ ks.load(null, null); // initialize
+ ks.setKeyEntry(alias, privateKey, password.toCharArray(),
+ new Certificate[]{cert});
+ saveKeyStore(ks, filename, password);
+ }
+
+ private static <T extends Certificate> void createTrustStore(
+ String filename, String password, Map<String, T> certs)
+ throws GeneralSecurityException, IOException {
+ KeyStore ks = KeyStore.getInstance("JKS");
+ ks.load(null, null); // initialize
+ for (Map.Entry<String, T> cert : certs.entrySet()) {
+ ks.setCertificateEntry(cert.getKey(), cert.getValue());
+ }
+ saveKeyStore(ks, filename, password);
+ }
+
+ private static void saveKeyStore(KeyStore ks, String filename,
+ String password)
+ throws GeneralSecurityException, IOException {
+ FileOutputStream out = new FileOutputStream(filename);
+ try {
+ ks.store(out, password.toCharArray());
+ } finally {
+ out.close();
+ }
+ }
+
+ /**
+ * Saves configuration to a file.
+ *
+ * @param file File to save
+ * @param conf Configuration contents to write to file
+ * @throws IOException if there is an I/O error saving the file
+ */
+ private static void saveConfig(File file, Configuration conf)
+ throws IOException {
+ Writer writer = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
+ try {
+ conf.writeXml(writer);
+ } finally {
+ writer.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java
deleted file mode 100644
index 35b9b7d..0000000
--- a/test/src/main/java/org/apache/sqoop/test/utils/SecurityUtils.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.test.utils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory;
-import org.apache.hadoop.security.ssl.SSLFactory;
-import org.bouncycastle.x509.X509V1CertificateGenerator;
-
-import javax.security.auth.x500.X500Principal;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.math.BigInteger;
-import java.security.GeneralSecurityException;
-import java.security.InvalidKeyException;
-import java.security.Key;
-import java.security.KeyPair;
-import java.security.KeyPairGenerator;
-import java.security.KeyStore;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.SecureRandom;
-import java.security.SignatureException;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateEncodingException;
-import java.security.cert.X509Certificate;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-
-public class SecurityUtils {
-
- public static final String CLIENT_KEYSTORE = "clientKS.jks";
- public static final String SERVER_KEYSTORE = "serverKS.jks";
- public static final String TRUSTSTORE = "trustKS.jks";
-
- public static final String CLIENT_KEY_PASSWORD = "client_key";
- public static final String CLIENT_KEY_STORE_PASSWORD = "client_keystore";
- public static final String SERVER_KEY_PASSWORD = "server_key";
- public static final String SERVER_KEY_STORE_PASSWORD = "server_keystore";
-
- /**
- * Performs complete setup of SSL configuration. This includes keys, certs,
- * keystores, truststores, the server SSL configuration file,
- * the client SSL configuration file.
- *
- * @param keystoresDir String directory to save keystores
- * @param sslConfDir String directory to save SSL configuration files
- * @param conf Configuration
- * @param useClientCert boolean true to make the client present a cert in the
- * SSL handshake
- * @param trustStore boolean true to create truststore, false not to create it
- */
- public static X509Certificate setupSSLConfig(String keystoresDir, String sslConfDir,
- Configuration conf, boolean useClientCert, boolean trustStore)
- throws Exception {
- String clientKeyStorePath = keystoresDir + "/" + CLIENT_KEYSTORE;
- String serverKeyStorePath = keystoresDir + "/" + SERVER_KEYSTORE;
- String serverPassword = "serverP";
- String trustKS = null;
- String trustPassword = "trustP";
-
- File sslClientConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-client"));
- File sslServerConfFile = new File(sslConfDir, getSSLConfigFileName("ssl-server"));
-
- Map<String, X509Certificate> certs = new HashMap<String, X509Certificate>();
-
- String hostname = SqoopUtils.getLocalHostName();
-
- if (useClientCert) {
- KeyPair cKP = generateKeyPair("RSA");
- X509Certificate cCert = generateCertificate("CN=" + hostname + ", O=client", cKP, 30, "SHA1withRSA");
- createKeyStore(clientKeyStorePath, CLIENT_KEY_PASSWORD, CLIENT_KEY_STORE_PASSWORD, "client", cKP.getPrivate(), cCert);
- certs.put("client", cCert);
- }
-
- KeyPair sKP = generateKeyPair("RSA");
- X509Certificate sCert = generateCertificate("CN=" + hostname + ", O=server", sKP, 30, "SHA1withRSA");
- createKeyStore(serverKeyStorePath, SERVER_KEY_PASSWORD, SERVER_KEY_STORE_PASSWORD, "server", sKP.getPrivate(), sCert);
- certs.put("server", sCert);
-
- if (trustStore) {
- trustKS = keystoresDir + TRUSTSTORE;
- createTrustStore(trustKS, trustPassword, certs);
- }
-
- Configuration clientSSLConf = createSSLConfig(
- SSLFactory.Mode.CLIENT, clientKeyStorePath, CLIENT_KEY_STORE_PASSWORD, CLIENT_KEY_PASSWORD, trustKS);
- Configuration serverSSLConf = createSSLConfig(
- SSLFactory.Mode.SERVER, serverKeyStorePath, SERVER_KEY_STORE_PASSWORD, SERVER_KEY_PASSWORD, trustKS);
-
- saveConfig(sslClientConfFile, clientSSLConf);
- saveConfig(sslServerConfFile, serverSSLConf);
-
- conf.set(SSLFactory.SSL_HOSTNAME_VERIFIER_KEY, "ALLOW_ALL");
- conf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile.getName());
- conf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile.getName());
- conf.setBoolean(SSLFactory.SSL_REQUIRE_CLIENT_CERT_KEY, useClientCert);
-
- return sCert;
- }
-
- /**
- * Returns an SSL configuration file name. Under parallel test
- * execution, this file name is parameterized by a unique ID to ensure that
- * concurrent tests don't collide on an SSL configuration file.
- *
- * @param base the base of the file name
- * @return SSL configuration file name for base
- */
- public static String getSSLConfigFileName(String base) {
- String testUniqueForkId = System.getProperty("test.unique.fork.id");
- String fileSuffix = testUniqueForkId != null ? "-" + testUniqueForkId : "";
- return base + fileSuffix + ".xml";
- }
-
- /**
- * Creates SSL configuration.
- *
- * @param mode SSLFactory.Mode mode to configure
- * @param keystore String keystore file
- * @param keyStorePassword String store password, or null to avoid setting store
- * password
- * @param keyPassword String key password, or null to avoid setting key
- * password
- * @param trustKS String truststore file
- * @return Configuration for SSL
- */
- public static Configuration createSSLConfig(SSLFactory.Mode mode,
- String keystore, String keyStorePassword,
- String keyPassword, String trustKS) {
- String trustPassword = "trustP";
-
- Configuration sslConf = new Configuration(false);
- if (keystore != null) {
- sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
- FileBasedKeyStoresFactory.SSL_KEYSTORE_LOCATION_TPL_KEY), keystore);
- }
-
- if (keyStorePassword != null) {
- sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
- FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), keyStorePassword);
- }
- if (keyPassword != null) {
- sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
- FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY),
- keyPassword);
- }
- if (trustKS != null) {
- sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
- FileBasedKeyStoresFactory.SSL_TRUSTSTORE_LOCATION_TPL_KEY), trustKS);
- }
- sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
- FileBasedKeyStoresFactory.SSL_TRUSTSTORE_PASSWORD_TPL_KEY),
- trustPassword);
- sslConf.set(FileBasedKeyStoresFactory.resolvePropertyName(mode,
- FileBasedKeyStoresFactory.SSL_TRUSTSTORE_RELOAD_INTERVAL_TPL_KEY), "1000");
-
- return sslConf;
- }
-
- public static KeyPair generateKeyPair(String algorithm)
- throws NoSuchAlgorithmException {
- KeyPairGenerator keyGen = KeyPairGenerator.getInstance(algorithm);
- keyGen.initialize(1024);
- return keyGen.genKeyPair();
- }
-
- /**
- * Create a self-signed X.509 Certificate.
- *
- * @param dn the X.509 Distinguished Name, eg "CN=Test, L=London, C=GB"
- * @param pair the KeyPair
- * @param days how many days from now the Certificate is valid for
- * @param algorithm the signing algorithm, eg "SHA1withRSA"
- * @return the self-signed certificate
- */
- public static X509Certificate generateCertificate(String dn, KeyPair pair, int days, String algorithm)
- throws CertificateEncodingException,
- InvalidKeyException,
- IllegalStateException,
- NoSuchProviderException, NoSuchAlgorithmException, SignatureException {
- Date from = new Date();
- Date to = new Date(from.getTime() + days * 86400000l);
- BigInteger sn = new BigInteger(64, new SecureRandom());
- KeyPair keyPair = pair;
- X509V1CertificateGenerator certGen = new X509V1CertificateGenerator();
- X500Principal dnName = new X500Principal(dn);
-
- certGen.setSerialNumber(sn);
- certGen.setIssuerDN(dnName);
- certGen.setNotBefore(from);
- certGen.setNotAfter(to);
- certGen.setSubjectDN(dnName);
- certGen.setPublicKey(keyPair.getPublic());
- certGen.setSignatureAlgorithm(algorithm);
-
- X509Certificate cert = certGen.generate(pair.getPrivate());
- return cert;
- }
-
- public static void createKeyStore(String filename,
- String keyPassword, String keyStorePassword,
- String alias, Key privateKey, Certificate cert)
- throws GeneralSecurityException, IOException {
- KeyStore ks = KeyStore.getInstance("JKS");
- ks.load(null, null); // initialize
- ks.setKeyEntry(alias, privateKey, keyPassword.toCharArray(),
- new Certificate[]{cert});
- saveKeyStore(ks, filename, keyStorePassword);
- }
-
- public static <T extends Certificate> void createTrustStore(
- String filename, String password, Map<String, T> certs)
- throws GeneralSecurityException, IOException {
- KeyStore ks = KeyStore.getInstance("JKS");
- ks.load(null, null); // initialize
- for (Map.Entry<String, T> cert : certs.entrySet()) {
- ks.setCertificateEntry(cert.getKey(), cert.getValue());
- }
- saveKeyStore(ks, filename, password);
- }
-
- public static void saveKeyStore(KeyStore ks, String filename,
- String password)
- throws GeneralSecurityException, IOException {
- FileOutputStream out = new FileOutputStream(filename);
- try {
- ks.store(out, password.toCharArray());
- } finally {
- out.close();
- }
- }
-
- /**
- * Saves configuration to a file.
- *
- * @param file File to save
- * @param conf Configuration contents to write to file
- * @throws IOException if there is an I/O error saving the file
- */
- public static void saveConfig(File file, Configuration conf)
- throws IOException {
- Writer writer = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
- try {
- conf.writeXml(writer);
- } finally {
- writer.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java
index 3e0566f..6614b19 100644
--- a/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java
+++ b/test/src/main/java/org/apache/sqoop/test/utils/SqoopUtils.java
@@ -18,7 +18,6 @@
package org.apache.sqoop.test.utils;
import java.net.InetAddress;
-import java.net.URL;
import java.net.UnknownHostException;
import java.util.Locale;
import java.util.Random;
@@ -45,7 +44,7 @@ public class SqoopUtils {
object.setName(prefix + rand.nextLong());
}
- // Retrieve the FQDN of the current host
+ //Retrieve the FQDN of the current host
public static String getLocalHostName() {
String fqdn;
try {
@@ -55,26 +54,4 @@ public class SqoopUtils {
}
return fqdn;
}
-
- // Retrieve the IP address of the current host
- public static String getLocalIpAddress() {
- String address;
- try {
- address = InetAddress.getLocalHost().getHostAddress();
- } catch (UnknownHostException e1) {
- address = "127.0.0.1";
- }
- return address;
- }
-
-
- @SuppressWarnings("rawtypes")
- public static String getClasspathDir(Class klass) throws Exception {
- String file = klass.getName();
- file = file.replace('.', '/') + ".class";
- URL url = Thread.currentThread().getContextClassLoader().getResource(file);
- String baseDir = url.toURI().getPath();
- baseDir = baseDir.substring(0, baseDir.length() - file.length() - 1);
- return baseDir;
- }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
new file mode 100644
index 0000000..7f0575b
--- /dev/null
+++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.integration.connectorloading;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.core.ConfigurationConstants;
+import org.apache.sqoop.test.infrastructure.Infrastructure;
+import org.apache.sqoop.test.infrastructure.SqoopTestCase;
+import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider;
+import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider;
+import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
+import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Test(groups = "no-real-cluster")
+@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class})
+public class BlacklistedConnectorTest extends SqoopTestCase {
+ private SqoopMiniCluster sqoopMiniCluster;
+
+ public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster {
+ public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception {
+ super(temporaryPath, configuration);
+ }
+
+ @Override
+ protected Map<String, String> getBlacklistedConnectorConfiguration() {
+ Map<String, String> properties = new HashMap<>();
+
+ properties.put(ConfigurationConstants.BLACKLISTED_CONNECTORS, "fake-connector:generic-jdbc-connector");
+ return properties;
+ }
+ }
+
+ @BeforeMethod(dependsOnMethods = { "init" })
+ public void startSqoopMiniCluster() throws Exception {
+ // And use them for new Derby repo instance
+ sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments(super
+ .getTemporaryPath(), getTestName()), getHadoopConf());
+ KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class);
+ if (kdcProvider != null) {
+ sqoopMiniCluster.setKdc(kdcProvider.getInstance());
+ }
+
+ // Start server
+ sqoopMiniCluster.start();
+
+ // Initialize Sqoop Client API
+ initSqoopClient(sqoopMiniCluster.getServerUrl());
+ }
+
+ @Test(expectedExceptions = {SqoopException.class})
+ public void testCreateLinkWithNonexistantConnector() throws Exception {
+ getClient().createLink("generic-jdbc-connector");
+ }
+
+ @AfterMethod
+ public void stopCluster() throws Exception {
+ sqoopMiniCluster.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
new file mode 100644
index 0000000..87f0eb1
--- /dev/null
+++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.integration.connectorloading;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.core.ConfigurationConstants;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.test.infrastructure.Infrastructure;
+import org.apache.sqoop.test.infrastructure.SqoopTestCase;
+import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider;
+import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider;
+import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider;
+import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
+import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
+import org.apache.sqoop.test.utils.ConnectorUtils;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Test(groups = "no-real-cluster")
+@Infrastructure(dependencies = {KdcInfrastructureProvider.class, DatabaseInfrastructureProvider.class, HadoopInfrastructureProvider.class})
+public class ClasspathTest extends SqoopTestCase {
+
+ private static final String TEST_CONNECTOR_JAR_NAME = "test-connector.jar";
+ private static final String TEST_DEPENDENCY_JAR_NAME = "test-dependency.jar";
+
+ private static final String[] CONNECTOR_SOURCE_FILES = {
+ "TestConnector/TestConnector.java",
+ "TestConnector/TestLinkConfiguration.java",
+ "TestConnector/TestLoader.java",
+ "TestConnector/TestToDestroyer.java",
+ "TestConnector/TestToInitializer.java",
+ "TestConnector/TestToJobConfiguration.java"
+ };
+
+ private static final String[] CONNECTOR_DEPENDENCY_SOURCE_FILES = {
+ "TestConnector/TestDependency.java"
+ };
+
+ private static final String[] CONNECTOR_PROPERTY_FILES = {
+ "TestConnector/sqoopconnector.properties"
+ };
+
+ private ClassLoader classLoader;
+ private SqoopMiniCluster sqoopMiniCluster;
+
+ public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster {
+
+ private String extraClasspath;
+ private String jobExtraClasspath;
+
+ public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration, String extraClasspath, String jobExtraClasspath) throws Exception {
+ super(temporaryPath, configuration);
+ this.extraClasspath = extraClasspath;
+ this.jobExtraClasspath = jobExtraClasspath;
+ }
+
+ @Override
+ protected Map<String, String> getClasspathConfiguration() {
+ Map<String, String> properties = new HashMap<>();
+
+ if (extraClasspath != null) {
+ properties.put(ConfigurationConstants.CLASSPATH, extraClasspath);
+ }
+ if (jobExtraClasspath != null) {
+ properties.put(ConfigurationConstants.JOB_CLASSPATH, jobExtraClasspath);
+ }
+
+
+ return properties;
+ }
+ }
+
+ public void startSqoopMiniCluster(String extraClasspath, String jobExtraClasspath) throws Exception {
+ // And use them for new Derby repo instance
+ sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments
+ (super.getTemporaryPath(), getTestName()), getHadoopConf(), extraClasspath, jobExtraClasspath);
+ KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class);
+ if (kdcProvider != null) {
+ sqoopMiniCluster.setKdc(kdcProvider.getInstance());
+ }
+
+ // Start server
+ sqoopMiniCluster.start();
+
+ // Initialize Sqoop Client API
+ initSqoopClient(sqoopMiniCluster.getServerUrl());
+ }
+
+ @BeforeMethod
+ public void captureClasspath() {
+ classLoader = Thread.currentThread().getContextClassLoader();
+ }
+
+ @AfterMethod
+ public void restoreClasspath(){
+ Thread.currentThread().setContextClassLoader(classLoader);
+ }
+
+ @Test
+ public void testClasspathSqoopProperties() throws Exception {
+ Map<String, String> jarMap = ConnectorUtils.compileTestConnectorAndDependency(
+ CONNECTOR_SOURCE_FILES,
+ CONNECTOR_DEPENDENCY_SOURCE_FILES,
+ CONNECTOR_PROPERTY_FILES,
+ TEST_CONNECTOR_JAR_NAME,
+ TEST_DEPENDENCY_JAR_NAME,
+ false);
+ startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), jarMap.get
+ (TEST_DEPENDENCY_JAR_NAME));
+ createAndLoadTableCities();
+
+ MJob job = prepareJob();
+
+ prepareDriverConfig(job);
+
+ saveJob(job);
+
+ executeJob(job);
+
+ stopSqoop();
+ ConnectorUtils.deleteJars(jarMap);
+ }
+
+ @Test
+ public void testClasspathDriverInput() throws Exception{
+ Map<String, String> jarMap = ConnectorUtils.compileTestConnectorAndDependency(
+ CONNECTOR_SOURCE_FILES,
+ CONNECTOR_DEPENDENCY_SOURCE_FILES,
+ CONNECTOR_PROPERTY_FILES,
+ TEST_CONNECTOR_JAR_NAME,
+ TEST_DEPENDENCY_JAR_NAME,
+ false);
+ startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), null);
+ createAndLoadTableCities();
+
+ MJob job = prepareJob();
+
+ MDriverConfig driverConfig = prepareDriverConfig(job);
+
+ List<String> extraJars = new ArrayList<>();
+ extraJars.add("file:" + jarMap.get(TEST_DEPENDENCY_JAR_NAME));
+ driverConfig.getListInput("jarConfig.extraJars").setValue(extraJars);
+
+ saveJob(job);
+
+ executeJob(job);
+
+ stopSqoop();
+ ConnectorUtils.deleteJars(jarMap);
+ }
+
+ private MJob prepareJob() {
+ MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector");
+ fillRdbmsLinkConfig(rdbmsConnection);
+ saveLink(rdbmsConnection);
+
+ MLink testConnection = getClient().createLink("test-connector");
+ saveLink(testConnection);
+
+ MJob job = getClient().createJob(rdbmsConnection.getName(), testConnection.getName());
+
+ fillRdbmsFromConfig(job, "id");
+
+ return job;
+ }
+
+ private MDriverConfig prepareDriverConfig(MJob job) {
+ MDriverConfig driverConfig = job.getDriverConfig();
+ driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
+
+ return driverConfig;
+ }
+
+ private void stopSqoop() throws Exception {
+ sqoopMiniCluster.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java
new file mode 100644
index 0000000..a82a4da
--- /dev/null
+++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ConnectorClasspathIsolationTest.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.integration.connectorloading;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.core.ConfigurationConstants;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.test.infrastructure.Infrastructure;
+import org.apache.sqoop.test.infrastructure.SqoopTestCase;
+import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider;
+import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider;
+import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
+import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
+import org.apache.sqoop.test.utils.ConnectorUtils;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+@Test(groups = "no-real-cluster")
+@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class})
+public class ConnectorClasspathIsolationTest extends SqoopTestCase {
+
+ private static final String TEST_FROM_CONNECTOR_JAR_NAME = "test-from-connector.jar";
+ private static final String TEST_TO_CONNECTOR_JAR_NAME = "test-to-connector.jar";
+ private static final String TEST_FROM_DEPENDENCY_JAR_NAME = "test-from-dependency.jar";
+ private static final String TEST_TO_DEPENDENCY_JAR_NAME = "test-to-dependency.jar";
+
+ private static final String[] FROM_CONNECTOR_SOURCE_FILES = {
+ "TestConnectorClasspathIsolation/from/TestFromConnector.java",
+ "TestConnectorClasspathIsolation/from/TestExtractor.java",
+ "TestConnectorClasspathIsolation/from/TestFromDestroyer.java",
+ "TestConnectorClasspathIsolation/from/TestFromInitializer.java",
+ "TestConnectorClasspathIsolation/from/TestFromJobConfiguration.java",
+ "TestConnectorClasspathIsolation/from/TestPartition.java",
+ "TestConnectorClasspathIsolation/from/TestPartitioner.java",
+ "TestConnectorClasspathIsolation/from/TestFromLinkConfiguration.java"
+ };
+
+ private static final String[] FROM_CONNECTOR_DEPENDENCY_SOURCE_FILES = {
+ "TestConnectorClasspathIsolation/from/TestClasspathIsolation.java"
+ };
+
+ private static final String[] FROM_CONNECTOR_PROPERTY_FILES = {
+ "TestConnectorClasspathIsolation/from/sqoopconnector.properties"
+ };
+
+ private static final String[] TO_CONNECTOR_SOURCE_FILES = {
+ "TestConnectorClasspathIsolation/to/TestToConnector.java",
+ "TestConnectorClasspathIsolation/to/TestLoader.java",
+ "TestConnectorClasspathIsolation/to/TestToDestroyer.java",
+ "TestConnectorClasspathIsolation/to/TestToInitializer.java",
+ "TestConnectorClasspathIsolation/to/TestToJobConfiguration.java",
+ "TestConnectorClasspathIsolation/to/TestToLinkConfiguration.java"
+ };
+
+ private static final String[] TO_CONNECTOR_DEPENDENCY_SOURCE_FILES = {
+ "TestConnectorClasspathIsolation/to/TestClasspathIsolation.java"
+ };
+
+ private static final String[] TO_CONNECTOR_PROPERTY_FILES = {
+ "TestConnectorClasspathIsolation/to/sqoopconnector.properties"
+ };
+
+ private ClassLoader classLoader;
+ private SqoopMiniCluster sqoopMiniCluster;
+
+ public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster {
+
+ private String extraClasspath;
+
+ public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration, String extraClasspath) throws Exception {
+ super(temporaryPath, configuration);
+ this.extraClasspath = extraClasspath;
+ }
+
+ @Override
+ protected Map<String, String> getClasspathConfiguration() {
+ Map<String, String> properties = new HashMap<>();
+
+ if (extraClasspath != null) {
+ properties.put(ConfigurationConstants.CLASSPATH, extraClasspath);
+ }
+
+ return properties;
+ }
+ }
+
+ public void startSqoopMiniCluster(String extraClasspath) throws Exception {
+ // And use them for new Derby repo instance
+ sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments(super.getTemporaryPath(), getTestName()), getHadoopConf(), extraClasspath);
+ KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class);
+ if (kdcProvider != null) {
+ sqoopMiniCluster.setKdc(kdcProvider.getInstance());
+ }
+
+ // Start server
+ sqoopMiniCluster.start();
+
+ // Initialize Sqoop Client API
+ initSqoopClient(sqoopMiniCluster.getServerUrl());
+ }
+
+ @BeforeMethod
+ public void captureClasspath() {
+ classLoader = Thread.currentThread().getContextClassLoader();
+ }
+
+ @AfterMethod
+ public void restoreClasspath(){
+ Thread.currentThread().setContextClassLoader(classLoader);
+ }
+
+ @Test
+ public void testConnectorClasspathIsolation() throws Exception {
+ Map<String, String> fromConnectorJarMap = ConnectorUtils.compileTestConnectorAndDependency(
+ FROM_CONNECTOR_SOURCE_FILES,
+ FROM_CONNECTOR_DEPENDENCY_SOURCE_FILES,
+ FROM_CONNECTOR_PROPERTY_FILES,
+ TEST_FROM_CONNECTOR_JAR_NAME,
+ TEST_FROM_DEPENDENCY_JAR_NAME,
+ true);
+ Map<String, String> toConnectorJarMap = ConnectorUtils.compileTestConnectorAndDependency(
+ TO_CONNECTOR_SOURCE_FILES,
+ TO_CONNECTOR_DEPENDENCY_SOURCE_FILES,
+ TO_CONNECTOR_PROPERTY_FILES,
+ TEST_TO_CONNECTOR_JAR_NAME,
+ TEST_TO_DEPENDENCY_JAR_NAME,
+ true);
+ startSqoopMiniCluster(
+ StringUtils.join(Arrays.asList(fromConnectorJarMap.get(TEST_FROM_CONNECTOR_JAR_NAME), toConnectorJarMap.get(TEST_TO_CONNECTOR_JAR_NAME)), ":"));
+
+ MJob job = prepareJob();
+
+ prepareDriverConfig(job);
+
+ saveJob(job);
+
+ executeJob(job);
+
+ stopSqoop();
+ ConnectorUtils.deleteJars(fromConnectorJarMap);
+ }
+
+ private MJob prepareJob() {
+ MLink rdbmsConnection = getClient().createLink("test-from-connector");
+ saveLink(rdbmsConnection);
+
+ MLink testConnection = getClient().createLink("test-to-connector");
+ saveLink(testConnection);
+
+ MJob job = getClient().createJob(rdbmsConnection.getName(), testConnection.getName());
+
+ return job;
+ }
+
+ private MDriverConfig prepareDriverConfig(MJob job) {
+ MDriverConfig driverConfig = job.getDriverConfig();
+ driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
+
+ return driverConfig;
+ }
+
+ private void stopSqoop() throws Exception {
+ sqoopMiniCluster.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java b/test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java
deleted file mode 100644
index 438e182..0000000
--- a/test/src/test/java/org/apache/sqoop/integration/serverproperties/BlacklistedConnectorTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.integration.serverproperties;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.sqoop.common.SqoopException;
-import org.apache.sqoop.core.ConfigurationConstants;
-import org.apache.sqoop.test.infrastructure.Infrastructure;
-import org.apache.sqoop.test.infrastructure.SqoopTestCase;
-import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider;
-import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider;
-import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
-import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
-import org.apache.sqoop.test.utils.HdfsUtils;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-@Test(groups = "no-real-cluster")
-@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class})
-public class BlacklistedConnectorTest extends SqoopTestCase {
- private SqoopMiniCluster sqoopMiniCluster;
-
- public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster {
- public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception {
- super(temporaryPath, configuration);
- }
-
- @Override
- protected Map<String, String> getBlacklistedConnectorConfiguration() {
- Map<String, String> properties = new HashMap<>();
-
- properties.put(ConfigurationConstants.BLACKLISTED_CONNECTORS, "fake-connector:generic-jdbc-connector");
- return properties;
- }
- }
-
- @BeforeMethod(dependsOnMethods = { "init" })
- public void startSqoopMiniCluster() throws Exception {
- // And use them for new Derby repo instance
- sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments(super
- .getTemporaryPath(), getTestName()), getHadoopConf());
- KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class);
- if (kdcProvider != null) {
- sqoopMiniCluster.setKdc(kdcProvider.getInstance());
- }
-
- // Start server
- sqoopMiniCluster.start();
-
- // Initialize Sqoop Client API
- initSqoopClient(sqoopMiniCluster.getServerUrl());
- }
-
- @Test(expectedExceptions = {SqoopException.class})
- public void testCreateLinkWithNonexistantConnector() throws Exception {
- getClient().createLink("generic-jdbc-connector");
- }
-
- @AfterMethod
- public void stopCluster() throws Exception {
- sqoopMiniCluster.stop();
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5fd80fd5/test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java b/test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java
deleted file mode 100644
index 1a33230..0000000
--- a/test/src/test/java/org/apache/sqoop/integration/serverproperties/ClasspathTest.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.sqoop.integration.serverproperties;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.sqoop.core.ConfigurationConstants;
-import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MLink;
-import org.apache.sqoop.test.infrastructure.Infrastructure;
-import org.apache.sqoop.test.infrastructure.SqoopTestCase;
-import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider;
-import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider;
-import org.apache.sqoop.test.infrastructure.providers.KdcInfrastructureProvider;
-import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
-import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
-import org.apache.sqoop.test.utils.ConnectorUtils;
-import org.apache.sqoop.test.utils.HdfsUtils;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-@Test(groups = "no-real-cluster")
-@Infrastructure(dependencies = {KdcInfrastructureProvider.class, DatabaseInfrastructureProvider.class, HadoopInfrastructureProvider.class})
-public class ClasspathTest extends SqoopTestCase {
-
- private static final String TEST_CONNECTOR_JAR_NAME = "test-connector.jar";
- private static final String TEST_DEPENDENCY_JAR_NAME = "test-dependency.jar";
-
- private static final String[] CONNECTOR_SOURCE_FILES = {
- "TestConnector/TestConnector.java",
- "TestConnector/TestLinkConfiguration.java",
- "TestConnector/TestLoader.java",
- "TestConnector/TestToDestroyer.java",
- "TestConnector/TestToInitializer.java",
- "TestConnector/TestToJobConfiguration.java"
- };
-
- private static final String[] CONNECTOR_DEPENDENCY_SOURCE_FILES = {
- "TestConnector/TestDependency.java"
- };
-
- private static final String[] CONNECTOR_PROPERTY_FILES = {
- "TestConnector/sqoopconnector.properties"
- };
-
- private ClassLoader classLoader;
- private SqoopMiniCluster sqoopMiniCluster;
-
- public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster {
-
- private String extraClasspath;
- private String jobExtraClasspath;
-
- public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration, String extraClasspath, String jobExtraClasspath) throws Exception {
- super(temporaryPath, configuration);
- this.extraClasspath = extraClasspath;
- this.jobExtraClasspath = jobExtraClasspath;
- }
-
- @Override
- protected Map<String, String> getClasspathConfiguration() {
- Map<String, String> properties = new HashMap<>();
-
- if (extraClasspath != null) {
- properties.put(ConfigurationConstants.CLASSPATH, extraClasspath);
- }
- if (jobExtraClasspath != null) {
- properties.put(ConfigurationConstants.JOB_CLASSPATH, jobExtraClasspath);
- }
-
-
- return properties;
- }
- }
-
- public void startSqoopMiniCluster(String extraClasspath, String jobExtraClasspath) throws Exception {
- // And use them for new Derby repo instance
- sqoopMiniCluster = new DerbySqoopMiniCluster(HdfsUtils.joinPathFragments
- (super.getTemporaryPath(), getTestName()), getHadoopConf(), extraClasspath, jobExtraClasspath);
- KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class);
- if (kdcProvider != null) {
- sqoopMiniCluster.setKdc(kdcProvider.getInstance());
- }
-
- // Start server
- sqoopMiniCluster.start();
-
- // Initialize Sqoop Client API
- initSqoopClient(sqoopMiniCluster.getServerUrl());
- }
-
- @BeforeMethod
- public void captureClasspath() {
- classLoader = Thread.currentThread().getContextClassLoader();
- }
-
- @AfterMethod
- public void restoreClasspath(){
- Thread.currentThread().setContextClassLoader(classLoader);
- }
-
- @Test
- public void testClasspathSqoopProperties() throws Exception {
- Map<String, String> jarMap = ConnectorUtils.compileTestConnectorAndDependency(
- CONNECTOR_SOURCE_FILES,
- CONNECTOR_DEPENDENCY_SOURCE_FILES,
- CONNECTOR_PROPERTY_FILES,
- TEST_CONNECTOR_JAR_NAME,
- TEST_DEPENDENCY_JAR_NAME,
- false);
- startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), jarMap.get
- (TEST_DEPENDENCY_JAR_NAME));
- createAndLoadTableCities();
-
- MJob job = prepareJob();
-
- prepareDriverConfig(job);
-
- saveJob(job);
-
- executeJob(job);
-
- stopSqoop();
- ConnectorUtils.deleteJars(jarMap);
- }
-
- @Test
- public void testClasspathDriverInput() throws Exception{
- Map<String, String> jarMap = ConnectorUtils.compileTestConnectorAndDependency(
- CONNECTOR_SOURCE_FILES,
- CONNECTOR_DEPENDENCY_SOURCE_FILES,
- CONNECTOR_PROPERTY_FILES,
- TEST_CONNECTOR_JAR_NAME,
- TEST_DEPENDENCY_JAR_NAME,
- false);
- startSqoopMiniCluster(jarMap.get(TEST_CONNECTOR_JAR_NAME), null);
- createAndLoadTableCities();
-
- MJob job = prepareJob();
-
- MDriverConfig driverConfig = prepareDriverConfig(job);
-
- List<String> extraJars = new ArrayList<>();
- extraJars.add("file:" + jarMap.get(TEST_DEPENDENCY_JAR_NAME));
- driverConfig.getListInput("jarConfig.extraJars").setValue(extraJars);
-
- saveJob(job);
-
- executeJob(job);
-
- stopSqoop();
- ConnectorUtils.deleteJars(jarMap);
- }
-
- private MJob prepareJob() {
- MLink rdbmsConnection = getClient().createLink("generic-jdbc-connector");
- fillRdbmsLinkConfig(rdbmsConnection);
- saveLink(rdbmsConnection);
-
- MLink testConnection = getClient().createLink("test-connector");
- saveLink(testConnection);
-
- MJob job = getClient().createJob(rdbmsConnection.getName(), testConnection.getName());
-
- fillRdbmsFromConfig(job, "id");
-
- return job;
- }
-
- private MDriverConfig prepareDriverConfig(MJob job) {
- MDriverConfig driverConfig = job.getDriverConfig();
- driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
-
- return driverConfig;
- }
-
- private void stopSqoop() throws Exception {
- sqoopMiniCluster.stop();
- }
-}