You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/12/08 08:11:56 UTC
sqoop git commit: SQOOP-2720: Sqoop2: Allow the integration tests to
be run against a real cluster
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 794adbcf3 -> 6c4743de1
SQOOP-2720: Sqoop2: Allow the integration tests to be run against a real cluster
(Abraham Fine via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/6c4743de
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/6c4743de
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/6c4743de
Branch: refs/heads/sqoop2
Commit: 6c4743de158e4ced43e57703a8bcee5fca6257c6
Parents: 794adbc
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Tue Dec 8 08:11:26 2015 +0100
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Tue Dec 8 08:11:26 2015 +0100
----------------------------------------------------------------------
.../org/apache/sqoop/client/SqoopClient.java | 17 +++++++
pom.xml | 51 +++++++++++++++-----
.../test/minicluster/RealSqoopCluster.java | 16 +++++-
.../sqoop/test/testcases/JettyTestCase.java | 21 +++++---
.../connector/hive/FromRDBMSToKiteHiveTest.java | 2 +-
.../connector/kafka/FromHDFSToKafkaTest.java | 2 +-
.../connector/kafka/FromRDBMSToKafkaTest.java | 2 +-
.../BlacklistedConnectorTest.java | 1 +
.../connectorloading/ClasspathTest.java | 1 +
.../derby/upgrade/Derby1_99_3UpgradeTest.java | 1 +
.../derby/upgrade/Derby1_99_4UpgradeTest.java | 3 ++
.../derby/upgrade/Derby1_99_5UpgradeTest.java | 1 +
.../derby/upgrade/Derby1_99_6UpgradeTest.java | 1 +
13 files changed, 94 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
index 9b9359b..4abee77 100644
--- a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
+++ b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
@@ -508,6 +508,23 @@ public class SqoopClient {
resourceRequests.deleteJob(String.valueOf(jobId));
}
+ public void deleteAllLinks(){
+ for (MJob job : getJobs()) {
+ deleteJob(job.getName());
+ }
+ }
+
+ public void deleteAllJobs(){
+ for (MLink link : getLinks()) {
+ deleteLink(link.getName());
+ }
+ }
+
+ public void deleteAllLinksAndJobs(){
+ deleteAllLinks();
+ deleteAllJobs();
+ }
+
/**
* Start job with given name.
*
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 91721ce..e66de4d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -230,6 +230,33 @@ limitations under the License.
</build>
</profile>
+ <profile>
+ <id>realcluster</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <excludedGroups>no-real-cluster</excludedGroups>
+ <reportNameSuffix>real-cluster-integration</reportNameSuffix>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <excludedGroups>no-real-cluster</excludedGroups>
+ <reportNameSuffix>real-cluster-integration</reportNameSuffix>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
<!-- Sign profile for releasing artifacts to Nexus repository -->
<profile>
<id>sign</id>
@@ -260,10 +287,10 @@ limitations under the License.
<artifactId>sqoop-client</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.sqoop</groupId>
- <artifactId>sqoop-shell</artifactId>
- <version>${project.version}</version>
+ <dependency>
+ <groupId>org.apache.sqoop</groupId>
+ <artifactId>sqoop-shell</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.sqoop</groupId>
@@ -500,10 +527,10 @@ limitations under the License.
<artifactId>commons-cli</artifactId>
<version>${commons-cli.version}</version>
</dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>${commons-io.version}</version>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons-io.version}</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
@@ -640,9 +667,9 @@ limitations under the License.
<version>${hadoop.2.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <version>${avro.version}</version>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>${avro.version}</version>
</dependency>
<dependency>
<groupId>net.sourceforge.findbugs</groupId>
@@ -660,7 +687,7 @@ limitations under the License.
<artifactId>jetty-servlet</artifactId>
<version>${jetty.version}</version>
</dependency>
- </dependencies>
+ </dependencies>
</dependencyManagement>
<modules>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java
index 325a6a9..cf19730 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java
@@ -18,6 +18,9 @@
package org.apache.sqoop.test.minicluster;
import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.client.SqoopClient;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
/**
*/
@@ -27,11 +30,15 @@ public class RealSqoopCluster extends SqoopMiniCluster {
private String serverUrl;
+ private SqoopClient client;
+
public RealSqoopCluster(String temporaryPath) throws Exception {
super(temporaryPath);
serverUrl = System.getProperty(SERVER_URL_KEY);
+ client = new SqoopClient(serverUrl);
+
if(serverUrl == null) {
throw new RuntimeException("Missing URL for real Sqoop 2 server: " + SERVER_URL_KEY);
}
@@ -44,16 +51,21 @@ public class RealSqoopCluster extends SqoopMiniCluster {
@Override
public void start() throws Exception {
- // Void operation
+ client.deleteAllLinksAndJobs();
}
@Override
public void stop() throws Exception {
- // Void operation
+ client.deleteAllLinksAndJobs();
}
@Override
public String getServerUrl() {
return serverUrl;
}
+
+ @Override
+ public String getConfigurationPath() {
+ return "/etc/hadoop/conf/";
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java
index e78adc0..e3ee300 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java
@@ -17,12 +17,9 @@
*/
package org.apache.sqoop.test.testcases;
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Method;
-
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.log4j.Logger;
import org.apache.sqoop.client.SqoopClient;
@@ -31,7 +28,9 @@ import org.apache.sqoop.test.hadoop.HadoopMiniClusterRunner;
import org.apache.sqoop.test.hadoop.HadoopRunner;
import org.apache.sqoop.test.hadoop.HadoopRunnerFactory;
import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
+import org.apache.sqoop.test.minicluster.RealSqoopCluster;
import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
+import org.apache.sqoop.test.minicluster.SqoopMiniClusterFactory;
import org.apache.sqoop.test.utils.HdfsUtils;
import org.testng.ITest;
import org.testng.ITestContext;
@@ -39,6 +38,10 @@ import org.testng.annotations.AfterSuite;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeSuite;
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+
/**
* Basic test case that will bootstrap Sqoop server running in embedded Jetty
* process.
@@ -82,7 +85,7 @@ abstract public class JettyTestCase implements ITest {
/**
* Jetty based Sqoop mini cluster
*/
- private static JettySqoopMiniCluster cluster;
+ private static SqoopMiniCluster cluster;
/**
* Sqoop client API.
@@ -97,8 +100,9 @@ abstract public class JettyTestCase implements ITest {
}
@BeforeMethod(alwaysRun = true)
- public void setMethodName(Method method) throws Exception {
+ public void setupMehodNameAndOutputPath(Method method) throws Exception {
methodName = method.getName();
+ hdfsClient.delete(new Path(getMapreduceDirectory()), true);
}
@BeforeSuite(alwaysRun = true)
@@ -154,8 +158,9 @@ abstract public class JettyTestCase implements ITest {
*
* @return New instance of test mini cluster
*/
- public JettySqoopMiniCluster createSqoopMiniCluster() throws Exception {
- return new JettySqoopMiniCluster(getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration());
+ public SqoopMiniCluster createSqoopMiniCluster() throws Exception {
+ return SqoopMiniClusterFactory.getSqoopMiniCluster(System.getProperties(), JettySqoopMiniCluster.class,
+ getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration());
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
index 0e46bf3..e964422 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
@@ -37,7 +37,7 @@ import org.testng.annotations.Test;
import java.lang.reflect.Method;
import java.util.List;
-@Test(groups = "slow")
+@Test(groups = {"slow", "no-real-cluster"})
public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase implements ITest {
private String testName;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java
index aa062fb..ea8c7b9 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java
@@ -23,7 +23,7 @@ import org.apache.sqoop.model.MLink;
import org.apache.sqoop.test.testcases.KafkaConnectorTestCase;
import org.testng.annotations.Test;
-
+@Test(groups = "no-real-cluster")
public class FromHDFSToKafkaTest extends KafkaConnectorTestCase {
public static final String[] input = {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java
index 6e78a13..4ecb018 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java
@@ -25,7 +25,7 @@ import org.apache.sqoop.model.MLink;
import org.apache.sqoop.test.testcases.KafkaConnectorTestCase;
import org.testng.annotations.Test;
-
+@Test(groups = "no-real-cluster")
public class FromRDBMSToKafkaTest extends KafkaConnectorTestCase {
private static final String[] input = {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
index 6228b0d..8b57d75 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
@@ -30,6 +30,7 @@ import org.testng.annotations.Test;
import java.util.HashMap;
import java.util.Map;
+@Test(groups = "no-real-cluster")
public class BlacklistedConnectorTest extends ConnectorTestCase {
public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster {
public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
index 4a2e7a4..e0879ed 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
@@ -53,6 +53,7 @@ import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
+@Test(groups = "no-real-cluster")
public class ClasspathTest extends ConnectorTestCase {
private static final String TEST_CONNECTOR_JAR_NAME = "test-connector.jar";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
index 8d02e24..2cfd43c 100644
--- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
@@ -41,6 +41,7 @@ import static org.testng.Assert.assertEquals;
* Job with id 3 has been disabled
* Job with id 1 has been run 5 times
*/
+@Test(groups = "no-real-cluster")
public class Derby1_99_3UpgradeTest extends DerbyRepositoryUpgradeTest {
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java
index b88940a..9f7b04b 100644
--- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java
@@ -17,6 +17,8 @@
*/
package org.apache.sqoop.integration.repository.derby.upgrade;
+import org.testng.annotations.Test;
+
import java.util.HashMap;
import java.util.Map;
@@ -42,6 +44,7 @@ import java.util.Map;
* Job with id 4 has been disabled
* Job with id 5 has been disabled
*/
+@Test(groups = "no-real-cluster")
public class Derby1_99_4UpgradeTest extends DerbyRepositoryUpgradeTest {
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java
index 1f3563d..3b1e214 100644
--- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java
@@ -51,6 +51,7 @@ import org.testng.annotations.Test;
* Job with id 4 has been disabled
* Job with id 5 has been disabled
*/
+@Test(groups = "no-real-cluster")
public class Derby1_99_5UpgradeTest extends DerbyRepositoryUpgradeTest {
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java
index a57a420..1081e50 100644
--- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java
@@ -52,6 +52,7 @@ import static org.testng.Assert.assertNull;
* Job with id 4 has been disabled
* Job with id 5 has been disabled
*/
+@Test(groups = "no-real-cluster")
public class Derby1_99_6UpgradeTest extends DerbyRepositoryUpgradeTest {
@Override