You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/03/25 01:41:37 UTC
sqoop git commit: SQOOP-2016: Sqoop2: Create integration test for
JDBC to Hive
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 d615b39c1 -> 45d1c32d7
SQOOP-2016: Sqoop2: Create integration test for JDBC to Hive
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/45d1c32d
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/45d1c32d
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/45d1c32d
Branch: refs/heads/sqoop2
Commit: 45d1c32d7ed4811452c42626940d80a4b091f797
Parents: d615b39
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Tue Mar 24 17:41:13 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Tue Mar 24 17:41:13 2015 -0700
----------------------------------------------------------------------
.../sqoop/common/test/db/HiveProvider.java | 101 ++++++++++++++
.../sqoop/common/test/utils/NetworkUtils.java | 36 +++++
.../connector/kite/KiteDatasetExecutor.java | 2 +-
.../sqoop/connector/kite/TestKiteExecutor.java | 13 +-
pom.xml | 12 ++
test/pom.xml | 12 ++
.../sqoop/test/hive/HiveServerRunner.java | 135 ++++++++++++++++++
.../test/hive/HiveServerRunnerFactory.java | 56 ++++++++
.../test/hive/InternalHiveServerRunner.java | 55 ++++++++
.../hive/InternalMetastoreServerRunner.java | 71 ++++++++++
.../sqoop/test/hive/MetastoreServerRunner.java | 137 +++++++++++++++++++
.../test/hive/MetastoreServerRunnerFactory.java | 56 ++++++++
.../minicluster/TomcatSqoopMiniCluster.java | 4 +
.../test/testcases/HiveConnectorTestCase.java | 88 ++++++++++++
.../connector/hive/FromRDBMSToKiteHiveTest.java | 96 +++++++++++++
.../connector/kite/FromRDBMSToKiteTest.java | 4 +-
16 files changed, 873 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java
----------------------------------------------------------------------
diff --git a/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java b/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java
new file mode 100644
index 0000000..dfe0f43
--- /dev/null
+++ b/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.common.test.db;
+
+/**
+ * MySQL Provider that will connect to remote MySQL server.
+ *
+ * JDBC can be configured via system properties. Default value is server running
+ * on the same box (localhost) that is access via sqoop/sqoop credentials.
+ */
+public class HiveProvider extends DatabaseProvider {
+
+ public static final String DRIVER = "org.apache.hive.jdbc.HiveDriver";
+
+ private static final String CONNECTION = System.getProperties().getProperty(
+ "sqoop.provider.hive.jdbc",
+ "jdbc:hive2://"
+ );
+
+ private static final String USERNAME = System.getProperties().getProperty(
+ "sqoop.provider.hive.username",
+ "sqoop"
+ );
+
+ private static final String PASSWORD = System.getProperties().getProperty(
+ "sqoop.provider.hive.password",
+ "sqoop"
+ );
+
+ private String jdbcUrl;
+
+ /**
+ * Use system properties to get JDBC URL.
+ */
+ public HiveProvider() {
+ this.jdbcUrl = CONNECTION;
+ }
+
+ /**
+ * Use JDBC URL provided.
+ *
+ * @param jdbcUrl hive server jdbc URL.
+ */
+ public HiveProvider(String jdbcUrl) {
+ this.jdbcUrl = jdbcUrl;
+ }
+
+ @Override
+ public String getConnectionUrl() {
+ return jdbcUrl;
+ }
+
+ @Override
+ public String getConnectionUsername() {
+ return USERNAME;
+ }
+
+ @Override
+ public String getConnectionPassword() {
+ return PASSWORD;
+ }
+
+ @Override
+ public String escapeColumnName(String columnName) {
+ return escape(columnName);
+ }
+
+ @Override
+ public String escapeTableName(String tableName) {
+ return escape(tableName);
+ }
+
+ @Override
+ public String escapeValueString(String value) {
+ return escape(value);
+ }
+
+ @Override
+ public String getJdbcDriver() {
+ return DRIVER;
+ }
+
+ public String escape(String entity) {
+ return entity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java
----------------------------------------------------------------------
diff --git a/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java b/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java
index 87534c7..7f0f750 100644
--- a/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java
+++ b/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java
@@ -17,14 +17,21 @@
*/
package org.apache.sqoop.common.test.utils;
+import org.apache.log4j.Logger;
+
import java.io.IOException;
+import java.net.InetAddress;
import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.concurrent.TimeoutException;
/**
* Network related utilities.
*/
public class NetworkUtils {
+ private static final Logger LOG = Logger.getLogger(NetworkUtils.class);
+
/**
* Create port number that is available on this machine for
* subsequent use.
@@ -58,6 +65,35 @@ public class NetworkUtils {
}
}
+ /**
+ * Create a socket and attempt to connect to ``hostname``:``port``
+ * ``numberOfAttempts`` amount of times with ``sleepTime`` milliseconds
+ * between each attempt.
+ *
+ * @param hostname host name to connect to
+ * @param port port to connect on
+ * @param numberOfAttempts number of tries to connect
+ * @param sleepTime time in between connection attempts in milliseconds
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ public static void waitForStartUp(String hostname, int port, int numberOfAttempts, long sleepTime)
+ throws InterruptedException, TimeoutException {
+ for (int i = 0; i < numberOfAttempts; ++i) {
+ try {
+ LOG.debug("Attempt " + (i + 1) + " to access " + hostname + ":" + port);
+ new Socket(InetAddress.getByName(hostname), port).close();
+ return;
+ } catch (Exception e) {
+ LOG.debug("Failed to connect to " + hostname + ":" + port, e);
+ }
+
+ Thread.sleep(sleepTime);
+ }
+
+ throw new TimeoutException("Couldn't access new server: " + hostname + ":" + port);
+ }
+
private NetworkUtils() {
// Instantiation is prohibited
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
index db8d4e6..6aa28be 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
@@ -189,7 +189,7 @@ public class KiteDatasetExecutor {
// Replace dataset name with temporary dataset name.
uriParts[1] = uriParts[1].substring(0, uriParts[1].lastIndexOf("/")) + "/" + temporaryDatasetName;
} else {
- uriParts[1] = temporaryDatasetName;
+ uriParts[1] = ":" + temporaryDatasetName;
}
} else {
uriParts[1] += "/" + temporaryDatasetName;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
index 44721a8..0e797f8 100644
--- a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
+++ b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
@@ -38,7 +38,8 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import static org.mockito.MockitoAnnotations.initMocks;
-import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
public class TestKiteExecutor {
@@ -173,6 +174,16 @@ public class TestKiteExecutor {
assertTrue(suggestedUri.length() > subURI.length());
assertTrue(suggestedUri.contains(subURI));
assertTrue(suggestedUri.endsWith(endURI));
+
+ endURI = "auth:host=metastore&auth:port=9083";
+ uri = "dataset:hive:sqoop?auth:host=metastore&auth:port=9083";
+ subURI = "dataset:hive:";
+ suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(new LinkConfig(), uri);
+ assertTrue(suggestedUri.length() > subURI.length());
+ assertTrue(suggestedUri.contains(subURI), suggestedUri);
+ assertTrue(suggestedUri.endsWith(endURI), suggestedUri);
+ assertFalse(suggestedUri.contains("sqoop"));
+ assertFalse(suggestedUri.contains("/"));
}
private static Schema createTwoFieldSchema() {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e3431bd..c608ca7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -368,6 +368,18 @@ limitations under the License.
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-service</artifactId>
+ <version>${hive.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>${hive.version}</version>
<scope>provided</scope>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index f743d25..d8fbfa2 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -113,6 +113,18 @@ limitations under the License.
</dependency>
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-service</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.codehaus.cargo</groupId>
<artifactId>cargo-core-container-tomcat</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java
new file mode 100644
index 0000000..8b355bd
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.test.utils.NetworkUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+/**
+ * Hive server runner for testing purpose.
+ *
+ * Runner provides methods for bootstrapping and using HiveServer2. This
+ * should allow providing a HiveServer2 minicluster or using a real server.
+ */
+public abstract class HiveServerRunner {
+ private static final Logger LOG = Logger.getLogger(HiveServerRunner.class);
+
+ private final String hostname;
+ private final int port;
+
+ public HiveServerRunner(String hostname, int port) throws Exception {
+ this.hostname = hostname;
+
+ if (port == 0) {
+ this.port = NetworkUtils.findAvailablePort();
+ } else {
+ this.port = port;
+ }
+
+ LOG.info("Hive Server will bind to port " + getPort());
+ }
+
+ /**
+ * Configuration object.
+ */
+ protected HiveConf config = null;
+
+ /**
+ * Start Hive server.
+ *
+ * @throws Exception
+ */
+ abstract public void start() throws Exception;
+
+ /**
+ * Stop Hive server.
+ *
+ * @throws Exception
+ */
+ abstract public void stop() throws Exception;
+
+ /**
+ * Return JDBC URL to be used with HiveServer2 instance.
+ *
+ * @return String
+ */
+ public String getUrl() {
+ return "jdbc:hive2://" + hostname + ":" + port + "/default";
+ }
+
+ /**
+ * Prepare configuration object. This method should be called once before the
+ * start method is called.
+ *
+ * @param config is the configuration object to prepare.
+ */
+ public Configuration prepareConfiguration(Configuration config) throws Exception {
+ config.set(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST.varname, getHostName());
+ config.setInt(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, getPort());
+ return config;
+ }
+
+ /**
+ * Get configuration.
+ *
+ * @return HiveConf
+ */
+ public HiveConf getConfiguration() {
+ return config;
+ }
+
+ /**
+ * Set the configuration object.
+ *
+ * @param config
+ */
+ public void setConfiguration(Configuration config) {
+ this.config = new HiveConf();
+ this.config.addResource(config);
+ this.printConfig();
+ }
+
+ /**
+ * Hostname used to start services on.
+ *
+ * @return String hostname
+ */
+ public String getHostName() {
+ return hostname;
+ }
+
+ /**
+ * Port hive service will be on.
+ *
+ * @return int port
+ */
+ public int getPort() {
+ return this.port;
+ }
+
+ private void printConfig() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ config.logVars(new PrintStream(baos));
+ LOG.debug("Hive server runner configuration:\n" + baos.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java
new file mode 100644
index 0000000..f6a2fa6
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import java.util.Properties;
+
+/**
+ * Create HiveServer2 runner.
+ */
+public class HiveServerRunnerFactory {
+
+ public static final String RUNNER_CLASS_PROPERTY = "sqoop.hive.runner.class";
+
+ public static final String HOSTNAME_PROPERTY = "sqoop.hive.server.hostname";
+
+ public static final String PORT_PROPERTY = "sqoop.hive.server.port";
+
+ public static final String DEFAULT_HOSTNAME = "127.0.0.1";
+
+ public static final String DEFAULT_PORT = "0";
+
+ public static HiveServerRunner getRunner(
+ Properties properties, Class<? extends HiveServerRunner> defaultRunner)
+ throws Exception {
+ Class<?> klass;
+
+ String hostname = properties.getProperty(HOSTNAME_PROPERTY, DEFAULT_HOSTNAME);
+ int port = Integer.parseInt(
+ properties.getProperty(PORT_PROPERTY, DEFAULT_PORT));
+
+ String className = properties.getProperty(RUNNER_CLASS_PROPERTY);
+ if(className == null) {
+ klass = defaultRunner;
+ } else {
+ klass = Class.forName(className);
+ }
+
+ return (HiveServerRunner)klass.getConstructor(String.class, int.class)
+ .newInstance(hostname, port);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java
new file mode 100644
index 0000000..3418525
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import org.apache.hive.service.server.HiveServer2;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.test.utils.NetworkUtils;
+
+/**
+ * Use HiveServer2 JDBC client to run all operations.
+ *
+ * @see org.apache.sqoop.test.hive.HiveServerRunner
+ */
+public class InternalHiveServerRunner extends HiveServerRunner {
+ private static final Logger LOG = Logger.getLogger(InternalHiveServerRunner.class);
+
+ private final HiveServer2 hiveServer2;
+
+ public InternalHiveServerRunner(String hostname, int port) throws Exception {
+ super(hostname, port);
+ hiveServer2 = new HiveServer2();
+ }
+
+ @Override
+ public void start() throws Exception {
+ Long start = System.currentTimeMillis();
+ hiveServer2.init(getConfiguration());
+ hiveServer2.start();
+ NetworkUtils.waitForStartUp(getHostName(), getPort(), 5, 100);
+ Long end = System.currentTimeMillis();
+ LOG.debug("Hive service took " + (end - start) + "ms to start");
+ }
+
+ @Override
+ public void stop() throws Exception {
+ if (hiveServer2 != null) {
+ hiveServer2.stop();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java
new file mode 100644
index 0000000..a8282ec
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.test.utils.NetworkUtils;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Start a Hive Metastore service on the specified hostname and port.
+ *
+ * @see org.apache.sqoop.test.hive.MetastoreServerRunner
+ */
+public class InternalMetastoreServerRunner extends MetastoreServerRunner {
+ private static final Logger LOG = Logger.getLogger(InternalMetastoreServerRunner.class);
+
+ private ExecutorService executor = Executors
+ .newSingleThreadExecutor();
+
+ public InternalMetastoreServerRunner(String hostname, int port) throws Exception {
+ super(hostname, port);
+ }
+
+ @Override
+ public void start() throws Exception {
+ Long start = System.currentTimeMillis();
+ final int metastorePort = getPort();
+ final HiveConf conf = getConfiguration();
+ Callable<Void> metastoreService = new Callable<Void>() {
+ public Void call() throws Exception {
+ try {
+ HiveMetaStore.startMetaStore(metastorePort,
+ ShimLoader.getHadoopThriftAuthBridge(), conf);
+ while(true);
+ } catch (Throwable e) {
+ throw new Exception("Error starting metastore", e);
+ }
+ }
+ };
+ executor.submit(metastoreService);
+ NetworkUtils.waitForStartUp(getHostName(), getPort(), 5, 1000);
+ Long end = System.currentTimeMillis();
+ LOG.debug("Metastore service took " + (end - start) + "ms to start");
+ }
+
+ @Override
+ public void stop() throws Exception {
+ executor.shutdownNow();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java
new file mode 100644
index 0000000..32a6b35
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.test.utils.NetworkUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.UUID;
+
+/**
+ * Metastore server runner for testing purpose.
+ *
+ * Runner provides methods for bootstrapping and using the Hive metastore. This
+ * should allow providing a Hive metastore minicluster or using a real server.
+ */
+public abstract class MetastoreServerRunner {
+ private static final Logger LOG = Logger.getLogger(MetastoreServerRunner.class);
+
+ private final String hostname;
+ private final int port;
+ private final String warehouseDirectory;
+
+ public MetastoreServerRunner(String hostname, int port) throws Exception {
+ this.hostname = hostname;
+ this.warehouseDirectory = "/user/hive/" + UUID.randomUUID();
+
+ if (port == 0) {
+ port = NetworkUtils.findAvailablePort();
+ }
+ LOG.info("Hive Metastore will bind to port " + port);
+
+ this.port = port;
+ }
+
+ /**
+ * Configuration object.
+ */
+ protected HiveConf config = null;
+
+ /**
+ * Start Hive server.
+ *
+ * @throws Exception
+ */
+ abstract public void start() throws Exception;
+
+ /**
+ * Stop Hive server.
+ *
+ * @throws Exception
+ */
+ abstract public void stop() throws Exception;
+
+ /**
+ * Metastore URI authority (ex. hostname:port).
+ *
+ * @return String metastore authority.
+ */
+ public String getAuthority() {
+ return getHostName() + ":" + getPort();
+ }
+
+ /**
+ * Prepare configuration object. This method should be called once before the
+ * start method is called.
+ *
+ * @param config is the configuration object to prepare.
+ */
+ public Configuration prepareConfiguration(Configuration config) throws Exception {
+ config.set(HiveConf.ConfVars.METASTOREURIS.varname, "thrift://" + getHostName() + ":" + getPort());
+ config.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehouseDirectory);
+ return config;
+ }
+
+ /**
+ * Get configuration.
+ *
+ * @return HiveConf
+ */
+ public HiveConf getConfiguration() {
+ return config;
+ }
+
+ /**
+ * Set the configuration object.
+ *
+ * @param config
+ */
+ public void setConfiguration(Configuration config) {
+ this.config = new HiveConf();
+ this.config.addResource(config);
+ this.printConfig();
+ }
+
+ /**
+ * Hostname used to start services on.
+ *
+ * @return String hostname
+ */
+ public String getHostName() {
+ return hostname;
+ }
+
+ /**
+ * Port meta store service will be on.
+ *
+ * @return int port
+ */
+ public int getPort() {
+ return this.port;
+ }
+
+ private void printConfig() {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ config.logVars(new PrintStream(baos));
+ LOG.debug("Hive server runner configuration:\n" + baos.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java
new file mode 100644
index 0000000..be3720f
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import java.util.Properties;
+
+/**
+ * Create HiveServer2 runner.
+ */
+public class MetastoreServerRunnerFactory {
+
+ public static final String RUNNER_CLASS_PROPERTY = "sqoop.hive.metastore.runner.class";
+
+ public static final String HOSTNAME_PROPERTY = "sqoop.hive.metastore.server.hostname";
+
+ public static final String PORT_PROPERTY = "sqoop.hive.metastore.server.port";
+
+ public static final String DEFAULT_HOSTNAME = "127.0.0.1";
+
+ public static final String DEFAULT_PORT = "0";
+
+ public static MetastoreServerRunner getRunner(
+ Properties properties, Class<? extends MetastoreServerRunner> defaultRunner)
+ throws Exception {
+ Class<?> klass;
+
+ String hostname = properties.getProperty(HOSTNAME_PROPERTY, DEFAULT_HOSTNAME);
+ int port = Integer.parseInt(
+ properties.getProperty(PORT_PROPERTY, DEFAULT_PORT));
+
+ String className = properties.getProperty(RUNNER_CLASS_PROPERTY);
+ if(className == null) {
+ klass = defaultRunner;
+ } else {
+ klass = Class.forName(className);
+ }
+
+ return (MetastoreServerRunner)klass.getConstructor(String.class, int.class)
+ .newInstance(hostname, port);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
index 648e2f6..4d27886 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
@@ -95,6 +95,7 @@ public class TomcatSqoopMiniCluster extends SqoopMiniCluster {
String []classpath = System.getProperty("java.class.path").split(":");
for(String jar : classpath) {
if(jar.contains("hadoop-") || // Hadoop jars
+ jar.contains("hive-") || // Hive jars
jar.contains("commons-") || // Apache Commons libraries
jar.contains("httpcore-") || // Apache Http Core libraries
jar.contains("httpclient-") || // Apache Http Client libraries
@@ -106,6 +107,7 @@ public class TomcatSqoopMiniCluster extends SqoopMiniCluster {
jar.contains("jackson-") || // Jackson
jar.contains("derby") || // Derby drivers
jar.contains("avro-") || // Avro
+ jar.contains("parquet-") || // Parquet
jar.contains("mysql") || // MySQL JDBC driver
jar.contains("postgre") || // PostgreSQL JDBC driver
jar.contains("oracle") || // Oracle driver
@@ -113,6 +115,8 @@ public class TomcatSqoopMiniCluster extends SqoopMiniCluster {
jar.contains("tdgs") || // Teradata driver
jar.contains("nzjdbc") || // Netezza driver
jar.contains("sqljdbc") || // Microsoft SQL Server driver
+ jar.contains("libfb303") || // Facebook thrift lib
+ jar.contains("datanucleus-") || // Data nucleus libs
jar.contains("google") // Google libraries (guava, ...)
) {
extraClassPath.add(jar);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java
new file mode 100644
index 0000000..628f484
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.testcases;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.test.db.HiveProvider;
+import org.apache.sqoop.test.hive.InternalHiveServerRunner;
+import org.apache.sqoop.test.hive.HiveServerRunner;
+import org.apache.sqoop.test.hive.HiveServerRunnerFactory;
+import org.apache.sqoop.test.hive.InternalMetastoreServerRunner;
+import org.apache.sqoop.test.hive.MetastoreServerRunner;
+import org.apache.sqoop.test.hive.MetastoreServerRunnerFactory;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+
+public class HiveConnectorTestCase extends ConnectorTestCase {
+ private static final Logger LOG = Logger.getLogger(HiveConnectorTestCase.class);
+
+ protected HiveServerRunner hiveServerRunner;
+ protected MetastoreServerRunner metastoreServerRunner;
+ protected HiveProvider hiveProvider;
+
+ private void ensureWarehouseDirectory(Configuration conf) throws Exception {
+ String warehouseDirectory = conf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname);
+ StringBuilder dir = new StringBuilder();
+ for (String part : warehouseDirectory.split("/")) {
+ dir.append(part).append("/");
+ Path path = new Path(dir.toString());
+ if (!hdfsClient.exists(path)) {
+ hdfsClient.mkdirs(path);
+ }
+ }
+ hdfsClient.setPermission(new Path(dir.toString()), new FsPermission((short)01777));
+ }
+
+ @BeforeMethod(alwaysRun = true)
+ public void startHive() throws Exception {
+ String databasePath = HdfsUtils.joinPathFragments(getTemporaryPath(), "metastore_db");
+ metastoreServerRunner = MetastoreServerRunnerFactory.getRunner(System.getProperties(), InternalMetastoreServerRunner.class);
+ metastoreServerRunner.setConfiguration(metastoreServerRunner.prepareConfiguration(hadoopCluster.getConfiguration()));
+ metastoreServerRunner.getConfiguration().set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname,
+ "jdbc:derby:;databaseName=" + databasePath + ";create=true");
+ ensureWarehouseDirectory(metastoreServerRunner.getConfiguration());
+ LOG.info("Starting Metastore Server: " + metastoreServerRunner.getClass().getName());
+ metastoreServerRunner.start();
+
+ hiveServerRunner = HiveServerRunnerFactory.getRunner(System.getProperties(), InternalHiveServerRunner.class);
+ hiveServerRunner.setConfiguration(hiveServerRunner.prepareConfiguration(metastoreServerRunner.getConfiguration()));
+ LOG.info("Starting Hive Server: " + hiveServerRunner.getClass().getName());
+ hiveServerRunner.start();
+
+ LOG.info("Starting Hive Provider: " + provider.getClass().getName());
+ hiveProvider = new HiveProvider(hiveServerRunner.getUrl());
+ hiveProvider.start();
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void stopHive() throws Exception {
+ LOG.info("Stopping Hive Provider: " + provider.getClass().getName());
+ hiveProvider.stop();
+
+ LOG.info("Stopping Hive Server: " + hiveServerRunner.getClass().getName());
+ hiveServerRunner.stop();
+
+ LOG.info("Stopping Metastore Server: " + metastoreServerRunner.getClass().getName());
+ metastoreServerRunner.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
new file mode 100644
index 0000000..e789ce0
--- /dev/null
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.integration.connector.hive;
+
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.common.test.asserts.ProviderAsserts;
+import org.apache.sqoop.common.test.db.TableName;
+import org.apache.sqoop.connector.common.FileFormat;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.test.testcases.HiveConnectorTestCase;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ */
+public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase {
+ private MLink rdbmsLink;
+ private MLink kiteLink;
+
+ @BeforeMethod(alwaysRun = true)
+ public void createTable() {
+ createAndLoadTableCities();
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void dropTable() {
+ super.dropTable();
+ }
+
+ @BeforeMethod(alwaysRun = true)
+ public void createLinks() {
+ // RDBMS link
+ rdbmsLink = getClient().createLink("generic-jdbc-connector");
+ fillRdbmsLinkConfig(rdbmsLink);
+ saveLink(rdbmsLink);
+
+ // Kite link
+ kiteLink = getClient().createLink("kite-connector");
+ kiteLink.getConnectorLinkConfig().getStringInput("linkConfig.authority")
+ .setValue(metastoreServerRunner.getAuthority());
+ saveLink(kiteLink);
+ }
+
+ @Test
+ public void testCities() throws Exception {
+ // Job creation
+ MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kiteLink.getPersistenceId());
+
+ // Set rdbms "FROM" config
+ MConfigList fromConfig = job.getJobConfig(Direction.FROM);
+ fillRdbmsFromConfig(job, "id");
+ // TODO: Kite have troubles with some data types, so we're limiting the columns to int only
+ fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id"));
+
+ // Fill the Kite "TO" config
+ MConfigList toConfig = job.getJobConfig(Direction.TO);
+ toConfig.getStringInput("toJobConfig.uri").setValue("dataset:hive:testtable");
+ toConfig.getEnumInput("toJobConfig.fileFormat").setValue(FileFormat.AVRO);
+
+ // driver config
+ MDriverConfig driverConfig = job.getDriverConfig();
+ driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(1);
+
+ saveJob(job);
+
+ executeJob(job);
+
+ // Assert correct output
+ ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 1}, "1");
+ ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 2}, "2");
+ ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 3}, "3");
+ ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 4}, "4");
+
+ hiveProvider.dropTable(new TableName("testtable"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
index 8ca1c3a..828d244 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
@@ -29,8 +29,7 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
-/**
- */
+@Test
public class FromRDBMSToKiteTest extends ConnectorTestCase {
@BeforeMethod(alwaysRun = true)
public void createTable() {
@@ -96,5 +95,4 @@ public class FromRDBMSToKiteTest extends ConnectorTestCase {
"\"4\""
);
}
-
}