You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/04/14 21:29:23 UTC
sqoop git commit: SQOOP-2269: Sqoop2: Parquet integration tests
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 5ea695a6d -> bfc10914c
SQOOP-2269: Sqoop2: Parquet integration tests
(Abraham Elmahrek via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/bfc10914
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/bfc10914
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/bfc10914
Branch: refs/heads/sqoop2
Commit: bfc10914c109261b151631c0129804dde60bc1df
Parents: 5ea695a
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Tue Apr 14 12:28:59 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Tue Apr 14 12:28:59 2015 -0700
----------------------------------------------------------------------
connector/connector-kite/pom.xml | 6 ++
.../connector/kite/KiteFromInitializer.java | 3 +-
.../sqoop/connector/kite/KiteToInitializer.java | 3 +-
.../connector/hive/FromRDBMSToKiteHiveTest.java | 63 ++++++++++++++++----
4 files changed, 61 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/bfc10914/connector/connector-kite/pom.xml
----------------------------------------------------------------------
diff --git a/connector/connector-kite/pom.xml b/connector/connector-kite/pom.xml
index 8593899..41441f5 100644
--- a/connector/connector-kite/pom.xml
+++ b/connector/connector-kite/pom.xml
@@ -80,6 +80,12 @@ limitations under the License.
<dependency>
<groupId>org.kitesdk</groupId>
<artifactId>kite-data-hive</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.twitter</groupId>
+ <artifactId>parquet-hive-bundle</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/bfc10914/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
index fd3017b..ed79314 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
@@ -75,12 +75,11 @@ public class KiteFromInitializer extends Initializer<LinkConfiguration,
jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors"));
jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse"));
jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat"));
- jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory"));
jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService"));
jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler"));
jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery"));
jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement"));
- jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe"));
+ jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"));
}
return jars;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/bfc10914/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
index e27c0b8..50daba0 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
@@ -83,12 +83,11 @@ public class KiteToInitializer extends Initializer<LinkConfiguration,
jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors"));
jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse"));
jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat"));
- jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory"));
jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService"));
jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler"));
jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery"));
jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement"));
- jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe"));
+ jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"));
}
return jars;
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/bfc10914/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
index a4a5b04..37a0a7d 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
@@ -18,6 +18,8 @@
package org.apache.sqoop.integration.connector.hive;
import org.apache.sqoop.common.test.asserts.ProviderAsserts;
+import org.apache.sqoop.common.test.db.DatabaseProvider;
+import org.apache.sqoop.common.test.db.DatabaseProviderFactory;
import org.apache.sqoop.common.test.db.TableName;
import org.apache.sqoop.connector.common.FileFormat;
import org.apache.sqoop.model.MConfigList;
@@ -25,15 +27,56 @@ import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.test.testcases.HiveConnectorTestCase;
+import org.testng.ITest;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Factory;
import org.testng.annotations.Test;
-/**
- */
-public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase {
+import java.lang.reflect.Method;
+
+@Test(groups = "slow")
+public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase implements ITest {
+ private String testName;
+
+ private FileFormat fileFormat;
+
private MLink rdbmsLink;
private MLink kiteLink;
+ private String hiveTableName;
+
+ @Factory(dataProvider="rdbms-to-kite-hive-test")
+ public FromRDBMSToKiteHiveTest(FileFormat fileFormat) {
+ this.fileFormat = fileFormat;
+ }
+
+ @DataProvider(name="rdbms-to-kite-hive-test", parallel=true)
+ public static Object[][] data() throws Exception {
+ DatabaseProvider provider = DatabaseProviderFactory.getProvider(System.getProperties());
+ return new Object[][]{
+ {FileFormat.AVRO},
+ {FileFormat.PARQUET}
+ };
+ }
+
+ @Override
+ public String getTestName() {
+ return testName + "[" + fileFormat.name() + "]";
+ }
+
+ public String getHiveTableName() {
+ return testName + "_" + fileFormat.name();
+ }
+
+ public String getDatasetURI() {
+ return "dataset:hive:" + getHiveTableName();
+ }
+
+ @BeforeMethod(alwaysRun = true)
+ public void setTestName(Method aMethod) {
+ this.testName = aMethod.getName();
+ }
@BeforeMethod(alwaysRun = true)
public void createTable() {
@@ -70,8 +113,8 @@ public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase {
// Fill the Kite "TO" config
MConfigList toConfig = job.getToJobConfig();
- toConfig.getStringInput("toJobConfig.uri").setValue("dataset:hive:testtable");
- toConfig.getEnumInput("toJobConfig.fileFormat").setValue(FileFormat.AVRO);
+ toConfig.getStringInput("toJobConfig.uri").setValue(getDatasetURI());
+ toConfig.getEnumInput("toJobConfig.fileFormat").setValue(this.fileFormat);
// driver config
MDriverConfig driverConfig = job.getDriverConfig();
@@ -81,11 +124,11 @@ public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase {
executeJob(job);
// Assert correct output
- ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 1}, "1");
- ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 2}, "2");
- ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 3}, "3");
- ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id", 4}, "4");
+ ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 1}, "1");
+ ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 2}, "2");
+ ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 3}, "3");
+ ProviderAsserts.assertRow(hiveProvider, new TableName(getHiveTableName()), new Object[]{"id", 4}, "4");
- hiveProvider.dropTable(new TableName("testtable"));
+ hiveProvider.dropTable(new TableName(getHiveTableName()));
}
}