You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by hs...@apache.org on 2013/07/31 08:29:47 UTC
git commit: SQOOP-1114. Sqoop2: Integration: Add tests for various
Generic JDBC Connector splitters
Updated Branches:
refs/heads/sqoop2 9c7adb4e1 -> 3631b29e0
SQOOP-1114. Sqoop2: Integration: Add tests for various Generic JDBC Connector splitters
(Jarek Jarcec Cecho via Hari Shreedharan)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3631b29e
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3631b29e
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3631b29e
Branch: refs/heads/sqoop2
Commit: 3631b29e06c5e78d666752a65b0fe01601dc3259
Parents: 9c7adb4
Author: Hari Shreedharan <hs...@apache.org>
Authored: Tue Jul 30 23:28:50 2013 -0700
Committer: Hari Shreedharan <hs...@apache.org>
Committed: Tue Jul 30 23:28:50 2013 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
.../apache/sqoop/test/asserts/HdfsAsserts.java | 11 ++
.../apache/sqoop/test/data/UbuntuReleases.java | 72 +++++++++++
.../sqoop/test/testcases/ConnectorTestCase.java | 15 +++
.../sqoop/test/testcases/TomcatTestCase.java | 9 ++
.../sqoop/test/utils/ParametrizedUtils.java | 91 ++++++++++++++
.../jdbc/generic/imports/PartitionerTest.java | 125 +++++++++++++++++++
7 files changed, 324 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 04cf479..c1ed306 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,7 +99,7 @@ limitations under the License.
<hadoop.2.version>2.0.2-alpha</hadoop.2.version>
<guava.version>11.0.2</guava.version>
<json-simple.version>1.1</json-simple.version>
- <junit.version>4.9</junit.version>
+ <junit.version>4.11</junit.version>
<mockito.version>1.9.5</mockito.version>
<log4j.version>1.2.16</log4j.version>
<servlet.version>2.5</servlet.version>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
index 056e612..c175272 100644
--- a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
+++ b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
@@ -30,6 +30,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
/**
@@ -76,6 +77,16 @@ public class HdfsAsserts {
}
}
+ /**
+ * Verify number of output mapreduce files.
+ *
+ * @param directory Mapreduce output directory
+ * @param expectedFiles Expected number of files
+ */
+ public static void assertMapreduceOutputFiles(String directory, int expectedFiles) {
+ String []files = HdfsUtils.getOutputMapreduceFiles(directory);
+ assertEquals(expectedFiles, files.length);
+ }
private HdfsAsserts() {
// Instantiation is prohibited
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java b/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java
new file mode 100644
index 0000000..a4730c2
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.data;
+
+import org.apache.sqoop.test.db.DatabaseProvider;
+
+/**
+ * Releases of Ubuntu Linux.
+ *
+ * Purpose of this set is to cover most common data types (varchar, int, numeric, date, boolean).
+ */
+public class UbuntuReleases extends DataSet {
+
+ public UbuntuReleases(DatabaseProvider provider, String tableBaseName) {
+ super(provider, tableBaseName);
+ }
+
+ @Override
+ public DataSet createTables() {
+ provider.createTable(
+ tableBaseName,
+ "id",
+ "id", "int",
+ "code_name", "varchar(64)",
+ "version", "numeric(4,2)",
+ "release_date", "date",
+ "lts", "boolean"
+ );
+
+ return this;
+ }
+
+ @Override
+ public DataSet loadBasicData() {
+ provider.insertRow(tableBaseName, 1, "Warty Warthog", 4.10, "2004-10-20", false);
+ provider.insertRow(tableBaseName, 2, "Hoary Hedgehog", 5.04, "2005-04-08", false);
+ provider.insertRow(tableBaseName, 3, "Breezy Badger", 5.10, "2005-10-13", false);
+ provider.insertRow(tableBaseName, 4, "Dapper Drake", 6.06, "2006-06-01", true);
+ provider.insertRow(tableBaseName, 5, "Edgy Eft", 6.10, "2006-10-26", false);
+ provider.insertRow(tableBaseName, 6, "Feisty Fawn", 7.04, "2007-04-19", false);
+ provider.insertRow(tableBaseName, 7, "Gutsy Gibbon", 7.10, "2007-10-18", false);
+ provider.insertRow(tableBaseName, 8, "Hardy Heron", 8.04, "2008-04-24", true);
+ provider.insertRow(tableBaseName, 9, "Intrepid Ibex", 8.10, "2008-10-18", false);
+ provider.insertRow(tableBaseName, 10, "Jaunty Jackalope", 9.04, "2009-04-23", false);
+ provider.insertRow(tableBaseName, 11, "Karmic Koala", 9.10, "2009-10-29", false);
+ provider.insertRow(tableBaseName, 12, "Lucid Lynx", 10.04, "2010-04-29", true);
+ provider.insertRow(tableBaseName, 13, "Maverick Meerkat",10.10, "2010-10-10", false);
+ provider.insertRow(tableBaseName, 14, "Natty Narwhal", 11.04, "2011-04-28", false);
+ provider.insertRow(tableBaseName, 15, "Oneiric Ocelot", 11.10, "2011-10-10", false);
+ provider.insertRow(tableBaseName, 16, "Precise Pangolin",12.04, "2012-04-26", true);
+ provider.insertRow(tableBaseName, 17, "Quantal Quetzal", 12.10, "2012-10-18", false);
+ provider.insertRow(tableBaseName, 18, "Raring Ringtail", 13.04, "2013-04-25", false);
+ provider.insertRow(tableBaseName, 19, "Saucy Salamander",13.10, "2013-10-17", false);
+
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
index bf1c91f..d10b942 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
@@ -28,6 +28,7 @@ import org.apache.sqoop.model.MPersistableEntity;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.test.asserts.ProviderAsserts;
import org.apache.sqoop.test.data.Cities;
+import org.apache.sqoop.test.data.UbuntuReleases;
import org.apache.sqoop.test.db.DatabaseProvider;
import org.apache.sqoop.test.db.DatabaseProviderFactory;
import org.apache.sqoop.validation.Status;
@@ -155,6 +156,20 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
}
/**
+ * Create table for ubuntu releases.
+ */
+ protected void createTableUbuntuReleases() {
+ new UbuntuReleases(provider, getTableName()).createTables();
+ }
+
+ /**
+ * Create table for ubuntu releases.
+ */
+ protected void createAndLoadTableUbuntuReleases() {
+ new UbuntuReleases(provider, getTableName()).createTables().loadBasicData();
+ }
+
+ /**
* Assert row in testing table.
*
* @param conditions Conditions in form that are expected by the database provider
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
index ca77e64..efdfed4 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
@@ -134,6 +134,15 @@ abstract public class TomcatTestCase {
}
/**
+ * Verify number of output mapreduce files.
+ *
+ * @param expectedFiles Expected number of files
+ */
+ protected void assertMapreduceOutputFiles(int expectedFiles) {
+ HdfsAsserts.assertMapreduceOutputFiles(getMapreduceDirectory(), expectedFiles);
+ }
+
+ /**
* Create mapreduce input file with specified content.
*
* @param filename Input file name
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java
new file mode 100644
index 0000000..656bc34
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.utils;
+
+import org.apache.commons.lang.ArrayUtils;
+
+import java.util.LinkedList;
+
+/**
+ * Various utils to help build expected structure for JUNIT Parametrized runner.
+ */
+public class ParametrizedUtils {
+
+ /**
+ * Join two arrays of parameters by providing their cross product to make
+ * testing matrix for Parameterized runner easier.
+ *
+ * This method accepts both arguments as array of arrays. In such case
+ * the inner array will be flattened during the join to meet the Parametrized
+ * runner requirements.
+ *
+ * @param v1 First array of test values
+ * @param v2 Second array of test values
+ * @return Cross product of v1 and v2
+ */
+ public static Iterable<Object []> crossProduct(Object[] v1, Object[] v2) {
+ LinkedList<Object []> ret = new LinkedList<Object []>();
+
+ for(Object o1 : v1) {
+ for(Object o2 : v2) {
+ ret.add(mergeObjects(o1, o2));
+ }
+ }
+
+ return ret;
+ }
+
+ /**
+ * Merge two objects into array.
+ *
+ * This method will flatten any argument that is already an array, e.g.:
+ *
+ * mergeObjects(1, 2) = [1, 2]
+ * mergeObjects([1], 2) = [1, 2]
+ * mergeObjects(1, [2]) = [1, 2]
+ * mergeObjects([1], [2]) = [1, 2]
+ *
+ * @param o1
+ * @param o2
+ * @return
+ */
+ public static Object [] mergeObjects(Object o1, Object o2) {
+ if(!o1.getClass().isArray() && !o2.getClass().isArray()) {
+ return new Object[] { o1, o2 };
+ }
+
+ Object []a1 = toArray(o1);
+ Object []a2 = toArray(o2);
+
+ return ArrayUtils.addAll(a1, a2);
+ }
+
+ /**
+ * Convert object into one element array if it's not an array already.
+ *
+ * @param o
+ * @return
+ */
+ public static Object[] toArray(Object o) {
+ if(o.getClass().isArray()) {
+ return (Object [])o;
+ }
+
+ return new Object[] { o };
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java
new file mode 100644
index 0000000..50fce74
--- /dev/null
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.integration.connector.jdbc.generic.imports;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.framework.configuration.OutputFormat;
+import org.apache.sqoop.framework.configuration.StorageType;
+import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.test.testcases.ConnectorTestCase;
+import org.apache.sqoop.test.utils.ParametrizedUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ *
+ */
+@RunWith(Parameterized.class)
+public class PartitionerTest extends ConnectorTestCase {
+
+ private static final Logger LOG = Logger.getLogger(PartitionerTest.class);
+
+ /**
+ * Columns that we will use as partition column with maximal number of
+ * partitions that can be created for such column.
+ */
+ public static Object[] COLUMNS = new Object [][] {
+ {"id", 13},
+ {"code_name", 13},
+ {"version", 13},
+ {"release_date", 13},
+ {"lts", 2},
+ };
+
+ /**
+ * Number of extractors that we will use to transfer the table.
+ */
+ public static Object [] EXTRACTORS = new Object[] {
+ 3, 5, 10, 13,
+ };
+
+ @Parameterized.Parameters(name = "{0}-{1}-{2}")
+ public static Iterable<Object[]> data() {
+ return ParametrizedUtils.crossProduct(COLUMNS, EXTRACTORS);
+ }
+
+ private String partitionColumn;
+ private int extractors;
+ private int maxOutputFiles;
+
+ public PartitionerTest(String partitionColumn, int expectedOutputFiles, int extractors) {
+ this.partitionColumn = partitionColumn;
+ this.maxOutputFiles = expectedOutputFiles;
+ this.extractors = extractors;
+ }
+
+ @Test
+ public void testSplitter() throws Exception {
+ createAndLoadTableUbuntuReleases();
+
+ // Connection creation
+ MConnection connection = getClient().newConnection(1L);
+ fillConnectionForm(connection);
+ createConnection(connection);
+
+ // Job creation
+ MJob job = getClient().newJob(connection.getPersistenceId(), MJob.Type.IMPORT);
+
+ // Connector values
+ MFormList forms = job.getConnectorPart();
+ forms.getStringInput("table.tableName").setValue(provider.escapeTableName(getTableName()));
+ forms.getStringInput("table.partitionColumn").setValue(provider.escapeColumnName(partitionColumn));
+ // Framework values
+ fillOutputForm(job, StorageType.HDFS, OutputFormat.TEXT_FILE);
+ forms = job.getFrameworkPart();
+ forms.getIntegerInput("throttling.extractors").setValue(extractors);
+ createJob(job);
+
+ runJob(job);
+
+ // Assert correct output
+ assertMapreduceOutputFiles((extractors > maxOutputFiles) ? maxOutputFiles : extractors);
+ assertMapreduceOutput(
+ "1,'Warty Warthog',4.10,2004-10-20,false",
+ "2,'Hoary Hedgehog',5.04,2005-04-08,false",
+ "3,'Breezy Badger',5.10,2005-10-13,false",
+ "4,'Dapper Drake',6.06,2006-06-01,true",
+ "5,'Edgy Eft',6.10,2006-10-26,false",
+ "6,'Feisty Fawn',7.04,2007-04-19,false",
+ "7,'Gutsy Gibbon',7.10,2007-10-18,false",
+ "8,'Hardy Heron',8.04,2008-04-24,true",
+ "9,'Intrepid Ibex',8.10,2008-10-18,false",
+ "10,'Jaunty Jackalope',9.04,2009-04-23,false",
+ "11,'Karmic Koala',9.10,2009-10-29,false",
+ "12,'Lucid Lynx',10.04,2010-04-29,true",
+ "13,'Maverick Meerkat',10.10,2010-10-10,false",
+ "14,'Natty Narwhal',11.04,2011-04-28,false",
+ "15,'Oneiric Ocelot',11.10,2011-10-10,false",
+ "16,'Precise Pangolin',12.04,2012-04-26,true",
+ "17,'Quantal Quetzal',12.10,2012-10-18,false",
+ "18,'Raring Ringtail',13.04,2013-04-25,false",
+ "19,'Saucy Salamander',13.10,2013-10-17,false"
+ );
+
+ // Clean up testing table
+ dropTable();
+ }
+}