You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2013/06/08 01:45:34 UTC
git commit: BIGTOP-1002. Complete porting Sqoop import tests to Sqoop
2
Updated Branches:
refs/heads/master 43467fee8 -> e2a49bb8b
BIGTOP-1002. Complete porting Sqoop import tests to Sqoop 2
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/e2a49bb8
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/e2a49bb8
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/e2a49bb8
Branch: refs/heads/master
Commit: e2a49bb8b9b1406a34a4cd0c684de806e1b8622e
Parents: 43467fe
Author: Anatoli Fomenko <an...@cloudera.com>
Authored: Fri Jun 7 16:19:52 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Fri Jun 7 16:44:45 2013 -0700
----------------------------------------------------------------------
.../bigtop/itest/sqoop/TestSqoopImport.groovy | 176 ++++++++++-----
1 files changed, 115 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/e2a49bb8/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopImport.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopImport.groovy b/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopImport.groovy
index 0aaaed6..00ef9fe 100644
--- a/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopImport.groovy
+++ b/bigtop-tests/test-artifacts/sqoop/src/main/groovy/org/apache/bigtop/itest/sqoop/TestSqoopImport.groovy
@@ -24,7 +24,6 @@ import org.apache.sqoop.validation.Status
import org.junit.Ignore
import static org.junit.Assert.assertEquals
-import static org.junit.Assert.assertNotNull
import static org.junit.Assert.assertNotSame
import static org.junit.Assert.assertTrue
import org.junit.AfterClass
@@ -157,7 +156,7 @@ class TestSqoopImport {
assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId());
}
- protected void runSqoopClient(String tableName, String partitionColumn) {
+ protected void runSqoopClient(String tableName=null, String partitionColumn=null, String tableColumns=null, String tableSQL=null, String outputSubdir=null, int extractors=0, int loaders=0) {
// Connection creation
MConnection connection = getClient().newConnection(1L);
fillConnectionForm(connection);
@@ -167,11 +166,43 @@ class TestSqoopImport {
MJob job = getClient().newJob(connection.getPersistenceId(), MJob.Type.IMPORT);
// Connector values
- MFormList forms = job.getConnectorPart();
- forms.getStringInput("table.tableName").setValue(tableName);
- forms.getStringInput("table.partitionColumn").setValue(partitionColumn);
+ MFormList connectorForms = job.getConnectorPart();
+
+ if(tableName != null) {
+ connectorForms.getStringInput("table.tableName").setValue(tableName);
+ }
+
+ if(partitionColumn != null) {
+ connectorForms.getStringInput("table.partitionColumn").setValue(partitionColumn);
+ }
+
+ if(tableColumns != null) {
+ connectorForms.getStringInput("table.columns").setValue(tableColumns);
+ }
+
+ if(tableSQL != null) {
+ connectorForms.getStringInput("table.sql").setValue(tableSQL);
+ }
+
// Framework values
- fillOutputForm(job, StorageType.HDFS, OutputFormat.TEXT_FILE, "$OUTPUT".toString() + "/" + tableName);
+ MFormList frameworkForms = job.getFrameworkPart();
+
+ if(extractors > 0) {
+ frameworkForms.getIntegerInput("throttling.extractors").setValue(extractors);
+ }
+
+ if(loaders > 0) {
+ frameworkForms.getIntegerInput("throttling.loaders").setValue(loaders);
+ }
+
+ String outSubdir;
+ if(outputSubdir == null) {
+ outSubdir = tableName;
+ } else {
+ outSubdir = outputSubdir;
+ }
+
+ fillOutputForm(job, StorageType.HDFS, OutputFormat.TEXT_FILE, "$OUTPUT".toString() + "/" + outSubdir);
createJob(job);
MSubmission submission = getClient().startSubmission(job.getPersistenceId());
@@ -187,11 +218,10 @@ class TestSqoopImport {
}
}
-
@Test
public void testBooleanImport() {
String tableName = "t_bool";
- String partitionColumn = "pri";
+ String partitionColumn = null;
runSqoopClient(tableName, partitionColumn);
@@ -200,11 +230,10 @@ class TestSqoopImport {
0, sh.exec("diff -u $DATA_DIR/sqoop-t_bool.out t_bool.out").getRet());
}
-
@Test
public void testIntegerImport() {
String tableName = "t_int";
- String partitionColumn = "pri";
+ String partitionColumn = null;
runSqoopClient(tableName, partitionColumn);
@@ -216,7 +245,7 @@ class TestSqoopImport {
@Test
public void testFixedPointFloatingPointImport() {
String tableName = "t_fp";
- String partitionColumn = "pri";
+ String partitionColumn = null;
runSqoopClient(tableName, partitionColumn);
@@ -228,7 +257,7 @@ class TestSqoopImport {
@Test
public void testDateTimeImport() {
String tableName = "t_date";
- String partitionColumn = "pri";
+ String partitionColumn = null;
runSqoopClient(tableName, partitionColumn);
@@ -240,76 +269,115 @@ class TestSqoopImport {
@Test
public void testStringImport() {
String tableName = "t_string";
- String partitionColumn = "pri";
+ String partitionColumn = null;
runSqoopClient(tableName, partitionColumn);
sh.exec("hadoop fs -cat $OUTPUT/t_string/part-* > t_string.out");
assertEquals("sqoop import did not write expected data",
0, sh.exec("diff -u $DATA_DIR/sqoop-t_string.out t_string.out").getRet());
- }
+ }
- @Ignore("Backward Compatibility")
@Test
- public void testAppendImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --target-dir $OUTPUT/append");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- //import again with append
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --append --target-dir $OUTPUT/append");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/append/part-* > append.out");
+ public void testColumnsImport() {
+ String tableName = "testtable";
+ String partitionColumn = null;
+ String tableColumns = "id,fname";
+
+ runSqoopClient(tableName, partitionColumn, tableColumns);
+
+ sh.exec("hadoop fs -cat $OUTPUT/testtable/part-* > columns.out");
assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-append.out append.out").getRet());
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-columns.out columns.out").getRet());
}
- @Ignore("Backward Compatibility")
@Test
- public void testColumnsImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --columns id,fname --target-dir $OUTPUT/columns");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/columns/part-* > columns.out");
+ public void testNumMappersImport() {
+ String tableName = "testtable";
+ String partitionColumn = null;
+ String tableColumns = null;
+ String tableSQL = null;
+ String outputSubdir = "testtable-nummappers";
+ int extractors = 1;
+ int loaders = 1;
+
+ runSqoopClient(tableName, partitionColumn, tableColumns, tableSQL, outputSubdir, extractors, loaders);
+
+ sh.exec("hadoop fs -cat $OUTPUT/$outputSubdir/part-*0 > num-mappers.out");
assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-columns.out columns.out").getRet());
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out num-mappers.out").getRet());
}
- @Ignore("Backward Compatibility")
@Test
- public void testDirectImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --direct --target-dir $OUTPUT/direct");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/direct/part-* > direct.out");
+ public void testQueryImport() {
+ String tableName = null;
+ String partitionColumn = "t1.id";
+ String tableColumns = null;
+ String tableSQL = 'select t1.id as id, t2.fname as fname from testtable as t1 join testtable2 as t2 on (t1.id = t2.id ) where t1.id < 3 AND \${CONDITIONS}';
+ String outputSubdir = "testtable-query";
+
+ runSqoopClient(tableName, partitionColumn, tableColumns, tableSQL, outputSubdir);
+
+ sh.exec("hadoop fs -cat $OUTPUT/$outputSubdir/part-* > query.out");
assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out direct.out").getRet());
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-query.out query.out").getRet());
}
+ @Test
+ public void testSplitByImport() {
+ String tableName = "testtable";
+ String partitionColumn = "id";
+ String tableColumns = null;
+ String tableSQL = null;
+ String outputSubdir = "testtable-split";
+
+ runSqoopClient(tableName, partitionColumn, tableColumns, tableSQL, outputSubdir);
+
+ sh.exec("hadoop fs -cat $OUTPUT/$outputSubdir/part-* > split-by.out");
+ assertEquals("sqoop import did not write expected data",
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out split-by.out").getRet());
+ }
+
+
+ //----------------------------------------@Ignore("Backward Compatibility")------------------------------------------//
+ // The functionality of the tests below is not currently supported by Sqoop 2.
+
+ //database name is hardcoded here
@Ignore("Backward Compatibility")
@Test
- public void testNumMappersImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --num-mappers 1 --target-dir $OUTPUT/num-mappers");
+ public void testImportAllTables() {
+ String SQOOP_CONNECTION_IMPORT_ALL =
+ "--connect jdbc:mysql://$MYSQL_HOST/mysqltestdb2 --username=$MYSQL_USER" +
+ (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
+
+ sh.exec("sqoop import-all-tables $SQOOP_CONNECTION_IMPORT_ALL --warehouse-dir $OUTPUT/alltables");
assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/num-mappers/part-*0 > num-mappers.out");
+ sh.exec("hadoop fs -cat $OUTPUT/alltables/testtable*/part-* > all-tables.out");
assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out num-mappers.out").getRet());
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-all-tables.out all-tables.out").getRet());
}
@Ignore("Backward Compatibility")
@Test
- public void testQueryImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --query 'select t1.id as id, t2.fname as fname from testtable as t1 join testtable2 as t2 on (t1.id = t2.id ) where t1.id < 3 AND \$CONDITIONS' --split-by t1.id --target-dir $OUTPUT/query");
+ public void testAppendImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --target-dir $OUTPUT/append");
assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/query/part-* > query.out");
+ //import again with append
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --append --target-dir $OUTPUT/append");
+ assertTrue("Sqoop job failed!", sh.getRet() == 0);
+ sh.exec("hadoop fs -cat $OUTPUT/append/part-* > append.out");
assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-query.out query.out").getRet());
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-append.out append.out").getRet());
}
@Ignore("Backward Compatibility")
@Test
- public void testSplityByImport() {
- sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --split-by fname --target-dir $OUTPUT/split-by");
+ public void testDirectImport() {
+ sh.exec("sqoop import $SQOOP_CONNECTION --table testtable --direct --target-dir $OUTPUT/direct");
assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/split-by/part-* > split-by.out");
+ sh.exec("hadoop fs -cat $OUTPUT/direct/part-* > direct.out");
assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out split-by.out").getRet());
+ 0, sh.exec("diff -u $DATA_DIR/sqoop-testtable.out direct.out").getRet());
}
@Ignore("Backward Compatibility")
@@ -352,18 +420,4 @@ class TestSqoopImport {
0, sh.exec("diff -u $DATA_DIR/sqoop-null-non-string.out non-null-string.out").getRet());
}
- //database name is hardcoded here
- @Ignore("Backward Compatibility")
- @Test
- public void testImportAllTables() {
- String SQOOP_CONNECTION_IMPORT_ALL =
- "--connect jdbc:mysql://$MYSQL_HOST/mysqltestdb2 --username=$MYSQL_USER" +
- (("".equals(MYSQL_PASSWORD)) ? "" : " --password=$MYSQL_PASSWORD");
-
- sh.exec("sqoop import-all-tables $SQOOP_CONNECTION_IMPORT_ALL --warehouse-dir $OUTPUT/alltables");
- assertTrue("Sqoop job failed!", sh.getRet() == 0);
- sh.exec("hadoop fs -cat $OUTPUT/alltables/testtable*/part-* > all-tables.out");
- assertEquals("sqoop import did not write expected data",
- 0, sh.exec("diff -u $DATA_DIR/sqoop-all-tables.out all-tables.out").getRet());
- }
}